{"id":"W4385570347","doi":"10.18653/v1/2023.findings-acl.826","title":"Data Sampling and (In)stability in Machine Translation Evaluation","year":2023,"lang":"en","type":"article","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Computer science; Machine translation; Ranking (information retrieval); Sampling (signal processing); Consistency (knowledge bases); Data mining; Sample (material); Data set; Artificial intelligence; Set (abstract data type); Data quality; Representation (politics); Stability (learning theory); Translation (biology); Machine learning; Quality (philosophy); Information retrieval; Natural language processing; Service (business)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1302491,0.0009769915,0.001386096,0.004384563,0.002776658,0.003997403,0.002244624,0.001861096,0.001346774],"category_scores_gemma":[0.4034241,0.0007863595,0.0008611521,0.005212314,0.004112952,0.005228742,0.004493123,0.002876244,0.0006075522],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002721975,"about_ca_system_score_gemma":0.001831614,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003448332,"about_ca_topic_score_gemma":0.004224519,"domain_scores_codex":[0.8313111,0.1327619,0.007835992,0.0105033,0.01596842,0.001619326],"domain_scores_gemma":[0.454688,0.4278782,0.02959966,0.04944628,0.03570896,0.002678788],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.008146262,0.001496559,0.3942745,0.001440387,0.001350926,0.0005253205,0.01406099,0.06812516,0.02751938,0.03838382,0.009269862,0.4354068],"study_design_scores_gemma":[0.001028071,0.004900346,0.2146989,0.001015045,0.001120866,0.001781093,0.007369083,0.5318165,0.08856688,0.1218411,0.02533286,0.000529412],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5191765,0.003071449,0.4675252,0.00230056,0.0003365478,0.0008911996,0.0009371223,0.0008971048,0.004864303],"genre_scores_gemma":[0.9116709,0.0002363643,0.08341328,0.0005138833,0.0002757666,0.0008838129,0.001529259,0.0003950858,0.001081559],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8697509,"threshold_uncertainty_score":0.6888316,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2005086114380628,"score_gpt":0.4040928662854281,"score_spread":0.2035842548473653,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}