{"id":"W4404781072","doi":"10.18653/v1/2024.wmt-1.25","title":"WMT24 Test Suite: Gender Resolution in Speaker-Listener Dialogue Roles","year":2024,"lang":"en","type":"article","venue":"","topic":"Language, Discourse, Communication Strategies","field":"Arts and Humanities","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Suite; Computer science; Test (biology); Test suite; Resolution (logic); Speech recognition; Artificial intelligence; Test case; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0002231554,0.0001358371,0.0001268173,0.0001593794,0.0001265523,0.0006265161,0.0002351326,0.00004629272,0.003675523],"category_scores_gemma":[0.00004147025,0.0001008426,0.00006280259,0.00007465696,0.0001983469,0.0004363307,0.00008553759,0.0001833635,0.0005841277],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005365212,"about_ca_system_score_gemma":0.00006714292,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00120779,"about_ca_topic_score_gemma":0.03300028,"domain_scores_codex":[0.9990882,0.00006343515,0.00025046,0.0002123013,0.0001630257,0.0002226304],"domain_scores_gemma":[0.9993008,0.0001982835,0.00002681367,0.0003886452,0.00004908552,0.00003636166],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000003714374,0.00007129402,0.0003083395,0.00003732821,0.00001738077,0.00002053306,0.02850465,0.00002348544,0.0001524382,0.9522586,0.01747817,0.001123993],"study_design_scores_gemma":[0.000413653,0.00006306442,0.005556358,0.0001488468,0.00005154163,0.00001455547,0.07590844,0.002393466,0.0002057098,0.04495895,0.8696899,0.0005955541],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.03712899,0.004011284,0.00006585601,0.002338094,0.0004638872,0.0001767791,0.00006591263,0.000391059,0.9553581],"genre_scores_gemma":[0.9657136,0.00009603938,0.0001086268,0.000246645,0.0004788166,0.00002901637,0.0000971791,0.00002359767,0.03320643],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9285846,"threshold_uncertainty_score":0.9972352,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06776917273351057,"score_gpt":0.292143731668265,"score_spread":0.2243745589347544,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}