{"id":"W3105424285","doi":"10.18653/v1/2020.emnlp-main.393","title":"Utility is in the Eye of the User: A Critique of NLP Leaderboards","year":2020,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":137,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Framing (construction); Inference; Artificial intelligence; Natural language processing; Machine learning","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002505281,0.00004454693,0.00009108146,0.00001006944,0.0000150525,0.00001276546,0.001115848,0.00002622953,0.00002686237],"category_scores_gemma":[0.00006826263,0.00002380646,0.00005177247,0.0002376137,0.00004150235,0.00009005058,0.0002118454,0.0001021561,0.000001636477],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000003237126,"about_ca_system_score_gemma":0.00005041426,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002873545,"about_ca_topic_score_gemma":0.00005812739,"domain_scores_codex":[0.999288,0.00009701332,0.0001823299,0.0001466684,0.0001997993,0.00008620908],"domain_scores_gemma":[0.99935,0.00005741465,0.00003552624,0.0005067227,0.00003487584,0.0000154489],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001560952,0.0002523826,0.09013876,0.0003249211,0.00002533758,0.000005962524,0.07658377,0.0005096516,0.00511145,0.7772859,0.007635475,0.04211077],"study_design_scores_gemma":[0.0004215962,0.00007597408,0.07623625,0.00005136239,0.000006720606,0.0000020245,0.001381982,0.8347639,0.06791053,0.009770371,0.009215135,0.0001641309],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3040075,0.00006647043,0.5872359,0.09834273,0.00008258969,0.0002152674,0.000002285163,0.00003086122,0.01001645],"genre_scores_gemma":[0.9859693,0.000002179834,0.008341765,0.005594423,0.00001320294,0.000001874907,2.681887e-8,0.000001373699,0.00007583188],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8342543,"threshold_uncertainty_score":0.2073541,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05896975066937212,"score_gpt":0.2934283674829232,"score_spread":0.2344586168135511,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}