{"id":"W3157225713","doi":"10.18653/v1/2021.naacl-main.324","title":"Dynabench: Rethinking Benchmarking in NLP","year":2021,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":true,"ca_institutions":"Simon Fraser University","funders":"Office of Naval Research; Defense Advanced Research Projects Agency","keywords":"Benchmarking; Benchmark (surveying); Computer science; Field (mathematics); Artificial intelligence; Data science; Open source; Machine learning; Programming language; Management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007730172,0.0002390093,0.0003412845,0.0002280947,0.00005376174,0.0006118998,0.001725264,0.0003347334,0.0001005939],"category_scores_gemma":[0.00006764681,0.0002522002,0.0001189695,0.0002788444,0.00001440405,0.0002791031,0.005231202,0.0008136063,0.000007023569],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001917964,"about_ca_system_score_gemma":0.0003560176,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007577003,"about_ca_topic_score_gemma":0.0006770159,"domain_scores_codex":[0.9975178,0.0001203367,0.0004732929,0.001084223,0.0004255738,0.0003787667],"domain_scores_gemma":[0.9981248,0.00009654409,0.0001323701,0.001501437,0.00007676391,0.00006813253],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002693441,0.0001904321,0.0115262,0.0007290297,0.00009351692,0.001404468,0.03321628,0.1601771,0.0005166627,0.3105358,0.0003859696,0.4812218],"study_design_scores_gemma":[0.00009128626,0.000005580342,0.0009592609,0.000471505,0.000002944468,0.00001059338,0.00006648852,0.9547245,0.0002130695,0.04297546,0.0001317908,0.0003475165],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05418746,0.0004499762,0.9210444,0.001052013,0.002186127,0.0001658672,2.910711e-7,0.0002229535,0.0206909],"genre_scores_gemma":[0.5845532,0.00003741402,0.4146688,0.0003833788,0.0001413341,0.00001523051,0.000006767039,0.000008917022,0.0001848993],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7945474,"threshold_uncertainty_score":0.999993,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04231095882403402,"score_gpt":0.2671027972964434,"score_spread":0.2247918384724094,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}