{"id":"W3186655327","doi":"10.18653/v1/2021.gem-1.10","title":"The GEM Benchmark: Natural Language Generation, its Evaluation and Metrics","year":2021,"lang":"en","type":"preprint","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":148,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Benchmark (surveying); Natural language generation; Natural language; Natural language processing; Geology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.008712871,0.004072922,0.002398913,0.007818285,0.001578446,0.004636199,0.00479501,0.003105307,0.00944643],"category_scores_gemma":[0.0283505,0.0008715646,0.002397244,0.005908105,0.00118903,0.005483366,0.003552546,0.002529169,0.006696726],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003380977,"about_ca_system_score_gemma":0.003861679,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02411037,"about_ca_topic_score_gemma":0.02141576,"domain_scores_codex":[0.9887658,0.004604396,0.00129834,0.001833157,0.002882845,0.0006154295],"domain_scores_gemma":[0.9848706,0.007933636,0.0005924458,0.002838465,0.002835921,0.0009289351],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.002453134,0.002350312,0.006666478,0.004662161,0.0009118736,0.0006537071,0.0005090008,0.08288432,0.005664715,0.005652303,0.4280253,0.4595667],"study_design_scores_gemma":[0.003612177,0.004047517,0.01481039,0.001261879,0.001056229,0.002636987,0.001492101,0.6626534,0.03066027,0.0304536,0.2468436,0.0004718384],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.3233838,0.04920667,0.1123847,0.005601618,0.006489156,0.005569758,0.209877,0.2076969,0.07979043],"genre_scores_gemma":[0.2248434,0.006147384,0.1792759,0.00156742,0.0005987667,0.00289335,0.5564252,0.00953288,0.01871566],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9912871,"threshold_uncertainty_score":0.04794008,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02864547784444409,"score_gpt":0.3241813393370074,"score_spread":0.2955358614925633,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}