{"id":"W3158244987","doi":"","title":"Evaluating Groundedness in Dialogue Systems: The BEGIN Benchmark","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Benchmark (surveying); Computer science; Metric (unit); Inference; Conversation; Natural language; Artificial intelligence; Natural language processing; Machine learning; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001224129,0.0002867614,0.0003740176,0.0002014366,0.0001791757,0.0004291268,0.002737081,0.0002625039,0.00001125081],"category_scores_gemma":[0.0001157407,0.0002782177,0.000158249,0.0006963102,0.00007223438,0.0004117226,0.003295724,0.0008434312,0.00001456095],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004403423,"about_ca_system_score_gemma":0.0004739605,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002247652,"about_ca_topic_score_gemma":0.000514439,"domain_scores_codex":[0.9971402,0.0006708796,0.0003141526,0.001275783,0.00019163,0.0004073418],"domain_scores_gemma":[0.9972137,0.0003135116,0.0002567524,0.001956982,0.0001702777,0.00008877102],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003551851,0.00003628083,0.00235618,0.00009608872,0.00003362467,0.0003087023,0.001105302,0.8586147,0.00001261924,0.1365973,0.00001753361,0.0008180981],"study_design_scores_gemma":[0.0002731108,0.00001378244,0.001227561,0.000275991,0.00002785879,0.000007943159,0.000722191,0.9859405,0.000006683704,0.01114346,0.00004910518,0.0003117848],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4930998,0.0002518982,0.5035352,0.0001351553,0.001527041,0.0003175915,0.000002321665,0.00008544576,0.001045501],"genre_scores_gemma":[0.997448,0.00006766256,0.001805902,0.0000772105,0.0001574822,0.000004464023,0.0000138716,0.00001342656,0.0004119244],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5043482,"threshold_uncertainty_score":0.999967,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1780607930847837,"score_gpt":0.2358178567770648,"score_spread":0.05775706369228115,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}