{"id":"W4313459318","doi":"10.1162/tacl_a_00529","title":"<scp>FaithDial</scp>: A Faithful Benchmark for Information-Seeking Dialogue","year":2022,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Topic Modeling","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University; Mila - Quebec Artificial Intelligence Institute; University of Alberta","funders":"Alberta Machine Intelligence Institute; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Benchmark (surveying); Computer science; Utterance; Hallucinating; Natural language processing; Artificial intelligence; Crowdsourcing; Machine learning; World Wide Web","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006931292,0.002594201,0.001037396,0.002702551,0.001745446,0.00275691,0.002896275,0.00326904,0.006216775],"category_scores_gemma":[0.02604859,0.0004618994,0.001275964,0.001293218,0.001696641,0.003148438,0.004144596,0.002948992,0.004460691],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001729801,"about_ca_system_score_gemma":0.001357193,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0092393,"about_ca_topic_score_gemma":0.01411737,"domain_scores_codex":[0.9907134,0.005456122,0.0004929999,0.001652681,0.001253155,0.0004316609],"domain_scores_gemma":[0.9827779,0.009158574,0.0006596957,0.003460013,0.002586874,0.001356835],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.006063689,0.004719162,0.01732304,0.005191254,0.0009193003,0.001523089,0.003971925,0.1258333,0.04382412,0.008591704,0.2475931,0.5344463],"study_design_scores_gemma":[0.0009337985,0.003904772,0.02005586,0.0004995764,0.0002050007,0.001377043,0.003203186,0.7813051,0.07424955,0.0169471,0.09688436,0.0004347437],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6336607,0.00894468,0.1675692,0.00377072,0.003110057,0.002551862,0.05512027,0.08624581,0.03902667],"genre_scores_gemma":[0.7580103,0.0006240549,0.1113841,0.001019298,0.0004117765,0.001138922,0.1132558,0.002625508,0.01153028],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0092393,"threshold_uncertainty_score":0.03665662,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01280205941306425,"score_gpt":0.2332753737500383,"score_spread":0.220473314336974,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}