{"id":"W4313459318","doi":"10.1162/tacl_a_00529","title":"<scp>FaithDial</scp>: A Faithful Benchmark for Information-Seeking Dialogue","year":2022,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Topic Modeling","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University; Mila - Quebec Artificial Intelligence Institute; University of Alberta","funders":"Alberta Machine Intelligence Institute; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Benchmark (surveying); Computer science; Utterance; Hallucinating; Natural language processing; Artificial intelligence; Crowdsourcing; Machine learning; World Wide Web","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007688726,0.0001161535,0.0001740415,0.0001578491,0.0009662766,0.0000965979,0.0007127409,0.00005648388,0.000003104415],"category_scores_gemma":[0.002465747,0.0001213055,0.0002528559,0.00040757,0.00001777358,0.0001732703,0.00005724137,0.0001745274,0.000001826298],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000365542,"about_ca_system_score_gemma":0.0002969421,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002490416,"about_ca_topic_score_gemma":0.000006852781,"domain_scores_codex":[0.9983848,0.00007455309,0.0005330361,0.0001780892,0.000602384,0.0002270817],"domain_scores_gemma":[0.9955707,0.002445505,0.0006061906,0.0002430055,0.001094459,0.00004010269],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003870693,0.00005351169,0.0003907658,0.00003100511,0.00006741825,3.215052e-8,0.001184771,0.8315013,0.000003914636,0.1643453,0.001452838,0.0009652646],"study_design_scores_gemma":[0.0006946858,0.00007170009,0.0007840109,0.000007370297,0.00005338154,0.000001091683,0.0001227608,0.7987117,0.00004273238,0.0804075,0.1190299,0.00007315829],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0004796078,0.00001491426,0.9940663,0.0004397011,0.002188818,0.0006486871,0.000530003,0.00007777368,0.001554195],"genre_scores_gemma":[0.8601776,9.902162e-7,0.1385152,0.0002864749,0.0001934335,0.0002317682,0.0001394159,0.00001077723,0.0004443153],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.859698,"threshold_uncertainty_score":0.7431916,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01280205941306425,"score_gpt":0.2332753737500383,"score_spread":0.220473314336974,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}