{"id":"W4389518784","doi":"10.18653/v1/2023.emnlp-main.397","title":"HaluEval: A Large-Scale Hallucination Evaluation Benchmark for Large Language Models","year":2023,"lang":"en","type":"article","venue":"","topic":"Mental Health via Writing","field":"Psychology","cited_by":234,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"Renmin University of China; National Natural Science Foundation of China","keywords":"Hallucinating; Benchmark (surveying); Computer science; Sampling (signal processing); Artificial intelligence; Face (sociological concept); Natural language processing; Machine learning; Computer vision; Sociology; Social science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.003665245,0.0001262007,0.0001571391,0.0001810343,0.0002124544,0.00002228276,0.000122075,0.0001336549,0.00402655],"category_scores_gemma":[0.00006925502,0.0001283033,0.00006773207,0.0003399854,0.000007510362,0.0001767019,0.00004985054,0.000102019,0.0008761364],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001338834,"about_ca_system_score_gemma":0.00004016989,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001122724,"about_ca_topic_score_gemma":0.0004249186,"domain_scores_codex":[0.9980316,0.0002004957,0.0003460902,0.0004032458,0.0003967317,0.0006218387],"domain_scores_gemma":[0.9991433,0.0002112886,0.00009893742,0.0003177678,0.000135586,0.0000931122],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005662174,0.002607386,0.004443826,0.001094217,0.0002189622,0.00003578703,0.1227495,0.001104526,0.002684799,0.3488343,0.2233346,0.2923259],"study_design_scores_gemma":[0.003692673,0.0002133207,0.004957475,0.00003925444,0.00004512429,0.000004992197,0.01744481,0.9612117,0.0001694839,0.007152353,0.004845741,0.000223058],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6295299,0.0006001403,0.08675959,0.001735726,0.001779925,0.004326097,0.000350952,0.000905393,0.2740123],"genre_scores_gemma":[0.9823322,0.000005420481,0.003259153,0.0008966025,0.0002379209,0.00107546,0.0009038027,0.00004336429,0.01124609],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9601072,"threshold_uncertainty_score":0.9999018,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08507588964639624,"score_gpt":0.4452306925761189,"score_spread":0.3601548029297226,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}