{"id":"W4389520397","doi":"10.18653/v1/2023.emnlp-main.593","title":"EpiK-Eval: Evaluation for Language Models as Epistemic Models","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal; Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada; Fonds de recherche du Québec – Nature et technologies; Alliance de recherche numérique du Canada","keywords":"Benchmark (surveying); Computer science; Narrative; Representation (politics); Artificial intelligence; Consolidation (business); Data science; Political science; Linguistics; Business; Geography","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.008592129,0.002441708,0.00102377,0.002165901,0.000827527,0.003631955,0.005184059,0.003204959,0.01116613],"category_scores_gemma":[0.06553539,0.0006397436,0.001275674,0.001369317,0.00120534,0.006327563,0.003970018,0.004037552,0.003947193],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002096788,"about_ca_system_score_gemma":0.002325217,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01049332,"about_ca_topic_score_gemma":0.01504413,"domain_scores_codex":[0.9903435,0.005548709,0.0008046509,0.001376129,0.001536499,0.0003904911],"domain_scores_gemma":[0.9636099,0.02852435,0.0006597876,0.003896612,0.002577717,0.0007315591],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004786835,0.002397426,0.01433416,0.007070282,0.001116386,0.0004653047,0.001567922,0.3469632,0.008318034,0.02872689,0.1606469,0.4236067],"study_design_scores_gemma":[0.0005060335,0.0007374962,0.002201535,0.00040999,0.0001408402,0.0002834118,0.000740041,0.9240652,0.01117027,0.02317202,0.03647875,0.00009427921],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.3837162,0.01018189,0.3213859,0.005678352,0.002475208,0.002691247,0.05644743,0.1387836,0.07864018],"genre_scores_gemma":[0.6037529,0.001341588,0.3073418,0.001286375,0.0001838728,0.00150265,0.07087964,0.005717825,0.007993476],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01116613,"threshold_uncertainty_score":0.04544008,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1251071811620084,"score_gpt":0.356596646379913,"score_spread":0.2314894652179046,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}