{"id":"W4404782347","doi":"10.18653/v1/2024.emnlp-main.734","title":"FAC2E: Better Understanding Large Language Model Capabilities by Dissociating Language and Cognition","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Computer science; Cognition; Cognitive science; Natural language processing; Language model; Artificial intelligence; Psychology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002428464,0.00009682904,0.00009114703,0.00005283726,0.0001158807,0.0003459048,0.000137211,0.00004841612,0.00002796628],"category_scores_gemma":[0.00003027664,0.00008538883,0.00002702061,0.0001124988,0.0000192161,0.0004756409,0.0001366474,0.0001193782,0.000009467995],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008331591,"about_ca_system_score_gemma":0.00001971999,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005290109,"about_ca_topic_score_gemma":0.00003693689,"domain_scores_codex":[0.9991391,0.00002610977,0.0001304349,0.0003016421,0.0001669035,0.0002358327],"domain_scores_gemma":[0.9996557,0.0001099765,0.00001753638,0.0001591349,0.00001182572,0.00004581933],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002769476,0.00006113735,0.0007915103,0.0004929328,0.0001066784,0.00008098481,0.2172224,0.0003392051,0.02859898,0.7105728,0.008096745,0.03363383],"study_design_scores_gemma":[0.0001076255,0.000006775463,0.000006285029,0.00004372768,0.000007170055,0.000003695975,0.009696046,0.9756727,0.0009130663,0.01337575,0.00003415732,0.0001329865],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1881182,0.0006096528,0.8053274,0.000990803,0.00007552599,0.00006596815,0.00002369823,0.0003269993,0.004461757],"genre_scores_gemma":[0.9836175,0.000005419082,0.01444698,0.0004876339,0.0000454853,0.00000656575,0.00001093523,0.000009876457,0.001369632],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9753335,"threshold_uncertainty_score":0.3482055,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02295772609323601,"score_gpt":0.2646405058209287,"score_spread":0.2416827797276927,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}