{"id":"W4404782347","doi":"10.18653/v1/2024.emnlp-main.734","title":"FAC2E: Better Understanding Large Language Model Capabilities by Dissociating Language and Cognition","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Computer science; Cognition; Cognitive science; Natural language processing; Language model; Artificial intelligence; Psychology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004288774,0.001682227,0.0006868525,0.002653701,0.0004948652,0.004125623,0.001465948,0.001362068,0.006834712],"category_scores_gemma":[0.02148008,0.0004247598,0.0009194314,0.001071891,0.0008617259,0.008700878,0.00405779,0.001566104,0.00127329],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001042466,"about_ca_system_score_gemma":0.001855711,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008342521,"about_ca_topic_score_gemma":0.01059152,"domain_scores_codex":[0.998489,0.0005876666,0.0001277053,0.0002903446,0.0003748674,0.0001304581],"domain_scores_gemma":[0.9875299,0.007913351,0.0007828182,0.002548756,0.0008652391,0.0003599196],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001041803,0.00071619,0.0437098,0.001088319,0.0004493045,0.0002832947,0.002959227,0.1127146,0.03008887,0.0365751,0.010695,0.7596784],"study_design_scores_gemma":[0.00005817314,0.0003435229,0.01262474,0.0001005849,0.0001405399,0.0002519324,0.001024952,0.8953321,0.02960085,0.04964565,0.0107365,0.0001405352],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1706866,0.0006026357,0.7960799,0.0007946799,0.00005765993,0.0004527986,0.001979792,0.01822416,0.01112169],"genre_scores_gemma":[0.6760209,0.0002127227,0.3185263,0.0001961307,0.00002102776,0.0002446783,0.002312976,0.0005378711,0.001927484],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.008342521,"threshold_uncertainty_score":0.0228644,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02295772609323601,"score_gpt":0.2646405058209287,"score_spread":0.2416827797276927,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}