{"id":"W4378474033","doi":"10.48550/arxiv.2305.14763","title":"Clever Hans or Neural Theory of Mind? Stress Testing Social Reasoning in Large Language Models","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"European Commission; Natural Sciences and Engineering Research Council of Canada; Defense Advanced Research Projects Agency; Naval Information Warfare Center Pacific; Bar-Ilan University; Canadian Institute for Advanced Research","keywords":"Heuristics; Theory of mind; Adversarial system; Psychology; Benchmark (surveying); Cognitive psychology; Artificial intelligence; Cognitive science; Computer science; Cognition","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007854685,0.0002845218,0.0003981019,0.000425536,0.0001745484,0.00009268773,0.001876223,0.0002740109,0.00002620245],"category_scores_gemma":[0.0002941322,0.0003158954,0.0001521359,0.001359188,0.0001177303,0.000623966,0.002393392,0.0006488765,0.00003530708],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001883389,"about_ca_system_score_gemma":0.000249302,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0009688034,"about_ca_topic_score_gemma":0.001462681,"domain_scores_codex":[0.9976327,0.0003309858,0.0003369527,0.0009568754,0.000151284,0.0005911699],"domain_scores_gemma":[0.9981143,0.0005402609,0.0003249419,0.0007500181,0.0001799645,0.00009055146],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00009564178,0.0001838456,0.006652685,0.0001538005,0.00005570435,0.002092854,0.01367504,0.6996599,0.0001944003,0.2744305,0.00003570973,0.002769888],"study_design_scores_gemma":[0.0001785848,0.0000341029,0.001351671,0.0002252251,0.0000229171,0.000001876512,0.004646182,0.9437663,0.001225555,0.04820426,0.000002900644,0.0003404468],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6449315,0.00002026647,0.3532459,0.00003589259,0.0002785939,0.0002171148,0.00004548818,0.0001600388,0.001065158],"genre_scores_gemma":[0.9958712,0.000009777717,0.002342775,0.000032364,0.00008017392,0.000001660348,0.00001034042,0.00002887857,0.001622802],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3509397,"threshold_uncertainty_score":0.9999293,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2058113727901079,"score_gpt":0.2491067126117638,"score_spread":0.04329533982165584,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}