{"id":"W4405903515","doi":"10.48550/arxiv.2412.19726","title":"Position: Theory of Mind Benchmarks are Broken for Large Language Models","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Instituto de Ciencias del Mar y Limnología, Universidad Nacional Autónoma de México; Canada Excellence Research Chairs, Government of Canada","keywords":"Context (archaeology); Computer science; Cognitive science; Psychology; History; Archaeology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04652021,0.002060207,0.002266308,0.004289959,0.002696388,0.01200181,0.006045281,0.005242526,0.01039914],"category_scores_gemma":[0.2449761,0.0009767928,0.0019277,0.002061076,0.01239145,0.03302665,0.009668084,0.01355834,0.003092559],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004258548,"about_ca_system_score_gemma":0.003603905,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002719164,"about_ca_topic_score_gemma":0.001380516,"domain_scores_codex":[0.9634944,0.01312108,0.002339872,0.007059865,0.01238483,0.001599973],"domain_scores_gemma":[0.7615302,0.1415781,0.01401038,0.05433262,0.02092171,0.007627007],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0003940782,0.0002087368,0.007670311,0.0005488291,0.0002133576,0.00009307588,0.001372971,0.007866264,0.001198509,0.8630365,0.02006543,0.09733191],"study_design_scores_gemma":[0.00006897913,0.0002627767,0.002300314,0.0003969187,0.00006242208,0.000177467,0.0005323139,0.03114078,0.002936126,0.9377987,0.0242143,0.00010886],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"commentary","genre_scores_codex":[0.1360478,0.006522804,0.6409194,0.09928551,0.003572437,0.0003748579,0.001750698,0.008422974,0.1031035],"genre_scores_gemma":[0.8280402,0.001321211,0.1507815,0.009858605,0.001418128,0.0007419768,0.001733089,0.002362999,0.003742206],"genre_candidate":"commentary","genre_consensus":null,"teacher_disagreement_score":0.9534798,"threshold_uncertainty_score":0.2460254,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05527061287191093,"score_gpt":0.1994037170977207,"score_spread":0.1441331042258098,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}