{"id":"W4411120250","doi":"10.18653/v1/2025.wnu-1.7","title":"A Theoretical Framework for Evaluating Narrative Surprise in Large Language Models","year":2025,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University","funders":"Social Sciences and Humanities Research Council of Canada","keywords":"Surprise; Narrative; Computer science; Natural language processing; Linguistics; Psychology; Philosophy; Communication","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0308545,0.001885183,0.00113954,0.00450755,0.001493211,0.004165257,0.002529525,0.001673541,0.003196764],"category_scores_gemma":[0.1295196,0.0008244696,0.001600971,0.002513391,0.00426907,0.006028774,0.004415776,0.002342744,0.0004190271],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002404009,"about_ca_system_score_gemma":0.001854366,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002114374,"about_ca_topic_score_gemma":0.001834924,"domain_scores_codex":[0.9832659,0.01169557,0.0008805582,0.001593603,0.002162518,0.0004019649],"domain_scores_gemma":[0.9018181,0.08150781,0.007636583,0.004329075,0.003643264,0.001065162],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0005576859,0.0005860202,0.03820621,0.0008557733,0.0005530199,0.0004496303,0.007013972,0.2807002,0.009596571,0.5540126,0.002329911,0.1051385],"study_design_scores_gemma":[0.00005388731,0.0004314092,0.003728351,0.00007920316,0.00008025587,0.0001535046,0.001006568,0.6653764,0.002052784,0.3246608,0.002304433,0.00007230171],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.03394982,0.0001182657,0.9623942,0.0004848749,0.0000155322,0.0002358739,0.000190171,0.000285764,0.002325471],"genre_scores_gemma":[0.640121,0.0001155593,0.3570317,0.0001662832,0.0000687993,0.001506678,0.0004693755,0.00008415081,0.0004363079],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.0308545,"threshold_uncertainty_score":0.1631762,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02994904019277738,"score_gpt":0.3667034877914282,"score_spread":0.3367544475986508,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}