{"id":"W3116202855","doi":"10.1101/2020.12.23.424091","title":"Narratives: fMRI data for evaluating models of naturalistic language comprehension","year":2020,"lang":"en","type":"preprint","venue":"bioRxiv (Cold Spring Harbor Laboratory)","topic":"Neurobiology of Language and Bilingualism","field":"Neuroscience","cited_by":35,"is_retracted":false,"has_abstract":true,"ca_institutions":"BC Children's Hospital; University of British Columbia; McGill University; Montreal Neurological Institute and Hospital","funders":"Advanced Research Projects Agency; Defense Advanced Research Projects Agency; National Institutes of Health; Intel Corporation","keywords":"Narrative; Computer science; Comprehension; Variety (cybernetics); Benchmark (surveying); Metadata; Data collection; Neuroimaging; Naturalism; Psychology; Natural language processing; Artificial intelligence; Linguistics; World Wide Web; Neuroscience","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006595966,0.0005514813,0.0008332198,0.000170837,0.000195069,0.00009813094,0.001921429,0.0004279882,0.00002645148],"category_scores_gemma":[0.002797579,0.0005235096,0.0001714617,0.0003451054,0.0002840929,0.0002576329,0.001854969,0.00085122,0.00001243349],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005643458,"about_ca_system_score_gemma":0.0004484562,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002748876,"about_ca_topic_score_gemma":0.000001550877,"domain_scores_codex":[0.9959502,0.0004622008,0.000741852,0.001899244,0.0004484142,0.0004981199],"domain_scores_gemma":[0.9957054,0.0006790371,0.0007823178,0.002318413,0.0003352574,0.0001795374],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.00008983984,0.00007713529,0.000009762054,0.0006449459,0.00003562508,0.0001545801,0.0002232301,0.0001983016,0.9976918,0.0005193897,0.0003526519,0.000002722156],"study_design_scores_gemma":[0.0008914496,0.000216929,0.0001316309,0.0005377871,0.0001576282,2.113958e-7,0.0000630017,0.03849282,0.9582061,0.0000846224,0.0004013451,0.0008165057],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9881824,0.001195254,0.003433223,0.0005752397,0.001224536,0.001551309,0.003471667,0.0003551203,0.00001123955],"genre_scores_gemma":[0.9833579,0.00005685107,0.01492939,0.001066151,0.0003927901,0.00007490037,0.000007462214,0.0001103872,0.000004148908],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03948575,"threshold_uncertainty_score":0.9997216,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1272530129184253,"score_gpt":0.3394178766284918,"score_spread":0.2121648637100665,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}