{"id":"W4398255874","doi":"10.1017/cjn.2024.213","title":"P.110 Evaluating AI performance in written neurosurgery exams: a comparative analysis of large language models","year":2024,"lang":"en","type":"article","venue":"Canadian Journal of Neurological Sciences / Journal Canadien des Sciences Neurologiques","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Calgary Laboratory Services","funders":"","keywords":"Perplexity; Neurosurgery; Medical physics; Medicine; Artificial intelligence; Psychology; Natural language processing; Radiology; Computer science; Language model","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02349636,0.001695964,0.001173192,0.003814617,0.000716201,0.005783673,0.001687633,0.00112447,0.005082291],"category_scores_gemma":[0.1088321,0.0004238635,0.003173447,0.002551437,0.001022739,0.004513941,0.002667118,0.002214276,0.001699749],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002318994,"about_ca_system_score_gemma":0.001360103,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00635088,"about_ca_topic_score_gemma":0.003733848,"domain_scores_codex":[0.9881827,0.007491634,0.00101371,0.001285387,0.001695761,0.0003308376],"domain_scores_gemma":[0.7435153,0.2413052,0.004014354,0.003997192,0.005662372,0.001505662],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.006461354,0.002518296,0.5316017,0.001738532,0.004592922,0.0009103349,0.006676139,0.1130119,0.003240134,0.003467816,0.01455315,0.3112277],"study_design_scores_gemma":[0.0004106997,0.003189013,0.2739329,0.0006445685,0.001155825,0.0007930552,0.005352316,0.694803,0.00358564,0.01080622,0.00503537,0.0002914442],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9578497,0.001512601,0.02793719,0.0007271814,0.0001658352,0.0006326975,0.003079258,0.001529,0.006566687],"genre_scores_gemma":[0.9816929,0.0003603438,0.01138665,0.0001257813,0.00004270747,0.0004892116,0.004778442,0.0001932239,0.0009307896],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9765037,"threshold_uncertainty_score":0.1242621,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2165060125666209,"score_gpt":0.4414506085346088,"score_spread":0.2249445959679879,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}