{"id":"W4403349599","doi":"10.1016/j.bas.2024.103540","title":"Can ChatGPT pass the Canadian Royal College written examination in Neurosurgery? A performance comparison of GPT-4 and PGY-6 residents","year":2024,"lang":"en","type":"article","venue":"Brain and Spine","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Manitoba; Ottawa Hospital; University of Ottawa","funders":"","keywords":"Medical education; Psychology; Medicine; Mathematics education","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.002987019,0.00032849,0.0006195696,0.001465367,0.002211961,0.00195743,0.001802033,0.001163084,0.004947925],"category_scores_gemma":[0.02006492,0.0003926434,0.0006414677,0.001832939,0.001123652,0.001332267,0.002443846,0.00122951,0.001500303],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.007885513,"about_ca_system_score_gemma":0.01629574,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.5773606,"about_ca_topic_score_gemma":0.7297559,"domain_scores_codex":[0.9966454,0.000293327,0.0002044299,0.0002556385,0.001402494,0.001198635],"domain_scores_gemma":[0.9867148,0.001548249,0.00197607,0.0002892026,0.004383563,0.005088141],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001250811,0.0007983326,0.9555963,0.00009181722,0.0001085118,0.0003734946,0.006647517,0.0003378363,0.0002822936,0.0002433391,0.006359872,0.0279098],"study_design_scores_gemma":[0.00003776621,0.0005403358,0.9905527,0.00006277137,0.00002712507,0.00011903,0.00652846,0.0002460094,0.00009039267,0.00006750608,0.001701789,0.00002609956],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9925925,0.000249146,0.00003298438,0.000693902,0.00005824852,0.00004240832,0.0003890172,0.000008677709,0.005933153],"genre_scores_gemma":[0.9960834,0.0002576874,0.0001337343,0.0003661942,0.00002325057,0.00002525467,0.0007235128,0.00001184668,0.002375083],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.997013,"threshold_uncertainty_score":0.8502569,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0748325211994584,"score_gpt":0.3696134618103719,"score_spread":0.2947809406109135,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}