{"id":"W4403349599","doi":"10.1016/j.bas.2024.103540","title":"Can ChatGPT pass the Canadian Royal College written examination in Neurosurgery? A performance comparison of GPT-4 and PGY-6 residents","year":2024,"lang":"en","type":"article","venue":"Brain and Spine","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Manitoba; Ottawa Hospital; University of Ottawa","funders":"","keywords":"Medical education; Psychology; Medicine; Mathematics education","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000393443,0.00006147286,0.0001508666,0.0001692734,0.00009131086,0.00002197127,0.0000270449,0.00005178631,0.00001257083],"category_scores_gemma":[0.000129301,0.00004557858,0.00001337795,0.0002270372,0.00006617784,0.00003359695,0.000009067359,0.0001284624,0.000001887801],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005075453,"about_ca_system_score_gemma":0.0002363553,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.06761125,"about_ca_topic_score_gemma":0.2115532,"domain_scores_codex":[0.9993527,0.00004571705,0.00024202,0.0001284443,0.00009487286,0.0001362799],"domain_scores_gemma":[0.9996662,0.0000960291,0.00002662436,0.0000820206,0.00003466268,0.0000944407],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00002422891,0.00004619651,0.2660936,0.0009817143,0.00001192278,0.00002582476,0.008752737,0.000006114947,0.0001354983,0.001258091,0.002205824,0.7204583],"study_design_scores_gemma":[0.00004831805,0.0001794173,0.9689522,0.0005837075,0.00001550423,0.00002236183,0.001072886,0.01899352,0.0003825021,0.00008243003,0.009600591,0.00006661249],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9724718,0.007594824,0.000002069828,0.01923063,0.0001558615,0.0002484012,0.000005286063,0.000008284235,0.0002828521],"genre_scores_gemma":[0.9973369,0.001589566,0.00000987876,0.0004297699,0.00007596476,0.00001500965,0.00001026602,0.000006705176,0.0005259278],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7203917,"threshold_uncertainty_score":0.9385976,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0748325211994584,"score_gpt":0.3696134618103719,"score_spread":0.2947809406109135,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}