{"id":"W4407920886","doi":"10.5489/cuaj.9020","title":"Use of AI (GPT-4)-generated multiple-choice questions for the examination of surgical subspecialty residents","year":2025,"lang":"en","type":"article","venue":"Canadian Urological Association Journal","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"University Health Network; Hospital for Sick Children; Women's College Hospital; University of Toronto","funders":"","keywords":"Subspecialty; United States Medical Licensing Examination; Multiple choice; Medical education; Medicine; Inclusion (mineral); Educational measurement; Medical school; Quality (philosophy); Graduate medical education; Urology; Medical physics; Curriculum; Family medicine; Internal medicine; Psychology; Pedagogy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02186863,0.000876448,0.0004697552,0.002769578,0.0006194486,0.001620332,0.001509705,0.001087929,0.008547353],"category_scores_gemma":[0.1066291,0.0003515467,0.001092517,0.001332466,0.001005522,0.001784374,0.002273572,0.001030807,0.002392651],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002879321,"about_ca_system_score_gemma":0.003778937,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004159679,"about_ca_topic_score_gemma":0.005406887,"domain_scores_codex":[0.9798679,0.01316263,0.001631553,0.001205878,0.003717208,0.0004147668],"domain_scores_gemma":[0.8912848,0.08156685,0.009733329,0.005682021,0.01031354,0.001419584],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00201103,0.003087759,0.2776362,0.001874512,0.0002070884,0.0005881281,0.0123756,0.02036767,0.008857708,0.004414329,0.02650705,0.642073],"study_design_scores_gemma":[0.001376156,0.01020426,0.5957811,0.002319289,0.0004961498,0.00300057,0.00907284,0.2042531,0.04058303,0.02049317,0.1117314,0.0006888423],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7980152,0.0003855176,0.1401366,0.002935563,0.0002246025,0.01807131,0.004857292,0.004169106,0.03120484],"genre_scores_gemma":[0.7279611,0.000327251,0.2564094,0.0006639814,0.00006063172,0.008212646,0.002592201,0.0001471774,0.003625548],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9781314,"threshold_uncertainty_score":0.1156538,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1269181480839119,"score_gpt":0.3828612384382713,"score_spread":0.2559430903543594,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}