{"id":"W4401632453","doi":"10.1016/j.acra.2024.08.009","title":"Evaluating Artificial Intelligence Competency in Education: Performance of ChatGPT-4 in the American Registry of Radiologic Technologists (ARRT) Radiography Certification Exam","year":2024,"lang":"en","type":"article","venue":"Academic Radiology","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University; Juravinski Hospital; Western University; University of British Columbia; McMaster University Medical Centre; Hamilton Health Sciences; University of Toronto; Trillium Health Centre","funders":"","keywords":"Certification; Medical physics; Radiography; Medicine; Medical education; Radiology; Management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00225875,0.0001632167,0.0005009799,0.001497104,0.00004503324,0.000005622668,0.0004374312,0.0002994365,0.0000163506],"category_scores_gemma":[0.0009783409,0.0001248571,0.00008968456,0.003822521,0.000998313,0.00009677022,0.00002327078,0.001083863,0.00001030454],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001353584,"about_ca_system_score_gemma":0.0007462321,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008292764,"about_ca_topic_score_gemma":0.00003467608,"domain_scores_codex":[0.9973252,0.0003517702,0.001356046,0.0004297989,0.000210338,0.0003267759],"domain_scores_gemma":[0.9980716,0.0008419647,0.0004221718,0.0005012275,0.0001209907,0.00004201832],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0002479721,0.0001768482,0.2376633,0.0004712591,0.00002165157,0.000005876153,0.005515549,0.000374611,0.02544297,0.01437562,0.0003044156,0.7153999],"study_design_scores_gemma":[0.0001269841,0.006108129,0.6741342,0.003300286,0.0002169596,0.001824858,0.05319075,0.05773313,0.1139057,0.08647844,0.002157606,0.0008229429],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9817404,0.008526198,0.0001739294,0.008071994,0.0005452775,0.0006545203,0.000002520995,0.00004168765,0.0002434314],"genre_scores_gemma":[0.9885943,0.009320748,0.001273607,0.0003000709,0.0002737903,0.0001824311,0.00003074505,0.00001208117,0.00001220875],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.714577,"threshold_uncertainty_score":0.5091527,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.189072774425262,"score_gpt":0.4709775486186016,"score_spread":0.2819047741933396,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}