{"id":"W4398190964","doi":"10.1148/radiol.232715","title":"Evaluation of Reliability, Repeatability, Robustness, and Confidence of GPT-3.5 and GPT-4 on a Radiology Board–style Examination","year":2024,"lang":"en","type":"article","venue":"Radiology","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":72,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; University Health Network; University of Toronto; Toronto General Hospital; Women's College Hospital; Mount Sinai Hospital","funders":"","keywords":"Medicine; Repeatability; Confidence interval; Reliability (semiconductor); Robustness (evolution); Medical physics; Radiology; Statistics; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02988531,0.0009643697,0.0007669529,0.001567432,0.0004102726,0.001258457,0.00126289,0.001425811,0.001222963],"category_scores_gemma":[0.1002777,0.0005681423,0.001114196,0.0007916763,0.0009615967,0.0007997837,0.001316615,0.0007983167,0.001013768],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004569789,"about_ca_system_score_gemma":0.0005026097,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00159975,"about_ca_topic_score_gemma":0.002371956,"domain_scores_codex":[0.9797442,0.01008519,0.002108099,0.002315006,0.005278726,0.0004687015],"domain_scores_gemma":[0.9144611,0.04815983,0.0059582,0.007836859,0.02239625,0.001187709],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.01575848,0.001263608,0.6161987,0.001603442,0.002442729,0.000737168,0.00673863,0.01134085,0.079118,0.0008942569,0.00822526,0.2556789],"study_design_scores_gemma":[0.0006517695,0.008525765,0.8450847,0.0003621386,0.001664613,0.003630314,0.001278005,0.04861085,0.07909374,0.00120292,0.009593429,0.0003017492],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.949245,0.001202176,0.04289406,0.0002089356,0.0002284986,0.0007837228,0.0009104974,0.001180369,0.003346743],"genre_scores_gemma":[0.9646751,0.0002293517,0.03235603,0.0001660108,0.00006847683,0.0005360807,0.0008258591,0.0002859341,0.0008571434],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9701147,"threshold_uncertainty_score":0.1580505,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1407291996203071,"score_gpt":0.427700663767374,"score_spread":0.2869714641470669,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}