{"id":"W4398190964","doi":"10.1148/radiol.232715","title":"Evaluation of Reliability, Repeatability, Robustness, and Confidence of GPT-3.5 and GPT-4 on a Radiology Board–style Examination","year":2024,"lang":"en","type":"article","venue":"Radiology","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":72,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; University Health Network; University of Toronto; Toronto General Hospital; Women's College Hospital; Mount Sinai Hospital","funders":"","keywords":"Medicine; Repeatability; Confidence interval; Reliability (semiconductor); Robustness (evolution); Medical physics; Radiology; Statistics; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004036153,0.0001022756,0.0003509068,0.0001719058,0.0000360688,0.000005187993,0.00004502184,0.0002018135,0.0000836229],"category_scores_gemma":[0.002611456,0.00008861943,0.00003572891,0.0001664053,0.0004954578,0.00006983953,0.0000244848,0.0001691901,0.000002878859],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001235788,"about_ca_system_score_gemma":0.0003151194,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004340839,"about_ca_topic_score_gemma":0.00005736446,"domain_scores_codex":[0.9981129,0.0006054077,0.0005133434,0.0003871461,0.0002315645,0.0001496394],"domain_scores_gemma":[0.9979917,0.001035672,0.0001076292,0.0002973044,0.0005024468,0.00006520554],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0006887009,0.0004690672,0.1115748,0.003168717,0.0002018453,0.00001554792,0.01248317,0.001807,0.02660432,0.03425296,0.001270286,0.8074636],"study_design_scores_gemma":[0.0006588084,0.006796081,0.6190028,0.001255499,0.001172498,0.001310604,0.004116328,0.2780842,0.03462359,0.05171648,0.0008256356,0.0004373915],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9921936,0.003733886,0.000779553,0.001800704,0.0004631334,0.0005641252,0.000005888532,0.0000245998,0.0004345354],"genre_scores_gemma":[0.998628,0.0007691943,0.0002918567,0.00006057883,0.0001394501,0.00003917914,0.00002254456,0.000008916863,0.00004034417],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8070262,"threshold_uncertainty_score":0.3613796,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1407291996203071,"score_gpt":0.427700663767374,"score_spread":0.2869714641470669,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}