{"id":"W4317910576","doi":"10.1101/2023.01.22.23284882","title":"Evaluating the Performance of ChatGPT in Ophthalmology: An Analysis of its Successes and Shortcomings","year":2023,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":104,"is_retracted":false,"has_abstract":true,"ca_institutions":"Centre intégré universitaire de santé et de services sociaux de l'Est-de-l'Île-de-Montréal; Centre Hospitalier de l’Université de Montréal; Université de Montréal; Centre Intégré Universitaire de Santé et de Services Sociaux du Centre-Sud-de-l'Île-de-Montréal; Hôpital Maisonneuve-Rosemont","funders":"","keywords":"Recall; Interpretation (philosophy); Ophthalmic pathology; Space (punctuation); Medicine; Computer science; Optometry; Ophthalmology; Medical physics; Psychology; Neuro-ophthalmology; Glaucoma; Cognitive psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"bench_or_experimental","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"grok","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"opus","categories":[],"domain":null,"study_design":"bench_or_experimental","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001948069,0.0001093992,0.0004985493,0.0004505148,0.0000385786,0.000006239883,0.0001574127,0.0001541535,0.0000574547],"category_scores_gemma":[0.0007076933,0.00008039653,0.00006911426,0.0006560373,0.0001067759,0.00004198269,0.0001213899,0.0003162548,0.000001744373],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003142154,"about_ca_system_score_gemma":0.0002114946,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00244429,"about_ca_topic_score_gemma":0.0003471421,"domain_scores_codex":[0.9985694,0.0001315245,0.0006364085,0.0002798456,0.0002362421,0.0001466325],"domain_scores_gemma":[0.9985949,0.0003880405,0.0002953757,0.0004035487,0.0002714077,0.00004672562],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00005971035,0.00005091626,0.985859,0.0008300646,0.000118661,0.000003218015,0.00477017,0.002400402,0.001311972,0.00001561546,9.318722e-7,0.004579375],"study_design_scores_gemma":[0.00001516025,0.0002729492,0.85649,0.0003538674,0.0004854047,0.000005425101,0.001223851,0.1277352,0.01326083,0.00009490005,0.000001581801,0.00006081957],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9977679,0.0005574798,0.000003923741,0.001004213,0.0002543448,0.0003602794,0.000006683822,0.00001021604,0.00003494281],"genre_scores_gemma":[0.9989496,0.000660933,0.0001027759,0.00002682116,0.0000701947,0.0000535787,0.0000403282,0.00001183446,0.00008391759],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.129369,"threshold_uncertainty_score":0.3695052,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3361547074696869,"score_gpt":0.5181460555126232,"score_spread":0.1819913480429363,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}