{"id":"W4390985458","doi":"10.2196/50842","title":"Performance of ChatGPT on Ophthalmology-Related Questions Across Various Examination Levels: Observational Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":47,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"United States Medical Licensing Examination; Medicine; Ophthalmology; Licensure; Medical school; Medical education; Family medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001063383,0.0001295975,0.0001975137,0.0001949092,0.0001339514,0.00002061846,0.0001045786,0.0002397764,0.001147376],"category_scores_gemma":[0.001117421,0.0001142468,0.00005338939,0.0006471655,0.0001220679,0.0001737777,0.00001876494,0.0004315542,0.0001899275],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002448353,"about_ca_system_score_gemma":0.002518379,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002389005,"about_ca_topic_score_gemma":0.00001155069,"domain_scores_codex":[0.9980312,0.0001259101,0.0006367041,0.0003218979,0.0006740275,0.0002102512],"domain_scores_gemma":[0.9987709,0.0002855455,0.00009804362,0.00025503,0.0003892427,0.0002012521],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001177348,0.006615671,0.1670719,0.0007076259,0.00009038798,0.00002230683,0.03348609,0.0000324839,0.0002324861,0.003774279,0.001865847,0.7859832],"study_design_scores_gemma":[0.00009467771,0.001488506,0.9813198,0.0009481208,0.00004168824,0.0001858222,0.004225662,0.009766608,0.0003614548,0.0005507887,0.000894725,0.000122083],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9888192,0.0002519218,0.00002243974,0.00617625,0.002496016,0.0009189378,0.000005081991,0.00008996587,0.001220158],"genre_scores_gemma":[0.9966108,0.00003949871,0.00007840787,0.0003765864,0.0004968355,0.0004041299,0.0001428894,0.00001939094,0.001831439],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.814248,"threshold_uncertainty_score":0.9997657,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2289267075375783,"score_gpt":0.5170309125922852,"score_spread":0.2881042050547069,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}