{"id":"W4390985458","doi":"10.2196/50842","title":"Performance of ChatGPT on Ophthalmology-Related Questions Across Various Examination Levels: Observational Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":47,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"United States Medical Licensing Examination; Medicine; Ophthalmology; Licensure; Medical school; Medical education; Family medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.008379051,0.0003884982,0.000851648,0.001594478,0.0008597219,0.001175447,0.001030534,0.001014203,0.002187496],"category_scores_gemma":[0.06162155,0.0005017378,0.0009466296,0.001088306,0.0009827857,0.002124719,0.002674953,0.001689784,0.0009299061],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001192363,"about_ca_system_score_gemma":0.001227465,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005942399,"about_ca_topic_score_gemma":0.005750203,"domain_scores_codex":[0.992455,0.002689938,0.001088147,0.001088468,0.001991709,0.000686708],"domain_scores_gemma":[0.9446294,0.02672099,0.01551753,0.002975556,0.006456677,0.003699904],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002836906,0.000879878,0.9833454,0.00009111228,0.00006157238,0.0001183899,0.005838208,0.00012742,0.0002415724,0.00002949782,0.0004884283,0.008494778],"study_design_scores_gemma":[0.00003541688,0.001502711,0.9903645,0.00006017941,0.00005956225,0.0004200044,0.004939858,0.001222801,0.000357388,0.00006873609,0.0009373014,0.00003146241],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.999238,0.00006685515,0.0001067094,0.00003671016,0.000004709865,0.00007263809,0.0001108277,0.000008305215,0.0003552364],"genre_scores_gemma":[0.9989656,0.00007098955,0.0002500692,0.00006162201,0.000008918646,0.0001030549,0.0002440309,0.000007997002,0.0002877403],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.008379051,"threshold_uncertainty_score":0.04431319,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2289267075375783,"score_gpt":0.5170309125922852,"score_spread":0.2881042050547069,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}