{"id":"W4384120659","doi":"10.1001/jamaophthalmol.2023.2754","title":"Performance of an Upgraded Artificial Intelligence Chatbot for Ophthalmic Knowledge Assessment","year":2023,"lang":"en","type":"letter","venue":"JAMA Ophthalmology","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":64,"is_retracted":false,"has_abstract":true,"ca_institutions":"St. Michael's Hospital; University of Toronto; Western University","funders":"","keywords":"Medicine; Chatbot; Medical physics; Optometry; Artificial intelligence; Computer science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01658127,0.0004162323,0.0004832996,0.001715024,0.001285041,0.00181906,0.0009216509,0.002158653,0.003824102],"category_scores_gemma":[0.127701,0.0005609259,0.000394268,0.0008451726,0.0005453684,0.001966736,0.001355855,0.002188641,0.003714207],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002084537,"about_ca_system_score_gemma":0.00173879,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008231164,"about_ca_topic_score_gemma":0.009738141,"domain_scores_codex":[0.9811321,0.009800439,0.002199366,0.001342908,0.004612009,0.0009132783],"domain_scores_gemma":[0.8175396,0.1211431,0.01929866,0.009425593,0.02740956,0.005183416],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001497686,0.008115882,0.893003,0.0001524706,0.00009925278,0.001001555,0.01017186,0.0003521005,0.002775553,0.0001386648,0.01183343,0.07085855],"study_design_scores_gemma":[0.0002209597,0.01067117,0.9607409,0.0001926655,0.0001153311,0.001904618,0.006496814,0.007378351,0.002833876,0.000188002,0.009127384,0.0001299297],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9920209,0.00006575279,0.0006010074,0.001600741,0.0001671691,0.0003346134,0.0004669123,0.0001575184,0.004585301],"genre_scores_gemma":[0.9895034,0.0000907222,0.002274514,0.002873731,0.0001692794,0.0004981065,0.0005176205,0.00005967362,0.004012951],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01658127,"threshold_uncertainty_score":0.08769119,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3044060482541513,"score_gpt":0.4875269611308485,"score_spread":0.1831209128766972,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}