{"id":"W4388464830","doi":"10.2196/47191","title":"Assessing the Performance of ChatGPT in Medical Biochemistry Using Clinical Case Vignettes: Observational Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Surprise; Observational study; Clearance; Usability; Medical education; United States Medical Licensing Examination; Vignette; Psychology; Medicine; Computer science; Medical school; Pathology; Social psychology; Human–computer interaction","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02392431,0.0006353217,0.0009556973,0.00229892,0.001408172,0.001828515,0.001329566,0.001476356,0.00217413],"category_scores_gemma":[0.1342539,0.0008156543,0.0009100479,0.001697806,0.001716595,0.002828936,0.002878341,0.002055316,0.0005774777],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002989691,"about_ca_system_score_gemma":0.001861399,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002216981,"about_ca_topic_score_gemma":0.00313342,"domain_scores_codex":[0.9690267,0.02312362,0.002549716,0.001297371,0.002993829,0.001008782],"domain_scores_gemma":[0.8697979,0.08389341,0.02797804,0.003969238,0.009146747,0.005214734],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00111484,0.006982965,0.7439272,0.002187254,0.0002104841,0.005150664,0.1998353,0.0007903267,0.002071149,0.0004112512,0.002143555,0.03517503],"study_design_scores_gemma":[0.0003233756,0.01130385,0.73234,0.001136422,0.0002977318,0.008014864,0.2271103,0.008508645,0.002993711,0.0004883697,0.007130376,0.0003523972],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9984768,0.0001280171,0.0005336164,0.00009733194,0.000008304707,0.0004677907,0.0000560967,0.000007722751,0.0002243307],"genre_scores_gemma":[0.9956182,0.0002777142,0.002603028,0.0001599717,0.0000255641,0.0009534141,0.0001643157,0.00001124865,0.0001865641],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02392431,"threshold_uncertainty_score":0.1265253,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4198719497903217,"score_gpt":0.6080220114771049,"score_spread":0.1881500616867832,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}