{"id":"W4388464830","doi":"10.2196/47191","title":"Assessing the Performance of ChatGPT in Medical Biochemistry Using Clinical Case Vignettes: Observational Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Surprise; Observational study; Clearance; Usability; Medical education; United States Medical Licensing Examination; Vignette; Psychology; Medicine; Computer science; Medical school; Pathology; Social psychology; Human–computer interaction","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003364884,0.000106122,0.0002453814,0.0001374346,0.0001308364,0.00001868309,0.0001576923,0.0002689224,0.0005110758],"category_scores_gemma":[0.005025584,0.00007855205,0.00005917954,0.0009941072,0.0002247865,0.0001466082,0.00004805014,0.0005998276,0.00002563219],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001325013,"about_ca_system_score_gemma":0.006986551,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006124953,"about_ca_topic_score_gemma":0.00006387882,"domain_scores_codex":[0.9972027,0.0001964476,0.001054147,0.0002720626,0.001040839,0.000233809],"domain_scores_gemma":[0.9981805,0.0007574231,0.0001888162,0.0003069958,0.000264358,0.0003018808],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00003104419,0.001845633,0.8360547,0.0002123649,0.0000147704,0.00009083487,0.003152057,0.000009071735,0.00007756487,0.00001800605,0.001334106,0.1571598],"study_design_scores_gemma":[0.0003473984,0.000407352,0.8211472,0.001552535,0.00005931037,0.001025616,0.05667513,0.117038,0.0008643412,0.000111751,0.0005722977,0.0001990649],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9848962,0.00007930662,0.00001833732,0.01325356,0.001056761,0.0005857151,5.477994e-7,0.00003366007,0.00007592093],"genre_scores_gemma":[0.9965634,0.00006575698,0.00009722772,0.001727878,0.001276662,0.0001314712,0.00004879692,0.00001307247,0.00007572171],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1569607,"threshold_uncertainty_score":0.9986429,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4198719497903217,"score_gpt":0.6080220114771049,"score_spread":0.1881500616867832,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}