{"id":"W4399851271","doi":"10.2196/52784","title":"Influence of Model Evolution and System Roles on ChatGPT’s Performance in Chinese Medical Licensing Exams: Comparative Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008287697,0.0001300789,0.0002820702,0.0002984611,0.00005863734,0.00001434672,0.00007689629,0.0001761439,0.00002569021],"category_scores_gemma":[0.0006020587,0.0001006427,0.00002448267,0.0004805333,0.000124964,0.0001759514,0.00002241206,0.0004385215,0.0000232261],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003247199,"about_ca_system_score_gemma":0.002650877,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004534638,"about_ca_topic_score_gemma":0.0002196128,"domain_scores_codex":[0.9980453,0.00009851568,0.0005710879,0.0002988191,0.0008128607,0.0001733635],"domain_scores_gemma":[0.999115,0.0001931643,0.00007166615,0.0001778196,0.0001571476,0.0002852248],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0006205231,0.004898819,0.6123163,0.00439645,0.00005999593,0.00002373092,0.1518578,0.002581342,0.0002488618,0.004285442,0.0004984509,0.2182124],"study_design_scores_gemma":[0.0001341664,0.0005653487,0.4403506,0.005903218,0.0000232347,0.00005743503,0.02202687,0.530477,0.000158116,0.0001505329,0.00003656646,0.000116866],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9953974,0.0006485431,0.00008151127,0.002295866,0.0003650592,0.0007858139,6.621072e-7,0.00005432494,0.0003707949],"genre_scores_gemma":[0.999089,0.00008493631,0.00003358626,0.0002350605,0.000329759,0.0001644825,0.00001089527,0.00001043656,0.00004181562],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5278957,"threshold_uncertainty_score":0.4702542,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07091824250082696,"score_gpt":0.4584233130009801,"score_spread":0.3875050705001531,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}