{"id":"W4389379994","doi":"10.2196/52202","title":"Performance Comparison of ChatGPT-4 and Japanese Medical Residents in the General Medicine In-Training Examination: Comparison Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":63,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Chiba University","keywords":"Expansive; Reliability (semiconductor); Medical knowledge; Training (meteorology); Medical education; Psychology; Computer science; Medicine; Geography","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003728871,0.0001307467,0.0004282018,0.0004611053,0.0000839924,0.000009283342,0.0002128695,0.000198651,0.000296326],"category_scores_gemma":[0.00289926,0.00009365833,0.00002033933,0.00121877,0.0002475731,0.0001049374,0.00003829642,0.0005760225,0.00002014166],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00009483992,"about_ca_system_score_gemma":0.001028742,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003196354,"about_ca_topic_score_gemma":0.001528416,"domain_scores_codex":[0.996496,0.000353133,0.001059045,0.0002826767,0.001536264,0.0002729185],"domain_scores_gemma":[0.9986337,0.0005514103,0.000171435,0.000257205,0.0001414108,0.0002448236],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00004175506,0.0009628402,0.5927741,0.0001479933,0.000004311788,0.000005176991,0.1879372,0.000002857793,0.00001133189,0.00004920321,0.001502397,0.2165608],"study_design_scores_gemma":[0.0002728046,0.0005098645,0.7918932,0.0006865597,0.00001278578,0.00001893618,0.189614,0.01666602,0.00002572178,0.0000497469,0.0001867935,0.00006355516],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9758956,0.0002746684,0.000001959196,0.02137663,0.0006823765,0.0009198358,2.206104e-7,0.00002557797,0.0008231339],"genre_scores_gemma":[0.997844,0.0001746087,0.00001539022,0.0007611194,0.0007076731,0.0002900168,0.00006509667,0.00001122541,0.000130809],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2164972,"threshold_uncertainty_score":0.4831952,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2033749244333059,"score_gpt":0.52524413778268,"score_spread":0.3218692133493741,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}