{"id":"W4380685958","doi":"10.2196/48002","title":"Performance of GPT-3.5 and GPT-4 on the Japanese Medical Licensing Examination: Comparison Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":289,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Shimane University","keywords":"Transformer; Competence (human resources); Generative grammar; Computer science; Psychology; Medical education; Internal medicine; Artificial intelligence; Medicine; Engineering; Electrical engineering; Social psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001816407,0.000107553,0.0002190843,0.0001643545,0.0001684999,0.00001330094,0.0001215166,0.0001653218,0.0006800072],"category_scores_gemma":[0.002958261,0.0000734196,0.00002889652,0.0006144504,0.0001877282,0.00006236805,0.00003579537,0.0004024036,0.0001081197],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006742551,"about_ca_system_score_gemma":0.001075269,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002972435,"about_ca_topic_score_gemma":0.00005457997,"domain_scores_codex":[0.9976728,0.0001810486,0.0005213896,0.000229636,0.001199692,0.000195431],"domain_scores_gemma":[0.9983681,0.000679268,0.0001241422,0.0002788992,0.0002371499,0.0003123677],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001231066,0.002497719,0.2179122,0.0002684223,0.00002990328,0.000004246562,0.06082637,0.000002761293,0.00004155349,0.0004410384,0.01265266,0.7052],"study_design_scores_gemma":[0.0002123385,0.001642607,0.8564005,0.00118826,0.00005719944,0.00004840835,0.1088072,0.02809837,0.0006182272,0.0000721109,0.002691845,0.0001629137],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9649219,0.00007144244,0.000001963935,0.03231273,0.0008262991,0.0007521637,3.130043e-7,0.0000542181,0.001058982],"genre_scores_gemma":[0.997081,0.0001240973,0.000009177703,0.001504631,0.0006958079,0.0001528424,0.00002823218,0.0000126344,0.0003915381],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7050371,"threshold_uncertainty_score":0.7445602,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1374971887477971,"score_gpt":0.4780372953633461,"score_spread":0.340540106615549,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}