{"id":"W4380685958","doi":"10.2196/48002","title":"Performance of GPT-3.5 and GPT-4 on the Japanese Medical Licensing Examination: Comparison Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":289,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Shimane University","keywords":"Transformer; Competence (human resources); Generative grammar; Computer science; Psychology; Medical education; Internal medicine; Artificial intelligence; Medicine; Engineering; Electrical engineering; Social psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00540562,0.001144814,0.0007273371,0.001119092,0.0003412121,0.0009332346,0.001213642,0.001350065,0.00215512],"category_scores_gemma":[0.01937244,0.000366172,0.001159413,0.000464903,0.000458585,0.001865824,0.002074213,0.001246287,0.001142638],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001294156,"about_ca_system_score_gemma":0.001627615,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01350542,"about_ca_topic_score_gemma":0.01040479,"domain_scores_codex":[0.9978328,0.0009941808,0.0001784143,0.0005070558,0.0003104517,0.0001770205],"domain_scores_gemma":[0.9925717,0.00436544,0.0003114717,0.0007122565,0.001329832,0.0007094295],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.008446206,0.002550291,0.4450471,0.0008960633,0.0009560551,0.0006880827,0.002289957,0.04389581,0.007056585,0.0008170756,0.01012684,0.4772299],"study_design_scores_gemma":[0.0006848263,0.006262982,0.3968668,0.0002005207,0.001166837,0.001411824,0.001776648,0.571263,0.01264451,0.002277886,0.005208181,0.0002361012],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9876139,0.0004040132,0.007443324,0.0002108037,0.00007983156,0.0002651765,0.0008265837,0.001005825,0.002150563],"genre_scores_gemma":[0.9900888,0.0001645053,0.006263377,0.0001069235,0.00002198937,0.000141163,0.002150421,0.00004021253,0.00102276],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01350542,"threshold_uncertainty_score":0.028588,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1374971887477971,"score_gpt":0.4780372953633461,"score_spread":0.340540106615549,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}