{"id":"W4387305434","doi":"10.2196/48023","title":"Accuracy of ChatGPT on Medical Questions in the National Medical Licensing Examination in Japan: Evaluation Study","year":2023,"lang":"en","type":"article","venue":"JMIR Formative Research","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":106,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Chiba University","keywords":"Medical diagnosis; Christian ministry; Welfare; Reliability (semiconductor); Medical education; Psychology; Medicine; Political science; Law; Pathology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02794395,0.00007241629,0.0001541804,0.001156001,0.0001434802,0.00001835232,0.000184791,0.0001431352,0.000295851],"category_scores_gemma":[0.02039658,0.00005165339,0.00002705875,0.002533223,0.0001535474,0.0002182472,0.00004879883,0.0009035325,0.0001739416],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004481164,"about_ca_system_score_gemma":0.001453754,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00063471,"about_ca_topic_score_gemma":0.001031621,"domain_scores_codex":[0.9912384,0.001965787,0.0006333594,0.0001853078,0.005666679,0.0003104432],"domain_scores_gemma":[0.9941337,0.004326371,0.00008348676,0.000161453,0.001190138,0.0001048986],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001765767,0.003254797,0.04160453,0.0001655484,0.00001629454,0.00001863427,0.2511573,0.00007523548,0.00005858664,0.00290378,0.003066467,0.6975023],"study_design_scores_gemma":[0.0002873244,0.0008330173,0.8647758,0.000563644,0.000002874448,0.000008775612,0.06617896,0.06543278,0.0001977043,0.001538194,0.0001330624,0.00004782422],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9799296,0.00002501754,0.000009585844,0.01595511,0.0001318416,0.001620143,0.000001705408,0.00001524615,0.002311728],"genre_scores_gemma":[0.9988645,0.00009011378,0.000003873549,0.0002099585,0.0001856754,0.0005604076,0.00005542609,0.00000658,0.0000234705],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8231713,"threshold_uncertainty_score":0.987855,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5099773349532368,"score_gpt":0.6438245307259427,"score_spread":0.1338471957727059,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}