{"id":"W4389379994","doi":"10.2196/52202","title":"Performance Comparison of ChatGPT-4 and Japanese Medical Residents in the General Medicine In-Training Examination: Comparison Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":63,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Chiba University","keywords":"Expansive; Reliability (semiconductor); Medical knowledge; Training (meteorology); Medical education; Psychology; Computer science; Medicine; Geography","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001847937,0.0003245936,0.0003788034,0.0009939469,0.0004502331,0.0004976371,0.0003301298,0.0004917261,0.0009978028],"category_scores_gemma":[0.00613795,0.0002030344,0.0004698403,0.0005049649,0.0004567572,0.0006026005,0.001200275,0.0002565117,0.0003903891],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005256625,"about_ca_system_score_gemma":0.0006453666,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007141071,"about_ca_topic_score_gemma":0.01370687,"domain_scores_codex":[0.9989384,0.0002756531,0.0001469573,0.0002397151,0.0002344188,0.0001648158],"domain_scores_gemma":[0.9967044,0.0005170096,0.0006821013,0.0001877456,0.0009473887,0.0009613127],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0003224069,0.0001591242,0.9874222,0.00004659925,0.00007337044,0.0001439854,0.003152158,0.00006606158,0.0009649408,0.000023288,0.0001552375,0.007470561],"study_design_scores_gemma":[0.00001026151,0.0004481749,0.996466,0.0000087851,0.00003263958,0.0002159317,0.002007375,0.0003075343,0.0002316351,0.00001427507,0.0002493429,0.000008014234],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9996706,0.00002961667,0.0000332914,0.000008964897,0.000003938783,0.000005794701,0.00002216816,0.000001668957,0.0002240393],"genre_scores_gemma":[0.9993136,0.00004993768,0.0001827082,0.00001715173,0.000006781721,0.00001371627,0.000118576,0.000001862304,0.0002957036],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.007141071,"threshold_uncertainty_score":0.01419902,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2033749244333059,"score_gpt":0.52524413778268,"score_spread":0.3218692133493741,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}