{"id":"W4387653139","doi":"10.1080/0142159x.2023.2249588","title":"ChatGPT-4: An assessment of an upgraded artificial intelligence chatbot in the United States Medical Licensing Examination","year":2023,"lang":"en","type":"article","venue":"Medical Teacher","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":112,"is_retracted":false,"has_abstract":true,"ca_institutions":"St. Michael's Hospital; University of Toronto","funders":"","keywords":"United States Medical Licensing Examination; Licensure; Medicine; Medical education; Multiple choice; Test (biology); Family medicine; Psychology; Medical school; Significant difference; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02606907,0.001125658,0.001046373,0.002621697,0.001351694,0.002409964,0.002576918,0.002080236,0.003664434],"category_scores_gemma":[0.1317699,0.0009445821,0.0009953433,0.001354868,0.001092207,0.00383899,0.005358359,0.002224293,0.002294288],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003452116,"about_ca_system_score_gemma":0.004134296,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007017177,"about_ca_topic_score_gemma":0.01176237,"domain_scores_codex":[0.97374,0.01614497,0.002258618,0.001830449,0.00463522,0.001390558],"domain_scores_gemma":[0.8742996,0.07230311,0.01120508,0.007859815,0.02028324,0.01404914],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.008772071,0.01549256,0.5170165,0.00264229,0.0005474827,0.001264866,0.04014695,0.006731154,0.008337187,0.0008009116,0.02690422,0.3713437],"study_design_scores_gemma":[0.0009509408,0.02281496,0.8904375,0.0008458449,0.0004367465,0.001117024,0.01442511,0.03115254,0.005357847,0.0009096157,0.03103181,0.0005200192],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9899336,0.0001082418,0.002526376,0.0004144023,0.00008974162,0.001666048,0.0008838967,0.001221042,0.003156799],"genre_scores_gemma":[0.9620433,0.0002689213,0.02284207,0.0008207529,0.0001523603,0.004164358,0.004597905,0.0003164415,0.004793808],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02606907,"threshold_uncertainty_score":0.137868,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2621084780040356,"score_gpt":0.5165753969505843,"score_spread":0.2544669189465487,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}