{"id":"W4417534853","doi":"10.1371/journal.pdig.0001156","title":"Evaluating ChatGPT-4 in the development of family medicine residency examinations","year":2025,"lang":"en","type":"article","venue":"PLOS Digital Health","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"St. Michael's Hospital; University of Toronto; University Health Network","funders":"","keywords":"Quality (philosophy); Educational measurement; MEDLINE; Medical school; Residency training","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.03756479,0.0009785985,0.0007785233,0.001695796,0.0005349594,0.001703151,0.001457908,0.0011662,0.002686067],"category_scores_gemma":[0.164304,0.0005383441,0.00116274,0.0007857183,0.0007018389,0.001658282,0.002574714,0.001160246,0.001205925],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00175244,"about_ca_system_score_gemma":0.002019956,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00146078,"about_ca_topic_score_gemma":0.003164413,"domain_scores_codex":[0.968249,0.02312831,0.002286687,0.001955052,0.003779183,0.0006017641],"domain_scores_gemma":[0.7400995,0.2044564,0.01700842,0.009103669,0.02312287,0.006209166],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.009265335,0.006535541,0.1835128,0.003833325,0.0006341884,0.0009026886,0.01593166,0.01672235,0.04243317,0.0009089589,0.0108321,0.708488],"study_design_scores_gemma":[0.003590904,0.04976309,0.6639001,0.001711541,0.001369848,0.003617041,0.009572842,0.1196667,0.08844463,0.003105641,0.05419033,0.001067302],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9603243,0.0004228679,0.02929391,0.0004767923,0.0001273486,0.003059758,0.0006058144,0.00209796,0.003591393],"genre_scores_gemma":[0.8218957,0.0004291358,0.1682004,0.0006132682,0.0001189392,0.00361264,0.001937086,0.00036885,0.002823996],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9624352,"threshold_uncertainty_score":0.1986639,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4848804161110564,"score_gpt":0.5470551834505694,"score_spread":0.06217476733951299,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}