{"id":"W4401611038","doi":"10.2196/56128","title":"Assessment of ChatGPT-4 in Family Medicine Board Examinations Using Advanced AI Learning and Analytical Methods: Observational Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"","keywords":"Preprint; Observational study; Psychology; Medicine; Medical education; Computer science; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01041305,0.000383404,0.0005696749,0.001960074,0.0008658103,0.001054476,0.000798859,0.0007452722,0.001542423],"category_scores_gemma":[0.04185862,0.0004086368,0.0007697439,0.0008519206,0.001001614,0.001123637,0.002180216,0.001206375,0.0005179975],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00147905,"about_ca_system_score_gemma":0.001787601,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004865316,"about_ca_topic_score_gemma":0.006573376,"domain_scores_codex":[0.9926909,0.003571455,0.0009088928,0.0006483227,0.001537571,0.0006429474],"domain_scores_gemma":[0.9665369,0.01109512,0.00994534,0.002908785,0.005980646,0.003533216],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002115647,0.0009841287,0.9795814,0.00008234054,0.00004825446,0.0002848919,0.00584955,0.0001868049,0.0004711128,0.00005047436,0.0003502536,0.0118994],"study_design_scores_gemma":[0.00002143887,0.001843626,0.9877639,0.00005838248,0.00004066854,0.0006199161,0.006210114,0.001147316,0.0007910804,0.00006693597,0.001402061,0.00003439396],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.999069,0.00006201874,0.0002658772,0.00003022523,0.000004266632,0.0001023011,0.00007568458,0.000008400172,0.0003822353],"genre_scores_gemma":[0.9986612,0.00008274114,0.0007088238,0.00003576838,0.000008429702,0.0001073104,0.000120788,0.000004733923,0.0002701264],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01041305,"threshold_uncertainty_score":0.05507016,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3176348765487993,"score_gpt":0.6310565019612359,"score_spread":0.3134216254124366,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}