{"id":"W4386457251","doi":"10.2196/50514","title":"Assessment of Resident and AI Chatbot Performance on the University of Toronto Family Medicine Residency Progress Test: Comparative Study","year":2023,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":58,"is_retracted":false,"has_abstract":true,"ca_institutions":"Hospital for Sick Children; University of Toronto","funders":"","keywords":"McNemar's test; Test (biology); Medicine; Pace; Multiple choice; Medical education; Chatbot; Family medicine; Cohort; Psychology; Artificial intelligence; Computer science; Internal medicine; Significant difference; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009020193,0.00008635875,0.0002392013,0.00006647875,0.0001241418,0.000003053612,0.0001145029,0.00006885507,0.0004514653],"category_scores_gemma":[0.0004269954,0.00005911106,0.00002107991,0.0002611358,0.0003357614,0.00007954574,0.00003147788,0.0001897565,0.000009880957],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001431395,"about_ca_system_score_gemma":0.001664728,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003557205,"about_ca_topic_score_gemma":0.0006814761,"domain_scores_codex":[0.9985299,0.0001219081,0.0003510202,0.0001885268,0.000680915,0.0001277415],"domain_scores_gemma":[0.9986278,0.0004568825,0.0001625096,0.0002601055,0.0003409163,0.0001517501],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001920637,0.002170312,0.8165617,0.0004643169,0.00004592702,0.000003688145,0.07923017,0.000001256248,0.0002549506,0.002506526,0.04186194,0.05670717],"study_design_scores_gemma":[0.0001060285,0.001776653,0.8642437,0.0006885641,0.00003131566,0.000001917659,0.1315684,0.0007485577,0.0001505457,0.0001520869,0.0004898176,0.00004233559],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9689605,0.0003147643,0.000003917779,0.02756769,0.00028197,0.001076544,9.536514e-7,0.00002140557,0.001772207],"genre_scores_gemma":[0.9982845,0.0006436075,0.00002821261,0.0003520808,0.0001868294,0.00004283191,0.00001471029,0.00000532509,0.0004419665],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05666484,"threshold_uncertainty_score":0.5377453,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.15186098784238,"score_gpt":0.5025409286234617,"score_spread":0.3506799407810817,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}