{"id":"W4391026420","doi":"10.3390/ime3010004","title":"Examining the Efficacy of ChatGPT in Marking Short-Answer Assessments in an Undergraduate Medical Program","year":2024,"lang":"en","type":"article","venue":"International Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":27,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"Rubric; Reliability (semiconductor); Medical education; Curriculum; Psychology; Computer science; Inter-rater reliability; Medicine; Mathematics education; Rating scale; Pedagogy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.05157968,0.0006933619,0.0006219563,0.002484776,0.0008364895,0.002257751,0.001438271,0.0007891125,0.001314358],"category_scores_gemma":[0.2123687,0.0003986742,0.0006277069,0.001170891,0.00103635,0.002327871,0.003090532,0.0009024862,0.0007294588],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001243416,"about_ca_system_score_gemma":0.001521054,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002129109,"about_ca_topic_score_gemma":0.006596153,"domain_scores_codex":[0.945327,0.03922158,0.002775318,0.00409799,0.007840144,0.0007378196],"domain_scores_gemma":[0.7162124,0.2157787,0.01805013,0.01331223,0.03232686,0.004319753],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.003496254,0.002028478,0.446068,0.0009393577,0.0003278828,0.0002576502,0.02549407,0.00447097,0.0196586,0.0009700476,0.002382248,0.4939065],"study_design_scores_gemma":[0.0004202764,0.01761767,0.8641589,0.0005353297,0.000424683,0.0009340492,0.01340032,0.06158813,0.03041647,0.001966792,0.008105731,0.0004317354],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9714842,0.0001120008,0.02406229,0.0001029463,0.00007110207,0.0007593731,0.0001120452,0.0003167202,0.002979367],"genre_scores_gemma":[0.9470696,0.0001049803,0.05054555,0.00009597952,0.00005092954,0.000726946,0.000205323,0.00006110577,0.001139588],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9484203,"threshold_uncertainty_score":0.2727827,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1665316257906498,"score_gpt":0.5365099278259731,"score_spread":0.3699783020353232,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}