{"id":"W4387124372","doi":"10.1177/23821205231204178","title":"Examining the Threat of ChatGPT to the Validity of Short Answer Assessments in an Undergraduate Medical Program","year":2023,"lang":"en","type":"article","venue":"Journal of Medical Education and Curricular Development","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":30,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"Cronbach's alpha; Bloom's taxonomy; Psychology; Taxonomy (biology); Cognition; Class (philosophy); Mathematics education; Medical education; Clinical psychology; Computer science; Psychometrics; Medicine; Artificial intelligence; Ecology; Psychiatry","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04732705,0.0004495163,0.000470409,0.001346746,0.0008123906,0.002095837,0.001125099,0.0009133849,0.0009084983],"category_scores_gemma":[0.3052572,0.0003325123,0.0005732605,0.0007865034,0.001317123,0.001902957,0.003143712,0.001320858,0.0003321097],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001570116,"about_ca_system_score_gemma":0.0019763,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002306242,"about_ca_topic_score_gemma":0.002645096,"domain_scores_codex":[0.9497475,0.02533671,0.00464916,0.002837127,0.01628794,0.001141461],"domain_scores_gemma":[0.654555,0.2492607,0.04021013,0.01242747,0.0373235,0.006223133],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001621873,0.0008070733,0.8674098,0.0003375323,0.0001424621,0.0004052165,0.02216115,0.001462532,0.003825261,0.0002290268,0.0006897246,0.1009084],"study_design_scores_gemma":[0.00009812727,0.008372488,0.9524001,0.0003426102,0.0001000397,0.00107923,0.010235,0.0136555,0.01060342,0.0006593802,0.002355147,0.00009894274],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9977745,0.00005630059,0.001181934,0.0001186153,0.00001830481,0.0000884922,0.00002752437,0.00003173941,0.0007025872],"genre_scores_gemma":[0.997696,0.00003433801,0.001810015,0.00005495069,0.00001466801,0.00007299987,0.00005560819,0.000009073746,0.0002524292],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.952673,"threshold_uncertainty_score":0.2502924,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2265374218918685,"score_gpt":0.5077038086447077,"score_spread":0.2811663867528392,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}