{"id":"W4396508255","doi":"10.36834/cmej.77596","title":"The impact of systematically repairing multiple choice questions with low discrimination on assessment reliability: an interrupted time series analysis","year":2024,"lang":"en","type":"article","venue":"Canadian Medical Education Journal","topic":"Medical Education and Admissions","field":"Medicine","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"","keywords":"Summative assessment; Reliability (semiconductor); Quality management; Consistency (knowledge bases); Regression analysis; Medicine; Multiple choice; Statistics; Psychology; Computer science; Significant difference; Mathematics; Internal medicine; Formative assessment; Operations management; Engineering; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.07195415,0.0004236846,0.0007196057,0.001713716,0.0006267782,0.001326092,0.001209306,0.0005994707,0.001376434],"category_scores_gemma":[0.235095,0.0003294585,0.001114523,0.002396942,0.0008894624,0.0008505244,0.001306683,0.001494392,0.0002179052],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002101886,"about_ca_system_score_gemma":0.002120851,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01006061,"about_ca_topic_score_gemma":0.007527858,"domain_scores_codex":[0.9367139,0.04144594,0.005340757,0.003679414,0.0114511,0.001368954],"domain_scores_gemma":[0.6722081,0.2393737,0.04923544,0.01588786,0.02063653,0.00265838],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.004095428,0.001347527,0.8749734,0.0005509203,0.001397479,0.0002334831,0.009055417,0.004073863,0.001798633,0.0007050744,0.001248901,0.1005198],"study_design_scores_gemma":[0.0001024015,0.003646423,0.9684986,0.0001448925,0.0005559516,0.0001404777,0.001619208,0.021625,0.001744418,0.0004741616,0.001384923,0.00006360599],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9846602,0.000379881,0.01313161,0.0003055102,0.00005035784,0.0002886838,0.0002779331,0.00005312207,0.000852633],"genre_scores_gemma":[0.9945352,0.00008080995,0.004277754,0.00006940332,0.00002567815,0.000420953,0.0003223458,0.00001869218,0.000249186],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.07195415,"threshold_uncertainty_score":0.3805345,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01210954641671757,"score_gpt":0.3767930397346411,"score_spread":0.3646834933179235,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}