{"id":"W3047987188","doi":"10.1097/acm.0000000000003659","title":"Sound Practices: An Exploratory Study of Building and Monitoring Multiple-Choice Exams at Canadian Undergraduate Medical Education Programs","year":2020,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Innovations in Medical Education","field":"Medicine","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University; McGill University Health Centre; Université de Sherbrooke","funders":"","keywords":"Medical education; Multiple choice; Quality (philosophy); Descriptive statistics; Psychology; Reliability (semiconductor); Licensure; Data collection; Applied psychology; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02080471,0.0007864393,0.0009315499,0.003456144,0.01847885,0.005211398,0.004712557,0.001852066,0.002254083],"category_scores_gemma":[0.0469759,0.00107981,0.0005529613,0.004096177,0.008388846,0.00212662,0.005500281,0.003800221,0.0002738569],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.06948735,"about_ca_system_score_gemma":0.1282376,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.8291944,"about_ca_topic_score_gemma":0.9146184,"domain_scores_codex":[0.9782851,0.008528231,0.0006584718,0.001669687,0.004707221,0.006151251],"domain_scores_gemma":[0.9558986,0.0188732,0.004111578,0.001126501,0.01177107,0.008218989],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"observational","study_design_scores_codex":[0.00004280221,0.0002863959,0.01847492,0.0001808684,0.000006541147,0.0005556032,0.9669759,0.0000451775,0.0006677101,0.0004475815,0.0007376645,0.01157881],"study_design_scores_gemma":[0.000009689698,0.0002375254,0.04262739,0.0002500816,0.00001142523,0.0001225004,0.9472513,0.0002091667,0.0004113007,0.0001263626,0.008703352,0.00003998448],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9941556,0.0002080889,0.0007888119,0.001105764,0.00001975255,0.000563389,0.0001638254,0.00001977837,0.002974961],"genre_scores_gemma":[0.9927718,0.0003931445,0.003124085,0.0008511643,0.00001429773,0.0004152313,0.0001250916,0.00002586482,0.002279251],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9791953,"threshold_uncertainty_score":0.5041684,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.135064679069751,"score_gpt":0.4276285432373625,"score_spread":0.2925638641676115,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}