{"id":"W2911682672","doi":"10.1097/acm.0000000000002627","title":"Ensuring the Quality of Multiple-Choice Tests: An Algorithm to Facilitate Decision Making for Difficult Questions","year":2019,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Educational Technology and Assessment","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"ca_institutions":"McGill University","funders":"","keywords":"Computer science; Quality (philosophy); Management science; Machine learning; Algorithm; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00143887,0.00009673443,0.0001840718,0.00009292306,0.0001030146,0.000004899524,0.0008799854,0.0001053297,0.000008254625],"category_scores_gemma":[0.00355939,0.00006113272,0.00002303212,0.0003372249,0.0000760857,0.0001660038,0.000140825,0.0003030394,0.00001046346],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000445335,"about_ca_system_score_gemma":0.00005379903,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006884764,"about_ca_topic_score_gemma":0.0000142613,"domain_scores_codex":[0.9987331,0.0000750139,0.0004080726,0.0003048023,0.0002967194,0.0001822973],"domain_scores_gemma":[0.9938757,0.005230665,0.0001469459,0.0005025626,0.0001841279,0.00005999956],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00003354188,0.0001071629,0.1166801,0.0001106469,0.00005909227,7.247697e-7,0.01489893,0.0009268797,0.03955163,0.2506863,0.00485524,0.5720897],"study_design_scores_gemma":[0.001655696,0.000661171,0.8417458,0.00104817,0.00002626231,0.00001640193,0.004306086,0.04706486,0.001385285,0.08333132,0.01837731,0.0003816159],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4199911,0.0001591054,0.5713033,0.007666435,0.0003924293,0.0003746179,0.000006559869,0.00006070202,0.0000457144],"genre_scores_gemma":[0.9482381,0.00001672039,0.05063529,0.0006974247,0.0001249864,0.00006335075,0.000004530086,0.000004273757,0.0002153521],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7250657,"threshold_uncertainty_score":0.4261179,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1164316970633832,"score_gpt":0.4311357888689092,"score_spread":0.3147040918055259,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}