{"id":"W2760364374","doi":"10.3102/0034654317726529","title":"Developing, Analyzing, and Using Distractors for Multiple-Choice Tests in Education: A Comprehensive Review","year":2017,"lang":"en","type":"review","venue":"Review of Educational Research","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":217,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Task (project management); Quality (philosophy); Process (computing); Psychology; Cognitive psychology; Multiple choice; Computer science; Applied psychology; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01750243,0.001710907,0.003064554,0.008125543,0.0005182126,0.002582664,0.002520105,0.001909873,0.003294997],"category_scores_gemma":[0.04834219,0.000737999,0.002303149,0.008366144,0.001660925,0.003310092,0.001323319,0.001862786,0.00117783],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001843823,"about_ca_system_score_gemma":0.007643739,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004528125,"about_ca_topic_score_gemma":0.009332346,"domain_scores_codex":[0.9865631,0.005324154,0.002952297,0.0008768407,0.004091548,0.0001920877],"domain_scores_gemma":[0.9229996,0.06711262,0.003674079,0.0008094588,0.005064957,0.0003392966],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00004491158,0.00007602288,0.0005907072,0.08480968,0.0002352328,0.00005912361,0.0002198515,0.0002855082,0.0002179695,0.002130728,0.004329993,0.9070003],"study_design_scores_gemma":[0.0001044696,0.0005575147,0.00884002,0.2961251,0.003484318,0.001868617,0.001113846,0.0009779353,0.002338011,0.01197102,0.6723614,0.0002577849],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0002482248,0.9974374,0.001021503,0.0004880391,0.0001021361,0.00005667032,0.00003884082,0.00001474836,0.0005923814],"genre_scores_gemma":[0.002155871,0.9933705,0.003885901,0.0002404659,0.0000924074,0.00007831497,0.00004749577,0.000007136405,0.0001219368],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9824976,"threshold_uncertainty_score":0.09256279,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9466681700442642,"score_gpt":0.7483343693634743,"score_spread":0.1983338006807899,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}