{"id":"W2760364374","doi":"10.3102/0034654317726529","title":"Developing, Analyzing, and Using Distractors for Multiple-Choice Tests in Education: A Comprehensive Review","year":2017,"lang":"en","type":"review","venue":"Review of Educational Research","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":217,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Task (project management); Quality (philosophy); Process (computing); Psychology; Cognitive psychology; Multiple choice; Computer science; Applied psychology; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.02083228,0.0003952302,0.003366808,0.001666955,0.0003738725,0.000191434,0.002159496,0.0001890464,0.0003463821],"category_scores_gemma":[0.7840396,0.0002798506,0.0005490006,0.00397793,0.0003703116,0.0002539061,0.0004563629,0.0006873726,0.00002744036],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004369001,"about_ca_system_score_gemma":0.01025622,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001951966,"about_ca_topic_score_gemma":0.00002684071,"domain_scores_codex":[0.9911433,0.00263995,0.002829629,0.001111194,0.001743468,0.000532414],"domain_scores_gemma":[0.5552396,0.4339007,0.003404998,0.00199499,0.005166262,0.0002933787],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[8.310841e-7,0.00006277508,0.002820321,0.244022,0.00002502783,1.784985e-7,0.000004707653,2.929615e-8,5.34082e-8,0.0005074039,0.009549277,0.7430074],"study_design_scores_gemma":[0.00004294793,0.00001620459,0.00431738,0.3530236,0.00008329793,0.0000227355,0.00001248145,0.000002338651,2.464538e-8,0.001217646,0.6410951,0.0001663086],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.00002014519,0.9935573,0.0000281169,0.001664191,0.0005952428,0.003529699,0.00007355806,0.000002952251,0.0005288105],"genre_scores_gemma":[0.000002039081,0.9802318,0.01797561,0.0001514336,0.0003283142,0.0006243188,0.0001061613,0.00002995994,0.0005504253],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.7632073,"threshold_uncertainty_score":0.9999654,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9466681700442642,"score_gpt":0.7483343693634743,"score_spread":0.1983338006807899,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}