{"id":"W3033487789","doi":"10.1186/s41039-020-00134-8","title":"Can automated item generation be used to develop high quality MCQs that assess application of knowledge?","year":2020,"lang":"en","type":"article","venue":"Research and Practice in Technology Enhanced Learning","topic":"Student Assessment and Feedback","field":"Social Sciences","cited_by":43,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta; University of Ottawa; Ottawa Hospital; Medical Council of Canada","funders":"","keywords":"Quality (philosophy); Multiple choice; Cognition; Recall; Item analysis; Wilcoxon signed-rank test; Sample (material); Computer science; Test (biology); Modalities; Psychology; Psychometrics; Significant difference; Statistics; Clinical psychology; Mathematics; Cognitive psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1104204,0.0007186647,0.001180289,0.003907979,0.0003039661,0.001530919,0.001437933,0.001078955,0.002209013],"category_scores_gemma":[0.2933639,0.0004121438,0.0008613601,0.002273442,0.0009010917,0.001761887,0.001216182,0.0006849696,0.000956236],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000771681,"about_ca_system_score_gemma":0.0009653654,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001084538,"about_ca_topic_score_gemma":0.001678451,"domain_scores_codex":[0.9288949,0.05319733,0.004009635,0.00205969,0.01125128,0.0005871926],"domain_scores_gemma":[0.5636616,0.3384749,0.03340413,0.02193826,0.04097394,0.00154721],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.003593423,0.001017375,0.2045776,0.001497953,0.000484505,0.0001066332,0.002819712,0.002929825,0.007329383,0.0005509155,0.004932938,0.7701597],"study_design_scores_gemma":[0.002731303,0.0172228,0.847783,0.002680758,0.0008468095,0.001433309,0.003323862,0.06446222,0.02868886,0.004984938,0.02538609,0.0004560268],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8454559,0.00223695,0.1357073,0.001861421,0.0004177094,0.005261413,0.0009607819,0.001622815,0.006475618],"genre_scores_gemma":[0.8174748,0.0006357256,0.1771541,0.000688999,0.0001795775,0.00277275,0.0005320229,0.0001058644,0.0004561275],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1104204,"threshold_uncertainty_score":0.5839657,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2606414184306209,"score_gpt":0.5297904732686232,"score_spread":0.2691490548380023,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}