{"id":"W2320000683","doi":"10.1080/08957347.2016.1171768","title":"Evaluating the Psychometric Characteristics of Generated Multiple-Choice Test Items","year":2016,"lang":"en","type":"article","venue":"Applied Measurement in Education","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":33,"is_retracted":false,"has_abstract":true,"ca_institutions":"Medical Council of Canada; University of Ottawa; University of Alberta","funders":"","keywords":"Item response theory; Item bank; Item analysis; Licensure; Psychology; Test (biology); Psychometrics; Cognition; Multiple choice; Differential item functioning; Applied psychology; Cognitive psychology; Computer science; Clinical psychology; Statistics; Medicine; Medical education","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02704178,0.0005458167,0.0004600293,0.002563701,0.0002821742,0.001159438,0.0009750884,0.0009342115,0.00230499],"category_scores_gemma":[0.1338513,0.0002502147,0.001010013,0.001698875,0.0007970747,0.001305045,0.001064548,0.0007313142,0.00078723],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000534819,"about_ca_system_score_gemma":0.000527347,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003816047,"about_ca_topic_score_gemma":0.0005343449,"domain_scores_codex":[0.9808663,0.008604432,0.003237362,0.0009974996,0.00581453,0.0004798383],"domain_scores_gemma":[0.8425999,0.1144378,0.01145136,0.009852418,0.02064885,0.001009723],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001210173,0.001015507,0.8620313,0.0002039495,0.0004323597,0.0001998793,0.001985159,0.004520538,0.006516681,0.001061137,0.001008292,0.1198151],"study_design_scores_gemma":[0.0001332789,0.00298945,0.9670588,0.0001180545,0.00008626943,0.0005332591,0.001089029,0.01337985,0.01059085,0.001792588,0.002165712,0.00006286136],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9837143,0.0001271725,0.01241734,0.000101322,0.00002857027,0.0003477771,0.0005550879,0.00008641482,0.002622001],"genre_scores_gemma":[0.9845279,0.00007408084,0.01341973,0.000061696,0.00001592673,0.0004001128,0.001055579,0.0000443769,0.0004005267],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02704178,"threshold_uncertainty_score":0.1430123,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1433803510187834,"score_gpt":0.4009891954820371,"score_spread":0.2576088444632537,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}