{"id":"W2320000683","doi":"10.1080/08957347.2016.1171768","title":"Evaluating the Psychometric Characteristics of Generated Multiple-Choice Test Items","year":2016,"lang":"en","type":"article","venue":"Applied Measurement in Education","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":33,"is_retracted":false,"has_abstract":true,"ca_institutions":"Medical Council of Canada; University of Ottawa; University of Alberta","funders":"","keywords":"Item response theory; Item bank; Item analysis; Licensure; Psychology; Test (biology); Psychometrics; Cognition; Multiple choice; Differential item functioning; Applied psychology; Cognitive psychology; Computer science; Clinical psychology; Statistics; Medicine; Medical education","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.001702891,0.0001067831,0.0002130775,0.0001468083,0.00003640998,0.000009104926,0.00008950831,0.0000664815,0.00005114227],"category_scores_gemma":[0.1396537,0.00006069112,0.00003918018,0.000575805,0.00004764235,0.0000193779,0.00001235349,0.0001046135,0.00003333936],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001921021,"about_ca_system_score_gemma":0.0004330451,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007539517,"about_ca_topic_score_gemma":0.00001383102,"domain_scores_codex":[0.998495,0.00004498626,0.0005079143,0.000222554,0.0005637342,0.0001657986],"domain_scores_gemma":[0.9879235,0.01080629,0.0002755947,0.0004049234,0.0004985972,0.00009109909],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00005615151,0.001016308,0.480281,0.00001842436,0.00001388515,4.796237e-8,0.00008952293,0.000001480237,0.1504353,0.00006678365,0.0005750424,0.367446],"study_design_scores_gemma":[0.001636498,0.0001918504,0.9893171,0.0009796123,0.00007346819,0.000001469535,0.00007776323,0.0000758118,0.006906064,0.0001018148,0.0005525064,0.00008602907],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9957502,0.0002490041,0.0001936668,0.00143513,0.0005333007,0.0006671475,0.000004273586,0.0000203872,0.001146885],"genre_scores_gemma":[0.9982314,0.0001157723,0.0005720544,0.0003872425,0.0003459881,0.0001939985,0.00002002709,0.00001471179,0.0001188512],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5090361,"threshold_uncertainty_score":0.8675934,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1433803510187834,"score_gpt":0.4009891954820371,"score_spread":0.2576088444632537,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}