{"id":"W2112868871","doi":"10.1111/medu.12202","title":"Evaluating the quality of medical multiple‐choice items created with automated processes","year":2013,"lang":"en","type":"article","venue":"Medical Education","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":41,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Quality (philosophy); Multiple choice; MEDLINE; Medical education; Psychology; Computer science; Medicine; Internal medicine; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02402544,0.00012585,0.0003346538,0.0001931885,0.0001738743,0.0001339012,0.001316855,0.0001785199,0.003670898],"category_scores_gemma":[0.8769526,0.00005760076,0.0000427623,0.003232302,0.0003574711,0.0002057845,0.0001164549,0.0003036934,0.00005959177],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002779891,"about_ca_system_score_gemma":0.002804662,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004044248,"about_ca_topic_score_gemma":0.0001664984,"domain_scores_codex":[0.990128,0.001631573,0.001117369,0.0004023587,0.006469827,0.0002508088],"domain_scores_gemma":[0.4947624,0.4897856,0.003396975,0.0026951,0.008199539,0.001160384],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00002770177,0.0004311092,0.4089982,0.00007718059,0.00002504313,3.668148e-7,0.001053189,0.00003043616,0.0001806782,0.0001137449,0.02723095,0.5618314],"study_design_scores_gemma":[0.0006574755,0.0002062282,0.8735582,0.0002453778,0.0000134465,0.00001727891,0.003261602,0.1177655,0.0001352538,0.001934577,0.002058564,0.0001465258],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9772696,0.000344146,0.003143558,0.0168192,0.0005743867,0.0003760415,0.000001131205,0.0001482262,0.001323676],"genre_scores_gemma":[0.9920486,0.00001871809,0.00589026,0.001432856,0.0002050563,0.0001210148,0.000007063007,0.000008458255,0.0002679313],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8529272,"threshold_uncertainty_score":0.9972399,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5970176373843434,"score_gpt":0.6240804864393285,"score_spread":0.02706284905498502,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}