{"id":"W3016806524","doi":"10.3352/jeehp.2020.17.12","title":"Performance of the Ebel standard-setting method for the spring 2019 Royal College of Physicians and Surgeons of Canada internal medicine certification examination consisting of multiple-choice questions","year":2020,"lang":"en","type":"article","venue":"Journal of Educational Evaluation for Health Professions","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":17,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Manitoba; University of Calgary; Royal College of Physicians and Surgeons of Canada","funders":"","keywords":"Specialty; Certification; Nuclear medicine; Medicine; Statistics; Psychology; Mathematics; Family medicine; Management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02215956,0.00008677334,0.0004276129,0.0002019808,0.0004353609,0.000004641288,0.0003573154,0.0000389358,0.00002910675],"category_scores_gemma":[0.2613388,0.00005065092,0.00009280076,0.0007777116,0.0001393381,0.0001124476,0.0000482555,0.0001755172,2.057026e-8],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001376313,"about_ca_system_score_gemma":0.004115891,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004487721,"about_ca_topic_score_gemma":0.003217783,"domain_scores_codex":[0.9951599,0.00122444,0.001987487,0.0001559841,0.001331244,0.0001409251],"domain_scores_gemma":[0.8772005,0.111593,0.005705876,0.0002040544,0.005226462,0.00007008405],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0006917389,0.0003004644,0.699519,0.001231465,0.0002347946,2.676518e-8,0.01028834,0.03501034,0.009224675,0.006654121,0.04833601,0.188509],"study_design_scores_gemma":[0.0007857529,0.0002607772,0.7978048,0.0007800138,0.00008201546,0.000001973346,0.006419925,0.191255,0.0007301784,0.001266325,0.0005688247,0.00004438232],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7749726,0.001078702,0.1043173,0.1151745,0.001943299,0.001818952,0.0005860908,0.000002388031,0.000106098],"genre_scores_gemma":[0.9655972,0.00002042175,0.03390514,0.0002166393,0.0001335962,0.00002843866,0.000004086492,0.000006358589,0.00008817913],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2391792,"threshold_uncertainty_score":0.7680104,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3982188306808459,"score_gpt":0.5391948166882257,"score_spread":0.1409759860073798,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}