{"id":"W2901897243","doi":"10.1007/s40037-018-0481-2","title":"Validity evidence for programmatic assessment in competency-based education","year":2018,"lang":"en","type":"article","venue":"Perspectives on Medical Education","topic":"Innovations in Medical Education","field":"Medicine","cited_by":58,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"","keywords":"Generalizability theory; Formative assessment; Summative assessment; Variance (accounting); Educational measurement; Reliability (semiconductor); Medical education; Psychology; Multilevel model; Computer science; Mathematics education; Medicine; Statistics; Curriculum; Mathematics; Pedagogy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.001777486,0.0001993471,0.0002891912,0.0005344491,0.0001629311,0.00003915326,0.0002158749,0.000201724,0.0009038929],"category_scores_gemma":[0.01882546,0.0001808065,0.00008848026,0.001011892,0.0004231908,0.0001645064,0.00001658547,0.0005206698,0.00005949988],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001653426,"about_ca_system_score_gemma":0.02720505,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009626993,"about_ca_topic_score_gemma":0.00004566326,"domain_scores_codex":[0.997372,0.0001595119,0.0005592561,0.0005692373,0.001014892,0.0003251558],"domain_scores_gemma":[0.9973344,0.0003902731,0.0001971693,0.0005152626,0.001360602,0.0002023484],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0007189871,0.02664013,0.04847063,0.001350618,0.00004940026,0.000001707601,0.01355838,0.000003858033,0.0003578447,0.17926,0.0439296,0.6856589],"study_design_scores_gemma":[0.006558915,0.008595199,0.734045,0.01524024,0.0003978574,0.00007773909,0.1048188,0.05882068,0.001300152,0.03619121,0.0326341,0.001320065],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7654682,0.0003960216,0.03940113,0.1771049,0.004518365,0.004591418,0.000002619819,0.000200209,0.008317161],"genre_scores_gemma":[0.8906108,0.00003912681,0.09804057,0.007293484,0.001882853,0.001706944,0.0001260941,0.00002883804,0.0002713173],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6855744,"threshold_uncertainty_score":0.9896992,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05731436957801039,"score_gpt":0.4721685536413467,"score_spread":0.4148541840633364,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}