{"id":"W2901897243","doi":"10.1007/s40037-018-0481-2","title":"Validity evidence for programmatic assessment in competency-based education","year":2018,"lang":"en","type":"article","venue":"Perspectives on Medical Education","topic":"Innovations in Medical Education","field":"Medicine","cited_by":58,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"","keywords":"Generalizability theory; Formative assessment; Summative assessment; Variance (accounting); Educational measurement; Reliability (semiconductor); Medical education; Psychology; Multilevel model; Computer science; Mathematics education; Medicine; Statistics; Curriculum; Mathematics; Pedagogy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.3261042,0.0006493203,0.0008936665,0.004495225,0.001730757,0.00489979,0.003559027,0.00182508,0.003834649],"category_scores_gemma":[0.6295704,0.0008510225,0.002752911,0.004155111,0.01112106,0.004131413,0.007239749,0.002492577,0.0005963586],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004874971,"about_ca_system_score_gemma":0.009463839,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005027233,"about_ca_topic_score_gemma":0.003613619,"domain_scores_codex":[0.6999385,0.1812526,0.01906716,0.0167501,0.08055839,0.002433223],"domain_scores_gemma":[0.1151886,0.7427014,0.06241901,0.03711746,0.04080797,0.001765618],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001087363,0.0005363841,0.8759406,0.002648946,0.001596497,0.0000957982,0.005887419,0.001461401,0.0004213743,0.01653434,0.001193702,0.09259611],"study_design_scores_gemma":[0.0003202427,0.002011394,0.9338296,0.005685986,0.0009717311,0.0006048979,0.004687963,0.01478027,0.002539176,0.0244339,0.01001071,0.0001242571],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8240938,0.0110997,0.08927304,0.009180598,0.0009078531,0.002520994,0.001384556,0.0002196584,0.06131983],"genre_scores_gemma":[0.9863478,0.0005353269,0.01106051,0.0004490013,0.00009516137,0.0006318919,0.0004621104,0.00005049459,0.0003676234],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3261042,"threshold_uncertainty_score":0.8310335,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05731436957801039,"score_gpt":0.4721685536413467,"score_spread":0.4148541840633364,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}