{"id":"W2810719550","doi":"10.3138/cjpe.31094","title":"Construct-Aligned Rating Scales Improve the Reliability of Program Evaluation Data","year":2018,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Innovations in Medical Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"Generalizability theory; Sophistication; Construct (python library); Reliability (semiconductor); Rating scale; Construct validity; Psychology; Independence (probability theory); Applied psychology; Computer science; Psychometrics; Statistics; Clinical psychology; Mathematics; Developmental psychology; Sociology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1435442,0.001024108,0.001566653,0.004109301,0.001264385,0.002649049,0.0009660439,0.001039472,0.002088885],"category_scores_gemma":[0.3709623,0.0007370074,0.001960615,0.005143682,0.001802234,0.003074186,0.003047616,0.002043981,0.00100062],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001016615,"about_ca_system_score_gemma":0.001556941,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001069304,"about_ca_topic_score_gemma":0.002327119,"domain_scores_codex":[0.7866536,0.1587692,0.0210358,0.008146566,0.02418514,0.001209669],"domain_scores_gemma":[0.5519769,0.2566808,0.04459223,0.05927104,0.08550067,0.001978263],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.004460974,0.002125855,0.5086483,0.001957572,0.0024471,0.0001699389,0.009383839,0.008163537,0.009114048,0.008950369,0.01111237,0.4334661],"study_design_scores_gemma":[0.0007912248,0.003335506,0.891521,0.000947351,0.0008118083,0.0002519762,0.003376709,0.04774731,0.01191918,0.01598616,0.02302404,0.000287621],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5364389,0.001094571,0.4167878,0.001020008,0.001000015,0.008878772,0.001496437,0.00196549,0.03131804],"genre_scores_gemma":[0.8128042,0.0002113719,0.1787812,0.0003287373,0.0001976773,0.005391757,0.0009745963,0.0003456243,0.0009648987],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8564558,"threshold_uncertainty_score":0.7591432,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1351104228610917,"score_gpt":0.4605146374006852,"score_spread":0.3254042145395935,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}