{"id":"W2810719550","doi":"10.3138/cjpe.31094","title":"Construct-Aligned Rating Scales Improve the Reliability of Program Evaluation Data","year":2018,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Innovations in Medical Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"Generalizability theory; Sophistication; Construct (python library); Reliability (semiconductor); Rating scale; Construct validity; Psychology; Independence (probability theory); Applied psychology; Computer science; Psychometrics; Statistics; Clinical psychology; Mathematics; Developmental psychology; Sociology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01629071,0.0001464018,0.0002879986,0.0003156908,0.000276533,0.00008232673,0.0004406318,0.000148956,0.0004605172],"category_scores_gemma":[0.01898615,0.000105683,0.00007632082,0.0009413355,0.0009882684,0.0003799688,0.00002873661,0.0003051748,0.000008260369],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006406739,"about_ca_system_score_gemma":0.008701171,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006161238,"about_ca_topic_score_gemma":0.00269502,"domain_scores_codex":[0.9962596,0.000514124,0.001150615,0.0002700845,0.001501875,0.0003037396],"domain_scores_gemma":[0.9880922,0.0001110705,0.00101775,0.0009097309,0.009636086,0.0002332075],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00004159142,0.0001380838,0.008356509,0.00005718797,0.00006132885,9.438064e-7,0.000744548,0.00001607101,0.0005427998,0.0001463498,0.003925213,0.9859694],"study_design_scores_gemma":[0.008073479,0.00741395,0.1949365,0.001252028,0.003662295,0.0005233113,0.008519897,0.705319,0.005379919,0.008015725,0.05632451,0.0005794428],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9826323,0.0003547172,0.0006702864,0.006323189,0.001861146,0.006852294,0.00001912935,0.00002351689,0.001263379],"genre_scores_gemma":[0.955987,0.000003765805,0.04223915,0.0002280348,0.001004576,0.0002462085,0.0002593132,0.00001755889,0.00001444096],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9853899,"threshold_uncertainty_score":0.9969186,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1351104228610917,"score_gpt":0.4605146374006852,"score_spread":0.3254042145395935,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}