{"id":"W2771770473","doi":"10.3138/cjpe.31128","title":"Using Rubrics for an Evaluation: A National Research Council Pilot","year":2017,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Rubric; Excellence; Relevance (law); Context (archaeology); Psychology; Political science; Computer science; Medical education; Pedagogy; Medicine; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"grok","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"opus","categories":["metaresearch"],"domain":"evaluation","study_design":"not_applicable","genre":"commentary","about_ca_system":true,"about_ca_topic":true,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1300691,0.001216859,0.001246421,0.003058083,0.005096379,0.002682816,0.004055235,0.001746361,0.0072915],"category_scores_gemma":[0.1566778,0.001163192,0.001229064,0.003113368,0.002406629,0.001725038,0.003911881,0.003443125,0.003210134],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01077891,"about_ca_system_score_gemma":0.03339886,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.09864382,"about_ca_topic_score_gemma":0.1187843,"domain_scores_codex":[0.9075254,0.06301938,0.005789492,0.00246958,0.01843175,0.002764232],"domain_scores_gemma":[0.7345852,0.07447094,0.003776928,0.02044761,0.159015,0.007704276],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"qualitative","study_design_scores_codex":[0.004874227,0.03326654,0.01731692,0.003434155,0.0001685315,0.001429257,0.03538558,0.007374888,0.02347897,0.006933751,0.09318785,0.7731494],"study_design_scores_gemma":[0.009778259,0.06273037,0.2169853,0.004035425,0.000615876,0.001322597,0.04846152,0.05575042,0.08119523,0.007255271,0.5108091,0.001060465],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4852779,0.0009471771,0.2034518,0.00441936,0.0009786898,0.2209303,0.005746122,0.005007837,0.07324082],"genre_scores_gemma":[0.3935453,0.0007542982,0.4916697,0.001236366,0.0001461085,0.0861664,0.005216519,0.001347969,0.01991746],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1300691,"threshold_uncertainty_score":0,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9730025773537614,"score_gpt":0.7170565067688764,"score_spread":0.2559460705848849,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}