{"id":"W2771770473","doi":"10.3138/cjpe.31128","title":"Using Rubrics for an Evaluation: A National Research Council Pilot","year":2017,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Rubric; Excellence; Relevance (law); Context (archaeology); Psychology; Political science; Computer science; Medical education; Pedagogy; Medicine; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"grok","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"opus","categories":["metaresearch"],"domain":"evaluation","study_design":"not_applicable","genre":"commentary","about_ca_system":true,"about_ca_topic":true,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","scholarly_communication","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2641856,0.0001314686,0.0002418981,0.0007846592,0.002282791,0.002524644,0.001249094,0.00009464572,0.001371724],"category_scores_gemma":[0.08519602,0.0001103941,0.0001192691,0.0004709685,0.0002790095,0.002254101,0.0000263442,0.0002909151,0.00004276623],"about_ca_system_candidate":true,"about_ca_system_consensus":true,"about_ca_system_score_codex":0.006140058,"about_ca_system_score_gemma":0.09513073,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001335582,"about_ca_topic_score_gemma":0.07478361,"domain_scores_codex":[0.9746389,0.001330455,0.001043045,0.0003373499,0.02220858,0.000441704],"domain_scores_gemma":[0.8787238,0.0005346013,0.001095552,0.0005820711,0.1185142,0.0005498141],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001230391,0.000183481,0.009522122,0.000009612614,0.00005556207,0.000002296221,0.001618347,0.01815687,0.0002416707,0.003980014,0.02736196,0.938745],"study_design_scores_gemma":[0.00209361,0.001551239,0.02851199,0.00004650364,0.00008726859,0.00003637074,0.0008288218,0.7624778,0.00004408643,0.09416685,0.1100019,0.0001535724],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8876423,0.001211147,0.01064296,0.00967995,0.006032709,0.01079587,0.0001042599,0.00002334202,0.07386744],"genre_scores_gemma":[0.9847043,0.000004604855,0.0138732,0.0001234335,0.0008852807,0.0002152292,0.00001282784,0.00001530535,0.0001658731],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9385915,"threshold_uncertainty_score":0.9995412,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9730025773537614,"score_gpt":0.7170565067688764,"score_spread":0.2559460705848849,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}