{"id":"W2593712670","doi":"10.24908/pceea.v0i0.6465","title":"Testing Inter-Rater Reliability in Rubrics for Large Scale Undergraduate Independent Projects","year":2017,"lang":"en","type":"article","venue":"Proceedings of the Canadian Engineering Education Association (CEEA)","topic":"Student Assessment and Feedback","field":"Social Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Rubric; Deliverable; Reliability (semiconductor); Peer assessment; Consistency (knowledge bases); Supervisor; Inter-rater reliability; Psychology; Computer science; Work (physics); Mathematics education; Engineering; Rating scale; Political science; Systems engineering; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2278346,0.001376256,0.001121089,0.004101294,0.002031753,0.002595753,0.002324671,0.001033905,0.00257645],"category_scores_gemma":[0.3804052,0.0007391216,0.002134868,0.003064576,0.002257646,0.002665482,0.004349313,0.001925272,0.001798535],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001842036,"about_ca_system_score_gemma":0.002305442,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001975975,"about_ca_topic_score_gemma":0.004246484,"domain_scores_codex":[0.7559672,0.1429275,0.02626096,0.01147703,0.06031649,0.003050983],"domain_scores_gemma":[0.4876661,0.265995,0.02600711,0.04838631,0.1692651,0.002680412],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.003868953,0.002650049,0.3655821,0.004401849,0.002636535,0.0004628506,0.07707155,0.007070543,0.0353415,0.009030243,0.02073023,0.4711536],"study_design_scores_gemma":[0.001019556,0.007324922,0.7511117,0.002358927,0.001043787,0.001182035,0.02442594,0.05597966,0.06095643,0.01128629,0.08247519,0.0008355536],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.709207,0.002098127,0.2486304,0.0006981516,0.001551998,0.01086744,0.001218954,0.001502995,0.02422488],"genre_scores_gemma":[0.8512404,0.0005102947,0.1303038,0.0003105887,0.00021454,0.01158691,0.001715663,0.0006488106,0.003469018],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7721654,"threshold_uncertainty_score":0.9522175,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02201015964286919,"score_gpt":0.2958311354151263,"score_spread":0.2738209757722572,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}