{"id":"W3137000688","doi":"10.3138/cjpe.69691","title":"Collaborative Evaluation Designs as an Authentic Course Assessment","year":2021,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta; Queen's University","funders":"","keywords":"Authentic assessment; Computer science; Evaluation methods; Graduate students; Authentic learning; Instructional design; Knowledge management; Psychology; Mathematics education; Pedagogy; Curriculum; Multimedia; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"grok","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"opus","categories":[],"domain":null,"study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.2294128,0.00113572,0.001051276,0.00286754,0.003295389,0.007476769,0.003949089,0.002280707,0.005639892],"category_scores_gemma":[0.3164621,0.0007833209,0.001171918,0.001374716,0.005404913,0.00681226,0.007427248,0.003030519,0.001181156],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004629832,"about_ca_system_score_gemma":0.009658656,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006546105,"about_ca_topic_score_gemma":0.001167194,"domain_scores_codex":[0.6149786,0.3411847,0.01141334,0.006925199,0.02399255,0.001505517],"domain_scores_gemma":[0.5953683,0.2741047,0.02084408,0.04705363,0.05749309,0.005136163],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"qualitative","study_design_scores_codex":[0.001100103,0.00178609,0.006713759,0.003672425,0.0003101906,0.0002722544,0.02609934,0.0103352,0.004783177,0.1818735,0.01136144,0.7516925],"study_design_scores_gemma":[0.003621397,0.01372976,0.01098641,0.00991104,0.0007039701,0.001116251,0.02165417,0.07528019,0.03983303,0.4162853,0.4061023,0.0007761609],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04818441,0.001157657,0.9039151,0.002714952,0.000656129,0.01408346,0.0001436123,0.0006242467,0.02852044],"genre_scores_gemma":[0.1973431,0.0003518197,0.7881314,0.0003943382,0.0001069346,0.01097407,0.00007165648,0.00008949026,0.002537124],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.2294128,"threshold_uncertainty_score":0,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3648231715123615,"score_gpt":0.5893866504709884,"score_spread":0.2245634789586268,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}