{"id":"W4388196622","doi":"10.22318/cscl2023.978554","title":"Towards Developing Scalable Assessments of Higher-Order Learning","year":2023,"lang":"en","type":"article","venue":"Computer-supported collaborative learning/˜The œComputer-Supported Collaborative Learning Conference","topic":"Educational Strategies and Epistemologies","field":"Psychology","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"Toronto Rehabilitation Institute; University of Toronto","funders":"","keywords":"Scalability; Computer science; Class (philosophy); Order (exchange); Cognition; Path (computing); Engineering ethics; Mathematics education; Artificial intelligence; Psychology; Engineering; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","sts","scholarly_communication","research_integrity","insufficient_payload"],"consensus_categories":["metaepi_narrow","insufficient_payload"],"category_scores_codex":[0.002833831,0.001701775,0.002201114,0.001015662,0.002426222,0.001138139,0.002207519,0.0009676947,0.002621895],"category_scores_gemma":[0.000541648,0.001530129,0.0003532855,0.01307459,0.001422144,0.0009110516,0.001237383,0.003856282,0.001080872],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000517617,"about_ca_system_score_gemma":0.005424978,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003745687,"about_ca_topic_score_gemma":0.00008556397,"domain_scores_codex":[0.9856085,0.004917381,0.002534587,0.002642775,0.001851552,0.002445242],"domain_scores_gemma":[0.9840031,0.002928411,0.003046191,0.001229827,0.008261102,0.0005313188],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","study_design_scores_codex":[0.001797633,0.001740861,0.1932482,0.0009296266,0.007906228,0.001511792,0.05823217,0.2283744,0.002847438,0.09659249,0.2157179,0.1911013],"study_design_scores_gemma":[0.004907246,0.007548367,0.3119936,0.0008523828,0.0004323737,0.00009243243,0.03293238,0.1032492,0.0006388471,0.00167386,0.5320211,0.003658218],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6096856,0.0009125404,0.3105793,0.006696894,0.02995003,0.004629702,0.0002263172,0.004607103,0.03271256],"genre_scores_gemma":[0.936536,0.0003732607,0.02992264,0.0004960715,0.004284209,0.0007463301,0.001689915,0.000320719,0.02563092],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3268504,"threshold_uncertainty_score":0.9998988,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04595326024166136,"score_gpt":0.3592451922201849,"score_spread":0.3132919319785235,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}