{"id":"W4386396810","doi":"10.1287/ited.2023.0020","title":"That’s Incorrect and Let Me Tell You Why: A Scalable Assessment to Evaluate Higher Order Thinking Skills","year":2023,"lang":"en","type":"article","venue":"INFORMS Transactions on Education","topic":"Online Learning and Analytics","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"Western University","funders":"","keywords":"Grading (engineering); Higher-order thinking; Computer science; Scalability; Critical thinking; Data science; Analytics; Mathematics education; Teaching method; Psychology; Database; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.008210645,0.001128561,0.0006521393,0.002468331,0.0005783775,0.003437643,0.001253179,0.001043969,0.009158984],"category_scores_gemma":[0.0343764,0.0003686765,0.0005743856,0.001258264,0.0005197196,0.002716944,0.003216267,0.00111386,0.005156097],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004496203,"about_ca_system_score_gemma":0.002261067,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008229485,"about_ca_topic_score_gemma":0.002683282,"domain_scores_codex":[0.9910671,0.003036422,0.0008062786,0.00109491,0.003684104,0.0003111302],"domain_scores_gemma":[0.9629034,0.01432757,0.005199677,0.003353982,0.01090632,0.00330921],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0008206933,0.001329485,0.04288243,0.0005342984,0.000112633,0.0003571699,0.001807144,0.002495903,0.02887795,0.001976013,0.03319121,0.8856151],"study_design_scores_gemma":[0.001466963,0.008736743,0.3763296,0.001812158,0.0004743775,0.004267856,0.00493328,0.2514391,0.1013924,0.03815912,0.2097732,0.00121515],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.384584,0.0008161537,0.5437325,0.002902366,0.001507821,0.004264605,0.004560335,0.02095826,0.03667388],"genre_scores_gemma":[0.379445,0.0003897842,0.6034541,0.0004285462,0.000263731,0.001921362,0.002043848,0.0007659508,0.01128768],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.009158984,"threshold_uncertainty_score":0.04342258,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01589931771445249,"score_gpt":0.3182597238451819,"score_spread":0.3023604061307295,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}