{"id":"W4386396810","doi":"10.1287/ited.2023.0020","title":"That’s Incorrect and Let Me Tell You Why: A Scalable Assessment to Evaluate Higher Order Thinking Skills","year":2023,"lang":"en","type":"article","venue":"INFORMS Transactions on Education","topic":"Online Learning and Analytics","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"Western University","funders":"","keywords":"Grading (engineering); Higher-order thinking; Computer science; Scalability; Critical thinking; Data science; Analytics; Mathematics education; Teaching method; Psychology; Database; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004724393,0.0001393912,0.0001246791,0.0003969613,0.0003170411,0.0003239337,0.0002802455,0.0000646812,0.00006046164],"category_scores_gemma":[0.00001866141,0.0001189866,0.00004283525,0.001164691,0.00001992884,0.000676011,0.0000148227,0.0002412792,0.000247442],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001148901,"about_ca_system_score_gemma":0.0003525316,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001434445,"about_ca_topic_score_gemma":0.00002686454,"domain_scores_codex":[0.9988786,0.00003144421,0.0001963029,0.0002718869,0.0003797884,0.0002419912],"domain_scores_gemma":[0.9992311,0.00008719334,0.00006665902,0.0003574521,0.0001380157,0.0001195519],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000009925152,0.0006419454,0.002191569,0.00006443698,0.00008543056,0.000001646944,0.008955577,0.082715,0.0001291985,0.005319419,0.009114145,0.8907717],"study_design_scores_gemma":[0.001547445,0.001013246,0.1227037,0.0006820819,0.0001750571,0.00004608744,0.004041189,0.5812892,0.001749758,0.03654115,0.2485523,0.001658861],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1079225,0.00001912893,0.8127891,0.06734919,0.003786356,0.0005059734,0.000007743301,0.0008082467,0.006811746],"genre_scores_gemma":[0.9363862,0.00006184337,0.03836339,0.00342695,0.000109832,0.0000852787,0.00002327562,0.00002066549,0.02152254],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8891128,"threshold_uncertainty_score":0.4852131,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01589931771445249,"score_gpt":0.3182597238451819,"score_spread":0.3023604061307295,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}