{"id":"W4213042376","doi":"10.1080/00221341.2021.2003848","title":"Reliability of the Reflective Learning Framework for Assessing Higher-Order Thinking in Geography and Sustainability Courses","year":2022,"lang":"en","type":"article","venue":"Journal of Geography","topic":"Reflective Practices in Education","field":"Social Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"Higher-order thinking; Inter-rater reliability; Experiential learning; Reliability (semiconductor); Mathematics education; Reflection (computer programming); Psychology; Consistency (knowledge bases); Critical thinking; Sustainability; Order (exchange); Higher education; Medical education; Computer science; Teaching method; Artificial intelligence; Medicine; Rating scale; Developmental psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.007860591,0.00009797935,0.0002471305,0.0003945383,0.001532581,0.0001185719,0.0003853486,0.00008291686,0.00003114236],"category_scores_gemma":[0.00742742,0.00007960014,0.0002333656,0.001774977,0.0005798222,0.0007613591,0.0001176715,0.001114485,1.573133e-8],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003345245,"about_ca_system_score_gemma":0.0006538177,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007657844,"about_ca_topic_score_gemma":0.00006973051,"domain_scores_codex":[0.9963915,0.00199351,0.0004623633,0.0001942416,0.0006999589,0.0002583873],"domain_scores_gemma":[0.9945077,0.002702385,0.001235521,0.0001648744,0.001336189,0.00005333413],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001062249,0.0002273777,0.9749684,0.00003290084,0.00003605096,5.75042e-7,0.01292863,0.001549358,0.0000183931,0.007196844,0.00001422051,0.002920995],"study_design_scores_gemma":[0.0001564517,0.0001282182,0.6120169,0.00003252253,0.00004041356,0.000001589355,0.03093584,0.00001065535,0.00001182462,0.3480401,0.008562644,0.00006277889],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9881297,0.0008337098,0.001682438,0.005616733,0.001131849,0.0004398432,0.000001541659,0.000009748671,0.002154469],"genre_scores_gemma":[0.9873156,0.0001033193,0.01229277,0.00009309672,0.0001466268,0.00002674644,2.676238e-7,0.000009281152,0.00001232852],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3629515,"threshold_uncertainty_score":0.9997673,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01661314349435251,"score_gpt":0.3961152365769527,"score_spread":0.3795020930826002,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}