{"id":"W2053410856","doi":"10.5054/tj.2011.269751","title":"Generalizability Theory as Evidence of Concerns About Fairness in Large‐Scale ESL Writing Assessments","year":2011,"lang":"en","type":"article","venue":"TESOL Journal","topic":"Student Assessment and Feedback","field":"Social Sciences","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Generalizability theory; Rating scale; Reliability (semiconductor); Psychology; Variation (astronomy); Scale (ratio); Inter-rater reliability; Mathematics education; Developmental psychology; Geography","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.005168122,0.0001243968,0.0002646917,0.00009610049,0.0003049865,0.00009423918,0.0005775945,0.00009981998,0.003447829],"category_scores_gemma":[0.0003160829,0.000113916,0.0001165055,0.0003005053,0.0002606109,0.0006609263,0.0001071371,0.0003385728,0.00002691081],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001747698,"about_ca_system_score_gemma":0.0003551178,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008492757,"about_ca_topic_score_gemma":0.0007354968,"domain_scores_codex":[0.9973231,0.0007844049,0.0005254059,0.0002009547,0.000694953,0.0004711601],"domain_scores_gemma":[0.9988185,0.0003186479,0.0003294123,0.0001633174,0.0002205764,0.0001495341],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00006782139,0.0002341641,0.942015,0.00001554358,0.00002511367,0.00001960578,0.04304248,0.000001954189,0.0003957491,0.01217577,0.0001399379,0.001866857],"study_design_scores_gemma":[0.001249652,0.0002290259,0.8849425,0.0005219363,0.00005465102,0.00001057563,0.0812472,0.00009288645,0.0005222206,0.03010993,0.0006777594,0.0003416729],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9515238,0.0007942573,0.001272261,0.0001115187,0.0004252742,0.0001954717,0.000002882173,0.00002330125,0.04565119],"genre_scores_gemma":[0.9971439,0.0003945986,0.001359184,0.0001285104,0.0003479832,0.000007716166,5.399007e-7,0.000009792549,0.000607781],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05707251,"threshold_uncertainty_score":0.9974632,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.112657399584718,"score_gpt":0.4387508983425095,"score_spread":0.3260934987577915,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}