{"id":"W1969690806","doi":"10.5539/elt.v3n1p108","title":"On Rater Agreement and Rater Training","year":2010,"lang":"en","type":"article","venue":"English Language Teaching","topic":"Educational Technology and Assessment","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Psychology; Inter-rater reliability; Task (project management); Applied psychology; Agreement; Selection (genetic algorithm); Interpretation (philosophy); Social psychology; Clinical psychology; Rating scale; Developmental psychology; Linguistics; Artificial intelligence; Computer science","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005273395,0.00008836019,0.00007427807,0.00007505664,0.0001498003,0.0001092348,0.0002958922,0.00005589489,0.00006277062],"category_scores_gemma":[0.0001707342,0.00007320118,0.00001792505,0.00004253583,0.00002726341,0.0002247679,0.0001013637,0.0005879297,0.00001542194],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001123244,"about_ca_system_score_gemma":0.00002214787,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00000805841,"about_ca_topic_score_gemma":0.00001055841,"domain_scores_codex":[0.9993432,0.00004922312,0.00009292721,0.0002365358,0.0001104014,0.0001677781],"domain_scores_gemma":[0.9994993,0.0001089708,0.00003259287,0.0003020177,0.00001140831,0.00004567372],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","study_design_scores_codex":[0.000002134024,0.00009541181,0.001127303,0.000006709784,0.00001935946,0.00003448012,0.1834427,0.000006169631,0.01631801,0.6889799,0.0008473289,0.1091205],"study_design_scores_gemma":[0.009977587,0.002024001,0.05689244,0.0007832241,0.0001247339,0.0004188494,0.3275294,0.02307596,0.2382794,0.1714188,0.1627893,0.006686338],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9758516,0.00002650004,0.008768959,0.001031854,0.0008164565,0.00006319986,6.653196e-7,0.0001887387,0.01325203],"genre_scores_gemma":[0.9735573,4.981219e-7,0.0251088,0.0005571817,0.0001998964,0.00001263432,0.000002855804,0.000005047332,0.0005557945],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5175611,"threshold_uncertainty_score":0.2985057,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.008705802459287353,"score_gpt":0.2739569706464495,"score_spread":0.2652511681871621,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}