{"id":"W1969690806","doi":"10.5539/elt.v3n1p108","title":"On Rater Agreement and Rater Training","year":2010,"lang":"en","type":"article","venue":"English Language Teaching","topic":"Educational Technology and Assessment","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Psychology; Inter-rater reliability; Task (project management); Applied psychology; Agreement; Selection (genetic algorithm); Interpretation (philosophy); Social psychology; Clinical psychology; Rating scale; Developmental psychology; Linguistics; Artificial intelligence; Computer science","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.5894499,0.001695282,0.002942887,0.01449664,0.004092019,0.00596616,0.004479031,0.002618058,0.00212086],"category_scores_gemma":[0.8197535,0.001525997,0.002319848,0.01053802,0.01036308,0.0103842,0.008994022,0.004893981,0.001416648],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.007577532,"about_ca_system_score_gemma":0.004545566,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00344923,"about_ca_topic_score_gemma":0.003041874,"domain_scores_codex":[0.1829964,0.6277314,0.05701445,0.02297273,0.1066103,0.002674709],"domain_scores_gemma":[0.05314705,0.8084497,0.02303172,0.02902705,0.08565732,0.0006872182],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002289923,0.0004466123,0.1041438,0.007072959,0.004026282,0.0004607937,0.06739537,0.009393895,0.005938633,0.09224124,0.01423185,0.6923587],"study_design_scores_gemma":[0.001192187,0.006617291,0.3125999,0.01879113,0.00382211,0.005751793,0.03703425,0.09796879,0.04095983,0.2450584,0.2275232,0.00268101],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1304044,0.01303814,0.732578,0.003518736,0.002746803,0.00848086,0.0008609815,0.0009913235,0.1073807],"genre_scores_gemma":[0.6949328,0.002977764,0.2861552,0.00176268,0.001211688,0.006517675,0.0007134379,0.001037468,0.004691357],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.4105501,"threshold_uncertainty_score":0.5062814,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.008705802459287353,"score_gpt":0.2739569706464495,"score_spread":0.2652511681871621,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}