{"id":"W2467932880","doi":"10.1177/0013164416654740","title":"An Unbiased Estimate of Global Interrater Agreement","year":2016,"lang":"en","type":"article","venue":"Educational and Psychological Measurement","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université du Québec à Trois-Rivières; University of Ottawa","funders":"","keywords":"Inter-rater reliability; Statistics; Agreement; Econometrics; Psychology; Mathematics; Rating scale; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1117375,0.001276331,0.002270862,0.006856731,0.001478942,0.003564422,0.00227307,0.001916428,0.004790479],"category_scores_gemma":[0.3101148,0.0007442116,0.001691294,0.004572264,0.002562439,0.004007297,0.004541899,0.002133486,0.002434815],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001036482,"about_ca_system_score_gemma":0.002387797,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001134559,"about_ca_topic_score_gemma":0.002481141,"domain_scores_codex":[0.8571511,0.08960382,0.01424609,0.01391398,0.02324045,0.001844551],"domain_scores_gemma":[0.7436058,0.1433543,0.02253111,0.03733405,0.05183033,0.001344452],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0008912151,0.0002817409,0.143088,0.005251277,0.003486235,0.0004055118,0.01290353,0.007129181,0.01392346,0.1020828,0.02162871,0.6889284],"study_design_scores_gemma":[0.0002903369,0.002512474,0.3621488,0.007763541,0.004024376,0.004488976,0.01203975,0.07911307,0.04470782,0.2954271,0.18643,0.001053705],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.04691038,0.002842282,0.921293,0.0007432716,0.0004953434,0.001575351,0.001454757,0.001210564,0.02347495],"genre_scores_gemma":[0.486115,0.001016943,0.5023735,0.0007373203,0.0003094455,0.003585681,0.001501552,0.0005038658,0.003856671],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.8882626,"threshold_uncertainty_score":0.5909312,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4022887300727808,"score_gpt":0.47602807594491,"score_spread":0.07373934587212921,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}