{"id":"W4224989672","doi":"10.3390/sym14020262","title":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","year":2022,"lang":"en","type":"article","venue":"Symmetry","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"ca_institutions":"Princess Margaret Cancer Centre; University Health Network; York University; Public Health Ontario; University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Statistics; Cohen's kappa; Kappa; Inter-rater reliability; Context (archaeology); Statistic; Mathematics; Summary statistics; Population; Medicine","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4741925,0.001023312,0.001661903,0.005086635,0.001088821,0.003651182,0.002885671,0.001891694,0.001744765],"category_scores_gemma":[0.6635643,0.0006570225,0.003429998,0.003639957,0.005916199,0.004861181,0.004026758,0.002150548,0.0003700602],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002824818,"about_ca_system_score_gemma":0.002020564,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008729018,"about_ca_topic_score_gemma":0.0007010337,"domain_scores_codex":[0.402117,0.5212848,0.02066888,0.01838351,0.03577919,0.001766434],"domain_scores_gemma":[0.09471259,0.8138121,0.04765073,0.02525181,0.01778239,0.0007903356],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.005161262,0.0006770984,0.6928131,0.004472841,0.01924338,0.0003157188,0.01292254,0.06222633,0.001897866,0.03379316,0.002907385,0.1635693],"study_design_scores_gemma":[0.0008938179,0.01064933,0.5547152,0.003304247,0.005116997,0.002176731,0.008420764,0.3098607,0.009924133,0.08408513,0.01017031,0.000682661],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.665527,0.003569033,0.3170608,0.0007714967,0.0003035242,0.001595012,0.0007596186,0.000225834,0.01018761],"genre_scores_gemma":[0.9725398,0.0001715298,0.02600907,0.00005739598,0.0000493703,0.000747653,0.0002453532,0.00003231053,0.0001474828],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5258075,"threshold_uncertainty_score":0.6484143,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2176509483273682,"score_gpt":0.4693801055011164,"score_spread":0.2517291571737482,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}