{"id":"W4224989672","doi":"10.3390/sym14020262","title":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","year":2022,"lang":"en","type":"article","venue":"Symmetry","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"ca_institutions":"Princess Margaret Cancer Centre; University Health Network; York University; Public Health Ontario; University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Statistics; Cohen's kappa; Kappa; Inter-rater reliability; Context (archaeology); Statistic; Mathematics; Summary statistics; Population; Medicine","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00426012,0.0001489175,0.0005104787,0.0002744568,0.0001462476,0.0000413809,0.0005726083,0.00002941571,0.0008843808],"category_scores_gemma":[0.0001716031,0.0001020988,0.0001208117,0.0004640614,0.0001911426,0.0001721051,0.0004314249,0.0001728876,0.000006171169],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008473315,"about_ca_system_score_gemma":0.00006441027,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005932449,"about_ca_topic_score_gemma":0.00001559554,"domain_scores_codex":[0.9959061,0.0009163117,0.0009047108,0.0004396951,0.00165083,0.0001823407],"domain_scores_gemma":[0.998004,0.000730164,0.0003642616,0.0006084795,0.00020157,0.00009159274],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00005934498,0.000742861,0.985733,0.0000145876,0.00005655237,0.000002190867,0.001472372,0.0008170482,0.007338954,0.0001689957,0.002049314,0.001544771],"study_design_scores_gemma":[0.0008799795,0.001255991,0.9689519,0.00001179751,0.00002042599,0.000001133823,0.01376021,0.008793017,0.002925923,0.001333653,0.001914007,0.0001519123],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.996466,0.00005958419,0.0009170115,0.0007839099,0.0003393613,0.0003294439,0.00006166577,0.000007940706,0.001035121],"genre_scores_gemma":[0.9976584,0.000002802045,0.001858026,0.0002645836,0.00001008012,0.00001954215,0.000008662695,0.000004361814,0.0001734957],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01678106,"threshold_uncertainty_score":0.968335,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2176509483273682,"score_gpt":0.4693801055011164,"score_spread":0.2517291571737482,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}