{"id":"W4394894160","doi":"10.1136/bmjoq-2023-002722","title":"Achieving high inter-rater reliability in establishing data labels: a retrospective chart review study","year":2024,"lang":"en","type":"article","venue":"BMJ Open Quality","topic":"Sepsis Diagnosis and Treatment","field":"Medicine","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"Alberta Health Services; University of Calgary","funders":"National Institute on Deafness and Other Communication Disorders; Canadian Institutes of Health Research","keywords":"Kappa; Inter-rater reliability; Reliability (semiconductor); Chart; Confidence interval; Medicine; Quality Score; Cohen's kappa; Medical record; Machine learning; Medical physics; Computer science; Artificial intelligence; Data mining; Statistics; Surgery; Operations management; Internal medicine; Mathematics; Rating scale; Engineering; Metric (unit)","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1849896,0.00053434,0.0008837315,0.003800665,0.001162587,0.001762727,0.001139503,0.0006229622,0.0004884693],"category_scores_gemma":[0.2596778,0.0005365423,0.0007848039,0.002803545,0.001901346,0.001603318,0.001616395,0.0006327704,0.0002515345],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001310696,"about_ca_system_score_gemma":0.002644974,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002426954,"about_ca_topic_score_gemma":0.003401208,"domain_scores_codex":[0.7665728,0.1593917,0.03018002,0.01176491,0.03028364,0.001806973],"domain_scores_gemma":[0.5148065,0.2576174,0.09164415,0.04612457,0.08737516,0.002432254],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0005097237,0.0002038517,0.9670373,0.0005027301,0.0004833401,0.0002130309,0.006593919,0.0003904953,0.001154768,0.0002371274,0.0005332173,0.02214071],"study_design_scores_gemma":[0.0001001602,0.00259879,0.975556,0.0006312599,0.0006818026,0.001416545,0.004963245,0.003518364,0.005357195,0.0004412945,0.004636551,0.00009879171],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9547696,0.002569374,0.03725367,0.0002163889,0.0001195728,0.002155943,0.0005011454,0.0001011466,0.002313098],"genre_scores_gemma":[0.9871256,0.0002952878,0.01130679,0.0001043899,0.00005549097,0.0007143556,0.0002569816,0.00003081597,0.0001102107],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8150104,"threshold_uncertainty_score":0.9783301,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4616052036959374,"score_gpt":0.5699567388274467,"score_spread":0.1083515351315093,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}