{"id":"W4394894160","doi":"10.1136/bmjoq-2023-002722","title":"Achieving high inter-rater reliability in establishing data labels: a retrospective chart review study","year":2024,"lang":"en","type":"article","venue":"BMJ Open Quality","topic":"Sepsis Diagnosis and Treatment","field":"Medicine","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"Alberta Health Services; University of Calgary","funders":"National Institute on Deafness and Other Communication Disorders; Canadian Institutes of Health Research","keywords":"Kappa; Inter-rater reliability; Reliability (semiconductor); Chart; Confidence interval; Medicine; Quality Score; Cohen's kappa; Medical record; Machine learning; Medical physics; Computer science; Artificial intelligence; Data mining; Statistics; Surgery; Operations management; Internal medicine; Mathematics; Rating scale; Engineering; Metric (unit)","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01121488,0.0002505699,0.001026706,0.00007666067,0.00006729596,0.0003941621,0.0007001428,0.00006580239,0.0007204501],"category_scores_gemma":[0.004001855,0.0001810229,0.00008342059,0.0005731288,0.00004129727,0.000976046,0.001580073,0.0004283831,0.000140934],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005257376,"about_ca_system_score_gemma":0.0002066591,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009482652,"about_ca_topic_score_gemma":0.001928298,"domain_scores_codex":[0.9960809,0.0009578156,0.0009983133,0.001202962,0.000464613,0.0002954227],"domain_scores_gemma":[0.9965085,0.0004902489,0.0001644588,0.002592063,0.0001198707,0.0001248334],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001361265,0.002727241,0.9503393,0.002245475,0.0002416709,0.0003393451,0.001269214,1.826256e-7,0.00001196909,0.0004639704,0.01773614,0.02448937],"study_design_scores_gemma":[0.001345746,0.0004220465,0.9772753,0.009949772,0.0002873424,0.00001281941,0.0007962531,0.00003026787,0.00002678197,0.0002154622,0.009377989,0.0002602125],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9551805,0.007318156,0.00001385275,0.02345333,0.00044307,0.01054257,0.0002058055,0.0001063477,0.002736341],"genre_scores_gemma":[0.9945651,0.0009990361,0.0004434301,0.002167681,0.0001404392,0.0009893646,0.0002762578,0.00003155,0.0003871034],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03938461,"threshold_uncertainty_score":0.9971133,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4616052036959374,"score_gpt":0.5699567388274467,"score_spread":0.1083515351315093,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}