{"meta":{"page":1,"per_page":50,"max_per_page":100,"total":305,"total_is_capped":false,"direct_labels_cover":3,"predictions_cover":305,"direct_label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline (scores rank; they never assert a category)","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"query_hash":"4f6799dd99f6","filters":{"topic":"Reliability and Agreement in Measurement"}},"results":[{"id":"W2061141931","doi":"10.1007/s10459-010-9222-y","title":"Likert scales, levels of measurement and the “laws” of statistics","year":2010,"lang":"en","type":"article","venue":"Advances in Health Sciences Education","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":4546,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McMaster University","funders":"","keywords":"Likert scale; Statistics; Parametric statistics; Econometrics; Nonparametric statistics; Variance (accounting); Regression analysis; Sample size determination; Sample (material); Ordinal data; Statistical analysis; Computer science; Psychology; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.1216247962799418,"gpt":0.4436029242752067,"spread":0.3219781279952649,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0272799,0.00008697971,0.0002794901,0.0001772075,0.0002022139,0.00004356246,0.0006373961,0.00002859026,0.00006132293],"category_scores_gemma":[0.005723535,0.00004940073,0.00002826904,0.0008494486,0.0016792,0.0004036236,0.00005528987,0.0001445575,0.000002658387],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004405821,"about_ca_system_score_gemma":0.0009322807,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003432234,"about_ca_topic_score_gemma":0.002922606,"domain_scores_codex":[0.9955663,0.0003928354,0.00117802,0.000342182,0.0023136,0.0002070824],"domain_scores_gemma":[0.996653,0.001446817,0.0007749992,0.0003876763,0.0006631131,0.00007439042],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00004613613,0.0002927226,0.1245341,0.0001915869,0.000002400513,2.890036e-8,0.002090147,0.0004946366,0.0008065314,0.1788625,0.0006521027,0.6920272],"study_design_scores_gemma":[0.0007861223,0.000326582,0.2945156,0.000258221,0.000005215687,0.000002623275,0.004815977,0.001441385,0.001583191,0.670577,0.02552611,0.000161961],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.901521,0.02831988,0.02362454,0.02686861,0.008683003,0.002981781,0.00007742801,0.00001802669,0.007905695],"genre_scores_gemma":[0.9794413,0.000846199,0.01932463,0.000290829,0.00003873431,0.00002479877,3.839893e-7,0.00000188644,0.00003118037],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6918652,"threshold_uncertainty_score":0.9454724,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2171762906","doi":"10.1016/j.jclinepi.2010.02.006","title":"The COSMIN study reached international consensus on taxonomy, terminology, and definitions of measurement properties for health-related patient-reported outcomes","year":2010,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":4442,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McMaster University","funders":"","keywords":"Terminology; Taxonomy (biology); Delphi method; Confusion; Delphi; Consistency (knowledge bases); Management science; Medicine; Psychology; Applied psychology; Computer science; Artificial intelligence; Engineering; Linguistics; Ecology","retraction":null,"screen_n_in":null,"score":{"opus":0.8170932433584581,"gpt":0.5349041107663476,"spread":0.2821891325921105,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1191484,0.0001579548,0.001689862,0.0001501894,0.0002225034,0.00002547965,0.000605055,0.0001784293,0.00002204227],"category_scores_gemma":[0.4071394,0.00007446431,0.0004844949,0.00009193378,0.000882184,0.0000519088,0.0001014085,0.000815491,0.000003880514],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004022543,"about_ca_system_score_gemma":0.0003749423,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003530072,"about_ca_topic_score_gemma":0.0002116949,"domain_scores_codex":[0.9816502,0.005626863,0.01087775,0.0004036547,0.001137715,0.0003037683],"domain_scores_gemma":[0.9297979,0.0577233,0.009249949,0.0006048247,0.00240005,0.0002240049],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001284202,0.0016583,0.8948935,0.000006726178,0.0005431209,0.000004863345,0.0002294557,0.00007202853,0.0001097611,0.0029336,0.01019291,0.0880715],"study_design_scores_gemma":[0.003562007,0.006070243,0.8864807,0.00006972581,0.0000838946,0.00006129876,0.001801442,0.0003389431,0.00003048756,0.07916848,0.02221304,0.0001197406],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9132114,0.0001667506,0.0002725753,0.08194643,0.003340952,0.000896962,0.000008605174,0.000005167129,0.0001511692],"genre_scores_gemma":[0.995866,0.0001172231,0.002555033,0.001294475,0.0000896816,0.00004611846,5.359728e-7,0.000005141414,0.00002580787],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.287991,"threshold_uncertainty_score":0.9070221,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gemma","categories":["metaresearch"],"domain":"methods","study_design":"qualitative","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low"},{"model":"gpt","categories":[],"domain":null,"study_design":"qualitative","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"},{"id":"W2155243985","doi":"10.20982/tqmp.08.1.p023","title":"Computing Inter-Rater Reliability for Observational Data: An Overview and Tutorial","year":2012,"lang":"en","type":"article","venue":"Tutorials in Quantitative Methods for Psychology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":3862,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"National Institute on Alcohol Abuse and Alcoholism","keywords":"Observational study; Reliability (semiconductor); Computer science; Syntax; Consistency (knowledge bases); Class (philosophy); Statistics; Multiple comparisons problem; Inter-rater reliability; Econometrics; Mathematics; Artificial intelligence; Power (physics); Rating scale","retraction":null,"screen_n_in":null,"score":{"opus":0.8909602875101015,"gpt":0.7046596688682544,"spread":0.1863006186418471,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.104791,0.0002587336,0.0008242232,0.000194368,0.0002033056,0.0001428099,0.001080057,0.0002083337,0.00008539781],"category_scores_gemma":[0.05203191,0.0001969741,0.0001187453,0.0004038689,0.0003119682,0.001165196,0.0002857676,0.0001914656,0.0000108579],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007392436,"about_ca_system_score_gemma":0.00006566958,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003845892,"about_ca_topic_score_gemma":0.00002727718,"domain_scores_codex":[0.9906228,0.005300933,0.001727441,0.001245264,0.000524117,0.0005794559],"domain_scores_gemma":[0.9703355,0.02654318,0.0005992816,0.001523943,0.0008366702,0.0001613716],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.003044948,0.001715911,0.1317137,0.0002072445,0.0001261889,4.06666e-7,0.006938288,0.0001156606,0.01395798,0.5343839,0.01367718,0.2941186],"study_design_scores_gemma":[0.002093903,0.0008371876,0.05533753,0.00003410186,0.00003822449,0.000001769483,0.0004913845,0.004685822,0.0002635129,0.4983064,0.4375931,0.000317039],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.1171288,0.001455173,0.853558,0.001351299,0.02446383,0.001694861,0.0002068283,0.00002735141,0.0001138219],"genre_scores_gemma":[0.02445561,0.00005041047,0.9720247,0.0006627998,0.002484455,0.0001572514,0.00009646177,0.00002029322,0.00004801908],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.423916,"threshold_uncertainty_score":0.9559532,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1970325005","doi":"10.1037/a0029315","title":"When can categorical variables be treated as continuous? A comparison of robust continuous and categorical SEM estimation methods under suboptimal conditions.","year":2012,"lang":"en","type":"article","venue":"Psychological Methods","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":2534,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of British Columbia","funders":"Social Sciences and Humanities Research Council of Canada","keywords":"Categorical variable; Statistics; Mathematics; Normality; Continuous variable; Sample size determination; Ordinal data; Sample (material); Latent variable; Confirmatory factor analysis; Econometrics; Structural equation modeling","retraction":null,"screen_n_in":null,"score":{"opus":0.35917610160414,"gpt":0.5393551820295689,"spread":0.1801790804254289,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01985353,0.0004308987,0.001415986,0.0002661441,0.0002947318,0.0002113082,0.0006962457,0.0004921583,0.001759439],"category_scores_gemma":[0.009291993,0.0002864,0.0002461573,0.0008991836,0.0006933506,0.0003313724,0.0002004708,0.0005656711,0.00003739318],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001212677,"about_ca_system_score_gemma":0.0000365577,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002334369,"about_ca_topic_score_gemma":0.000006330643,"domain_scores_codex":[0.9880808,0.007052256,0.001833522,0.0009840264,0.001299522,0.0007498622],"domain_scores_gemma":[0.9887494,0.008602803,0.0007382056,0.0008331003,0.0005398568,0.0005366234],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0008796751,0.008410262,0.2674582,0.000064655,0.0004405153,0.0000159655,0.004969495,0.004061446,0.03602914,0.1547667,0.02701853,0.4958854],"study_design_scores_gemma":[0.003436225,0.0029655,0.2208232,0.00004391505,0.0005982096,0.0002133696,0.005875295,0.02839177,0.01431786,0.7070454,0.01488032,0.001408929],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2184741,0.0006506965,0.7715866,0.002958117,0.0006295535,0.0006335424,0.00001984766,0.00009562784,0.004951901],"genre_scores_gemma":[0.5634009,0.00001465114,0.4358103,0.0003829806,0.00006637195,0.00005947203,0.00001744137,0.00001211629,0.0002357817],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.5522787,"threshold_uncertainty_score":0.9999588,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2069378996","doi":"10.1016/j.jclinepi.2010.03.002","title":"Guidelines for Reporting Reliability and Agreement Studies (GRRAS) were proposed","year":2010,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":2097,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto; Western University","funders":"","keywords":"Reliability (semiconductor); Psychology; Medicine; Statistics; Mathematics; Physics","retraction":null,"screen_n_in":null,"score":{"opus":0.8672390330587612,"gpt":0.6777059197779776,"spread":0.1895331132807836,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4506437,0.0002183659,0.003410418,0.0001230641,0.0001964599,0.00003667506,0.0007325085,0.0003323937,0.0001168744],"category_scores_gemma":[0.9416817,0.0001143521,0.001038846,0.0001520215,0.0008075084,0.0002271974,0.0002278197,0.00106282,0.00001204692],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002862946,"about_ca_system_score_gemma":0.0002935071,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008643571,"about_ca_topic_score_gemma":0.00006009123,"domain_scores_codex":[0.9589608,0.004705852,0.03348568,0.000917647,0.001396254,0.0005337567],"domain_scores_gemma":[0.4239996,0.443244,0.0934272,0.003587839,0.03416727,0.001574159],"domain_codex":null,"domain_gemma":"methods","domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0004155023,0.0002534834,0.7635147,0.00005419919,0.0001614057,0.000008256494,0.00009904367,0.0002754978,0.0004237409,0.001983501,0.0986567,0.134154],"study_design_scores_gemma":[0.001543044,0.001719869,0.2529599,0.0001089483,0.0001033491,0.00006973462,0.0009580551,0.001820644,0.00009909769,0.5412202,0.1992001,0.0001970611],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8651575,0.0008193071,0.009263499,0.1182095,0.005886674,0.0004645776,0.000003651123,0.000008726315,0.0001865142],"genre_scores_gemma":[0.7472031,0.001173061,0.2394508,0.007989421,0.003441096,0.00001409962,8.075982e-7,0.00001280716,0.0007148376],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5392367,"threshold_uncertainty_score":0.5656779,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1968667761","doi":"10.1016/j.ijnurstu.2011.01.016","title":"Guidelines for Reporting Reliability and Agreement Studies (GRRAS) were proposed","year":2011,"lang":"en","type":"article","venue":"International Journal of Nursing Studies","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":985,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto; Western University","funders":"","keywords":"Agreement; Reliability (semiconductor); Psychology; Physics","retraction":null,"screen_n_in":null,"score":{"opus":0.7846223701604345,"gpt":0.5880428148793927,"spread":0.1965795552810418,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01518753,0.0001760273,0.0005931967,0.0002300148,0.0002157184,0.00009756815,0.0005820048,0.00003846576,0.00002446975],"category_scores_gemma":[0.07175019,0.000106274,0.0002996476,0.0001168644,0.0003942005,0.0003901228,0.0001281075,0.0001301936,0.000003108024],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003029382,"about_ca_system_score_gemma":0.00007838821,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000004845205,"about_ca_topic_score_gemma":0.000008293466,"domain_scores_codex":[0.9935377,0.0001501552,0.00361348,0.0003457238,0.002161008,0.0001919939],"domain_scores_gemma":[0.9776951,0.001062042,0.003860365,0.0002522715,0.01705962,0.00007057136],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.001596805,0.001425078,0.101982,0.0001073279,0.004539798,0.00009426978,0.05253648,0.0004452967,0.002085849,0.003146331,0.2053537,0.626687],"study_design_scores_gemma":[0.002678173,0.001739557,0.05835431,0.003473157,0.0004409759,0.0002740987,0.1377573,0.0003956322,0.006433009,0.7658302,0.02210369,0.0005198542],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9141483,0.01670005,0.006428286,0.05032753,0.0104042,0.0006028932,0.00001076575,0.00002146442,0.001356537],"genre_scores_gemma":[0.9526653,0.00125354,0.04442557,0.0004265851,0.0007139606,0.000009631301,3.941298e-7,0.000008539731,0.0004964456],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7626839,"threshold_uncertainty_score":0.9360688,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2074840499","doi":"10.1207/s15324796abm2603_02","title":"Classification and regression tree analysis in public health: Methodological review and comparison with logistic regression","year":2003,"lang":"en","type":"review","venue":"Annals of Behavioral Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":898,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"St. Stephen's University","funders":"","keywords":"Logistic regression; Public health; Identification (biology); Population; Nonparametric statistics; Logistic model tree; Regression analysis; Health psychology; Decision tree; Statistics; Stepwise regression; Regression; Psychology; Medicine; Applied psychology; Computer science; Data mining; Mathematics; Environmental health; Pathology; Biology","retraction":null,"screen_n_in":null,"score":{"opus":0.9360556982721563,"gpt":0.6560495613872027,"spread":0.2800061368849536,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02797909,0.0004877235,0.005896827,0.001094474,0.0001099208,0.00004510117,0.0005161337,0.0002885011,0.0001984542],"category_scores_gemma":[0.003630626,0.0002161447,0.0002546688,0.003338685,0.0007066207,0.000152871,0.0001143423,0.0005625115,0.000002948841],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000581188,"about_ca_system_score_gemma":0.000171734,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000107103,"about_ca_topic_score_gemma":0.0001492383,"domain_scores_codex":[0.9876019,0.005634188,0.003242534,0.001107163,0.002021246,0.0003930097],"domain_scores_gemma":[0.9930894,0.0020388,0.002948306,0.001001739,0.0005595306,0.0003622246],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001420041,0.0002527144,0.008471751,0.004681008,0.00005774329,0.000004920176,0.00009569665,2.233681e-7,6.982272e-7,0.0000760193,0.004362924,0.9819821],"study_design_scores_gemma":[0.0006145511,0.002701468,0.02850194,0.08580411,0.0047183,0.00002528984,0.0007334967,0.00002974035,6.732441e-7,0.0006841428,0.8756337,0.0005525787],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0003618745,0.9890538,0.0004718181,0.008739704,0.00007034805,0.001181475,0.00001449428,0.00001100743,0.00009547257],"genre_scores_gemma":[0.001937244,0.9959616,0.001339186,0.0004784644,0.00002289516,0.00007289957,0.00008441168,0.00001161945,0.00009165607],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9814295,"threshold_uncertainty_score":0.969705,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low"},{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low"}],"label_agreement":"split"},{"id":"W2105852486","doi":"10.1186/1472-6963-8-277","title":"Reliability of the interRAI suite of assessment instruments: a 12-country study of an integrated health information system","year":2008,"lang":"en","type":"article","venue":"BMC Health Services Research","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":677,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"Homewood Research Institute; University of Waterloo","funders":"University of Waterloo; Curtin University of Technology","keywords":"Health informatics; Reliability (semiconductor); Medicine; Nursing research; Cohen's kappa; Kappa; Suite; Health care; Health administration; Nursing; Family medicine; Public health; Statistics","retraction":null,"screen_n_in":null,"score":{"opus":0.1940493002678006,"gpt":0.4783379459711168,"spread":0.2842886457033162,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0445347,0.0001830821,0.0007992351,0.0007941765,0.00054222,0.00006614435,0.002208464,0.00009120582,0.00005470052],"category_scores_gemma":[0.0006047749,0.000106522,0.0001080067,0.003513541,0.0003095359,0.0009829114,0.0005730197,0.0005437569,0.000008744917],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007132391,"about_ca_system_score_gemma":0.002786337,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02217947,"about_ca_topic_score_gemma":0.00641921,"domain_scores_codex":[0.9793022,0.007465601,0.003884483,0.0005111161,0.008233674,0.000602919],"domain_scores_gemma":[0.9918459,0.0006854472,0.001824899,0.002084461,0.003295711,0.0002635986],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.000383511,0.002080101,0.9487247,0.00464759,0.00002404433,1.254046e-7,0.03534415,0.0009892086,0.00004543488,0.000124894,0.0001608749,0.007475389],"study_design_scores_gemma":[0.001036573,0.00260266,0.8290715,0.0006726564,0.00000268614,0.000001876834,0.1558597,0.009736136,0.0001032368,0.000104893,0.0007397592,0.00006833016],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9953597,0.00003887503,0.0002426449,0.0002146578,0.0002855801,0.003340838,0.00009046346,0.00001772861,0.0004094983],"genre_scores_gemma":[0.9988275,0.00005020455,0.0008285342,0.0001578911,0.00001830429,0.00007439555,0.00001545145,0.000006900932,0.00002083],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1205156,"threshold_uncertainty_score":0.9843319,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2007345088","doi":"10.1371/journal.pone.0073990","title":"The Case for Using the Repeatability Coefficient When Calculating Test–Retest Reliability","year":2013,"lang":"en","type":"article","venue":"PLoS ONE","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":474,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Dalhousie University","funders":"Curtin University of Technology","keywords":"Intraclass correlation; Repeatability; Reliability (semiconductor); Strengths and weaknesses; Computer science; Pearson product-moment correlation coefficient; Statistics; Test (biology); Measure (data warehouse); Reliability engineering; Correlation coefficient; Data mining; Medical physics; Machine learning; Mathematics; Reproducibility; Medicine; Psychology; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.2965950402626327,"gpt":0.3528709709396792,"spread":0.0562759306770465,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":[],"category_scores_codex":[0.01775398,0.0002072145,0.0003321327,0.0000356502,0.001985018,0.000793622,0.00102685,0.00007934347,0.000290215],"category_scores_gemma":[0.06078167,0.0000963998,0.0002036914,0.0004053282,0.0004938233,0.0002770914,0.000315456,0.0002649582,0.00009350516],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002070842,"about_ca_system_score_gemma":0.00009606268,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005538755,"about_ca_topic_score_gemma":0.0001701672,"domain_scores_codex":[0.9948629,0.0006426198,0.001220265,0.0007740533,0.001976232,0.0005239233],"domain_scores_gemma":[0.9732891,0.02082653,0.0004852084,0.002953324,0.002271377,0.0001744721],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003539921,0.0180725,0.781798,0.0006240397,0.0005908543,0.00005420441,0.009725716,0.01553076,0.06705296,0.004561966,0.01762033,0.08401469],"study_design_scores_gemma":[0.0008324501,0.0005734155,0.01553934,0.0001776564,0.0002609665,0.00003672189,0.006709621,0.8500034,0.01109514,0.1113677,0.002813156,0.0005904653],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9828515,0.0001199228,0.005306322,0.008109681,0.0001549659,0.002943767,0.00003138279,0.00004853826,0.0004339385],"genre_scores_gemma":[0.9882015,0.000002757638,0.01070322,0.0002668677,0.0001405079,0.0002682327,0.000001706097,0.00001289647,0.0004023364],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8344726,"threshold_uncertainty_score":0.9993142,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1829168467","doi":"10.1191/0962280204sm365ra","title":"Sample size requirements for the design of reliability study: review and new results","year":2004,"lang":"en","type":"article","venue":"Statistical Methods in Medical Research","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":468,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Western University","funders":"","keywords":"Intraclass correlation; Sample size determination; Reliability (semiconductor); Variance (accounting); Statistics; Sample (material); Computer science; Mathematics; Reliability engineering; Psychometrics","retraction":null,"screen_n_in":null,"score":{"opus":0.7388653050513273,"gpt":0.6767105014196586,"spread":0.0621548036316687,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3059061,0.0001586218,0.0007157914,0.0001192253,0.000234616,0.00008057058,0.001493691,0.0001173259,0.001230869],"category_scores_gemma":[0.885231,0.0000838829,0.00006633768,0.001371154,0.001272959,0.00008711299,0.0005117015,0.0007985488,0.00001097812],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001576554,"about_ca_system_score_gemma":0.00102483,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0009682967,"about_ca_topic_score_gemma":0.00008592699,"domain_scores_codex":[0.9661128,0.0174778,0.003102248,0.001373613,0.01100778,0.0009257876],"domain_scores_gemma":[0.2118028,0.781992,0.000361751,0.002526177,0.002214304,0.001102982],"domain_codex":null,"domain_gemma":"methods","domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0008900702,0.000996033,0.001086405,0.0003816549,0.0000367946,0.000009792651,0.0006431302,0.0001015617,0.00006885325,0.01230432,0.01676285,0.9667186],"study_design_scores_gemma":[0.002557727,0.001583491,0.01666613,0.0004758434,0.00004343197,9.721886e-7,0.000704837,0.001384857,0.00008773887,0.9667127,0.009666867,0.0001153959],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.0002772174,0.001785707,0.9649975,0.02878453,0.0001465486,0.00372231,0.00007045415,0.000007060383,0.0002086929],"genre_scores_gemma":[0.01529603,0.002371657,0.9812514,0.000588629,0.0000658162,0.0002700231,0.000001357453,0.00001042662,0.000144685],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.9666032,"threshold_uncertainty_score":0.9996821,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2091019309","doi":"10.1002/sim.5466","title":"Sample size formulas for estimating intraclass correlation coefficients with precision and assurance","year":2012,"lang":"en","type":"article","venue":"Statistics in Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":467,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Robarts Clinical Trials; Western University","funders":"","keywords":"Intraclass correlation; Confidence interval; Reliability (semiconductor); Sample size determination; Statistics; Interval (graph theory); Mathematics; Coverage probability; Limit (mathematics); Interval estimation; Sample (material); Correlation coefficient; Correlation; Computer science; Reproducibility; Power (physics); Mathematical analysis; Combinatorics; Physics","retraction":null,"screen_n_in":null,"score":{"opus":0.0854428886232476,"gpt":0.3958068923931848,"spread":0.3103640037699372,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.00701403,0.0001294434,0.0002927156,0.00009737493,0.0001445583,0.00004037315,0.0001777118,0.00004867782,0.0001137755],"category_scores_gemma":[0.05895115,0.00007927259,0.00001140496,0.0003191426,0.0001817037,0.0001946525,0.00003749401,0.000128265,0.000007101997],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006403356,"about_ca_system_score_gemma":0.00003125366,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003909961,"about_ca_topic_score_gemma":0.0000768278,"domain_scores_codex":[0.9972705,0.000117423,0.0006695832,0.0002901726,0.001354668,0.0002976892],"domain_scores_gemma":[0.9771791,0.02183926,0.0002568303,0.0002580502,0.000351443,0.0001153645],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004922879,0.0003229198,0.4425733,0.0001626575,0.00002307319,0.000002307115,0.005014019,0.02067447,0.000284349,0.02675664,0.02059298,0.483101],"study_design_scores_gemma":[0.003485309,0.0009511887,0.2150431,0.0005213569,0.00005061744,0.000006225203,0.001208445,0.5956204,0.00005170642,0.1763625,0.006426179,0.00027296],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.03243521,0.0001012836,0.9651323,0.0006145685,0.0007373785,0.0006140475,0.00008665168,0.00001027044,0.000268267],"genre_scores_gemma":[0.5394728,0.000008728366,0.4602003,0.00008952562,0.0001099004,0.00002323863,0.00001157251,0.000005826205,0.00007804557],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.5749459,"threshold_uncertainty_score":0.9489757,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3172444956","doi":"10.1109/access.2021.3084050","title":"The Matthews Correlation Coefficient (MCC) is More Informative Than Cohen’s Kappa and Brier Score in Binary Classification Assessment","year":2021,"lang":"en","type":"article","venue":"IEEE Access","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":429,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto","funders":"","keywords":"Brier score; Kappa; Cohen's kappa; Correlation; Computer science; Statistics; Artificial intelligence; Binary classification; Metric (unit); Binary number; Odds; Machine learning; Mathematics; Natural language processing; Logistic regression; Support vector machine; Arithmetic","retraction":null,"screen_n_in":null,"score":{"opus":0.1877563290803927,"gpt":0.4289589341158113,"spread":0.2412026050354187,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.003729351,0.0001569236,0.0002169389,0.0001314416,0.0004161778,0.001092042,0.000685083,0.0000848515,0.0001510596],"category_scores_gemma":[0.0004900451,0.00009778181,0.00006522963,0.0008687254,0.0001973012,0.0008947063,0.0002116759,0.0002486232,0.0000557343],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000169706,"about_ca_system_score_gemma":0.0001908761,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006648133,"about_ca_topic_score_gemma":0.0002266232,"domain_scores_codex":[0.9964032,0.0003539039,0.0008376134,0.0004393026,0.001714478,0.000251458],"domain_scores_gemma":[0.9974095,0.0009074268,0.0003736406,0.0006763844,0.0005540413,0.00007899453],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00007983487,0.0003187938,0.906014,0.0000313883,0.00003205771,0.000009942843,0.006527978,0.008035145,0.0009332365,0.001957368,0.02247477,0.05358554],"study_design_scores_gemma":[0.0004122015,0.00003853438,0.9109122,0.00006662534,0.0000103665,0.000003421452,0.004770107,0.06985734,0.001062906,0.004363029,0.008355946,0.0001472793],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9791437,0.0001607185,0.008619796,0.007217608,0.0008075006,0.0005620633,0.00001814902,0.00001726408,0.00345317],"genre_scores_gemma":[0.9980797,0.0001387113,0.0001284418,0.0008357337,0.00003697666,0.00007367897,0.00001260196,0.000006193337,0.0006879289],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0618222,"threshold_uncertainty_score":0.9999449,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2030560722","doi":"10.1136/bmj.b4012","title":"Risk of bias versus quality assessment of randomised controlled trials: cross sectional study","year":2009,"lang":"en","type":"article","venue":"BMJ","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":345,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Alberta","funders":"","keywords":"Jadad scale; Medicine; Kappa; Publication bias; Funnel plot; Sample size determination; Cohen's kappa; Statistics; Meta-analysis; Correlation; Relative risk; Internal medicine; Mathematics; Confidence interval","retraction":null,"screen_n_in":null,"score":{"opus":0.553386919390064,"gpt":0.5654401656422524,"spread":0.0120532462521884,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.199935,0.0001504685,0.001801988,0.0001492118,0.0001181519,0.0001104418,0.0005073936,0.00007140502,0.0007189066],"category_scores_gemma":[0.1150397,0.00008583317,0.0009039165,0.0003627609,0.00009896282,0.0001252716,0.00004390231,0.0001395854,0.00001193417],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006103941,"about_ca_system_score_gemma":0.0002037268,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001173596,"about_ca_topic_score_gemma":0.00004265311,"domain_scores_codex":[0.9796926,0.01014238,0.005047834,0.0004388659,0.004502318,0.0001760182],"domain_scores_gemma":[0.9670328,0.02743007,0.003404265,0.0008295099,0.001228437,0.00007488337],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.06133809,0.002756515,0.9219787,0.000006583417,0.0006031921,0.000001047696,0.0003418553,0.004842432,0.001043994,0.0005006932,0.002012091,0.004574792],"study_design_scores_gemma":[0.1508288,0.0009760218,0.8422504,0.000006319858,0.0001074987,1.02201e-7,0.0007150553,0.0005874126,0.0002633729,0.004070699,0.000109377,0.00008495322],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9869348,0.00004731954,0.001874721,0.0003370717,0.001012539,0.003510251,0.00004029324,0.00001196677,0.006231035],"genre_scores_gemma":[0.9988416,0.000008171422,0.0006682576,0.00002479627,0.0001418974,0.00007114175,0.000001665499,0.000002430903,0.000240031],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.08949069,"threshold_uncertainty_score":0.8924147,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2036060563","doi":"10.1186/1471-2288-10-82","title":"Inter-rater agreement and reliability of the COSMIN (COnsensus-based Standards for the selection of health status Measurement Instruments) Checklist","year":2010,"lang":"en","type":"article","venue":"BMC Medical Research Methodology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":289,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"McMaster University","funders":"Vrije Universiteit Amsterdam; University of Oxford; McMaster University; Goddard Space Flight Center; University of Washington","keywords":"Checklist; Kappa; Inter-rater reliability; Intraclass correlation; Cohen's kappa; Reliability (semiconductor); Medicine; Quality (philosophy); Agreement; Terminology; Psychology; Applied psychology; Physical therapy; Medical physics; Clinical psychology; Statistics; Psychometrics; Rating scale; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.5948043968788056,"gpt":0.5666070490965269,"spread":0.02819734778227867,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.394015,0.0001699311,0.0006664849,0.0002072472,0.0004807004,0.000051671,0.001216097,0.0002002344,0.001058631],"category_scores_gemma":[0.3844943,0.00007941615,0.0002122274,0.0007389415,0.002842776,0.00004208873,0.0004454363,0.00094143,0.000002441472],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003325753,"about_ca_system_score_gemma":0.00629961,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001463431,"about_ca_topic_score_gemma":0.009785491,"domain_scores_codex":[0.9401533,0.03720393,0.003069788,0.001145841,0.01720643,0.001220689],"domain_scores_gemma":[0.9302254,0.06034505,0.0008108895,0.001648001,0.006530828,0.000439812],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002453449,0.0008920706,0.3676997,0.0008563165,0.00009789977,3.107363e-7,0.001033717,0.0001358515,0.01665502,0.002569515,0.01801015,0.589596],"study_design_scores_gemma":[0.007085396,0.004468236,0.5668253,0.0006157815,0.0000784599,0.000008793487,0.005424237,0.02663061,0.07637274,0.07185221,0.2402325,0.0004056925],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7360402,0.0002679026,0.2360404,0.02366712,0.001168196,0.00253549,0.00008794098,0.000008788287,0.0001840239],"genre_scores_gemma":[0.9704114,0.00007649354,0.02881524,0.0003273527,0.00007062288,0.0001740064,0.000001368642,0.000008853127,0.000114639],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5891903,"threshold_uncertainty_score":0.9998709,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1983075605","doi":"10.1038/nmeth.2659","title":"Error bars","year":2013,"lang":"en","type":"article","venue":"Nature Methods","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":289,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Canada's Michael Smith Genome Sciences Centre","funders":"","keywords":"Computer science; Chemistry","retraction":null,"screen_n_in":null,"score":{"opus":0.2218999693596393,"gpt":0.5448362889887357,"spread":0.3229363196290964,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0149639,0.0001371238,0.0002547374,0.0001393383,0.0001262492,0.0002553707,0.0009416313,0.0003343286,0.008351367],"category_scores_gemma":[0.01221792,0.00008143799,0.0001410055,0.000651349,0.00007608941,0.0002913088,0.0001245385,0.0007337779,0.001975726],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000375629,"about_ca_system_score_gemma":0.00003401992,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002112763,"about_ca_topic_score_gemma":0.000003200607,"domain_scores_codex":[0.9956036,0.001476625,0.0004981103,0.0004922366,0.001656818,0.0002726161],"domain_scores_gemma":[0.9959865,0.002325654,0.0001327875,0.0009141018,0.0004989491,0.0001420523],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000007581722,0.00008243261,0.00383696,0.000004519363,0.00001699553,0.000001283399,0.0002007431,0.00005860048,0.01161407,0.003860106,0.2206148,0.7597019],"study_design_scores_gemma":[0.000180449,0.00005323208,0.0472484,0.000007372477,0.000008434353,0.000002326228,0.0003266779,0.0009819947,0.007390748,0.2261581,0.7174686,0.0001736415],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.2005043,0.004842004,0.4237961,0.03945188,0.01175299,0.002200432,0.00001384809,0.0003114,0.317127],"genre_scores_gemma":[0.2024047,0.000009355923,0.7835223,0.00391298,0.0002409957,0.00005051714,0.000001610348,0.00001113014,0.009846417],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.7595283,"threshold_uncertainty_score":0.9988014,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2088910496","doi":"10.3109/0142159x.2012.703791","title":"Generalizability theory for the perplexed: A practical introduction and guide: AMEE Guide No. 68","year":2012,"lang":"en","type":"article","venue":"Medical Teacher","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":250,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McMaster University","funders":"","keywords":"Generalizability theory; Generalization; Reliability (semiconductor); Computer science; Variance (accounting); Psychology; Management science; Cognitive psychology; Statistics; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.1690163772069375,"gpt":0.4518711929609048,"spread":0.2828548157539673,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.05644593,0.0001599456,0.0002587284,0.0000386504,0.0003011688,0.0001296919,0.0004890902,0.0001678344,0.0142133],"category_scores_gemma":[0.1232879,0.00007815242,0.0001405412,0.0001769917,0.0005986888,0.000310262,0.0002065149,0.0003672504,0.0003808298],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008642022,"about_ca_system_score_gemma":0.0001184141,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003782767,"about_ca_topic_score_gemma":0.00001005719,"domain_scores_codex":[0.9942915,0.001344817,0.0007941172,0.000506057,0.002628412,0.0004351201],"domain_scores_gemma":[0.9950133,0.003105376,0.0001586488,0.000925715,0.0004222743,0.0003746947],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00008474184,0.0003795153,0.03989009,0.00001038858,0.0000402429,4.49781e-7,0.0006533791,0.000003159799,0.0001149553,0.005778612,0.8784801,0.0745644],"study_design_scores_gemma":[0.0003445919,0.00007146253,0.007194897,0.000004889312,0.00004198372,0.00001228238,0.0008709682,0.003345006,0.00005488939,0.01215599,0.9757878,0.0001152438],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.3717804,0.003499884,0.2375172,0.3563158,0.008717683,0.002736955,0.00001120786,0.0001683677,0.01925247],"genre_scores_gemma":[0.8307664,0.0003239797,0.02060918,0.01058083,0.02246063,0.0006459841,0.0000158687,0.00005816309,0.1145389],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.458986,"threshold_uncertainty_score":0.9866878,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2095752327","doi":"10.1002/(sici)1097-0258(20000315)19:5<723::aid-sim379>3.0.co;2-a","title":"Interval estimation for Cohen's kappa as a measure of agreement","year":2000,"lang":"en","type":"article","venue":"Statistics in Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":217,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Western University","funders":"","keywords":"Confidence interval; Statistics; Mathematics; Measure (data warehouse); Kappa; Statistic; Variance (accounting); Delta method; Coverage probability; Asymptotic analysis; Computation; Asymptotic distribution; Cohen's kappa; Applied mathematics; Computer science; Algorithm","retraction":null,"screen_n_in":null,"score":{"opus":0.1194851870437682,"gpt":0.4260948892710158,"spread":0.3066097022272476,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.007252351,0.0001381291,0.0004369779,0.000180623,0.00005625889,0.0000204042,0.0004393494,0.00005284913,0.006227912],"category_scores_gemma":[0.008945916,0.00009497673,0.00004560084,0.0003774818,0.0002492155,0.00007386786,0.00002588437,0.0001163929,0.0001100698],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007963402,"about_ca_system_score_gemma":0.00007118362,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001450716,"about_ca_topic_score_gemma":0.0001421121,"domain_scores_codex":[0.9958637,0.0001877172,0.001203565,0.0003503025,0.002177019,0.0002176944],"domain_scores_gemma":[0.9970811,0.001633237,0.0002395192,0.0004441712,0.0005190375,0.00008291486],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0004506093,0.0003054597,0.003310242,0.000110228,0.0000390168,0.00000870051,0.003275936,0.004834862,0.0005035896,0.02728542,0.09017988,0.869696],"study_design_scores_gemma":[0.004283762,0.002271252,0.01531838,0.0006632088,0.00008327864,0.000004389221,0.00209458,0.08427042,0.0005615975,0.8536577,0.03650804,0.000283358],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.09164632,0.0003324673,0.8774809,0.005971984,0.0009288197,0.001759466,0.0002601117,0.00002090377,0.02159907],"genre_scores_gemma":[0.934467,0.00005486016,0.06239018,0.0004828379,0.0001126822,0.00006699333,0.00004351584,0.00000935096,0.002372566],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8694127,"threshold_uncertainty_score":0.9994022,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2044059462","doi":"10.1097/acm.0b013e31822a6cf8","title":"Rater-Based Assessments as Social Judgments: Rethinking the Etiology of Rater Errors","year":2011,"lang":"en","type":"review","venue":"Academic Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":194,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Northern British Columbia","funders":"","keywords":"Categorical variable; Psychology; Categorization; Impression formation; Inter-rater reliability; Social psychology; Construct (python library); Cognitive psychology; Social perception; Rating scale; Computer science; Perception; Developmental psychology; Artificial intelligence; Machine learning","retraction":null,"screen_n_in":null,"score":{"opus":0.6437870321086094,"gpt":0.5584685302281239,"spread":0.08531850188048551,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02608813,0.0005650882,0.002658526,0.000413971,0.0002885668,0.00002132307,0.003290462,0.001061323,0.00400032],"category_scores_gemma":[0.004496257,0.0002534132,0.0004501408,0.0009071265,0.001110094,0.0001294991,0.0003261665,0.002259265,0.0004188964],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001726529,"about_ca_system_score_gemma":0.0005659985,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006995632,"about_ca_topic_score_gemma":0.000004247121,"domain_scores_codex":[0.9884249,0.003578073,0.003471011,0.0009705822,0.003017773,0.0005376614],"domain_scores_gemma":[0.9924318,0.003139077,0.002767846,0.001093861,0.0004354803,0.0001319354],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00009914445,0.00006793999,0.0004498042,0.003062672,0.0004655869,0.00001226354,0.004852303,0.000002718505,0.00003674251,0.004058517,0.1223053,0.8645869],"study_design_scores_gemma":[0.0004968644,0.0003320122,0.00003835015,0.006003218,0.0006597808,0.000006666482,0.0005592464,0.000007683276,0.00001196749,0.02569501,0.9659476,0.0002416267],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0004319819,0.9837942,0.0007982573,0.004180676,0.002854487,0.001687736,0.00001881627,0.00003452356,0.006199378],"genre_scores_gemma":[0.001708759,0.9919846,0.00007063522,0.002715185,0.0008688346,0.0001505089,0.00005791099,0.00003561417,0.002407931],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.8643453,"threshold_uncertainty_score":0.9999918,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2056109249","doi":"10.1080/00016470412331294445","title":"How reliable are reliability studies of fracture classifications?A systematic review of their methodologies","year":2004,"lang":"en","type":"review","venue":"Acta Orthopaedica Scandinavica","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":184,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Health Sciences Centre; McMaster University Medical Centre","funders":"","keywords":"Checklist; Medicine; Reliability (semiconductor); Kappa; Cohen's kappa; MEDLINE; Statistics; Sample size determination; Medical physics; Data mining; Computer science; Mathematics; Psychology; Cognitive psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.3789064394489871,"gpt":0.4667575180918639,"spread":0.08785107864287689,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03297559,0.001061327,0.01164354,0.0005983827,0.0002825702,0.0001483779,0.003706552,0.0007016168,0.0001521223],"category_scores_gemma":[0.104112,0.0005488904,0.003017348,0.002986034,0.001245539,0.0004181942,0.0006461345,0.001054215,0.00002738791],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005567155,"about_ca_system_score_gemma":0.001083206,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001648936,"about_ca_topic_score_gemma":0.00000671254,"domain_scores_codex":[0.9813682,0.005176373,0.006532552,0.001725344,0.004578433,0.0006190836],"domain_scores_gemma":[0.9618553,0.01625462,0.01230188,0.005852583,0.003470611,0.0002650033],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.000009375584,0.0002024773,0.00006945527,0.9754391,0.0005787341,0.000003351079,0.0001365575,0.000002584139,0.000001095375,0.0001483253,0.008458263,0.01495067],"study_design_scores_gemma":[0.0001255918,0.0001323019,0.00003088827,0.6429625,0.002544132,0.00001207205,0.001014108,4.884353e-7,0.000007906009,0.004710937,0.3480912,0.0003679447],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.000005939002,0.9886809,0.0003195693,0.004711944,0.0005466359,0.005126765,0.0002513237,0.00006688498,0.0002900293],"genre_scores_gemma":[0.0004052568,0.9949648,0.002864145,0.0002213549,0.00007818227,0.0007394561,0.00005089731,0.00004831462,0.0006275662],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.3396329,"threshold_uncertainty_score":0.9996963,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1989853314","doi":"10.1186/1471-2288-9-5","title":"Measuring agreement of administrative data with chart data using prevalence unadjusted and adjusted kappa","year":2009,"lang":"en","type":"article","venue":"BMC Medical Research Methodology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":177,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":true},"ca_institutions":"Calgary General Hospital; Alberta Bone and Joint Health Institute; University of Calgary","funders":"Canadian Institutes of Health Research; Fondation pour la Recherche Médicale","keywords":"Kappa; Cohen's kappa; Statistics; Medicine; Sampling (signal processing); Sample size determination; Sample (material); Mathematics; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.9684346225908497,"gpt":0.6570201568897845,"spread":0.3114144657010651,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","open_science","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1828558,0.0002265619,0.0008053505,0.000405469,0.00026327,0.00009626042,0.005753323,0.0002279883,0.001188401],"category_scores_gemma":[0.2484596,0.0001428505,0.00003819934,0.001287191,0.001629139,0.0005030331,0.00259774,0.0008296284,0.00001981652],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007229082,"about_ca_system_score_gemma":0.002306484,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002220004,"about_ca_topic_score_gemma":0.001191019,"domain_scores_codex":[0.9596533,0.02368828,0.001591617,0.002099676,0.01202403,0.0009430707],"domain_scores_gemma":[0.9358465,0.05494029,0.0005086852,0.005941864,0.001830552,0.0009320845],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.005504302,0.002212081,0.1304403,0.001494355,0.000405046,0.0003023226,0.002959083,0.0002245794,0.02797928,0.007033973,0.01093176,0.8105129],"study_design_scores_gemma":[0.009283043,0.009673474,0.6395493,0.004378966,0.0004193304,0.000432117,0.0224212,0.2441572,0.01166218,0.04065389,0.01580456,0.001564746],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5973017,0.002872126,0.3907014,0.005393614,0.0002711269,0.001746694,0.0002083691,0.00003211892,0.001472812],"genre_scores_gemma":[0.7302495,0.0007399202,0.2682155,0.0002326822,0.0002154479,0.00001454449,0.00005388672,0.00001269744,0.0002658691],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8089482,"threshold_uncertainty_score":0.9997246,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2123033778","doi":"10.1177/0962280210383082","title":"Closed-form confidence intervals for functions of the normal mean and standard deviation","year":2010,"lang":"en","type":"article","venue":"Statistical Methods in Medical Research","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":172,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"Robarts Clinical Trials; Western University","funders":"Ontario Ministry of Research and Innovation; Natural Sciences and Engineering Research Council of Canada","keywords":"Standard deviation; Confidence interval; Percentile; Mathematics; Statistics; Coefficient of variation; Robust confidence intervals; Normal distribution; Geometric standard deviation; Standard error; CDF-based nonparametric confidence interval; Variance (accounting); Mean difference; Range (aeronautics); Limits of agreement; Relative standard deviation; Medicine; Detection limit; Nuclear medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.3547369148091911,"gpt":0.6128973161725231,"spread":0.258160401363332,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1100707,0.00008988532,0.000310686,0.0001826272,0.0002334441,0.00009938571,0.0008777013,0.0001570354,0.002872102],"category_scores_gemma":[0.3485041,0.00004952302,0.00006353475,0.0007182384,0.001710188,0.0001020159,0.0003706187,0.00111577,0.00001005962],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004761361,"about_ca_system_score_gemma":0.0004875911,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001127896,"about_ca_topic_score_gemma":0.001055198,"domain_scores_codex":[0.9890402,0.003324019,0.001047715,0.0004582904,0.005692075,0.0004377214],"domain_scores_gemma":[0.9306525,0.06664287,0.0001372804,0.0006660515,0.001538578,0.000362738],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0001761867,0.00007759803,0.00546384,0.00004866521,0.000008367719,0.000001232246,0.0004047066,0.000001825551,0.002088815,0.1264373,0.002488403,0.8628031],"study_design_scores_gemma":[0.0006617922,0.0004105788,0.06661344,0.00009610029,0.00000979521,0.000002325616,0.0009008228,0.01095056,0.003013095,0.8975269,0.01972528,0.00008934994],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05266859,0.00004051549,0.9382982,0.005628624,0.0006493448,0.0006895743,0.00009816025,0.000005139726,0.001921882],"genre_scores_gemma":[0.6721897,0.0000249475,0.3271143,0.0001489413,0.00009579927,0.0001044804,0.00000213681,0.000006607864,0.0003130741],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8627138,"threshold_uncertainty_score":0.9980394,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1949714730","doi":"10.5435/00124635-200207000-00007","title":"Classification Systems in Orthopaedics","year":2002,"lang":"en","type":"review","venue":"Journal of the American Academy of Orthopaedic Surgeons","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":164,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of British Columbia","funders":"","keywords":"Medicine; Reliability (semiconductor); Documentation; Kappa; Measure (data warehouse); Orthopedic surgery; Medical physics; Cohen's kappa; Reliability engineering; Data mining; Machine learning; Computer science; Surgery","retraction":null,"screen_n_in":null,"score":{"opus":0.2429347439420307,"gpt":0.4163155049620289,"spread":0.1733807610199982,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.02288431,0.0004883427,0.004111054,0.001513609,0.0001170122,0.0001014212,0.003444245,0.0002928653,0.0001232133],"category_scores_gemma":[0.008399463,0.0002562041,0.002983319,0.004785118,0.0009039257,0.0002530377,0.0002721943,0.002195874,0.00005074701],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000355113,"about_ca_system_score_gemma":0.0005715385,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004790025,"about_ca_topic_score_gemma":0.000005970609,"domain_scores_codex":[0.9829429,0.003911429,0.006631374,0.0005237982,0.005510183,0.0004803664],"domain_scores_gemma":[0.9766468,0.004920145,0.01693531,0.0008044552,0.0004396182,0.0002536088],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0000194342,0.0001995295,0.01505993,0.0007169535,0.0001593684,0.00001228964,0.00005352837,0.0001177387,0.000001487047,0.0001447858,0.01512919,0.9683858],"study_design_scores_gemma":[0.0002010545,0.00008689235,0.01029567,0.007176855,0.0004253225,0.000199344,0.0002973972,0.0000807675,2.675929e-7,0.0002333529,0.9807619,0.0002411685],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.00195711,0.9936304,0.00006027457,0.001633776,0.001534059,0.0007050229,0.00003851066,0.000007682295,0.0004331656],"genre_scores_gemma":[0.006966587,0.9919982,0.00008118189,0.0001391443,0.0003101501,0.00001873159,8.2998e-7,0.00003261169,0.0004525431],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9681446,"threshold_uncertainty_score":0.999989,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2152177373","doi":"10.2106/jbjs.h.01624","title":"Evaluating Agreement: Conducting a Reliability Study","year":2009,"lang":"en","type":"article","venue":"Journal of Bone and Joint Surgery","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":162,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Sunnybrook Health Science Centre; University of Toronto; McMaster University","funders":"","keywords":"Reliability (semiconductor); Sample size determination; Computer science; Reliability engineering; Context (archaeology); Test (biology); Psychology; Statistics; Engineering; Mathematics; Power (physics)","retraction":null,"screen_n_in":null,"score":{"opus":0.5353733677564685,"gpt":0.4567027472231148,"spread":0.07867062053335366,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.07968286,0.0001595904,0.001012014,0.0004823384,0.0002311266,0.0002435048,0.00009593881,0.00004766479,0.0002746231],"category_scores_gemma":[0.02232389,0.0001022674,0.0004188722,0.0004610953,0.00006045834,0.0005913446,0.00006207136,0.0003206627,0.00001409761],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006093482,"about_ca_system_score_gemma":0.0001484881,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001126288,"about_ca_topic_score_gemma":0.000003174543,"domain_scores_codex":[0.9920106,0.0009673272,0.003397904,0.000342414,0.003017287,0.0002644559],"domain_scores_gemma":[0.9940507,0.002198091,0.001990377,0.000438238,0.001140123,0.0001824006],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0004087916,0.002725828,0.4889885,0.00003908601,0.000102337,0.0001957571,0.003877291,0.0008973652,0.01671037,0.00005209028,0.01654116,0.4694614],"study_design_scores_gemma":[0.0015204,0.002728166,0.9408506,0.0003613296,0.0001334101,0.000201676,0.01291084,0.0006483971,0.001311331,0.03651258,0.00242726,0.000394006],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9942325,0.0005769378,0.0002791395,0.003648982,0.0005936014,0.0002260087,7.519571e-7,0.000006859284,0.0004352307],"genre_scores_gemma":[0.9982053,0.00005181001,0.001007356,0.0003651157,0.0002317847,0.000001297626,1.519202e-7,0.000003600182,0.0001336023],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4690674,"threshold_uncertainty_score":0.9859115,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2113321769","doi":"10.1136/eb-2013-101243","title":"The two-step Fagan's nomogram: ad hoc interpretation of a diagnostic test result without calculation","year":2013,"lang":"en","type":"article","venue":"Evidence-Based Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":136,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Prince Edward Island","funders":"","keywords":"Nomogram; Fagan inspection; Computer science; Test (biology); Medicine; Software engineering; Geology","retraction":null,"screen_n_in":null,"score":{"opus":0.1152710325688526,"gpt":0.3861066713332384,"spread":0.2708356387643858,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.009809416,0.0002045441,0.0003932458,0.0001851621,0.0002321672,0.0001303396,0.0007973243,0.00006535591,0.0004274227],"category_scores_gemma":[0.1928435,0.0001024716,0.0001187076,0.0008718849,0.0006280133,0.0005231996,0.00006214061,0.0002067242,0.0002528871],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001080826,"about_ca_system_score_gemma":0.0001592554,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004606884,"about_ca_topic_score_gemma":0.0003156608,"domain_scores_codex":[0.9941412,0.000623463,0.001401503,0.0004933754,0.003042186,0.0002982831],"domain_scores_gemma":[0.9470395,0.04936058,0.0007290411,0.001052393,0.00164439,0.0001740887],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0003383303,0.0002226146,0.1642155,0.00007757479,0.00003319109,0.000004540905,0.001785972,0.003179232,0.009445188,0.000175027,0.0354628,0.78506],"study_design_scores_gemma":[0.005642888,0.00479252,0.6165152,0.01038941,0.0002930458,0.000006661123,0.003598252,0.3022066,0.004936134,0.01405987,0.03686929,0.0006901431],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7494009,0.01181111,0.1040706,0.1273637,0.001963361,0.00380952,0.00001070384,0.0001079809,0.001462136],"genre_scores_gemma":[0.9969175,0.0003676884,0.0009826577,0.0008622043,0.0001921153,0.000141341,0.000005971085,0.000009324641,0.0005211396],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7843698,"threshold_uncertainty_score":0.8139555,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"medium"}],"label_agreement":"split"},{"id":"W1970971885","doi":"10.1016/j.jclinepi.2011.10.019","title":"A confidence interval approach to sample size estimation for interobserver agreement studies with multiple raters and outcomes","year":2012,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":131,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Western University; York University","funders":"","keywords":"Confidence interval; Triage; Sample size determination; Reliability (semiconductor); Medicine; Estimation; Medical physics; Sample (material); Interval estimation; Statistics; Scale (ratio); Mathematics; Psychiatry; Power (physics)","retraction":null,"screen_n_in":null,"score":{"opus":0.7278829789470725,"gpt":0.5892656106725227,"spread":0.1386173682745498,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.09069584,0.0001803115,0.002053035,0.00009117781,0.0000845443,0.00003203431,0.0004777616,0.0001088144,0.00002711038],"category_scores_gemma":[0.5352219,0.00008927724,0.0003826142,0.0001218673,0.0003215185,0.0003679059,0.0001625698,0.0002908761,0.00001083593],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005151686,"about_ca_system_score_gemma":0.0000494451,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001758943,"about_ca_topic_score_gemma":0.0000239838,"domain_scores_codex":[0.9912939,0.002782632,0.004568639,0.0003588049,0.0006178538,0.0003781493],"domain_scores_gemma":[0.6780325,0.3153101,0.003940808,0.0005730192,0.001509238,0.0006343322],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001103802,0.0003167372,0.9745274,0.00003082355,0.0002538756,3.571976e-7,0.0006800868,0.001985873,0.000007718636,0.001437612,0.005383465,0.01427224],"study_design_scores_gemma":[0.002237365,0.00287811,0.9346225,0.0001292804,0.00009984955,0.00001553144,0.002355095,0.004781458,0.00001742879,0.04831425,0.004381427,0.0001676564],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5510898,0.0002165327,0.4377364,0.009640635,0.0009378351,0.0003496362,0.000005271525,0.000003412427,0.00002046452],"genre_scores_gemma":[0.7412383,0.00005265777,0.2546178,0.003840247,0.0001733146,0.00001950056,3.934305e-7,0.000004288615,0.0000535332],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4445261,"threshold_uncertainty_score":0.93632,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1978765295","doi":"10.2202/1557-4679.1275","title":"Sample Size Requirements for Interval Estimation of the Kappa Statistic for Interobserver Agreement Studies with a Binary Outcome and Multiple Raters","year":2010,"lang":"en","type":"article","venue":"The International Journal of Biostatistics","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":127,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Western University","funders":"","keywords":"Statistics; Cohen's kappa; Confidence interval; Kappa; Intraclass correlation; Mathematics; Interval estimation; Statistic; Sample size determination; Sample (material); Binary number; Limit (mathematics); Interval (graph theory); Psychometrics; Combinatorics","retraction":null,"screen_n_in":null,"score":{"opus":0.2216258336390316,"gpt":0.4380071238304545,"spread":0.2163812901914229,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.004113966,0.0001335865,0.0002824377,0.00007955485,0.0001356879,0.0001308151,0.0009536272,0.00002662906,0.00003126899],"category_scores_gemma":[0.02575179,0.00006102618,0.0001129646,0.00009102379,0.0003308829,0.0001707311,0.0001740547,0.00014133,9.10318e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007405372,"about_ca_system_score_gemma":0.0000755367,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002003962,"about_ca_topic_score_gemma":0.0001547477,"domain_scores_codex":[0.9970289,0.0001053058,0.00116739,0.0001661636,0.001400865,0.0001313647],"domain_scores_gemma":[0.9857738,0.0104351,0.001269922,0.0002525876,0.002223991,0.00004462379],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.01482201,0.002080193,0.547716,0.00074401,0.006213073,0.00002400472,0.0172447,0.03111898,0.07179476,0.04494015,0.06721252,0.1960896],"study_design_scores_gemma":[0.01558455,0.006142671,0.2742167,0.001239144,0.0010018,0.0001133647,0.01439831,0.1746508,0.01906436,0.4832476,0.009562142,0.0007785992],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6872414,0.00003646775,0.304521,0.005011971,0.002003114,0.0005767656,0.0006028786,0.000002112089,0.000004281791],"genre_scores_gemma":[0.8858047,0.00001290703,0.1137049,0.000299639,0.0000769631,0.00001203056,0.000003935047,0.000006858631,0.00007806447],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4383075,"threshold_uncertainty_score":0.9824547,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2044481915","doi":"10.1038/modpathol.3800496","title":"Interobserver agreement and reproducibility in classification of invasive breast carcinoma: an NCI breast cancer family registry study","year":2005,"lang":"en","type":"article","venue":"Modern Pathology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":116,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Mount Sinai Hospital; University of Toronto; Jewish General Hospital; Statistics Canada","funders":"National Cancer Institute; National Institutes of Health; National Institute of Informatics","keywords":"Medicine; Reproducibility; Breast cancer; Pathology; Breast carcinoma; Cancer registry; Cancer; Oncology; Radiology; Internal medicine; Epidemiology","retraction":null,"screen_n_in":null,"score":{"opus":0.1848026354599528,"gpt":0.3738318780764844,"spread":0.1890292426165316,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01032803,0.0001995051,0.0004693821,0.0001788561,0.00007587572,0.00004995114,0.0006467848,0.0001164724,0.00008361291],"category_scores_gemma":[0.000703709,0.0001558972,0.00006273251,0.0003027861,0.0002690189,0.0003654748,0.0002693345,0.0002159462,0.00001317068],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002082196,"about_ca_system_score_gemma":0.00012579,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000774084,"about_ca_topic_score_gemma":0.009485027,"domain_scores_codex":[0.9944434,0.001088146,0.001190479,0.002076925,0.0009134698,0.0002875441],"domain_scores_gemma":[0.995874,0.0001717498,0.0004246087,0.00292276,0.0005076555,0.00009921764],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001717822,0.001403417,0.9166543,0.00001569428,0.000009188963,0.00001197017,0.004692938,0.0004854364,0.02469855,0.00002856402,0.0001742542,0.05165387],"study_design_scores_gemma":[0.0008592479,0.0002229116,0.9899136,0.00001887829,0.00001736796,0.00003423497,0.00371461,0.001893681,0.0002886348,0.00286961,0.00002525495,0.0001419292],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9958175,0.0002205886,0.0003348846,0.002367548,0.0002095079,0.0007483629,0.00008484458,0.00001294675,0.0002038275],"genre_scores_gemma":[0.9992701,0.00001912527,0.0002031265,0.0001841834,0.00007187137,0.00008605952,0.000005619534,0.000008187208,0.0001517453],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0732593,"threshold_uncertainty_score":0.6357303,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2155433306","doi":"10.1590/s1413-35552006000200002","title":"Importance and clarification of measurement properties in rehabilitation","year":2006,"lang":"en","type":"article","venue":"Brazilian Journal of Physical Therapy","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":109,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Alberta","funders":"Canadian Institutes of Health Research; Coordenação de Aperfeiçoamento de Pessoal de Nível Superior; University of Alberta","keywords":"Reliability (semiconductor); Rehabilitation; Quality (philosophy); Clinical Practice; Validity; Field (mathematics); Psychology; Applied psychology; Computer science; Management science; Psychometrics; Medicine; Power (physics); Physical therapy; Clinical psychology; Engineering; Mathematics; Epistemology","retraction":null,"screen_n_in":null,"score":{"opus":0.1405243827265361,"gpt":0.3289915533775897,"spread":0.1884671706510536,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005683126,0.00009158583,0.0003075088,0.0001456914,0.00003341588,0.0000432901,0.0002565675,0.00002792096,0.00001200177],"category_scores_gemma":[0.0008374867,0.00005498101,0.0001260508,0.0003275616,0.0001545904,0.000321173,0.00001014634,0.0001278193,0.000002129911],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006050646,"about_ca_system_score_gemma":0.00006159822,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001476753,"about_ca_topic_score_gemma":0.00002060828,"domain_scores_codex":[0.9966618,0.0003404073,0.0009648737,0.0001704825,0.001756354,0.0001060655],"domain_scores_gemma":[0.9980953,0.0002112517,0.0005915935,0.0002240087,0.0008390243,0.00003880622],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"observational","study_design_scores_codex":[0.0005995022,0.002023137,0.2399451,0.00003554198,0.00002798956,0.000001917044,0.002508861,0.001487307,0.5687832,0.002454255,0.0006281472,0.1815051],"study_design_scores_gemma":[0.001048273,0.000920958,0.8344077,0.0001336213,0.000004984463,0.000001793555,0.0005450579,0.0003543655,0.03370433,0.1276754,0.001110766,0.00009271057],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9967065,0.0009016003,0.0002616797,0.001711323,0.00005960467,0.0001928585,7.016931e-7,0.000002264547,0.0001634695],"genre_scores_gemma":[0.9993198,0.00003507407,0.0004901625,0.00003283509,0.0001014064,0.000003719356,1.158528e-7,0.000003980996,0.00001288741],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5944626,"threshold_uncertainty_score":0.2242061,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2470750097","doi":"","title":"Enhancing retrieval of best evidence for health care from bibliographic databases: calibration of the hand search of the literature.","year":2001,"lang":"en","type":"article","venue":"PubMed","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":102,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"McMaster University; Hamilton Health Sciences","funders":"","keywords":"Cohen's kappa; Statistic; Information retrieval; Reliability (semiconductor); Computer science; MEDLINE; Health care; Test (biology); Medical education; Database; Medicine; Statistics; Machine learning; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.267435042832476,"gpt":0.4016326631986881,"spread":0.1341976203662121,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00624426,0.00008429028,0.0002416047,0.0003813077,0.0001774336,0.00009243298,0.0008583294,0.00003946256,0.00001692839],"category_scores_gemma":[0.00499357,0.00004140383,0.0001949498,0.005210837,0.0002331401,0.0003498762,0.000169196,0.0001131711,2.263711e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003019228,"about_ca_system_score_gemma":0.0001593642,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004008315,"about_ca_topic_score_gemma":0.001000766,"domain_scores_codex":[0.9962166,0.0005633832,0.0007905021,0.0002990311,0.001914658,0.0002158298],"domain_scores_gemma":[0.9963071,0.001481035,0.0004735831,0.000929822,0.0007513001,0.00005715108],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001087706,0.0004057723,0.9126067,0.0006913567,0.00008460243,3.445851e-7,0.01351191,0.0005203877,0.02722429,0.0002764221,0.00419822,0.03939233],"study_design_scores_gemma":[0.0005340591,0.0001128118,0.7333352,0.0008832099,0.00003812107,5.043844e-7,0.002758601,0.0001789597,0.2592857,0.001447282,0.001329144,0.00009647669],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9879133,0.004823403,0.002388302,0.002592412,0.0003742858,0.001579554,0.0002995568,0.000002784444,0.00002644639],"genre_scores_gemma":[0.9990651,0.0004075948,0.0002116443,0.0001325316,0.00005629508,0.00005969403,0.000006563852,0.000003773233,0.00005678141],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2320614,"threshold_uncertainty_score":0.597813,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1887790309","doi":"10.1186/1742-5573-3-11","title":"New patient-oriented summary measure of net total gain in certainty for dichotomous diagnostic tests","year":2006,"lang":"en","type":"article","venue":"Epidemiologic Perspectives & Innovations","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of British Columbia","funders":"","keywords":"Reciprocal; Measure (data warehouse); Certainty; Youden's J statistic; Index (typography); Medicine; Statistics; Population; Disease; Computer science; Test (biology); Predictive value; Mathematics; Data mining; Internal medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.08524577208288033,"gpt":0.3642964651287134,"spread":0.2790506930458331,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.007246373,0.0002673558,0.0006775666,0.0005269886,0.0001905718,0.00003829348,0.0004737959,0.000165434,0.0002027199],"category_scores_gemma":[0.09875732,0.0001972619,0.0002216839,0.002628006,0.0002386268,0.0002554021,0.0001042011,0.0002862617,0.00001848899],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003044399,"about_ca_system_score_gemma":0.0002280609,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001038702,"about_ca_topic_score_gemma":0.0005350477,"domain_scores_codex":[0.9950262,0.000767392,0.001964834,0.0008364648,0.0009080434,0.0004970699],"domain_scores_gemma":[0.9818092,0.01505455,0.0008118937,0.0007049766,0.001533801,0.00008556204],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001670198,0.001187596,0.7242026,0.00001522707,0.00005017987,0.000005356021,0.001880554,0.02167976,0.0031169,0.1207655,0.1130004,0.013929],"study_design_scores_gemma":[0.001081347,0.0005228269,0.7158669,0.00008200976,0.0000192911,0.000002497048,0.006719258,0.001101922,0.0002890593,0.2709298,0.003057321,0.0003278165],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9134075,0.00147047,0.06910617,0.007762569,0.0004755518,0.002007333,0.0002351916,0.00007834889,0.005456832],"genre_scores_gemma":[0.9896062,0.00001562284,0.009395287,0.0002475347,0.0001425282,0.0001873564,0.00007279379,0.000008522932,0.0003241547],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1501643,"threshold_uncertainty_score":0.9088342,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2016242508","doi":"10.1016/j.jclinepi.2010.12.001","title":"The difference between reliability and agreement","year":2011,"lang":"en","type":"letter","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":93,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto","funders":"","keywords":"Scopus; Inter-rater reliability; Intraclass correlation; Reliability (semiconductor); Kappa; Cohen's kappa; Statistics; Psychology; Medicine; Mathematics; MEDLINE; Physics; Psychometrics; Political science","retraction":null,"screen_n_in":null,"score":{"opus":0.6917992205636752,"gpt":0.5490965920628558,"spread":0.1427026285008194,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","research_integrity"],"consensus_categories":["metaresearch","research_integrity"],"category_scores_codex":[0.2983632,0.0004179197,0.004917853,0.0001585916,0.0002974881,0.00009326921,0.003278539,0.001673133,0.0002906569],"category_scores_gemma":[0.4813201,0.0001817077,0.001647482,0.0002033678,0.002081532,0.0001178354,0.0005648548,0.007275918,0.0001339988],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006771901,"about_ca_system_score_gemma":0.0003666239,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004514266,"about_ca_topic_score_gemma":0.00001582638,"domain_scores_codex":[0.9308183,0.04178641,0.02202808,0.001287097,0.003088524,0.0009915679],"domain_scores_gemma":[0.5006245,0.4758477,0.01830079,0.002482411,0.002227405,0.0005172578],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00006318836,0.00003419346,0.3578837,0.00001340376,0.0001043645,0.00001639474,0.0000180379,0.00000147778,1.890721e-7,0.00009793718,0.5578282,0.08393882],"study_design_scores_gemma":[0.0002075643,0.0005425882,0.3219838,0.00004897955,0.00007399208,0.00000903469,0.00001205528,0.000009234634,5.966838e-7,0.2173314,0.459684,0.00009669408],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.03808391,0.001321954,0.005125985,0.9488102,0.005243683,0.0003347505,0.00001731159,0.000006484633,0.001055723],"genre_scores_gemma":[0.03287585,0.008884266,0.00660981,0.9198707,0.02535897,0.00001664701,0.000006767917,0.00003713779,0.006339813],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.4340613,"threshold_uncertainty_score":0.9996229,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2099954138","doi":"10.1016/j.ygyno.2005.07.040","title":"Kappa statistics to measure interrater and intrarater agreement for 1790 cervical biopsy specimens among twelve pathologists: Qualitative histopathologic analysis and methodologic issues","year":2005,"lang":"en","type":"article","venue":"Gynecologic Oncology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":90,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of British Columbia","funders":"National Cancer Institute","keywords":"Medicine; Biopsy; Inter-rater reliability; Cohen's kappa; Cervical cancer; Radiology; Kappa; Family medicine; Cancer; Statistics; Internal medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.2655803757781627,"gpt":0.4873603395796752,"spread":0.2217799638015124,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02167428,0.0005250087,0.001603962,0.0006906674,0.0003879808,0.0002371023,0.0007766491,0.0005269216,0.002428644],"category_scores_gemma":[0.01218319,0.0003445577,0.0002358359,0.0008973108,0.001106595,0.0002400235,0.0005930126,0.000406082,0.0001980606],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005820693,"about_ca_system_score_gemma":0.00008526498,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005595708,"about_ca_topic_score_gemma":0.001815171,"domain_scores_codex":[0.9901624,0.004570346,0.001741697,0.001755707,0.0009148847,0.0008549701],"domain_scores_gemma":[0.9889414,0.008439044,0.0006521557,0.0007358625,0.0008530112,0.0003784896],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002547797,0.003529766,0.3018108,0.0001602781,0.002038486,0.0004830072,0.08651789,0.001950942,0.0177167,0.01679632,0.08312747,0.4833205],"study_design_scores_gemma":[0.002624947,0.009718316,0.7209008,0.00002290404,0.001303673,0.00005871608,0.03155709,0.001814052,0.001078077,0.1313717,0.09815412,0.00139556],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8078763,0.0006070869,0.1714945,0.01557534,0.0006080382,0.001816253,0.0002511194,0.00006817509,0.001703133],"genre_scores_gemma":[0.7277134,0.00009657118,0.2670175,0.002841398,0.0001637549,0.0004214282,0.00003377517,0.00001292224,0.001699268],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.481925,"threshold_uncertainty_score":0.9999006,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2146976162","doi":"10.1002/(sici)1097-0258(20000215)19:3<373::aid-sim337>3.0.co;2-y","title":"Testing the equality of two dependent kappa statistics","year":2000,"lang":"en","type":"article","venue":"Statistics in Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":89,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Guelph; Western University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Statistics; Kappa; Correlation; Monte Carlo method; Mathematics; Cohen's kappa; Goodness of fit; Dependency (UML); Sample size determination; Variable (mathematics); Computer science; Artificial intelligence","retraction":null,"screen_n_in":null,"score":{"opus":0.2670104097609483,"gpt":0.4556427207423286,"spread":0.1886323109813803,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01463562,0.0001596583,0.0004365018,0.0001061005,0.0001287112,0.00003519339,0.0008215466,0.00003758503,0.00546636],"category_scores_gemma":[0.01668274,0.00008785782,0.00002191709,0.0007223649,0.0006210688,0.00005408334,0.00007111597,0.0002935005,0.0001087362],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006649305,"about_ca_system_score_gemma":0.00009942054,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001001015,"about_ca_topic_score_gemma":0.0006780621,"domain_scores_codex":[0.9938334,0.0007102268,0.001599391,0.0003923825,0.003155049,0.0003095853],"domain_scores_gemma":[0.9887842,0.009418523,0.0003409826,0.0007828973,0.000582,0.00009143108],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00009540518,0.0002852184,0.09400722,0.00006967088,0.00003091168,0.00004357227,0.003204483,0.00447572,0.00116882,0.09661052,0.04216598,0.7578425],"study_design_scores_gemma":[0.001895219,0.0004329739,0.1491368,0.0001718927,0.00005242628,0.000006656131,0.001584292,0.01274943,0.0001400724,0.8244635,0.00915303,0.0002137071],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1872347,0.0005463707,0.7453173,0.003448757,0.001312718,0.001222724,0.001363933,0.00003883952,0.05951462],"genre_scores_gemma":[0.922231,0.00005878457,0.07571973,0.0005517206,0.0001681844,0.00001450345,0.00001900848,0.00001007761,0.001226982],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7576288,"threshold_uncertainty_score":0.9954428,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3035592618","doi":"10.1186/s12874-020-01022-x","title":"Using multiple agreement methods for continuous repeated measures data: a tutorial for practitioners","year":2020,"lang":"en","type":"article","venue":"BMC Medical Research Methodology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":88,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; Emory University","keywords":"Concordance; Cohen's kappa; Computer science; Concordance correlation coefficient; Data mining; Statistics; Strengths and weaknesses; Correlation coefficient; Pulmonary disease; Limits of agreement; Medical physics; Medicine; Mathematics; Machine learning; Psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.9644013985579888,"gpt":0.7181859517571121,"spread":0.2462154468008767,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3470378,0.0002609011,0.001126708,0.0003171687,0.000635898,0.0002511844,0.003297167,0.000447857,0.0008096859],"category_scores_gemma":[0.8918822,0.0001864536,0.0003135111,0.001076753,0.0008022512,0.0003499154,0.001207392,0.0007865009,0.00004570411],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001853395,"about_ca_system_score_gemma":0.002523184,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002758091,"about_ca_topic_score_gemma":0.0003406374,"domain_scores_codex":[0.9347641,0.05104046,0.002333362,0.002412321,0.008000678,0.001449097],"domain_scores_gemma":[0.4472745,0.5396779,0.0008955213,0.003221999,0.006813199,0.002116955],"domain_codex":null,"domain_gemma":"methods","domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.009065401,0.0005422449,0.002173008,0.0003620243,0.0003355102,0.00001661116,0.001600174,0.0005165896,0.0571048,0.005255338,0.1233234,0.7997049],"study_design_scores_gemma":[0.003950355,0.001112296,0.00008803976,0.00005651392,0.00006757223,0.000008103986,0.002799793,0.2378906,0.004435614,0.03224042,0.7170838,0.0002669609],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.002484589,0.0004510766,0.9788967,0.01254353,0.002125253,0.003102865,0.0001233271,0.00005018777,0.0002224705],"genre_scores_gemma":[0.005849201,0.00007368642,0.9896866,0.001105719,0.002447798,0.00047534,0.00008486683,0.00002762176,0.0002491588],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.7994379,"threshold_uncertainty_score":0.8865492,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2143965974","doi":"10.1093/ije/dyp370","title":"Forest plots in reports of systematic reviews: a cross-sectional study reviewing current practice","year":2010,"lang":"en","type":"article","venue":"International Journal of Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":87,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"Cancer Research UK; University of Ottawa","keywords":"Cross-sectional study; Current (fluid); Medicine; MEDLINE; Systematic review; Environmental health; Geography; Political science; Engineering; Pathology","retraction":null,"screen_n_in":null,"score":{"opus":0.5444736336084718,"gpt":0.5819775103364283,"spread":0.03750387672795652,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1578574,0.0001501814,0.001296312,0.0004952039,0.00003885533,0.00006031524,0.00117878,0.00008551392,0.0001858839],"category_scores_gemma":[0.5019255,0.00009108941,0.000419121,0.0002845149,0.0001341087,0.0006136806,0.0001493383,0.0007360405,0.00003024081],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001227253,"about_ca_system_score_gemma":0.0001470282,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004663333,"about_ca_topic_score_gemma":0.0001192886,"domain_scores_codex":[0.9800361,0.0055925,0.01116383,0.0004576092,0.002502323,0.0002475987],"domain_scores_gemma":[0.9625373,0.02249492,0.009888959,0.0006168906,0.00434673,0.0001152209],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00008294608,0.0006904327,0.9940318,0.0002940831,0.00008050904,0.00006757125,0.0001960741,0.0006581921,0.0001307052,0.0007980984,0.0006878863,0.002281706],"study_design_scores_gemma":[0.0005244666,0.0003291442,0.9345525,0.002676393,0.00005228991,0.001187476,0.0002224623,0.000202508,0.00001037533,0.03380745,0.0263103,0.0001246225],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9765968,0.007376542,0.002569671,0.001538771,0.01056196,0.001061165,0.000002921299,0.000003717205,0.0002884869],"genre_scores_gemma":[0.9956835,0.0008873563,0.002594403,0.0003123213,0.0004542052,0.000033097,0.000001097353,0.00000519523,0.00002883119],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3440681,"threshold_uncertainty_score":0.8671631,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2107006322","doi":"10.1370/afm.1195","title":"Methods to Achieve High Interrater Reliability in Data Collection From Primary Care Medical Records","year":2011,"lang":"en","type":"article","venue":"The Annals of Family Medicine","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":83,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Élisabeth Bruyère Hospital; Bruyère; University of Ottawa","funders":"","keywords":"Medicine; Inter-rater reliability; Reliability (semiconductor); Quality assurance; Chart; Confidence interval; Audit; Statistics; Sample size determination; Data collection; Control chart; Quality (philosophy); Sample (material); Quality management; Statistic; Medical physics; Operations management; Computer science; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.7358387871238369,"gpt":0.5387267113294011,"spread":0.1971120757944358,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03377779,0.0001919913,0.0006673181,0.000233398,0.0000861706,0.00001622972,0.003048468,0.0001331717,0.001460184],"category_scores_gemma":[0.01376385,0.0000971556,0.0000697654,0.001058348,0.0004213185,0.0002245524,0.000875101,0.0003745536,0.00004197165],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006376385,"about_ca_system_score_gemma":0.0001830394,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009034255,"about_ca_topic_score_gemma":0.001082433,"domain_scores_codex":[0.9923752,0.00239038,0.001500649,0.0007730561,0.002670198,0.0002905233],"domain_scores_gemma":[0.993553,0.002528254,0.0002948843,0.002709132,0.0007049104,0.0002098185],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.004025218,0.0006367561,0.04423352,0.00007976044,0.0001085009,0.00001515597,0.04014425,0.0000318631,0.005974001,0.0001152385,0.3169057,0.5877301],"study_design_scores_gemma":[0.00120859,0.00158491,0.8841581,0.0006690299,0.0000490107,0.000001419501,0.01228859,0.000493831,0.00280815,0.06336588,0.03311352,0.0002590028],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9377157,0.001027701,0.01361397,0.03347982,0.002000295,0.0006954911,0.00005253811,0.00002450557,0.01138996],"genre_scores_gemma":[0.9692243,0.0004519374,0.01807741,0.01138403,0.0003401645,0.00003537812,0.00007191067,0.00001453107,0.0004003511],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8399245,"threshold_uncertainty_score":0.9994526,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2134076777","doi":"10.1016/j.jclinepi.2009.08.005","title":"A valid and reliable belief elicitation method for Bayesian priors","year":2009,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":80,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"Women's College Hospital; University of Toronto; Toronto General Hospital; University Health Network; Toronto Western Hospital; Hospital for Sick Children; Public Health Ontario","funders":"Canadian Institutes of Health Research","keywords":"Prior probability; Bayesian probability; Expert elicitation; Statistics; Computer science; Econometrics; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.5853364478923457,"gpt":0.6222949697557054,"spread":0.03695852186335968,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2154536,0.0001357565,0.001906673,0.0001666694,0.0001121269,0.00003448045,0.0005043676,0.0002767187,0.00007204201],"category_scores_gemma":[0.3805293,0.00008129854,0.0006667847,0.0002097568,0.0001336114,0.0002224041,0.00003371562,0.0004813214,0.00001087375],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002492255,"about_ca_system_score_gemma":0.0001342671,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000006124475,"about_ca_topic_score_gemma":0.000004321033,"domain_scores_codex":[0.9856853,0.00556186,0.007261669,0.0004422971,0.0007113589,0.0003375357],"domain_scores_gemma":[0.8742157,0.1192714,0.00450992,0.0004257427,0.001192671,0.0003846222],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0009934983,0.0004438891,0.1248169,0.00001524373,0.00008388076,0.000005222131,0.0002200946,0.003069977,0.0002441427,0.01834535,0.06141168,0.7903501],"study_design_scores_gemma":[0.001159015,0.002943571,0.1722225,0.00004468986,0.00005043001,0.00002441364,0.000118149,0.01177134,0.00002731548,0.7238445,0.08769877,0.00009533406],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.09627637,0.0004883627,0.822445,0.07867032,0.001217888,0.0002859249,0.000002508805,0.000007147788,0.0006065065],"genre_scores_gemma":[0.2231001,0.0006096329,0.7577446,0.01718985,0.0009963679,0.000003860361,8.164579e-7,0.000006769892,0.0003479862],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.7902548,"threshold_uncertainty_score":0.8078556,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2177903965","doi":"10.24095/hpcdp.32.1.05","title":"2008 Niday Perinatal Database quality audit: report of a quality assurance project","year":2011,"lang":"en","type":"article","venue":"Chronic diseases and injuries in Canada","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":71,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"Ottawa Hospital; Ottawa Public Health; University of Ottawa; Champlain Regional College; Children's Hospital of Eastern Ontario; Ontario Stroke Network","funders":"","keywords":"Intraclass correlation; Quality assurance; Cohen's kappa; Data quality; Kappa; Audit; Database; Statistic; Statistics; Quality (philosophy); Medicine; Sample (material); Reliability (semiconductor); Mathematics; Computer science; Reproducibility; Accounting; Operations management; Physics; Business; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.1220847053585366,"gpt":0.3818689116768931,"spread":0.2597842063183565,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003154149,0.0001872393,0.0004965353,0.00008136389,0.000117044,0.00004446628,0.0005564489,0.00003565543,0.0007542467],"category_scores_gemma":[0.004015033,0.0001416601,0.00008955582,0.0003981222,0.000393236,0.0003789649,0.0002476678,0.0001279449,0.00000212523],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005499055,"about_ca_system_score_gemma":0.006877664,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.8429173,"about_ca_topic_score_gemma":0.892978,"domain_scores_codex":[0.9955185,0.0003973772,0.00138267,0.0006457436,0.001722283,0.0003334135],"domain_scores_gemma":[0.9973741,0.0005629056,0.0005342944,0.001097896,0.0002666949,0.0001641656],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002438685,0.0002250375,0.9581692,0.000427631,0.00003842975,0.0001157438,0.0003875525,0.00001301952,0.00008142735,0.001543811,0.02400376,0.01475048],"study_design_scores_gemma":[0.0004367048,0.0001008364,0.966229,0.0001198652,0.00001464288,0.000005176015,0.001545045,0.00004331113,0.0004468995,0.002274009,0.02852862,0.0002558632],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9910793,0.003736627,0.00009229638,0.0002848078,0.000611229,0.0003712107,0.001053831,0.00001018025,0.00276054],"genre_scores_gemma":[0.9991556,0.0002335047,0.000176094,0.00009430497,0.00005192902,0.0000389952,0.00001647657,0.000006092675,0.0002269805],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05006075,"threshold_uncertainty_score":0.9987524,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2592866053","doi":"10.1186/s13643-017-0441-7","title":"Effect of standardized training on the reliability of the Cochrane risk of bias assessment tool: a prospective study","year":2017,"lang":"en","type":"article","venue":"Systematic Reviews","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":65,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Alberta; St. Michael's Hospital; University of Toronto","funders":"","keywords":"Medicine; Kappa; Reliability (semiconductor); Physical therapy; Randomized controlled trial; Cohen's kappa; Risk assessment; MEDLINE; Clinical psychology; Statistics; Surgery","retraction":null,"screen_n_in":null,"score":{"opus":0.2307356376426931,"gpt":0.4609718262253066,"spread":0.2302361885826134,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2333902,0.000260131,0.002960625,0.00005200931,0.0004066641,0.000115745,0.002209717,0.00006080738,0.00008590473],"category_scores_gemma":[0.1990856,0.00008028286,0.0008971494,0.0003205758,0.0003737987,0.0001146932,0.0002303493,0.0002838175,0.00001294993],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000994119,"about_ca_system_score_gemma":0.0001167205,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006071364,"about_ca_topic_score_gemma":0.00002830655,"domain_scores_codex":[0.9651013,0.02518062,0.00429227,0.0005036243,0.004734975,0.0001872093],"domain_scores_gemma":[0.9689186,0.01664128,0.008014901,0.005526992,0.000863271,0.00003498728],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0005409111,0.001780873,0.8551649,0.1044754,0.0007904719,0.000001276525,0.02143411,0.0003538141,0.001040458,0.002762141,0.001027072,0.01062854],"study_design_scores_gemma":[0.006353878,0.009482225,0.8459819,0.09302914,0.002627683,0.000003410328,0.01512998,0.001266922,0.009308466,0.01538805,0.000807728,0.0006206254],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9699941,0.0004119044,0.0003685017,0.0002565146,0.0004060498,0.02576275,0.00002941522,0.000004742484,0.002766029],"genre_scores_gemma":[0.9987981,0.0001019979,0.0001476091,0.000009002571,0.0000168618,0.0008320037,1.019976e-7,0.000006427144,0.00008794246],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03430468,"threshold_uncertainty_score":0.8076609,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2022913092","doi":"10.1007/s11135-013-9919-0","title":"On the choice of measures of reliability and validity in the content-analysis of texts","year":2013,"lang":"en","type":"article","venue":"Quality & Quantity","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":61,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Memorial University of Newfoundland","funders":"","keywords":"Reliability (semiconductor); Rhetorical question; Interpretation (philosophy); Reading comprehension; Complement (music); Reading (process); Kappa; Computer science; Measure (data warehouse); Content (measure theory); Psychology; Natural language processing; Content validity; Comprehension; Linguistics; Statistics; Social psychology; Cognitive psychology; Mathematics; Psychometrics; Data mining; Philosophy","retraction":null,"screen_n_in":null,"score":{"opus":0.6107004055274742,"gpt":0.4566016238261046,"spread":0.1540987817013695,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04846977,0.0001511311,0.000825978,0.0001943807,0.0001007961,0.00006156922,0.001148752,0.0000835285,0.0004174048],"category_scores_gemma":[0.03930073,0.00006999595,0.00040318,0.001591698,0.0007542975,0.0001974649,0.0001229438,0.0002251109,0.00001167729],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002848448,"about_ca_system_score_gemma":0.00004681244,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01672968,"about_ca_topic_score_gemma":0.004504059,"domain_scores_codex":[0.9890718,0.00519361,0.001951117,0.0004576677,0.003132679,0.0001931365],"domain_scores_gemma":[0.9763601,0.01916376,0.001126609,0.002017598,0.001278828,0.00005305616],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00005679812,0.0006928099,0.9720294,0.00004776779,0.0001313018,6.744084e-8,0.001274483,0.0002717421,0.003363539,0.0199721,0.0006180998,0.001541912],"study_design_scores_gemma":[0.000191868,0.0001277928,0.9583821,0.00002055731,0.0001301091,4.047712e-8,0.001875709,0.0004187029,0.002590917,0.03605396,0.0001333665,0.00007483747],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9949781,0.0000858313,0.0003830842,0.003135792,0.00007052761,0.0006311501,0.00005094063,0.000004376049,0.0006602306],"genre_scores_gemma":[0.999601,0.00002178792,0.00007083907,0.0002471567,0.000007917351,0.00001809591,0.00000143766,0.000002375308,0.00002936979],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01608186,"threshold_uncertainty_score":0.989818,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2004544084","doi":"10.1016/j.jclinepi.2006.02.006","title":"Interrater reliability of measurements of comorbid illness should be reported","year":2006,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":59,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"Baycrest Hospital; Queen's University","funders":"Cancer Care Ontario","keywords":"Inter-rater reliability; Intraclass correlation; Medicine; Reliability (semiconductor); Rating scale; Comorbidity; Physical therapy; Psychometrics; Statistics; Psychiatry; Clinical psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.7802261328157349,"gpt":0.5869670242980766,"spread":0.1932591085176584,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2371966,0.0002150168,0.003775955,0.0002274138,0.00005269617,0.00001065476,0.001260061,0.0004145406,0.000707714],"category_scores_gemma":[0.3217455,0.0001248407,0.001502039,0.0004045442,0.001014284,0.0002259124,0.0001841061,0.0008228759,0.000009920836],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005471427,"about_ca_system_score_gemma":0.0002783862,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001835824,"about_ca_topic_score_gemma":0.0000607679,"domain_scores_codex":[0.9584736,0.0132464,0.02447588,0.0006357,0.002738953,0.0004295227],"domain_scores_gemma":[0.9320835,0.04372802,0.01774651,0.001412649,0.004764829,0.0002644878],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0006809109,0.0009739255,0.9587648,0.00002925989,0.0001028918,0.000007685612,0.00004322561,0.001834429,0.0008886383,0.0002822942,0.03088149,0.005510469],"study_design_scores_gemma":[0.001527958,0.001336743,0.8491168,0.0001416367,0.0001179344,0.00002933792,0.000141251,0.0003180862,0.002744949,0.1269759,0.01738408,0.0001653738],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9735734,0.0001961042,0.008541066,0.01220525,0.002896884,0.0002063208,0.00001019564,0.000005623547,0.002365165],"genre_scores_gemma":[0.9925303,0.00005262574,0.005791944,0.001088283,0.000336233,0.000001971002,0.0000020151,0.000007775091,0.0001888633],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1266936,"threshold_uncertainty_score":0.7854667,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2111817119","doi":"10.1016/j.jpainsymman.2009.06.013","title":"Evaluating Correlation and Interrater Reliability for Four Performance Scales in the Palliative Care Setting","year":2010,"lang":"en","type":"article","venue":"Journal of Pain and Symptom Management","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":56,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Health Sciences Centre; Sunnybrook Health Science Centre","funders":"","keywords":"Inter-rater reliability; Correlation; Medicine; Scale (ratio); Reliability (semiconductor); Kappa; Palliative care; Population; Statistics; Spearman's rank correlation coefficient; Intraclass correlation; Psychometrics; Clinical psychology; Rating scale; Nursing; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.1194752482627371,"gpt":0.3982255808639538,"spread":0.2787503326012167,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.03358426,0.00009892001,0.0001725387,0.0001316323,0.0001887233,0.0002190872,0.0002878144,0.00003965884,0.00001401585],"category_scores_gemma":[0.001965117,0.0000533709,0.00006521247,0.0001336053,0.00007116329,0.0002376993,0.00007225476,0.0002375125,0.000001077913],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003674344,"about_ca_system_score_gemma":0.00001328595,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002074124,"about_ca_topic_score_gemma":0.00004702144,"domain_scores_codex":[0.9974122,0.0005633524,0.000745528,0.0002094212,0.0009262624,0.0001432215],"domain_scores_gemma":[0.9974408,0.00155321,0.0004036577,0.0002014167,0.0003624297,0.00003852221],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001002803,0.00005532001,0.6799285,0.0001609719,0.00002314671,0.000002231876,0.01009642,0.001824028,0.000150117,0.0004527379,0.0008865212,0.3063197],"study_design_scores_gemma":[0.000842412,0.0005216997,0.8873509,0.0001697858,0.00003021938,0.000002735583,0.01885628,0.07913987,0.00001996081,0.01072968,0.002237629,0.00009883405],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9882253,0.00009389439,0.008151303,0.002227382,0.0003075124,0.0005040256,0.000001079568,0.00000227391,0.0004872141],"genre_scores_gemma":[0.9928301,0.0000492477,0.006674001,0.0003035268,0.00007651515,0.00001661538,5.649089e-7,0.000003048908,0.00004640264],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3062209,"threshold_uncertainty_score":0.9951284,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2099077995","doi":"10.1177/0962280215601133","title":"Assessing agreement between two measurement systems: An alternative to the limits of agreement approach","year":2015,"lang":"en","type":"article","venue":"Statistical Methods in Medical Research","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":55,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"Interchangeability; Agreement; Metric (unit); Limits of agreement; Computer science; Units of measurement; Physics; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.8455312682296875,"gpt":0.6788734831139276,"spread":0.1666577851157599,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3749643,0.0003073855,0.0009721339,0.0006872685,0.0002732591,0.000609609,0.003568112,0.0001736044,0.0006258779],"category_scores_gemma":[0.2304376,0.0001750129,0.00009620861,0.002449658,0.0009993035,0.0003155131,0.0009649156,0.001505947,0.0001720804],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001002482,"about_ca_system_score_gemma":0.001365161,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001354066,"about_ca_topic_score_gemma":0.0002028967,"domain_scores_codex":[0.9017674,0.04603641,0.003419851,0.001767464,0.0454216,0.00158723],"domain_scores_gemma":[0.9661933,0.02439417,0.0003365949,0.001891713,0.005061493,0.002122768],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0001650629,0.001129789,0.007303854,0.00009439813,0.00009324845,0.00003767438,0.003018375,0.002225057,0.0002898197,0.0350492,0.007562968,0.9430305],"study_design_scores_gemma":[0.00698043,0.006074854,0.08956563,0.001806341,0.0001365812,0.00001517509,0.0811056,0.2376251,0.003379,0.4907228,0.08113442,0.001454089],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0114914,0.0003286806,0.9649674,0.004965167,0.0006921373,0.001886587,0.00003972745,0.00001259065,0.01561636],"genre_scores_gemma":[0.6608213,0.00001554669,0.3380594,0.0001872084,0.0004664141,0.0003149985,0.000007411359,0.00001721429,0.0001105187],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9415765,"threshold_uncertainty_score":0.7760447,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2079234813","doi":"10.1111/j.0006-341x.2002.00209.x","title":"Interval Estimation for a Difference Between Intraclass Kappa Statistics","year":2002,"lang":"en","type":"article","venue":"Biometrics","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":48,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"Western University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Statistics; Confidence interval; Cohen's kappa; Mathematics; Interval estimation; Kappa; Statistic; Sample size determination; Intraclass correlation; Reproducibility","retraction":null,"screen_n_in":null,"score":{"opus":0.4256073761305363,"gpt":0.418085093842297,"spread":0.007522282288239368,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.002863789,0.0001300698,0.0002579732,0.000938941,0.0001393758,0.0003218254,0.0006484619,0.00008446605,0.000353466],"category_scores_gemma":[0.01615218,0.000095891,0.00008377169,0.003177648,0.00008518841,0.0001359684,0.00008892419,0.00009148603,0.000359454],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001003356,"about_ca_system_score_gemma":0.0000153386,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008432401,"about_ca_topic_score_gemma":0.000003485761,"domain_scores_codex":[0.9968911,0.0001013549,0.0007154003,0.000390026,0.001642851,0.0002592664],"domain_scores_gemma":[0.9951007,0.003571343,0.0002495965,0.0004297738,0.000527841,0.0001207044],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000006430146,0.0001370199,0.02443453,0.00002071175,0.0000147404,6.265117e-7,0.0001486627,0.00003575697,0.0001770106,0.001330446,0.03159675,0.9420973],"study_design_scores_gemma":[0.001672704,0.001356842,0.3019794,0.00005898427,0.0001106805,0.000002231893,0.0002880545,0.3562871,0.00173128,0.1739916,0.1617533,0.0007679071],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.09006494,0.0001208995,0.9074925,0.0007320164,0.0005018935,0.0003744795,0.0003080836,0.00003074051,0.0003744372],"genre_scores_gemma":[0.9084426,0.00002447549,0.08995767,0.0001014242,0.000128353,0.00002741038,0.00002230548,0.000007736553,0.001287997],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9413294,"threshold_uncertainty_score":0.9921352,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2500917129","doi":"10.1016/j.diii.2016.05.014","title":"Agreement studies in radiology research","year":2016,"lang":"en","type":"article","venue":"Diagnostic and Interventional Imaging","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":48,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Alberta Hospital; Université de Montréal; Hôpital Notre-Dame; Health Sciences Centre; Centre Hospitalier de l’Université de Montréal","funders":"","keywords":"Medicine; Confidence interval; Medical physics; Reliability (semiconductor); Sample size determination; Nuclear medicine; Statistics; Internal medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.512024444147994,"gpt":0.5438569514778593,"spread":0.03183250732986531,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01137222,0.00008949568,0.0001970586,0.0003396524,0.0001198375,0.0000683927,0.0002986299,0.00002023692,0.0008703622],"category_scores_gemma":[0.03050197,0.00004924597,0.00008179222,0.0002554522,0.0005667501,0.0002633537,0.0002844772,0.000107922,0.0003159951],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001185455,"about_ca_system_score_gemma":0.00002200383,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002622939,"about_ca_topic_score_gemma":0.00004732952,"domain_scores_codex":[0.9969463,0.0006546952,0.0006305856,0.0004392775,0.001036883,0.0002922488],"domain_scores_gemma":[0.9868995,0.01222662,0.00007974542,0.0002356279,0.0004949858,0.00006354898],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00002214012,0.0002192602,0.7568604,0.00001852466,0.00002932751,0.00003881653,0.0003844635,0.000005895562,0.0004743188,0.01737137,0.03482472,0.1897507],"study_design_scores_gemma":[0.00105732,0.0001834781,0.5171729,0.001054122,0.000006353092,0.0000226497,0.003078101,0.0001218991,0.0002873397,0.4556624,0.0211958,0.000157575],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9513464,0.01385302,0.004425569,0.02610916,0.0009188117,0.0003620393,0.00001190185,0.00001384226,0.002959252],"genre_scores_gemma":[0.997516,0.0006429974,0.0001585512,0.0001573804,0.00008025744,0.00005795064,7.861142e-7,0.000003096834,0.001382982],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.438291,"threshold_uncertainty_score":0.9776645,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2142892303","doi":"10.2466/pr0.2000.87.3f.1171","title":"Creating Comparability among Reliability Coefficients: The Case of Cronbach Alpha and Cohen Kappa","year":2000,"lang":"en","type":"article","venue":"Psychological Reports","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":44,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Winnipeg","funders":"","keywords":"Cronbach's alpha; Comparability; Kappa; Psychology; Reliability (semiconductor); Alpha (finance); Estimator; Statistics; Metric (unit); Cohen's kappa; Contrast (vision); Complement (music); Psychometrics; Social psychology; Clinical psychology; Mathematics; Computer science; Artificial intelligence; Combinatorics; Chemistry; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.123497322714613,"gpt":0.4136398494609489,"spread":0.2901425267463359,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01787973,0.0002156262,0.0005186254,0.00004251756,0.0004156884,0.0001606943,0.0004464971,0.0001222812,0.003060602],"category_scores_gemma":[0.008336258,0.0001087738,0.0001912712,0.0005517434,0.001423992,0.0001615709,0.0001278966,0.0003192729,0.00002983916],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004075881,"about_ca_system_score_gemma":0.00001708257,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002522452,"about_ca_topic_score_gemma":0.00004526365,"domain_scores_codex":[0.9943228,0.0009504688,0.001911232,0.001173495,0.00128992,0.0003520704],"domain_scores_gemma":[0.9945118,0.002436244,0.0005674455,0.001886081,0.0004053784,0.0001930394],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001529893,0.001531631,0.7840953,0.00001716386,0.00001970005,0.0007870522,0.0007493444,0.001205812,0.000156622,0.0001035955,0.002928434,0.2082523],"study_design_scores_gemma":[0.0002777298,0.0002716358,0.9581445,0.00002333445,0.0000252622,0.0009881625,0.0005281726,0.000840351,0.000102027,0.03252944,0.006086323,0.0001829979],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9727799,0.0001407886,0.0004625562,0.0006268174,0.0002174479,0.0007009123,0.000006876814,0.00003869915,0.02502604],"genre_scores_gemma":[0.998752,0.0000168657,0.0005081419,0.0002054093,0.00003613178,0.00003271312,0.000001634229,0.000005089743,0.0004420025],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2080693,"threshold_uncertainty_score":0.9979882,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2353892345","doi":"10.1111/hir.12140","title":"Inter‐rater reliability of h‐index scores calculated by Web of Science and Scopus for clinical epidemiology scientists","year":2016,"lang":"en","type":"article","venue":"Health Information & Libraries Journal","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":42,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":true},"ca_institutions":"Ottawa Hospital","funders":"Ottawa Hospital Research Institute","keywords":"Scopus; Rank correlation; Bibliometrics; Reliability (semiconductor); Web of science; Statistics; Spearman's rank correlation coefficient; Index (typography); Inter-rater reliability; Psychology; Medicine; Meta-analysis; MEDLINE; Mathematics; Library science; Computer science; World Wide Web; Physics; Political science; Internal medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.2502676357798633,"gpt":0.4679676814020811,"spread":0.2177000456222178,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.07015967,0.0001262578,0.0006458722,0.0004525889,0.0004965671,0.0001952282,0.0007578423,0.0001006568,0.0001465177],"category_scores_gemma":[0.05151013,0.000066812,0.0001259902,0.0007205642,0.00366928,0.004899823,0.0002041469,0.0001857215,0.000009988275],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001046687,"about_ca_system_score_gemma":0.002312075,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001396774,"about_ca_topic_score_gemma":0.000002876945,"domain_scores_codex":[0.9921054,0.0008884937,0.004726019,0.0002940516,0.001531693,0.0004543304],"domain_scores_gemma":[0.9893441,0.004225103,0.002969501,0.0004958393,0.0025534,0.0004120151],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0004954676,0.00009902647,0.7097659,0.0001612409,0.00001244648,8.478492e-8,0.0006958957,0.00003634431,0.0001896786,0.005553535,0.09810676,0.1848837],"study_design_scores_gemma":[0.004290813,0.002349595,0.5953082,0.0007516743,0.00001291828,0.00003986194,0.001057307,0.01071749,0.00221412,0.1060425,0.2768967,0.0003188376],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8506948,0.0005317553,0.118724,0.02680264,0.002076392,0.0007543032,0.0001531319,0.00001711257,0.0002457619],"genre_scores_gemma":[0.9925712,0.0003904513,0.005154111,0.00177128,0.00004840301,0.000006838216,0.000002800319,0.000003069498,0.00005183873],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1845648,"threshold_uncertainty_score":0.9990422,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1881569770","doi":"10.1002/9781118445112.stat05243","title":"Kappa and Its Dependence on Marginal Rates","year":2014,"lang":"en","type":"other","venue":"Wiley StatsRef: Statistics Reference Online","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":40,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"Kappa; Cohen's kappa; Statistic; Statistics; Measure (data warehouse); Mathematics; Econometrics; Computer science; Data mining; Geometry","retraction":null,"screen_n_in":null,"score":{"opus":0.1823082849611272,"gpt":0.4110886459547939,"spread":0.2287803609936667,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.002285355,0.0007128985,0.001000612,0.0005485315,0.0001852634,0.0003799318,0.001310515,0.0003935274,0.009830364],"category_scores_gemma":[0.003746122,0.0005163636,0.00007073601,0.0003936506,0.0002900217,0.00009445089,0.0002854445,0.0007835634,0.002275656],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008925405,"about_ca_system_score_gemma":0.0002403428,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001180352,"about_ca_topic_score_gemma":0.001236471,"domain_scores_codex":[0.9924707,0.0005656208,0.001164314,0.001546849,0.003596165,0.0006563935],"domain_scores_gemma":[0.9951077,0.001774304,0.0009361625,0.001192255,0.0005924616,0.0003970887],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001078388,0.0003925737,0.0006701011,0.0001675936,0.00007513898,0.00004489989,0.00005681715,0.00004343169,0.00007411763,0.03829796,0.8876674,0.07240217],"study_design_scores_gemma":[0.0007756625,0.0007877319,0.001588316,0.000936644,0.00007608961,0.000007344966,0.0001297936,0.002740203,0.00003855474,0.04744902,0.9445784,0.0008922537],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.004570795,0.01286955,0.2473367,0.00371092,0.006400554,0.005937757,0.1129844,0.0009963459,0.605193],"genre_scores_gemma":[0.01658486,0.01084318,0.07746396,0.001489056,0.001057718,0.00009329261,0.002206324,0.0005930181,0.8896686],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.2844756,"threshold_uncertainty_score":0.9997288,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2075431957","doi":"10.1007/s10877-008-9127-y","title":"A Measure of Confidence in Bland–Altman Analysis for the Interchangeability of Two Methods of Measurement","year":2008,"lang":"en","type":"article","venue":"Journal of Clinical Monitoring and Computing","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":40,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University Health Network","funders":"","keywords":"Interchangeability; Measure (data warehouse); Confidence interval; Limits of agreement; Medicine; Bland–Altman plot; Statistics; Medical physics; Computer science; Mathematics; Data mining; Internal medicine; Nuclear medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.6837745116735408,"gpt":0.587837256182952,"spread":0.09593725549058874,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.0847835,0.0001077749,0.001170716,0.000238141,0.00007848511,0.00001707705,0.0006272217,0.00006513083,0.00000448544],"category_scores_gemma":[0.02485948,0.00006105094,0.0006585852,0.0007118956,0.0003074612,0.00008113493,0.0001177018,0.0003228871,7.093408e-8],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003148504,"about_ca_system_score_gemma":0.0001398025,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000882181,"about_ca_topic_score_gemma":0.00001910132,"domain_scores_codex":[0.9918317,0.001603539,0.004150102,0.0002471784,0.00200798,0.0001594916],"domain_scores_gemma":[0.9794673,0.01453553,0.002804093,0.0003946279,0.00271166,0.00008684712],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002388316,0.0002317374,0.89879,0.00004055446,0.0003107105,7.921353e-7,0.001091199,0.005470381,0.00113298,0.00003343655,0.000008122311,0.09265123],"study_design_scores_gemma":[0.001344785,0.0006852077,0.9732242,0.0005118746,0.0003112931,0.000004274532,0.001589928,0.00998935,0.008223477,0.003976218,0.00005914558,0.0000803079],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8203797,0.001002282,0.177654,0.0002213203,0.0005770855,0.0001412803,0.000001110114,0.000001270325,0.00002192169],"genre_scores_gemma":[0.9768368,0.0001288017,0.02279784,0.000007144,0.0002229632,0.000001042103,2.47503e-8,0.00000276551,0.00000262445],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1564571,"threshold_uncertainty_score":0.9833546,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2140519023","doi":"10.1002/9781118445112.stat05255","title":"Receiver Operating Characteristic (<scp>ROC</scp>) Curves","year":2014,"lang":"en","type":"other","venue":"Wiley StatsRef: Statistics Reference Online","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":39,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University","funders":"","keywords":"Receiver operating characteristic; Fraction (chemistry); Boundary (topology); Mathematics; Statistics; Arbitrariness; Mathematical analysis; Chromatography","retraction":null,"screen_n_in":null,"score":{"opus":0.14698234004203,"gpt":0.3834437404836891,"spread":0.236461400441659,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.003664086,0.001026172,0.001711817,0.0006109214,0.0003001078,0.0005895417,0.002332314,0.0005343438,0.01133992],"category_scores_gemma":[0.01877448,0.0007828132,0.0001695085,0.0007227904,0.0004615955,0.0001809119,0.0004506607,0.001158488,0.003986207],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001728875,"about_ca_system_score_gemma":0.000462956,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002378425,"about_ca_topic_score_gemma":0.0008958002,"domain_scores_codex":[0.9890569,0.001057661,0.002304821,0.001942531,0.004601142,0.00103695],"domain_scores_gemma":[0.9906703,0.003419924,0.001959656,0.002192457,0.001234954,0.0005227502],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000007775878,0.0003170399,0.0006814643,0.0004224709,0.00009622963,0.00002820218,0.0001065961,0.00001924169,0.00009424642,0.00196285,0.9655343,0.03072955],"study_design_scores_gemma":[0.0004989415,0.0003346853,0.002524454,0.002894649,0.0001339104,0.000005831051,0.0002218357,0.001260663,0.00001193023,0.007441921,0.984187,0.0004841623],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.001331515,0.01177419,0.4011115,0.001347668,0.007572323,0.004924501,0.1418205,0.001187389,0.4289304],"genre_scores_gemma":[0.0008927052,0.01837516,0.09989526,0.002584248,0.001475605,0.0001269672,0.01002014,0.0007116849,0.8659182],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.4369878,"threshold_uncertainty_score":0.9994623,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null}]}