{"id":"W1517951603","doi":"10.1186/1471-2288-6-14","title":"Reviewer agreement trends from four years of electronic submissions of conference abstract","year":2006,"lang":"en","type":"article","venue":"BMC Medical Research Methodology","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"Hamilton Health Sciences; McMaster University; St. Paul's Hospital; Providence Health Care; Winnipeg Regional Health Authority; University of Alberta","funders":"Canadian Institutes of Health Research; China Academy of Engineering Physics; University of Alberta","keywords":"Blinding; Intraclass correlation; Medicine; Trial registration; Quality Score; Family medicine; Clinical trial; Clinical psychology; Psychometrics; Internal medicine","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"evaluation","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"evaluation","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4882578,0.0001890749,0.004088545,0.0008249643,0.0000605167,0.00005563838,0.002903882,0.0002706096,0.1562312],"category_scores_gemma":[0.6725697,0.00009921267,0.001298453,0.002014551,0.0006061092,0.00006358069,0.0003653776,0.0006871716,0.0007765949],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000467199,"about_ca_system_score_gemma":0.001346406,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001195082,"about_ca_topic_score_gemma":0.001874719,"domain_scores_codex":[0.7725514,0.1752846,0.01694099,0.001885377,0.03195224,0.001385404],"domain_scores_gemma":[0.6775684,0.3001767,0.005439303,0.006167108,0.009793005,0.000855532],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005786436,0.0002701044,0.01364918,0.0001934587,0.0002194865,0.00002226514,0.0001808417,0.00001487814,0.003015781,0.02557228,0.1751655,0.7816384],"study_design_scores_gemma":[0.0008708362,0.0004035351,0.4097653,0.0004655136,0.0002131716,0.00001039804,0.0006065716,0.003155919,0.001092931,0.1645575,0.4185813,0.0002769917],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5629222,0.01560916,0.3824409,0.00283419,0.0007571874,0.001060252,0.00007654056,0.000008313179,0.03429129],"genre_scores_gemma":[0.9119006,0.0009240026,0.07002809,0.00007579639,0.0003235994,0.00006155316,0.00002492496,0.0000143108,0.01664715],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7813614,"threshold_uncertainty_score":0.9981815,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9586731752727339,"score_gpt":0.6946784667617693,"score_spread":0.2639947085109646,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}