{"id":"W1517951603","doi":"10.1186/1471-2288-6-14","title":"Reviewer agreement trends from four years of electronic submissions of conference abstract","year":2006,"lang":"en","type":"article","venue":"BMC Medical Research Methodology","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"Hamilton Health Sciences; McMaster University; St. Paul's Hospital; Providence Health Care; Winnipeg Regional Health Authority; University of Alberta","funders":"Canadian Institutes of Health Research; China Academy of Engineering Physics; University of Alberta","keywords":"Blinding; Intraclass correlation; Medicine; Trial registration; Quality Score; Family medicine; Clinical trial; Clinical psychology; Psychometrics; Internal medicine","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"evaluation","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"evaluation","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1902501,0.0008158945,0.001848861,0.01122117,0.001186774,0.004001346,0.002363811,0.001504851,0.001159306],"category_scores_gemma":[0.5292588,0.0008424597,0.002450023,0.007187987,0.0009531407,0.002835373,0.003805523,0.001397218,0.0009478502],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002019066,"about_ca_system_score_gemma":0.002944292,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002608188,"about_ca_topic_score_gemma":0.002914468,"domain_scores_codex":[0.7223503,0.1033062,0.08316848,0.02053783,0.06826125,0.002375845],"domain_scores_gemma":[0.2078727,0.3336851,0.2013649,0.02593072,0.2275855,0.003561128],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.003926058,0.0001227535,0.892804,0.004038631,0.00287831,0.0002730852,0.009462995,0.001125334,0.002669731,0.0002156163,0.005804425,0.0766791],"study_design_scores_gemma":[0.0001762345,0.001726597,0.9761449,0.001315816,0.000879246,0.001249387,0.003382384,0.002671799,0.003287834,0.000383568,0.008572135,0.000210114],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9221473,0.03486571,0.0185565,0.00346266,0.00140771,0.00119379,0.008057578,0.001179075,0.009129662],"genre_scores_gemma":[0.9810196,0.002767318,0.008404847,0.0004470019,0.0004483521,0.001050843,0.004284224,0.0002201563,0.001357588],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8097499,"threshold_uncertainty_score":0.9985658,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9586731752727339,"score_gpt":0.6946784667617693,"score_spread":0.2639947085109646,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}