{"id":"W4285090825","doi":"10.1101/2022.07.11.22277498","title":"Assessment of concordance between related systematic reviews and between related guideline recommendations: protocol for a methodological survey","year":2022,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; St. Michael's Hospital; Toronto Rehabilitation Institute; University Health Network; University of Toronto; McMaster University; Impact","funders":"","keywords":"Concordance; Guideline; Protocol (science); Systematic review; Clinical Practice; Medicine; MEDLINE; Psychology; Management science; Family medicine; Alternative medicine; Pathology; Political science; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"evaluation","study_design":"observational","genre":"protocol","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch","metaepi_broad"],"domain":"methods","study_design":"systematic_review","genre":"protocol","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","metaepi_broad","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.660358,0.0008566904,0.0188283,0.0006986972,0.0002753828,0.0004407091,0.003637003,0.00051125,0.00741879],"category_scores_gemma":[0.3335123,0.0004349516,0.003035546,0.001988798,0.0001493676,0.000141976,0.001786624,0.001156778,0.00007973811],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001553435,"about_ca_system_score_gemma":0.0003118722,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005096101,"about_ca_topic_score_gemma":0.00001699406,"domain_scores_codex":[0.6925623,0.2369532,0.06086918,0.003366568,0.005699142,0.0005496744],"domain_scores_gemma":[0.864613,0.08453851,0.0413548,0.006538465,0.002684472,0.00027078],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00001434601,0.0001127434,0.9079399,0.03639773,0.002100011,0.000002284894,0.0002436376,0.0001343435,0.00001487147,0.0008643735,0.04689319,0.005282592],"study_design_scores_gemma":[0.002175899,0.0006344153,0.5992535,0.01407953,0.004956615,0.00001386637,0.0003579083,0.01348007,0.00001332834,0.04874614,0.3146966,0.00159218],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"protocol","genre_gemma":"protocol","genre_scores_codex":[0.01346352,0.001910778,0.2398243,0.002164835,0.0007092966,0.7389176,0.001384686,0.00004528887,0.001579676],"genre_scores_gemma":[0.01691113,0.000185667,0.2684286,0.0001293896,0.0001226705,0.7073992,0.001204408,0.00008170299,0.005537204],"genre_candidate":"protocol","genre_consensus":"protocol","teacher_disagreement_score":0.3268458,"threshold_uncertainty_score":0.9998102,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.928813207935435,"score_gpt":0.6732457410896231,"score_spread":0.2555674668458119,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}