{"id":"W2112548834","doi":"10.1016/j.jclinepi.2011.01.010","title":"Testing a tool for the classification of study designs in systematic reviews of interventions and exposures showed moderate reliability and low accuracy","year":2011,"lang":"en","type":"article","venue":"Journal of Clinical Epidemiology","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":73,"is_retracted":false,"has_abstract":false,"ca_institutions":"McMaster University; University of Alberta","funders":"Agency for Healthcare Research and Quality","keywords":"Inter-rater reliability; Reliability (semiconductor); Context (archaeology); Test (biology); Systematic review; Research design; CLARITY; Statistics; Computer science; Medicine; Reliability engineering; Medical physics; MEDLINE; Mathematics; Engineering; Rating scale","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"methods","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.6654222,0.007993676,0.01337927,0.02707238,0.007646862,0.01431489,0.007789045,0.01597847,0.005060365],"category_scores_gemma":[0.9100288,0.004757635,0.03608596,0.01884107,0.01332095,0.01396319,0.01291148,0.01135813,0.001889302],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006070981,"about_ca_system_score_gemma":0.006808592,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002961547,"about_ca_topic_score_gemma":0.005611694,"domain_scores_codex":[0.1531859,0.5246934,0.202779,0.04459685,0.0714457,0.003299108],"domain_scores_gemma":[0.01802103,0.9048796,0.0253947,0.0333475,0.0177527,0.0006045208],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.01314019,0.001306571,0.4430478,0.05147596,0.1519488,0.001358899,0.009773741,0.01479029,0.003630313,0.02413956,0.02531048,0.2600775],"study_design_scores_gemma":[0.01555173,0.010203,0.2513797,0.02803474,0.1256511,0.004548378,0.003407939,0.3173673,0.02234298,0.1549756,0.0636298,0.002907796],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1356708,0.02140008,0.7799601,0.00500191,0.004624434,0.02996,0.007515602,0.006139153,0.009727832],"genre_scores_gemma":[0.5040521,0.00119898,0.4649611,0.002329652,0.0005350864,0.02234663,0.002833266,0.0008607259,0.0008825149],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.3345778,"threshold_uncertainty_score":0.412594,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9765515537496716,"score_gpt":0.6909558342193558,"score_spread":0.2855957195303158,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}