{"id":"W4379469178","doi":"10.1186/s13643-023-02247-9","title":"Ensemble of deep learning language models to support the creation of living systematic reviews for the COVID-19 literature","year":2023,"lang":"en","type":"article","venue":"Systematic Reviews","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Canadian Institutes of Health Research; Horizon 2020 Framework Programme; European Commission; Innosuisse - Schweizerische Agentur für Innovationsförderung; Université de Genève; Schweizerischer Nationalfonds zur Förderung der Wissenschaftlichen Forschung; National Science Foundation","keywords":"Artificial intelligence; Machine learning; Ensemble forecasting; Computer science; Ranking (information retrieval); Task (project management); Natural language processing; Ensemble learning; Class (philosophy); Triage; Medicine; Data science","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4339638,0.0006105696,0.01275398,0.0005836178,0.0004063605,0.0006903612,0.003452495,0.000137345,0.000737305],"category_scores_gemma":[0.5151258,0.0001889856,0.004649191,0.004892418,0.00006860255,0.0003019758,0.0002656557,0.0002277632,0.002099053],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007935805,"about_ca_system_score_gemma":0.00008933954,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002010879,"about_ca_topic_score_gemma":0.00006496756,"domain_scores_codex":[0.8888609,0.06963702,0.03406344,0.001252724,0.005589193,0.0005967422],"domain_scores_gemma":[0.86234,0.1013755,0.02615889,0.00813794,0.001689321,0.0002983141],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.000004606057,0.00002902144,0.00008907125,0.8930158,0.0003771104,0.000002120463,0.04897286,0.003161342,0.0001792093,0.003707934,0.04826627,0.002194653],"study_design_scores_gemma":[0.0004378172,0.0005451345,0.00007316386,0.4131305,0.009261531,0.0002419002,0.06447985,0.3191918,0.00004565572,0.004626843,0.1866492,0.001316712],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0009421693,0.186735,0.698339,0.001447525,0.0007221605,0.1084617,0.00004677521,0.00004484402,0.003260843],"genre_scores_gemma":[0.8115697,0.0160406,0.01235493,0.003671224,0.0005747083,0.05852374,0.00008158499,0.0002191508,0.09696437],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8106275,"threshold_uncertainty_score":0.9986779,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.6019366327301586,"score_gpt":0.5165044718164059,"score_spread":0.08543216091375272,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}