{"id":"W4378190436","doi":"10.1002/jrsm.1636","title":"A real‐world evaluation of the implementation of <scp>NLP</scp> technology in abstract screening of a systematic review","year":2023,"lang":"en","type":"review","venue":"Research Synthesis Methods","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":27,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; McGill University; The Quebec Population Health Research Network; University of Calgary; McGill University Health Centre","funders":"Canadian Medical Association; Robert Koch Institut; Koch Institute for Integrative Cancer Research, Massachusetts Institute of Technology; Public Health Agency; Rhodes Scholarships; Public Health Agency of Canada; World Health Organization","keywords":"Computer science; Context (archaeology); Systematic review; Recall; Inclusion (mineral); Machine learning; Artificial intelligence; Precision and recall; Natural language processing; MEDLINE; Psychology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_broad","bibliometrics","open_science","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.7924761,0.0005418818,0.02192546,0.006694605,0.000106096,0.0001180679,0.005887507,0.0003379669,0.001115339],"category_scores_gemma":[0.6473593,0.0002396812,0.00497539,0.02428611,0.0003244466,0.0001430654,0.0008002021,0.0007978036,0.0001324078],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002766563,"about_ca_system_score_gemma":0.001399005,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002656039,"about_ca_topic_score_gemma":0.0002496475,"domain_scores_codex":[0.3393281,0.5496169,0.06618163,0.002935802,0.04058953,0.001347954],"domain_scores_gemma":[0.6196575,0.3096524,0.0459981,0.01370651,0.01081035,0.0001750841],"domain_codex":"methods","domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[2.718284e-7,0.00003513919,0.00004542404,0.5491176,0.0006149151,5.89168e-7,0.00007635008,0.000003158423,0.00001430671,0.0004677289,0.0006085563,0.4490159],"study_design_scores_gemma":[0.0001006456,0.00004125576,0.0002989462,0.9622547,0.01152956,0.000006992964,0.00288978,0.00050366,0.0003500725,0.005102103,0.0167788,0.000143458],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.00005191979,0.9846416,0.0007387397,0.00009802077,0.00007072409,0.01285862,0.00008134662,0.000005913158,0.001453116],"genre_scores_gemma":[0.0001219904,0.9768335,0.01903173,0.000002681187,0.00001939297,0.003248387,0.000006365093,0.00005102469,0.0006849103],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.4488725,"threshold_uncertainty_score":0.9997978,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9514931780577904,"score_gpt":0.7593710306945818,"score_spread":0.1921221473632087,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}