{"id":"W4214697769","doi":"10.1101/2022.02.24.22268947","title":"A real-world evaluation of the implementation of NLP technology in abstract screening of a systematic review","year":2022,"lang":"en","type":"review","venue":"medRxiv","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary; University of Toronto; McGill University","funders":"University of Calgary","keywords":"Recall; Systematic review; Context (archaeology); Inclusion (mineral); Machine learning; Artificial intelligence; Precision and recall; Computer science; Natural language processing; MEDLINE; Psychology; Cognitive psychology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.48294,0.002001786,0.003375061,0.007966403,0.002126251,0.006988643,0.00344428,0.003836569,0.003467195],"category_scores_gemma":[0.7354315,0.002521798,0.006765726,0.00862382,0.002526556,0.006043579,0.005147051,0.002478133,0.000783913],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006567298,"about_ca_system_score_gemma":0.01119253,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002364366,"about_ca_topic_score_gemma":0.004961654,"domain_scores_codex":[0.3714978,0.5213327,0.07214608,0.009122319,0.02439126,0.001509828],"domain_scores_gemma":[0.1021281,0.7885453,0.04252717,0.03204103,0.03328703,0.001471509],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.05067544,0.004782888,0.04508778,0.2192134,0.02442856,0.001621696,0.02574118,0.03073373,0.01735811,0.005868444,0.01947703,0.5550118],"study_design_scores_gemma":[0.1070644,0.132014,0.185547,0.1019821,0.07706457,0.005761769,0.01266879,0.1796322,0.04265107,0.01784247,0.1330767,0.004695003],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6304975,0.02747633,0.2081537,0.01163625,0.001702311,0.092319,0.01076957,0.007007557,0.01043783],"genre_scores_gemma":[0.4952568,0.00345516,0.4423678,0.001769789,0.0002417447,0.05372892,0.002358259,0.0004129012,0.000408609],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.51706,"threshold_uncertainty_score":0.637627,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.8193420800030563,"score_gpt":0.6182600224401161,"score_spread":0.2010820575629402,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}