{"id":"W3214639705","doi":"10.1111/hir.12413","title":"Using an artificial intelligence tool can be as accurate as human assessors in level one screening for a systematic review","year":2021,"lang":"en","type":"review","venue":"Health Information & Libraries Journal","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; Global Affairs Canada; Ottawa Hospital","funders":"","keywords":"Systematic review; Computer science; Artificial intelligence; Identification (biology); Set (abstract data type); Machine learning; MEDLINE; Meta-analysis; Medicine; Pathology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"methods","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","metaepi_broad","sts","scholarly_communication","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1428647,0.000905188,0.01783365,0.001940752,0.0014882,0.01254159,0.0028888,0.0003530109,0.002962174],"category_scores_gemma":[0.06486132,0.0005295946,0.00336731,0.003697598,0.00009398299,0.007468342,0.0002614322,0.001233396,0.0002497933],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000398681,"about_ca_system_score_gemma":0.006728174,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006287682,"about_ca_topic_score_gemma":0.00006398218,"domain_scores_codex":[0.9270195,0.02002432,0.04557979,0.0007257338,0.005821603,0.0008290619],"domain_scores_gemma":[0.9399847,0.004770609,0.04898413,0.003006447,0.002529665,0.0007243983],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.000003703424,0.00005312825,0.000001213121,0.6194819,0.0003226909,0.000007435846,0.001580811,0.0001311768,1.464358e-8,0.03676222,0.002210136,0.3394456],"study_design_scores_gemma":[0.000090287,0.0001584412,0.000001319027,0.6239491,0.001523504,0.0009071935,0.003899202,0.003818893,3.548965e-7,0.006015812,0.3589642,0.000671731],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.000006013359,0.7829767,0.2088833,0.001089689,0.0003315602,0.006380142,0.0001588163,0.00001109185,0.0001625745],"genre_scores_gemma":[0.00001676565,0.9549794,0.03795555,0.005460322,0.0002962117,0.0004498022,0.0005419767,0.00005011667,0.0002498132],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.3567541,"threshold_uncertainty_score":0.9998117,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.945914799977756,"score_gpt":0.6376514516269239,"score_spread":0.308263348350832,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}