{"id":"W4405750350","doi":"10.1016/j.jval.2024.10.2421","title":"MSR187 Automated Data Extraction in Systematic Literature Reviews (SLRs): Assessing the Accuracy and Reliability of a Large Language Model (LLM)","year":2024,"lang":"en","type":"article","venue":"Value in Health","topic":"Artificial Intelligence in Healthcare","field":"Health Professions","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"Thermo Fisher Scientific (Canada)","funders":"","keywords":"Computer science; Data extraction; Reliability (semiconductor); Natural language processing; Artificial intelligence; Data mining; MEDLINE; Chemistry","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2975529,0.002834897,0.01015493,0.04166402,0.003032309,0.008327347,0.004912342,0.003620414,0.008248477],"category_scores_gemma":[0.7054225,0.003580704,0.02697626,0.02134186,0.003063675,0.008192613,0.01002653,0.002658119,0.002014945],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004792726,"about_ca_system_score_gemma":0.02737533,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005099252,"about_ca_topic_score_gemma":0.01941638,"domain_scores_codex":[0.5209242,0.2668307,0.1697882,0.01693431,0.02431773,0.001204819],"domain_scores_gemma":[0.1504925,0.74233,0.05455166,0.0321754,0.01911193,0.001338541],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00733305,0.0003631473,0.04547852,0.5419404,0.114469,0.0006817752,0.005784465,0.006705114,0.003641965,0.006154889,0.02143384,0.2460138],"study_design_scores_gemma":[0.01738989,0.006573766,0.08017766,0.2228311,0.4239161,0.002491613,0.004362352,0.06618527,0.009789945,0.03236314,0.1313957,0.002523492],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.150919,0.2047813,0.3305582,0.01382895,0.002365953,0.1052112,0.1671779,0.01607943,0.009078021],"genre_scores_gemma":[0.2836557,0.0144207,0.5854664,0.002470394,0.0004770339,0.08288481,0.02859024,0.001034193,0.00100058],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7024471,"threshold_uncertainty_score":0.8662423,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2714836057692865,"score_gpt":0.5738235770727436,"score_spread":0.302339971303457,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}