{"id":"W4405750350","doi":"10.1016/j.jval.2024.10.2421","title":"MSR187 Automated Data Extraction in Systematic Literature Reviews (SLRs): Assessing the Accuracy and Reliability of a Large Language Model (LLM)","year":2024,"lang":"en","type":"article","venue":"Value in Health","topic":"Artificial Intelligence in Healthcare","field":"Health Professions","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"Thermo Fisher Scientific (Canada)","funders":"","keywords":"Computer science; Data extraction; Reliability (semiconductor); Natural language processing; Artificial intelligence; Data mining; MEDLINE; Chemistry","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01904538,0.0002129889,0.0007820838,0.0002699189,0.0003554623,0.00006012166,0.0004305824,0.0002868671,0.00002120937],"category_scores_gemma":[0.007836827,0.0001414068,0.00004749072,0.001159495,0.00005395871,0.000788587,0.0002535942,0.001732249,0.00003240766],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005485258,"about_ca_system_score_gemma":0.001231965,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01060975,"about_ca_topic_score_gemma":0.002242185,"domain_scores_codex":[0.9893317,0.006396514,0.002688466,0.000623035,0.0003179669,0.0006423612],"domain_scores_gemma":[0.9921486,0.005548198,0.0006600237,0.001403282,0.0001162985,0.0001236233],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005006768,0.0003217995,0.02764113,0.7902298,0.00001253525,0.00005152589,0.1638884,0.005882444,0.0002090878,0.004442436,0.003479119,0.00379168],"study_design_scores_gemma":[0.00007738034,0.00002491837,0.001783242,0.0991049,0.0000102383,0.000006075278,0.008141849,0.8895165,0.000005884207,0.0009246594,0.000305303,0.00009911262],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8330544,0.1456723,0.001416486,0.00889916,0.001267294,0.008793271,0.0003323989,0.0003953544,0.0001693808],"genre_scores_gemma":[0.9891622,0.007699359,0.001809175,0.0007309973,0.0001248909,0.0002726809,0.00008723886,0.00003673749,0.00007670117],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.883634,"threshold_uncertainty_score":0.9959787,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2714836057692865,"score_gpt":0.5738235770727436,"score_spread":0.302339971303457,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}