{"id":"W4414189579","doi":"10.1136/bmjdhai-2025-000014","title":"Optimising large language models for clinical information extraction: a benchmarking study in the context of ulcerative colitis research","year":2025,"lang":"en","type":"article","venue":"BMJ Digital Health & AI","topic":"Biomedical Text Mining and Ontologies","field":"Biochemistry, Genetics and Molecular Biology","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Artificial Intelligence in Medicine (Canada)","funders":"Clinical and Translational Science Institute, University of California, Los Angeles; Clinical and Translational Science Institute, University of California, San Francisco","keywords":"Benchmarking; Context (archaeology); Adaptation (eye); Oracle; Language model; Set (abstract data type); Predictive modelling; Colonoscopy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01933746,0.001641262,0.0008698837,0.00171419,0.0005075546,0.002025023,0.002256734,0.001649255,0.002071202],"category_scores_gemma":[0.06170137,0.0007287963,0.001675796,0.002000408,0.0008421748,0.003231189,0.001940519,0.002243378,0.001107178],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002321859,"about_ca_system_score_gemma":0.002025967,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01207981,"about_ca_topic_score_gemma":0.01277549,"domain_scores_codex":[0.9874959,0.009346471,0.0007720692,0.001550165,0.0006260784,0.0002092643],"domain_scores_gemma":[0.9216779,0.06996682,0.001141272,0.003795288,0.002983576,0.0004351848],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002823057,0.001226999,0.02448993,0.002419522,0.001124247,0.0007399291,0.001728066,0.5579401,0.005996359,0.002992375,0.01683334,0.381686],"study_design_scores_gemma":[0.0003690925,0.000635031,0.005395252,0.0001491724,0.0003079748,0.0002538403,0.0004409589,0.976561,0.00591034,0.004334175,0.005552892,0.00009026466],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6816127,0.005819432,0.2755574,0.003322324,0.000408588,0.001193441,0.007375512,0.01981895,0.0048918],"genre_scores_gemma":[0.771923,0.0009569341,0.2078477,0.0005901522,0.0001064467,0.000671251,0.01540913,0.0009125505,0.001582864],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9806625,"threshold_uncertainty_score":0.1022675,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1149706351165215,"score_gpt":0.5230203048782074,"score_spread":0.4080496697616859,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}