{"id":"W4409451928","doi":"10.1016/j.jbi.2025.104825","title":"Benchmarking domain-specific pretrained language models to identify the best model for methodological rigor in clinical studies","year":2025,"lang":"en","type":"review","venue":"Journal of Biomedical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":6,"is_retracted":false,"has_abstract":false,"ca_institutions":"McMaster University; Hamilton Health Sciences","funders":"EBSCO; Alliance de recherche numérique du Canada; Mitacs","keywords":"Benchmarking; Computer science; Natural language processing; Domain (mathematical analysis); Artificial intelligence; Language model; Data science; Machine learning; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01101454,0.0002982919,0.002486733,0.0006934035,0.0001133526,0.00004973337,0.0004775679,0.000628327,0.00001197912],"category_scores_gemma":[0.00336078,0.000157249,0.0008777824,0.0007611554,0.0002734876,0.0001609146,0.0001366659,0.001408255,0.000009734516],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003628212,"about_ca_system_score_gemma":0.001492219,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000038357,"about_ca_topic_score_gemma":0.000006798373,"domain_scores_codex":[0.992855,0.000356498,0.005566578,0.0001565723,0.0006669373,0.0003983909],"domain_scores_gemma":[0.9904773,0.006784609,0.001479508,0.0003559804,0.0005534373,0.0003491398],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00009082086,0.0001918598,0.000001336076,0.01277304,0.0002211357,0.00001468408,0.01110216,0.00005865551,2.591233e-7,0.0004409945,0.007871273,0.9672338],"study_design_scores_gemma":[0.0003095043,0.001333887,0.000002161731,0.05431475,0.001225756,0.0002470467,0.03923481,0.01883619,0.000001368211,0.007238696,0.8769211,0.0003347235],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0003893457,0.7355736,0.2589486,0.001583747,0.001830421,0.001555624,0.00002862771,0.000008773339,0.00008126551],"genre_scores_gemma":[0.00004111983,0.8432924,0.1542636,0.000909917,0.001300692,0.00007390512,0.00003103905,0.00001387813,0.00007344638],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.966899,"threshold_uncertainty_score":0.6412427,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.741286052634215,"score_gpt":0.6411235352893042,"score_spread":0.1001625173449108,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}