{"id":"W4410985212","doi":"10.1200/cci-24-00331","title":"Clinical Trial Design Approach to Auditing Language Models in Health Care Setting","year":2025,"lang":"en","type":"article","venue":"JCO Clinical Cancer Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Provincial Health Services Authority; University of British Columbia","funders":"","keywords":"Audit; Health care; Medicine; Computer science; Psychology; Business; Accounting; Economics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4546672,0.001984579,0.002322856,0.003569866,0.002176418,0.005513234,0.004520275,0.004279033,0.01025618],"category_scores_gemma":[0.5573549,0.001543343,0.002673473,0.002518285,0.005469596,0.004864547,0.005586406,0.006030152,0.001865067],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004436578,"about_ca_system_score_gemma":0.01934237,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001291221,"about_ca_topic_score_gemma":0.001479331,"domain_scores_codex":[0.3413847,0.6145635,0.01492063,0.01067716,0.01684275,0.001611283],"domain_scores_gemma":[0.347463,0.4863277,0.04299773,0.07294577,0.04444237,0.005823351],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.05769399,0.006451144,0.03077401,0.01188576,0.006482319,0.0009831553,0.008559367,0.04889134,0.005202271,0.2280181,0.02203274,0.5730259],"study_design_scores_gemma":[0.0546477,0.05074408,0.01247529,0.004307461,0.004674315,0.0008036895,0.002206768,0.3483234,0.02106019,0.4251745,0.07488866,0.0006939865],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.01996612,0.0007501853,0.8566231,0.005251086,0.001198681,0.1071642,0.0008617137,0.001214108,0.006970778],"genre_scores_gemma":[0.1716122,0.0001964281,0.6937068,0.002349803,0.0002982701,0.130474,0.0002232021,0.0001000696,0.001039215],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.5453328,"threshold_uncertainty_score":0.6724924,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4717397137646442,"score_gpt":0.6009811590416988,"score_spread":0.1292414452770546,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}