{"id":"W4410985212","doi":"10.1200/cci-24-00331","title":"Clinical Trial Design Approach to Auditing Language Models in Health Care Setting","year":2025,"lang":"en","type":"article","venue":"JCO Clinical Cancer Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Provincial Health Services Authority; University of British Columbia","funders":"","keywords":"Audit; Health care; Medicine; Computer science; Psychology; Business; Accounting; Economics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005112699,0.0001622549,0.0006837858,0.0001972582,0.0001250631,0.00004018061,0.0001848944,0.0002926885,0.00001674412],"category_scores_gemma":[0.003154321,0.0001447109,0.0001799365,0.0005373138,0.0000699692,0.0001765844,0.00007886322,0.0008983132,0.00003106745],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004295871,"about_ca_system_score_gemma":0.002854872,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000893046,"about_ca_topic_score_gemma":0.00016428,"domain_scores_codex":[0.9947212,0.0003228353,0.003943808,0.0002453224,0.0002991667,0.0004677098],"domain_scores_gemma":[0.9970114,0.001508141,0.0004978061,0.000408593,0.0002430152,0.00033109],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.006594182,0.0004339718,0.01456195,0.001371086,0.00003785465,0.000001760569,0.03062377,0.008940587,2.342572e-7,0.0005673543,0.0138828,0.9229845],"study_design_scores_gemma":[0.02214693,0.007605011,0.009776098,0.006581224,0.0002817764,0.000009539273,0.3141612,0.5988801,0.0002025069,0.002706945,0.03637353,0.001275249],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6540832,0.002104805,0.2893183,0.01372804,0.006541711,0.007963871,0.00003456599,0.0002630227,0.02596255],"genre_scores_gemma":[0.8722273,0.0006155566,0.08982379,0.03465296,0.001874735,0.0003473477,0.00006258261,0.0000240865,0.0003716625],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9217092,"threshold_uncertainty_score":0.5901137,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4717397137646442,"score_gpt":0.6009811590416988,"score_spread":0.1292414452770546,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}