{"id":"W4408672934","doi":"10.1101/2025.03.17.25324157","title":"Real-World Evaluation of Large Language Models in Healthcare (RWE-LLM): A New Realm of AI Safety & Validation","year":2025,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"Realm; Health care; Computer science; Political science; Law","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003685297,0.000234265,0.000739919,0.0007704474,0.00004120504,0.000007984418,0.0001908827,0.000391777,0.0001909733],"category_scores_gemma":[0.0008178956,0.000239604,0.0001597745,0.0007240938,0.00004056972,0.00008979695,0.0001307667,0.0007780077,0.000009756269],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006496693,"about_ca_system_score_gemma":0.004474927,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.09006192,"about_ca_topic_score_gemma":0.03560385,"domain_scores_codex":[0.9962081,0.0005578664,0.001539544,0.0005110342,0.0008689645,0.0003145463],"domain_scores_gemma":[0.996904,0.0001883995,0.0005903554,0.0009489462,0.001213106,0.0001552229],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001744054,0.001843801,0.7007406,0.01381809,0.000242776,0.000008045928,0.06084749,0.02782107,0.002312165,0.02504021,0.003818875,0.1617628],"study_design_scores_gemma":[0.002282561,0.0007785925,0.3460141,0.02606995,0.002105736,0.00001047286,0.007380731,0.1838086,0.1318324,0.296061,0.002066131,0.001589711],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9784648,0.001168299,0.002718966,0.01061224,0.000935044,0.002136873,0.0001299924,0.00004867657,0.003785115],"genre_scores_gemma":[0.9945552,0.0009247489,0.001106931,0.0003190756,0.0002827146,0.00009570171,0.001166448,0.00002329198,0.001525834],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3547266,"threshold_uncertainty_score":0.9819939,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2160216630493233,"score_gpt":0.4999122890355706,"score_spread":0.2838906259862473,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}