{"id":"W4393529891","doi":"10.1001/jamanetworkopen.2024.4630","title":"Quality of Large Language Model Responses to Radiation Oncology Patient Care Questions","year":2024,"lang":"en","type":"article","venue":"JAMA Network Open","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":105,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"National Cancer Institute; Canadian Institutes of Health Research","keywords":"Readability; Likert scale; Correctness; Radiation oncology; Harm; Medicine; Computer science; Psychology; Medical education; Radiation therapy; Internal medicine; Social psychology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03346045,0.000330381,0.0004804009,0.001371398,0.0003366853,0.00139893,0.0005669827,0.0006687562,0.002136334],"category_scores_gemma":[0.234955,0.0001989787,0.0006460256,0.001172038,0.0007374949,0.001306077,0.001630844,0.000559344,0.0005376532],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009862554,"about_ca_system_score_gemma":0.0006319237,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001100846,"about_ca_topic_score_gemma":0.001244753,"domain_scores_codex":[0.9603009,0.03059476,0.003045496,0.001988129,0.003678309,0.0003923577],"domain_scores_gemma":[0.700902,0.2468971,0.0290835,0.008777482,0.01325794,0.001081961],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0030668,0.0004832827,0.7913016,0.00136467,0.0004618432,0.0004004297,0.03691998,0.005588641,0.005383526,0.0005168547,0.00458182,0.1499306],"study_design_scores_gemma":[0.0003286549,0.002842702,0.9004965,0.0007131874,0.0005167216,0.001376493,0.02233816,0.04394438,0.01363809,0.002124938,0.01143698,0.0002431222],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9895738,0.0001653263,0.006424612,0.0005227132,0.00002769916,0.0002553767,0.0007548293,0.0002778725,0.001997742],"genre_scores_gemma":[0.9944215,0.00006473601,0.004067636,0.000170168,0.00001398826,0.0002324016,0.0006430148,0.00004611378,0.0003404864],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03346045,"threshold_uncertainty_score":0.1769579,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1616338542101096,"score_gpt":0.5333206311682803,"score_spread":0.3716867769581708,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}