{"id":"W4410424798","doi":"10.2196/66917","title":"Benchmarking the Confidence of Large Language Models in Answering Clinical Questions: Cross-Sectional Evaluation Study","year":2025,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":33,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Confidence interval; Low Confidence; Medicine; Statistics; Benchmarking; Mean difference; Psychology; Internal medicine; Mathematics; Social psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.04300991,0.0005400546,0.0007459981,0.0015721,0.0004697046,0.001902855,0.001058286,0.001249253,0.001280046],"category_scores_gemma":[0.1080011,0.0004788483,0.001383702,0.0008866874,0.001103237,0.002565176,0.001663023,0.00131529,0.0007419265],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00100429,"about_ca_system_score_gemma":0.0007034795,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001708314,"about_ca_topic_score_gemma":0.001391298,"domain_scores_codex":[0.9809301,0.01136003,0.001887591,0.001761378,0.003344451,0.0007164676],"domain_scores_gemma":[0.8609707,0.08359089,0.0270382,0.007906476,0.01592938,0.004564326],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0003828338,0.0008393101,0.9891232,0.00004417742,0.0001674861,0.00005450248,0.002409612,0.000577725,0.0001535131,0.00004443207,0.000325786,0.005877391],"study_design_scores_gemma":[0.00007439819,0.004412362,0.9781961,0.00006492184,0.0001752896,0.0003933717,0.002937161,0.01140119,0.0007269498,0.0001948438,0.001362481,0.00006091781],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.998672,0.00007424854,0.0005572843,0.00006326556,0.000005454667,0.00007559442,0.0001263663,0.00001835502,0.0004074035],"genre_scores_gemma":[0.998984,0.00004152922,0.0004775262,0.0000422171,0.000006837129,0.00008103894,0.000253321,0.000009088962,0.0001044964],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.04300991,"threshold_uncertainty_score":0.2274609,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1903462553477343,"score_gpt":0.5662406282886161,"score_spread":0.3758943729408818,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}