{"id":"W4414499338","doi":"10.1056/aidbp2500120","title":"Assessment of Large Language Models in Clinical Reasoning: A Novel Benchmarking Study","year":2025,"lang":"en","type":"article","venue":"NEJM AI","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":28,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Benchmarking; Language model; Quality (philosophy); Identification (biology); Measure (data warehouse)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.001824198,0.0001187788,0.0005705086,0.000134377,0.00002523914,0.000009989755,0.0001069131,0.0001307134,0.00006355077],"category_scores_gemma":[0.01151043,0.0001015288,0.0001445427,0.0003522196,0.00005165481,0.0000374901,0.000116729,0.0004715862,0.000002757449],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005317431,"about_ca_system_score_gemma":0.000342717,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002803307,"about_ca_topic_score_gemma":0.0001370681,"domain_scores_codex":[0.9982597,0.00009676005,0.0007728121,0.0003445124,0.0002825901,0.0002436148],"domain_scores_gemma":[0.9948824,0.004381389,0.0001139133,0.0004301701,0.00008817848,0.0001039705],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00008877677,0.0057655,0.9790066,0.00003033466,0.00008874205,0.00008244335,0.0005864494,0.0001060854,0.00005785091,0.005909915,0.0008411314,0.007436201],"study_design_scores_gemma":[0.006883582,0.0006166971,0.9599111,0.00204986,0.0001365213,0.00000252638,0.001283881,0.02868949,0.00001736664,0.0001774471,0.0001439315,0.0000875621],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.975984,0.000154302,0.00419541,0.0005577447,0.0002851665,0.0004757138,0.000009428488,0.00003408246,0.0183041],"genre_scores_gemma":[0.9949498,0.00002560698,0.002928823,0.001478453,0.0001061561,0.00002542743,0.00001732058,0.00001082869,0.0004575415],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02858341,"threshold_uncertainty_score":0.996816,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03709534502479742,"score_gpt":0.4589540775897755,"score_spread":0.4218587325649781,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}