{"id":"W4410521503","doi":"10.1038/s41557-025-01815-x","title":"A framework for evaluating the chemical knowledge and reasoning abilities of large language models against the expertise of chemists","year":2025,"lang":"en","type":"article","venue":"Nature Chemistry","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":55,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Consejo Superior de Investigaciones Científicas; HORIZON EUROPE Framework Programme; Slezská Univerzita v Opavě; Helmholtz Association; Office of Multicultural Interests Department of Local Government and Communities; Ministerio de Economía y Competitividad; Southeastern Ontario Academic Medical Organization; Schweizerischer Nationalfonds zur Förderung der Wissenschaftlichen Forschung; European Commission; Ministerio de Ciencia, Innovación y Universidades; Deutsche Forschungsgemeinschaft; UK Research and Innovation; Agencia Estatal de Investigación; U.S. Department of State; US-UK Fulbright Commission; Friedrich-Schiller-Universität Jena; Carl-Zeiss-Stiftung; Government of the United Kingdom; Fulbright Association; National Science Foundation","keywords":"Benchmarking; Harm; Process (computing); Value (mathematics); Chemistry; Management science; Computer science; Cognitive science; Psychology; Engineering; Social psychology; Machine learning; Programming language; Management","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001679927,0.0001474485,0.0002473746,0.0000112768,0.0001532588,0.0000459145,0.0006278685,0.0002617033,0.00005043324],"category_scores_gemma":[0.004655097,0.0000889065,0.00006804886,0.0001563316,0.0003734982,0.00005760198,0.0002809739,0.000383196,3.503571e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000364472,"about_ca_system_score_gemma":0.0001161798,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00000656634,"about_ca_topic_score_gemma":4.508438e-7,"domain_scores_codex":[0.9988182,0.0000646683,0.000312519,0.0003167436,0.0002411114,0.000246765],"domain_scores_gemma":[0.9980587,0.001015924,0.0002140123,0.0005100702,0.0001715252,0.00002973525],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0000498011,0.00002821007,0.00005528593,0.0006633469,0.000006258946,1.463559e-7,0.003463572,0.0001203414,0.9921905,0.002986714,0.0001054982,0.0003303732],"study_design_scores_gemma":[0.0002112751,0.000008186949,0.00004437487,0.0004047674,0.00001790174,0.000001980308,0.0009887418,0.02382019,0.9708449,0.003523032,0.00004824732,0.00008644408],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9911196,0.004512458,0.001661876,0.0003148286,0.0001385571,0.0002112742,0.00004190342,0.00003239565,0.0019671],"genre_scores_gemma":[0.9891422,0.00001118214,0.01029507,0.0001182354,0.00009128636,0.00006604907,0.00000574192,0.00001057099,0.0002596594],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02369985,"threshold_uncertainty_score":0.5572922,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01131941292029752,"score_gpt":0.3572969330387604,"score_spread":0.3459775201184628,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}