{"id":"W4410830098","doi":"10.1039/d5dd00090d","title":"Evaluating the performance and robustness of LLMs in materials science Q&A and property predictions","year":2025,"lang":"en","type":"article","venue":"Digital Discovery","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Robustness (evolution); Property (philosophy); Economics; Biology; Genetics; Epistemology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01098333,0.001075965,0.0005190008,0.001924581,0.0006866925,0.001620297,0.001174918,0.001944751,0.002131571],"category_scores_gemma":[0.04431769,0.0002591518,0.0006005058,0.0008419003,0.001068957,0.001790561,0.001914648,0.001482175,0.001097249],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001279349,"about_ca_system_score_gemma":0.001293574,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006685538,"about_ca_topic_score_gemma":0.003579961,"domain_scores_codex":[0.995902,0.001455569,0.0002932748,0.0007933217,0.001174249,0.0003815625],"domain_scores_gemma":[0.9666756,0.02171242,0.00259087,0.004768528,0.003341375,0.000911162],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003899903,0.0009132395,0.1066033,0.0004120889,0.0005041983,0.0002140159,0.0002255074,0.7081661,0.01848101,0.006553142,0.006515867,0.1475116],"study_design_scores_gemma":[0.00005414958,0.0003510303,0.007777417,0.00003509334,0.00002713441,0.00004161858,0.00006250131,0.9661027,0.02152433,0.002771307,0.001220238,0.0000324609],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.885124,0.001489685,0.08972839,0.002095815,0.0002804162,0.0002407825,0.002057355,0.009066798,0.009916699],"genre_scores_gemma":[0.9755355,0.00009597435,0.02200535,0.0002082242,0.00003826603,0.00005985577,0.001080349,0.0001003389,0.0008762912],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01098333,"threshold_uncertainty_score":0.0580861,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02812550467023981,"score_gpt":0.3167299339162702,"score_spread":0.2886044292460304,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}