{"id":"W4362700002","doi":"10.1038/s41524-023-01012-9","title":"A critical examination of robustness and generalizability of machine learning prediction of materials properties","year":2023,"lang":"en","type":"article","venue":"npj Computational Materials","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":97,"is_retracted":false,"has_abstract":true,"ca_institutions":"Natural Resources Canada; University of Toronto; University of New Brunswick","funders":"Office of Energy Research and Development; Natural Resources Canada","keywords":"Generalizability theory; Robustness (evolution); Computer science; Machine learning; Artificial intelligence; Benchmark (surveying); Artificial neural network; Feature engineering; Data mining; Feature vector; Test data; Deep learning; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0036873,0.0001756383,0.0005532556,0.0002186555,0.0001110439,0.00007245548,0.0002237981,0.00009372801,0.0007431196],"category_scores_gemma":[0.001617237,0.0001535969,0.00003074733,0.0002870194,0.0005858637,0.0003266852,0.0002169308,0.00004627044,0.00001100141],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002255273,"about_ca_system_score_gemma":0.00006510128,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001856694,"about_ca_topic_score_gemma":0.000001250891,"domain_scores_codex":[0.9967133,0.001021935,0.001059973,0.0003670262,0.0006132622,0.0002245266],"domain_scores_gemma":[0.9983001,0.0004021301,0.0005338933,0.0001813213,0.0005334024,0.00004916338],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.00008146885,0.00005143054,0.0007374625,0.0009916582,0.000005762042,7.848718e-7,0.000433692,0.1337088,0.8619773,0.001924273,0.000008647384,0.0000786836],"study_design_scores_gemma":[0.0003025169,0.0001896747,0.07226273,0.000146596,0.00002038197,0.00001023131,0.00005773078,0.03594043,0.8896718,0.001282892,0.000004597134,0.0001104027],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9954081,0.0000370613,0.002823131,0.00009063621,0.0006693289,0.0002804945,0.000554583,0.0001087298,0.00002794475],"genre_scores_gemma":[0.9913435,0.00001143692,0.008360286,0.000007190144,0.00006682398,0.00003058167,0.0001323285,0.00001995831,0.00002788292],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.09776834,"threshold_uncertainty_score":0.8136639,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03174563863098726,"score_gpt":0.2746000237751183,"score_spread":0.2428543851441311,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}