{"id":"W4307415438","doi":"10.48550/arxiv.2210.13597","title":"A critical examination of robustness and generalizability of machine learning prediction of materials properties","year":2022,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"Natural Resources Canada; University of Toronto","funders":"","keywords":"Generalizability theory; Robustness (evolution); Computer science; Machine learning; Artificial intelligence; Benchmark (surveying); Artificial neural network; Feature engineering; Data mining; Feature vector; Test data; Deep learning; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.002394443,0.0002112124,0.0005779934,0.0002067911,0.0001282577,0.0000331426,0.0005186517,0.0001539334,0.001083831],"category_scores_gemma":[0.0008434008,0.0002163905,0.00006495199,0.0002291086,0.0007956377,0.0002390689,0.001359589,0.0002352245,9.056186e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008134177,"about_ca_system_score_gemma":0.0001030812,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008538581,"about_ca_topic_score_gemma":0.000008220209,"domain_scores_codex":[0.9969515,0.001412766,0.000548857,0.0006649435,0.0002255696,0.0001963199],"domain_scores_gemma":[0.9982889,0.000162761,0.0006984281,0.0004676385,0.0003262013,0.0000560675],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0000964549,0.00008042028,0.005420908,0.001349886,0.000007876618,0.000003396861,0.000345799,0.4627094,0.5277373,0.002229594,0.000001123221,0.00001784917],"study_design_scores_gemma":[0.0004768736,0.000399461,0.02908528,0.0002934612,0.0001637806,0.000006537898,0.0003614872,0.4025562,0.5651538,0.001180265,0.00001215876,0.0003106233],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9910169,0.00007585534,0.007600533,0.00001504704,0.0005203977,0.0002917755,0.0003132434,0.00005595858,0.0001103048],"genre_scores_gemma":[0.9985629,0.00006618309,0.001161673,0.000002460663,0.00002462865,0.000003095181,0.0000354322,0.00001652842,0.0001270912],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.06015317,"threshold_uncertainty_score":0.9998293,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06758512803701792,"score_gpt":0.2070397022359092,"score_spread":0.1394545741988913,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}