{"id":"W6979354307","doi":"","title":"Automated Capability Evaluation of Foundation Models","year":2025,"lang":"en","type":"article","venue":"ArXiv.org","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute","funders":"","keywords":"Foundation (evidence); Domain (mathematical analysis); Software deployment; Aggregate (composite); Limiting; Domain knowledge","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.004340612,0.0001166825,0.0001988735,0.00009776395,0.0001376031,0.00005128273,0.0003590567,0.00006643596,0.001150284],"category_scores_gemma":[0.001107592,0.0001065026,0.00003745327,0.0003613354,0.0001851638,0.0004355702,0.0001165752,0.00006561926,0.000231902],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001346625,"about_ca_system_score_gemma":0.0002549638,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000381302,"about_ca_topic_score_gemma":0.00002986169,"domain_scores_codex":[0.997893,0.0005261573,0.0004268071,0.0003840592,0.0005597904,0.0002101384],"domain_scores_gemma":[0.9986526,0.0001123512,0.0002125335,0.0005043772,0.0004839183,0.00003423523],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001746621,0.00005389591,0.06749627,0.00005429781,0.000003670133,1.611526e-7,0.000203747,0.06177614,0.8684454,0.001285583,0.0001142583,0.0005491032],"study_design_scores_gemma":[0.0002907193,0.00002360674,0.2807786,0.00003890452,0.00003497333,5.754468e-7,0.00003036396,0.5341447,0.1793278,0.005189898,0.00004578206,0.00009399372],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9897071,0.00002525828,0.004496778,0.0002537419,0.0008400367,0.0003314227,0.000004419846,0.0003337999,0.004007443],"genre_scores_gemma":[0.99756,0.00000169946,0.002147419,0.00007693775,0.00002549377,0.00003704077,0.00001012197,0.000006789399,0.0001344696],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6891176,"threshold_uncertainty_score":0.9997628,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05992947960825695,"score_gpt":0.3532751462383975,"score_spread":0.2933456666301405,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}