{"id":"W6979354307","doi":"","title":"Automated Capability Evaluation of Foundation Models","year":2025,"lang":"en","type":"article","venue":"ArXiv.org","topic":"Machine Learning in Materials Science","field":"Materials Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute","funders":"","keywords":"Foundation (evidence); Domain (mathematical analysis); Software deployment; Aggregate (composite); Limiting; Domain knowledge","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005241718,0.001823087,0.0009349225,0.002798911,0.0005610698,0.002544271,0.001598804,0.001254561,0.00417088],"category_scores_gemma":[0.03106527,0.0003626494,0.001027603,0.001439637,0.0008766551,0.005038836,0.003559773,0.001551824,0.001436111],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001499197,"about_ca_system_score_gemma":0.00237594,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005121496,"about_ca_topic_score_gemma":0.008135033,"domain_scores_codex":[0.9961125,0.001562149,0.0002297569,0.000655011,0.001095907,0.0003446353],"domain_scores_gemma":[0.9871458,0.00684964,0.0008743753,0.00222447,0.002348843,0.0005569482],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0009081105,0.0006107223,0.04246041,0.001386094,0.0003719692,0.0002806334,0.0005709518,0.452242,0.008427597,0.03122444,0.03093001,0.4305869],"study_design_scores_gemma":[0.00004999049,0.0002028287,0.003179972,0.0001580686,0.00003630443,0.00007328328,0.0002721719,0.9574804,0.006812778,0.02508861,0.006608957,0.00003662737],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.4236523,0.002520789,0.5156917,0.001156072,0.0002695093,0.0004888884,0.009217164,0.01766544,0.0293381],"genre_scores_gemma":[0.8735779,0.0003787699,0.1114312,0.0001722117,0.00004249758,0.0002671845,0.01177609,0.000628223,0.001725929],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.005241718,"threshold_uncertainty_score":0.02772123,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05992947960825695,"score_gpt":0.3532751462383975,"score_spread":0.2933456666301405,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}