{"id":"W4403289443","doi":"10.29007/hgfv","title":"ARCH-COMP 2024 Category Report: Falsification","year":2024,"lang":"en","type":"article","venue":"EPiC series in computing","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"ACT-X; JST-Mirai Program; Exploratory Research for Advanced Technology; Electronic Components and Systems for European Leadership; Defense Advanced Research Projects Agency; Universität Salzburg; Alliance de recherche numérique du Canada; Japan Society for the Promotion of Science; Austrian Science Fund; Division of Civil, Mechanical and Manufacturing Innovation; European Commission; Technische Universität Wien; Core Research for Evolutional Science and Technology; National Science Foundation","keywords":"Benchmark (surveying); Arch; Competition (biology); Computer science; Software engineering; Data science; Artificial intelligence; Engineering; Civil engineering; Geography","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001243984,0.0001852254,0.0002135791,0.0002733766,0.0001751,0.0003421745,0.0008060817,0.00008011295,0.0000276394],"category_scores_gemma":[0.0003785394,0.0001907191,0.00006658433,0.0009703235,0.00008608417,0.000694479,0.0006159556,0.0005988058,0.00006982544],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001401611,"about_ca_system_score_gemma":0.0001311769,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001433079,"about_ca_topic_score_gemma":0.00001282863,"domain_scores_codex":[0.9979203,0.0001722143,0.0005233354,0.0007043467,0.0003004151,0.0003794256],"domain_scores_gemma":[0.9986911,0.000441475,0.0001127332,0.0006538398,0.00004372276,0.00005717254],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000009788376,0.00006791383,0.01520373,0.0002725917,0.00004724041,0.003303781,0.00975739,0.1226662,0.0002748089,0.5092142,0.00203677,0.3371456],"study_design_scores_gemma":[0.0001435369,0.00004205305,0.005645029,0.0002095026,0.000005229732,0.0006687283,0.0002394602,0.9449988,0.0001699478,0.02661945,0.02093267,0.000325578],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02331081,0.0006876829,0.9537005,0.002182298,0.005280898,0.0001508379,3.344815e-7,0.000743971,0.01394268],"genre_scores_gemma":[0.9216639,0.000008701184,0.07662903,0.00009018485,0.0004748884,0.000006391669,0.000006393703,0.00002149398,0.001099003],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8983531,"threshold_uncertainty_score":0.7777301,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01676143807099343,"score_gpt":0.295545062689395,"score_spread":0.2787836246184015,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}