{"id":"W4200324797","doi":"10.29007/xwl1","title":"ARCH-COMP 2021 Category Report: Falsification with Validation of Results","year":2021,"lang":"en","type":"article","venue":"EPiC series in computing","topic":"Physical Unclonable Functions (PUFs) and Hardware Security","field":"Computer Science","cited_by":23,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"JST-Mirai Program; Core Research for Evolutional Science and Technology; Exploratory Research for Advanced Technology; National Science Foundation; Division of Civil, Mechanical and Manufacturing Innovation; Division of Industrial Innovation and Partnerships; Defense Advanced Research Projects Agency; ACT-X; Japan Society for the Promotion of Science; European Commission","keywords":"Novelty; Computer science; Competition (biology); Arch; Work (physics); Artificial intelligence; Software engineering; Engineering; Psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"bench_or_experimental","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"simulation_or_modeling","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005400447,0.0001233334,0.00023445,0.00009749112,0.0001290242,0.00007378824,0.0003211531,0.00004733749,0.000009220465],"category_scores_gemma":[0.0003221038,0.0001200627,0.00004725238,0.0009982918,0.00007868694,0.0004571013,0.0002426595,0.0001912874,0.000006438318],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005825745,"about_ca_system_score_gemma":0.0002186185,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001447297,"about_ca_topic_score_gemma":0.00005736901,"domain_scores_codex":[0.998296,0.0001579156,0.0005179859,0.0005058295,0.000296595,0.0002257038],"domain_scores_gemma":[0.9983699,0.0002408764,0.0002826281,0.0007473236,0.0003108075,0.00004846187],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0004312572,0.002637374,0.03405776,0.0005402302,0.0002189993,0.002375664,0.02409297,0.06069861,0.01603873,0.6253725,0.002011708,0.2315241],"study_design_scores_gemma":[0.005975002,0.001109022,0.1982771,0.001212821,0.00007044544,0.001905097,0.003190427,0.185597,0.4528891,0.1134536,0.03414891,0.002171479],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5138463,0.0001213876,0.4712156,0.001635282,0.0007594478,0.0001757739,0.000005217888,0.0001185472,0.01212251],"genre_scores_gemma":[0.9726507,0.000008973596,0.02688376,0.0000380403,0.0001413366,0.000004314078,0.00006308453,0.000006832911,0.0002029545],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.511919,"threshold_uncertainty_score":0.4896014,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0166020675808463,"score_gpt":0.2502045175981701,"score_spread":0.2336024500173238,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}