{"id":"W4403024154","doi":"10.1109/lad62341.2024.10691745","title":"Toward Hardware Security Benchmarking of LLMs","year":2024,"lang":"en","type":"article","venue":"","topic":"Security and Verification in Computing","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"Natural Sciences and Engineering Research Council of Canada; Intel Corporation","keywords":"Benchmarking; Computer science; Computer security; Business","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002850952,0.00006858993,0.00008976746,0.0000843958,0.00004723823,0.0001627335,0.0005296892,0.00003975235,0.0001132958],"category_scores_gemma":[0.00001950619,0.0000622074,0.00006665839,0.0004220227,0.0000246977,0.000300497,0.000192992,0.0001069683,0.00004406168],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001909744,"about_ca_system_score_gemma":0.000058598,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003046129,"about_ca_topic_score_gemma":0.000001662044,"domain_scores_codex":[0.9991918,0.00002759333,0.000192192,0.0002700675,0.0001865755,0.0001317933],"domain_scores_gemma":[0.9994525,0.0001252658,0.00002966728,0.0002980522,0.00005465829,0.00003989798],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[6.594656e-7,0.00002020526,0.0001456117,0.0001326661,0.00001475653,0.000009296517,0.004884975,0.00004148587,0.0001695458,0.9480518,0.003618789,0.04291023],"study_design_scores_gemma":[0.00009740623,0.00004470673,0.0005157006,0.0001423655,0.000004894634,0.00002795417,0.0001215009,0.9014097,0.01069844,0.03602431,0.05072162,0.0001914316],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01305529,0.0006754609,0.959574,0.001124229,0.001372566,0.00006714136,0.000001761164,0.0004475026,0.02368207],"genre_scores_gemma":[0.9745001,0.00001574086,0.02515284,0.0001208968,0.0001235138,0.000002382466,0.000001431944,0.000003518932,0.00007962745],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9614447,"threshold_uncertainty_score":0.2536744,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03368345729125524,"score_gpt":0.2739716618867946,"score_spread":0.2402882045955393,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}