{"id":"W4379115941","doi":"10.23919/date56975.2023.10137086","title":"Benchmarking Large Language Models for Automated Verilog RTL Code Generation","year":2023,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":142,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"Army Research Office; National Science Foundation","keywords":"Computer science; Verilog; Programming language; Compiler; Correctness; Construct (python library); Scripting language; Embedded system; Field-programmable gate array","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004561578,0.00007555897,0.00007874929,0.0001511978,0.0000943391,0.0001272431,0.0003906119,0.00004643714,0.0000155434],"category_scores_gemma":[0.0001612441,0.00007219242,0.00003421051,0.0005272821,0.000005495281,0.0003137777,0.0001835597,0.00005527071,0.00007145921],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004191095,"about_ca_system_score_gemma":0.00003816379,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001331399,"about_ca_topic_score_gemma":0.00001718343,"domain_scores_codex":[0.9990453,0.00001717352,0.0001073749,0.0002593394,0.0002209021,0.0003499587],"domain_scores_gemma":[0.9992737,0.0002674088,0.00001465932,0.0003305822,0.00005782831,0.00005581035],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000008672411,0.00009163808,0.0008778537,0.0001470927,0.00007772968,0.00008488772,0.004962403,0.5315608,0.04558487,0.1493678,0.2362184,0.03101785],"study_design_scores_gemma":[0.0001901076,0.00002508303,0.000683044,0.000004684805,8.896644e-7,0.000001586263,0.000009845852,0.9952943,0.002724799,0.0002228591,0.0007517421,0.00009106944],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05158882,0.00002705856,0.9432599,0.0001917494,0.0003422408,0.0001915688,0.00001158535,0.004194652,0.0001924073],"genre_scores_gemma":[0.8888408,0.000004799848,0.1098777,0.00009529419,0.0001689264,0.00008488132,0.00006952556,0.00001711352,0.0008409075],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.837252,"threshold_uncertainty_score":0.2943921,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04206808642623612,"score_gpt":0.3182129701439342,"score_spread":0.2761448837176981,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}