{"id":"W4414196974","doi":"10.1109/lcn65610.2025.11146057","title":"SIMCODE: A Benchmark for Natural Language to ns-3 Network Simulation Code Generation","year":2025,"lang":"en","type":"article","venue":"","topic":"Simulation Techniques and Applications","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University","funders":"","keywords":"Scripting language; Benchmark (surveying); Code (set theory); Focus (optics); Natural language; Natural language generation; Automation; Code generation","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003416328,0.001575274,0.0004995569,0.001071768,0.0006395741,0.001022804,0.003388701,0.00134333,0.006478118],"category_scores_gemma":[0.01557847,0.0006144116,0.00101651,0.001368552,0.001084179,0.001387828,0.001331773,0.001710763,0.002771235],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001393231,"about_ca_system_score_gemma":0.003446813,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01446713,"about_ca_topic_score_gemma":0.01275368,"domain_scores_codex":[0.9976398,0.0008972473,0.0002652898,0.00029413,0.0006954774,0.0002080033],"domain_scores_gemma":[0.9920229,0.004934499,0.0003265573,0.001032696,0.001410748,0.0002727338],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001394834,0.000942843,0.01295611,0.002831417,0.000370913,0.0006553697,0.0009253421,0.6390618,0.01156407,0.03032322,0.1644247,0.1345493],"study_design_scores_gemma":[0.00032667,0.0002569935,0.001332545,0.0001348645,0.00003265496,0.0001145843,0.0001562443,0.9347247,0.01428449,0.009720399,0.03886726,0.00004854344],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.2648419,0.002700954,0.358099,0.002570054,0.0009706707,0.001793627,0.03142078,0.28828,0.04932308],"genre_scores_gemma":[0.4828788,0.001087788,0.4236304,0.0009299497,0.00006225456,0.00175147,0.05671668,0.02762625,0.0053164],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.01446713,"threshold_uncertainty_score":0.0287658,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1025791354737176,"score_gpt":0.4863095816006462,"score_spread":0.3837304461269286,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}