{"id":"W4411523084","doi":"10.1145/3728947","title":"The First Prompt Counts the Most! An Evaluation of Large Language Models on Iterative Example-Based Code Generation","year":2025,"lang":"en","type":"article","venue":"Proceedings of the ACM on software engineering.","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"Simon Fraser University","funders":"","keywords":"Code (set theory); Computer science; Benchmark (surveying); Iterative and incremental development; Code generation; Process (computing); Natural language generation; Natural language; Programming language; Software engineering; Artificial intelligence; Computer security; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0106398,0.001395787,0.0008974175,0.001340981,0.0005595933,0.002291641,0.002423454,0.001558181,0.003706945],"category_scores_gemma":[0.07462703,0.0004931991,0.001055517,0.001093288,0.00128266,0.004128648,0.002586553,0.002170048,0.001450717],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001466617,"about_ca_system_score_gemma":0.00235127,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004488434,"about_ca_topic_score_gemma":0.005330537,"domain_scores_codex":[0.9839138,0.008508175,0.0007676229,0.001453467,0.004884723,0.0004721425],"domain_scores_gemma":[0.9319447,0.0454809,0.002142277,0.01044764,0.008788329,0.001196193],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003286815,0.002274633,0.02494879,0.003034739,0.0004879333,0.0005993812,0.002728818,0.2093764,0.03705904,0.01642852,0.03163591,0.6681389],"study_design_scores_gemma":[0.0004802959,0.002503114,0.008158939,0.0004516304,0.0002104257,0.0004314078,0.001215738,0.8979132,0.04633655,0.010257,0.0318669,0.0001747384],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6797331,0.003220061,0.266373,0.002674025,0.0004121914,0.0008985984,0.002134926,0.02459657,0.0199575],"genre_scores_gemma":[0.6968561,0.0006539879,0.2905954,0.0005145909,0.00004288229,0.0004022537,0.005660266,0.002079155,0.003195341],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0106398,"threshold_uncertainty_score":0.05626935,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03728653452721785,"score_gpt":0.2970827131643091,"score_spread":0.2597961786370912,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}