{"id":"W4391986589","doi":"10.1145/3643991.3645074","title":"Can ChatGPT Support Developers? An Empirical Evaluation of Large Language Models for Code Generation","year":2024,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University","funders":"","keywords":"Computer science; Programming language; Code generation; Code (set theory); Code review; Software engineering; Static program analysis; Software; Software development; Operating system","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002215589,0.0001946654,0.000257607,0.0001672148,0.00004852636,0.0001553936,0.0006823918,0.0002331258,0.00005322041],"category_scores_gemma":[0.00004742272,0.0001776338,0.00009877482,0.0001090241,0.00001015019,0.0001854687,0.0008896223,0.0002187969,0.00000515329],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001921955,"about_ca_system_score_gemma":0.00135338,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001170346,"about_ca_topic_score_gemma":0.001215187,"domain_scores_codex":[0.9977139,0.0001206671,0.0004379413,0.0007917656,0.0006926059,0.0002431742],"domain_scores_gemma":[0.9984157,0.00002037394,0.0001167342,0.0008679633,0.0004989458,0.00008029882],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000009216854,0.0002405009,0.00005870633,0.0005859429,0.0001718183,0.000008389588,0.06168821,0.6155557,0.003419619,0.1777237,0.005644848,0.1348933],"study_design_scores_gemma":[0.0002127841,0.00002915399,0.00000895688,0.00003034724,0.00004879327,0.000002416677,0.00007913447,0.9614232,0.003109215,0.03478133,0.00007656504,0.0001980836],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1010241,0.0001443027,0.8941077,0.001683471,0.0007614216,0.0007513896,0.00008905843,0.0001525613,0.001285939],"genre_scores_gemma":[0.7950365,0.000004986888,0.2033764,0.0004082016,0.0002426973,0.0001843708,0.000299333,0.00001851936,0.0004290171],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.6940123,"threshold_uncertainty_score":0.7243695,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1782814047691914,"score_gpt":0.4102592759568877,"score_spread":0.2319778711876963,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}