{"id":"W4404952914","doi":"10.1109/issre62328.2024.00030","title":"Assessing the Performance of AI-Generated Code: A Case Study on GitHub Copilot","year":2024,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Natural Science Foundation of China","keywords":"Computer science; Code (set theory); Computer security; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004273068,0.0009542251,0.0004406628,0.002079192,0.0008778747,0.001164295,0.002692188,0.001202516,0.00102534],"category_scores_gemma":[0.02732149,0.0004179853,0.0006385326,0.002709116,0.002229292,0.0013403,0.001524225,0.001822276,0.0008817649],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002178922,"about_ca_system_score_gemma":0.001830699,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01248895,"about_ca_topic_score_gemma":0.01941939,"domain_scores_codex":[0.995285,0.001560108,0.0002612486,0.0007508689,0.001885876,0.000256962],"domain_scores_gemma":[0.966576,0.02321876,0.001700729,0.003951603,0.003855513,0.0006973378],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.005450218,0.005278026,0.1289821,0.005672443,0.00060858,0.01404837,0.02148925,0.1939746,0.05073551,0.0111432,0.1146923,0.4479254],"study_design_scores_gemma":[0.001263436,0.004381435,0.2297662,0.0008747423,0.0002946501,0.00402494,0.006433326,0.5483323,0.08320577,0.01089097,0.1100601,0.0004720335],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9734344,0.0004712509,0.00980584,0.0007333914,0.00006420976,0.0004330931,0.003479966,0.007425858,0.004152032],"genre_scores_gemma":[0.9131528,0.0004049114,0.06030755,0.0005953745,0.00003727112,0.0007368449,0.01774064,0.003677036,0.003347513],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01248895,"threshold_uncertainty_score":0.02483255,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06393546958244994,"score_gpt":0.3681672486282733,"score_spread":0.3042317790458234,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}