{"id":"W4312438588","doi":"10.1145/3524842.3528470","title":"An empirical evaluation of GitHub copilot's code suggestions","year":2022,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":281,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Correctness; Programmer; JavaScript; Programming language; Code (set theory); Source code; Java; Software engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001368524,0.00004407225,0.0000624772,0.0001265906,0.00009562796,0.00002947079,0.0006430002,0.00001293496,0.0004093496],"category_scores_gemma":[0.0003774992,0.00004434972,0.00002058691,0.0005572715,0.00001900438,0.0001715608,0.0002375834,0.0001181026,0.00001370715],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001110573,"about_ca_system_score_gemma":0.0002820739,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002928842,"about_ca_topic_score_gemma":0.000004466845,"domain_scores_codex":[0.998365,0.0001970749,0.0001108457,0.0001901653,0.001005088,0.0001317675],"domain_scores_gemma":[0.9989434,0.0002947631,0.00001907461,0.000489862,0.0001888047,0.00006402117],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001745244,0.001256602,0.1768616,0.00003016896,0.00005676014,0.00001364317,0.003287126,0.6344412,0.006143182,0.0360005,0.04883482,0.09305701],"study_design_scores_gemma":[0.0002531164,0.0002309631,0.1278446,0.000001440867,0.000004405301,0.000009505768,0.00003351298,0.8681604,0.0009433668,0.0009797139,0.001463136,0.00007588071],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.466973,0.00004330958,0.5307108,0.000966954,0.0002149161,0.0001927928,0.000007009488,0.0002927281,0.000598519],"genre_scores_gemma":[0.9802628,4.620944e-7,0.01947636,0.00005286479,0.00001418224,0.00006432943,0.000005642994,0.000004879568,0.0001184774],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5132898,"threshold_uncertainty_score":0.4482091,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09023633677771457,"score_gpt":0.395196726944413,"score_spread":0.3049603901666985,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}