{"id":"W4389988584","doi":"10.1109/scam59687.2023.00020","title":"Explaining Transformer-based Code Models: What Do They Learn? When They Do Not Work?","year":2023,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University; University of Calgary","funders":"","keywords":"Computer science; KPI-driven code analysis; Source code; Code (set theory); Code generation; Downstream (manufacturing); Code review; Transformer; Security token; Static program analysis; Artificial neural network; Software engineering; Set (abstract data type); Software; Artificial intelligence; Programming language; Machine learning; Software development; Key (lock); Computer security; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.008735626,0.001390414,0.0005720167,0.001198773,0.0006206965,0.002540661,0.002453432,0.00217064,0.003991065],"category_scores_gemma":[0.07734156,0.0008211014,0.0009475622,0.0008717244,0.00214445,0.01465341,0.001819655,0.004561517,0.001603607],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00174298,"about_ca_system_score_gemma":0.001715109,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00720846,"about_ca_topic_score_gemma":0.009859064,"domain_scores_codex":[0.9959546,0.002394281,0.0001425771,0.0006532292,0.0006460112,0.0002093295],"domain_scores_gemma":[0.9560376,0.03210513,0.001592662,0.006518589,0.003087456,0.0006584729],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004267811,0.0004954899,0.05748381,0.001588672,0.0004023432,0.0003545665,0.005525099,0.1050723,0.01165303,0.06026798,0.01788055,0.7388495],"study_design_scores_gemma":[0.0001016853,0.0003463363,0.00814067,0.0004409461,0.0001704712,0.000237813,0.002464759,0.7837906,0.01483862,0.1727511,0.01661613,0.0001009234],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2616311,0.003225918,0.6848073,0.02632416,0.0002956589,0.0004682098,0.001439317,0.006205893,0.01560249],"genre_scores_gemma":[0.7703958,0.001527853,0.2201446,0.001879823,0.00009661802,0.0002916401,0.00178234,0.0006527575,0.003228667],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.008735626,"threshold_uncertainty_score":0.04619896,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06524125755685459,"score_gpt":0.2859045553251602,"score_spread":0.2206632977683056,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}