{"id":"W7128191878","doi":"10.1145/3745764","title":"NLPerturbator: Studying the Robustness of Code LLMs to Natural Language Variations","year":2025,"lang":"en","type":"article","venue":"ACM Transactions on Software Engineering and Methodology","topic":"Topic Modeling","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University","funders":"National Key Research and Development Program of China","keywords":"Robustness (evolution); Natural language; Coding (social sciences); Code (set theory); Natural language generation","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005143621,0.00009760055,0.0001683028,0.0001741854,0.0001124462,0.0000210919,0.0004749312,0.00005697499,0.000002324475],"category_scores_gemma":[0.0008484024,0.00007744001,0.0000430438,0.0003548432,0.00001558891,0.00006858545,0.00003322722,0.0002157753,5.473391e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000217407,"about_ca_system_score_gemma":0.00002996438,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000029414,"about_ca_topic_score_gemma":0.00001232117,"domain_scores_codex":[0.9992532,0.0001250271,0.0001669877,0.0002257296,0.00007721876,0.0001518964],"domain_scores_gemma":[0.99714,0.002207373,0.00002303408,0.0005528016,0.00004264775,0.00003412976],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000009012992,0.0000311336,0.0000437924,0.00004911114,0.00009801998,0.000001626304,0.003937786,0.7792239,0.001739888,0.007589368,0.00003256967,0.2072438],"study_design_scores_gemma":[0.000793063,0.0001306648,0.004298685,0.0001662844,0.0001239487,0.00004698677,0.0008129802,0.9805472,0.009059025,0.0007909384,0.002783077,0.0004471689],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.01728769,0.0003379351,0.9799492,0.000945655,0.001181855,0.0001216403,0.000004288167,0.0001675191,0.0000042519],"genre_scores_gemma":[0.3331617,0.000007505887,0.6664926,0.0001528827,0.00001699344,0.00002582656,3.461287e-7,0.000004609355,0.0001374724],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.315874,"threshold_uncertainty_score":0.3157912,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05664408697152098,"score_gpt":0.3164152668179596,"score_spread":0.2597711798464386,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}