{"id":"W4205678875","doi":"10.1109/ase51524.2021.9678871","title":"DeepMemory: Model-based Memorization Analysis of Deep Neural Language Models","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Perplexity; Memorization; Language model; Artificial intelligence; Artificial neural network; Machine learning; Data modeling; Robustness (evolution); Natural language processing; Database","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001729058,0.001069164,0.0007092992,0.001229561,0.0003448018,0.001033433,0.001502185,0.0006577314,0.001271956],"category_scores_gemma":[0.01037721,0.0003643798,0.0009969985,0.0005662029,0.0005183704,0.002939186,0.001371772,0.002028399,0.000376172],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001090988,"about_ca_system_score_gemma":0.001071509,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003994116,"about_ca_topic_score_gemma":0.005709801,"domain_scores_codex":[0.9991678,0.0002087232,0.00008302704,0.0002353313,0.0002094276,0.00009579878],"domain_scores_gemma":[0.9957151,0.002162507,0.0006785806,0.0007986831,0.0005353322,0.0001097685],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000870982,0.0004005059,0.01971267,0.0004055548,0.0003745064,0.0005269333,0.0005264099,0.4233956,0.02339659,0.009290509,0.007040185,0.5140596],"study_design_scores_gemma":[0.00001165077,0.00009126831,0.0008535247,0.00001108236,0.0000281488,0.00004982919,0.00003471318,0.9850172,0.007703545,0.005690846,0.0004949721,0.00001327977],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.208484,0.001013444,0.7779106,0.0007086614,0.0001013011,0.0001739109,0.001016314,0.009314076,0.001277754],"genre_scores_gemma":[0.8941908,0.0003689225,0.1011194,0.0002658123,0.00005876254,0.0001998825,0.001600919,0.0002246765,0.001970847],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.003994116,"threshold_uncertainty_score":0.009144247,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02093544744193205,"score_gpt":0.2813157815033691,"score_spread":0.2603803340614371,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}