{"id":"W4205678875","doi":"10.1109/ase51524.2021.9678871","title":"DeepMemory: Model-based Memorization Analysis of Deep Neural Language Models","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Perplexity; Memorization; Language model; Artificial intelligence; Artificial neural network; Machine learning; Data modeling; Robustness (evolution); Natural language processing; Database","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002559315,0.000342417,0.0004792422,0.0007976549,0.00008544307,0.0002081061,0.001458509,0.0001596659,0.0002874976],"category_scores_gemma":[0.001195549,0.0003837176,0.0002436445,0.001584979,0.00003395767,0.0005660487,0.0002780159,0.0003917725,0.00001514586],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001991317,"about_ca_system_score_gemma":0.0002311468,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003025627,"about_ca_topic_score_gemma":0.00001342079,"domain_scores_codex":[0.9974718,0.0001009432,0.0005421742,0.000685477,0.0008475365,0.0003520678],"domain_scores_gemma":[0.9975896,0.0003326032,0.0002613609,0.0009663867,0.0007102626,0.0001397099],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001194178,0.00007088531,0.0002178251,0.00002722557,0.0003740459,0.00009183781,0.0005704781,0.9836665,0.002624585,0.01018736,0.00002444048,0.002132921],"study_design_scores_gemma":[0.0004226485,0.0000266105,0.0005009705,0.00006984201,0.000132059,0.000005173441,0.0000418289,0.9938751,0.004407512,0.0001637672,0.000007091915,0.000347406],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.03385437,0.00003888,0.9633803,0.0003742805,0.001022556,0.0001039176,0.00006217151,0.0009697759,0.0001936831],"genre_scores_gemma":[0.8163967,0.000005853403,0.1828574,0.0001436774,0.0000620167,0.00001858116,0.0003681162,0.00003283482,0.0001149332],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7825422,"threshold_uncertainty_score":0.9998615,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02093544744193205,"score_gpt":0.2813157815033691,"score_spread":0.2603803340614371,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}