{"id":"W4386566901","doi":"10.18653/v1/2023.eacl-main.19","title":"Understanding Transformer Memorization Recall Through Idioms","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"European Commission; Canadian Institute for Advanced Research","keywords":"Memorization; Computer science; Transformer; Recall; Phrase; Artificial intelligence; Natural language processing; Security token; Machine learning; Speech recognition; Cognitive psychology; Psychology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001640905,0.00005900041,0.00006339019,0.00005718943,0.00008634139,0.00007489351,0.0002788808,0.00003646432,0.00003614013],"category_scores_gemma":[0.00000915555,0.00005143339,0.00002994191,0.0005661742,0.000009586674,0.0005595254,0.00003107206,0.00004756335,0.0001352771],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005796361,"about_ca_system_score_gemma":0.00002153354,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001685488,"about_ca_topic_score_gemma":0.000009664267,"domain_scores_codex":[0.9992776,0.00001538337,0.0001243754,0.0002133199,0.0001836975,0.000185633],"domain_scores_gemma":[0.9996839,0.00003255749,0.00001549501,0.0002260043,0.00001364121,0.00002844245],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[7.036512e-7,0.000004405872,0.00006107337,0.000008848779,0.000005477845,0.000004289562,0.001704482,0.002130792,0.0009501579,0.9888171,0.00120654,0.005106121],"study_design_scores_gemma":[0.0002355607,0.00002144239,0.00007124884,0.00001372016,0.000002438214,0.000004443531,0.0002796861,0.8003422,0.003989787,0.1896302,0.005244827,0.0001644751],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.001136668,0.000006606875,0.9530684,0.003000563,0.0004110324,0.00007069374,2.167586e-7,0.0005880172,0.04171774],"genre_scores_gemma":[0.9477819,0.00003654023,0.04912652,0.0003853858,0.00006764079,0.000004962385,0.000002354186,0.000007328392,0.002587344],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9466453,"threshold_uncertainty_score":0.2097393,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.237016441629294,"score_gpt":0.2989166739255986,"score_spread":0.06190023229630456,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}