{"id":"W3176904855","doi":"10.1609/aaai.v35i15.17608","title":"On the Softmax Bottleneck of Recurrent Language Models","year":2021,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Artificial Intelligence","topic":"Topic Modeling","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Western Canada Research Grid; Compute Canada","keywords":"Perplexity; Softmax function; Bottleneck; Rank (graph theory); Computer science; Monotonic function; Word (group theory); Correlation; Language model; Mathematics; Artificial intelligence; Artificial neural network; Combinatorics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00797704,0.001474303,0.001618761,0.0009919953,0.00074653,0.002242385,0.002008923,0.001778387,0.003419125],"category_scores_gemma":[0.03418868,0.001253324,0.001097045,0.0009420485,0.001759473,0.00620389,0.00270342,0.004213871,0.001408601],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001714164,"about_ca_system_score_gemma":0.001565942,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00641345,"about_ca_topic_score_gemma":0.004854272,"domain_scores_codex":[0.9973597,0.001437652,0.0001261063,0.0004301177,0.0003990446,0.0002474178],"domain_scores_gemma":[0.9846042,0.01257236,0.0006191686,0.0009221662,0.0009699545,0.0003122025],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004983866,0.0001229758,0.001714293,0.0002429454,0.0001340734,0.0002824352,0.0003129003,0.7751242,0.004250511,0.1163288,0.004117387,0.09687114],"study_design_scores_gemma":[0.000007196802,0.00002683759,0.0001221388,0.00001402752,0.000008133074,0.00002092473,0.00001046459,0.9659191,0.0007718874,0.03275678,0.0003323645,0.000009987245],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04657959,0.001027933,0.9456083,0.001121092,0.00007168399,0.00004062249,0.0002029808,0.001304782,0.004042957],"genre_scores_gemma":[0.8868228,0.001568538,0.1008496,0.000711217,0.0002605008,0.0002612631,0.0007769778,0.0005442305,0.00820482],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.00797704,"threshold_uncertainty_score":0.04218709,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1059550927922049,"score_gpt":0.2987821668527856,"score_spread":0.1928270740605807,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}