{"id":"W6929093138","doi":"10.48448/ek8y-z972","title":"On the Softmax Bottleneck of Recurrent Language Models","year":2021,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"","keywords":"Softmax function; Perplexity; Bottleneck; Correlation; Monotonic function; Rank (graph theory); Piecewise; Matrix (chemical analysis)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004552203,0.00129098,0.001133901,0.000733439,0.0005681864,0.001829375,0.001553206,0.00140294,0.004180497],"category_scores_gemma":[0.02047959,0.0008988304,0.0007360181,0.0007302949,0.001321917,0.004501988,0.002201619,0.003143598,0.001825097],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001262471,"about_ca_system_score_gemma":0.001327104,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005716007,"about_ca_topic_score_gemma":0.004919039,"domain_scores_codex":[0.9984913,0.0008044239,0.00006423533,0.0002606913,0.0002355637,0.0001437293],"domain_scores_gemma":[0.9919695,0.006305302,0.0003619459,0.0005750165,0.0005898772,0.0001983624],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0004782004,0.000123208,0.001823168,0.0002428328,0.0001283255,0.0002756233,0.0002389524,0.7224838,0.005303279,0.1154663,0.006416044,0.1470202],"study_design_scores_gemma":[0.000005718416,0.00002375819,0.000128599,0.00001341035,0.000007011149,0.00001988026,0.000009215568,0.9686185,0.0009162904,0.02977538,0.0004738107,0.000008378315],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04822898,0.001230819,0.9401994,0.001371161,0.00008199312,0.00003581888,0.0002557936,0.001664513,0.006931527],"genre_scores_gemma":[0.8816215,0.001627321,0.102485,0.0006767694,0.0002398413,0.0001910039,0.0007895364,0.0005963176,0.01177267],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.005716007,"threshold_uncertainty_score":0.02407461,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04317076503652927,"score_gpt":0.3134520370440302,"score_spread":0.2702812720075009,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}