{"id":"W2966610483","doi":"10.48550/arxiv.1907.12009","title":"Representation Degeneration Problem in Training Natural Language\\n Generation Models","year":2019,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":105,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Computer science; Regularization (linguistics); Artificial intelligence; Machine translation; Representation (politics); Maximization; Natural language processing; Language model; Natural language; Natural language understanding; Tying; Machine learning; Mathematical optimization; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.000288553,0.0002060728,0.000238338,0.0003094582,0.00006745218,0.0001585489,0.0007421596,0.0002132209,0.000004371086],"category_scores_gemma":[0.00001722493,0.000251687,0.00009507909,0.0004106828,0.00001633186,0.0009980003,0.0005865202,0.0004601567,0.00001525536],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002730654,"about_ca_system_score_gemma":0.0001943824,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003640939,"about_ca_topic_score_gemma":0.0003201675,"domain_scores_codex":[0.9981728,0.0001759017,0.0002465287,0.001038751,0.0001149812,0.0002510736],"domain_scores_gemma":[0.9987831,0.00003567788,0.0001968262,0.0008366675,0.000096424,0.00005129923],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00000406317,0.00001426261,0.0002845716,0.00002096465,0.00001028827,0.00003720374,0.002928295,0.93083,0.000853451,0.06195381,0.00002403459,0.003039053],"study_design_scores_gemma":[0.0003303697,0.000009521723,0.0001018047,0.00004682305,0.00001068281,0.000002447915,0.0001962523,0.984295,0.000502028,0.01424763,0.000004149183,0.0002532615],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3980981,0.00008715241,0.6002331,0.00007103327,0.0004477675,0.0002795338,0.000001533189,0.00009296404,0.000688791],"genre_scores_gemma":[0.9737179,0.00003546387,0.02515784,0.0000664446,0.0001525816,0.000002599884,0.00007518705,0.00001298065,0.0007790116],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5756198,"threshold_uncertainty_score":0.9999936,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1649596966506116,"score_gpt":0.2195593484739972,"score_spread":0.05459965182338553,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}