{"id":"W3167021948","doi":"10.48550/arxiv.2106.03795","title":"Heavy Tails in SGD and Compressibility of Overparametrized Neural Networks","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Stochastic Gradient Optimization Techniques","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; Canada Research Chairs; University of Toronto","funders":"","keywords":"Pruning; Artificial neural network; Generalization; Compressibility; Computer science; Limit (mathematics); Compression (physics); Node (physics); Computation; Algorithm; Mathematics; Applied mathematics; Mathematical optimization; Artificial intelligence; Mathematical analysis; Physics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.000290514,0.0002349894,0.0004864072,0.0003490895,0.00004234841,0.00009196826,0.0009612984,0.0002569211,0.00001015698],"category_scores_gemma":[0.0001154554,0.0002854288,0.0001179471,0.001009374,0.0001687271,0.0003047086,0.002206649,0.0004832058,2.812942e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000120722,"about_ca_system_score_gemma":0.00008677121,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002825886,"about_ca_topic_score_gemma":0.00004076531,"domain_scores_codex":[0.998153,0.0002144045,0.0003149932,0.0009796809,0.00008556831,0.0002523701],"domain_scores_gemma":[0.9980819,0.0002718356,0.0003008524,0.001064429,0.0001730092,0.0001079747],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00004333656,0.0001409872,0.01418079,0.00007568812,0.00002893171,0.00009087243,0.0002283129,0.9563918,0.00001359741,0.02833296,0.0000234273,0.0004493533],"study_design_scores_gemma":[0.0004663169,0.00004029095,0.006427227,0.0001100075,0.0000170864,0.000003074344,0.00003543912,0.9840279,0.0001451457,0.008500104,0.000001282241,0.0002261896],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3456908,0.0001811044,0.6534768,0.00001725975,0.0001707803,0.0002515107,0.000003339121,0.00009951233,0.0001089478],"genre_scores_gemma":[0.9827753,0.0001422347,0.01698996,0.00003664444,0.00001033339,0.000001329285,0.00001051384,0.000008965416,0.00002474862],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6370845,"threshold_uncertainty_score":0.9999598,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05854675172088644,"score_gpt":0.1973297937004245,"score_spread":0.1387830419795381,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}