{"id":"W3167021948","doi":"10.48550/arxiv.2106.03795","title":"Heavy Tails in SGD and Compressibility of Overparametrized Neural Networks","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Stochastic Gradient Optimization Techniques","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; Canada Research Chairs; University of Toronto","funders":"","keywords":"Pruning; Artificial neural network; Generalization; Compressibility; Computer science; Limit (mathematics); Compression (physics); Node (physics); Computation; Algorithm; Mathematics; Applied mathematics; Mathematical optimization; Artificial intelligence; Mathematical analysis; Physics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002527424,0.0008965048,0.001014679,0.001400384,0.000730004,0.00107818,0.001328968,0.001500268,0.001985645],"category_scores_gemma":[0.02136985,0.0005528269,0.0007499834,0.0006454865,0.003158051,0.002908837,0.002100409,0.002737769,0.0002099452],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001632744,"about_ca_system_score_gemma":0.0009537393,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004316257,"about_ca_topic_score_gemma":0.003496972,"domain_scores_codex":[0.9994144,0.0001828561,0.00004864355,0.0001125815,0.000166951,0.00007461561],"domain_scores_gemma":[0.9903924,0.006920122,0.0008914325,0.0007803363,0.0006883824,0.0003272678],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00009486899,0.00003115321,0.002968619,0.0001943499,0.00004746665,0.0003154357,0.0002277037,0.8733603,0.003305648,0.104675,0.001153594,0.0136259],"study_design_scores_gemma":[0.000004361753,0.00001249199,0.0003269129,0.00001581885,0.000003026968,0.00002026755,0.0000087924,0.9721249,0.0004142658,0.0269181,0.0001452662,0.000005738405],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2011436,0.002162733,0.787823,0.002153807,0.0001006947,0.00008358161,0.0003293239,0.0006356168,0.005567638],"genre_scores_gemma":[0.9463862,0.001407892,0.0475188,0.0003926801,0.00009567019,0.0001712471,0.0003949067,0.0001609464,0.003471727],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.004316257,"threshold_uncertainty_score":0.01336652,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05854675172088644,"score_gpt":0.1973297937004245,"score_spread":0.1387830419795381,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}