{"id":"W4399115506","doi":"10.48550/arxiv.2405.16325","title":"SLoPe: Double-Pruned Sparse Plus Lazy Low-Rank Adapter Pretraining of LLMs","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Network Packet Processing and Optimization","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Department of Mechanical Engineering, University of Texas at Austin; Alliance de recherche numérique du Canada; DeepMind; Canada Research Chairs; University of Texas at Austin; National Science Foundation","keywords":"Adapter (computing); Rank (graph theory); Computer science; Psychology; Mathematics; Combinatorics; Computer hardware","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.000359445,0.0003656997,0.000449481,0.0003503838,0.0001218364,0.0002386393,0.001580653,0.0003669675,0.00003489557],"category_scores_gemma":[0.00001379707,0.0004041181,0.0002228528,0.001027448,0.0001418282,0.0003637763,0.002153598,0.0007403251,0.00007195191],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000170014,"about_ca_system_score_gemma":0.0004886379,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006488863,"about_ca_topic_score_gemma":0.00002694723,"domain_scores_codex":[0.9978544,0.00008362634,0.0003225681,0.001181183,0.000164406,0.0003938554],"domain_scores_gemma":[0.9981428,0.00007003979,0.0003301064,0.001075513,0.0002299913,0.0001515019],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000099898,0.00005037698,0.0001230105,0.0004424388,0.0001227665,0.0002287497,0.001451931,0.9279059,0.00001993692,0.06622057,0.0005944712,0.002739967],"study_design_scores_gemma":[0.0008047126,0.00004462931,0.00003720747,0.0008388981,0.0001065539,0.000005067423,0.00006545623,0.9782819,0.0002719643,0.01882118,0.0002714877,0.0004509436],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1070051,0.000440303,0.8688657,0.000157862,0.001762149,0.0003660489,0.00001178331,0.000564781,0.02082633],"genre_scores_gemma":[0.9867548,0.0001741901,0.008930844,0.00007308504,0.0001378838,0.000001305639,0.00002092631,0.0000333525,0.003873603],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8797497,"threshold_uncertainty_score":0.9998411,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07694164391039529,"score_gpt":0.1976297395737091,"score_spread":0.1206880956633138,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}