{"id":"W4392828353","doi":"10.48550/arxiv.2403.08763","title":"Simple and Scalable Strategies to Continually Pre-train Large Language Models","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Oak Ridge National Laboratory; Natural Sciences and Engineering Research Council of Canada; Office of Science; Fonds de recherche du Québec – Nature et technologies; Université de Montréal; Deutscher Akademischer Austauschdienst; Canadian Institute for Advanced Research; Canada Excellence Research Chairs, Government of Canada; U.S. Department of Energy","keywords":"Simple (philosophy); Computer science; Scalability; Epistemology; Philosophy","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003154886,0.0002798403,0.0003013973,0.0002291021,0.00009442744,0.0004947787,0.001179408,0.000209782,0.00001725336],"category_scores_gemma":[0.00001391148,0.0003145891,0.00009821475,0.0003284663,0.00003624984,0.0004507272,0.004251353,0.0004884298,0.00004470096],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008952162,"about_ca_system_score_gemma":0.0002315622,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004316615,"about_ca_topic_score_gemma":0.0003667198,"domain_scores_codex":[0.9980027,0.00007291517,0.0001788934,0.00121464,0.0001009725,0.0004298571],"domain_scores_gemma":[0.9986016,0.00005494053,0.0000722527,0.0009928999,0.00007837551,0.0001999129],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00000780666,0.00002710588,0.00009473859,0.0001432048,0.00005048494,0.0003036496,0.003521157,0.3737903,0.0001032126,0.6209195,0.0003300714,0.0007087233],"study_design_scores_gemma":[0.0001607158,0.00002277376,0.00008439267,0.00007901746,0.00002617234,0.00000268157,0.000579281,0.754938,0.00004453912,0.2435463,0.0002338231,0.0002823022],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4070206,0.0001685058,0.5880277,0.00009125839,0.0001721542,0.0002229778,0.00003127295,0.0002764063,0.00398918],"genre_scores_gemma":[0.9913347,0.00003202719,0.004992869,0.0001485687,0.0000661926,0.000001688778,0.000008053054,0.00001975477,0.003396158],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5843141,"threshold_uncertainty_score":0.9999306,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0430427905600358,"score_gpt":0.2101212016528687,"score_spread":0.1670784110928329,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}