{"id":"W4388328907","doi":"10.48550/arxiv.2311.00913","title":"Self-Influence Guided Data Reweighting for Language Model Pre-training","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Computer science; Novelty; Artificial intelligence; Context (archaeology); Task (project management); Relevance (law); Language model; Machine learning; Sample (material); Training set; Point (geometry); Stability (learning theory); Quality (philosophy); Natural language processing; Mathematics","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","open_science"],"consensus_categories":["open_science"],"category_scores_codex":[0.0008617544,0.0003369945,0.0003737075,0.0002416659,0.0002292079,0.0002081717,0.006238803,0.0002846483,0.000001899007],"category_scores_gemma":[0.0002385628,0.000408749,0.0001413479,0.0004013221,0.00003862007,0.0008501175,0.008190099,0.0004957358,0.00002742235],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001835049,"about_ca_system_score_gemma":0.0004825988,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001827078,"about_ca_topic_score_gemma":0.00005643209,"domain_scores_codex":[0.9967604,0.00007370521,0.0003392312,0.002113472,0.0001454624,0.0005677398],"domain_scores_gemma":[0.9950642,0.0002412303,0.0003239721,0.004036603,0.0001733325,0.0001606724],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005229729,0.00001863442,0.0001596544,0.0001875643,0.00006847112,0.0001036269,0.002545869,0.943401,0.00005264822,0.05264835,0.0003071427,0.0005018016],"study_design_scores_gemma":[0.0002913331,0.00001023485,0.0000435696,0.0001432047,0.00005921212,0.000003174506,0.0001360373,0.9580985,0.00003708876,0.04061638,0.0001419454,0.0004193754],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1334643,0.00003457514,0.8635287,0.0001350112,0.0003972923,0.0004863364,0.00007674258,0.001210873,0.0006662001],"genre_scores_gemma":[0.7897966,0.00004431701,0.207424,0.0001017546,0.0001496452,0.000003465869,0.0000750001,0.00003870308,0.002366557],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.6563323,"threshold_uncertainty_score":0.9998364,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2694218530808317,"score_gpt":0.2625380589666247,"score_spread":0.006883794114206976,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}