{"id":"W6910435407","doi":"10.48448/w0ps-0t42","title":"BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training","year":2024,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Thomson Reuters (Canada)","funders":"","keywords":"Vocabulary; Downstream (manufacturing); Encoding (memory); Training (meteorology); Simple (philosophy); Harm; Compromise","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.001984078,0.0008888555,0.0007370525,0.002946451,0.0003922342,0.0004937675,0.001709506,0.0003267539,0.002713536],"category_scores_gemma":[0.0002179602,0.0007984058,0.0001897371,0.002188228,0.001683043,0.000143171,0.001043882,0.0009190933,0.03043274],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00110468,"about_ca_system_score_gemma":0.001228936,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002362643,"about_ca_topic_score_gemma":0.0004914015,"domain_scores_codex":[0.9924785,0.00007740922,0.0007707321,0.002215703,0.002708168,0.001749455],"domain_scores_gemma":[0.9974193,0.0000392087,0.000422417,0.001444752,0.0001564422,0.0005178814],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001015854,0.001071623,0.00007980371,0.001827517,0.000650687,0.002513236,0.01056315,0.006483618,0.1068531,0.006347385,0.845058,0.01845026],"study_design_scores_gemma":[0.00148006,0.0001896143,0.0002076459,0.003007298,0.0003160182,0.0001967674,0.001483829,0.02380714,0.002515179,0.0003667564,0.9639923,0.0024374],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.008881121,0.006897545,0.0005720438,0.0007132732,0.005182668,0.00149954,0.0004233723,0.003773225,0.9720572],"genre_scores_gemma":[0.1205276,0.00008981964,0.01262168,0.0003840401,0.002109895,0.0001203302,0.0001601888,0.003323706,0.8606627],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.1189343,"threshold_uncertainty_score":0.9994467,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03139523454099037,"score_gpt":0.2981416731004584,"score_spread":0.266746438559468,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}