{"id":"W4385572901","doi":"10.18653/v1/2022.emnlp-main.418","title":"TemporalWiki: A Lifelong Benchmark for Training and Evaluating Ever-Evolving Language Models","year":2022,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":42,"is_retracted":false,"has_abstract":true,"ca_institutions":"Kootenay Association for Science & Technology","funders":"Institute for Information and Communications Technology Promotion; Ministry of Science and ICT, South Korea; Korea Advanced Institute of Science and Technology","keywords":"Perplexity; Computer science; Benchmark (surveying); Snapshot (computer storage); Machine learning; Lifelong learning; Artificial intelligence; Training set; Adaptability; Language model; Training (meteorology); Database","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001271795,0.000112698,0.0001561703,0.0000942187,0.000448918,0.0001450186,0.0004772264,0.00002300859,0.0000542293],"category_scores_gemma":[0.00008984076,0.0001152143,0.00004973532,0.0001727737,0.0000110796,0.0005883981,0.0005911583,0.0001358663,5.150344e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005469774,"about_ca_system_score_gemma":0.0001093608,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009246902,"about_ca_topic_score_gemma":0.00001175667,"domain_scores_codex":[0.9986038,0.00007669861,0.0002331906,0.0004496354,0.0003348241,0.0003018026],"domain_scores_gemma":[0.9992477,0.0002175975,0.00007708825,0.0003408015,0.00004218622,0.00007461371],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001419,0.0000674494,0.0007688137,0.00009287426,0.00005914257,0.00002825786,0.1206561,0.1767001,0.004356809,0.1373153,0.001580841,0.5583602],"study_design_scores_gemma":[0.0003236298,0.00008249877,0.00003949512,0.000009054155,0.000004836345,0.00001805224,0.002825957,0.9872457,0.00004686764,0.009091736,0.0001570181,0.0001550892],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1715496,0.0004569822,0.8244066,0.0006142827,0.0001829178,0.0002951788,0.000003022529,0.0001656906,0.002325737],"genre_scores_gemma":[0.6690934,7.911308e-7,0.329888,0.0004426177,0.00005541323,0.00009001001,0.000002682153,0.000008299417,0.0004188046],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8105457,"threshold_uncertainty_score":0.4698302,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09207660360202875,"score_gpt":0.3187674719700207,"score_spread":0.226690868367992,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}