{"id":"W4400488479","doi":"10.48550/arxiv.2407.06172","title":"On Speeding Up Language Model Evaluation","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Army Research Office; Division of Materials Research; Office of Naval Research; Multidisciplinary University Research Initiative; Materials Research Science and Engineering Center, Harvard University; Natural Sciences and Engineering Research Council of Canada; Cornell Center for Materials Research; Defense Advanced Research Projects Agency; National Institute of Food and Agriculture; National Science Foundation","keywords":"Computer science; Natural language processing","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0186655,0.004188025,0.002690346,0.003302648,0.001648686,0.004799916,0.00387865,0.003657217,0.01937405],"category_scores_gemma":[0.0950387,0.001565798,0.001792024,0.002452812,0.001795775,0.009056916,0.00508259,0.006038388,0.01179402],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002537695,"about_ca_system_score_gemma":0.004433266,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008182959,"about_ca_topic_score_gemma":0.01705424,"domain_scores_codex":[0.9790897,0.01367386,0.0009752794,0.00213003,0.00333027,0.0008008525],"domain_scores_gemma":[0.9390286,0.04794817,0.001409553,0.006441703,0.004355982,0.0008160712],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001836715,0.0007712698,0.004967382,0.0009448242,0.0003845257,0.0003204136,0.000419305,0.1865887,0.009186564,0.0227407,0.04470708,0.7271324],"study_design_scores_gemma":[0.0002379902,0.0002064499,0.0005699216,0.0001151732,0.00007226513,0.0001544277,0.0001554208,0.9559526,0.00449151,0.03100609,0.006995835,0.0000422995],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0404929,0.00446063,0.8954949,0.002811408,0.0005200093,0.0006213997,0.001210438,0.04572721,0.008661109],"genre_scores_gemma":[0.2379081,0.0007585678,0.746025,0.001728075,0.0002852966,0.0007293746,0.003136691,0.004099443,0.00532947],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01937405,"threshold_uncertainty_score":0.09871382,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0959286585888844,"score_gpt":0.2504746523215263,"score_spread":0.1545459937326419,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}