{"id":"W3190342989","doi":"10.48550/arxiv.2104.01604","title":"Timers and Such: A Practical Benchmark for Spoken Language Understanding\\n with Numbers","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Intelligent Tutoring Systems and Adaptive Learning","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal; McGill University; Mila - Quebec Artificial Intelligence Institute","funders":"","keywords":"Benchmark (surveying); Spoken language; Computer science; Natural language processing; Linguistics; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002981963,0.000261909,0.0003199997,0.0001335246,0.0001941398,0.000307489,0.0003591238,0.0001858116,0.00002114067],"category_scores_gemma":[0.00006168211,0.0002669664,0.000120288,0.0002544087,0.00008766974,0.0003704565,0.0006045193,0.0004695099,0.000006507075],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002653105,"about_ca_system_score_gemma":0.0002505357,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002498569,"about_ca_topic_score_gemma":0.00007594675,"domain_scores_codex":[0.9983511,0.000109006,0.0001375746,0.0009571262,0.0001039764,0.0003412464],"domain_scores_gemma":[0.9987537,0.0003184061,0.000200719,0.0004656857,0.0001100751,0.000151387],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001036487,0.00006944751,0.004780221,0.0002671981,0.0003570835,0.001333685,0.003961601,0.01284146,0.00003235817,0.975912,0.0001975776,0.0001437483],"study_design_scores_gemma":[0.005254298,0.001379153,0.003660704,0.004747217,0.001123583,0.00048211,0.0837533,0.8644639,0.0007872566,0.01637167,0.01287462,0.005102191],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1563157,0.00006720544,0.8403775,0.0001258942,0.0002811162,0.0003017145,0.000004755683,0.0001006558,0.002425442],"genre_scores_gemma":[0.9872879,0.00002943965,0.007323937,0.00004666217,0.00007610282,0.000001721847,0.00001482562,0.00001984368,0.005199594],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9595403,"threshold_uncertainty_score":0.9999782,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1272331395489106,"score_gpt":0.222093520618043,"score_spread":0.09486038106913239,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}