{"id":"W2404260248","doi":"10.21437/interspeech.2013-610","title":"Automatic human utility evaluation of ASR systems: does WER really predict performance?","year":2013,"lang":"en","type":"article","venue":"","topic":"Speech Recognition and Synthesis","field":"Computer Science","cited_by":39,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canada Research Chairs; University of Toronto","funders":"","keywords":"Metric (unit); Computer science; Task (project management); Context (archaeology); Audit; Word error rate; Word (group theory); Speech recognition; Performance metric; Artificial intelligence; Natural language processing; Range (aeronautics); Machine learning; Mathematics; Engineering; Operations management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001419983,0.00009064009,0.0001604888,0.00009854641,0.00008009377,0.0001093011,0.0003801198,0.00005226444,0.003018296],"category_scores_gemma":[0.00008893591,0.0000577559,0.00004762361,0.0001542696,0.00003668024,0.0006358674,0.00006408303,0.00004666872,0.0002512726],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003895596,"about_ca_system_score_gemma":0.0000701477,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002299775,"about_ca_topic_score_gemma":0.00001621876,"domain_scores_codex":[0.9983839,0.0001994424,0.0003665693,0.0002145658,0.0006881984,0.000147324],"domain_scores_gemma":[0.9987407,0.00007269074,0.0001284824,0.0004865932,0.0005103908,0.0000611213],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000001296533,0.0001993608,0.01080528,0.0002564394,0.00005867655,4.188882e-7,0.0004827705,0.00001569038,0.001507037,0.003363134,0.004382809,0.9789271],"study_design_scores_gemma":[0.0001846774,0.0000386964,0.1514828,0.00005604841,0.00001602972,0.000003097745,0.00008135857,0.84322,0.00369461,0.001084152,0.00006014788,0.00007837619],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9288341,0.00001528342,0.008048197,0.0001416831,0.0002656145,0.000545492,0.00000145478,0.0002201838,0.06192794],"genre_scores_gemma":[0.9932519,0.000003693698,0.006104193,0.0000390361,0.0000257854,0.00009842264,0.00000228212,0.000003877406,0.000470827],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9788487,"threshold_uncertainty_score":0.9978931,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04054442211315303,"score_gpt":0.273337642310252,"score_spread":0.232793220197099,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}