{"id":"W4385822952","doi":"10.21437/interspeech.2023-1087","title":"Speech Self-Supervised Representation Benchmarking: Are We Doing it Right?","year":2023,"lang":"en","type":"article","venue":"","topic":"Speech Recognition and Synthesis","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"Mila - Quebec Artificial Intelligence Institute","funders":"Grand Équipement National De Calcul Intensif","keywords":"Benchmarking; Computer science; Representation (politics); Artificial intelligence; Speech recognition; Natural language processing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0003270466,0.0001229447,0.000145676,0.0002592795,0.0001759991,0.0002650857,0.0004975687,0.00006489752,0.00117541],"category_scores_gemma":[0.00004193,0.0001074576,0.00009516026,0.001168087,0.00001333726,0.0004958197,0.0001769589,0.00008534887,0.00184974],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003364221,"about_ca_system_score_gemma":0.00003029324,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002491404,"about_ca_topic_score_gemma":0.00003364763,"domain_scores_codex":[0.9985605,0.00008639686,0.0002337028,0.0004484613,0.000390518,0.0002803682],"domain_scores_gemma":[0.9990068,0.0002554169,0.00007091974,0.0004736059,0.00009094413,0.0001023173],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005267623,0.0001134532,0.00215574,0.00004258353,0.00006447695,0.0003985855,0.001617753,0.00002531991,0.002032657,0.007914457,0.1092431,0.8763866],"study_design_scores_gemma":[0.000879802,0.00004927842,0.009165164,0.0001418933,0.00002496223,0.0001080975,0.001304726,0.7602191,0.06786589,0.008672309,0.1507922,0.0007765755],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"other","genre_gemma":"methods","genre_scores_codex":[0.06975109,0.00004729432,0.355865,0.06349503,0.002540143,0.0008053989,0.000006914068,0.007148433,0.5003407],"genre_scores_gemma":[0.1596297,0.0004340499,0.8284182,0.002599495,0.0004858414,0.00006355769,0.00002202228,0.00003368337,0.008313484],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8756101,"threshold_uncertainty_score":0.9997377,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05260799574573499,"score_gpt":0.2907351813080752,"score_spread":0.2381271855623402,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}