{"id":"W3137010024","doi":"10.1162/tacl_a_00447","title":"Quality at a Glance: An Audit of Web-Crawled Multilingual Datasets","year":2022,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":167,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo; Google (Canada)","funders":"Agence Nationale de la Recherche","keywords":"Computer science; USable; Audit; Natural language processing; Quality (philosophy); Artificial intelligence; Information retrieval; World Wide Web; Accounting","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.06712028,0.001397105,0.00132719,0.01715647,0.00343512,0.007021653,0.003319533,0.001607887,0.001370947],"category_scores_gemma":[0.2108094,0.001615657,0.001202456,0.01735238,0.003590734,0.005904365,0.007036,0.002938509,0.003048406],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002321183,"about_ca_system_score_gemma":0.005634047,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02001247,"about_ca_topic_score_gemma":0.02837876,"domain_scores_codex":[0.8996423,0.02861887,0.01683081,0.01018274,0.04248993,0.002235285],"domain_scores_gemma":[0.4999964,0.1270439,0.03055844,0.1397169,0.1957293,0.006955071],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0021886,0.00153723,0.2703964,0.004656477,0.001347916,0.002693572,0.01417526,0.006669717,0.03967949,0.004891357,0.302007,0.349757],"study_design_scores_gemma":[0.0004033604,0.0008357639,0.4684419,0.002261519,0.0007260713,0.002622361,0.00593911,0.05185007,0.08352879,0.004778695,0.3778376,0.0007747139],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7020456,0.005848777,0.05948382,0.009382596,0.001744382,0.002667416,0.1168523,0.0827925,0.0191827],"genre_scores_gemma":[0.5110751,0.001643284,0.1286201,0.002577777,0.0003854511,0.00200854,0.3251124,0.01962509,0.00895226],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9328797,"threshold_uncertainty_score":0.3549702,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02197055172116473,"score_gpt":0.3306852318074935,"score_spread":0.3087146800863287,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}