{"id":"W4285148079","doi":"10.18653/v1/2022.findings-acl.293","title":"Local Structure Matters Most: Perturbation Study in NLU","year":2022,"lang":"en","type":"article","venue":"Findings of the Association for Computational Linguistics: ACL 2022","topic":"Topic Modeling","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canadian Institute for Advanced Research; McGill University; Polytechnique Montréal; Mila - Quebec Artificial Intelligence Institute","funders":"Canada First Research Excellence Fund; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Computer science; Perturbation (astronomy); Word order; Phenomenon; Artificial neural network; Invariant (physics); Artificial intelligence; Natural language processing; Mathematics; Physics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008035807,0.0001145385,0.000179522,0.0001791077,0.0003586842,0.00006285012,0.0008767091,0.00004115809,0.00002454364],"category_scores_gemma":[0.001440549,0.0001151224,0.00008468026,0.000545637,0.00001336483,0.00006333106,0.0005073393,0.000263776,0.000001487505],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009062517,"about_ca_system_score_gemma":0.0001563808,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004721613,"about_ca_topic_score_gemma":0.00001672237,"domain_scores_codex":[0.9980074,0.0001483875,0.0004467425,0.0003124869,0.0008855384,0.0001994513],"domain_scores_gemma":[0.9983345,0.0006135527,0.0004359642,0.0002246009,0.0003665736,0.00002484093],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001491363,0.0001354663,0.0468686,0.00001451087,0.00004401222,8.918118e-7,0.00228234,0.8679314,0.00001800249,0.08018631,0.002198341,0.0003052493],"study_design_scores_gemma":[0.00107038,0.0001219933,0.04783331,0.000009183535,0.00002198444,0.000001532055,0.0004924844,0.842502,0.00004544515,0.1037828,0.003933719,0.0001852107],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4382632,0.00006623469,0.5405857,0.007873446,0.008687241,0.002811902,0.0007858576,0.0001764083,0.0007500194],"genre_scores_gemma":[0.9907287,1.424115e-7,0.008002929,0.0005906869,0.000103684,0.00004059561,0.0000646445,0.00001197687,0.0004565819],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5524656,"threshold_uncertainty_score":0.4694555,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01015079981417298,"score_gpt":0.2441797445615391,"score_spread":0.2340289447473661,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}