{"id":"W4388555939","doi":"10.48550/arxiv.2311.04900","title":"How Abstract Is Linguistic Generalization in Large Language Models? Experiments with Argument Structure","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Northwestern University; University of Toronto; Stony Brook University; National Science Foundation; Yale University; Heinrich-Heine-Universität Düsseldorf","keywords":"Argument (complex analysis); Verb; Word order; Generalization; Linguistics; Object (grammar); Computer science; Subject (documents); Noun; Natural language processing; Space (punctuation); Artificial intelligence; Mathematics; Philosophy","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0001066964,0.0002847759,0.0002539638,0.000287527,0.00007422583,0.0002140536,0.001036047,0.0002251309,0.00001405817],"category_scores_gemma":[0.00001354794,0.0003004798,0.00006806949,0.0003793706,0.00002054637,0.0003130362,0.0009636484,0.0004190395,0.000008288338],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000265048,"about_ca_system_score_gemma":0.0001191867,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003700395,"about_ca_topic_score_gemma":0.0002922106,"domain_scores_codex":[0.9982185,0.00004500261,0.0001553536,0.001063511,0.0001571858,0.0003603932],"domain_scores_gemma":[0.9985828,0.00001942162,0.0001810361,0.001031847,0.00008751433,0.00009734914],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001558826,0.00005781834,0.00209597,0.00008586386,0.00005540229,0.0007194856,0.004985627,0.9394501,0.00008453476,0.0522368,0.0000467351,0.0001660944],"study_design_scores_gemma":[0.0005118732,0.00001998182,0.0007205953,0.0001215521,0.00001737486,0.00000142566,0.0002588977,0.9802645,0.0003224653,0.01736616,0.00004267836,0.0003525244],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4244821,0.00005301273,0.5744789,0.00007274323,0.000351833,0.0002074548,0.00001994412,0.0001534732,0.0001805471],"genre_scores_gemma":[0.9933695,0.00003863298,0.004789201,0.00009626259,0.000090207,0.000001515082,0.00004455147,0.00002629894,0.001543864],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5696898,"threshold_uncertainty_score":0.9999447,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07583491214773017,"score_gpt":0.2133967284531725,"score_spread":0.1375618163054423,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}