{"id":"W4385573565","doi":"10.18653/v1/2022.blackboxnlp-1.22","title":"On the Compositional Generalization Gap of In-Context Learning","year":2022,"lang":"en","type":"article","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Generalization; Computer science; Parsing; Artificial intelligence; Task (project management); Context (archaeology); Generative grammar; Natural language processing; Machine learning; Benchmark (surveying); Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01695427,0.002002392,0.001745961,0.001257668,0.001095811,0.00227591,0.002098992,0.002563057,0.0028819],"category_scores_gemma":[0.05273409,0.0008127642,0.001092,0.0009658962,0.003051131,0.009847098,0.005189442,0.005809372,0.0009759755],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001551796,"about_ca_system_score_gemma":0.001614284,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004539289,"about_ca_topic_score_gemma":0.005519936,"domain_scores_codex":[0.9940156,0.003157365,0.0002854467,0.001273274,0.0009144178,0.0003538845],"domain_scores_gemma":[0.9514384,0.03688664,0.001084422,0.00792301,0.001759512,0.0009080544],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.001760151,0.0008233413,0.02481637,0.000642326,0.0005625113,0.0004696999,0.001444976,0.5438079,0.009897571,0.03847812,0.01181832,0.3654786],"study_design_scores_gemma":[0.00007823919,0.0004501621,0.002879588,0.00008735059,0.00009417893,0.0001960512,0.0002202172,0.9306833,0.004992737,0.05809046,0.00218685,0.00004093547],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5644477,0.009669134,0.3927682,0.008473645,0.0003562401,0.0001964588,0.0008430863,0.006406487,0.01683906],"genre_scores_gemma":[0.9429053,0.001174617,0.05107097,0.001188611,0.0001726742,0.000169419,0.001079447,0.0006015507,0.001637409],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01695427,"threshold_uncertainty_score":0.08966386,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01643434899607517,"score_gpt":0.2584125213705757,"score_spread":0.2419781723745005,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}