{"id":"W4385573565","doi":"10.18653/v1/2022.blackboxnlp-1.22","title":"On the Compositional Generalization Gap of In-Context Learning","year":2022,"lang":"en","type":"article","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Generalization; Computer science; Parsing; Artificial intelligence; Task (project management); Context (archaeology); Generative grammar; Natural language processing; Machine learning; Benchmark (surveying); Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002216823,0.00003197982,0.00003982924,0.00006097215,0.0001046287,0.00001961625,0.0003679468,0.000008098174,0.0001403971],"category_scores_gemma":[0.00002201692,0.00002225236,0.00001473999,0.0002460444,0.00001135361,0.00007515466,0.0001817043,0.00013408,0.000001418879],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003114541,"about_ca_system_score_gemma":0.00001708256,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003680652,"about_ca_topic_score_gemma":0.000003963212,"domain_scores_codex":[0.9994982,0.00009850255,0.000081066,0.00009001739,0.00017735,0.00005480093],"domain_scores_gemma":[0.9997405,0.00008367732,0.00004434071,0.0001026331,0.00002278674,0.000006003793],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.000002538929,0.00001839799,0.0001753229,0.000001541272,9.867484e-7,0.000001580586,0.0003400988,0.001285125,0.005345164,0.9883432,0.0006125369,0.003873458],"study_design_scores_gemma":[0.0003366564,0.0003609927,0.0006998436,0.00003718066,0.000002045627,0.00003010751,0.0002817395,0.3841747,0.1488407,0.463378,0.001610256,0.0002477448],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08397897,0.0003755213,0.9076948,0.003892267,0.00007700046,0.000128666,8.186944e-7,0.0002032064,0.003648822],"genre_scores_gemma":[0.9653992,6.179395e-7,0.03338047,0.0009933771,0.000004940782,0.00001314427,0.000002642153,0.000001978226,0.0002036198],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8814203,"threshold_uncertainty_score":0.153725,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01643434899607517,"score_gpt":0.2584125213705757,"score_spread":0.2419781723745005,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}