{"meta":{"query_hash":"b4e2a36aa5f9","filters":{"venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing"},"cohort_total":19,"direct_labels_cover":0,"predictions_cover":19,"exported":19,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/b4e2a36aa5f9","api":"https://metacan.xera.ac/api/v1/cohort?venue=Proceedings+of+the+2021+Conference+on+Empirical+Methods+in+Natural+Language+Processing"},"results":[{"id":"W3093808828","doi":"10.18653/v1/2021.emnlp-main.740","title":"BARThez: a Skilled Pretrained French Sequence-to-Sequence Model","year":2021,"lang":"en","type":"preprint","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Centre National de la Recherche Scientifique","keywords":"Automatic summarization; Discriminative model; Computer science; Generative grammar; Benchmark (surveying); Transfer of learning; Artificial intelligence; Natural language processing; Sequence (biology); Code (set theory); Generative model; Field (mathematics); Language model; Machine learning; Programming language; Cartography","score_opus":0.08687262609204593,"score_gpt":0.4277476868274459,"score_spread":0.34087506073539997,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3093808828","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14060694,0.03566253,0.78819007,0.019854493,0.0023254175,0.004801399,0.00009138572,0.0017664462,0.006701295],"genre_scores_gemma":[0.46731678,0.000037006153,0.5311901,0.00086599693,0.00008034822,0.00016151415,0.0000061928313,0.000046447156,0.0002956625],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9927507,0.0003607739,0.0015321848,0.002555933,0.001614623,0.0011858093],"domain_scores_gemma":[0.99474806,0.00041774142,0.0013404053,0.0012599659,0.001909174,0.00032465247],"candidate_categories":["metaepi_narrow","scholarly_communication","open_science","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.0027934515,0.0011502268,0.0016720735,0.00083421654,0.00027742746,0.001448615,0.007593039,0.0009976652,0.00003296914],"category_scores_gemma":[0.00664901,0.000843959,0.0005170854,0.0032682326,0.00038045947,0.0010676049,0.00628146,0.004780704,0.0000028478246],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008709292,0.00036928788,0.0003753769,0.00228173,0.00007971298,0.00006905394,0.030187275,0.00050307025,0.29946235,0.0067964043,0.00019147513,0.65959716],"study_design_scores_gemma":[0.00044206684,0.00010317959,0.00011763639,0.007939691,0.00006280664,0.000055061115,0.00079209224,0.7507824,0.1646333,0.07368595,0.0000121358025,0.00137369],"about_ca_topic_score_codex":0.00009818989,"about_ca_topic_score_gemma":0.00002488599,"teacher_disagreement_score":0.7502793,"about_ca_system_score_codex":0.0005872741,"about_ca_system_score_gemma":0.0016550608,"threshold_uncertainty_score":0.99958795},"labels":[],"label_agreement":null},{"id":"W3120832022","doi":"10.18653/v1/2021.emnlp-main.526","title":"Towards Zero-Shot Knowledge Distillation for Natural Language Processing","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Computer science; Distillation; Task (project management); Benchmark (surveying); Artificial intelligence; Knowledge transfer; Natural language processing; Transfer of learning; Domain knowledge; Variety (cybernetics); Shot (pellet); Machine learning; Domain (mathematical analysis); Knowledge management; Engineering","score_opus":0.08645918045858457,"score_gpt":0.43361835862493486,"score_spread":0.3471591781663503,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3120832022","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1905611,0.051287748,0.7307347,0.008224932,0.002156371,0.0014370196,0.000014652728,0.00039260485,0.015190856],"genre_scores_gemma":[0.6142212,0.000008536137,0.38495246,0.00019955976,0.00013007359,0.000034524703,0.0000030057174,0.00001990828,0.0004307117],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99706334,0.00013088551,0.00070938846,0.0009708176,0.0005270311,0.0005985322],"domain_scores_gemma":[0.9977541,0.00031450202,0.00046953396,0.0003396566,0.0010184111,0.00010382273],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0013285801,0.00038496582,0.00059069716,0.00021300408,0.00025489365,0.000499028,0.0014221527,0.000209614,0.000012403382],"category_scores_gemma":[0.0032070745,0.0002803313,0.00021534758,0.0015336786,0.0001243279,0.00083383406,0.0006889859,0.000869882,0.0000015020086],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000036701902,0.00009015189,0.0003310889,0.00060374883,0.0000120885425,0.0000043116283,0.010251276,0.000015307478,0.07508254,0.0038357514,0.000031381496,0.90970564],"study_design_scores_gemma":[0.00076117826,0.00005352912,0.0013310539,0.0014792483,0.000039234634,0.000040276977,0.0029607364,0.81705743,0.16615693,0.009350971,0.00019211783,0.0005772885],"about_ca_topic_score_codex":0.0000126149935,"about_ca_topic_score_gemma":0.0000113038095,"teacher_disagreement_score":0.90912837,"about_ca_system_score_codex":0.0001943538,"about_ca_system_score_gemma":0.00052634027,"threshold_uncertainty_score":0.9999649},"labels":[],"label_agreement":null},{"id":"W3152698349","doi":"10.18653/v1/2021.emnlp-main.230","title":"Masked Language Modeling and the Distributional Hypothesis: Order Word Matters Pre-training for Little","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":182,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Computer science; Word (group theory); Natural language processing; Artificial intelligence; Word order; Language model; Downstream (manufacturing); Order (exchange); Parametric statistics; Linguistics; Mathematics","score_opus":0.07636691541448742,"score_gpt":0.38622034162426394,"score_spread":0.3098534262097765,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3152698349","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.18909124,0.004977314,0.76629406,0.03801954,0.00032757383,0.0005947074,0.000010680456,0.00007990595,0.00060497364],"genre_scores_gemma":[0.5688958,0.000008683237,0.43004707,0.0007740436,0.00006291036,0.00004386071,0.0000012252092,0.000009917489,0.00015646307],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.997997,0.00012891719,0.00045948048,0.0005843429,0.00043907177,0.00039119244],"domain_scores_gemma":[0.99812096,0.0008075233,0.00023246577,0.00023417597,0.00054546853,0.000059383074],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0018708508,0.0002261407,0.00040330164,0.00007884572,0.0002148773,0.00036855292,0.00091182423,0.00011151393,0.000007884013],"category_scores_gemma":[0.0044719214,0.00013779076,0.00011663786,0.00071158423,0.00015897393,0.0003659466,0.0004990299,0.0005436991,3.2089687e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00020650224,0.00007319539,0.0002971615,0.00043674922,0.000047844365,0.0000045183388,0.035891727,0.00037527792,0.030542105,0.026374882,0.000036464517,0.90571356],"study_design_scores_gemma":[0.0008619824,0.000012325301,0.00016280483,0.0005348476,0.000026482228,0.00002456478,0.007942137,0.9533445,0.011306745,0.025527125,0.00003071303,0.0002257317],"about_ca_topic_score_codex":0.000014254026,"about_ca_topic_score_gemma":0.0000037589691,"teacher_disagreement_score":0.95296925,"about_ca_system_score_codex":0.00006224908,"about_ca_system_score_gemma":0.0002046844,"threshold_uncertainty_score":0.5618944},"labels":[],"label_agreement":null},{"id":"W3152747197","doi":"10.18653/v1/2021.emnlp-main.234","title":"Linguistic Dependencies and Statistical Dependence","year":2021,"lang":"en","type":"preprint","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Language and cultural evolution","field":"Social Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Dependency (UML); Pointwise mutual information; Pointwise; Computer science; Context (archaeology); Natural language processing; ENCODE; Artificial intelligence; Simple (philosophy); Linguistics; Mathematics; Mutual information; Geography","score_opus":0.06719722973920086,"score_gpt":0.44821170864457643,"score_spread":0.38101447890537554,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3152747197","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8854271,0.05347152,0.004001977,0.0060669654,0.0023649428,0.0017207322,0.000057504116,0.00020065837,0.046688594],"genre_scores_gemma":[0.89089835,0.0002990557,0.1077754,0.00020513088,0.00026651792,0.00003345385,0.0000069527714,0.0000188002,0.0004963257],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.9968927,0.00036521378,0.0005656108,0.0007877686,0.0008700239,0.000518674],"domain_scores_gemma":[0.9978799,0.00049250055,0.0004842958,0.0001712188,0.0008284105,0.0001436615],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0018928474,0.00035695918,0.00063928403,0.00012846048,0.00036814096,0.0005529243,0.0008518521,0.00049585034,0.0001389382],"category_scores_gemma":[0.013341642,0.00024801394,0.00013084739,0.00059718365,0.0006588843,0.00024555536,0.0009597686,0.0019565737,0.0000010895079],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00014906295,0.00023622983,0.0071284296,0.0020920632,0.00007269245,0.000065024244,0.2518945,0.000009264505,0.021324335,0.018744383,0.00009850261,0.6981855],"study_design_scores_gemma":[0.0026278005,0.00037990286,0.055977877,0.030537125,0.001220928,0.00013822176,0.6095311,0.05497144,0.060452744,0.17668968,0.0014277826,0.006045369],"about_ca_topic_score_codex":0.0009217671,"about_ca_topic_score_gemma":0.00062889134,"teacher_disagreement_score":0.69214016,"about_ca_system_score_codex":0.00023097674,"about_ca_system_score_gemma":0.0007049905,"threshold_uncertainty_score":0.9999972},"labels":[],"label_agreement":null},{"id":"W3153046263","doi":"10.18653/v1/2021.emnlp-main.168","title":"Neural Path Hunter: Reducing Hallucination in Dialogue Systems via Path Grounding","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":75,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Institute for Advanced Research; University of Alberta","funders":"Alberta Machine Intelligence Institute; Compute Canada; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Computer science; Path (computing); Security token; Artificial neural network; Focus (optics); Artificial intelligence; Suite; Graph; Deep neural networks; Machine learning; Theoretical computer science; History","score_opus":0.055657809110537304,"score_gpt":0.3775607902916954,"score_spread":0.3219029811811581,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3153046263","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.62996,0.008041443,0.35244358,0.0035547055,0.0020975778,0.00065350666,0.000002554086,0.00014634685,0.0031002976],"genre_scores_gemma":[0.7652932,0.000014913778,0.23432404,0.00015586673,0.00010109196,0.000024593277,0.0000012546035,0.000014770925,0.000070299764],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99704653,0.0002790141,0.00076402735,0.0008446011,0.0005632493,0.0005026054],"domain_scores_gemma":[0.9984741,0.00029180315,0.00044280785,0.00030072228,0.00041643472,0.0000741248],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0019093886,0.00028656068,0.0005212372,0.0002752946,0.00013057402,0.00045880975,0.0011451626,0.00017597845,0.0000056605527],"category_scores_gemma":[0.0018802838,0.00021975127,0.000102878104,0.0015725876,0.00006375878,0.0009438494,0.00061167555,0.0009892657,8.689562e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002654382,0.00014552184,0.0058254013,0.0006139281,0.000011125509,0.000042454365,0.012920466,0.0005030039,0.12665172,0.0065711197,0.0000070410956,0.84668165],"study_design_scores_gemma":[0.00037322013,0.000033428125,0.003220503,0.0017020814,0.00000958943,0.000054738422,0.0019764758,0.9759703,0.013618458,0.0027321174,0.0000081442195,0.00030091923],"about_ca_topic_score_codex":0.00010942011,"about_ca_topic_score_gemma":0.000012181859,"teacher_disagreement_score":0.9754673,"about_ca_system_score_codex":0.0002579718,"about_ca_system_score_gemma":0.00017445357,"threshold_uncertainty_score":0.89611965},"labels":[],"label_agreement":null},{"id":"W3197275000","doi":"10.18653/v1/2021.emnlp-main.181","title":"Unsupervised Conversation Disentanglement through Co-Training","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Conversation; Computer science; Classifier (UML); Session (web analytics); Artificial intelligence; Machine learning; Reinforcement learning; Training set; Speech recognition; World Wide Web","score_opus":0.11382687623688474,"score_gpt":0.4306470469350119,"score_spread":0.3168201706981271,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3197275000","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.43537003,0.0062070945,0.500186,0.028671913,0.0015600033,0.0009410989,0.000007579742,0.00027798014,0.026778262],"genre_scores_gemma":[0.607336,0.00001357089,0.3918849,0.00060676114,0.000051039497,0.000011791305,0.000001250987,0.000008463164,0.00008621411],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.997815,0.00012802299,0.0005064066,0.00063182856,0.0005403863,0.00037833233],"domain_scores_gemma":[0.99880373,0.00021127253,0.00028037935,0.00026255831,0.0003817378,0.000060335296],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008683087,0.0002234103,0.00036635366,0.00008999485,0.00012578923,0.00024693718,0.000992402,0.0001086068,0.000046453468],"category_scores_gemma":[0.0011015233,0.0001652533,0.00011474552,0.0009596451,0.00009094131,0.0006800954,0.00041499938,0.0005677343,0.0000020517325],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002758117,0.00012359773,0.0037938124,0.00029576445,0.000024954204,0.000014540935,0.05404985,0.00003052326,0.15512958,0.020970374,0.00004038207,0.76549906],"study_design_scores_gemma":[0.00121523,0.000064813925,0.0024153185,0.0014376863,0.00003535653,0.00004029653,0.02620299,0.44262588,0.4933661,0.031691123,0.00026579204,0.0006394196],"about_ca_topic_score_codex":0.0000114251125,"about_ca_topic_score_gemma":0.0000030880537,"teacher_disagreement_score":0.7648596,"about_ca_system_score_codex":0.00012883,"about_ca_system_score_gemma":0.00025799207,"threshold_uncertainty_score":0.6738834},"labels":[],"label_agreement":null},{"id":"W3198457396","doi":"10.18653/v1/2021.emnlp-main.259","title":"Learning from Multiple Noisy Augmented Data Sets for Better Cross-Lingual Spoken Language Understanding","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Simon Fraser University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Benchmark (surveying); Focus (optics); Noise (video); Code (set theory); Artificial intelligence; Training set; Machine learning; Spoken language; Resource (disambiguation); Noise reduction; Natural language processing; Speech recognition; Image (mathematics); Set (abstract data type)","score_opus":0.14898561212540867,"score_gpt":0.4560963868956432,"score_spread":0.3071107747702345,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3198457396","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44080988,0.02410701,0.5258783,0.005320973,0.0009976893,0.0012054,0.00010439155,0.0007135327,0.0008628137],"genre_scores_gemma":[0.53064686,0.000010007207,0.46862525,0.00039632712,0.00010510193,0.000021675607,0.000038275364,0.000029740118,0.0001267683],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99609756,0.00020898915,0.0007934426,0.0014590296,0.0007228011,0.00071819016],"domain_scores_gemma":[0.99654776,0.0013769855,0.0006789412,0.0006639127,0.0006115467,0.00012083983],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0018549068,0.00047639856,0.00069077546,0.00024828562,0.0003758682,0.0009428226,0.003264668,0.00031638067,0.00003166546],"category_scores_gemma":[0.009259205,0.00035984613,0.00017977551,0.0014481993,0.00021416224,0.0013927897,0.0022947222,0.0015283681,0.0000016626717],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022591883,0.00021453477,0.011999776,0.00060842576,0.00008400754,0.00005283427,0.014338124,0.000012536128,0.28329906,0.0015489712,0.00012318512,0.6874926],"study_design_scores_gemma":[0.0017002478,0.00011197763,0.0010342674,0.0026053076,0.0000780221,0.000036663074,0.0079550855,0.43087,0.5311243,0.023486817,0.000081262675,0.0009160277],"about_ca_topic_score_codex":0.000064845866,"about_ca_topic_score_gemma":0.00003291322,"teacher_disagreement_score":0.6865766,"about_ca_system_score_codex":0.00026491712,"about_ca_system_score_gemma":0.0003164459,"threshold_uncertainty_score":0.9998854},"labels":[],"label_agreement":null},{"id":"W3200130628","doi":"10.18653/v1/2021.emnlp-main.122","title":"Conditional probing: measuring usable information beyond a baseline","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Baseline (sea); USable; Representation (politics); Computer science; Word (group theory); Property (philosophy); Identity (music); Natural language processing; Artificial intelligence; Speech recognition; Mathematics","score_opus":0.05992423051901477,"score_gpt":0.36906375505046807,"score_spread":0.3091395245314533,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3200130628","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07204081,0.004151265,0.89369583,0.011621268,0.0007981763,0.00061849697,0.000008526075,0.00019067095,0.016874932],"genre_scores_gemma":[0.562755,0.000010007928,0.43641344,0.00062356493,0.000050154133,0.000017848599,0.0000027468761,0.000006057494,0.00012118188],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979332,0.00011275773,0.0005825192,0.00042434112,0.0006028978,0.00034430492],"domain_scores_gemma":[0.9983127,0.00021149674,0.00035170984,0.00022159849,0.00083161227,0.000070872484],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0016360783,0.00020624154,0.00031942694,0.0001802948,0.00015200781,0.00035910512,0.00092678814,0.00011712633,0.00003433302],"category_scores_gemma":[0.0022997512,0.00015371575,0.00009881689,0.0011551871,0.00007211012,0.0015964272,0.00040735686,0.0006685505,0.0000035074063],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000028851404,0.000117081196,0.0010805606,0.00047674213,0.000018490788,0.0000064240367,0.0068681315,0.00026317016,0.08288294,0.034350622,0.000076714976,0.87383026],"study_design_scores_gemma":[0.00057473587,0.000029215591,0.00060442305,0.0008219858,0.000015828471,0.000055564524,0.0013905724,0.7257151,0.23061241,0.039471637,0.00033700577,0.00037151962],"about_ca_topic_score_codex":0.0000070111378,"about_ca_topic_score_gemma":0.000002072315,"teacher_disagreement_score":0.87345874,"about_ca_system_score_codex":0.0000972817,"about_ca_system_score_gemma":0.00031834812,"threshold_uncertainty_score":0.62683463},"labels":[],"label_agreement":null},{"id":"W3200172683","doi":"10.18653/v1/2021.emnlp-main.516","title":"Mind the Context: The Impact of Contextualization in Neural Module Networks for Grounding Visual Referring Expressions","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Contextualization; Computer science; Generalization; Context (archaeology); Set (abstract data type); Exploit; Parameterized complexity; Artificial intelligence; Test set; Embedding; Cube (algebra); Machine learning; Algorithm; Mathematics; Programming language","score_opus":0.06160955148695172,"score_gpt":0.46076189392771477,"score_spread":0.39915234244076303,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3200172683","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7771295,0.0023884634,0.21577853,0.0034026213,0.0001741653,0.0007532968,0.0000034566315,0.00002680726,0.0003431587],"genre_scores_gemma":[0.94096375,0.000007461574,0.05866545,0.00015411808,0.000058563495,0.00008852058,0.0000021254234,0.0000149499365,0.00004506943],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99802566,0.000286964,0.000576772,0.00046347428,0.00029910463,0.00034800547],"domain_scores_gemma":[0.99729437,0.0013682339,0.00053393893,0.0002793272,0.00047931157,0.00004482003],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0015876108,0.00021828129,0.00037177064,0.0001176688,0.00025774882,0.00022227729,0.0012608126,0.00011509023,0.000008913917],"category_scores_gemma":[0.0035043857,0.00011241762,0.00017693908,0.0014430127,0.00015899647,0.00035705577,0.00047821773,0.00082958216,1.915006e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008803403,0.00018873761,0.01570872,0.00010718204,0.000027047416,0.0000011122762,0.013974718,0.0047122845,0.081775874,0.006997934,0.000018553397,0.8763998],"study_design_scores_gemma":[0.00036658603,0.000045202207,0.031917617,0.0004655492,0.000011480159,0.000008360422,0.0025869687,0.95139205,0.011768366,0.0012924487,0.0000067968585,0.00013855698],"about_ca_topic_score_codex":0.00017905192,"about_ca_topic_score_gemma":0.000031433072,"teacher_disagreement_score":0.9466798,"about_ca_system_score_codex":0.00011061916,"about_ca_system_score_gemma":0.00015226178,"threshold_uncertainty_score":0.45842573},"labels":[],"label_agreement":null},{"id":"W3200285169","doi":"10.18653/v1/2021.emnlp-main.779","title":"PICARD: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models","year":2021,"lang":"en","type":"preprint","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; University of Toronto","funders":"","keywords":"Computer science; Decoding methods; Parsing; Language model; Natural language processing; Artificial intelligence; Code (set theory); Programming language; Compiler; Algorithm; Set (abstract data type)","score_opus":0.07064381815737095,"score_gpt":0.41447000363355646,"score_spread":0.3438261854761855,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3200285169","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.11116094,0.07104825,0.8051189,0.004927959,0.0017632968,0.0028646134,0.00014504201,0.0008216914,0.0021492823],"genre_scores_gemma":[0.47375345,0.00003715678,0.52538437,0.00042963607,0.00013886682,0.00013327863,0.000025393821,0.00004665595,0.000051170337],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99412996,0.00032145123,0.0014149236,0.0020824054,0.0010842885,0.00096695],"domain_scores_gemma":[0.9949386,0.0009993778,0.0018072284,0.0008305313,0.0012363284,0.00018789788],"candidate_categories":["metaepi_narrow","scholarly_communication","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.002074232,0.0009895816,0.001612364,0.000519568,0.0003218084,0.0015184209,0.0045733307,0.0008734342,0.000024176128],"category_scores_gemma":[0.003512418,0.0007533261,0.0005878755,0.0012356442,0.0003693929,0.0012156512,0.0040618777,0.0030769226,5.450121e-7],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00015738013,0.0002309112,0.00028955893,0.0015787422,0.00015073887,0.00004285729,0.022532279,0.000073529365,0.24129973,0.004694725,0.000093513394,0.728856],"study_design_scores_gemma":[0.00086749427,0.00007771164,0.00008703404,0.010017151,0.00014385671,0.000025342779,0.0048919083,0.5860032,0.31734395,0.07935237,0.000008317571,0.0011816536],"about_ca_topic_score_codex":0.00018125711,"about_ca_topic_score_gemma":0.00002823347,"teacher_disagreement_score":0.72767437,"about_ca_system_score_codex":0.0005212605,"about_ca_system_score_gemma":0.0009505381,"threshold_uncertainty_score":0.9995181},"labels":[],"label_agreement":null},{"id":"W3200853751","doi":"10.18653/v1/2021.emnlp-main.71","title":"Predicting emergent linguistic compositions through time: Syntactic frame extension via multimodal chaining","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Language, Metaphor, and Cognition","field":"Psychology","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Vector Institute; University of Toronto","funders":"Social Sciences and Humanities Research Council of Canada; Natural Sciences and Engineering Research Council of Canada; University of Toronto","keywords":"Computer science; Chaining; Natural language processing; Artificial intelligence; Linguistics; Noun; Psychology","score_opus":0.05792580951332454,"score_gpt":0.4257146922206785,"score_spread":0.36778888270735394,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3200853751","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.93022156,0.011027136,0.013380217,0.0022680191,0.0022733472,0.0007757909,0.000024664942,0.00021955493,0.039809696],"genre_scores_gemma":[0.8924829,0.000018391163,0.105897486,0.0006321503,0.00030781963,0.00005037672,0.000014246161,0.00004150176,0.00055513595],"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99711007,0.00032466304,0.0007249986,0.0007983195,0.0004948641,0.00054709165],"domain_scores_gemma":[0.9977543,0.00055866624,0.0005129711,0.00026059337,0.0008179729,0.00009554067],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009603692,0.00035624657,0.0005887008,0.00015279355,0.00030384742,0.00012333185,0.0004369729,0.00027460753,0.0007537924],"category_scores_gemma":[0.0038715096,0.00026906832,0.0002133709,0.0010951961,0.00016808951,0.00023392295,0.00025254342,0.0013010103,0.00002180495],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00038286534,0.0009096667,0.0040675844,0.00050438696,0.00016752024,0.00010464869,0.06421575,0.000013831797,0.63533586,0.0026609432,0.000078709025,0.29155824],"study_design_scores_gemma":[0.005494845,0.00077599665,0.03511653,0.011351194,0.0013699355,0.00094199047,0.10278655,0.30687657,0.47960335,0.052164905,0.00033304142,0.0031851148],"about_ca_topic_score_codex":0.000087187575,"about_ca_topic_score_gemma":0.0000067462993,"teacher_disagreement_score":0.3068627,"about_ca_system_score_codex":0.00010474414,"about_ca_system_score_gemma":0.00011188749,"threshold_uncertainty_score":0.99997616},"labels":[],"label_agreement":null},{"id":"W3201341200","doi":"10.18653/v1/2021.emnlp-main.42","title":"Mitigating Language-Dependent Ethnic Bias in BERT","year":2021,"lang":"en","type":"preprint","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Kootenay Association for Science & Technology","funders":"Ministry of Science and ICT, South Korea; National Research Foundation of Korea; National Research Foundation","keywords":"Categorical variable; Computer science; Ethnic group; German; Turkish; Natural language processing; Metric (unit); Language model; Linguistics; Arabic; Gender bias; Word (group theory); Artificial intelligence; Psychology; Machine learning; Sociology; Social psychology","score_opus":0.13897384373430055,"score_gpt":0.44294323317028,"score_spread":0.30396938943597945,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3201341200","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.921169,0.024980145,0.041056376,0.005124675,0.001551102,0.0010095628,0.000005599928,0.00019017675,0.0049133534],"genre_scores_gemma":[0.6134404,0.00004622275,0.38579944,0.00036510278,0.00010384405,0.00005339186,0.0000022165614,0.000027441396,0.00016192807],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99493945,0.0004199713,0.0012634317,0.0015953397,0.0010288828,0.0007529196],"domain_scores_gemma":[0.9971314,0.0005690432,0.0009790106,0.0006969838,0.0005005389,0.000123007],"candidate_categories":["metaepi_narrow","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.00341118,0.0006098373,0.0010831157,0.0004929094,0.000104092425,0.00065598276,0.0034561965,0.00060019264,0.000028021605],"category_scores_gemma":[0.004886336,0.00046947322,0.00029442896,0.0016285192,0.00014065276,0.0005375389,0.004271333,0.004159542,0.0000015110885],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024069146,0.00017065047,0.00330039,0.0014050343,0.000029944813,0.00005567894,0.048187338,0.0003724118,0.0475841,0.0009772104,0.000007799454,0.8978854],"study_design_scores_gemma":[0.0008848872,0.0000528455,0.004773921,0.012559289,0.00005037079,0.00005099981,0.017086415,0.80579907,0.14614752,0.011273537,0.000006764019,0.0013143765],"about_ca_topic_score_codex":0.00027259244,"about_ca_topic_score_gemma":0.00011052002,"teacher_disagreement_score":0.896571,"about_ca_system_score_codex":0.00036253204,"about_ca_system_score_gemma":0.00061795313,"threshold_uncertainty_score":0.9997757},"labels":[],"label_agreement":null},{"id":"W3203338521","doi":"10.18653/v1/2021.emnlp-main.328","title":"Language-Aligned Waypoint (LAW) Supervision for Vision-and-Language Navigation in Continuous Environments","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":39,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Simon Fraser University","funders":"Western Canada Research Grid; Compute Canada; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Waypoint; Computer science; Task (project management); Human–computer interaction; Embodied cognition; Path (computing); Natural language; Work (physics); Measure (data warehouse); Metric (unit); Shortest path problem; Artificial intelligence; Multimedia; Programming language; Real-time computing; Engineering; Theoretical computer science; Database","score_opus":0.026221236821911142,"score_gpt":0.3976567197951307,"score_spread":0.37143548297321954,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3203338521","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95542055,0.0035468012,0.033580136,0.0054782904,0.00014735707,0.0007318463,0.0000064785386,0.00006364192,0.0010249206],"genre_scores_gemma":[0.68806607,0.000011220822,0.31140107,0.00031142888,0.000028916706,0.0000620965,0.000005809175,0.000014803764,0.00009858154],"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99781007,0.00016589418,0.0005381184,0.00074742857,0.0003774822,0.00036099317],"domain_scores_gemma":[0.9986753,0.00050186424,0.00029430623,0.00029112352,0.00015977194,0.00007767023],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0012299895,0.00026055935,0.00043381331,0.00013839352,0.0001489283,0.00023176106,0.0007892852,0.0001559829,0.000011625351],"category_scores_gemma":[0.0015611261,0.0001927003,0.000100444246,0.0007765341,0.00012331865,0.00043543262,0.00055074284,0.000525122,0.0000017723673],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003682004,0.00016050955,0.0016719733,0.00022782115,0.000007340699,0.000006621366,0.01576307,0.000017833077,0.49917465,0.0060883234,0.0000062364543,0.4768388],"study_design_scores_gemma":[0.0018901702,0.00017035185,0.02438025,0.0018473371,0.0000330355,0.000047828788,0.009068174,0.48672062,0.46417862,0.0108879255,0.0001040176,0.0006716737],"about_ca_topic_score_codex":0.000150018,"about_ca_topic_score_gemma":0.0000143421075,"teacher_disagreement_score":0.4867028,"about_ca_system_score_codex":0.00010405682,"about_ca_system_score_gemma":0.00007817574,"threshold_uncertainty_score":0.78580904},"labels":[],"label_agreement":null},{"id":"W3209039755","doi":"10.18653/v1/2021.emnlp-main.821","title":"IndoNLI: A Natural Language Inference Dataset for Indonesian","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Universitas Indonesia; York University; Samsung; National Science Foundation","keywords":"Annotation; Indonesian; Sentence; Test set; Set (abstract data type); Inference; Data set","score_opus":0.06648941278896932,"score_gpt":0.43358885394360996,"score_spread":0.36709944115464066,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3209039755","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.48122483,0.024016103,0.46411476,0.021123433,0.0029697416,0.0024204864,0.00028274357,0.00037496397,0.0034729294],"genre_scores_gemma":[0.5931575,0.00000937684,0.4058751,0.0006722309,0.000085953776,0.000039061564,0.000019489973,0.000014566608,0.00012673397],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9971326,0.0001426829,0.00064917293,0.00093871227,0.0005470959,0.00058972545],"domain_scores_gemma":[0.99781746,0.0005991604,0.00042227667,0.00049702247,0.0005575809,0.000106494685],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0011891038,0.00035333005,0.0005679241,0.00021566618,0.00017660006,0.00040956322,0.00204899,0.00019113263,0.000016412954],"category_scores_gemma":[0.004242244,0.00025638888,0.00015444477,0.0014218155,0.0001262284,0.0008144082,0.0010016627,0.0010353437,0.0000017891622],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000060705057,0.00014500585,0.0016916188,0.00048374326,0.000025231011,0.000018798342,0.007102371,0.000013537218,0.10528915,0.0072734295,0.00013712312,0.8777593],"study_design_scores_gemma":[0.0018937938,0.000118705495,0.0029914184,0.0018981792,0.00006099793,0.00009603138,0.0064839455,0.657691,0.31473932,0.012254466,0.00062939234,0.0011427263],"about_ca_topic_score_codex":0.000024839108,"about_ca_topic_score_gemma":0.000021472986,"teacher_disagreement_score":0.87661654,"about_ca_system_score_codex":0.00010720356,"about_ca_system_score_gemma":0.00039760428,"threshold_uncertainty_score":0.99998885},"labels":[],"label_agreement":null},{"id":"W3211384195","doi":"10.18653/v1/2021.emnlp-main.130","title":"Translation-based Supervision for Policy Generation in Simultaneous Neural Machine Translation","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Simon Fraser University","funders":"","keywords":"Computer science; Machine translation; Artificial intelligence; Translation (biology); Security token; Oracle; Reinforcement learning; Heuristic; Action (physics); Inference; Machine learning; Quality (philosophy); Sentence; Natural language processing; Programming language","score_opus":0.07715591203049524,"score_gpt":0.42118249622465037,"score_spread":0.3440265841941551,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3211384195","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.041468292,0.054860458,0.86638176,0.034387883,0.00043190556,0.001632315,0.000023284536,0.00031597132,0.0004981517],"genre_scores_gemma":[0.5263477,0.000009176552,0.47311282,0.0004008157,0.000059569305,0.000028524984,0.000008839016,0.0000134821075,0.000019076557],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9975741,0.00017658532,0.0006664563,0.0007102178,0.00046275297,0.0004098934],"domain_scores_gemma":[0.9979856,0.00085919746,0.0002883534,0.0002352291,0.0005692727,0.00006236022],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010495918,0.00030527214,0.00043782534,0.00042402925,0.00013908527,0.00028622829,0.0009426899,0.00022610702,0.0000074097024],"category_scores_gemma":[0.0031108817,0.00022842271,0.0001566325,0.0022344969,0.00008386987,0.0006805769,0.00011089519,0.00068409316,2.5485866e-7],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005781275,0.00009037625,0.00019657765,0.00021789414,0.000003210317,0.0000051601533,0.0018182662,0.00037704298,0.18315478,0.0011332423,0.0000033494255,0.81294227],"study_design_scores_gemma":[0.0005604483,0.00005151132,0.00004465067,0.00029860283,0.000008282763,0.000006755138,0.00009840327,0.7843733,0.20909731,0.005218614,0.000023853103,0.00021826463],"about_ca_topic_score_codex":0.00004663869,"about_ca_topic_score_gemma":0.00006829936,"teacher_disagreement_score":0.81272405,"about_ca_system_score_codex":0.00016590752,"about_ca_system_score_gemma":0.0003734848,"threshold_uncertainty_score":0.93148077},"labels":[],"label_agreement":null},{"id":"W3211439810","doi":"10.18653/v1/2021.emnlp-main.835","title":"Types of Out-of-Distribution Texts and How to Detect Them","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"York University","keywords":"Computer science; Categorization; Artificial intelligence; Calibration; Natural language processing; Data mining; Statistics; Mathematics","score_opus":0.0736749441821645,"score_gpt":0.3899331424867225,"score_spread":0.31625819830455804,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3211439810","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5934435,0.0041736877,0.39328957,0.006707844,0.0003993262,0.00032166083,0.00000781529,0.000043408832,0.0016131924],"genre_scores_gemma":[0.6291451,0.0000073164897,0.37071267,0.0000510056,0.000017085107,0.000003390461,2.6422725e-7,0.0000037964571,0.000059385584],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99878806,0.00007261799,0.00027908324,0.0003694251,0.0002969694,0.00019385127],"domain_scores_gemma":[0.9988232,0.00019954903,0.00025520657,0.00017177852,0.00050136173,0.00004886835],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00075882574,0.00013590329,0.00033354806,0.00006521167,0.000042543837,0.00008310398,0.0006012589,0.0000874218,0.0000051741877],"category_scores_gemma":[0.002204647,0.00009509638,0.000060164322,0.0007099042,0.000074263015,0.00022906414,0.00048277466,0.00029564038,2.7203842e-7],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001725014,0.000024475641,0.00082618196,0.0002549466,0.000008080502,0.0000012861827,0.0033025302,0.0000058730675,0.19622058,0.0035319666,0.000010486442,0.79579633],"study_design_scores_gemma":[0.00023419443,0.00005779259,0.0017704153,0.0009370066,0.000017431435,0.000016878112,0.0011056446,0.059098355,0.9247481,0.011775164,0.000041837207,0.0001971791],"about_ca_topic_score_codex":0.0000044612493,"about_ca_topic_score_gemma":0.0000028348138,"teacher_disagreement_score":0.79559916,"about_ca_system_score_codex":0.000033005384,"about_ca_system_score_gemma":0.00011322375,"threshold_uncertainty_score":0.38779178},"labels":[],"label_agreement":null},{"id":"W3213180921","doi":"10.18653/v1/2021.emnlp-main.603","title":"Universal-KD: Attention-based Output-Grounded Intermediate Layer Knowledge Distillation","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":18,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Layer (electronics); Distillation; Matching (statistics); Interpretability; Projection (relational algebra); Base (topology); Space (punctuation); Architecture; Artificial intelligence; Deep learning; Computer architecture; Machine learning; Algorithm; Mathematics; Nanotechnology; Chromatography; Operating system; Materials science; Chemistry","score_opus":0.07139261601228206,"score_gpt":0.3974343703737435,"score_spread":0.32604175436146143,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3213180921","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.34356523,0.003309703,0.63129365,0.006508523,0.00137135,0.00044017236,0.0000044757517,0.00019277618,0.0133141205],"genre_scores_gemma":[0.7555471,0.0000048563156,0.24373057,0.00017276227,0.00006304575,0.000007893663,0.000001577716,0.000011241461,0.00046091416],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99799776,0.00016112166,0.00046506617,0.00064523786,0.0003916377,0.0003391556],"domain_scores_gemma":[0.9983684,0.00029903685,0.00032434365,0.00027150643,0.0006595681,0.00007716658],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008527695,0.00023675608,0.00035837354,0.00019084726,0.00012900961,0.00027003567,0.0010213733,0.00014112894,0.000019280726],"category_scores_gemma":[0.0013985368,0.00017683407,0.00014414817,0.001190857,0.00010360875,0.0005192872,0.0004914308,0.00061713386,0.0000034379834],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000044302666,0.0001948558,0.005383664,0.00040794184,0.000026256668,0.000011988434,0.0054624905,0.000083853774,0.066367194,0.011968293,0.00003621514,0.91001296],"study_design_scores_gemma":[0.00070802664,0.00003823698,0.011042436,0.0010908477,0.000032044547,0.000009347238,0.0015741579,0.92532575,0.052498385,0.0071804635,0.00011466854,0.0003856334],"about_ca_topic_score_codex":0.000011929927,"about_ca_topic_score_gemma":0.000012571486,"teacher_disagreement_score":0.9252419,"about_ca_system_score_codex":0.00018139777,"about_ca_system_score_gemma":0.0003234548,"threshold_uncertainty_score":0.7211084},"labels":[],"label_agreement":null},{"id":"W3214455632","doi":"10.18653/v1/2021.emnlp-main.77","title":"Contextualized Query Embeddings for Conversational Search","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":41,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Canada First Research Excellence Fund","keywords":"Computer science; Leverage (statistics); Inference; Security token; Relevance (law); Pipeline (software); Information retrieval; Query expansion; Query language; Artificial intelligence","score_opus":0.092333595524727,"score_gpt":0.43953950920379287,"score_spread":0.34720591367906584,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3214455632","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.16445453,0.0043315613,0.80447227,0.018419666,0.0009631083,0.0009185073,0.000010290694,0.00014777896,0.006282305],"genre_scores_gemma":[0.5235511,0.0000062723507,0.4754031,0.00058032956,0.000058104437,0.000024740419,0.0000011449099,0.000009250772,0.0003659665],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978356,0.000113370326,0.00049764535,0.00065516366,0.00050295884,0.000395313],"domain_scores_gemma":[0.99783945,0.00062824326,0.00024118759,0.00022244625,0.0009941956,0.00007447814],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0015283576,0.00020929985,0.00040646305,0.00014104442,0.00014253538,0.00026635116,0.0010952986,0.0001382064,0.000033920034],"category_scores_gemma":[0.0029334442,0.00015548433,0.00015743493,0.0008547008,0.00011436923,0.00049997173,0.0004836491,0.000560144,0.0000012780304],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010311058,0.00013931197,0.0014744961,0.00052654097,0.000033502452,0.0000062058125,0.01286653,0.000039792147,0.18560295,0.09630091,0.00011956037,0.7027871],"study_design_scores_gemma":[0.001147327,0.00004950101,0.0006241996,0.00073326775,0.000018150831,0.000027199912,0.0041156258,0.69759446,0.27195448,0.023139939,0.00022312935,0.00037269693],"about_ca_topic_score_codex":0.000011178025,"about_ca_topic_score_gemma":0.0000028225627,"teacher_disagreement_score":0.7024144,"about_ca_system_score_codex":0.00010968695,"about_ca_system_score_gemma":0.00040461705,"threshold_uncertainty_score":0.63404673},"labels":[],"label_agreement":null},{"id":"W4231122779","doi":"10.18653/v1/2021.emnlp-main.288","title":"Automated Generation of Accurate & Fluent Medical X-ray Reports","year":2021,"lang":"en","type":"article","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","topic":"Topic Modeling","field":"Computer Science","cited_by":38,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Fluency; Embedding; Generator (circuit theory); Transformer; Natural language processing; Artificial intelligence; Information retrieval; Linguistics","score_opus":0.09840819144765348,"score_gpt":0.4311545131288209,"score_spread":0.33274632168116747,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4231122779","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6914437,0.0046860916,0.28919226,0.009736587,0.0013887722,0.0004262011,0.000001946038,0.0002790397,0.0028454214],"genre_scores_gemma":[0.59064543,0.000013056849,0.40890208,0.00025392364,0.00006156299,0.000008629899,0.0000012468282,0.000007533083,0.00010650193],"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9970958,0.0001773294,0.0008771544,0.0006425262,0.00089733326,0.00030982905],"domain_scores_gemma":[0.997941,0.00019812622,0.00060991687,0.00035369585,0.0008021703,0.000095089774],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0020992893,0.00021072701,0.00045113117,0.00014069985,0.000085482396,0.00015365363,0.0009055932,0.0001951491,0.000055270302],"category_scores_gemma":[0.006296829,0.00014745083,0.0001156545,0.0011900491,0.00010448968,0.00043011623,0.0006095886,0.0006226657,7.792056e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015499445,0.00018842366,0.0016507838,0.00037197795,0.000026342928,0.000044101333,0.0052281395,0.00022071236,0.51846725,0.003524655,0.0001438938,0.47011822],"study_design_scores_gemma":[0.00020895716,0.000019176057,0.00083824655,0.00072994793,0.000013306457,0.00007523274,0.0003159424,0.79576975,0.19972742,0.0020168112,0.00007821244,0.00020701089],"about_ca_topic_score_codex":0.000017344173,"about_ca_topic_score_gemma":0.00001060311,"teacher_disagreement_score":0.79554904,"about_ca_system_score_codex":0.00007274555,"about_ca_system_score_gemma":0.000476146,"threshold_uncertainty_score":0.7538347},"labels":[],"label_agreement":null}]}