{"meta":{"query_hash":"7a3547c421a9","filters":{"venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)"},"cohort_total":6,"direct_labels_cover":0,"predictions_cover":6,"exported":6,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/7a3547c421a9","api":"https://metacan.xera.ac/api/v1/cohort?venue=2021+36th+IEEE%2FACM+International+Conference+on+Automated+Software+Engineering+%28ASE%29"},"results":[{"id":"W3196824794","doi":"10.1109/ase51524.2021.9678596","title":"Leveraging Code Clones and Natural Language Processing for Log Statement Prediction","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Source code; Statement (logic); Debugging; Java; Context (archaeology); Programming language; Data mining; Database; Artificial intelligence; Information retrieval","score_opus":0.02070793589757158,"score_gpt":0.28549241407539305,"score_spread":0.26478447817782147,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196824794","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.29151326,0.0006711504,0.69955134,0.0010649115,0.0044932715,0.00042392357,0.00017550435,0.0020572739,0.000049369624],"genre_scores_gemma":[0.94881785,0.000062627754,0.050183322,0.00013499394,0.00019625886,0.00009517954,0.00018855627,0.000025296244,0.00029593796],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9980912,0.00003796307,0.00045493865,0.00061886234,0.00045495754,0.00034206398],"domain_scores_gemma":[0.9985705,0.00020429386,0.0001372812,0.00044198404,0.00053937593,0.00010658382],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00034288142,0.00026988995,0.0002553904,0.00017300797,0.00015982108,0.00039543177,0.00054806983,0.00010661291,0.000039358325],"category_scores_gemma":[0.00061967305,0.00025862342,0.00007581462,0.00024286746,0.000028576414,0.00056651817,0.00017989182,0.00024885492,0.000015255765],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00048960245,0.0017006164,0.056723956,0.0074278875,0.00221477,0.0012616696,0.034223985,0.21694441,0.084016845,0.014729983,0.018588342,0.56167793],"study_design_scores_gemma":[0.0006888503,0.00006241144,0.006659476,0.00053637783,0.000013655116,0.000063565116,0.00017888265,0.98361,0.006553419,0.00009085777,0.0012296516,0.00031290445],"about_ca_topic_score_codex":0.00001572753,"about_ca_topic_score_gemma":0.0000052422897,"teacher_disagreement_score":0.7666655,"about_ca_system_score_codex":0.00020729152,"about_ca_system_score_gemma":0.00023856797,"threshold_uncertainty_score":0.9999866},"labels":[],"label_agreement":null},{"id":"W4205513494","doi":"10.1109/ase51524.2021.9678640","title":"Is Historical Data an Appropriate Benchmark for Reviewer Recommendation Systems? : A Case Study of the Gerrit Community","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; Polytechnique Montréal; McGill University","funders":"","keywords":"Context (archaeology); Pessimism; Computer science; Benchmark (surveying); Replicate; Task (project management); Data science; History; Epistemology; Management","score_opus":0.1459378586426111,"score_gpt":0.3631536798642139,"score_spread":0.21721582122160277,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205513494","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.42436033,0.0002227622,0.5577116,0.00142473,0.011867951,0.0019524385,0.00079358526,0.0016258986,0.00004072053],"genre_scores_gemma":[0.9731131,0.000035850862,0.025690082,0.000086418986,0.0002015348,0.00021321708,0.00036010082,0.00005427002,0.0002453751],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9968402,0.00046456544,0.0007486186,0.0007575616,0.00081173686,0.00037735672],"domain_scores_gemma":[0.99284387,0.0015900192,0.00025397277,0.0035101245,0.0016282978,0.00017374141],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0014584186,0.00034074808,0.00045271596,0.00024629146,0.0002544318,0.00034375786,0.0038659591,0.00014541166,0.00012315215],"category_scores_gemma":[0.008897702,0.00030891312,0.000106723775,0.00070871034,0.000024017116,0.0006367081,0.0012852107,0.00076236675,0.000010936372],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00051110564,0.030868022,0.05423665,0.010536271,0.008833637,0.006679065,0.055373672,0.3209176,0.010403581,0.016834589,0.26878434,0.21602146],"study_design_scores_gemma":[0.00091320736,0.00029200132,0.0019542007,0.00048341235,0.000044726028,0.00031261204,0.00045822444,0.98970103,0.0003712407,0.000054500146,0.004971264,0.0004435856],"about_ca_topic_score_codex":0.0007942355,"about_ca_topic_score_gemma":0.00006435008,"teacher_disagreement_score":0.6687834,"about_ca_system_score_codex":0.00063256436,"about_ca_system_score_gemma":0.0003487794,"threshold_uncertainty_score":0.9999363},"labels":[],"label_agreement":null},{"id":"W4205678875","doi":"10.1109/ase51524.2021.9678871","title":"DeepMemory: Model-based Memorization Analysis of Deep Neural Language Models","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Perplexity; Memorization; Language model; Artificial intelligence; Artificial neural network; Machine learning; Data modeling; Robustness (evolution); Natural language processing; Database","score_opus":0.020935447441932053,"score_gpt":0.2813157815033691,"score_spread":0.26038033406143707,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205678875","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.03385437,0.000038879996,0.96338034,0.0003742805,0.0010225559,0.00010391765,0.00006217151,0.0009697759,0.0001936831],"genre_scores_gemma":[0.81639665,0.0000058534033,0.18285736,0.00014367737,0.0000620167,0.000018581157,0.0003681162,0.00003283482,0.00011493317],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9974718,0.00010094322,0.00054217415,0.00068547705,0.0008475365,0.00035206781],"domain_scores_gemma":[0.99758965,0.0003326032,0.00026136087,0.0009663867,0.00071026257,0.0001397099],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002559315,0.00034241704,0.00047924215,0.0007976549,0.00008544307,0.00020810607,0.0014585094,0.00015966588,0.00028749765],"category_scores_gemma":[0.0011955494,0.00038371765,0.00024364449,0.0015849795,0.00003395767,0.00056604866,0.00027801585,0.00039177245,0.000015145862],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011941777,0.00007088531,0.00021782506,0.00002722557,0.00037404586,0.00009183781,0.0005704781,0.9836665,0.0026245848,0.010187356,0.000024440482,0.002132921],"study_design_scores_gemma":[0.00042264853,0.000026610498,0.0005009705,0.00006984201,0.00013205895,0.000005173441,0.000041828902,0.9938751,0.0044075116,0.0001637672,0.000007091915,0.000347406],"about_ca_topic_score_codex":0.00003025627,"about_ca_topic_score_gemma":0.000013420788,"teacher_disagreement_score":0.7825422,"about_ca_system_score_codex":0.0001991317,"about_ca_system_score_gemma":0.00023114677,"threshold_uncertainty_score":0.9998615},"labels":[],"label_agreement":null},{"id":"W4205689130","doi":"10.1109/ase51524.2021.9678888","title":"SMARTIAN: Enhancing Smart Contract Fuzzing with Static and Dynamic Data-Flow Analyses","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":165,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Kootenay Association for Science & Technology","funders":"","keywords":"Fuzz testing; Computer science; Database transaction; Static analysis; Code (set theory); Code coverage; Construct (python library); Smart contract; Software bug; Database; Software engineering; Programming language; Software; Set (abstract data type)","score_opus":0.03912733686987249,"score_gpt":0.32688754569295964,"score_spread":0.28776020882308717,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205689130","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.018982572,0.00012164236,0.97689676,0.000457383,0.0007690157,0.00017585447,0.00014310917,0.0023817688,0.00007187376],"genre_scores_gemma":[0.61066175,0.000112767615,0.38859072,0.00018172008,0.000044420474,0.000038048525,0.00020917434,0.000039035516,0.00012236982],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99742615,0.00007422055,0.0004914137,0.0009693343,0.00061441533,0.0004244607],"domain_scores_gemma":[0.99732715,0.00042162,0.00020525641,0.0013070168,0.00055216835,0.00018675982],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00027528006,0.00039572318,0.0003920621,0.00035173938,0.00013387037,0.00052800745,0.0012636383,0.00011391206,0.0001569082],"category_scores_gemma":[0.0012769732,0.0003950314,0.00005430794,0.0005355242,0.00004872261,0.0011147062,0.00048090267,0.0004385722,0.00003263143],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00033547048,0.0010914619,0.003126163,0.0012197166,0.002992508,0.007601625,0.002628324,0.3823534,0.31757155,0.016791958,0.0030300487,0.2612578],"study_design_scores_gemma":[0.0004290436,0.00010506115,0.0012288789,0.00053538376,0.000030266512,0.00019049227,0.000043064127,0.96637636,0.02977128,0.00022245856,0.00058797916,0.00047970397],"about_ca_topic_score_codex":0.000042717213,"about_ca_topic_score_gemma":0.00010225225,"teacher_disagreement_score":0.59167916,"about_ca_system_score_codex":0.00020959621,"about_ca_system_score_gemma":0.00028775723,"threshold_uncertainty_score":0.99985015},"labels":[],"label_agreement":null},{"id":"W4205795599","doi":"10.1109/ase51524.2021.9678554","title":"Automatically Annotating Sentences for Task-specific Bug Report Summarization","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Lethbridge; University of Alberta","funders":"","keywords":"Automatic summarization; Computer science; Task (project management); DevOps; Annotation; Software engineering; Information retrieval; World Wide Web; Natural language processing; Software; Artificial intelligence; Programming language; Engineering","score_opus":0.03620246813544579,"score_gpt":0.3043772340185386,"score_spread":0.2681747658830928,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205795599","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.020491324,0.00008376284,0.9698473,0.0011460492,0.0040595676,0.00040479284,0.0001196565,0.0036885163,0.00015902627],"genre_scores_gemma":[0.5101559,0.00007793626,0.48700243,0.00015478063,0.0004116122,0.0002811982,0.0007286748,0.000103721555,0.0010837576],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.995818,0.000078924335,0.00089203374,0.0011526325,0.0013110171,0.00074739923],"domain_scores_gemma":[0.9946885,0.0018838427,0.00023695681,0.0012900616,0.00161263,0.00028798153],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007018221,0.00048462435,0.00045171904,0.00049632107,0.00019496807,0.00091856753,0.0018369529,0.00022492852,0.000255864],"category_scores_gemma":[0.011287477,0.00053505966,0.00020211063,0.0009795424,0.00005140596,0.0006839476,0.00039855693,0.00049336697,0.00013817842],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012510428,0.0012391604,0.01151106,0.0011115617,0.0015625163,0.008048239,0.0023044848,0.68809354,0.055010304,0.12651452,0.044605643,0.059873838],"study_design_scores_gemma":[0.00058690895,0.00009069535,0.004142082,0.00048155498,0.0000118506505,0.0002890988,0.000044903383,0.9819797,0.007267535,0.00046620268,0.0040203277,0.00061909226],"about_ca_topic_score_codex":0.000013086426,"about_ca_topic_score_gemma":0.0000030053566,"teacher_disagreement_score":0.48966458,"about_ca_system_score_codex":0.000361947,"about_ca_system_score_gemma":0.0005567026,"threshold_uncertainty_score":0.9997101},"labels":[],"label_agreement":null},{"id":"W4210294742","doi":"10.1109/ase51524.2021.9678520","title":"Subtle Bugs Everywhere: Generating Documentation for Data Wrangling Code","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Documentation; Code (set theory); Programming language; Reuse; Plug-in; Software bug; Database; Software engineering; Software; Engineering; Set (abstract data type)","score_opus":0.07020492043842771,"score_gpt":0.3442052189351064,"score_spread":0.27400029849667873,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4210294742","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.03265987,0.00015699788,0.95941275,0.00082676555,0.0038332124,0.00034807372,0.00040942954,0.0022971616,0.000055748213],"genre_scores_gemma":[0.40273798,0.00010858827,0.59270114,0.00026360905,0.0006972066,0.00020498988,0.0020206403,0.00012301795,0.0011428015],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99655217,0.000069316935,0.0005892811,0.0011590428,0.0009913463,0.0006388228],"domain_scores_gemma":[0.99525565,0.001610262,0.0001472965,0.0018411358,0.00092422793,0.0002214442],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00065008353,0.00040727592,0.00034616934,0.00033476372,0.00017995355,0.0010030476,0.0030154411,0.00017769035,0.00038974415],"category_scores_gemma":[0.007174456,0.00047013568,0.00011320762,0.0006143586,0.000026568154,0.0011410422,0.0007834036,0.00044633247,0.00014544868],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0001207111,0.00072869484,0.0057889246,0.00086045696,0.0015697994,0.001130918,0.0017681376,0.7149779,0.12826967,0.032360297,0.053100914,0.059323523],"study_design_scores_gemma":[0.00077484903,0.000058046593,0.0007263186,0.00029170173,0.000015666461,0.0000466082,0.000030585616,0.9784154,0.015267132,0.0001371189,0.0037367349,0.000499825],"about_ca_topic_score_codex":0.000036466496,"about_ca_topic_score_gemma":0.000014425998,"teacher_disagreement_score":0.37007812,"about_ca_system_score_codex":0.00034925627,"about_ca_system_score_gemma":0.0004978822,"threshold_uncertainty_score":0.99977505},"labels":[],"label_agreement":null}]}