{"meta":{"query_hash":"7a3547c421a9","filters":{"venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)"},"cohort_total":6,"direct_labels_cover":0,"predictions_cover":6,"exported":6,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/7a3547c421a9","api":"https://metacan.xera.ac/api/v1/cohort?venue=2021+36th+IEEE%2FACM+International+Conference+on+Automated+Software+Engineering+%28ASE%29"},"results":[{"id":"W3196824794","doi":"10.1109/ase51524.2021.9678596","title":"Leveraging Code Clones and Natural Language Processing for Log Statement Prediction","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Source code; Statement (logic); Debugging; Java; Context (archaeology); Programming language; Data mining; Database; Artificial intelligence; Information retrieval","score_opus":0.02070793589757158,"score_gpt":0.28549241407539305,"score_spread":0.26478447817782147,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196824794","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.40352458,0.00097164424,0.56016135,0.0008244008,0.00007607491,0.00049480255,0.0040848223,0.027699955,0.0021623436],"genre_scores_gemma":[0.6872912,0.00033000496,0.30123594,0.0002293753,0.00004781324,0.00034930962,0.008550915,0.0007404284,0.0012249385],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9974784,0.00048437424,0.00021476744,0.00086345914,0.00084441405,0.00011457661],"domain_scores_gemma":[0.97274506,0.016060712,0.0047878926,0.0022806027,0.0037683886,0.00035743247],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0018462883,0.0011829907,0.0006567198,0.0045269467,0.00052246056,0.0012708603,0.0011849051,0.0010376286,0.0006846739],"category_scores_gemma":[0.024773765,0.00046548233,0.0009052115,0.0020659086,0.0007297627,0.0035650674,0.0011616754,0.0015066213,0.0006622943],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0006011467,0.0006989743,0.19971576,0.0010434173,0.00017184849,0.0024497132,0.0037562626,0.045259282,0.048522066,0.0037874964,0.0087505,0.6852436],"study_design_scores_gemma":[0.000054022337,0.00030123827,0.04700322,0.00012448923,0.00013903363,0.0010555058,0.00076567486,0.89727163,0.034627423,0.010040841,0.008507118,0.000109806686],"about_ca_topic_score_codex":0.006767068,"about_ca_topic_score_gemma":0.010358205,"teacher_disagreement_score":0.006767068,"about_ca_system_score_codex":0.00081323175,"about_ca_system_score_gemma":0.0017403249,"threshold_uncertainty_score":0.013455331},"labels":[],"label_agreement":null},{"id":"W4205513494","doi":"10.1109/ase51524.2021.9678640","title":"Is Historical Data an Appropriate Benchmark for Reviewer Recommendation Systems? : A Case Study of the Gerrit Community","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; Polytechnique Montréal; McGill University","funders":"","keywords":"Context (archaeology); Pessimism; Computer science; Benchmark (surveying); Replicate; Task (project management); Data science; History; Epistemology; Management","score_opus":0.1459378586426111,"score_gpt":0.3631536798642139,"score_spread":0.21721582122160277,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205513494","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9839527,0.0011480495,0.0072677666,0.0018040782,0.00005674351,0.00043698473,0.0005413618,0.00026592854,0.0045263325],"genre_scores_gemma":[0.98261225,0.0003752549,0.014023932,0.00031556145,0.00007086336,0.0002358108,0.0006293292,0.00012573601,0.0016111885],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.95386857,0.030837,0.0025901757,0.0037093388,0.008021301,0.00097354257],"domain_scores_gemma":[0.60254973,0.27839342,0.02805292,0.020271076,0.06323623,0.0074966024],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.039902847,0.0004654721,0.0007826616,0.004814048,0.0038229723,0.0033390282,0.0019040874,0.0020959128,0.0012943348],"category_scores_gemma":[0.19410795,0.00047927347,0.00055477093,0.0048914477,0.0014764583,0.0049091033,0.0018036686,0.0014910189,0.0007009175],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.001634384,0.0019466062,0.45260215,0.0028609708,0.00036229112,0.0071781348,0.18413486,0.00649564,0.009586056,0.0031119406,0.015753541,0.31433347],"study_design_scores_gemma":[0.00051459274,0.005599404,0.56155044,0.002055454,0.00045276823,0.0071232314,0.20917334,0.06415552,0.0165475,0.005892315,0.12601088,0.0009246325],"about_ca_topic_score_codex":0.020884013,"about_ca_topic_score_gemma":0.04176582,"teacher_disagreement_score":0.96009713,"about_ca_system_score_codex":0.0029612037,"about_ca_system_score_gemma":0.002903485,"threshold_uncertainty_score":0.211029},"labels":[],"label_agreement":null},{"id":"W4205678875","doi":"10.1109/ase51524.2021.9678871","title":"DeepMemory: Model-based Memorization Analysis of Deep Neural Language Models","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Perplexity; Memorization; Language model; Artificial intelligence; Artificial neural network; Machine learning; Data modeling; Robustness (evolution); Natural language processing; Database","score_opus":0.020935447441932053,"score_gpt":0.2813157815033691,"score_spread":0.26038033406143707,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205678875","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.20848402,0.0010134439,0.7779106,0.0007086614,0.00010130114,0.00017391088,0.0010163136,0.009314076,0.0012777541],"genre_scores_gemma":[0.89419085,0.00036892245,0.10111942,0.0002658123,0.00005876254,0.00019988247,0.0016009192,0.00022467648,0.0019708467],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9991678,0.0002087232,0.00008302704,0.00023533133,0.00020942763,0.00009579878],"domain_scores_gemma":[0.9957151,0.0021625066,0.0006785806,0.0007986831,0.0005353322,0.00010976852],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001729058,0.0010691636,0.00070929923,0.001229561,0.00034480175,0.001033433,0.0015021855,0.0006577314,0.0012719557],"category_scores_gemma":[0.0103772115,0.00036437984,0.0009969985,0.0005662029,0.0005183704,0.0029391858,0.0013717723,0.0020283987,0.00037617196],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00087098195,0.00040050593,0.019712674,0.00040555483,0.00037450643,0.00052693335,0.00052640995,0.42339557,0.023396594,0.009290509,0.007040185,0.5140596],"study_design_scores_gemma":[0.000011650767,0.00009126831,0.0008535247,0.00001108236,0.000028148796,0.000049829192,0.000034713175,0.9850172,0.007703545,0.005690846,0.0004949721,0.000013279771],"about_ca_topic_score_codex":0.0039941156,"about_ca_topic_score_gemma":0.0057098013,"teacher_disagreement_score":0.0039941156,"about_ca_system_score_codex":0.0010909879,"about_ca_system_score_gemma":0.0010715092,"threshold_uncertainty_score":0.009144247},"labels":[],"label_agreement":null},{"id":"W4205689130","doi":"10.1109/ase51524.2021.9678888","title":"SMARTIAN: Enhancing Smart Contract Fuzzing with Static and Dynamic Data-Flow Analyses","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":165,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Kootenay Association for Science & Technology","funders":"","keywords":"Fuzz testing; Computer science; Database transaction; Static analysis; Code (set theory); Code coverage; Construct (python library); Smart contract; Software bug; Database; Software engineering; Programming language; Software; Set (abstract data type)","score_opus":0.03912733686987249,"score_gpt":0.32688754569295964,"score_spread":0.28776020882308717,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205689130","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14753729,0.00048588248,0.7597989,0.0010203428,0.000102155354,0.0005715412,0.0011472367,0.086369924,0.002966794],"genre_scores_gemma":[0.46529862,0.00027645277,0.5248548,0.0006153135,0.00005541509,0.0003131874,0.0026518013,0.003945327,0.0019891167],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99501294,0.0010800699,0.0003935301,0.0009728833,0.0022344242,0.0003060036],"domain_scores_gemma":[0.98067,0.011203746,0.0015901054,0.0044137766,0.0018335386,0.0002888094],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004066743,0.0015993364,0.00086044474,0.003293842,0.00070550607,0.001757971,0.0022979053,0.001358383,0.0021095392],"category_scores_gemma":[0.02960371,0.0009920557,0.0013768203,0.0012683535,0.0023205823,0.0058503626,0.0030606189,0.0020200394,0.0008283956],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0010397792,0.00095683994,0.05981112,0.0010471126,0.0003734866,0.0011522315,0.0022149729,0.14930516,0.09352568,0.02743243,0.01897952,0.64416164],"study_design_scores_gemma":[0.00008811581,0.00023927497,0.004033957,0.0000870938,0.0000748774,0.0003668302,0.00016563713,0.91438985,0.05192642,0.020662034,0.007877143,0.00008874325],"about_ca_topic_score_codex":0.0044035194,"about_ca_topic_score_gemma":0.006199884,"teacher_disagreement_score":0.0044035194,"about_ca_system_score_codex":0.0010491357,"about_ca_system_score_gemma":0.0028881142,"threshold_uncertainty_score":0.021507263},"labels":[],"label_agreement":null},{"id":"W4205795599","doi":"10.1109/ase51524.2021.9678554","title":"Automatically Annotating Sentences for Task-specific Bug Report Summarization","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Lethbridge; University of Alberta","funders":"","keywords":"Automatic summarization; Computer science; Task (project management); DevOps; Annotation; Software engineering; Information retrieval; World Wide Web; Natural language processing; Software; Artificial intelligence; Programming language; Engineering","score_opus":0.03620246813544579,"score_gpt":0.3043772340185386,"score_spread":0.2681747658830928,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4205795599","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1024886,0.0030717568,0.7949791,0.0038748542,0.0021543233,0.0014136005,0.03194063,0.048880737,0.011196372],"genre_scores_gemma":[0.14510994,0.0010763622,0.7696705,0.00071287935,0.0009311578,0.0014815573,0.06965651,0.0045882915,0.00677276],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9909019,0.00417686,0.0012998725,0.0015033435,0.0017301923,0.00038786232],"domain_scores_gemma":[0.9471923,0.021772387,0.006037434,0.004130394,0.019840287,0.0010272377],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009600126,0.0023829082,0.0012147884,0.008525165,0.0014664857,0.0027368965,0.0015674857,0.0016336404,0.004272044],"category_scores_gemma":[0.045108963,0.0008044557,0.0010476869,0.0040256507,0.0005918401,0.0037960478,0.0026523543,0.0020380586,0.005055931],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0005924125,0.00031661207,0.015494099,0.004914821,0.0002792693,0.0012761202,0.011063268,0.0063450597,0.109378435,0.0065516075,0.27659723,0.56719106],"study_design_scores_gemma":[0.0002338581,0.0011421167,0.048833117,0.0016211713,0.0011536417,0.0021090005,0.00940651,0.18965702,0.14073984,0.028025704,0.57642996,0.0006481782],"about_ca_topic_score_codex":0.0024613836,"about_ca_topic_score_gemma":0.004875251,"teacher_disagreement_score":0.009600126,"about_ca_system_score_codex":0.0008242657,"about_ca_system_score_gemma":0.0024738477,"threshold_uncertainty_score":0.05077094},"labels":[],"label_agreement":null},{"id":"W4210294742","doi":"10.1109/ase51524.2021.9678520","title":"Subtle Bugs Everywhere: Generating Documentation for Data Wrangling Code","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Documentation; Code (set theory); Programming language; Reuse; Plug-in; Software bug; Database; Software engineering; Software; Engineering; Set (abstract data type)","score_opus":0.07020492043842771,"score_gpt":0.3442052189351064,"score_spread":0.27400029849667873,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4210294742","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.048732873,0.00022775849,0.7281107,0.00063254486,0.00017629714,0.0006041682,0.0027160503,0.21332934,0.0054702456],"genre_scores_gemma":[0.1759035,0.0002449866,0.7641879,0.0003582699,0.00006428248,0.0007065772,0.006920453,0.044447545,0.00716644],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9964803,0.0008949452,0.00039457338,0.0006477146,0.001415122,0.0001673214],"domain_scores_gemma":[0.9633649,0.016951736,0.0028775204,0.011755317,0.0043978365,0.0006526492],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0047372524,0.0015558752,0.0005639002,0.002839777,0.00078943034,0.0018317517,0.002091566,0.0012972789,0.0070668063],"category_scores_gemma":[0.037257142,0.0013855625,0.0009914874,0.0013981222,0.0010102422,0.002981326,0.003048274,0.0019005528,0.0033989763],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0008525086,0.00087594293,0.018171597,0.0016454142,0.00015110656,0.0021257238,0.0076438673,0.01778179,0.06844699,0.014070249,0.08685815,0.78137654],"study_design_scores_gemma":[0.0006822046,0.0013404515,0.011906907,0.001176003,0.000246298,0.0035500012,0.0015096895,0.39948985,0.29122156,0.03177271,0.2566788,0.00042551087],"about_ca_topic_score_codex":0.00090691994,"about_ca_topic_score_gemma":0.0015849681,"teacher_disagreement_score":0.0070668063,"about_ca_system_score_codex":0.00054004585,"about_ca_system_score_gemma":0.0015289882,"threshold_uncertainty_score":0.025053322},"labels":[],"label_agreement":null}]}