{"meta":{"page":1,"per_page":50,"max_per_page":100,"total":6,"total_is_capped":false,"direct_labels_cover":0,"predictions_cover":6,"direct_label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline (scores rank; they never assert a category)","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"query_hash":"7a3547c421a9","filters":{"venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)"}},"results":[{"id":"W4205689130","doi":"10.1109/ase51524.2021.9678888","title":"SMARTIAN: Enhancing Smart Contract Fuzzing with Static and Dynamic Data-Flow Analyses","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":165,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Kootenay Association for Science & Technology","funders":"","keywords":"Fuzz testing; Computer science; Database transaction; Static analysis; Code (set theory); Code coverage; Construct (python library); Smart contract; Software bug; Database; Software engineering; Programming language; Software; Set (abstract data type)","retraction":null,"screen_n_in":null,"score":{"opus":0.03912733686987249,"gpt":0.3268875456929596,"spread":0.2877602088230872,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002752801,0.0003957232,0.0003920621,0.0003517394,0.0001338704,0.0005280075,0.001263638,0.0001139121,0.0001569082],"category_scores_gemma":[0.001276973,0.0003950314,0.00005430794,0.0005355242,0.00004872261,0.001114706,0.0004809027,0.0004385722,0.00003263143],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002095962,"about_ca_system_score_gemma":0.0002877572,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004271721,"about_ca_topic_score_gemma":0.0001022522,"domain_scores_codex":[0.9974262,0.00007422055,0.0004914137,0.0009693343,0.0006144153,0.0004244607],"domain_scores_gemma":[0.9973271,0.00042162,0.0002052564,0.001307017,0.0005521683,0.0001867598],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003354705,0.001091462,0.003126163,0.001219717,0.002992508,0.007601625,0.002628324,0.3823534,0.3175716,0.01679196,0.003030049,0.2612578],"study_design_scores_gemma":[0.0004290436,0.0001050612,0.001228879,0.0005353838,0.00003026651,0.0001904923,0.00004306413,0.9663764,0.02977128,0.0002224586,0.0005879792,0.000479704],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01898257,0.0001216424,0.9768968,0.000457383,0.0007690157,0.0001758545,0.0001431092,0.002381769,0.00007187376],"genre_scores_gemma":[0.6106617,0.0001127676,0.3885907,0.0001817201,0.00004442047,0.00003804853,0.0002091743,0.00003903552,0.0001223698],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.5916792,"threshold_uncertainty_score":0.9998502,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4210294742","doi":"10.1109/ase51524.2021.9678520","title":"Subtle Bugs Everywhere: Generating Documentation for Data Wrangling Code","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":25,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University; University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Documentation; Code (set theory); Programming language; Reuse; Plug-in; Software bug; Database; Software engineering; Software; Engineering; Set (abstract data type)","retraction":null,"screen_n_in":null,"score":{"opus":0.07020492043842771,"gpt":0.3442052189351064,"spread":0.2740002984966787,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006500835,0.0004072759,0.0003461693,0.0003347637,0.0001799536,0.001003048,0.003015441,0.0001776904,0.0003897441],"category_scores_gemma":[0.007174456,0.0004701357,0.0001132076,0.0006143586,0.00002656815,0.001141042,0.0007834036,0.0004463325,0.0001454487],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003492563,"about_ca_system_score_gemma":0.0004978822,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000364665,"about_ca_topic_score_gemma":0.000014426,"domain_scores_codex":[0.9965522,0.00006931693,0.0005892811,0.001159043,0.0009913463,0.0006388228],"domain_scores_gemma":[0.9952556,0.001610262,0.0001472965,0.001841136,0.0009242279,0.0002214442],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001207111,0.0007286948,0.005788925,0.000860457,0.001569799,0.001130918,0.001768138,0.7149779,0.1282697,0.0323603,0.05310091,0.05932352],"study_design_scores_gemma":[0.000774849,0.00005804659,0.0007263186,0.0002917017,0.00001566646,0.0000466082,0.00003058562,0.9784154,0.01526713,0.0001371189,0.003736735,0.000499825],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.03265987,0.0001569979,0.9594128,0.0008267656,0.003833212,0.0003480737,0.0004094295,0.002297162,0.00005574821],"genre_scores_gemma":[0.402738,0.0001085883,0.5927011,0.0002636091,0.0006972066,0.0002049899,0.00202064,0.0001230179,0.001142801],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.3700781,"threshold_uncertainty_score":0.9997751,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4205513494","doi":"10.1109/ase51524.2021.9678640","title":"Is Historical Data an Appropriate Benchmark for Reviewer Recommendation Systems? : A Case Study of the Gerrit Community","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo; Polytechnique Montréal; McGill University","funders":"","keywords":"Context (archaeology); Pessimism; Computer science; Benchmark (surveying); Replicate; Task (project management); Data science; History; Epistemology; Management","retraction":null,"screen_n_in":null,"score":{"opus":0.1459378586426111,"gpt":0.3631536798642139,"spread":0.2172158212216028,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001458419,0.0003407481,0.000452716,0.0002462915,0.0002544318,0.0003437579,0.003865959,0.0001454117,0.0001231521],"category_scores_gemma":[0.008897702,0.0003089131,0.0001067238,0.0007087103,0.00002401712,0.0006367081,0.001285211,0.0007623667,0.00001093637],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006325644,"about_ca_system_score_gemma":0.0003487794,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007942355,"about_ca_topic_score_gemma":0.00006435008,"domain_scores_codex":[0.9968402,0.0004645654,0.0007486186,0.0007575616,0.0008117369,0.0003773567],"domain_scores_gemma":[0.9928439,0.001590019,0.0002539728,0.003510125,0.001628298,0.0001737414],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005111056,0.03086802,0.05423665,0.01053627,0.008833637,0.006679065,0.05537367,0.3209176,0.01040358,0.01683459,0.2687843,0.2160215],"study_design_scores_gemma":[0.0009132074,0.0002920013,0.001954201,0.0004834124,0.00004472603,0.000312612,0.0004582244,0.989701,0.0003712407,0.00005450015,0.004971264,0.0004435856],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4243603,0.0002227622,0.5577116,0.00142473,0.01186795,0.001952438,0.0007935853,0.001625899,0.00004072053],"genre_scores_gemma":[0.9731131,0.00003585086,0.02569008,0.00008641899,0.0002015348,0.0002132171,0.0003601008,0.00005427002,0.0002453751],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6687834,"threshold_uncertainty_score":0.9999363,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4205678875","doi":"10.1109/ase51524.2021.9678871","title":"DeepMemory: Model-based Memorization Analysis of Deep Neural Language Models","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Queen's University; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Perplexity; Memorization; Language model; Artificial intelligence; Artificial neural network; Machine learning; Data modeling; Robustness (evolution); Natural language processing; Database","retraction":null,"screen_n_in":null,"score":{"opus":0.02093544744193205,"gpt":0.2813157815033691,"spread":0.2603803340614371,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002559315,0.000342417,0.0004792422,0.0007976549,0.00008544307,0.0002081061,0.001458509,0.0001596659,0.0002874976],"category_scores_gemma":[0.001195549,0.0003837176,0.0002436445,0.001584979,0.00003395767,0.0005660487,0.0002780159,0.0003917725,0.00001514586],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001991317,"about_ca_system_score_gemma":0.0002311468,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003025627,"about_ca_topic_score_gemma":0.00001342079,"domain_scores_codex":[0.9974718,0.0001009432,0.0005421742,0.000685477,0.0008475365,0.0003520678],"domain_scores_gemma":[0.9975896,0.0003326032,0.0002613609,0.0009663867,0.0007102626,0.0001397099],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001194178,0.00007088531,0.0002178251,0.00002722557,0.0003740459,0.00009183781,0.0005704781,0.9836665,0.002624585,0.01018736,0.00002444048,0.002132921],"study_design_scores_gemma":[0.0004226485,0.0000266105,0.0005009705,0.00006984201,0.000132059,0.000005173441,0.0000418289,0.9938751,0.004407512,0.0001637672,0.000007091915,0.000347406],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.03385437,0.00003888,0.9633803,0.0003742805,0.001022556,0.0001039176,0.00006217151,0.0009697759,0.0001936831],"genre_scores_gemma":[0.8163967,0.000005853403,0.1828574,0.0001436774,0.0000620167,0.00001858116,0.0003681162,0.00003283482,0.0001149332],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7825422,"threshold_uncertainty_score":0.9998615,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3196824794","doi":"10.1109/ase51524.2021.9678596","title":"Leveraging Code Clones and Natural Language Processing for Log Statement Prediction","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Source code; Statement (logic); Debugging; Java; Context (archaeology); Programming language; Data mining; Database; Artificial intelligence; Information retrieval","retraction":null,"screen_n_in":null,"score":{"opus":0.02070793589757158,"gpt":0.2854924140753931,"spread":0.2647844781778215,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003428814,0.0002698899,0.0002553904,0.000173008,0.0001598211,0.0003954318,0.0005480698,0.0001066129,0.00003935833],"category_scores_gemma":[0.0006196731,0.0002586234,0.00007581462,0.0002428675,0.00002857641,0.0005665182,0.0001798918,0.0002488549,0.00001525576],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002072915,"about_ca_system_score_gemma":0.000238568,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001572753,"about_ca_topic_score_gemma":0.00000524229,"domain_scores_codex":[0.9980912,0.00003796307,0.0004549387,0.0006188623,0.0004549575,0.000342064],"domain_scores_gemma":[0.9985705,0.0002042939,0.0001372812,0.000441984,0.0005393759,0.0001065838],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004896024,0.001700616,0.05672396,0.007427888,0.00221477,0.00126167,0.03422398,0.2169444,0.08401684,0.01472998,0.01858834,0.5616779],"study_design_scores_gemma":[0.0006888503,0.00006241144,0.006659476,0.0005363778,0.00001365512,0.00006356512,0.0001788826,0.98361,0.006553419,0.00009085777,0.001229652,0.0003129044],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2915133,0.0006711504,0.6995513,0.001064911,0.004493271,0.0004239236,0.0001755043,0.002057274,0.00004936962],"genre_scores_gemma":[0.9488178,0.00006262775,0.05018332,0.0001349939,0.0001962589,0.00009517954,0.0001885563,0.00002529624,0.000295938],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7666655,"threshold_uncertainty_score":0.9999866,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4205795599","doi":"10.1109/ase51524.2021.9678554","title":"Automatically Annotating Sentences for Task-specific Bug Report Summarization","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Lethbridge; University of Alberta","funders":"","keywords":"Automatic summarization; Computer science; Task (project management); DevOps; Annotation; Software engineering; Information retrieval; World Wide Web; Natural language processing; Software; Artificial intelligence; Programming language; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.03620246813544579,"gpt":0.3043772340185386,"spread":0.2681747658830928,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007018221,0.0004846244,0.000451719,0.0004963211,0.0001949681,0.0009185675,0.001836953,0.0002249285,0.000255864],"category_scores_gemma":[0.01128748,0.0005350597,0.0002021106,0.0009795424,0.00005140596,0.0006839476,0.0003985569,0.000493367,0.0001381784],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000361947,"about_ca_system_score_gemma":0.0005567026,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001308643,"about_ca_topic_score_gemma":0.000003005357,"domain_scores_codex":[0.995818,0.00007892433,0.0008920337,0.001152633,0.001311017,0.0007473992],"domain_scores_gemma":[0.9946885,0.001883843,0.0002369568,0.001290062,0.00161263,0.0002879815],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001251043,0.00123916,0.01151106,0.001111562,0.001562516,0.008048239,0.002304485,0.6880935,0.0550103,0.1265145,0.04460564,0.05987384],"study_design_scores_gemma":[0.000586909,0.00009069535,0.004142082,0.000481555,0.00001185065,0.0002890988,0.00004490338,0.9819797,0.007267535,0.0004662027,0.004020328,0.0006190923],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02049132,0.00008376284,0.9698473,0.001146049,0.004059568,0.0004047928,0.0001196565,0.003688516,0.0001590263],"genre_scores_gemma":[0.5101559,0.00007793626,0.4870024,0.0001547806,0.0004116122,0.0002811982,0.0007286748,0.0001037216,0.001083758],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.4896646,"threshold_uncertainty_score":0.9997101,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null}]}