{"meta":{"query_hash":"1f0d90f13f76","filters":{"venue":"Empirical Software Engineering"},"cohort_total":371,"direct_labels_cover":1,"predictions_cover":371,"exported":371,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/1f0d90f13f76","api":"https://metacan.xera.ac/api/v1/cohort?venue=Empirical+Software+Engineering"},"results":[{"id":"W1024906986","doi":"10.1007/s10664-015-9387-3","title":"A contextual approach towards more accurate duplicate bug report detection and ranking","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":90,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Data deduplication; Software bug; Android (operating system); Software; Ranking (information retrieval); Context (archaeology); Data science; Software engineering; Data mining; World Wide Web; Information retrieval; Database","score_opus":0.05477320630831465,"score_gpt":0.3061968965378645,"score_spread":0.25142369022954986,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1024906986","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.22463568,0.00034069308,0.77258176,0.00021937954,0.00034691428,0.00021428405,0.0000014349498,0.0016322973,0.000027578746],"genre_scores_gemma":[0.9243937,0.0000056009962,0.07508279,0.000076077595,0.00019852132,0.00010052384,0.0000058349606,0.0000507888,0.0000861686],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9974777,0.00003774341,0.00039065932,0.00075621804,0.00070172135,0.00063595397],"domain_scores_gemma":[0.9980202,0.0004887518,0.00007024991,0.000699261,0.00019542992,0.0005261325],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008912401,0.00032233514,0.0003470919,0.0002500158,0.00009526451,0.00027957678,0.0006283331,0.00017416386,0.0000020519622],"category_scores_gemma":[0.0053116707,0.00031004768,0.00008355189,0.00086442253,0.00004947561,0.0005899671,0.0005696282,0.00053506054,0.000016979162],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00020703678,0.0006498655,0.3816082,0.0011766716,0.0007486867,0.004422944,0.017391771,0.3535842,0.0045904582,0.0007169185,0.0061728167,0.22873046],"study_design_scores_gemma":[0.001988021,0.00025837056,0.26924115,0.00010403301,0.000029350678,0.0033896358,0.000086414366,0.69794285,0.0037028936,0.0002270546,0.02147916,0.0015510354],"about_ca_topic_score_codex":0.000051479794,"about_ca_topic_score_gemma":7.3931136e-7,"teacher_disagreement_score":0.69975805,"about_ca_system_score_codex":0.00022414923,"about_ca_system_score_gemma":0.000120055956,"threshold_uncertainty_score":0.99993515},"labels":[],"label_agreement":null},{"id":"W1174118978","doi":"10.1007/s10664-015-9388-2","title":"Fresh apps: an empirical study of frequently-updated mobile apps in the Google play store","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Digital Marketing and Social Media","field":"Social Sciences","cited_by":201,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; Queen's University","funders":"","keywords":"Mobile apps; Computer science; World Wide Web; App store; Internet privacy","score_opus":0.049901752931140124,"score_gpt":0.34997055647660247,"score_spread":0.30006880354546234,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1174118978","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.996354,0.00018211182,0.00030533166,0.00016357178,0.0004240498,0.0005834021,0.000011297815,0.00028054795,0.0016956746],"genre_scores_gemma":[0.99890673,0.0000042572924,0.00038610096,0.000119306846,0.00035610818,0.0001047191,0.000017590386,0.000027749486,0.00007745355],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99739414,0.0004830782,0.00040238458,0.0003236624,0.0009002916,0.00049645535],"domain_scores_gemma":[0.9983602,0.0008291262,0.00007786727,0.00030040825,0.00011109734,0.00032129997],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0019813254,0.0001965524,0.00034052535,0.00011267341,0.00014236194,0.0001039551,0.00062229246,0.0001838754,0.000037707494],"category_scores_gemma":[0.0033291439,0.00015947699,0.00007273137,0.0009417017,0.00012493397,0.00029012482,0.00007990459,0.00040145323,0.00001613121],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000042853146,0.001638409,0.74764866,0.0000174131,0.000037639325,0.00005919478,0.2303734,0.0041019856,0.000005646836,0.00006687397,0.012793379,0.003214531],"study_design_scores_gemma":[0.003548012,0.0024877433,0.36094338,0.00018541607,0.00008899039,0.000005823897,0.32147795,0.0009646017,0.000025800866,0.0008759728,0.30791202,0.0014843019],"about_ca_topic_score_codex":0.0011345478,"about_ca_topic_score_gemma":0.0010527219,"teacher_disagreement_score":0.3867053,"about_ca_system_score_codex":0.00020331271,"about_ca_system_score_gemma":0.00017925336,"threshold_uncertainty_score":0.6503283},"labels":[],"label_agreement":null},{"id":"W1260237309","doi":"10.1007/s10664-015-9364-x","title":"Coevolution of variability models and related software artifacts","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":57,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Software evolution; Coevolution; Variation (astronomy); Artifact (error); Computer science; Software; Feature (linguistics); Representation (politics); Personalization; Software system; Kernel (algebra); Data science; Machine learning; Artificial intelligence; Data mining; Software construction","score_opus":0.07639605922882918,"score_gpt":0.30006684688225754,"score_spread":0.22367078765342835,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1260237309","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0361114,0.00045405087,0.9608349,0.0000899545,0.00054921716,0.00015228012,0.000003860027,0.0017852966,0.000019045423],"genre_scores_gemma":[0.35648057,0.0000074547906,0.6434177,0.000018144363,0.00002090911,0.0000122719675,0.0000026499133,0.000021057398,0.000019252138],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9981149,0.00014397086,0.0004578941,0.0005058736,0.00036559394,0.0004117791],"domain_scores_gemma":[0.9967667,0.002021857,0.00010191497,0.00061927334,0.00018628394,0.00030397082],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0013976243,0.00026376074,0.00040585193,0.00014899627,0.000042744574,0.000034794284,0.00046124356,0.00019763673,0.000002668445],"category_scores_gemma":[0.013513289,0.00026206314,0.00007383943,0.00061555987,0.00006870327,0.0008209376,0.000422818,0.00033141562,0.0000061888927],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007185487,0.00003719354,0.0072027957,0.00007950404,0.000027710787,0.000012056491,0.0007433485,0.9830219,0.00010302726,0.002029599,0.00013523993,0.006600441],"study_design_scores_gemma":[0.001285193,0.00034167105,0.044680044,0.0001957596,0.000041566534,0.00017142906,0.000033774828,0.7177072,0.002518475,0.23056884,0.001114929,0.0013410755],"about_ca_topic_score_codex":0.000007886524,"about_ca_topic_score_gemma":2.3971646e-7,"teacher_disagreement_score":0.32036915,"about_ca_system_score_codex":0.00016146163,"about_ca_system_score_gemma":0.00008771522,"threshold_uncertainty_score":0.99998313},"labels":[],"label_agreement":null},{"id":"W1491417663","doi":"10.1023/a:1009859121816","title":"Ethics and Empirical Studies of Software Engineering","year":2000,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Ethics in Business and Education","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Computer science","score_opus":0.311473937096708,"score_gpt":0.4635035632400121,"score_spread":0.15202962614330406,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1491417663","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.943381,0.003058798,0.04958199,0.002675009,0.0008412149,0.00012975882,0.000011305,0.00029511604,0.000025768062],"genre_scores_gemma":[0.94306624,0.0005487309,0.054981206,0.0005373856,0.00035000851,0.00002563246,0.0000050229696,0.000052785694,0.0004329873],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99693024,0.00010248588,0.000815348,0.00055366044,0.0011988328,0.00039941235],"domain_scores_gemma":[0.9815997,0.017072288,0.00010523677,0.0005009699,0.0005221355,0.00019968538],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0029608838,0.00028034014,0.00055335596,0.0002670086,0.00019696793,0.00011241724,0.00043319663,0.00034444514,0.0002992526],"category_scores_gemma":[0.07665189,0.00022788373,0.00012261725,0.0010761837,0.0001534104,0.0003132165,0.00016339199,0.0010564661,0.00005583435],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005388496,0.0002249981,0.7682125,0.0007971791,0.00021451956,0.00003181196,0.026277281,0.14501402,0.00018925579,0.00053557294,0.012040178,0.0464088],"study_design_scores_gemma":[0.00076657324,0.0002077964,0.8274815,0.0007836146,0.0000903707,0.000084693565,0.0011356105,0.011183532,0.00078679394,0.00768312,0.14843705,0.001359312],"about_ca_topic_score_codex":0.000012191106,"about_ca_topic_score_gemma":0.0000052296436,"teacher_disagreement_score":0.13639688,"about_ca_system_score_codex":0.000073222705,"about_ca_system_score_gemma":0.00014678606,"threshold_uncertainty_score":0.9311259},"labels":[],"label_agreement":null},{"id":"W1493296086","doi":"10.1007/s10664-014-9308-x","title":"Understanding the impact of rapid releases on software quality","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":59,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Polytechnique Montréal","funders":"","keywords":"Computer science; Software release life cycle; Software engineering; Crash; Software quality; Software; Quality (philosophy); Software bug; Software quality analyst; Software quality assurance; Software development; Operating system","score_opus":0.11284751522180525,"score_gpt":0.3485465269864667,"score_spread":0.23569901176466146,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1493296086","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14531282,0.000103917446,0.85297877,0.00015152169,0.00025819638,0.00016300636,0.0000061740684,0.0009962821,0.000029325927],"genre_scores_gemma":[0.97668684,0.000007352282,0.022983994,0.00007417658,0.00015118015,0.000022632788,0.0000036291678,0.0000466944,0.000023521636],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99743265,0.00015856168,0.00044813007,0.0005020965,0.0007895418,0.00066900917],"domain_scores_gemma":[0.9897571,0.008606198,0.00010267872,0.0011819344,0.00009881131,0.00025323813],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0011215553,0.00033289494,0.00040502223,0.00024496965,0.00015068913,0.00012737037,0.0013360364,0.00013451862,0.000036036083],"category_scores_gemma":[0.014921135,0.00023278291,0.00033875098,0.0010271922,0.00007175255,0.00025799265,0.0003355134,0.000564737,0.000037341822],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000045931276,0.0002250129,0.39427716,0.00021573967,0.00023520707,0.000017538165,0.00093602564,0.5817055,0.000258738,0.006005829,0.006413215,0.009664102],"study_design_scores_gemma":[0.0007531255,0.00089785684,0.9131991,0.00022560732,0.000012201603,0.00002499912,0.000018313509,0.08027246,0.00094335375,0.0016860175,0.00116639,0.0008005334],"about_ca_topic_score_codex":0.000049838793,"about_ca_topic_score_gemma":7.000228e-7,"teacher_disagreement_score":0.831374,"about_ca_system_score_codex":0.00048628123,"about_ca_system_score_gemma":0.000098826495,"threshold_uncertainty_score":0.9933766},"labels":[],"label_agreement":null},{"id":"W1497968089","doi":"10.1023/a:1024424811345","title":"Fault Prediction Modeling for Software Quality Estimation: Comparing Commonly Used Techniques","year":2003,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Reliability and Analysis Research","field":"Computer Science","cited_by":128,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"McGill University","keywords":"Computer science; Software quality; Software; Artificial neural network; Approximation error; Reliability engineering; Data mining; Artificial intelligence; Machine learning; Algorithm; Software development; Engineering","score_opus":0.07721747522635733,"score_gpt":0.35017777725256943,"score_spread":0.27296030202621213,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1497968089","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.053146422,0.00015347908,0.94354683,0.0001609792,0.00017580987,0.00040069985,0.000010654374,0.0023896769,0.000015474858],"genre_scores_gemma":[0.4940997,0.0000050260546,0.50558245,0.000055510933,0.00005706008,0.00013402355,0.000022531862,0.000024001802,0.000019716093],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.997311,0.00013075804,0.00066744105,0.0006625305,0.000638091,0.00059017184],"domain_scores_gemma":[0.99729365,0.0013593758,0.0000840328,0.00073612266,0.00028215235,0.00024469342],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0014008299,0.00029060853,0.0004805184,0.00022537248,0.00031160493,0.00027641992,0.0006873557,0.00020127918,0.000008864174],"category_scores_gemma":[0.005380118,0.00029285802,0.00026569795,0.00082408136,0.000032555752,0.00066360127,0.0001687635,0.00041583023,0.000011261964],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010647868,0.00016720335,0.094680086,0.00030849248,0.00008055412,0.0000038885787,0.0003951948,0.8871707,0.00007651346,0.0023423545,0.00036640986,0.014397975],"study_design_scores_gemma":[0.00030616747,0.00006204323,0.0018328445,0.000075264645,0.000014416909,0.000008300578,0.000010899327,0.99122363,0.0011579024,0.0022703018,0.002696077,0.00034214844],"about_ca_topic_score_codex":0.000029579869,"about_ca_topic_score_gemma":0.000004850769,"teacher_disagreement_score":0.44095328,"about_ca_system_score_codex":0.0002562927,"about_ca_system_score_gemma":0.000112179325,"threshold_uncertainty_score":0.9999524},"labels":[],"label_agreement":null},{"id":"W1507760270","doi":"10.1023/a:1011966430523","title":"Getting to the Source of Ethical Issues","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Social and Cultural Studies","field":"Social Sciences","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Computer science","score_opus":0.04003005214578316,"score_gpt":0.3469797730443074,"score_spread":0.30694972089852424,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1507760270","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.88549495,0.0013413372,0.017563593,0.09288235,0.00052693015,0.00025920855,0.0000013943716,0.00052268,0.0014075453],"genre_scores_gemma":[0.99389094,0.000075074284,0.0016937098,0.0015135824,0.0011034801,0.000018089473,5.2992834e-7,0.000011534791,0.001693045],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9990851,0.000050330444,0.00013958653,0.000118026765,0.00033019847,0.00027670152],"domain_scores_gemma":[0.9993224,0.00041920293,0.000023531897,0.00007249837,0.00006841882,0.000093907955],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003790616,0.000081028535,0.00014012324,0.000015866368,0.00040718372,0.00002797136,0.00020763885,0.00011490132,0.00007895825],"category_scores_gemma":[0.0031549188,0.000054332355,0.00007346904,0.00042769927,0.00007157036,0.00003623494,0.00007836792,0.00025206426,0.0000414028],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003249875,0.00009565058,0.4681406,0.00005654714,0.00017166299,0.000018643623,0.33528307,0.011590143,0.00044007477,0.0048674913,0.095464736,0.08383889],"study_design_scores_gemma":[0.000039638442,0.000016527187,0.04554537,0.000022230643,0.0000070384767,5.891535e-7,0.0028950488,0.000026201978,0.000049986673,0.00011129457,0.951172,0.00011405142],"about_ca_topic_score_codex":0.0009527394,"about_ca_topic_score_gemma":0.0002238634,"teacher_disagreement_score":0.8557073,"about_ca_system_score_codex":0.000045314304,"about_ca_system_score_gemma":0.000015467089,"threshold_uncertainty_score":0.37769604},"labels":[],"label_agreement":null},{"id":"W1549553848","doi":"10.1023/a:1009815306478","title":"Replicated Case Studies for Investigating Quality Factors in Object-Oriented Designs","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":172,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec à Montréal; Carleton University","funders":"","keywords":"Computer science; Cohesion (chemistry); Software engineering; Quality (philosophy); Software quality; Set (abstract data type); Software; Data science; Data mining; Programming language; Software development","score_opus":0.1714365202462394,"score_gpt":0.4017966966694224,"score_spread":0.23036017642318302,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1549553848","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5189525,0.00018192614,0.479303,0.000103146114,0.00020666675,0.0002790871,0.0000031252198,0.00096923305,0.0000013381607],"genre_scores_gemma":[0.86920244,0.000009449936,0.13033313,0.00007308872,0.0000812809,0.00020414614,0.000007590294,0.00004896575,0.000039943283],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99725264,0.00008446789,0.0006259132,0.0007796744,0.00040370997,0.000853573],"domain_scores_gemma":[0.99150825,0.007170237,0.00007874339,0.00073504477,0.0002076808,0.00030004632],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009119512,0.0003459275,0.0004776801,0.000364631,0.00014851349,0.00009232421,0.0005686528,0.00015023837,0.0000034061964],"category_scores_gemma":[0.029265344,0.00033511576,0.0001259639,0.0018955828,0.000048900423,0.00041327454,0.00033485226,0.00045270668,0.000006748259],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010129972,0.00009263487,0.9558115,0.00024103398,0.00009073354,0.00092056615,0.0032813428,0.036205363,0.0009832871,0.0004024138,0.00046728615,0.0014936802],"study_design_scores_gemma":[0.0032571093,0.0005775087,0.78425235,0.0005509747,0.000033212888,0.0016687668,0.00053455634,0.18499532,0.012207504,0.0010128538,0.007923006,0.0029868393],"about_ca_topic_score_codex":0.00010451151,"about_ca_topic_score_gemma":0.000017988825,"teacher_disagreement_score":0.35024995,"about_ca_system_score_codex":0.00042733224,"about_ca_system_score_gemma":0.00008439602,"threshold_uncertainty_score":0.9999101},"labels":[],"label_agreement":null},{"id":"W1564555743","doi":"10.1023/a:1011998412776","title":"Why and How Research Ethics Matters to You. Yes, You!","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Ethics in Clinical Research","field":"Medicine","cited_by":29,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Computer science; Medicine","score_opus":0.5129399488320844,"score_gpt":0.562356893618366,"score_spread":0.04941694478628167,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1564555743","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2728018,0.0007525473,0.06370893,0.66089123,0.00036825318,0.00069743936,0.000009386676,0.0005149161,0.00025550023],"genre_scores_gemma":[0.71474046,0.002762354,0.11859428,0.1396841,0.002743841,0.0002885208,0.000045110082,0.00047365006,0.020667687],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.996122,0.00015157579,0.00027639113,0.00062192464,0.0019418423,0.0008862655],"domain_scores_gemma":[0.982476,0.015497013,0.000019942958,0.0006017203,0.00057983235,0.0008254843],"candidate_categories":["metaresearch","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.0061997077,0.00020847534,0.00037133397,0.00034078234,0.00017889796,0.0001242688,0.00029669682,0.0006877581,0.00014089864],"category_scores_gemma":[0.08489306,0.00018865755,0.00008840185,0.0009998748,0.0001946722,0.00009127271,0.00048684288,0.006550981,0.00013039056],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00054919306,0.00045937064,0.53000534,0.0029479265,0.0002732783,0.0018658304,0.0045122677,0.00091063755,0.0032038703,0.0014675944,0.44298378,0.010820918],"study_design_scores_gemma":[0.00071327656,0.0005613864,0.058300905,0.00072624674,0.000021466321,0.00011357158,0.00014468963,0.00051325496,0.00032297545,0.000829051,0.9374166,0.00033659078],"about_ca_topic_score_codex":0.00007713331,"about_ca_topic_score_gemma":0.000038302765,"teacher_disagreement_score":0.52120715,"about_ca_system_score_codex":0.00022783334,"about_ca_system_score_gemma":0.00021791129,"threshold_uncertainty_score":0.99574095},"labels":[],"label_agreement":null},{"id":"W1576984811","doi":"10.1023/a:1025320418915","title":"An Externally Replicated Experiment for Evaluating the Learning Effectiveness of Using Simulations in Software Project Management Education","year":2003,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"European Commission; Technische Universität Kaiserslautern","keywords":"Computer science; COCOMO; Debriefing; Software project management; Empirical research; Project management; Software; Software development; Process (computing); Software engineering; Knowledge management; Engineering management; Engineering; Systems engineering; Software construction; Psychology","score_opus":0.07108431282324686,"score_gpt":0.42070935125680736,"score_spread":0.3496250384335605,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1576984811","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44666272,0.00014626041,0.5521446,0.0000039475512,0.00013089384,0.0007049319,8.6026137e-7,0.00020409415,0.000001691323],"genre_scores_gemma":[0.69453835,0.0000011154782,0.30509984,0.000007103883,0.000019990366,0.00028454734,0.0000058869255,0.000031978223,0.0000112061125],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99792904,0.00029009613,0.0003913087,0.00053005625,0.0004379628,0.00042152556],"domain_scores_gemma":[0.99603724,0.0029753065,0.000087015804,0.0006409273,0.00018248512,0.000077048215],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0015904806,0.00021524321,0.00023854071,0.00033017178,0.00013492527,0.00009795148,0.0005919104,0.00007729711,0.0000030938659],"category_scores_gemma":[0.0041467436,0.00019396096,0.000080435704,0.0010831868,0.000019682915,0.00032327624,0.00011766101,0.0002668126,8.696973e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015105165,0.00015861142,0.08134501,0.00025468913,0.000025529369,0.0000014210711,0.00046571562,0.911322,0.0033425346,0.0004015722,0.0000033084273,0.002664538],"study_design_scores_gemma":[0.00089347275,0.00044085504,0.16841729,0.0005836771,0.000022554344,0.000015379792,0.00008038101,0.8181104,0.010176625,0.00043028634,0.00033111894,0.000497975],"about_ca_topic_score_codex":0.000030292169,"about_ca_topic_score_gemma":6.328894e-7,"teacher_disagreement_score":0.24787562,"about_ca_system_score_codex":0.00035722213,"about_ca_system_score_gemma":0.0002186752,"threshold_uncertainty_score":0.7909499},"labels":[],"label_agreement":null},{"id":"W1581639473","doi":"10.1007/s10664-012-9209-9","title":"Automated topic naming","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; University of British Columbia; University of Alberta","funders":"","keywords":"Computer science; Latent Dirichlet allocation; Topic model; Commit; Categorization; Software; Context (archaeology); Information retrieval; Domain (mathematical analysis); Artificial intelligence; Natural language processing; Data science; Database; Programming language","score_opus":0.0256654668367926,"score_gpt":0.2953701151461752,"score_spread":0.2697046483093826,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1581639473","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2284016,0.0005390106,0.7618927,0.00015155341,0.0010352217,0.00010344171,8.6386365e-7,0.007840191,0.00003541306],"genre_scores_gemma":[0.85822403,0.00000325243,0.14115793,0.00011013879,0.00031992287,0.00003233766,0.0000026090975,0.00003676363,0.00011302379],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99802995,0.000026750093,0.00023811581,0.00030946915,0.00045464156,0.0009410584],"domain_scores_gemma":[0.9978342,0.0011042304,0.000026205473,0.00058067695,0.000050903534,0.00040375098],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00037645802,0.00022706161,0.00020879786,0.00020781445,0.00008223888,0.00011562112,0.0007678248,0.00012302959,0.000032182503],"category_scores_gemma":[0.0025646968,0.00022544221,0.000088103596,0.0008306269,0.000014756464,0.00073641626,0.00039183418,0.00035683546,0.0002588585],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000001694712,0.000107995074,0.97366345,0.00011126098,0.00005558391,0.00004167399,0.0009761376,0.011116342,0.00047884934,0.00088078954,0.0058173877,0.006748844],"study_design_scores_gemma":[0.00026345171,0.000039738956,0.841128,0.000052413652,0.0000055491328,0.000076615426,0.00000417658,0.11647921,0.0014472018,0.000023060784,0.039908703,0.0005718668],"about_ca_topic_score_codex":0.0000064123483,"about_ca_topic_score_gemma":1.2447866e-7,"teacher_disagreement_score":0.62982243,"about_ca_system_score_codex":0.00018381016,"about_ca_system_score_gemma":0.000035762547,"threshold_uncertainty_score":0.91932666},"labels":[],"label_agreement":null},{"id":"W1589080523","doi":"10.1023/a:1023014713253","title":"An Investigation on the Occurrence of Service Requests in Commercial Software Applications","year":2003,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Reliability and Analysis Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Process (computing); Software quality; Service (business); Identification (biology); Software; Data mining; Reliability engineering; Software development; Engineering","score_opus":0.05197182144963392,"score_gpt":0.3105143093181651,"score_spread":0.2585424878685312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1589080523","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.26259613,0.0000639882,0.7357953,0.0009933586,0.000052160096,0.0002511404,0.0000074814466,0.00023096433,0.000009467744],"genre_scores_gemma":[0.9566641,0.0000076501865,0.04234078,0.0007805287,0.00003243275,0.00014679268,0.000013463335,0.000010975854,0.000003313556],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983722,0.0001996501,0.00033259403,0.0003778434,0.0004247532,0.00029295243],"domain_scores_gemma":[0.99736685,0.0015005817,0.000061006842,0.0007750814,0.00015504994,0.00014141829],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007843614,0.00015170129,0.00019987725,0.00016817312,0.00009778313,0.00006655514,0.00088919664,0.000093882365,0.000015654163],"category_scores_gemma":[0.0016468933,0.00012225869,0.00006225789,0.0020100342,0.0000473778,0.00028756904,0.000078975034,0.00038700964,0.000024974124],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008271277,0.00049538026,0.84782034,0.00023138936,0.000033030836,0.000007555244,0.0021897992,0.10595814,0.00031401054,0.017958106,0.0004947129,0.024489282],"study_design_scores_gemma":[0.0011982288,0.0005419247,0.7858499,0.0005522298,0.000036456495,0.000021964166,0.00013730333,0.15706402,0.014028675,0.02164332,0.017156469,0.0017695006],"about_ca_topic_score_codex":0.000054132855,"about_ca_topic_score_gemma":0.00002955944,"teacher_disagreement_score":0.69406796,"about_ca_system_score_codex":0.000087955195,"about_ca_system_score_gemma":0.0001225398,"threshold_uncertainty_score":0.49855646},"labels":[],"label_agreement":null},{"id":"W1600717382","doi":"10.1007/s10664-012-9200-5","title":"Understanding Ajax applications by connecting client and server-side execution traces","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Ajax; Computer science; World Wide Web; Web application; Client-side; Web 2.0; Web-based simulation; Field (mathematics); Web page; Web API","score_opus":0.06906239645443948,"score_gpt":0.29564332158658657,"score_spread":0.2265809251321471,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1600717382","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.13167274,0.0012146923,0.86532855,0.0002122781,0.00023137634,0.00020380424,0.0000036562765,0.0011141535,0.00001873308],"genre_scores_gemma":[0.9589939,0.000023070023,0.040623788,0.000058249607,0.00015728502,0.000076750846,0.000007658337,0.000034596673,0.00002468391],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99829453,0.000028794222,0.00024680738,0.00037869543,0.00037584506,0.0006753008],"domain_scores_gemma":[0.99769574,0.0014992625,0.000044135777,0.00035742842,0.000034454693,0.00036896757],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000502093,0.00021551001,0.00019072794,0.00015216568,0.0002063678,0.00017728955,0.0003783918,0.000115801406,0.000007655523],"category_scores_gemma":[0.0008539484,0.000226435,0.00005335289,0.0005988206,0.000024533498,0.0007244474,0.00026586957,0.00034024572,0.000022561893],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000120213135,0.0004196391,0.913027,0.00067038444,0.00020585752,0.000020181731,0.0050460733,0.031405315,0.0044958494,0.016928924,0.007831379,0.019937411],"study_design_scores_gemma":[0.003025227,0.00046756762,0.52073723,0.0005998411,0.000111945345,0.00064366177,0.00079786923,0.19632642,0.017676288,0.0023744474,0.25193158,0.0053079086],"about_ca_topic_score_codex":0.000010025547,"about_ca_topic_score_gemma":6.364444e-7,"teacher_disagreement_score":0.8273212,"about_ca_system_score_codex":0.00034645107,"about_ca_system_score_gemma":0.000019637582,"threshold_uncertainty_score":0.9233752},"labels":[],"label_agreement":null},{"id":"W1774183706","doi":"10.1023/a:1011485205161","title":"Quantitative Measurements of the Influence of Participant Roles during Peer Review Meetings","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Innovative Teaching and Learning Methods","field":"Psychology","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Division of Materials Research","keywords":"Artifact (error); Computer science; Protocol (science); Interpretation (philosophy); Content analysis; Focus (optics); Supervisor; Task (project management); Data science; Quality (philosophy); Knowledge management; Artificial intelligence; Engineering; Epistemology; Management","score_opus":0.1633991765249903,"score_gpt":0.4226693604507548,"score_spread":0.2592701839257645,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1774183706","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.99088436,0.0018395399,0.0062478683,0.0004748536,0.00014554527,0.00014637176,0.0000022340437,0.000085346095,0.00017388437],"genre_scores_gemma":[0.9908586,0.0000147141345,0.00855893,0.00014685022,0.000027020933,0.000021657674,0.0000010467137,0.000023990075,0.00034718696],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981006,0.00042001036,0.0004979731,0.00020189285,0.00049914035,0.00028042318],"domain_scores_gemma":[0.998564,0.00046793406,0.00023217345,0.00030483617,0.0003883464,0.000042693722],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.002540142,0.00015056865,0.00035210792,0.00006250569,0.00006270232,0.0000044473836,0.00025770548,0.00006419195,0.00008563364],"category_scores_gemma":[0.011569866,0.00010947461,0.00011046427,0.0005845227,0.000063893654,0.000047999813,0.00007026372,0.00038859574,0.000010280837],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000047337202,0.00010239769,0.9569728,0.0006062889,0.00017726018,0.000004901712,0.005918234,0.020126086,0.014855932,0.0001268798,0.000532771,0.00052915106],"study_design_scores_gemma":[0.00020897987,0.00005682362,0.9932513,0.0015850011,0.000052452124,0.0000082449515,0.00006252124,0.000026674525,0.0020643016,0.000012816915,0.0025327883,0.00013805335],"about_ca_topic_score_codex":0.00004755595,"about_ca_topic_score_gemma":0.0000011009865,"teacher_disagreement_score":0.036278583,"about_ca_system_score_codex":0.000032258613,"about_ca_system_score_gemma":0.000018901084,"threshold_uncertainty_score":0.9967561},"labels":[],"label_agreement":null},{"id":"W1838805003","doi":"10.1023/a:1011487332587","title":"Modelling the Likelihood of Software Process Improvement: An Exploratory Study","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":63,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Capability Maturity Model; Computer science; Sample (material); Process (computing); Empirical research; Software; Process management; Data science; Engineering; Mathematics; Statistics","score_opus":0.03668895162732284,"score_gpt":0.2952584062826417,"score_spread":0.2585694546553189,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1838805003","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.47593856,0.00014883178,0.52248305,0.000042089046,0.00019846391,0.000334937,0.0000017811805,0.0008506225,0.0000016709268],"genre_scores_gemma":[0.9717931,0.000009426023,0.02766735,0.00005799026,0.0001771641,0.00019885773,0.000003390578,0.00007517644,0.000017551587],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99674714,0.00007152919,0.00055713346,0.0007302206,0.0010569717,0.000837027],"domain_scores_gemma":[0.9968573,0.0010673101,0.00009799814,0.0013536988,0.00029532088,0.00032838178],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00091889367,0.00038473398,0.00038501967,0.00026656777,0.00018056927,0.00017139243,0.0021332873,0.000119553064,0.000015637415],"category_scores_gemma":[0.0011863139,0.00030945294,0.00011996839,0.0015391221,0.000042137857,0.0009538535,0.0004884697,0.0006160152,0.000019864867],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000129603,0.0005122872,0.11283214,0.000097159755,0.00007421065,0.00006342389,0.006870897,0.8731896,0.00009262175,0.000020585861,0.00006514456,0.006168955],"study_design_scores_gemma":[0.0014166399,0.0016127987,0.022680473,0.00015477223,0.00004434434,0.000048554895,0.0009372555,0.9681733,0.0018169231,0.000600022,0.0012495025,0.0012653666],"about_ca_topic_score_codex":0.000038842576,"about_ca_topic_score_gemma":0.0000050770204,"teacher_disagreement_score":0.49585453,"about_ca_system_score_codex":0.00012303966,"about_ca_system_score_gemma":0.00014954407,"threshold_uncertainty_score":0.99993575},"labels":[],"label_agreement":null},{"id":"W1909497710","doi":"10.1007/s10664-015-9396-2","title":"Towards building a universal defect prediction model with rank transformed predictors","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":116,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"University of Victoria","keywords":"Computer science; Software; Rank (graph theory); Workflow; Predictive modelling; Context (archaeology); Eclipse; Data mining; Software development; Obstacle; Software bug; Software engineering; Machine learning; Database; Programming language; Mathematics","score_opus":0.02760184776548904,"score_gpt":0.2626829102734268,"score_spread":0.23508106250793773,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1909497710","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.20122969,0.00008754126,0.7953404,0.00013353393,0.00029044764,0.00023702411,0.000009530021,0.0026337202,0.00003806324],"genre_scores_gemma":[0.7468565,0.0000049000946,0.25282267,0.00003269863,0.00011652349,0.00004811522,0.000007085848,0.000057843026,0.000053639396],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99732697,0.000033019944,0.0002783145,0.0006072115,0.0009985345,0.0007559236],"domain_scores_gemma":[0.9981569,0.0004277504,0.000034452907,0.000548059,0.00017663697,0.00065622304],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004967626,0.00034708492,0.00032045876,0.00035326788,0.000083798644,0.00014888294,0.0008487701,0.0001723722,0.000004977282],"category_scores_gemma":[0.0013119916,0.00031001252,0.00013470055,0.0011451978,0.00004109129,0.0008990088,0.00018490569,0.00053048856,0.000016687836],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003941064,0.00004898979,0.043967664,0.00006089411,0.00009547183,0.00006731994,0.0012146629,0.95154595,0.00012505853,0.00032175487,0.0009971692,0.0015156238],"study_design_scores_gemma":[0.001483678,0.00037804595,0.016195603,0.00010688604,0.00002944782,0.00008541723,0.000014836862,0.9768612,0.0008202167,0.00015294291,0.0033599953,0.0005117554],"about_ca_topic_score_codex":0.000022381697,"about_ca_topic_score_gemma":0.0000016455015,"teacher_disagreement_score":0.5456268,"about_ca_system_score_codex":0.00055933953,"about_ca_system_score_gemma":0.0004254069,"threshold_uncertainty_score":0.9999352},"labels":[],"label_agreement":null},{"id":"W1914969610","doi":"10.1007/s10664-015-9393-5","title":"An in-depth study of the promises and perils of mining GitHub","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":270,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Computer science; Software; Point (geometry); Data science; World Wide Web; Set (abstract data type); Event (particle physics); Empirical research; The Internet; Quality (philosophy)","score_opus":0.05189227763191294,"score_gpt":0.31378800903066856,"score_spread":0.2618957313987556,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1914969610","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95690686,0.00019010458,0.042313367,0.00003113473,0.00014767566,0.00024866522,7.5769844e-7,0.00015651772,0.0000049166783],"genre_scores_gemma":[0.98104054,0.0000011549059,0.018880783,0.00000702895,0.000025435054,0.00002119375,1.9065985e-7,0.000017559703,0.000006089214],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99863636,0.00005753996,0.00029211806,0.00028782734,0.0004642531,0.00026188573],"domain_scores_gemma":[0.99852276,0.0006166723,0.000049222766,0.0005647362,0.00009146906,0.00015515639],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004735464,0.00014106055,0.00025362102,0.00016573192,0.000021222544,0.000036967373,0.00075269514,0.000054554795,0.0000011300592],"category_scores_gemma":[0.002513944,0.00010949884,0.000030619387,0.0007739405,0.000034144567,0.00026668937,0.00037155193,0.00018937465,5.066326e-7],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000047811377,0.00015806504,0.96640176,0.000044115033,0.000010812448,0.0000071144595,0.0070909373,0.02469545,0.0002057069,0.0000108307995,0.000028269036,0.0013421893],"study_design_scores_gemma":[0.00058985763,0.00037663843,0.97949326,0.00006407967,0.0000038030357,0.000008968999,0.0003049112,0.01791116,0.0010103154,0.000015041484,0.000074848314,0.0001470959],"about_ca_topic_score_codex":0.00003816221,"about_ca_topic_score_gemma":0.000012373299,"teacher_disagreement_score":0.024133703,"about_ca_system_score_codex":0.000056948043,"about_ca_system_score_gemma":0.00009241549,"threshold_uncertainty_score":0.4465233},"labels":[],"label_agreement":null},{"id":"W1963574127","doi":"10.1007/s10664-014-9303-2","title":"Mining system logs to learn error predictors: a case study of a telemetry system","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":43,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Telemetry; Data mining; Computer science; Support vector machine; Robustness (evolution); Reliability (semiconductor); Word error rate; Software; Machine learning; Artificial intelligence","score_opus":0.0179966043784878,"score_gpt":0.2613391865108126,"score_spread":0.24334258213232482,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1963574127","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5692862,0.000026362863,0.428531,0.000012992623,0.00064646313,0.00038284756,0.0000014371283,0.0010989236,0.00001374735],"genre_scores_gemma":[0.97373575,9.168691e-8,0.02587168,0.000021907228,0.00021672682,0.00010264085,7.088583e-7,0.00003394126,0.000016582038],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9972941,0.00014909785,0.0007891047,0.0006681313,0.0005831048,0.000516438],"domain_scores_gemma":[0.9976931,0.0006126524,0.0001470715,0.0010460878,0.0001684897,0.00033255466],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010783526,0.00031859695,0.0006264704,0.00030401355,0.0001448489,0.00007871458,0.00074563106,0.00015120534,0.0000018069926],"category_scores_gemma":[0.00070762826,0.00026904163,0.00012544614,0.001128162,0.000015856318,0.0002692092,0.00042830422,0.00024109267,0.000032881962],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018716559,0.0003699878,0.8943866,0.0031349547,0.00016012046,0.0008723317,0.014711765,0.08009302,0.00004137607,0.00008515254,0.00047108915,0.0056548985],"study_design_scores_gemma":[0.003314951,0.0044535236,0.11683118,0.0027076772,0.0001599273,0.0065831216,0.018798517,0.84023505,0.0004817106,0.0000018772797,0.004373662,0.0020587845],"about_ca_topic_score_codex":0.00021662659,"about_ca_topic_score_gemma":0.00001098405,"teacher_disagreement_score":0.7775554,"about_ca_system_score_codex":0.0002784177,"about_ca_system_score_gemma":0.000050680068,"threshold_uncertainty_score":0.99997616},"labels":[],"label_agreement":null},{"id":"W1969265968","doi":"10.1007/s10664-007-9054-4","title":"Analysis of attribute weighting heuristics for analogy-based software effort estimation method AQUA+","year":2007,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":77,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Weighting; Heuristics; Data mining; Set (abstract data type); Computer science; Selection (genetic algorithm); Estimation; Exploit; A-weighting; Machine learning; Engineering","score_opus":0.02959484286795332,"score_gpt":0.3421866429248011,"score_spread":0.3125918000568478,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1969265968","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.054991204,0.0001324139,0.94293934,0.000073540905,0.00033726305,0.00032916103,0.000041354648,0.001154139,0.0000015725672],"genre_scores_gemma":[0.30237654,9.4977156e-7,0.6973307,0.00005554195,0.00007078233,0.000033223445,0.000078897734,0.00004126346,0.000012087116],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99677527,0.000035121855,0.0008217567,0.0006957007,0.0007634934,0.0009086732],"domain_scores_gemma":[0.98792017,0.010339959,0.00018370693,0.0008329268,0.00039813103,0.00032513894],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0023225804,0.00036468892,0.0007332424,0.0013939566,0.00012779486,0.00008787278,0.00095577014,0.00024423227,0.000011631899],"category_scores_gemma":[0.011102329,0.00038111108,0.00045800747,0.0041720485,0.000034514327,0.00025784032,0.00021475078,0.00037499875,0.0000057721218],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012552878,0.000058426882,0.28947613,0.00019578534,0.00035777275,0.00001816565,0.00011112313,0.70044535,0.000116392766,0.00015229308,0.00015349756,0.008902516],"study_design_scores_gemma":[0.00036175916,0.0001028426,0.22258024,0.00004785302,0.0001984567,0.0000039949273,0.0000023898435,0.7718169,0.0036386717,0.000059988466,0.0008268434,0.00036007143],"about_ca_topic_score_codex":0.000020051166,"about_ca_topic_score_gemma":0.0000049091677,"teacher_disagreement_score":0.24738534,"about_ca_system_score_codex":0.00028111637,"about_ca_system_score_gemma":0.0001279791,"threshold_uncertainty_score":0.9998641},"labels":[],"label_agreement":null},{"id":"W1975236650","doi":"10.1007/s10664-008-9101-9","title":"A subject-based empirical evaluation of SSUCD’s performance in reducing inconsistencies in use case models","year":2008,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Quality and Management","field":"Decision Sciences","cited_by":23,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Subject (documents); Empirical research; Statistics; Mathematics; World Wide Web","score_opus":0.4411743086357212,"score_gpt":0.4184957454988292,"score_spread":0.022678563136892038,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1975236650","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9595954,0.000097536,0.03960092,0.00016467895,0.00013247886,0.0002834544,0.0000137467,0.00006190276,0.00004988566],"genre_scores_gemma":[0.990956,0.000007691159,0.008783903,0.00013443007,0.000020383797,0.000044632507,0.000008936475,0.000013576553,0.000030452802],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99626523,0.000297752,0.0010059993,0.0004952896,0.0015888251,0.00034692904],"domain_scores_gemma":[0.99684197,0.0021196643,0.00013193836,0.00056885334,0.00023619483,0.00010136779],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0054510557,0.00018426434,0.00040780843,0.0007623269,0.00006795343,0.00005551094,0.00034529486,0.00010371626,0.00004820182],"category_scores_gemma":[0.010492348,0.00016437264,0.00008969029,0.0016790298,0.00007475567,0.00093052397,0.00017143998,0.0002596431,0.000014071563],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000022451099,0.00010849999,0.23521264,0.000027580456,0.0000048058755,0.00018463978,0.0015901679,0.7592541,0.000009636747,0.000012535744,0.00035455296,0.0032183772],"study_design_scores_gemma":[0.00066521455,0.000047364938,0.2587697,0.0000832254,0.000010366755,0.000070832684,0.00019593397,0.73913395,0.00014735738,0.0001514614,0.0005274688,0.0001971343],"about_ca_topic_score_codex":0.00038918082,"about_ca_topic_score_gemma":0.00028440292,"teacher_disagreement_score":0.031360596,"about_ca_system_score_codex":0.0002724915,"about_ca_system_score_gemma":0.00022191186,"threshold_uncertainty_score":0.99784267},"labels":[],"label_agreement":null},{"id":"W1979991500","doi":"10.1007/s10664-012-9224-x","title":"Configuring latent Dirichlet allocation based feature location","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":92,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"U.S. Department of Education; National Science Foundation","keywords":"Latent Dirichlet allocation; Computer science; Feature (linguistics); Source code; Heuristics; Context (archaeology); Java; Artificial intelligence; Topic model; Code (set theory); Measure (data warehouse); Data mining; Natural language processing; Information retrieval; Programming language","score_opus":0.024902158345490696,"score_gpt":0.2743756658225636,"score_spread":0.24947350747707292,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1979991500","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059462793,0.00075892924,0.9360986,0.0008566431,0.0007749467,0.00020410572,0.0000014435713,0.0018269736,0.000015591597],"genre_scores_gemma":[0.90142494,0.0000037369437,0.09778346,0.00021896734,0.0003326907,0.00007093789,0.000016982434,0.000044790395,0.000103507125],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9979314,0.000039658604,0.0002351536,0.000390513,0.0005963015,0.00080696656],"domain_scores_gemma":[0.9977708,0.0009816218,0.00004497102,0.00065416313,0.00015603304,0.00039244967],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00050539227,0.00027360665,0.00021188828,0.00025733552,0.000097354074,0.00014245596,0.000673337,0.0001727861,0.000021975486],"category_scores_gemma":[0.002307262,0.00027506167,0.000083705665,0.0011306858,0.000017708548,0.00072021544,0.00017420958,0.00045036533,0.00016250178],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000075586454,0.00020564262,0.7764213,0.00028123692,0.000058450438,0.000017516428,0.0006431837,0.20783025,0.0011965436,0.00077072874,0.0063440353,0.006223564],"study_design_scores_gemma":[0.0003851537,0.000047527337,0.826615,0.00010860064,0.000010406488,0.000022648957,0.0000028682682,0.14128198,0.0057517183,0.000014084213,0.025170034,0.00059000676],"about_ca_topic_score_codex":0.000007759668,"about_ca_topic_score_gemma":3.133222e-7,"teacher_disagreement_score":0.84196216,"about_ca_system_score_codex":0.00029608846,"about_ca_system_score_gemma":0.000076927274,"threshold_uncertainty_score":0.99997014},"labels":[],"label_agreement":null},{"id":"W1980130600","doi":"10.1007/s10664-014-9329-5","title":"Special issue on program comprehension","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Program comprehension; Computer science; Comprehension; Data science; Psychology; Programming language; Software","score_opus":0.021811196545500226,"score_gpt":0.29366010915216767,"score_spread":0.27184891260666744,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1980130600","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08910202,0.000039512837,0.8985508,0.0007765267,0.0037397693,0.00049191294,0.0000016458937,0.0065642046,0.0007335932],"genre_scores_gemma":[0.438844,0.0000074968034,0.52990043,0.0008834523,0.029095111,0.00023870938,0.000018141896,0.00019670647,0.00081593555],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9977339,0.000046604808,0.00026006793,0.00059181266,0.00068921165,0.00067843194],"domain_scores_gemma":[0.9970353,0.0017421567,0.00003102828,0.00078872096,0.000076964134,0.00032579206],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00030675484,0.00029175766,0.00028086265,0.00023681167,0.000098930475,0.00019542998,0.0009592364,0.0001415884,0.00008118547],"category_scores_gemma":[0.0027072846,0.00027852468,0.00011347811,0.0007316675,0.000025994199,0.00022823775,0.00034371277,0.0005190719,0.0007632417],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024449531,0.0005494031,0.077232294,0.0001999068,0.00006486598,0.00008138821,0.0005140287,0.142138,0.00012413297,0.0017184659,0.14884326,0.62850976],"study_design_scores_gemma":[0.0003148482,0.00036525735,0.13084953,0.00006234234,0.0000026067955,0.000011156703,8.538762e-7,0.08411576,0.00041017312,0.0000514992,0.7834205,0.00039548575],"about_ca_topic_score_codex":0.0000034949928,"about_ca_topic_score_gemma":3.3978452e-7,"teacher_disagreement_score":0.6345772,"about_ca_system_score_codex":0.00012750998,"about_ca_system_score_gemma":0.000027226926,"threshold_uncertainty_score":0.9999667},"labels":[],"label_agreement":null},{"id":"W1981075560","doi":"10.1007/s10664-013-9274-8","title":"Studying the relationship between logging characteristics and the code quality of platform software","year":2013,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":83,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Software; Product metric; Process (computing); Relation (database); Source lines of code; Logging; Software quality; Code (set theory); Database; Quality (philosophy); Product (mathematics); Software development; Software engineering; Data mining; Operating system; Programming language; Set (abstract data type)","score_opus":0.1071977212306796,"score_gpt":0.3313187722431654,"score_spread":0.2241210510124858,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1981075560","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.50311464,0.00016495473,0.49533394,0.000576482,0.00013185015,0.00030220608,0.000005694165,0.00036813976,0.00000209906],"genre_scores_gemma":[0.97429264,0.0000060028187,0.025304144,0.0000858357,0.00014586384,0.000094033996,0.0000049355326,0.000032348402,0.000034211494],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99772894,0.00012049367,0.00063360995,0.00037224806,0.0006386825,0.00050604256],"domain_scores_gemma":[0.9680029,0.030735046,0.00014768876,0.0008201418,0.00013706852,0.00015714028],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0017604147,0.00025515092,0.00043133504,0.00012646597,0.0002871379,0.00024914966,0.0011420447,0.00012090842,0.0000089370205],"category_scores_gemma":[0.02213654,0.00016099619,0.00011269707,0.00060617714,0.00017677147,0.00047630214,0.00070792146,0.0007497124,0.000023999457],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000035145993,0.000010286147,0.99438894,0.00010620411,0.00003620528,0.0000012432237,0.000999135,0.00066993444,0.000007713138,0.00092506537,0.00013308946,0.0027186454],"study_design_scores_gemma":[0.00043651584,0.00002056346,0.99294794,0.0000580676,0.000010339071,0.0000059314093,0.000026128118,0.005069331,0.000029456705,0.0009366482,0.00025611254,0.00020296972],"about_ca_topic_score_codex":0.00007780483,"about_ca_topic_score_gemma":0.0000011816257,"teacher_disagreement_score":0.471178,"about_ca_system_score_codex":0.00008526451,"about_ca_system_score_gemma":0.000050965344,"threshold_uncertainty_score":0.98610044},"labels":[],"label_agreement":null},{"id":"W1981621595","doi":"10.1007/s10664-010-9132-x","title":"Testing the theory of relative defect proneness for closed-source software","year":2010,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Children's Hospital of Eastern Ontario","funders":"","keywords":"Open source software; Software; Open source; Computer science; Software engineering; Data science; Programming language","score_opus":0.0349592160170909,"score_gpt":0.28196653910387043,"score_spread":0.24700732308677953,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1981621595","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.20703787,0.00016295003,0.79043317,0.00012305474,0.000546767,0.0005202464,0.0000072251414,0.0011608421,0.000007897891],"genre_scores_gemma":[0.6399484,6.529411e-7,0.3594208,0.000051039162,0.00021192401,0.00020882516,0.0000034063826,0.00007197061,0.00008292928],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9976843,0.00007339687,0.00042068717,0.0005749086,0.00053631177,0.0007104018],"domain_scores_gemma":[0.9759064,0.022464162,0.00011308575,0.001005699,0.00032391053,0.0001867845],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0015577896,0.0003288211,0.00035407403,0.00018555565,0.00021371106,0.00011138044,0.0016047909,0.00020301843,0.000009388659],"category_scores_gemma":[0.04230084,0.00025211033,0.00025140378,0.0011166777,0.000110566754,0.00040600574,0.0004977867,0.00092229294,0.000013761483],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007928372,0.00031720474,0.67945296,0.0010771969,0.0004909778,0.000035678167,0.0044331225,0.09814982,0.012514143,0.01438901,0.0013980618,0.18766257],"study_design_scores_gemma":[0.0017040196,0.0006327985,0.85972345,0.00040274792,0.00010273215,0.0001636397,0.000045043413,0.08449407,0.015578199,0.009582237,0.025744302,0.0018267804],"about_ca_topic_score_codex":0.000008553082,"about_ca_topic_score_gemma":0.0000011977335,"teacher_disagreement_score":0.4329106,"about_ca_system_score_codex":0.000070011985,"about_ca_system_score_gemma":0.00015139184,"threshold_uncertainty_score":0.9999931},"labels":[],"label_agreement":null},{"id":"W1982036946","doi":"10.1007/s10664-013-9260-1","title":"An experimental investigation on the effects of context on source code identifiers splitting and expansion","year":2013,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Identifier; Computer science; Source code; Program comprehension; Context (archaeology); Documentation; Acronym; Unique identifier; Internal documentation; Code (set theory); Set (abstract data type); Software documentation; World Wide Web; Information retrieval; Software; Programming language; Software system; Software development; Software development process; Linguistics; Software construction","score_opus":0.021987243302096875,"score_gpt":0.26804226194432085,"score_spread":0.24605501864222398,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1982036946","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8269589,0.00014846341,0.17179823,0.0001797046,0.00022943004,0.00030566013,4.7579834e-7,0.00037718652,0.000001978847],"genre_scores_gemma":[0.99421,0.0000020624416,0.0053193956,0.00026760437,0.00005782186,0.00008834815,0.0000015839353,0.000029491981,0.000023661092],"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.9985512,0.00007236265,0.00021992072,0.00038621417,0.0004531449,0.0003171652],"domain_scores_gemma":[0.99550956,0.0036755502,0.000048623613,0.00050905533,0.00005241338,0.00020476565],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00028069338,0.00019758429,0.00017762581,0.00013835357,0.00010526831,0.00015732681,0.00048521569,0.00008255862,0.000008768182],"category_scores_gemma":[0.0027387342,0.00015374528,0.00004646792,0.0003043611,0.00005734771,0.00035580705,0.0001497258,0.00030260772,0.000026247251],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000048776546,0.0006785723,0.31766257,0.0011468731,0.00019425916,0.0000656466,0.028452767,0.096542604,0.439699,0.0040259,0.008019487,0.10346356],"study_design_scores_gemma":[0.00076358195,0.0011730817,0.27252394,0.00069910113,0.000007669824,0.000013737881,0.00019832459,0.19954246,0.52384466,0.00016661476,0.00042112547,0.0006456858],"about_ca_topic_score_codex":0.000027933538,"about_ca_topic_score_gemma":2.2886432e-7,"teacher_disagreement_score":0.16725115,"about_ca_system_score_codex":0.00007987529,"about_ca_system_score_gemma":0.00001956101,"threshold_uncertainty_score":0.62695503},"labels":[],"label_agreement":null},{"id":"W1985892950","doi":"10.1007/s10664-015-9371-y","title":"An empirical study of integration activities in distributions of open source software","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":29,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; Queen's University; Polytechnique Montréal","funders":"","keywords":"Computer science; Component (thermodynamics); Reuse; System integration; Software engineering; Software; Context (archaeology); Software quality; Component-based software engineering; Software development; Systems engineering; Engineering; Operating system","score_opus":0.06030563834253202,"score_gpt":0.3592406911326763,"score_spread":0.2989350527901443,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1985892950","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49736136,0.000033714274,0.5019266,0.00003320527,0.000107120584,0.00026968523,0.000007043487,0.0002587615,0.0000025079962],"genre_scores_gemma":[0.94102144,0.000001052593,0.058776855,0.000011322704,0.00003952617,0.00008511166,0.0000150568385,0.00003308343,0.00001656316],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99759775,0.00014892792,0.0005891406,0.0005109319,0.00071181153,0.00044145936],"domain_scores_gemma":[0.99711585,0.0013714879,0.00010382796,0.00089141174,0.00021568612,0.0003017212],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008662375,0.00025902805,0.0005150956,0.00042363507,0.0000435207,0.00014687894,0.0019511235,0.00013433919,0.000006790422],"category_scores_gemma":[0.005065159,0.00025370254,0.00006658448,0.0016664905,0.000046855996,0.0011782649,0.00087283057,0.00045922646,0.0000035565913],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027678934,0.001172528,0.9035181,0.00004398621,0.000030390358,0.000018299585,0.0060234615,0.08377262,0.00017125029,0.00005257028,0.00039594783,0.0047731875],"study_design_scores_gemma":[0.0020676944,0.0017852407,0.92369264,0.00018174348,0.000016626456,0.000019157018,0.0010754688,0.06584404,0.003699983,0.00026190045,0.00068712345,0.00066840136],"about_ca_topic_score_codex":0.00030689203,"about_ca_topic_score_gemma":0.000048737817,"teacher_disagreement_score":0.44366005,"about_ca_system_score_codex":0.00028010184,"about_ca_system_score_gemma":0.00022994286,"threshold_uncertainty_score":0.99999154},"labels":[],"label_agreement":null},{"id":"W1987409374","doi":"10.1007/s10664-008-9084-6","title":"Another viewpoint on “evaluating web software reliability based on workload and failure data extracted from server logs”","year":2008,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Reliability and Analysis Research","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Workload; Reliability (semiconductor); Computer science; Replication (statistics); Reliability engineering; Software quality; Web application; Term (time); Data mining; Software; World Wide Web; Engineering; Statistics; Software development; Operating system","score_opus":0.08123406468878247,"score_gpt":0.3199451369208011,"score_spread":0.23871107223201865,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1987409374","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4123276,0.00029956765,0.58386445,0.0019493002,0.00015044305,0.00028601897,0.000058160527,0.0010561008,0.000008349882],"genre_scores_gemma":[0.6820945,0.00005738727,0.315537,0.0017709819,0.00023850794,0.000049224425,0.000103502156,0.00007242258,0.000076496224],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.995521,0.00028980718,0.0005774604,0.0016340318,0.0012844811,0.0006932264],"domain_scores_gemma":[0.9922481,0.0043278993,0.00010631193,0.0028032626,0.00017868105,0.00033570617],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0011863708,0.00046357463,0.00056984177,0.00022878601,0.00030892485,0.0001907357,0.0016807633,0.0002955817,0.00017193695],"category_scores_gemma":[0.0076682307,0.00040468964,0.00017101326,0.0010892368,0.00010921696,0.00059105863,0.00080516544,0.0010101192,0.00011904037],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00024070432,0.0016884321,0.58530873,0.00041480656,0.00028075514,0.00042440457,0.0011868854,0.2951661,0.0006537586,0.00005798776,0.009971132,0.10460629],"study_design_scores_gemma":[0.0010132845,0.0003014305,0.1601306,0.0004338396,0.000028605331,0.000010990271,0.000008477783,0.8281066,0.00030684826,0.00023919152,0.008633146,0.00078700134],"about_ca_topic_score_codex":0.00007434332,"about_ca_topic_score_gemma":0.000012527767,"teacher_disagreement_score":0.53294045,"about_ca_system_score_codex":0.00021110011,"about_ca_system_score_gemma":0.0002269843,"threshold_uncertainty_score":0.9998405},"labels":[],"label_agreement":null},{"id":"W1987810861","doi":"10.1007/s10664-005-1288-4","title":"Requirements Engineering and Downstream Software Development: Findings from a Case Study","year":2005,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":53,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"University of Victoria","keywords":"Software project management; Software development process; Software development; Software engineering; Computer science; Requirements analysis; Requirement prioritization; Software requirements; Requirements engineering; Process management; Requirement; Documentation; Personal software process; Team software process; Systems engineering; Engineering; Software construction; Software","score_opus":0.030852546566778504,"score_gpt":0.28626038873216897,"score_spread":0.2554078421653905,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1987810861","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5011283,0.00028690213,0.4959051,0.000063795844,0.00018912503,0.00021228995,0.0000032852076,0.0022092524,0.0000019424654],"genre_scores_gemma":[0.5251875,0.000007060004,0.47447497,0.000074497744,0.00012226331,0.00006809314,0.0000046029277,0.000043347663,0.000017643899],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9975104,0.000029022212,0.000546841,0.0008156011,0.00045811915,0.0006400513],"domain_scores_gemma":[0.99793595,0.00090841844,0.00007870655,0.0006588402,0.00006287538,0.00035519683],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004100577,0.00049756805,0.00040069668,0.00028951347,0.00020028745,0.00036955182,0.0006630176,0.00017201761,0.000028367607],"category_scores_gemma":[0.0009603121,0.0005127014,0.000074409116,0.00057204906,0.000014752444,0.0012724367,0.0006508952,0.00050465536,0.000033444678],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000030276327,0.0011563206,0.6846979,0.0002773206,0.00082187616,0.008603943,0.020905212,0.05155873,0.0004278786,0.00013241451,0.0020952187,0.2292929],"study_design_scores_gemma":[0.0066193035,0.001349052,0.28989604,0.0011020779,0.00034646253,0.008608603,0.00058761,0.31528902,0.009479703,0.000149711,0.35763082,0.008941601],"about_ca_topic_score_codex":0.00010938651,"about_ca_topic_score_gemma":0.000014159871,"teacher_disagreement_score":0.39480188,"about_ca_system_score_codex":0.0002053465,"about_ca_system_score_gemma":0.00005484547,"threshold_uncertainty_score":0.99973243},"labels":[],"label_agreement":null},{"id":"W1987963388","doi":"10.1007/s10664-015-9370-z","title":"Studying high impact fix-inducing changes","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":46,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Change impact analysis; Computer science; Software; Quality (philosophy); Risk analysis (engineering); Data science; Software engineering","score_opus":0.07085915187211694,"score_gpt":0.32367575130540127,"score_spread":0.25281659943328433,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1987963388","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44260684,0.00032972108,0.55375916,0.00035781227,0.00074884197,0.00015624863,0.000002811834,0.0020287747,0.000009757317],"genre_scores_gemma":[0.92032164,0.00000449314,0.07894745,0.00010328161,0.0004195968,0.000050427945,0.0000053452313,0.00006656037,0.00008118697],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9972064,0.000049021502,0.00027086324,0.00063897274,0.000878246,0.0009565202],"domain_scores_gemma":[0.99692327,0.0012798931,0.000048037247,0.0008556065,0.00017345099,0.0007197659],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00076357194,0.0003694192,0.0003773892,0.0003835828,0.000091762464,0.00028800647,0.0012070418,0.0001526289,0.000020975678],"category_scores_gemma":[0.0055309413,0.00034029866,0.00011146755,0.0012941854,0.00002012224,0.0005284084,0.0007081624,0.0005732003,0.0001482945],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020239653,0.00021707651,0.6126105,0.00015333493,0.00026431892,0.0005481728,0.004489284,0.339859,0.00073785405,0.00028269287,0.015603761,0.025213765],"study_design_scores_gemma":[0.0026938128,0.0012364441,0.82861435,0.00036251007,0.00003544955,0.00031641684,0.000088105924,0.13592498,0.0053059617,0.0004891879,0.022062754,0.0028700414],"about_ca_topic_score_codex":0.000102342456,"about_ca_topic_score_gemma":0.0000027982762,"teacher_disagreement_score":0.4777148,"about_ca_system_score_codex":0.0005485412,"about_ca_system_score_gemma":0.00016831016,"threshold_uncertainty_score":0.99990493},"labels":[],"label_agreement":null},{"id":"W1991263523","doi":"10.1007/s10664-009-9110-3","title":"Guest editors introduction: special issue on mining software repositories","year":2009,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Mining Algorithms and Applications","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"European Metrology Programme for Innovation and Research","keywords":"Computer science; Software engineering; Data science; World Wide Web","score_opus":0.010820047409559233,"score_gpt":0.25210937070345196,"score_spread":0.24128932329389274,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1991263523","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.003174739,0.00006901939,0.95199597,0.0075319475,0.034595285,0.00017493362,0.000013105956,0.0021964977,0.00024849403],"genre_scores_gemma":[0.001644549,0.0000048523566,0.611892,0.0004076793,0.38538793,0.000028720007,0.000038026432,0.000027353368,0.00056893006],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99833965,0.00001428651,0.0003203386,0.0005899723,0.00036691973,0.00036881768],"domain_scores_gemma":[0.9987952,0.00020078591,0.000068567686,0.00068537996,0.00007302989,0.00017704596],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00015090748,0.00023093399,0.00021119406,0.000111361005,0.00022201205,0.00027096068,0.0006512321,0.00010468785,0.000029201565],"category_scores_gemma":[0.0007870589,0.00022875289,0.00007153706,0.00062613894,0.000021170998,0.00045099345,0.00012061492,0.00026610764,0.00011734478],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000003856412,0.00009959805,0.0010055848,0.000010190056,0.000014576052,0.000030034531,0.00047291507,0.0018775125,0.000027660364,0.0004423961,0.89495623,0.101059444],"study_design_scores_gemma":[0.000111467765,0.00016345698,0.0064461804,0.000030489537,0.000005881554,0.000030268146,0.000009546988,0.0029027294,0.00069105433,0.00006105864,0.9892403,0.0003075947],"about_ca_topic_score_codex":0.0000029894873,"about_ca_topic_score_gemma":2.3839135e-7,"teacher_disagreement_score":0.35079265,"about_ca_system_score_codex":0.000097583325,"about_ca_system_score_gemma":0.000035061992,"threshold_uncertainty_score":0.9328272},"labels":[],"label_agreement":null},{"id":"W1991613282","doi":"10.1007/s10664-010-9150-8","title":"A field study of API learning obstacles","year":2010,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":352,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"McGill University; Microsoft Research","keywords":"Documentation; Computer science; Programmer; Presentation (obstetrics); Field (mathematics); World Wide Web; Software engineering; Software documentation; Internal documentation; Multimedia; Software development; Software; Programming language; Software development process","score_opus":0.017733432877838518,"score_gpt":0.2866929103016268,"score_spread":0.2689594774237883,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1991613282","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6592954,0.000033682245,0.33926836,0.000055461238,0.00041165124,0.0001304024,2.7348602e-7,0.0007940628,0.000010749989],"genre_scores_gemma":[0.96007633,0.000001360776,0.039677966,0.00002276893,0.000102198996,0.000030874315,6.6833024e-7,0.00002952015,0.000058304107],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99831146,0.000030027883,0.00030320624,0.000400881,0.0005209421,0.00043350633],"domain_scores_gemma":[0.9962434,0.0027953342,0.00004697959,0.0006287479,0.00010003984,0.00018550693],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00035128547,0.00019817156,0.00026172752,0.00023916886,0.000070976246,0.00008278649,0.0009648062,0.00012082106,0.000036475318],"category_scores_gemma":[0.007420908,0.00019523875,0.00007719741,0.00076068833,0.0000148682,0.00025491643,0.00043594753,0.0010204004,0.000024045394],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000066515604,0.00028240986,0.9691461,0.00006276706,0.00004848695,0.000058911763,0.0019837385,0.017026572,0.0025194918,0.000133475,0.00033801256,0.008393438],"study_design_scores_gemma":[0.0012731875,0.0017456238,0.9171814,0.00007218267,0.000017157621,0.00005230115,0.00012641276,0.057675816,0.0093411235,0.00009826649,0.011464572,0.0009519694],"about_ca_topic_score_codex":0.000041670384,"about_ca_topic_score_gemma":0.0000065153804,"teacher_disagreement_score":0.30078098,"about_ca_system_score_codex":0.0000268347,"about_ca_system_score_gemma":0.000041322495,"threshold_uncertainty_score":0.8884056},"labels":[],"label_agreement":null},{"id":"W1991842606","doi":"10.1007/s10664-014-9302-3","title":"Waste identification and elimination in information technology organizations","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Quality and Supply Management","field":"Business, Management and Accounting","cited_by":32,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Categorization; Identification (biology); Lean manufacturing; Knowledge management; Process management; Business; Engineering; Computer science; Operations management; Artificial intelligence","score_opus":0.007171616111908883,"score_gpt":0.21267061111200308,"score_spread":0.2054989950000942,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1991842606","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.34453842,0.000011841912,0.6517862,0.0029729595,0.000136394,0.00013514789,5.26128e-7,0.0002817528,0.00013675765],"genre_scores_gemma":[0.99803716,0.0000026181233,0.0012593591,0.00046311968,0.00011988869,0.000020377038,0.000051014486,0.000009075498,0.000037381324],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9994133,0.000003975179,0.00023325832,0.00010833689,0.0001112757,0.00012983171],"domain_scores_gemma":[0.99971277,0.000040096118,0.000056263845,0.00010590512,0.00007782721,0.0000071243608],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00026718344,0.00007987501,0.00008453611,0.00052059925,0.000059746053,0.00019215963,0.00008450236,0.00006359375,0.000016385351],"category_scores_gemma":[0.0013276506,0.00008597075,0.000009823546,0.00096665166,0.000012984376,0.0016776776,0.00008866719,0.000087629436,0.000077590725],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013122659,0.0001320267,0.5566769,0.0011671833,0.000027009197,0.0000025331253,0.0006854499,0.10700191,0.00039804928,0.26140967,0.0029417982,0.06954429],"study_design_scores_gemma":[0.00071162597,0.000013003422,0.55703896,0.000080048725,0.000028458824,0.0000017307492,0.00037382814,0.27601177,0.00018923909,0.006228122,0.15887126,0.00045193662],"about_ca_topic_score_codex":0.00002112966,"about_ca_topic_score_gemma":0.00001174761,"teacher_disagreement_score":0.65349877,"about_ca_system_score_codex":0.000029195087,"about_ca_system_score_gemma":0.0000030727167,"threshold_uncertainty_score":0.35057855},"labels":[],"label_agreement":null},{"id":"W1991867644","doi":"10.1007/s10664-015-9375-7","title":"Analyzing and automatically labelling the types of user issues that are raised in mobile app reviews","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":188,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; Queen's University","funders":"","keywords":"App store; Computer science; World Wide Web; Download; Internet privacy; Mobile apps; Mobile device; Analytics; Notice; Data science","score_opus":0.052240163040090756,"score_gpt":0.31900221406896706,"score_spread":0.2667620510288763,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1991867644","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3242166,0.015309047,0.6590646,0.00033720687,0.0001803387,0.0003807817,0.0000012863121,0.00050357496,0.000006593834],"genre_scores_gemma":[0.8334458,0.0004142507,0.16570622,0.00007416616,0.00010376146,0.00012286681,0.0000016576662,0.000046311605,0.00008497946],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9984935,0.000066632856,0.000363211,0.00032589165,0.00037182288,0.00037894445],"domain_scores_gemma":[0.9976998,0.0014629494,0.00007170482,0.00051313604,0.00007704385,0.00017537775],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0011705625,0.00019976564,0.0004114475,0.00018562216,0.000035154608,0.00009705221,0.0006658933,0.00008780358,0.0000063106677],"category_scores_gemma":[0.0038889665,0.0001468129,0.00006150576,0.0008552639,0.00004147855,0.0002726501,0.0003394883,0.00033268516,0.000016288112],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008667386,0.00008764648,0.86811346,0.00052882015,0.00005258004,0.000048157395,0.0033705172,0.1150562,0.00016670483,0.00023089079,0.0022885986,0.010047775],"study_design_scores_gemma":[0.0013673436,0.0002518017,0.56509656,0.0015238412,0.000029557299,0.000045868303,0.00012995522,0.33433014,0.0019413367,0.000435739,0.09367935,0.0011684825],"about_ca_topic_score_codex":0.000013589568,"about_ca_topic_score_gemma":0.0000023314471,"teacher_disagreement_score":0.5092292,"about_ca_system_score_codex":0.00007326922,"about_ca_system_score_gemma":0.000041014828,"threshold_uncertainty_score":0.5986856},"labels":[],"label_agreement":null},{"id":"W1996342119","doi":"10.1007/s10664-011-9188-2","title":"Introduction to the special issue on software repository mining in 2009","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Software engineering; Software; Data science; Operating system","score_opus":0.022772899001632055,"score_gpt":0.257169182466531,"score_spread":0.23439628346489894,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1996342119","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.16146901,0.00015567633,0.82652175,0.0024864404,0.0064860824,0.00060035207,0.0000035277394,0.0020448861,0.00023228994],"genre_scores_gemma":[0.2893514,0.000014008304,0.64781666,0.0015509895,0.05786722,0.00046632707,0.000010191939,0.0002643847,0.0026588144],"study_design_codex":"not_applicable","study_design_gemma":"observational","domain_scores_codex":[0.9975032,0.000063611486,0.00038430092,0.00073735224,0.0005996591,0.00071186206],"domain_scores_gemma":[0.9978546,0.0007710361,0.0000408255,0.0010036518,0.00007790783,0.0002520004],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005524296,0.00029574923,0.00025719986,0.00041970756,0.00012601906,0.00012774134,0.001259102,0.0001358598,0.000104102684],"category_scores_gemma":[0.0045558247,0.00025164458,0.00008836644,0.0014430741,0.00002370229,0.00031342456,0.00037972906,0.00060821336,0.0003835974],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000112095666,0.00043807784,0.275442,0.00013046863,0.00008830503,0.0006764974,0.016655037,0.1856306,0.00014753196,0.00040888705,0.44552523,0.07474526],"study_design_scores_gemma":[0.00041218312,0.00043430706,0.6116317,0.0001432523,0.0000071151944,0.00009666196,0.000045383033,0.0061204797,0.0032434028,0.00004389797,0.37697592,0.0008457024],"about_ca_topic_score_codex":0.000033469296,"about_ca_topic_score_gemma":0.0000059947315,"teacher_disagreement_score":0.3361897,"about_ca_system_score_codex":0.0002896845,"about_ca_system_score_gemma":0.000059060796,"threshold_uncertainty_score":0.99999356},"labels":[],"label_agreement":null},{"id":"W1998785137","doi":"10.1007/s10664-008-9063-y","title":"Triangulation as a basis for knowledge discovery in software engineering","year":2008,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Artificial Intelligence in Education","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Software engineering; Triangulation; Basis (linear algebra); Data science; Cartography; Mathematics; Geography","score_opus":0.05327967590589569,"score_gpt":0.3145633665390798,"score_spread":0.26128369063318413,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1998785137","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3001591,0.00018574463,0.6980349,0.00013451945,0.0007800966,0.00025447077,0.0000024370759,0.00044195264,0.0000067888272],"genre_scores_gemma":[0.81834286,0.0000092443115,0.1808414,0.00006981952,0.00032267184,0.00017984037,0.000009235014,0.000039609222,0.00018530824],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99839485,0.000020857655,0.00043401093,0.0004689371,0.00021282272,0.00046853552],"domain_scores_gemma":[0.9978148,0.0015209373,0.000057867714,0.0003833625,0.000090153175,0.00013292683],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00025592285,0.0002350374,0.0002554971,0.00032523088,0.000104877334,0.00009521798,0.00048909045,0.00013925682,0.000010299129],"category_scores_gemma":[0.0058568297,0.0002559698,0.00013247402,0.0008982507,0.000018124456,0.0010097749,0.00011869147,0.00021282348,0.00006195061],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006529497,0.00081567746,0.20565066,0.0003919132,0.000083206505,0.00008143189,0.02396342,0.71947455,0.0014962916,0.0054248283,0.0032299869,0.039322745],"study_design_scores_gemma":[0.00063352427,0.0002647145,0.22950624,0.00038233516,0.000019969035,0.00010339056,0.000098427765,0.7291818,0.015722835,0.0011587847,0.021353574,0.0015743935],"about_ca_topic_score_codex":0.000032013533,"about_ca_topic_score_gemma":0.000008343844,"teacher_disagreement_score":0.51818377,"about_ca_system_score_codex":0.0003541365,"about_ca_system_score_gemma":0.00016796173,"threshold_uncertainty_score":0.9999893},"labels":[],"label_agreement":null},{"id":"W2002641269","doi":"10.1007/s10664-014-9315-y","title":"An empirical study on the importance of source code entities for requirements traceability","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":39,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; University of Waterloo","funders":"","keywords":"Computer science; Information retrieval; Traceability; Source code; Weighting; Search engine indexing; Rank (graph theory); Data mining; Software engineering; Programming language","score_opus":0.05999686482834988,"score_gpt":0.3489345172867725,"score_spread":0.2889376524584226,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2002641269","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.50610065,0.000016225382,0.49250516,0.00024943016,0.00016138238,0.00041022454,0.0000044615467,0.0005493321,0.00000311823],"genre_scores_gemma":[0.9789586,4.94707e-7,0.0204433,0.00017962334,0.00013664397,0.00019589622,0.0000031257077,0.000045291687,0.0000370387],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9973527,0.0001433274,0.00049680634,0.0006553447,0.0007836292,0.00056815316],"domain_scores_gemma":[0.99295914,0.005191865,0.00008613623,0.0014041784,0.00015696057,0.00020173112],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0018730862,0.0002842368,0.0003668174,0.00013648927,0.00014180935,0.000118878685,0.0015087691,0.00009747656,0.000013370467],"category_scores_gemma":[0.007115993,0.00021920518,0.00014479291,0.000529683,0.00005986869,0.000294295,0.00022121357,0.0003525707,0.000008430259],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000016150132,0.00045258785,0.9780486,0.00006369089,0.000042438656,0.0000019678937,0.0013584237,0.018067926,0.000088353256,0.00017302515,0.0008524997,0.0008343082],"study_design_scores_gemma":[0.0006402925,0.0013696891,0.85266435,0.000043262244,0.000014112167,0.0000030438994,0.000092010225,0.13933255,0.0009954225,0.00034998596,0.0040889825,0.0004062811],"about_ca_topic_score_codex":0.0000056666977,"about_ca_topic_score_gemma":0.000003908701,"teacher_disagreement_score":0.47285792,"about_ca_system_score_codex":0.0001298583,"about_ca_system_score_gemma":0.000051556497,"threshold_uncertainty_score":0.89389277},"labels":[],"label_agreement":null},{"id":"W2013851052","doi":"10.1007/s10664-006-9031-3","title":"A practical approach to testing GUI systems","year":2006,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":35,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Component (thermodynamics); Set (abstract data type); Graph; Graphical user interface; Software engineering; Programming language; Theoretical computer science","score_opus":0.060666448574408956,"score_gpt":0.297102360528753,"score_spread":0.23643591195434405,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2013851052","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0028374242,0.000077122866,0.9698866,0.0002629286,0.00030675152,0.00022248867,0.0000013363268,0.025578536,0.00082681555],"genre_scores_gemma":[0.2356102,1.2194582e-7,0.7637761,0.00016568328,0.00026885746,0.000082154926,0.0000024079275,0.00002993466,0.00006452641],"study_design_codex":"not_applicable","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9980383,0.00004196115,0.0003577538,0.0005757607,0.00041954877,0.0005667061],"domain_scores_gemma":[0.99748415,0.0014893449,0.00006131068,0.0006127401,0.00013114723,0.00022133162],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00046672722,0.0002568958,0.00026891378,0.00018931033,0.00011697963,0.00033708868,0.00059890916,0.00012003678,7.6655033e-7],"category_scores_gemma":[0.005435978,0.00024465655,0.00006342433,0.0011456606,0.000014414478,0.00027855468,0.0003455993,0.0003012332,0.000050097395],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008425411,0.0007264138,0.19109246,0.00036824337,0.000049092363,0.00027173868,0.00040577305,0.26204482,0.00024584236,0.018188946,0.5183488,0.008249395],"study_design_scores_gemma":[0.00024864974,0.00017860137,0.03210743,0.00022612516,0.000014951265,0.0006707009,0.0000027851872,0.93158793,0.00017004681,0.0023264624,0.031379864,0.0010864591],"about_ca_topic_score_codex":0.00019548635,"about_ca_topic_score_gemma":2.5199813e-7,"teacher_disagreement_score":0.6695431,"about_ca_system_score_codex":0.0001283614,"about_ca_system_score_gemma":0.00006809859,"threshold_uncertainty_score":0.9976805},"labels":[],"label_agreement":null},{"id":"W2019257047","doi":"10.1007/s10664-015-9381-9","title":"An empirical study of the impact of modern code review practices on software quality","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":320,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"Code review; Software quality; Computer science; Software engineering; Software quality analyst; Software inspection; Software peer review; Software quality management; Static program analysis; Software construction; Software quality assurance; Software development; Software; Programming language","score_opus":0.14670699522306,"score_gpt":0.4503907650299953,"score_spread":0.30368376980693534,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2019257047","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.75370634,0.001328314,0.24338152,0.00016502557,0.00023310847,0.0006276666,0.000015810592,0.0005368525,0.0000053369267],"genre_scores_gemma":[0.9837691,0.000026619227,0.015903078,0.00010032329,0.00007771851,0.000054068896,0.0000033766312,0.000051812287,0.000013907172],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9958455,0.00046300664,0.000833388,0.00068463763,0.0016196611,0.0005538131],"domain_scores_gemma":[0.99363226,0.0027514463,0.0004842771,0.0021868276,0.00048416125,0.0004610076],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0020218617,0.0003918155,0.00075779145,0.00018602189,0.00006585287,0.000066176326,0.0023676748,0.00014360901,0.000010305646],"category_scores_gemma":[0.027878877,0.0002695756,0.00029109154,0.0015531786,0.000054545515,0.0005753416,0.00058657886,0.0007254057,0.000010387882],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000029916888,0.001192056,0.84076154,0.0003871029,0.00011056736,0.000012672783,0.0016446578,0.1516697,0.000051940435,0.000008303816,0.0018663297,0.0022652028],"study_design_scores_gemma":[0.00083818595,0.0018566679,0.9666544,0.0006076573,0.000033730135,0.000020320029,0.000031152013,0.028839394,0.0002485382,0.00012018495,0.00027497733,0.0004747876],"about_ca_topic_score_codex":0.00017916058,"about_ca_topic_score_gemma":0.000007992588,"teacher_disagreement_score":0.23006274,"about_ca_system_score_codex":0.00031863112,"about_ca_system_score_gemma":0.00041090627,"threshold_uncertainty_score":0.9999756},"labels":[],"label_agreement":null},{"id":"W2020503784","doi":"10.1007/s10664-014-9340-x","title":"The kanban approach, between agility and leanness: a systematic review","year":2014,"lang":"en","type":"review","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":64,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Kanban; Agile software development; Variety (cybernetics); Field (mathematics); Process management; Scrum; Computer science; Management science; Lean software development; Product (mathematics); Software; Software development; Engineering; Knowledge management; Software development process; Software engineering; Control (management); Artificial intelligence; Mathematics","score_opus":0.04499939975791214,"score_gpt":0.3342892531106428,"score_spread":0.28928985335273066,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2020503784","genre_codex":"review","genre_gemma":"review","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"review","genre_consensus":"review","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[5.165557e-8,0.53217405,0.46570045,0.00004913378,0.00013511523,0.000868321,0.000003665832,0.0010622181,0.0000070028555],"genre_scores_gemma":[0.0000021740145,0.9417745,0.0571713,0.000091979506,0.00019822296,0.000622384,0.000016851167,0.00007653569,0.00004603721],"study_design_codex":"systematic_review","study_design_gemma":"not_applicable","domain_scores_codex":[0.9965385,0.00038124292,0.0012042777,0.0008059415,0.0004840931,0.0005859546],"domain_scores_gemma":[0.99011594,0.0073259375,0.0004802901,0.0017464298,0.00006883712,0.0002625769],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0025583224,0.000744421,0.0029819654,0.00012717073,0.00023038537,0.00043924229,0.0021752687,0.00035292807,0.0000011378045],"category_scores_gemma":[0.006344378,0.00046339957,0.00052043836,0.0008014624,0.000048203892,0.00031016758,0.00076525594,0.0010770772,0.000024352243],"study_design_candidate":"systematic_review","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[1.2416749e-7,0.000008631492,0.000012166805,0.90952563,0.00015825102,0.000005116409,0.000020790992,0.000008413735,8.868951e-10,0.00030179654,0.0009372551,0.08902183],"study_design_scores_gemma":[0.000027223688,0.000027118109,0.000008343384,0.20817229,0.0007208662,0.00009658684,3.924709e-7,0.00059658237,4.343157e-8,0.000054979286,0.7897448,0.00055078417],"about_ca_topic_score_codex":0.0000041065177,"about_ca_topic_score_gemma":1.3549997e-7,"teacher_disagreement_score":0.7888075,"about_ca_system_score_codex":0.00013094558,"about_ca_system_score_gemma":0.00008071696,"threshold_uncertainty_score":0.9997818},"labels":[],"label_agreement":null},{"id":"W2023617068","doi":"10.1007/s10664-009-9125-9","title":"An empirical study on the efficiency of different design pattern representations in UML class diagrams","year":2010,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":56,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Université de Montréal; Computer Research Institute of Montréal","funders":"","keywords":"Computer science; Applications of UML; Unified Modeling Language; Class diagram; UML tool; Notation; Program comprehension; Class (philosophy); Software design pattern; Visualization; Software engineering; Programming language; Software; Data mining; Artificial intelligence; Software system","score_opus":0.04501694003319926,"score_gpt":0.33526054527152854,"score_spread":0.29024360523832926,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2023617068","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.516008,0.0000060738835,0.4828225,0.00027026614,0.0002555872,0.00036276897,0.0000012019791,0.00027133097,0.0000022194527],"genre_scores_gemma":[0.99499905,9.0733664e-7,0.004608792,0.0000758206,0.00008859588,0.00018074659,0.0000016062513,0.000037533795,0.000006972063],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9972416,0.00021628727,0.0004720125,0.00064782554,0.0008379799,0.00058430794],"domain_scores_gemma":[0.99298364,0.005234628,0.00005829906,0.0014084796,0.00008441715,0.00023053208],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00090317114,0.0002968506,0.00032581255,0.0003280538,0.0000910768,0.00013850805,0.0016577764,0.00012788945,0.000021986547],"category_scores_gemma":[0.004022007,0.0002127529,0.000097252014,0.0011849502,0.000055850032,0.00021422449,0.00029402925,0.0010862184,0.000020152747],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000005417594,0.0010325601,0.96404916,0.000013239052,0.000017969705,0.00004094577,0.0021265498,0.028725188,0.0009765547,0.000049449685,0.00011625175,0.0028467283],"study_design_scores_gemma":[0.0002997721,0.00040653106,0.84314716,0.0000210023,0.0000039038014,0.0000048180373,0.00003518975,0.15427768,0.0015027187,0.00005206732,0.00002610747,0.0002230634],"about_ca_topic_score_codex":0.000031081618,"about_ca_topic_score_gemma":0.000018736535,"teacher_disagreement_score":0.478991,"about_ca_system_score_codex":0.000085727756,"about_ca_system_score_gemma":0.000059449656,"threshold_uncertainty_score":0.8675812},"labels":[],"label_agreement":null},{"id":"W2024695825","doi":"10.1007/s10664-011-9163-y","title":"Qualitative research in software engineering","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":59,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Qualitative research; Phenomenon; Ethnography; Context (archaeology); Social phenomenon; Social research; Participant observation; Grounded theory; Knowledge management; Software; Data science; Management science; Sociology; Qualitative property; Computer science; Epistemology; Engineering ethics; Social science; Engineering","score_opus":0.19842369192862866,"score_gpt":0.4223873846295447,"score_spread":0.22396369270091607,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2024695825","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.22838868,0.0005239232,0.76753825,0.0001267614,0.00063861697,0.00038877834,0.0000051178313,0.0023370096,0.000052868985],"genre_scores_gemma":[0.6593238,0.000014260633,0.34006467,0.00004713377,0.00014766902,0.00020882842,0.0000047145218,0.00010600536,0.00008291239],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99490863,0.00024445783,0.0006567051,0.0010683249,0.001318015,0.0018038808],"domain_scores_gemma":[0.99105996,0.0068983636,0.000048536505,0.001184635,0.00027904997,0.0005294487],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0031812917,0.00047356883,0.0005105726,0.0015264374,0.00012151158,0.00016731948,0.0021089513,0.00030674064,0.00006518123],"category_scores_gemma":[0.013548694,0.00050574244,0.00015832495,0.003920027,0.00007691721,0.00088900427,0.0010101084,0.0018253303,0.00029850073],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009320532,0.0011455627,0.7901309,0.0013752986,0.00030786544,0.0029162327,0.12088977,0.053386178,0.00071743625,0.010737887,0.005201716,0.013097932],"study_design_scores_gemma":[0.0022451798,0.0007452239,0.90829045,0.0011577954,0.000013628137,0.0001816685,0.000616064,0.060266457,0.0055171577,0.0026365547,0.014955233,0.003374616],"about_ca_topic_score_codex":0.0001340064,"about_ca_topic_score_gemma":0.000006013443,"teacher_disagreement_score":0.4309351,"about_ca_system_score_codex":0.0005914202,"about_ca_system_score_gemma":0.00017719288,"threshold_uncertainty_score":0.9997394},"labels":[],"label_agreement":null},{"id":"W2026170849","doi":"10.1007/s10664-014-9338-4","title":"On rapid releases and software testing: a case study and a semi-systematic literature review","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":113,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Context (archaeology); Test suite; Systematic review; Scope (computer science); Software engineering; Software; Software bug; Software testing; Test case; Operating system","score_opus":0.027427958343474917,"score_gpt":0.2857589749436667,"score_spread":0.25833101660019175,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2026170849","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.41494712,0.058347806,0.5212908,0.0002618277,0.00030546347,0.0019203012,0.0000063729326,0.0029152834,0.0000050690724],"genre_scores_gemma":[0.9220911,0.00074076676,0.0756472,0.0007801689,0.00017032494,0.00039609475,0.0000036293993,0.00011369785,0.00005700815],"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","domain_scores_codex":[0.99735653,0.00020091688,0.0004872451,0.00083498005,0.00056797074,0.00055238005],"domain_scores_gemma":[0.9886906,0.009689499,0.000081166916,0.00093139627,0.00016278881,0.00044451037],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001026332,0.00044823397,0.00069977966,0.00028733016,0.00016898247,0.00036248678,0.00048126746,0.00012133078,0.0000036187914],"category_scores_gemma":[0.039578114,0.00037686643,0.00007158864,0.0012870382,0.000029848197,0.00033175258,0.0004650722,0.00061876065,0.00001179168],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000036161404,0.0015000134,0.41408092,0.44142714,0.0006947088,0.033318378,0.010189863,0.0076796366,0.000067327666,0.00038419996,0.009574081,0.08104759],"study_design_scores_gemma":[0.00710534,0.010257113,0.1406277,0.4322082,0.0006936993,0.088916935,0.00024388613,0.30660215,0.00012216262,0.0006346379,0.004396993,0.008191195],"about_ca_topic_score_codex":0.000010064211,"about_ca_topic_score_gemma":0.0000011050872,"teacher_disagreement_score":0.507144,"about_ca_system_score_codex":0.000071535986,"about_ca_system_score_gemma":0.000032962747,"threshold_uncertainty_score":0.99986833},"labels":[],"label_agreement":null},{"id":"W2031459347","doi":"10.1007/s10664-010-9152-6","title":"Using grounded theory to study the experience of software development","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":236,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Grounded theory; Computer science; Context (archaeology); Software; Management science; Development theory; Qualitative research; Engineering; Sociology; Social science","score_opus":0.1170929886314408,"score_gpt":0.32886256978482953,"score_spread":0.21176958115338873,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2031459347","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2997518,0.00007931393,0.69881785,0.000011915633,0.0002513737,0.00027592803,4.829866e-7,0.00080231804,0.000008996391],"genre_scores_gemma":[0.48151955,7.8289776e-7,0.51830095,0.00007210313,0.000022574608,0.00005614834,2.0516794e-7,0.000019202353,0.000008507656],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99827313,0.000079000885,0.00042298407,0.00041859646,0.0004110894,0.00039520345],"domain_scores_gemma":[0.9981063,0.00075342745,0.00009733394,0.00079435034,0.00009142062,0.00015719493],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00075112825,0.0002640863,0.00025625425,0.00014899203,0.00015561222,0.00006870282,0.0014496577,0.00007216864,0.000034903413],"category_scores_gemma":[0.0012066639,0.0002017595,0.00007262918,0.0008093377,0.000025794088,0.00041893887,0.0006597943,0.00024831673,0.000010313727],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00015441935,0.002019734,0.41493306,0.00020998738,0.000765454,0.00028895316,0.37615988,0.022644242,0.00065254554,0.0067136968,0.00035273476,0.1751053],"study_design_scores_gemma":[0.0010429969,0.0011809992,0.9132613,0.0003161496,0.000120076365,0.00016023168,0.0018207094,0.01105406,0.04517988,0.0032398845,0.01969292,0.0029307953],"about_ca_topic_score_codex":0.00005347087,"about_ca_topic_score_gemma":0.0000017999607,"teacher_disagreement_score":0.49832824,"about_ca_system_score_codex":0.00008136939,"about_ca_system_score_gemma":0.00006052926,"threshold_uncertainty_score":0.82275134},"labels":[],"label_agreement":null},{"id":"W2034314383","doi":"10.1007/s10664-012-9212-1","title":"Preface to the special issue on program comprehension","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Intelligent Tutoring Systems and Adaptive Learning","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Program comprehension; Computer science; Comprehension; Programming language; Software","score_opus":0.029704598744702493,"score_gpt":0.29145600796233845,"score_spread":0.26175140921763596,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2034314383","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06445963,0.00013379165,0.9266888,0.001422838,0.0049289516,0.00058473746,8.492068e-7,0.001223132,0.0005572974],"genre_scores_gemma":[0.82978916,0.000002258241,0.13131091,0.0012150358,0.031342007,0.00015027609,0.0000026562984,0.000067739915,0.0061199646],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9987666,0.000039202187,0.00016551513,0.00023391249,0.00032315063,0.00047164384],"domain_scores_gemma":[0.9991767,0.00022057115,0.000029328205,0.00034970968,0.000037509773,0.00018618081],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00024877346,0.00016275949,0.00013853092,0.000058058857,0.00014133725,0.00011776228,0.00047663462,0.00005404046,0.000029839985],"category_scores_gemma":[0.00024230138,0.000112061796,0.00006712066,0.0003139436,0.000005986196,0.00017880708,0.00020844274,0.000278886,0.0009222677],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024800736,0.0005520602,0.063123405,0.00010374174,0.00008613384,0.000020925512,0.012369933,0.42231977,0.00022937855,0.022295617,0.09879458,0.38007966],"study_design_scores_gemma":[0.000040665996,0.00010811131,0.03639875,0.00005150449,0.0000017105857,0.0000047189396,0.000010038165,0.0041180365,0.0003563281,0.000002473207,0.9587491,0.00015851876],"about_ca_topic_score_codex":0.000006452523,"about_ca_topic_score_gemma":3.4791032e-7,"teacher_disagreement_score":0.8599546,"about_ca_system_score_codex":0.000079509635,"about_ca_system_score_gemma":0.000009326405,"threshold_uncertainty_score":0.99985564},"labels":[],"label_agreement":null},{"id":"W2034628356","doi":"10.1007/s10664-005-1290-x","title":"Studying Software Engineers: Data Collection Techniques for Software Field Studies","year":2005,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":491,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada; University of Ottawa","funders":"","keywords":"Computer science; Software engineering; Field (mathematics); Software; Task (project management); Data science; Taxonomy (biology); Data collection; Systems engineering; Engineering","score_opus":0.09495362100580118,"score_gpt":0.36248098551184676,"score_spread":0.2675273645060456,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2034628356","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.008946556,0.0026928354,0.977892,0.0012363454,0.00097019254,0.0009746229,0.000030876792,0.0072539425,0.0000026760297],"genre_scores_gemma":[0.042701505,0.00012975531,0.9548208,0.0004926078,0.00083713204,0.0005770689,0.00005776165,0.00015098244,0.0002323669],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9958319,0.000050649935,0.00071520003,0.00137236,0.0008338888,0.001195997],"domain_scores_gemma":[0.9884626,0.0085501475,0.00010899531,0.002107579,0.00042533458,0.00034530598],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0011207438,0.0006254537,0.0006877778,0.0005966342,0.0003645115,0.00031479786,0.002870119,0.00031818936,0.000016170537],"category_scores_gemma":[0.03039933,0.00063883915,0.00020050736,0.0014826632,0.00004296835,0.001603572,0.0017516238,0.0007454109,0.00003728066],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012351302,0.00078127004,0.11114891,0.0024189793,0.0016528997,0.00015074469,0.005861739,0.0866128,0.00043737856,0.00035085846,0.3012945,0.4891664],"study_design_scores_gemma":[0.0041344394,0.0023748535,0.021267597,0.0018530939,0.0002861102,0.00035720007,0.00030350828,0.40645215,0.033741392,0.0009596472,0.52144754,0.00682248],"about_ca_topic_score_codex":0.0000130084345,"about_ca_topic_score_gemma":0.000007728564,"teacher_disagreement_score":0.48234394,"about_ca_system_score_codex":0.0005884301,"about_ca_system_score_gemma":0.00017848729,"threshold_uncertainty_score":0.9996063},"labels":[],"label_agreement":null},{"id":"W2039052699","doi":"10.1007/s10664-012-9228-6","title":"Studying re-opened bugs in open source software","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":138,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"Software bug; Eclipse; Software regression; Dimension (graph theory); Open source; Rework; Computer science; Software; Software quality; Software engineering; Engineering; Software development; Operating system; Mathematics","score_opus":0.05923128895567467,"score_gpt":0.32261791254277666,"score_spread":0.26338662358710196,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2039052699","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.22314107,0.0006899379,0.7728356,0.00025647413,0.00073235866,0.00053132925,0.0000022153602,0.0017737449,0.000037241327],"genre_scores_gemma":[0.7904896,0.00000865811,0.20834197,0.0002780212,0.00027890198,0.00019607872,0.000006064134,0.000110694295,0.00029001577],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99613583,0.000104217404,0.00056437665,0.0007711113,0.0007919224,0.0016325298],"domain_scores_gemma":[0.9953523,0.0026794185,0.00007004588,0.0011964061,0.00008384399,0.00061798404],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0015755278,0.00045463475,0.00054485124,0.00047652086,0.0001504359,0.00052618916,0.003348088,0.00021630495,0.00007423441],"category_scores_gemma":[0.007043353,0.00046951056,0.000119591445,0.0019177918,0.000029875073,0.0022038016,0.003153432,0.00086764584,0.0002372252],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008732182,0.00019005971,0.96301186,0.000075760276,0.000034436733,0.000046170124,0.0018339555,0.02364812,0.00006682758,0.000089201654,0.0018141103,0.009180743],"study_design_scores_gemma":[0.0014029121,0.0001256324,0.9322138,0.00027118067,0.0000107772385,0.0000607441,0.000070712136,0.012886767,0.0008795312,0.000084845844,0.050593916,0.0013992005],"about_ca_topic_score_codex":0.000113515634,"about_ca_topic_score_gemma":0.000007333398,"teacher_disagreement_score":0.56734854,"about_ca_system_score_codex":0.00047666806,"about_ca_system_score_gemma":0.00011313528,"threshold_uncertainty_score":0.99977565},"labels":[],"label_agreement":null},{"id":"W2040411904","doi":"10.1007/s10664-014-9312-1","title":"Do topics make sense to managers and developers?","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":34,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Traceability; Latent Dirichlet allocation; Computer science; Documentation; Relevance (law); Requirements traceability; Perception; Topic model; Control (management); Software engineering; Requirements engineering; Requirements elicitation; BitTorrent tracker; Software; Data science; Information retrieval; Requirement; Artificial intelligence","score_opus":0.019201760776928176,"score_gpt":0.26889327776064786,"score_spread":0.2496915169837197,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2040411904","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.18767184,0.000080408005,0.80980295,0.0009946241,0.0003349278,0.0001229702,7.566868e-7,0.00094261946,0.000048910915],"genre_scores_gemma":[0.56278646,0.0000071332183,0.43579954,0.0007363839,0.00024005042,0.00003333049,0.0000013356548,0.00004752026,0.00034823208],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.9983169,0.000027719525,0.00019775977,0.0005259612,0.00038068127,0.0005509809],"domain_scores_gemma":[0.9980699,0.0008782824,0.000015606493,0.00056239887,0.000060539747,0.00041329593],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00039349738,0.0002227543,0.00021735944,0.00023450064,0.00007543591,0.00023164222,0.0004779002,0.0000969085,0.000008010523],"category_scores_gemma":[0.0035284408,0.00022271049,0.000045597648,0.00069125637,0.000016611744,0.00015696287,0.00056082814,0.00027453224,0.00008287613],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000019668601,0.00010626434,0.37347597,0.0005923352,0.00017067796,0.00046919333,0.005144475,0.105485946,0.0005539747,0.007916953,0.012575374,0.49348918],"study_design_scores_gemma":[0.00054051424,0.00018186047,0.5559425,0.00013600923,0.000009078528,0.00013900014,0.000015950916,0.05516102,0.0007469191,0.0002623122,0.3856927,0.0011721813],"about_ca_topic_score_codex":0.0000063118205,"about_ca_topic_score_gemma":9.89234e-7,"teacher_disagreement_score":0.492317,"about_ca_system_score_codex":0.0000938957,"about_ca_system_score_gemma":0.000024281315,"threshold_uncertainty_score":0.908187},"labels":[],"label_agreement":null},{"id":"W2043780253","doi":"10.1007/s10664-010-9131-y","title":"An empirical investigation into open source web applications’ implementation vulnerabilities","year":2010,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Web Application Security Vulnerabilities","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Open source; Computer science; World Wide Web; Web application; Empirical research; Computer security; Operating system; Software","score_opus":0.022168539556753533,"score_gpt":0.33476904965194676,"score_spread":0.31260051009519324,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2043780253","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3910864,0.000012470089,0.6051014,0.001949145,0.00015929337,0.00063586293,0.0000056695008,0.001024402,0.000025367171],"genre_scores_gemma":[0.71621966,0.0000014055082,0.28134847,0.0008347515,0.0002928216,0.0011357586,0.00006523751,0.000040370505,0.00006153785],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9975877,0.0001168162,0.00056905265,0.0008401142,0.00046050837,0.00042583767],"domain_scores_gemma":[0.9973711,0.00060773804,0.00012197105,0.0012953797,0.00020535063,0.0003984786],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008585892,0.00029799156,0.0002778758,0.00019467199,0.000352974,0.0008465758,0.0021274735,0.00018324843,0.000116888856],"category_scores_gemma":[0.00032989573,0.000311907,0.00007377767,0.00080897706,0.00010984959,0.0018619824,0.00060776155,0.0006157208,0.00009070307],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001917529,0.0006005758,0.75028116,0.0003262153,0.00008900838,0.0000044564695,0.029888045,0.010450316,0.04075719,0.056630366,0.012264432,0.09868907],"study_design_scores_gemma":[0.0016646085,0.0003419522,0.15555675,0.000030436197,0.000033510692,0.00004894076,0.001099603,0.14583461,0.030238535,0.04594379,0.6169504,0.0022568246],"about_ca_topic_score_codex":0.00022587039,"about_ca_topic_score_gemma":0.000110805006,"teacher_disagreement_score":0.60468596,"about_ca_system_score_codex":0.00014317466,"about_ca_system_score_gemma":0.00026157082,"threshold_uncertainty_score":0.9999333},"labels":[],"label_agreement":null},{"id":"W2044807274","doi":"10.1007/s10664-010-9151-7","title":"Design evolution metrics for defect prediction in object oriented systems","year":2010,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":50,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Software metric; Eclipse; Data mining; Software quality; Quality (philosophy); Identification (biology); Software; Software system; Reliability engineering; Machine learning; Software engineering; Software development; Programming language; Engineering","score_opus":0.026845936819820337,"score_gpt":0.27604758517063077,"score_spread":0.24920164835081043,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2044807274","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0724314,0.0003040208,0.9226257,0.00003161769,0.002414666,0.0006997724,0.0000069620064,0.0014829814,0.0000028589898],"genre_scores_gemma":[0.7971088,0.0000018957702,0.20225191,0.0000084758085,0.00021048792,0.0003283549,0.000007719484,0.000042275446,0.000040016497],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978503,0.00005091988,0.00037304097,0.00055793446,0.0005043926,0.000663431],"domain_scores_gemma":[0.99524826,0.0037805103,0.00004604631,0.00055222044,0.00017046061,0.0002025023],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012553686,0.00024171031,0.00026653215,0.0007736584,0.00008101254,0.00014102613,0.0006036989,0.00023798384,0.0000037198724],"category_scores_gemma":[0.011420873,0.00024824234,0.00011456071,0.002245849,0.00001641983,0.00044804535,0.00014556172,0.00063445314,0.000022447564],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025106347,0.00013697753,0.33674636,0.00025080325,0.000057172718,0.000029215478,0.000316728,0.654538,0.0025685458,0.0012075717,0.0018136977,0.0023098325],"study_design_scores_gemma":[0.000569244,0.00015769883,0.11041824,0.000049400038,0.0000072294815,0.000030504258,0.0000035094106,0.88470834,0.0005329423,0.0000572442,0.003163129,0.00030253426],"about_ca_topic_score_codex":0.000030483585,"about_ca_topic_score_gemma":0.000002440441,"teacher_disagreement_score":0.72467744,"about_ca_system_score_codex":0.00040263683,"about_ca_system_score_gemma":0.0001287015,"threshold_uncertainty_score":0.99999696},"labels":[],"label_agreement":null},{"id":"W2045837563","doi":"10.1007/s10664-012-9219-7","title":"Static test case prioritization using topic models","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":156,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Test suite; Computer science; Code coverage; Test case; Source code; Black box; Test Management Approach; Programming language; White-box testing; Test (biology); Test data; Operating system; Software; Software development; Artificial intelligence; Machine learning","score_opus":0.06432137932201937,"score_gpt":0.3036697114581389,"score_spread":0.23934833213611956,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2045837563","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07237958,0.00019094166,0.91803277,0.000064403575,0.00028314855,0.00009874723,0.0000017364483,0.008934137,0.000014525311],"genre_scores_gemma":[0.5370809,0.0000011681489,0.46265408,0.00012494679,0.000105368155,0.000007784058,0.0000011274052,0.000015419158,0.000009191915],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9988658,0.00002193053,0.00022477267,0.00023234509,0.00020371418,0.00045146263],"domain_scores_gemma":[0.9985216,0.0008052508,0.000046362056,0.00037276084,0.000059430306,0.00019457041],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00024203016,0.00017304359,0.00015985416,0.00011911046,0.0001122495,0.000097553835,0.00025512563,0.00008151158,0.000003972741],"category_scores_gemma":[0.0014001232,0.00017392183,0.000048634698,0.00044923666,0.00001219212,0.00079739053,0.00016901048,0.00017064612,0.000008227141],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000002932833,0.00056414015,0.76315516,0.00042846313,0.000058909205,0.0012897438,0.0058282507,0.17192984,0.0002808286,0.0032404785,0.009944203,0.043277074],"study_design_scores_gemma":[0.000111651236,0.000036636575,0.0033980168,0.000075157615,0.000011008084,0.0014713063,0.000002664606,0.99075234,0.00024905204,0.0029967315,0.0005388826,0.0003565747],"about_ca_topic_score_codex":0.000036821933,"about_ca_topic_score_gemma":2.968668e-7,"teacher_disagreement_score":0.8188225,"about_ca_system_score_codex":0.00012609163,"about_ca_system_score_gemma":0.00003353302,"threshold_uncertainty_score":0.7092326},"labels":[],"label_agreement":null},{"id":"W2046022618","doi":"10.1023/b:emse.0000013514.19567.ad","title":"An Industrial Case Study of Immediate Benefits of Requirements Engineering Process Improvement at the Australian Center for Unisys Software","year":2004,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":55,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Requirements management; Process management; Engineering; Process (computing); Requirements engineering; Requirements elicitation; Capability Maturity Model; Session (web analytics); Dimension (graph theory); Systems engineering; Engineering management; Computer science; Software","score_opus":0.06675324205872442,"score_gpt":0.3308906027842591,"score_spread":0.2641373607255347,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2046022618","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.64558303,0.00004321622,0.3526047,0.00007721862,0.00041257008,0.0007844893,0.000039046176,0.00045547404,2.5268412e-7],"genre_scores_gemma":[0.948673,0.0000042417664,0.05082768,0.000027284836,0.0001621707,0.00022580645,0.000015820535,0.000057320452,0.0000066560574],"study_design_codex":"simulation_or_modeling","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99763596,0.000026133175,0.00073238526,0.0005386599,0.0005241764,0.00054268405],"domain_scores_gemma":[0.99798304,0.00049555756,0.00027661832,0.00084877666,0.00019282187,0.00020317234],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005810565,0.0003804865,0.00042826866,0.00019314441,0.00012937014,0.00007558726,0.0009864478,0.00017650485,0.0000059250074],"category_scores_gemma":[0.00068332563,0.0003169543,0.0001313991,0.0006229546,0.000024790144,0.00064870156,0.0003016024,0.00033469062,9.318204e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006730821,0.0011891287,0.03934662,0.00045009027,0.00025777644,0.00015552276,0.005129149,0.9412384,0.0006751423,0.000093516355,0.00014392874,0.011253424],"study_design_scores_gemma":[0.0799102,0.06586272,0.10256092,0.006091005,0.0021753877,0.005738581,0.0064382292,0.16033885,0.5270867,0.0013933924,0.025721414,0.01668263],"about_ca_topic_score_codex":0.00017510743,"about_ca_topic_score_gemma":0.00002043947,"teacher_disagreement_score":0.7808995,"about_ca_system_score_codex":0.0001999428,"about_ca_system_score_gemma":0.00007311802,"threshold_uncertainty_score":0.99992824},"labels":[],"label_agreement":null},{"id":"W2046157646","doi":"10.1007/s10664-015-9377-5","title":"An empirical study of software release notes","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":51,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Empirical research; Computer science; Software; Software engineering; Programming language; Mathematics; Statistics","score_opus":0.05553808588465858,"score_gpt":0.33739025458356287,"score_spread":0.2818521686989043,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2046157646","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5142955,0.000114313676,0.48352367,0.00007511979,0.00032140524,0.00023373865,0.0000031926008,0.0014309052,0.0000021595768],"genre_scores_gemma":[0.875527,0.0000015579873,0.124051794,0.00007717381,0.00018331573,0.000061236446,0.0000065758863,0.0000739954,0.000017326762],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.996149,0.00014381114,0.0006325065,0.0008884175,0.0013651752,0.0008211087],"domain_scores_gemma":[0.9944654,0.002387245,0.00009185595,0.001636222,0.00039971748,0.001019545],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00082460255,0.00043557916,0.00057361287,0.00045856735,0.00008143298,0.00015945155,0.0018960987,0.00020658443,0.000015323436],"category_scores_gemma":[0.012705362,0.00042394258,0.00012707652,0.0016813437,0.000046054087,0.00080775894,0.00060910836,0.000649989,0.00006623588],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000021215668,0.00095549494,0.8833331,0.00005593013,0.000054570228,0.00015993364,0.0030880552,0.10745992,0.00006798806,0.0000125165025,0.0011228829,0.003668385],"study_design_scores_gemma":[0.0029725193,0.0036064626,0.90284836,0.00010889529,0.000041285573,0.00009599749,0.00023815414,0.08305744,0.0017129532,0.00022655915,0.003563974,0.0015273968],"about_ca_topic_score_codex":0.00008156755,"about_ca_topic_score_gemma":0.000005985472,"teacher_disagreement_score":0.3612315,"about_ca_system_score_codex":0.00023819713,"about_ca_system_score_gemma":0.00025070686,"threshold_uncertainty_score":0.99982125},"labels":[],"label_agreement":null},{"id":"W2049875335","doi":"10.1007/s10664-011-9179-3","title":"Computer-mediated communication to support distributed requirements elicitations and negotiations tasks","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":49,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Requirements elicitation; Computer science; Negotiation; Technical communication; Knowledge management; Empirical research; Requirements management; Stakeholder; Requirements engineering; Process management; Software; Engineering","score_opus":0.06055152737650241,"score_gpt":0.30205757710763703,"score_spread":0.24150604973113463,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2049875335","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.018577395,0.0000594795,0.9786735,0.00044182868,0.0001634898,0.00021094752,0.000022063105,0.0018113084,0.000039944374],"genre_scores_gemma":[0.38893005,0.000015739653,0.61057925,0.00030839964,0.00002469764,0.000055835928,0.000060669016,0.000017730099,0.000007634493],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9988664,0.000036068803,0.00029681335,0.00030936493,0.00019645112,0.0002949473],"domain_scores_gemma":[0.99845314,0.00057400495,0.00006634515,0.00056892826,0.000093636125,0.00024393346],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00024886563,0.0001842745,0.0001692723,0.00017053913,0.00013257739,0.00011878139,0.00059291784,0.00008696973,0.000023998604],"category_scores_gemma":[0.00069280603,0.00019649405,0.000040607665,0.00065653113,0.000016642132,0.00063912815,0.00039334883,0.00021000084,0.00004172166],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009339803,0.0021314083,0.5622766,0.0006235287,0.0009965536,0.00025518067,0.03957572,0.06253464,0.0015575059,0.06019246,0.11617866,0.15358433],"study_design_scores_gemma":[0.0007919328,0.0008318637,0.7358867,0.00018506624,0.00005540538,0.00007803972,0.000028464261,0.13704444,0.000832005,0.0013666599,0.12143911,0.0014603114],"about_ca_topic_score_codex":0.00004003309,"about_ca_topic_score_gemma":0.0000033409483,"teacher_disagreement_score":0.37035266,"about_ca_system_score_codex":0.00006574027,"about_ca_system_score_gemma":0.000027701533,"threshold_uncertainty_score":0.8012795},"labels":[],"label_agreement":null},{"id":"W2051162588","doi":"10.1007/s10664-014-9311-2","title":"Modelling the ‘hurried’ bug report reading process to summarize bug reports","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":70,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Automatic summarization; Computer science; Process (computing); Sentence; Reading (process); Software bug; Quality (philosophy); Natural language processing; Artificial intelligence; Data science; Information retrieval; Software; Programming language; Linguistics","score_opus":0.02334213703925641,"score_gpt":0.2861944876746135,"score_spread":0.26285235063535706,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2051162588","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.121932246,0.00007543393,0.8735599,0.0008662003,0.0009720354,0.0003474402,8.854642e-7,0.0021745318,0.00007135067],"genre_scores_gemma":[0.9004614,0.0000026964358,0.09816115,0.00028104233,0.000497157,0.00017295702,0.0000060836846,0.0000977186,0.00031979868],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9959021,0.00005694954,0.00073941937,0.0010876688,0.0011499054,0.0010639443],"domain_scores_gemma":[0.9950438,0.002181959,0.00012408193,0.0018431303,0.0002639076,0.00054310245],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0019549702,0.00042461307,0.00042291786,0.00028554568,0.00027658435,0.00041487187,0.0014145414,0.00017166343,0.00001244268],"category_scores_gemma":[0.01059985,0.00034971218,0.00015775248,0.0016360191,0.00003408215,0.0004805636,0.0006321632,0.00077048934,0.000060254348],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000003663344,0.000022229417,0.028963167,0.000067265144,0.000025483476,0.0003926068,0.0005347938,0.9676506,0.000056339737,0.00011217348,0.0013592686,0.00081244146],"study_design_scores_gemma":[0.0001548962,0.00010460993,0.0074430136,0.00019433402,0.000015056886,0.0011213011,0.000010771658,0.93510926,0.0012113192,0.000748519,0.053030346,0.00085654727],"about_ca_topic_score_codex":0.000059467202,"about_ca_topic_score_gemma":8.313399e-7,"teacher_disagreement_score":0.77852917,"about_ca_system_score_codex":0.00020855182,"about_ca_system_score_gemma":0.00012759306,"threshold_uncertainty_score":0.9998955},"labels":[],"label_agreement":null},{"id":"W2052776473","doi":"10.1007/s10664-014-9324-x","title":"A Large-Scale Empirical Study of the Relationship between Build Technology and Build Maintenance","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":57,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"Software maintenance; Computer science; Deliverable; Abstraction; Software engineering; Software; Source code; Scale (ratio); Source lines of code; Code (set theory); Systems engineering; Software system; Engineering; Programming language","score_opus":0.026836341569620767,"score_gpt":0.2976999388033339,"score_spread":0.2708635972337131,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2052776473","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.55582726,0.00005205954,0.44268072,0.00066426065,0.00012211659,0.0001958792,0.0000020776358,0.00045207536,0.0000035836142],"genre_scores_gemma":[0.97839767,7.246653e-7,0.021313472,0.000054246757,0.00009049835,0.000048916958,7.525378e-7,0.000034566867,0.000059126665],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99775857,0.00009725241,0.00041642142,0.00057447935,0.00054993056,0.0006033425],"domain_scores_gemma":[0.9945648,0.003994671,0.0000770396,0.0010706036,0.00011759745,0.00017532663],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0007183663,0.00025274532,0.0003765836,0.00039799293,0.0001671288,0.000062465464,0.0012786608,0.00020884218,0.0000024807707],"category_scores_gemma":[0.011535884,0.00019851589,0.00007100898,0.0025086706,0.00007548875,0.00018694348,0.0011036396,0.00081185735,0.000008917296],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000015656784,0.00007953785,0.9964204,0.000036650526,0.000016642221,0.000002584085,0.00066687306,0.0017310132,0.000010909456,0.00037742214,0.00028506428,0.00037134075],"study_design_scores_gemma":[0.00052184076,0.00017736314,0.98831207,0.00006593348,0.000010804487,0.000013528497,0.00003855319,0.0069526215,0.000086494816,0.00061590585,0.0029815694,0.00022331697],"about_ca_topic_score_codex":0.000006927105,"about_ca_topic_score_gemma":0.0000048119605,"teacher_disagreement_score":0.42257044,"about_ca_system_score_codex":0.00008695703,"about_ca_system_score_gemma":0.000045796867,"threshold_uncertainty_score":0.99679035},"labels":[],"label_agreement":null},{"id":"W2055777130","doi":"10.1007/s10664-015-9366-8","title":"Investigating technical and non-technical factors influencing modern code review","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":107,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; Université de Montréal","funders":"","keywords":"Computer science; Code review; Process (computing); Code (set theory); Key (lock); Source code; Variety (cybernetics); Empirical research; Software engineering; Component (thermodynamics); Data science; Static program analysis; Software development; Computer security; Software; Artificial intelligence; Programming language","score_opus":0.05327895845453611,"score_gpt":0.31370679174594,"score_spread":0.2604278332914039,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2055777130","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2304231,0.003761867,0.7625388,0.00046039122,0.00018470471,0.00033385225,0.000003142399,0.002277186,0.000016970247],"genre_scores_gemma":[0.77950025,0.00011229469,0.21969423,0.00044896224,0.00009736359,0.00006090251,0.000005574281,0.00006694008,0.000013480127],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99710333,0.0000458668,0.00052495557,0.00073061197,0.0008499883,0.00074521726],"domain_scores_gemma":[0.9966091,0.0014486834,0.000069317626,0.00079637853,0.00015699778,0.0009195241],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009029174,0.00038983184,0.0005226991,0.0002046596,0.00008899043,0.00016650901,0.0010577269,0.00022609555,0.0000033230415],"category_scores_gemma":[0.014507653,0.0003631996,0.00010197641,0.0010495207,0.00007610304,0.00058735645,0.0010909907,0.0009250625,0.000021346441],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000031720747,0.00009396079,0.96022815,0.0016638727,0.00004995832,0.000123563,0.0010250699,0.019162036,0.0038121243,0.00021292495,0.009914507,0.0037106604],"study_design_scores_gemma":[0.0013960866,0.00049121294,0.8487226,0.005364517,0.00006240759,0.0005570401,0.000020572708,0.12972523,0.0020581807,0.0012743114,0.0076054023,0.002722468],"about_ca_topic_score_codex":0.000013485817,"about_ca_topic_score_gemma":0.0000019358638,"teacher_disagreement_score":0.54907715,"about_ca_system_score_codex":0.0003597514,"about_ca_system_score_gemma":0.00019090893,"threshold_uncertainty_score":0.999882},"labels":[],"label_agreement":null},{"id":"W2056894403","doi":"10.1007/s10664-012-9231-y","title":"What are developers talking about? An analysis of topics and trends in Stack Overflow","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":613,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Latent Dirichlet allocation; World Wide Web; Topic model; Popularity; Data science; Leverage (statistics); Android (operating system); Information retrieval; Artificial intelligence","score_opus":0.03051975966812,"score_gpt":0.3086728286717767,"score_spread":0.2781530690036567,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2056894403","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8032327,0.0010401821,0.19502683,0.00008378682,0.00033037039,0.000045766225,0.0000026034477,0.00023527471,0.0000024353908],"genre_scores_gemma":[0.9597165,0.000043809894,0.040042285,0.000044376353,0.00007817997,0.00001177434,0.000008252698,0.000022233693,0.0000325804],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982768,0.00003708535,0.0003198783,0.00036189283,0.0004098033,0.00059450004],"domain_scores_gemma":[0.9985987,0.0005140968,0.000052792115,0.0004914957,0.000056900568,0.0002860132],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00045856697,0.00021036902,0.00040468344,0.0010952902,0.00003457412,0.00015475629,0.00047114212,0.00012312457,0.000025445926],"category_scores_gemma":[0.00076639396,0.0002137244,0.000088267974,0.0030522917,0.000024749068,0.0016459612,0.00027457086,0.0002798129,0.0000018257015],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000019674842,0.00004744007,0.9412918,0.00003633161,0.00008597357,0.000009748611,0.0017625622,0.031906392,0.000024767285,0.00006435311,0.0000211205,0.024747528],"study_design_scores_gemma":[0.00013413918,0.000022457873,0.9266498,0.00005539847,0.000023134202,0.000003411829,0.000049172246,0.07192862,0.00015440052,0.0000033540762,0.00074439176,0.00023173355],"about_ca_topic_score_codex":0.000028877836,"about_ca_topic_score_gemma":0.000018427547,"teacher_disagreement_score":0.15648375,"about_ca_system_score_codex":0.00015807946,"about_ca_system_score_gemma":0.000019088337,"threshold_uncertainty_score":0.8715428},"labels":[],"label_agreement":null},{"id":"W2061072593","doi":"10.1007/s10664-013-9264-x","title":"SWordNet: Inferring semantically related words from software context","year":2013,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":59,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; WordNet; Program comprehension; Information retrieval; Software; Context (archaeology); Java; Ranking (information retrieval); Natural language processing; Code (set theory); Precision and recall; Software maintenance; Artificial intelligence; Software development; Programming language; Software system","score_opus":0.01656489003974,"score_gpt":0.2521907926293967,"score_spread":0.2356259025896567,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2061072593","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39893028,0.00044353455,0.59559226,0.00045559043,0.0006725201,0.00031782364,0.000005164055,0.0035629075,0.000019882473],"genre_scores_gemma":[0.8508537,0.000011676059,0.14801934,0.00036305937,0.00021069417,0.00014720218,0.000020206751,0.000117469346,0.0002566418],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99588984,0.00006979508,0.0007592578,0.001049811,0.00094308786,0.0012882317],"domain_scores_gemma":[0.9937361,0.003930639,0.00008669266,0.001334375,0.00018914089,0.00072304497],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00034052474,0.00057507306,0.00061667396,0.000371074,0.0001585595,0.000568375,0.0018555953,0.00037568787,0.00051582005],"category_scores_gemma":[0.0063436027,0.00057638774,0.0002349484,0.0013743994,0.000065558124,0.0011964465,0.0010020556,0.0011440825,0.0017162303],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008506382,0.0002010181,0.87804365,0.0001475997,0.00034307732,0.00031289557,0.0023987961,0.028362164,0.0009444081,0.0006480292,0.006907315,0.08168252],"study_design_scores_gemma":[0.001058283,0.00014412866,0.89673805,0.00037680002,0.00002382016,0.00006268504,0.000030012896,0.08654048,0.0010373442,0.0023413016,0.010010688,0.0016364043],"about_ca_topic_score_codex":0.00029363472,"about_ca_topic_score_gemma":0.000006110088,"teacher_disagreement_score":0.4519234,"about_ca_system_score_codex":0.0002917064,"about_ca_system_score_gemma":0.00011961429,"threshold_uncertainty_score":0.9996688},"labels":[],"label_agreement":null},{"id":"W2062937278","doi":"10.1023/b:emse.0000048324.12188.a2","title":"An Empirical Exploration of the Distributions of the Chidamber and Kemerer Object-Oriented Metrics Suite","year":2004,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":71,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Collinearity; Suite; Computer science; Data mining; Correlation; Parametric statistics; Variance (accounting); Empirical research; Set (abstract data type); Test suite; Regression analysis; Statistics; Machine learning; Mathematics; Test case","score_opus":0.0281820532597251,"score_gpt":0.2982924489141738,"score_spread":0.27011039565444867,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2062937278","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4366406,0.00008631312,0.5623271,0.0003992105,0.0002406809,0.00015285885,0.000011262318,0.00013984286,0.0000021737042],"genre_scores_gemma":[0.9792188,0.000006300759,0.020632073,0.00003756375,0.000047898935,0.0000221909,0.000003944287,0.000019772433,0.000011493878],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99827594,0.000059466627,0.0003519178,0.00032465902,0.00065083534,0.00033717268],"domain_scores_gemma":[0.99805754,0.00068617007,0.00008328519,0.0008561565,0.00017667662,0.0001401638],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00031188416,0.00018574159,0.00022105903,0.00014034352,0.00012591852,0.00004699541,0.000840054,0.00010991083,0.0000039603456],"category_scores_gemma":[0.0035952155,0.00012260301,0.0001222338,0.0025367655,0.00010217308,0.0005039986,0.0004720044,0.00037807654,0.0000018815151],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00000910901,0.00031800027,0.86571586,0.00013822832,0.00007314424,0.00000474911,0.0025514332,0.1240992,0.0020567281,0.0030664443,0.00028689462,0.0016801953],"study_design_scores_gemma":[0.0004103586,0.00008718168,0.97319967,0.0000728976,0.00001545497,0.000013176729,0.0000146939,0.010471392,0.014206816,0.00040268412,0.00089523545,0.00021043172],"about_ca_topic_score_codex":0.000034407516,"about_ca_topic_score_gemma":0.0000043310015,"teacher_disagreement_score":0.54257816,"about_ca_system_score_codex":0.00013922098,"about_ca_system_score_gemma":0.00012516294,"threshold_uncertainty_score":0.49996057},"labels":[],"label_agreement":null},{"id":"W2066687630","doi":"10.1007/s10664-013-9292-6","title":"Towards improving statistical modeling of software engineering data: think locally, act globally!","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":37,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Cluster analysis; Multivariate adaptive regression splines; Data mining; Machine learning; Software; Statistical model; Parametric statistics; Data modeling; Artificial intelligence; Data science; Regression analysis; Nonparametric regression; Mathematics; Statistics; Software engineering","score_opus":0.03219726578772242,"score_gpt":0.2902385702837676,"score_spread":0.2580413044960452,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2066687630","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.026696634,0.00035773427,0.96884334,0.00012551612,0.0008839309,0.00029350945,0.00006042791,0.0027260063,0.000012894075],"genre_scores_gemma":[0.4842553,0.000009064082,0.5152208,0.00006663864,0.0002713797,0.000018611336,0.000051053164,0.00009743574,0.000009656332],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99461335,0.00007837607,0.00095920323,0.0013756492,0.0015846945,0.0013887429],"domain_scores_gemma":[0.99317664,0.0031658674,0.00012237446,0.002581164,0.0002928786,0.000661063],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0016381002,0.00066944933,0.0008349661,0.0004124803,0.00012450498,0.00033760132,0.0043196073,0.00034566887,0.000026004973],"category_scores_gemma":[0.026230985,0.0006862371,0.00016953809,0.001170493,0.00005613797,0.0012025399,0.002985732,0.0010481357,0.000045250763],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001610107,0.00011725877,0.019650731,0.0007580407,0.00013195381,0.000068236586,0.00028769425,0.94386876,0.0002455895,0.0015840195,0.0006822437,0.032589376],"study_design_scores_gemma":[0.00046249028,0.000119648976,0.013672689,0.00018772164,0.000024639172,0.000050077866,0.000004224997,0.9827684,0.00025068963,0.00014161253,0.0015908713,0.0007268979],"about_ca_topic_score_codex":0.00012862413,"about_ca_topic_score_gemma":0.0000032465189,"teacher_disagreement_score":0.4575587,"about_ca_system_score_codex":0.0003223441,"about_ca_system_score_gemma":0.0003243718,"threshold_uncertainty_score":0.99955887},"labels":[],"label_agreement":null},{"id":"W2068712694","doi":"10.1007/s10664-009-9121-0","title":"Applying empirical software engineering to software architecture: challenges and lessons learned","year":2009,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Social software engineering; Resource-oriented architecture; Software development; Software peer review; Personal software process; Software construction; Architecture tradeoff analysis method; Software architecture; Reference architecture; Software Engineering Process Group","score_opus":0.10052347010825755,"score_gpt":0.34551098358241233,"score_spread":0.2449875134741548,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2068712694","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.009351314,0.0051118694,0.9719918,0.004240139,0.00074843125,0.00072123465,0.000011334538,0.007816431,0.000007443619],"genre_scores_gemma":[0.028789947,0.00030154322,0.96883565,0.0011420107,0.00040262804,0.0003083893,0.00000922267,0.0001553807,0.000055202145],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9947203,0.0001273448,0.0007651257,0.0018200151,0.00083902344,0.0017282368],"domain_scores_gemma":[0.9935502,0.003783447,0.00012522802,0.001413375,0.00014534587,0.0009824447],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00083818304,0.0010661936,0.0010908112,0.0007381791,0.00025950765,0.00027747807,0.0016238487,0.00047967318,0.0000072563676],"category_scores_gemma":[0.015058465,0.0010940722,0.0002726422,0.0012370513,0.00005463447,0.00071704295,0.0009388127,0.0013891849,0.000044365202],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020475767,0.00007750428,0.0011163205,0.00019839309,0.00006816599,0.00015183141,0.0018215182,0.6007149,0.00049943564,0.0005349062,0.0003215909,0.39447492],"study_design_scores_gemma":[0.006268428,0.004205232,0.4933577,0.0035457686,0.0003373371,0.0023301141,0.00032733273,0.049224924,0.009829522,0.052423738,0.3586349,0.019515038],"about_ca_topic_score_codex":0.0000031840382,"about_ca_topic_score_gemma":0.0000017627431,"teacher_disagreement_score":0.55149,"about_ca_system_score_codex":0.00030152735,"about_ca_system_score_gemma":0.0000942269,"threshold_uncertainty_score":0.99915093},"labels":[],"label_agreement":null},{"id":"W2070873282","doi":"10.1007/s10664-011-9169-5","title":"The evolution of Java build systems","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":73,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Java; Executable; Software evolution; Software maintenance; Software system; Software engineering; Source code; Software development; Legacy system; Overhead (engineering); Codebase; Source lines of code; Software; Operating system; Distributed computing; Software construction","score_opus":0.028469191643593714,"score_gpt":0.2551634501681812,"score_spread":0.2266942585245875,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2070873282","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059028164,0.0015929262,0.9373577,0.00004019936,0.000963094,0.00015880799,0.0000014899188,0.0007936824,0.000063921856],"genre_scores_gemma":[0.9703323,0.000010673394,0.029384367,0.0000058569676,0.00008973648,0.000040624123,4.5116772e-7,0.00002553796,0.000110443],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983843,0.00003675936,0.00032805334,0.00028924583,0.00048341957,0.00047819605],"domain_scores_gemma":[0.9974864,0.0014203289,0.000056242516,0.0007518241,0.00013527818,0.00014991367],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005388668,0.00016722784,0.00018654144,0.0001387904,0.000101548125,0.0000685442,0.0012214079,0.00009371116,0.0000048328275],"category_scores_gemma":[0.0023843762,0.00012740736,0.00008820135,0.0007623974,0.000043731103,0.00024305856,0.00031687942,0.00027972308,0.000037264082],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000026659485,0.00019846243,0.8853792,0.00040252178,0.00022925298,0.00007460085,0.0024143527,0.054966472,0.0008150096,0.042571176,0.005379547,0.0075427773],"study_design_scores_gemma":[0.0004706901,0.000272287,0.8313681,0.00019728646,0.000015079805,0.00008487248,0.000063691834,0.14342912,0.002371727,0.0005970853,0.020387003,0.0007430642],"about_ca_topic_score_codex":0.00012841294,"about_ca_topic_score_gemma":0.0000010363901,"teacher_disagreement_score":0.9113041,"about_ca_system_score_codex":0.00023217361,"about_ca_system_score_gemma":0.00008235101,"threshold_uncertainty_score":0.5195522},"labels":[],"label_agreement":null},{"id":"W2071255138","doi":"10.1007/s10664-014-9323-y","title":"Recommending reference API documentation","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":67,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Documentation; Computer science; Categorization; Programmer; Set (abstract data type); Internal documentation; Information retrieval; World Wide Web; Software; Artificial intelligence; Programming language; Software system","score_opus":0.03239301421113391,"score_gpt":0.30504308565920896,"score_spread":0.27265007144807507,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2071255138","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.058455,0.000059332695,0.93835914,0.00043916423,0.00053973234,0.000105717365,9.1049026e-7,0.0019124419,0.00012854632],"genre_scores_gemma":[0.802001,0.0000076020183,0.19741641,0.00021048037,0.00016948696,0.00004465742,0.0000065365193,0.000035534817,0.000108291635],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981727,0.000050304323,0.000273319,0.00048528877,0.0004573567,0.00056104193],"domain_scores_gemma":[0.9971373,0.0019191589,0.00004075105,0.0005940804,0.0000659112,0.00024280981],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005380241,0.00022090279,0.00020571523,0.00022927616,0.00010014483,0.00025521696,0.00087092427,0.00010535998,0.000059716625],"category_scores_gemma":[0.0032194473,0.00022567561,0.00006325347,0.00068605074,0.000013942237,0.0006679474,0.00032711634,0.00040316465,0.00025278382],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000019629171,0.00023800135,0.6039171,0.00048802147,0.00016726762,0.0000797748,0.0018989821,0.0968254,0.0033139472,0.020404335,0.024443652,0.24820389],"study_design_scores_gemma":[0.0012037578,0.000344912,0.5122635,0.0002302285,0.000014606073,0.000082733735,0.000012184811,0.21058877,0.0053419573,0.0016263283,0.26658213,0.001708903],"about_ca_topic_score_codex":0.000011831944,"about_ca_topic_score_gemma":5.590958e-7,"teacher_disagreement_score":0.743546,"about_ca_system_score_codex":0.00018106727,"about_ca_system_score_gemma":0.000026527117,"threshold_uncertainty_score":0.9202785},"labels":[],"label_agreement":null},{"id":"W2072143380","doi":"10.1007/s10664-015-9361-0","title":"Evaluating the impact of design pattern and anti-pattern dependencies on changes and faults","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université de Montréal; Polytechnique Montréal; Queen's University","funders":"Fonds Québécois de la Recherche sur la Nature et les Technologies; Canada Research Chairs","keywords":"Software design pattern; Structural pattern; Maintainability; Computer science; Design pattern; Flexibility (engineering); Architectural pattern; Data mining; Software design; Software; Software engineering; Software development; Programming language; Mathematics; Statistics","score_opus":0.1456268245412086,"score_gpt":0.381946844692815,"score_spread":0.23632002015160639,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2072143380","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.52713734,0.00028858968,0.47205496,0.0001479371,0.000073227144,0.00012763582,0.0000022226484,0.00016727205,8.0796514e-7],"genre_scores_gemma":[0.9875284,0.000011413323,0.012281047,0.000044536657,0.00007452991,0.000024586769,8.969792e-7,0.000026383175,0.000008203441],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99837863,0.00008821565,0.00018705698,0.00037158982,0.0005674772,0.00040701745],"domain_scores_gemma":[0.99658865,0.0026015332,0.00005026673,0.0004231094,0.00010652653,0.00022990911],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010230416,0.00023150157,0.00025159534,0.00015786848,0.000062771214,0.00010135298,0.00048882706,0.000080732236,0.0000029117236],"category_scores_gemma":[0.0033058175,0.00015877718,0.000046362475,0.0003461154,0.000044650857,0.00017418254,0.00038994002,0.00029564157,0.000005200353],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012701668,0.00004930282,0.6266164,0.00009625548,0.00009065377,0.00003668867,0.0034189054,0.23787174,0.0007354886,0.0000060009916,0.00037853082,0.13068733],"study_design_scores_gemma":[0.00042101808,0.00084527675,0.5362682,0.000107212305,0.0000070727287,0.000060071357,0.000018877347,0.46085992,0.0010870656,0.000039475264,0.00002153528,0.00026430452],"about_ca_topic_score_codex":0.000099934405,"about_ca_topic_score_gemma":0.000001716776,"teacher_disagreement_score":0.46039104,"about_ca_system_score_codex":0.00008944839,"about_ca_system_score_gemma":0.00006893265,"threshold_uncertainty_score":0.6474745},"labels":[],"label_agreement":null},{"id":"W2074917598","doi":"10.1007/s10664-014-9353-5","title":"The effects of visualization and interaction techniques on feature model configuration","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Athabasca University; Simon Fraser University","funders":"","keywords":"Feature model; Feature (linguistics); Software product line; Computer science; Visualization; Artifact (error); Process (computing); Software visualization; Domain (mathematical analysis); Feature-oriented domain analysis; Data mining; Software; Domain engineering; Domain analysis; Set (abstract data type); Software engineering; Artificial intelligence; Software system; Software development; Component-based software engineering; Software construction","score_opus":0.03807687604875666,"score_gpt":0.33070934099790766,"score_spread":0.292632464949151,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2074917598","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.01188614,0.00030541825,0.9863544,0.00012525395,0.0003798865,0.0001494661,3.7467305e-7,0.00078944577,0.0000096087615],"genre_scores_gemma":[0.31186953,0.000101165766,0.6876804,0.00010298605,0.0000798361,0.000058192807,0.0000033027222,0.000028776769,0.0000757786],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9992722,0.00004643011,0.0001433039,0.00019279079,0.00019622136,0.00014903143],"domain_scores_gemma":[0.9978502,0.0016919915,0.00006119178,0.00023792821,0.00009051505,0.00006822136],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00026789843,0.00013257582,0.00013613545,0.00007637601,0.000050348335,0.000052584706,0.00020853408,0.00008534648,6.429575e-8],"category_scores_gemma":[0.0043384247,0.00009972952,0.000027246442,0.00021668873,0.000021272916,0.0003056239,0.00006587624,0.00017296913,8.098839e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000051155137,0.000063700165,0.00071815343,0.00030968897,0.000050446604,0.00001061085,0.0033675442,0.863751,0.0070613613,0.018665368,0.0028667357,0.10308423],"study_design_scores_gemma":[0.00041002128,0.0005837276,0.0018173661,0.00028904283,0.000014141462,0.000020372509,0.000030202806,0.79578584,0.18767564,0.0072285626,0.00566315,0.00048192442],"about_ca_topic_score_codex":8.418904e-7,"about_ca_topic_score_gemma":1.7593221e-7,"teacher_disagreement_score":0.29998338,"about_ca_system_score_codex":0.00006571232,"about_ca_system_score_gemma":0.000019513473,"threshold_uncertainty_score":0.5193813},"labels":[],"label_agreement":null},{"id":"W2075269190","doi":"10.1007/s10664-014-9350-8","title":"Linguistic antipatterns: what they are and how developers perceive them","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":116,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Lexicon; Documentation; Source code; Cognitive dissonance; Computer science; Open source; Code (set theory); Empirical research; Affect (linguistics); Code review; Data science; Linguistics; Psychology; Artificial intelligence; Software development; Static program analysis; Social psychology; Software; Programming language; Communication; Epistemology","score_opus":0.064961403368231,"score_gpt":0.28485054955810457,"score_spread":0.21988914618987357,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2075269190","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38949472,0.0009100485,0.60548884,0.0015115726,0.0010013534,0.00017440248,0.0000025304048,0.0014025903,0.0000139572],"genre_scores_gemma":[0.93576324,0.000068073874,0.06354639,0.00019838374,0.00023964493,0.000030502326,0.0000028940535,0.000054605658,0.00009624722],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9979285,0.000042941134,0.00017238682,0.0006045542,0.0006051355,0.00064646156],"domain_scores_gemma":[0.99751353,0.0011253974,0.00004318007,0.0005571034,0.00022145474,0.00053931633],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004484697,0.00032294853,0.00031302086,0.00016819981,0.00008009019,0.0008416695,0.0008287748,0.00013667406,0.0000053024337],"category_scores_gemma":[0.0075708027,0.00029545228,0.0000630758,0.00043636194,0.000036146568,0.0008455758,0.00054677005,0.00044476372,0.00006794325],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010536525,0.00006630292,0.9622895,0.00021074244,0.000103866645,0.00049018074,0.013608111,0.00808102,0.00006107798,0.00019718152,0.0025044957,0.012376991],"study_design_scores_gemma":[0.0015438506,0.00022468764,0.88431066,0.00090860645,0.000027099277,0.0004744604,0.00070347183,0.06907811,0.0005806216,0.00051370624,0.039490312,0.0021444028],"about_ca_topic_score_codex":0.000011493253,"about_ca_topic_score_gemma":0.0000021913802,"teacher_disagreement_score":0.5462685,"about_ca_system_score_codex":0.00019121822,"about_ca_system_score_gemma":0.00010036647,"threshold_uncertainty_score":0.99994975},"labels":[],"label_agreement":null},{"id":"W2087670325","doi":"10.1007/s10664-014-9333-9","title":"Improving bug management using correlations in crash reports","year":2014,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":25,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"Crash; Computer science; Eclipse; Precision and recall; Identification (biology); Software bug; Software; Data mining; Artificial intelligence; Programming language","score_opus":0.019100434507084794,"score_gpt":0.2706619834027131,"score_spread":0.25156154889562826,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2087670325","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19575357,0.0000569631,0.80238974,0.00006011493,0.0006259146,0.00016935857,3.3295717e-7,0.0009089115,0.000035100507],"genre_scores_gemma":[0.7345207,0.0000015847968,0.26519454,0.00005021404,0.000099324854,0.000030666757,0.000002141307,0.000036229194,0.00006460624],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978176,0.000035402212,0.00043408351,0.0005918124,0.00048469784,0.00063641],"domain_scores_gemma":[0.998072,0.00085261755,0.00006128122,0.00076537183,0.00005103015,0.00019769603],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00066274905,0.00023182611,0.00023737788,0.00046487487,0.000082654435,0.00015247233,0.0005222484,0.000109236615,0.000010720587],"category_scores_gemma":[0.002259102,0.00025535873,0.00007800389,0.001157508,0.00001602656,0.0004202141,0.0005032823,0.0004143367,0.000024003983],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000010605884,0.00005183597,0.5273074,0.000117201154,0.000018880502,0.00028616248,0.00017081882,0.46356782,0.00018901529,0.00039080955,0.00019720323,0.007701816],"study_design_scores_gemma":[0.00021596081,0.000025253004,0.32773194,0.000098481505,0.0000055794735,0.00008628682,0.0000032806395,0.66838676,0.00014606284,0.00019482804,0.0027316916,0.000373889],"about_ca_topic_score_codex":0.000038343653,"about_ca_topic_score_gemma":0.0000023016537,"teacher_disagreement_score":0.5387671,"about_ca_system_score_codex":0.00030742912,"about_ca_system_score_gemma":0.00003652613,"threshold_uncertainty_score":0.99998987},"labels":[],"label_agreement":null},{"id":"W2090432523","doi":"10.1007/s10664-008-9076-6","title":"“Cloning considered harmful” considered harmful: patterns of cloning in software","year":2008,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":363,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Cloning (programming); Source code; Computer science; Software engineering; Code (set theory); Software system; Software; clone (Java method); Maintainability; Web application; Codebase; Programming language; World Wide Web; Biology; Genetics","score_opus":0.044662327963924,"score_gpt":0.2815639292607685,"score_spread":0.2369016012968445,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2090432523","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49146175,0.0004283489,0.5060543,0.00018198081,0.0004281427,0.00025509123,0.000012868421,0.0011706305,0.0000068995946],"genre_scores_gemma":[0.85962284,0.000037189344,0.13982724,0.00015345716,0.000112413894,0.000072621886,0.000012693668,0.000099759876,0.00006179701],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.995464,0.00011497835,0.0010379485,0.0010163429,0.0010360249,0.0013307082],"domain_scores_gemma":[0.9931839,0.004786635,0.00017045281,0.0011115333,0.00028943003,0.00045802654],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00063403,0.0005862429,0.0008906313,0.0008212666,0.00016376408,0.00010731257,0.0013043842,0.0003110822,0.00009247373],"category_scores_gemma":[0.011222485,0.0006485433,0.00025025677,0.0014678097,0.00010484615,0.00070020254,0.0008272546,0.0011249504,0.00005015781],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000134123,0.00011382822,0.9472035,0.000254443,0.00006843892,0.0009116988,0.0016107813,0.04721961,0.00060031295,0.00008519979,0.00083993125,0.0010787974],"study_design_scores_gemma":[0.0022371167,0.0002040457,0.9482715,0.0007903987,0.000016330143,0.0006366497,0.000045929213,0.02974329,0.014747475,0.0001725155,0.001720953,0.0014137677],"about_ca_topic_score_codex":0.00017314179,"about_ca_topic_score_gemma":0.00001634555,"teacher_disagreement_score":0.36816108,"about_ca_system_score_codex":0.00032416606,"about_ca_system_score_gemma":0.00033916047,"threshold_uncertainty_score":0.9995966},"labels":[],"label_agreement":null},{"id":"W2093400716","doi":"10.1007/s10664-015-9379-3","title":"What are mobile developers asking about? A large scale study using stack overflow","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":311,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Computer science; Mobile device; World Wide Web; Mobile computing; Latent Dirichlet allocation; Context (archaeology); Software; Popularity; Mobile Web; Data science; Mobile technology; Software development; Topic model; Telecommunications; Artificial intelligence; Operating system","score_opus":0.0514213064798872,"score_gpt":0.32439770776851695,"score_spread":0.27297640128862977,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2093400716","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5360192,0.00066045776,0.4604915,0.00003418926,0.0012144799,0.00039289644,0.000003237804,0.0011814946,0.0000025508873],"genre_scores_gemma":[0.89612275,0.000012798809,0.10312206,0.0001315801,0.00027573627,0.00013109874,0.0000042401257,0.00011004237,0.0000897048],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9959553,0.000093484494,0.00048875465,0.00093418173,0.0012899626,0.0012383385],"domain_scores_gemma":[0.99705404,0.00075951807,0.000087833396,0.0010646546,0.00031421863,0.0007197637],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001120452,0.0004744451,0.00051881856,0.00039258925,0.00017589638,0.00091137923,0.0013758519,0.00017872365,0.000018103225],"category_scores_gemma":[0.002266109,0.00048369882,0.0001300408,0.0017669824,0.000026946213,0.0020174538,0.0011220932,0.00067740853,0.000101076235],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011575212,0.00041524842,0.7260981,0.00008403755,0.00009483498,0.00031313984,0.0100102965,0.2600571,0.00004521414,0.000007871424,0.0010632832,0.0017993113],"study_design_scores_gemma":[0.0036799444,0.0006789127,0.46294424,0.0008305109,0.00004796964,0.00016821668,0.004623929,0.49581155,0.00071000267,0.000050896848,0.027763532,0.0026902864],"about_ca_topic_score_codex":0.000024832467,"about_ca_topic_score_gemma":0.00000977492,"teacher_disagreement_score":0.36010355,"about_ca_system_score_codex":0.0007542131,"about_ca_system_score_gemma":0.00026902626,"threshold_uncertainty_score":0.99976146},"labels":[],"label_agreement":null},{"id":"W2094392229","doi":"10.1023/a:1009809824225","title":"Beg, Borrow, or Steal: Using Multidisciplinary Approaches in Empirical Software Engineering Research","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":41,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"National Research Council Canada; University of Toronto; University of Victoria; University of New Brunswick","funders":"University of Victoria","keywords":"Computer science","score_opus":0.286019139000558,"score_gpt":0.4116135689242767,"score_spread":0.12559442992371872,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2094392229","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25625172,0.00069439923,0.73859084,0.00042316856,0.000522577,0.00046401453,0.0000053893323,0.0030319118,0.000015963684],"genre_scores_gemma":[0.34951428,0.00007872253,0.6494584,0.00007267588,0.00041640288,0.00016994706,0.000011847757,0.000167606,0.00011010101],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99461794,0.00019563433,0.0008965337,0.0013417315,0.0011538875,0.0017942548],"domain_scores_gemma":[0.9938998,0.003956448,0.000118521566,0.0013140043,0.00016055894,0.0005506386],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0021132715,0.000704124,0.0007299541,0.0011724212,0.0002806886,0.00041273798,0.0018924376,0.00053149724,0.000043708576],"category_scores_gemma":[0.005409092,0.00066177116,0.00021769678,0.003870739,0.00007076548,0.0015529532,0.0014547369,0.00205372,0.000040007875],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012631525,0.00058357767,0.5266621,0.00048708988,0.00010251148,0.0020364658,0.0026806442,0.45678604,0.00023063445,0.00028548067,0.0016302781,0.00838888],"study_design_scores_gemma":[0.0009362838,0.00038879478,0.121816665,0.0006165264,0.00002409834,0.00070442964,0.00008468013,0.8378141,0.000490397,0.000223714,0.035145346,0.0017550015],"about_ca_topic_score_codex":0.00010893487,"about_ca_topic_score_gemma":0.000016945825,"teacher_disagreement_score":0.40484545,"about_ca_system_score_codex":0.0007084841,"about_ca_system_score_gemma":0.0002614472,"threshold_uncertainty_score":0.99958336},"labels":[],"label_agreement":null},{"id":"W2100925270","doi":"10.1007/s10664-011-9171-y","title":"An exploratory study of the impact of antipatterns on class change- and fault-proneness","year":2011,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":394,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"Odds; Computer science; Machine learning; Logistic regression","score_opus":0.10704019854817991,"score_gpt":0.3282979284401009,"score_spread":0.221257729891921,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2100925270","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9257967,0.000051904415,0.07339596,0.000009180326,0.00018842235,0.000324949,0.000004722265,0.00022643141,0.0000017309052],"genre_scores_gemma":[0.9983754,0.0000025945992,0.0014803963,0.000012303531,0.000046687735,0.000050576426,4.6605282e-7,0.000029781715,0.0000017957967],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9985618,0.000074208656,0.00025836966,0.000359866,0.00042322243,0.00032252056],"domain_scores_gemma":[0.99840736,0.0003675955,0.00006838126,0.00091331144,0.00008901172,0.00015432954],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00028437396,0.0002122619,0.0002803308,0.00018796699,0.0000421935,0.000024401488,0.0009705887,0.00007704553,0.000006882579],"category_scores_gemma":[0.00040165696,0.0001496581,0.00008254809,0.00059980084,0.00004612625,0.00034814575,0.00035382755,0.00026878947,0.0000019759607],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011632566,0.00045895393,0.9817401,0.00004953308,0.000045635144,0.000010230655,0.009926309,0.005829439,0.0003553885,0.000016441925,0.00002096232,0.001535354],"study_design_scores_gemma":[0.00031836223,0.0009442537,0.98627347,0.00005901608,0.0000049805117,0.0000041230064,0.00006518595,0.009982081,0.0021639129,0.000011400567,0.000006430147,0.00016677004],"about_ca_topic_score_codex":0.000110316236,"about_ca_topic_score_gemma":0.000004510009,"teacher_disagreement_score":0.0725787,"about_ca_system_score_codex":0.00006161538,"about_ca_system_score_gemma":0.000036825633,"threshold_uncertainty_score":0.61028796},"labels":[],"label_agreement":null},{"id":"W2102772625","doi":"10.1007/s10664-006-9006-4","title":"Replaying development history to assess the effectiveness of change propagation tools","year":2006,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":44,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; University of Victoria","funders":"","keywords":"Computer science; Dependency (UML); Notice; Source code; Software engineering; Code review; Open source; Empirical research; Software development; Software; Dependency graph; Code (set theory); Change impact analysis; Data science; Static program analysis; Database; Programming language","score_opus":0.11189261393545909,"score_gpt":0.2994290649800117,"score_spread":0.18753645104455263,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2102772625","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2917388,0.00020677618,0.7067032,0.00005624595,0.0003866018,0.00041285972,6.5991503e-7,0.00047851252,0.000016343804],"genre_scores_gemma":[0.9248052,7.067122e-7,0.07457839,0.000049424,0.00012884024,0.00036918433,0.0000049614728,0.000030494695,0.00003277297],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982477,0.00010253431,0.00030107624,0.00038393866,0.0005645792,0.00040018753],"domain_scores_gemma":[0.99577236,0.0034164374,0.00004706141,0.0005277815,0.00013734821,0.00009898187],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0014120699,0.00019093235,0.00022184619,0.00019833464,0.00006440722,0.00006176432,0.0007599452,0.00007386487,0.000005140332],"category_scores_gemma":[0.0027290066,0.00015637449,0.00006034343,0.000631643,0.000018198574,0.00037382558,0.00032656296,0.00023233812,0.000021429247],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008120212,0.00036581044,0.7329366,0.0021018672,0.000140497,0.00011543036,0.0032689278,0.17962433,0.021473376,0.0047799437,0.0033357716,0.051776234],"study_design_scores_gemma":[0.000143859,0.00004429459,0.96470743,0.00017875733,0.0000030017968,0.000009139776,0.0000016899611,0.0031013296,0.01569612,0.000020812975,0.015838832,0.0002547194],"about_ca_topic_score_codex":0.00003267315,"about_ca_topic_score_gemma":0.0000011953949,"teacher_disagreement_score":0.6330664,"about_ca_system_score_codex":0.0007377683,"about_ca_system_score_gemma":0.00012387159,"threshold_uncertainty_score":0.63767666},"labels":[],"label_agreement":null},{"id":"W2109156518","doi":"10.1007/s10664-013-9258-8","title":"Bug characteristics in open source software","year":2013,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":232,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"National Science Foundation","keywords":"Software bug; Computer science; Concurrency; Security bug; Software regression; Linux kernel; Operating system; Software; Source code; Software engineering; Software system; Software security assurance; Software construction","score_opus":0.025828708118874232,"score_gpt":0.2819716111627644,"score_spread":0.2561429030438902,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2109156518","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.27261257,0.00010041974,0.7243778,0.00041483899,0.00043360243,0.0005183171,0.0000030446683,0.0015225651,0.000016832844],"genre_scores_gemma":[0.76524806,0.000011099848,0.23290114,0.00043936356,0.00022210144,0.00041811963,0.0000143693605,0.00012271349,0.0006230417],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99699336,0.000053122294,0.0005558701,0.0007850696,0.0005668098,0.0010457624],"domain_scores_gemma":[0.99642515,0.0018879195,0.0000666293,0.0010519816,0.00013851804,0.00042982164],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00048581988,0.00038975023,0.00049945735,0.00038159214,0.000090193804,0.00086892117,0.0031432502,0.00020484539,0.0001660376],"category_scores_gemma":[0.006162914,0.00039890542,0.00009619017,0.0013441136,0.00003196756,0.0013609937,0.0021198993,0.00076051214,0.0007149973],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000034438951,0.000115952294,0.9631999,0.00009513162,0.00002517549,0.000081950915,0.00047759837,0.0060061556,0.00011153323,0.00007429805,0.0047656093,0.025043216],"study_design_scores_gemma":[0.0004979551,0.0000682349,0.94850606,0.00013271887,0.0000028039358,0.000036415506,0.00000728441,0.03037778,0.00019381901,0.00017025042,0.019323817,0.0006828372],"about_ca_topic_score_codex":0.00019807881,"about_ca_topic_score_gemma":0.0000031226343,"teacher_disagreement_score":0.4926355,"about_ca_system_score_codex":0.0002995658,"about_ca_system_score_gemma":0.00012295708,"threshold_uncertainty_score":0.9998463},"labels":[],"label_agreement":null},{"id":"W2109942136","doi":"10.1007/s10664-008-9104-6","title":"A study of the non-linear adjustment for analogy based software cost estimation","year":2009,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":88,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of Saskatchewan; University of Wisconsin-Madison","keywords":"Categorical variable; Analogy; Flexibility (engineering); Computer science; Software; Estimation; Artificial intelligence; Artificial neural network; Machine learning; Data mining; Statistics; Mathematics; Engineering","score_opus":0.035501370270765505,"score_gpt":0.320520460559485,"score_spread":0.2850190902887195,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2109942136","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.17569157,0.000049550163,0.82202256,0.00023672079,0.00039607217,0.0011177226,0.0000063936195,0.00047842966,0.0000010061101],"genre_scores_gemma":[0.7837989,5.569163e-7,0.21574914,0.00015283517,0.000083568906,0.00016805693,0.0000054932198,0.000025559671,0.000015897846],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99796176,0.000040571227,0.00041565477,0.0004724407,0.0006003602,0.00050921587],"domain_scores_gemma":[0.99700016,0.0016789653,0.00008733684,0.0008949732,0.00018251294,0.00015606804],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00037762162,0.0002629807,0.00033353895,0.00021448448,0.00011008827,0.00004509681,0.0011297021,0.0001076663,0.0000045398165],"category_scores_gemma":[0.0038083938,0.00021283644,0.0001558816,0.0010381247,0.000019681369,0.00019698248,0.00017783939,0.0002906972,0.0000057268207],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001454208,0.00047308943,0.045636546,0.00007657426,0.000035977155,0.0000079289075,0.0005080796,0.9369976,0.00008286043,0.00003774022,0.0007461756,0.015382868],"study_design_scores_gemma":[0.0011410663,0.0006397379,0.43763202,0.0000661989,0.00001847554,0.0000058315895,0.000009491065,0.55879515,0.00094045344,0.000053665637,0.00043721142,0.00026069712],"about_ca_topic_score_codex":0.0000107351725,"about_ca_topic_score_gemma":0.0000022541697,"teacher_disagreement_score":0.6081073,"about_ca_system_score_codex":0.00018891382,"about_ca_system_score_gemma":0.00012335902,"threshold_uncertainty_score":0.8679218},"labels":[],"label_agreement":null},{"id":"W2110915013","doi":"10.1007/s10664-012-9205-0","title":"Studying the impact of social interactions on software quality","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":54,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Software quality; Software; Quality (philosophy); Source code; Software metric; Data science; Software development; Code review; Software engineering; Data mining","score_opus":0.08293084537093437,"score_gpt":0.3962824852350411,"score_spread":0.31335163986410675,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2110915013","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4948909,0.00009033333,0.50356203,0.00012312485,0.00049827504,0.0001519238,0.000007387189,0.00065749796,0.000018499402],"genre_scores_gemma":[0.983136,0.0000013540057,0.016279554,0.00004532964,0.00039780964,0.000047405432,0.0000034123793,0.000038495724,0.00005066695],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9977696,0.00011426398,0.0004100409,0.0003059959,0.00064439245,0.00075568596],"domain_scores_gemma":[0.9949265,0.00398101,0.00009528692,0.0006555191,0.00012046514,0.00022121923],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00088961597,0.00028169702,0.00032841548,0.00020799928,0.00021007653,0.00009123925,0.0010003963,0.00009316301,0.000053085227],"category_scores_gemma":[0.005485809,0.00020203684,0.00033621318,0.00095816865,0.000044166074,0.0005289955,0.0004188246,0.00069614296,0.0000677914],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000021060383,0.0004534882,0.9407433,0.00008227299,0.00025705516,0.0000055253645,0.005348,0.039460327,0.00030654957,0.000977701,0.0050465656,0.0072981524],"study_design_scores_gemma":[0.00019441193,0.00008503765,0.99520123,0.00002922416,0.00000641019,0.000011147012,0.00002271943,0.0022696431,0.00035670388,0.000027311047,0.0015276422,0.0002685455],"about_ca_topic_score_codex":0.000055280318,"about_ca_topic_score_gemma":6.6201295e-7,"teacher_disagreement_score":0.48824507,"about_ca_system_score_codex":0.0003689542,"about_ca_system_score_gemma":0.00007233857,"threshold_uncertainty_score":0.82388234},"labels":[],"label_agreement":null},{"id":"W2126432237","doi":"10.1023/a:1011930801390","title":"Is it Ethical to Evaluate Web-based Learning Tools using Students?","year":2001,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Usability and User Interface Design","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Computer science","score_opus":0.12063872378271584,"score_gpt":0.3809342550198918,"score_spread":0.26029553123717597,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2126432237","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.41434267,0.000018835655,0.58218616,0.0026708343,0.00021449936,0.0001273097,8.9429676e-7,0.00042662103,0.000012175408],"genre_scores_gemma":[0.8555423,0.0000019127071,0.13807225,0.0060830577,0.00010577189,0.000024069035,0.0000012654932,0.000030961237,0.00013837358],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99748075,0.0001278632,0.00036330233,0.00061400037,0.00083089713,0.0005831831],"domain_scores_gemma":[0.9982422,0.00073819805,0.000041473544,0.0005138598,0.00013462227,0.00032965143],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010151395,0.00026416546,0.0002851689,0.00015320121,0.00015436833,0.0004405564,0.0011054958,0.00024340232,0.0001674218],"category_scores_gemma":[0.0020264776,0.00026567493,0.00013823934,0.0008204386,0.000019324681,0.00031887877,0.00039394503,0.00093715766,0.00028703557],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027365106,0.00012836237,0.17421062,0.000045840734,0.000044757602,0.000074883086,0.001819017,0.8155792,0.0014784418,0.000033942237,0.0030149124,0.0035426605],"study_design_scores_gemma":[0.0006500401,0.000255542,0.030984366,0.00016954802,0.000020054256,0.000023891003,0.000014880463,0.8753403,0.001837193,0.0000515818,0.09001263,0.0006399926],"about_ca_topic_score_codex":0.000012390411,"about_ca_topic_score_gemma":0.0000024935075,"teacher_disagreement_score":0.4441139,"about_ca_system_score_codex":0.00022904815,"about_ca_system_score_gemma":0.00010797192,"threshold_uncertainty_score":0.99997956},"labels":[],"label_agreement":null},{"id":"W2142920827","doi":"10.1007/s10664-015-9398-0","title":"Introduction to the special issue on software maintenance and evolution research","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Software development; Open-source software development; Software engineering; Software analytics; Computer science; World Wide Web; Software; Source code; Open source software; Engineering; Software development process; Operating system","score_opus":0.04403391377686007,"score_gpt":0.31887500522402334,"score_spread":0.2748410914471633,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2142920827","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.03403651,0.00019002575,0.94519895,0.01640549,0.002594786,0.0004850949,0.000003878303,0.0010417447,0.00004352033],"genre_scores_gemma":[0.55690885,0.000034568136,0.35020924,0.0012959536,0.08588583,0.0006080543,0.000016923479,0.00027110166,0.004769465],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99673116,0.00012828903,0.00025414187,0.00077380845,0.0012478923,0.00086472224],"domain_scores_gemma":[0.9967187,0.0013876152,0.000024461693,0.0009422304,0.00039500924,0.0005319936],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0020873642,0.00024478754,0.00021492905,0.00041980905,0.00022430293,0.00028904766,0.0010317584,0.00012607043,0.000022431796],"category_scores_gemma":[0.02225215,0.0001937818,0.00004980873,0.0018806716,0.00006527189,0.00034961486,0.000718008,0.000902098,0.0007231746],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004730062,0.00007176883,0.023468358,0.000046315563,0.00002567611,0.000046680394,0.0017404718,0.09405034,0.000027131773,0.0019778695,0.8581157,0.02038243],"study_design_scores_gemma":[0.00034977793,0.00043910832,0.10408749,0.000067687564,0.000003130455,0.00006375028,0.000047894016,0.010573698,0.0002015333,0.00045754103,0.883309,0.00039941273],"about_ca_topic_score_codex":0.000028971986,"about_ca_topic_score_gemma":0.000004805914,"teacher_disagreement_score":0.5949897,"about_ca_system_score_codex":0.0007107561,"about_ca_system_score_gemma":0.00012809125,"threshold_uncertainty_score":0.98598385},"labels":[],"label_agreement":null},{"id":"W2144641356","doi":"10.1007/s10664-006-7552-4","title":"A flexible method for software effort estimation by analogy","year":2006,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":156,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Analogy; Data mining; Computer science; Similarity (geometry); Quality (philosophy); Feature (linguistics); Adaptation (eye); Estimation; Machine learning; Artificial intelligence; Engineering; Image (mathematics)","score_opus":0.017182083426203802,"score_gpt":0.31183334560405335,"score_spread":0.29465126217784954,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2144641356","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.006412457,0.00041103162,0.9886331,0.00031636533,0.00045720374,0.00044124195,0.00002487047,0.003294779,0.000008959392],"genre_scores_gemma":[0.03624903,0.0000021147457,0.9625425,0.00013776474,0.00021005108,0.0002906281,0.000093533126,0.00008158017,0.00039282904],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9973494,0.000028724522,0.0004523614,0.0007366197,0.0005286978,0.00090419286],"domain_scores_gemma":[0.99599534,0.0028984193,0.000066356384,0.00065921166,0.00015149191,0.0002292066],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00059440435,0.00036115933,0.00039499858,0.00031567973,0.0001483906,0.00020736792,0.0009191816,0.00022044002,0.000017176519],"category_scores_gemma":[0.0026657858,0.00037819427,0.00018831404,0.0009994789,0.00002252279,0.00047729726,0.0002450797,0.0003355072,0.00004321653],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000021960124,0.00022807014,0.08731848,0.00044842332,0.000105397376,0.000036663536,0.00015873993,0.75355285,0.001086753,0.0020672441,0.109902866,0.045072533],"study_design_scores_gemma":[0.0011536175,0.00025951696,0.05936804,0.000081692226,0.000026315718,0.00007418278,0.0000018984085,0.8433763,0.009967305,0.0026874195,0.08186958,0.0011341376],"about_ca_topic_score_codex":0.00007066281,"about_ca_topic_score_gemma":0.0000017928346,"teacher_disagreement_score":0.089823425,"about_ca_system_score_codex":0.0002613706,"about_ca_system_score_gemma":0.00009024113,"threshold_uncertainty_score":0.999867},"labels":[],"label_agreement":null},{"id":"W2145766063","doi":"10.1007/s10664-012-9206-z","title":"Introduction to the Special Issue on Mining Software Repositories in 2010","year":2012,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"European Metrology Programme for Innovation and Research; Universität des Saarlandes; University of Calgary","keywords":"Computer science; Software engineering; Data science; Software; Operating system","score_opus":0.018376988051948446,"score_gpt":0.27463512077081664,"score_spread":0.2562581327188682,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2145766063","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2207487,0.00032622163,0.74715334,0.009976009,0.019354092,0.000563923,0.000003861337,0.0018290629,0.00004481031],"genre_scores_gemma":[0.4244236,0.00001584701,0.38940367,0.0012476294,0.18151124,0.00046294314,0.000017082435,0.00024358858,0.0026744043],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.997412,0.000060307288,0.00035495768,0.00053918,0.0006863044,0.0009472399],"domain_scores_gemma":[0.99700683,0.001627288,0.000038771188,0.00092811056,0.00007246,0.00032656183],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00080053805,0.00029625182,0.00026066526,0.00035261858,0.0001429813,0.00019750946,0.0009720333,0.0001402159,0.00006703734],"category_scores_gemma":[0.008884975,0.00024643613,0.000080325255,0.0015351017,0.000021717899,0.00056322897,0.00042782046,0.0006357779,0.00043024792],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000031504725,0.00018563741,0.45662418,0.000072567345,0.000039130682,0.0000505288,0.006898194,0.116676085,0.00009638554,0.00022939095,0.38782164,0.031274747],"study_design_scores_gemma":[0.00017089266,0.00011233214,0.40421268,0.00005638342,0.0000037329316,0.000041334173,0.00002632823,0.0011472068,0.0011534566,0.000009543106,0.5926427,0.00042339752],"about_ca_topic_score_codex":0.000018304876,"about_ca_topic_score_gemma":0.000004255085,"teacher_disagreement_score":0.35774967,"about_ca_system_score_codex":0.0003539207,"about_ca_system_score_gemma":0.00004439482,"threshold_uncertainty_score":0.9999988},"labels":[],"label_agreement":null},{"id":"W2149695823","doi":"10.1007/s10664-006-9026-0","title":"Empirical evaluation of optimization algorithms when used in goal-oriented automated test data generation techniques","year":2006,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":59,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Test Management Approach; Computer science; Keyword-driven testing; Test data; Software development; Test harness; Software; Software reliability testing; Automation; Test strategy; Non-regression testing; Data mining; Software construction; Software engineering; Engineering; Programming language","score_opus":0.06930117347575526,"score_gpt":0.3370234376690021,"score_spread":0.26772226419324685,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2149695823","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.023737116,0.00011352865,0.9594879,0.00020466899,0.00015120562,0.0003798705,0.000012809566,0.01589891,0.000014003437],"genre_scores_gemma":[0.28893465,0.0000018665515,0.71057355,0.000043479107,0.0001174625,0.00005884009,0.00023976287,0.000025783756,0.0000046083715],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99747235,0.00012694104,0.00063536945,0.0006465216,0.0007920311,0.0003268057],"domain_scores_gemma":[0.9977652,0.0006679233,0.00015920405,0.0010084735,0.00033026538,0.000068961286],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0016009898,0.00024243946,0.0002972333,0.0003764406,0.000060230865,0.000090504545,0.0007741569,0.00019236018,0.000008212254],"category_scores_gemma":[0.004106378,0.00024748358,0.000044804056,0.0010644776,0.000028901242,0.0006699534,0.00036161428,0.0002089286,0.0000024096983],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000003725941,0.00049932185,0.40218502,0.000056213736,0.000016599637,0.000015993315,0.00035537785,0.549433,0.000918107,0.000035283803,0.03208046,0.014400875],"study_design_scores_gemma":[0.00028150438,0.00006412833,0.025896102,0.00008579692,0.000015791678,0.000010141789,7.7046417e-7,0.97015774,0.0026729552,0.00033546623,0.00022148008,0.00025809265],"about_ca_topic_score_codex":0.00018628343,"about_ca_topic_score_gemma":0.000012714499,"teacher_disagreement_score":0.42072475,"about_ca_system_score_codex":0.00025154403,"about_ca_system_score_gemma":0.0001476353,"threshold_uncertainty_score":0.99999774},"labels":[],"label_agreement":null},{"id":"W2200054515","doi":"10.1007/s10664-015-9409-1","title":"On the unreliability of bug severity data","year":2015,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":51,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Waterloo","funders":"","keywords":"Computer science; Software bug; Reliability (semiconductor); Eclipse; Software; Data mining; Data science; Programming language","score_opus":0.10725834666028859,"score_gpt":0.32480638545586177,"score_spread":0.21754803879557316,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2200054515","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2803766,0.00010922356,0.7170793,0.0010237716,0.00048945186,0.00017395482,0.00001597993,0.00069969776,0.000032010274],"genre_scores_gemma":[0.96267134,0.0000020376972,0.037016742,0.00014355166,0.00008573214,0.00001630037,0.0000075462303,0.000023824206,0.000032939322],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9979551,0.000069703194,0.00027906592,0.0005092919,0.000784423,0.00040246153],"domain_scores_gemma":[0.9923173,0.0047104806,0.000041297106,0.0025325776,0.00014663108,0.00025170165],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0015083958,0.00019119414,0.0002281005,0.00009041676,0.000044846638,0.00006778585,0.0029723905,0.00008867232,0.000015317723],"category_scores_gemma":[0.024736537,0.00013968548,0.000057227786,0.00081651844,0.000052862422,0.00032395704,0.0014317007,0.00045929648,0.000062895095],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006591679,0.0007469023,0.6372493,0.0003682106,0.00022303786,0.00010854248,0.0022336117,0.18035664,0.00017363379,0.0089635635,0.15989903,0.009611604],"study_design_scores_gemma":[0.0010057939,0.00045015517,0.47985402,0.0001936899,0.000018645997,0.000044944492,0.000027269321,0.46093985,0.0027245437,0.004699059,0.04890792,0.001134114],"about_ca_topic_score_codex":0.000029387427,"about_ca_topic_score_gemma":8.3058023e-7,"teacher_disagreement_score":0.6822947,"about_ca_system_score_codex":0.0001295937,"about_ca_system_score_gemma":0.00014895898,"threshold_uncertainty_score":0.98347855},"labels":[],"label_agreement":null},{"id":"W2214818406","doi":"10.1007/s10664-013-9284-6","title":"Management of community contributions","year":2013,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":34,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; Polytechnique Montréal; Queen's University","funders":"","keywords":"Thriving; Linux kernel; Android (operating system); Computer science; Empirical research; Knowledge management; Source code; Open source; Software; Software engineering; World Wide Web; Data science; Operating system","score_opus":0.019567496537484873,"score_gpt":0.27656324933417,"score_spread":0.25699575279668513,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2214818406","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14171073,0.000028388904,0.8564705,0.0006984117,0.00014034654,0.00019465011,0.0000051213306,0.00053465215,0.00021719965],"genre_scores_gemma":[0.807785,0.0000013406926,0.19190274,0.00013873792,0.00001579936,0.00005898756,0.0000063358707,0.000009990261,0.000081096616],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.999078,0.000041618277,0.00029013792,0.00011979429,0.00021237669,0.00025812333],"domain_scores_gemma":[0.9986436,0.00034214344,0.00005553637,0.00064368104,0.00022322983,0.00009183134],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00022823259,0.00012068624,0.00016210313,0.000097909484,0.00013190096,0.00007827581,0.0008218547,0.00005602811,0.000052766816],"category_scores_gemma":[0.00026640034,0.00012078906,0.00006159529,0.0008870058,0.00003153347,0.00044550642,0.0004802206,0.00030195137,0.00013527762],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007957673,0.002561788,0.2433563,0.0012602868,0.0013574851,0.00008817522,0.008490864,0.05979973,0.0020773262,0.39350745,0.0759546,0.21153803],"study_design_scores_gemma":[0.0007201305,0.00011002724,0.9524592,0.00016403609,0.000023124918,0.000020638141,0.00012066505,0.01127196,0.002091126,0.0071923872,0.025195392,0.0006313128],"about_ca_topic_score_codex":0.00005072332,"about_ca_topic_score_gemma":3.1321883e-7,"teacher_disagreement_score":0.7091029,"about_ca_system_score_codex":0.00007237258,"about_ca_system_score_gemma":0.000015779677,"threshold_uncertainty_score":0.49256352},"labels":[],"label_agreement":null},{"id":"W2343606968","doi":"10.1007/s10664-015-9416-2","title":"The impact of domain knowledge on the effectiveness of requirements engineering activities","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":28,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Domain (mathematical analysis); Systems engineering; Computer science; Domain engineering; Engineering; Engineering management; Knowledge management; Software development; Operating system; Mathematics","score_opus":0.02712372993167587,"score_gpt":0.3155331697157048,"score_spread":0.2884094397840289,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2343606968","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5765483,0.00017572584,0.42249912,0.00006591905,0.0002450846,0.00021485506,0.0000053861386,0.00022889224,0.00001673411],"genre_scores_gemma":[0.99744755,0.000013250776,0.0023364625,0.0000023102943,0.00006109021,0.00007575161,3.8246418e-7,0.00003873542,0.000024451489],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99811125,0.00017720445,0.0003356688,0.00031701918,0.0005002071,0.00055863085],"domain_scores_gemma":[0.9720847,0.026734276,0.00007464522,0.0008845037,0.00010541232,0.000116483636],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0018506432,0.00028003508,0.00032218717,0.00017725368,0.00009178612,0.000051132378,0.0012995126,0.00008798475,0.000007362074],"category_scores_gemma":[0.006177712,0.00013351077,0.00025346872,0.0007365993,0.00008510244,0.00024301455,0.000390848,0.0002567236,0.000011748154],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00060086435,0.0009518157,0.4620376,0.0015998221,0.0022090618,0.00005645735,0.0026794332,0.19231611,0.2630286,0.03862915,0.0026990778,0.033192012],"study_design_scores_gemma":[0.0010289103,0.0007846162,0.8663787,0.0014818849,0.000010894145,0.000015312962,0.000011048283,0.008511168,0.119349666,0.0006499262,0.0011608561,0.0006169888],"about_ca_topic_score_codex":0.000010903075,"about_ca_topic_score_gemma":2.8225682e-7,"teacher_disagreement_score":0.4208993,"about_ca_system_score_codex":0.00033365315,"about_ca_system_score_gemma":0.00010613127,"threshold_uncertainty_score":0.7395745},"labels":[],"label_agreement":null},{"id":"W2411521827","doi":"10.1007/s10664-016-9439-3","title":"Foreword to the special issue on empirical evidence on software product line engineering","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Toronto Metropolitan University","funders":"","keywords":"Software product line; Product line; Computer science; Software engineering; Software; Engineering; Manufacturing engineering; Software development; Operating system","score_opus":0.09659113352451718,"score_gpt":0.33783418186220177,"score_spread":0.2412430483376846,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2411521827","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.009344942,0.00022661516,0.9717185,0.010582061,0.0039118943,0.0006577006,0.000020243366,0.0035280657,0.000010013327],"genre_scores_gemma":[0.03659859,0.000049260056,0.94503784,0.002781305,0.013985943,0.00035189057,0.0000031015331,0.00022532918,0.00096675317],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9952849,0.0001364018,0.0006781311,0.0014925193,0.0010686287,0.0013393961],"domain_scores_gemma":[0.9876109,0.009515814,0.00010897644,0.0020828722,0.00016933358,0.00051210704],"candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0011014641,0.0008168167,0.0006641229,0.00041894166,0.0002144322,0.00015042421,0.0025179852,0.0002216016,0.000059801838],"category_scores_gemma":[0.04602789,0.00051511667,0.00027095102,0.0015079032,0.0000547187,0.000669512,0.00083443854,0.00085529994,0.0007852871],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000093774106,0.000110267414,0.0025951117,0.00009575698,0.000069594986,0.00011734074,0.00067521684,0.80293226,0.00042625854,0.0005025828,0.03806733,0.15431447],"study_design_scores_gemma":[0.0013362321,0.0027998057,0.06554804,0.0034284953,0.000056486668,0.00015796731,0.000014697492,0.0135762915,0.030407285,0.0009671412,0.8776037,0.0041038487],"about_ca_topic_score_codex":0.0000024209935,"about_ca_topic_score_gemma":0.0000017225284,"teacher_disagreement_score":0.83953637,"about_ca_system_score_codex":0.0006023004,"about_ca_system_score_gemma":0.000102534046,"threshold_uncertainty_score":0.9999927},"labels":[],"label_agreement":null},{"id":"W2417033663","doi":"10.1007/s10664-016-9438-4","title":"License usage and changes: a large-scale study on gitHub","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":45,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"European Commission; National Science Foundation","keywords":"License; Computer science; Traceability; Commit; Java; Software engineering; Python (programming language); AspectJ; Software; JavaScript; Secure coding; World Wide Web; Empirical research; Software development; Reuse; Computer security; Database; Programming language; Engineering; Software security assurance; Operating system","score_opus":0.027664631968529942,"score_gpt":0.28769272460390743,"score_spread":0.2600280926353775,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2417033663","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49967954,0.00008067326,0.49798194,0.0006380529,0.0002428062,0.00023998503,0.000005478624,0.0011236755,0.000007864155],"genre_scores_gemma":[0.9849562,0.000011987519,0.014207747,0.00020105614,0.00019442636,0.00010140713,6.2390876e-7,0.000055716173,0.00027081757],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9977093,0.000050914474,0.00020616752,0.0007123543,0.00057393924,0.0007473274],"domain_scores_gemma":[0.9965803,0.0021883796,0.00002954285,0.0007814944,0.0000586349,0.00036165467],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005245593,0.000298434,0.00028209857,0.000282607,0.00009871573,0.00013474768,0.0006630383,0.00010415772,0.000022951228],"category_scores_gemma":[0.0021166406,0.0002173584,0.00005405847,0.00060963264,0.000024383446,0.0002699479,0.00055116066,0.00027695257,0.0001287071],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027526,0.00068186905,0.96259165,0.00008657457,0.00011080343,0.00035764073,0.005315075,0.00061068236,0.0011086104,0.00014109367,0.00272407,0.026244398],"study_design_scores_gemma":[0.001651768,0.0009718962,0.9681646,0.00018984092,0.000009958145,0.000040947645,0.00005090283,0.0038462523,0.0009293067,0.00003767289,0.023360671,0.00074617355],"about_ca_topic_score_codex":0.0000042984257,"about_ca_topic_score_gemma":0.0000050525014,"teacher_disagreement_score":0.48527667,"about_ca_system_score_codex":0.00012812529,"about_ca_system_score_gemma":0.00002774307,"threshold_uncertainty_score":0.88636184},"labels":[],"label_agreement":null},{"id":"W2472751774","doi":"10.1007/s10664-016-9435-7","title":"An empirical study of emergency updates for top android mobile apps","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Mobile and Web Applications","field":"Computer Science","cited_by":40,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Android (operating system); Mobile apps; Computer science; Empirical research; Operating system; World Wide Web; Statistics","score_opus":0.021899480142806616,"score_gpt":0.31454526076882966,"score_spread":0.29264578062602303,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2472751774","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.41814676,0.00005557896,0.5809185,0.000092752554,0.00014082271,0.00039317983,0.0000078613675,0.00023926418,0.0000053099934],"genre_scores_gemma":[0.96778715,0.000013167882,0.031258166,0.000036742273,0.0001305688,0.00070491777,0.0000040262075,0.000021450678,0.00004381712],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9986142,0.000023461864,0.00038305562,0.00046117743,0.00021202912,0.00030608187],"domain_scores_gemma":[0.9986532,0.00028099687,0.0000635304,0.00072708825,0.00010205753,0.00017317617],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001829965,0.0001679204,0.00023448982,0.00009396976,0.00007206375,0.000023667784,0.0007253819,0.000073175244,0.000043265394],"category_scores_gemma":[0.00012787852,0.00012236647,0.000087319735,0.0004072473,0.000011162279,0.0003026324,0.00013744681,0.00007062764,0.000016928067],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000026302758,0.0032816841,0.86498994,0.00012365186,0.00013306283,0.000007930474,0.0031577097,0.0072288453,0.008504996,0.0010338866,0.01233341,0.099178605],"study_design_scores_gemma":[0.0064677135,0.009121352,0.5021666,0.00017700397,0.00015050861,0.000030727228,0.00043639526,0.10060167,0.026651174,0.004908194,0.3460318,0.003256845],"about_ca_topic_score_codex":0.0000061635856,"about_ca_topic_score_gemma":0.0000031879172,"teacher_disagreement_score":0.5496603,"about_ca_system_score_codex":0.000031814416,"about_ca_system_score_gemma":0.000028479093,"threshold_uncertainty_score":0.498996},"labels":[],"label_agreement":null},{"id":"W2518473846","doi":"10.1007/s10664-016-9451-7","title":"Naming the pain in requirements engineering","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":271,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"Queen's University; Eesti Teadusagentuur; Queen's University Belfast","keywords":"Relevance (law); Dependency (UML); Context (archaeology); Status quo; Empirical research; Computer science; Management science; Requirements engineering; Complement (music); Data science; Criticality; Engineering ethics; Knowledge management; Risk analysis (engineering); Engineering; Software; Epistemology; Artificial intelligence; Political science; Business","score_opus":0.025830632765819712,"score_gpt":0.27321335161251814,"score_spread":0.24738271884669844,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2518473846","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.020915302,0.00026877032,0.97443324,0.0020267405,0.00044233637,0.00016155631,0.0000011109793,0.0017376078,0.000013339551],"genre_scores_gemma":[0.72614336,0.000041507403,0.2728988,0.00047391013,0.00019257376,0.000119919,8.9603503e-7,0.00005702051,0.0000720274],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99827546,0.00006024534,0.00035001113,0.00040858248,0.00033560637,0.0005700957],"domain_scores_gemma":[0.99672735,0.0024063478,0.000058036538,0.00064598944,0.000043891083,0.00011839598],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0014296601,0.0002589348,0.0002059674,0.00018949978,0.00006024302,0.00011986448,0.0010517105,0.000109004664,0.000016327713],"category_scores_gemma":[0.004909373,0.00016380406,0.00008539291,0.0007033683,0.000014170056,0.0007453635,0.0003223338,0.00029132926,0.000025960375],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000030084333,0.0002969261,0.24967737,0.00037732077,0.00022355306,0.0006673874,0.0020714502,0.0779789,0.008314798,0.013146569,0.012752672,0.63446295],"study_design_scores_gemma":[0.0015149456,0.0004376111,0.1442563,0.001562126,0.000022874925,0.00015657739,0.000016910353,0.29685655,0.0053387037,0.0013296704,0.5459978,0.002509947],"about_ca_topic_score_codex":0.000013858077,"about_ca_topic_score_gemma":0.0000015769024,"teacher_disagreement_score":0.70522803,"about_ca_system_score_codex":0.00018841795,"about_ca_system_score_gemma":0.000028500881,"threshold_uncertainty_score":0.6679735},"labels":[],"label_agreement":null},{"id":"W2531425405","doi":"10.1007/s10664-016-9456-2","title":"Which log level should developers choose for a new logging statement?","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":107,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Statement (logic); Logging; Computer science; Brier score; Leverage (statistics); Login; Database; Data mining; Information retrieval; Operating system; Machine learning; Forestry","score_opus":0.07458787435700132,"score_gpt":0.30730650497298284,"score_spread":0.23271863061598153,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2531425405","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.032812923,0.000114356306,0.961664,0.002996464,0.00097428873,0.00036045257,0.000011113538,0.0010485585,0.000017837283],"genre_scores_gemma":[0.61757344,0.00002065504,0.3805265,0.00046654794,0.00040614,0.00012377264,0.000005380797,0.00005415153,0.0008233834],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9978303,0.000021276726,0.00048440963,0.0006138583,0.0003499492,0.0007002497],"domain_scores_gemma":[0.998212,0.0006708302,0.0000760552,0.00057712296,0.00015138344,0.0003126105],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00055273075,0.0002907008,0.00031248992,0.00013358935,0.00012792026,0.00009892006,0.000759066,0.00015549961,0.000024630684],"category_scores_gemma":[0.0010820044,0.00019492136,0.00011980509,0.0004988045,0.000017601444,0.0006798557,0.00024959908,0.00015018917,0.00009000645],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010478643,0.0002287915,0.49844545,0.0009798551,0.00040486243,0.000024828054,0.0031631778,0.011051798,0.0015789046,0.0011458944,0.07487842,0.40799323],"study_design_scores_gemma":[0.006227609,0.0006402509,0.3924212,0.0011844542,0.00006857841,0.00008662456,0.00004686266,0.036174674,0.009502307,0.0023505923,0.54795617,0.0033406704],"about_ca_topic_score_codex":0.000021941709,"about_ca_topic_score_gemma":0.000007582728,"teacher_disagreement_score":0.58476055,"about_ca_system_score_codex":0.00028898384,"about_ca_system_score_gemma":0.0002895386,"threshold_uncertainty_score":0.79486626},"labels":[],"label_agreement":null},{"id":"W2534933448","doi":"10.1007/s10664-016-9467-z","title":"Towards just-in-time suggestions for log changes","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":79,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Commit; Computer science; Logging; Random forest; Set (abstract data type); Directory; Source code; Process (computing); Machine learning; Data mining; Data science; Database; Artificial intelligence; Programming language; Operating system","score_opus":0.023848696009115034,"score_gpt":0.2737739051977656,"score_spread":0.24992520918865058,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2534933448","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09583456,0.000118687116,0.8990081,0.0031774456,0.0006659028,0.0003018029,0.000011648837,0.00086584856,0.000015976899],"genre_scores_gemma":[0.87659043,0.000028946935,0.12145888,0.00038473328,0.0005944898,0.0003634685,0.000005025533,0.000043726737,0.00053029123],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9986228,0.000019132289,0.00024266787,0.00040730636,0.00023758541,0.0004705243],"domain_scores_gemma":[0.9986909,0.0006152492,0.00003674742,0.0004479264,0.00007240068,0.00013677555],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00040921362,0.0001831552,0.00024846764,0.00014702139,0.00006190284,0.00004870311,0.00061537663,0.00013720522,0.000021990654],"category_scores_gemma":[0.00095495494,0.00012348752,0.000092256014,0.00039243605,0.000023579822,0.00027771617,0.0001703143,0.000098679026,0.00010688234],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000664752,0.00057135354,0.6466935,0.0010079398,0.00013000086,0.00006771218,0.002483106,0.006869541,0.0032662412,0.00228073,0.025941713,0.31062165],"study_design_scores_gemma":[0.00443765,0.0009126471,0.7040953,0.0013555702,0.000038939503,0.00009052242,0.000018073068,0.04874685,0.009568935,0.0024625144,0.2256741,0.0025989024],"about_ca_topic_score_codex":0.0000076147094,"about_ca_topic_score_gemma":0.0000043013974,"teacher_disagreement_score":0.7807559,"about_ca_system_score_codex":0.00016590078,"about_ca_system_score_gemma":0.00005990014,"threshold_uncertainty_score":0.5035675},"labels":[],"label_agreement":null},{"id":"W2536305519","doi":"10.1007/s10664-016-9462-4","title":"Multi-objective reverse engineering of variability-safe feature models based on code dependencies of system variants","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure","funders":"Austrian Science Fund","keywords":"Reverse engineering; Exploit; Computer science; Software product line; Software; Feature engineering; Source code; Feature (linguistics); Software engineering; Data mining; Consolidation (business); Software system; Machine learning; Artificial intelligence; Software development; Programming language","score_opus":0.025584771873456095,"score_gpt":0.2587096846354994,"score_spread":0.23312491276204333,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2536305519","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.02795051,0.00006378608,0.96995765,0.00011518831,0.0005113642,0.00035291372,0.00007292396,0.00096052774,0.000015145039],"genre_scores_gemma":[0.74436635,0.0000027490405,0.25543028,0.000015336822,0.000048599304,0.000043459717,0.0000020650805,0.000052853946,0.000038293954],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.996899,0.00011896217,0.00056698296,0.00076831266,0.00095554435,0.00069118757],"domain_scores_gemma":[0.99174064,0.006237442,0.00014017786,0.0012534686,0.000344765,0.00028348705],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012877609,0.00043499682,0.0006579906,0.0005026208,0.000049705704,0.000034972698,0.0012113303,0.0002971958,0.000008947841],"category_scores_gemma":[0.005936269,0.00034875877,0.00023032393,0.0010520542,0.000049232087,0.0004960987,0.00033669037,0.0004460627,0.000012381449],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004197215,0.00019181833,0.01916886,0.0010443452,0.00011005455,0.000058457215,0.0003600926,0.96840155,0.008470653,0.0009844018,0.00017544313,0.0009923445],"study_design_scores_gemma":[0.00096526876,0.00016542594,0.041811544,0.001217364,0.000017119466,0.000017556618,0.000008322312,0.94372267,0.0114286365,0.00002565696,0.00014460935,0.00047585595],"about_ca_topic_score_codex":0.00001999895,"about_ca_topic_score_gemma":8.378313e-7,"teacher_disagreement_score":0.7164159,"about_ca_system_score_codex":0.00052364165,"about_ca_system_score_gemma":0.00019391564,"threshold_uncertainty_score":0.99989647},"labels":[],"label_agreement":null},{"id":"W2543971965","doi":"10.1007/s10664-016-9452-6","title":"Review participation in modern code review","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":116,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; McGill University","funders":"","keywords":"Software quality; Code review; Android (operating system); Open source software; Computer science; Process (computing); Source code; Open source; Software; Best practice; Quality (philosophy); Set (abstract data type); Data science; Software development; Political science; Operating system","score_opus":0.04737496011315095,"score_gpt":0.34805441343046845,"score_spread":0.3006794533173175,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2543971965","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0037014051,0.09283011,0.8979808,0.0039767413,0.00019397268,0.000369559,0.0000030137962,0.00093518314,0.00000923275],"genre_scores_gemma":[0.5990868,0.2116793,0.1704314,0.015489158,0.00049896387,0.001893965,0.00001954613,0.00030407164,0.00059677946],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.997958,0.00006174019,0.00046254642,0.00049151777,0.00043795136,0.0005882306],"domain_scores_gemma":[0.99745685,0.0014461213,0.000045871588,0.0007488307,0.00008275956,0.00021953962],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0007575117,0.000221119,0.00038019396,0.0001409669,0.000025219539,0.000028796461,0.0007753494,0.0000718062,0.000043098175],"category_scores_gemma":[0.008980447,0.00016515366,0.00009599431,0.00096524047,0.000015591006,0.0004580875,0.00026205453,0.00022967078,0.00015286573],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008134658,0.00034140516,0.32344222,0.023361055,0.000079701975,0.0002644088,0.000269605,0.0055149016,0.0007857035,0.00054965727,0.08005099,0.56533223],"study_design_scores_gemma":[0.0018412092,0.0002573348,0.50774705,0.15539958,0.000058263184,0.0001135589,7.6998026e-7,0.041458126,0.0011844737,0.0009348129,0.28829962,0.0027052038],"about_ca_topic_score_codex":0.000003395321,"about_ca_topic_score_gemma":0.0000017855357,"teacher_disagreement_score":0.7275494,"about_ca_system_score_codex":0.00022987455,"about_ca_system_score_gemma":0.000062474035,"threshold_uncertainty_score":0.99936736},"labels":[],"label_agreement":null},{"id":"W2559532455","doi":"10.1007/s10664-016-9487-8","title":"Analysis of license inconsistency in large collections of open source projects","year":2016,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":35,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Japan Society for the Promotion of Science","keywords":"License; Computer science; Source code; Header; MIT License; Computer security; Software; Open source; Reuse; World Wide Web; Open source software; Resource (disambiguation); Software engineering; Programming language; Engineering; Operating system","score_opus":0.032707972927695436,"score_gpt":0.309570352120259,"score_spread":0.27686237919256357,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2559532455","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.33172408,0.00006754155,0.6675812,0.000103951934,0.00008064764,0.00020840045,0.0000127582025,0.00019648302,0.000024966233],"genre_scores_gemma":[0.97226864,0.000008477428,0.027356401,0.000016879183,0.00001427651,0.000053772255,0.0000015773396,0.000020942618,0.00025900514],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982472,0.000055740322,0.0004819468,0.00040451088,0.00037923534,0.0004313708],"domain_scores_gemma":[0.99637294,0.0025798338,0.000087040644,0.00068168185,0.00015076651,0.00012775633],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00062278984,0.00016470184,0.0005055631,0.0013817701,0.00004091491,0.000054506494,0.0012507676,0.00009831233,0.000038251972],"category_scores_gemma":[0.0050999257,0.00013365831,0.00015137564,0.0073539373,0.000029953164,0.00037600414,0.0008896447,0.00016291875,0.0000045013767],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00000974345,0.00018740054,0.9820506,0.000055098542,0.00026250997,0.000012411836,0.0008437089,0.013931203,0.00072041946,0.0001716359,0.000309041,0.0014462539],"study_design_scores_gemma":[0.0009981963,0.00014017428,0.95123434,0.0002240564,0.00006852744,0.0000063260795,0.000023552253,0.04158415,0.003055959,0.000044852117,0.0022370736,0.00038278592],"about_ca_topic_score_codex":0.00012697713,"about_ca_topic_score_gemma":0.000065460124,"teacher_disagreement_score":0.6405446,"about_ca_system_score_codex":0.00016002868,"about_ca_system_score_gemma":0.00015870697,"threshold_uncertainty_score":0.6105456},"labels":[],"label_agreement":null},{"id":"W2559885217","doi":"10.1007/s10664-017-9512-6","title":"Curating GitHub for engineered software projects","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":345,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Software; Classifier (UML); Software engineering; Software development; Software bug; Software metric; Data mining; Machine learning; Data science; Software quality; Artificial intelligence; Programming language","score_opus":0.05729360034467611,"score_gpt":0.325474775312802,"score_spread":0.2681811749681259,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2559885217","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.062632956,0.00022178733,0.9315885,0.00051808724,0.0013024096,0.0006809021,0.000017836477,0.003022398,0.000015124628],"genre_scores_gemma":[0.48461446,0.000004651521,0.51404285,0.000082974926,0.00053190376,0.0003617467,0.000011752869,0.00011280661,0.00023682835],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99671316,0.000022005532,0.0004611829,0.00093106454,0.0006578228,0.0012147463],"domain_scores_gemma":[0.9944837,0.002630208,0.00014691602,0.00205374,0.0002637099,0.00042174192],"candidate_categories":["metaresearch","metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0006123101,0.0004914973,0.0004844389,0.00028593073,0.0008796388,0.001475019,0.0029207307,0.00024542442,0.000011763595],"category_scores_gemma":[0.028839998,0.0004977081,0.00026111375,0.00035068078,0.00007040808,0.0012942852,0.00087995443,0.00055667246,0.000055275545],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000761463,0.0004828598,0.7440752,0.0025983471,0.00064197363,0.00047266917,0.0041477964,0.10835054,0.0027209336,0.0037592466,0.029699635,0.10297465],"study_design_scores_gemma":[0.003512656,0.00060278876,0.56188333,0.00074633164,0.00005348927,0.00017665605,0.000024000876,0.32764307,0.011271558,0.00086314854,0.08960683,0.0036161467],"about_ca_topic_score_codex":0.000019017916,"about_ca_topic_score_gemma":0.0000021822323,"teacher_disagreement_score":0.4219815,"about_ca_system_score_codex":0.00022010955,"about_ca_system_score_gemma":0.00021721121,"threshold_uncertainty_score":0.99974746},"labels":[],"label_agreement":null},{"id":"W2569512077","doi":"10.1007/s10664-016-9475-z","title":"Group versus individual use of power-only EPMcreate as a creativity enhancement technique for requirements elicitation","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Creativity; Brainstorming; Group (periodic table); Requirements elicitation; Raw data; Affect (linguistics); Computer science; Psychology; Social psychology; Requirements analysis; Artificial intelligence; Communication","score_opus":0.10535207857372979,"score_gpt":0.3668034348591423,"score_spread":0.2614513562854125,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2569512077","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07860037,0.00004747221,0.91948557,0.00014692191,0.00048449187,0.0005988914,0.000017729697,0.0005891448,0.00002943663],"genre_scores_gemma":[0.48407647,0.000014603796,0.5155599,0.000032487045,0.000042060703,0.00021302064,0.000010958547,0.000025214129,0.000025255436],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99835354,0.000031232536,0.00037038018,0.00044486256,0.00041415467,0.00038583073],"domain_scores_gemma":[0.9971017,0.0013966901,0.00030222436,0.00093750237,0.00013115362,0.00013071697],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006063773,0.00026837998,0.0003049723,0.00015231548,0.00022002203,0.00034220325,0.0010017469,0.00015446043,0.000012113811],"category_scores_gemma":[0.0047180764,0.00027449615,0.00013678608,0.00016018203,0.000036818474,0.0016599214,0.00041891044,0.00020210107,0.0000048071647],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00312957,0.004258337,0.32233796,0.0032114654,0.0038756307,0.0003270216,0.009193731,0.012123021,0.13511646,0.11440137,0.019023232,0.37300223],"study_design_scores_gemma":[0.0074170125,0.010293364,0.29575002,0.0017507795,0.00040210955,0.00009742719,0.000030407897,0.025618277,0.42433894,0.006484055,0.22331433,0.0045032892],"about_ca_topic_score_codex":0.00007870433,"about_ca_topic_score_gemma":0.000002355079,"teacher_disagreement_score":0.40547612,"about_ca_system_score_codex":0.00010436749,"about_ca_system_score_gemma":0.000057034904,"threshold_uncertainty_score":0.99997073},"labels":[],"label_agreement":null},{"id":"W2586191823","doi":"10.1007/s10664-017-9499-z","title":"Reengineering legacy applications into software product lines: a systematic mapping","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":143,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure","funders":"Conselho Nacional de Desenvolvimento Científico e Tecnológico; Austrian Science Fund; Coordenação de Aperfeiçoamento de Pessoal de Nível Superior","keywords":"Business process reengineering; Code refactoring; Computer science; Software product line; Software engineering; Process (computing); Process management; Product (mathematics); Reuse; Software; Data science; Software development; Systems engineering; Engineering; Manufacturing engineering","score_opus":0.05418621084206881,"score_gpt":0.3191469998959,"score_spread":0.2649607890538312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2586191823","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0022613106,0.0015763744,0.98798674,0.0003985906,0.0011032728,0.0010317723,0.0000030933259,0.00562409,0.000014767997],"genre_scores_gemma":[0.074399985,0.000030424468,0.92386675,0.00007246322,0.0005203854,0.00085509365,0.0000060964308,0.00011271421,0.00013606256],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99648637,0.0000748533,0.00083388656,0.0011140123,0.00057945604,0.00091141264],"domain_scores_gemma":[0.99393755,0.001678226,0.000341417,0.0034762337,0.00022320737,0.00034337185],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009862885,0.0006476646,0.00092624326,0.00038155253,0.0007665456,0.0009617576,0.0034726001,0.00019689083,0.0000045033585],"category_scores_gemma":[0.017852759,0.00063958764,0.0002537704,0.0006209207,0.00008112571,0.002275034,0.0012567181,0.0006985113,0.00008508932],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013432487,0.00020076992,0.014804557,0.036625955,0.00047122376,0.00024061122,0.0029708252,0.91332364,0.0020026602,0.003187585,0.00056869525,0.02559007],"study_design_scores_gemma":[0.008037717,0.001158664,0.18716253,0.050764684,0.0007023229,0.0022248162,0.00069700833,0.53789955,0.023025924,0.043698143,0.117232025,0.027396608],"about_ca_topic_score_codex":0.000014059364,"about_ca_topic_score_gemma":0.0000015142733,"teacher_disagreement_score":0.37542406,"about_ca_system_score_codex":0.00030564275,"about_ca_system_score_gemma":0.00009325794,"threshold_uncertainty_score":0.99960554},"labels":[],"label_agreement":null},{"id":"W2600697011","doi":"10.1007/s10664-017-9501-9","title":"Documenting and sharing software knowledge using screencasts","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":32,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Documentation; Software; Reputation; Software documentation; Knowledge sharing; Multimedia; World Wide Web; Persona; Best practice; Software development; Human–computer interaction; Knowledge management; Software development process","score_opus":0.0560603512164832,"score_gpt":0.33611254988620826,"score_spread":0.2800521986697251,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2600697011","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1887572,0.0002145243,0.809359,0.0001481839,0.00050136895,0.00019616255,0.0000026536288,0.00072480005,0.000096069365],"genre_scores_gemma":[0.50116694,0.000002474312,0.498311,0.000066686174,0.00020602962,0.000024246929,0.0000021752878,0.00004573643,0.0001747204],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99793476,0.000018589477,0.00039316027,0.00073485123,0.00029220298,0.0006264189],"domain_scores_gemma":[0.9978563,0.00028416436,0.00018249078,0.0012899657,0.00014931205,0.00023775655],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0005029605,0.00031566634,0.00031152114,0.00020986276,0.0010894731,0.0014966162,0.0016976546,0.00013679167,0.000016624672],"category_scores_gemma":[0.002347402,0.00033814422,0.00008426329,0.00036617066,0.00007401359,0.0018205266,0.0025030866,0.0003959258,0.000036953574],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000031916031,0.000075255186,0.9606953,0.00018069908,0.000088392015,0.000088708635,0.0014813471,0.003812498,0.00057794724,0.002081627,0.00029567484,0.030619338],"study_design_scores_gemma":[0.0013663957,0.00009317789,0.72833043,0.0009683029,0.000058723363,0.00038561976,0.00003687241,0.23917833,0.0015954281,0.0021007755,0.023644643,0.0022412934],"about_ca_topic_score_codex":0.00004953783,"about_ca_topic_score_gemma":0.0000042417555,"teacher_disagreement_score":0.31240973,"about_ca_system_score_codex":0.00012887448,"about_ca_system_score_gemma":0.000064093045,"threshold_uncertainty_score":0.9999071},"labels":[],"label_agreement":null},{"id":"W2604153767","doi":"10.1007/s10664-017-9510-8","title":"An empirical study of unspecified dependencies in make-based build systems","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":40,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; Polytechnique Montréal; McGill University; Queen's University","funders":"","keywords":"Deliverable; Computer science; Software engineering; Software deployment; sync; Software; Systems engineering; Programming language; Engineering","score_opus":0.05014827432941954,"score_gpt":0.3386470064963655,"score_spread":0.28849873216694594,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2604153767","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.603466,0.00008316529,0.39513943,0.00005953497,0.00045446394,0.00031187834,0.0000021943765,0.00047549486,0.000007877466],"genre_scores_gemma":[0.98214877,0.0000010523007,0.01756451,0.000021640084,0.00012651428,0.00006546595,0.0000020803936,0.000049916816,0.000020028012],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9966729,0.00009355914,0.0006334427,0.0007526159,0.0010832174,0.00076422375],"domain_scores_gemma":[0.99568444,0.0013710471,0.00014157649,0.0022907092,0.00017368123,0.0003385517],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012216358,0.00034597333,0.00054707105,0.00059091055,0.00017119372,0.00042370468,0.0028989075,0.00020687608,0.000005891222],"category_scores_gemma":[0.0034354916,0.0003415852,0.000091898066,0.0006245866,0.000057848974,0.00052748254,0.00046497036,0.00063574594,0.000012525795],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014238715,0.00043029123,0.8449474,0.00007838905,0.00001612406,0.00019079483,0.0005401034,0.15293145,0.00013430734,0.00003366713,0.000059597645,0.00062366505],"study_design_scores_gemma":[0.0009211146,0.00039774214,0.88012904,0.000072496085,0.000004971868,0.000008765889,0.000060917806,0.11720367,0.0004516084,0.000010100741,0.0003785277,0.00036103965],"about_ca_topic_score_codex":0.00030150678,"about_ca_topic_score_gemma":0.00006804349,"teacher_disagreement_score":0.37868282,"about_ca_system_score_codex":0.00023525322,"about_ca_system_score_gemma":0.00013001844,"threshold_uncertainty_score":0.9999036},"labels":[],"label_agreement":null},{"id":"W2604794021","doi":"10.1007/s10664-017-9514-4","title":"What do developers search for on the web?","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":177,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of British Columbia","funders":"Ministry of Science and Technology of the People's Republic of China; National Natural Science Foundation of China; Baidu","keywords":"Computer science; Debugging; World Wide Web; Reuse; Software bug; Software; Search engine optimization; Application programming interface; Information retrieval; Software engineering; Search engine; Data science; Programming language; Engineering","score_opus":0.06330645191543763,"score_gpt":0.3294800053743171,"score_spread":0.2661735534588795,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2604794021","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1979769,0.00032321477,0.78963447,0.008382531,0.0020708477,0.0005717629,0.0000048411857,0.0010004314,0.00003501315],"genre_scores_gemma":[0.94867384,0.00008883079,0.049696263,0.00044179423,0.00038312408,0.0002583623,0.0000023481814,0.00007585793,0.00037958304],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9978727,0.000027350197,0.00020249453,0.0005397423,0.0006126438,0.0007450482],"domain_scores_gemma":[0.99382246,0.0041240705,0.00003828916,0.0016520243,0.0001414963,0.00022167712],"candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00090931694,0.00025326,0.00021051957,0.00013782542,0.0006276443,0.0020007333,0.003000235,0.00011727623,0.000020124748],"category_scores_gemma":[0.006646141,0.00018615543,0.00013459449,0.00024241142,0.00006187339,0.0008738113,0.00067339354,0.00048449857,0.00012560599],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012886574,0.00046684756,0.38967273,0.0008008007,0.00070530263,0.00034820917,0.007045595,0.1540725,0.0013896371,0.04551778,0.08367986,0.31617188],"study_design_scores_gemma":[0.0019140465,0.0005409784,0.47558597,0.0010206464,0.00001855874,0.000058230886,0.0001240804,0.20687756,0.013284202,0.0006471526,0.29774904,0.0021795398],"about_ca_topic_score_codex":0.0000042276583,"about_ca_topic_score_gemma":6.997392e-7,"teacher_disagreement_score":0.75069696,"about_ca_system_score_codex":0.00016015704,"about_ca_system_score_gemma":0.00013460129,"threshold_uncertainty_score":0.9990353},"labels":[],"label_agreement":null},{"id":"W2605594016","doi":"10.1007/s10664-017-9516-2","title":"Data Transformation in Cross-project Defect Prediction","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":65,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Transformation (genetics); Computer science; Data mining; Data transformation; Software; Rank (graph theory); Predictive modelling; Reliability engineering; Machine learning; Data warehouse; Mathematics; Engineering","score_opus":0.08894280679107658,"score_gpt":0.37317781031934233,"score_spread":0.28423500352826575,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2605594016","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25425628,0.0001023149,0.74373174,0.00014015041,0.0005769487,0.00021983135,0.000037086727,0.00089876936,0.000036907717],"genre_scores_gemma":[0.959863,0.00001743026,0.039794873,0.000019395336,0.00015666906,0.000043974844,0.00004400021,0.000030538908,0.000030125168],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980905,0.000025158375,0.00032432273,0.0005734624,0.0004723774,0.00051416154],"domain_scores_gemma":[0.99696773,0.0005777204,0.000052238614,0.0022368124,0.000051461346,0.000114034],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008426551,0.00019988965,0.00020090357,0.00027216604,0.00021078042,0.00069389597,0.0029560938,0.00013884841,0.0000064668193],"category_scores_gemma":[0.005576007,0.00020420697,0.00006931787,0.0003379153,0.000033588207,0.0027743545,0.00076606125,0.00045886807,0.000039214945],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000087079125,0.000053057087,0.9460662,0.00012728568,0.000026993992,0.000056371475,0.00050097593,0.043334007,0.00006402205,0.00016629952,0.0008515659,0.008744516],"study_design_scores_gemma":[0.00036311548,0.000029387742,0.7375204,0.000051004517,0.0000022774216,0.000015372505,0.0000013202255,0.25625303,0.00009609805,0.000022033633,0.005468369,0.0001775797],"about_ca_topic_score_codex":0.00009053733,"about_ca_topic_score_gemma":0.000015227371,"teacher_disagreement_score":0.7056067,"about_ca_system_score_codex":0.00016563952,"about_ca_system_score_gemma":0.000088353896,"threshold_uncertainty_score":0.83273184},"labels":[],"label_agreement":null},{"id":"W2612705982","doi":"10.1007/s10664-017-9522-4","title":"Identifying self-admitted technical debt in open source projects using text mining","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":190,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia; Concordia University","funders":"Ministry of Science and Technology of the People's Republic of China; National Natural Science Foundation of China","keywords":"Technical debt; Computer science; Classifier (UML); Source code; Code review; Baseline (sea); Open source; Artificial intelligence; F1 score; Machine learning; Data mining; Natural language processing; Software; Software quality; Software development; Programming language","score_opus":0.07815053665037296,"score_gpt":0.3568263279761991,"score_spread":0.27867579132582615,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2612705982","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44006473,0.00009959622,0.55759555,0.00013050558,0.0003604548,0.00040454746,8.095723e-7,0.00130687,0.000036950136],"genre_scores_gemma":[0.6484872,0.0000036766153,0.3511574,0.000045427074,0.000116344156,0.0000661286,0.0000012457091,0.00006922843,0.000053368498],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9966274,0.00006131829,0.0005212922,0.0009986634,0.0006878369,0.0011035012],"domain_scores_gemma":[0.9963688,0.0012379334,0.00014148948,0.0018137419,0.00009576895,0.00034221378],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0010925541,0.00040080832,0.0005087902,0.00048197652,0.0004576864,0.0020191793,0.005158322,0.00026218616,0.000012552298],"category_scores_gemma":[0.007963097,0.00042761242,0.00011677738,0.0008401823,0.000046433353,0.0018432857,0.0044640456,0.00080856826,0.0000406439],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010465908,0.00015931365,0.9702091,0.00017637502,0.000050068833,0.00040777045,0.0013099731,0.01984857,0.0015882117,0.00010917261,0.00039572341,0.0057352325],"study_design_scores_gemma":[0.0010357423,0.0000718739,0.74635154,0.00057309924,0.000012171715,0.00017469635,0.000025996755,0.24722911,0.0012146077,0.00007214633,0.0022862481,0.0009527431],"about_ca_topic_score_codex":0.00017417768,"about_ca_topic_score_gemma":0.000015832218,"teacher_disagreement_score":0.22738054,"about_ca_system_score_codex":0.00055130565,"about_ca_system_score_gemma":0.00024769132,"threshold_uncertainty_score":0.99981755},"labels":[],"label_agreement":null},{"id":"W2612865512","doi":"10.1007/s10664-017-9520-6","title":"An empirical study of the integration time of fixed issues","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; McGill University","funders":"","keywords":"Eclipse; Computer science; Fixed cost; Heuristics; Fixed point; Fixed effects model; Operations research; Engineering; Business; Mathematics; Accounting; Operating system; Statistics","score_opus":0.038382145223893084,"score_gpt":0.3498034475928034,"score_spread":0.3114213023689103,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2612865512","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.79375637,0.00004011403,0.20519759,0.00021699104,0.0002739426,0.00024238366,0.0000024112892,0.0002643496,0.0000058461655],"genre_scores_gemma":[0.9844309,0.0000012248521,0.015348965,0.000014943013,0.00008247446,0.00001987108,0.0000010771648,0.000024400948,0.000076123746],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981914,0.00007759748,0.00037658244,0.00037217303,0.00067959965,0.00030261214],"domain_scores_gemma":[0.9968916,0.00063647423,0.0001430359,0.0020284017,0.00017593091,0.00012456613],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00047463883,0.00019783688,0.00033128768,0.00012761832,0.00016483705,0.0001356493,0.0026476646,0.00010223911,0.000016340575],"category_scores_gemma":[0.0046754573,0.00014631775,0.00010287603,0.0003527955,0.00006379243,0.0005173124,0.0006193132,0.00033123052,0.00001174503],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008238863,0.00046330073,0.97756,0.000041851763,0.000046455923,0.00000842927,0.0036026267,0.012099968,0.0023651638,0.000026699177,0.0006515442,0.0031257307],"study_design_scores_gemma":[0.00030638042,0.0003257503,0.9410421,0.000056637586,0.000008166344,0.0000037189552,0.000027275377,0.050944798,0.006862004,0.000034686418,0.0002167566,0.00017169677],"about_ca_topic_score_codex":0.00007880238,"about_ca_topic_score_gemma":0.00000539623,"teacher_disagreement_score":0.19067454,"about_ca_system_score_codex":0.000059679027,"about_ca_system_score_gemma":0.000056594465,"threshold_uncertainty_score":0.59666646},"labels":[],"label_agreement":null},{"id":"W2623177746","doi":"10.1007/s10664-017-9526-0","title":"An exploratory qualitative and quantitative analysis of emotions in issue report comments of open source systems","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":63,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Regione Autonoma della Sardegna","keywords":"Sadness; Gratitude; Recall; Psychology; Emotion classification; Cognitive psychology; Anger; Computer science; Social psychology","score_opus":0.10398874082538329,"score_gpt":0.43043162378672245,"score_spread":0.32644288296133916,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2623177746","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4810251,0.00008375615,0.51847583,0.00008607134,0.00009309101,0.0001529731,0.000010285342,0.00006778987,0.000005070915],"genre_scores_gemma":[0.9559738,0.000006173669,0.043901753,0.000007391785,0.0000123830105,0.00004021944,0.0000092636665,0.000018532644,0.000030479992],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982592,0.00015824004,0.0004952095,0.00042932166,0.00039270788,0.00026533086],"domain_scores_gemma":[0.99645036,0.0017654839,0.0002582906,0.0011710655,0.00019567886,0.0001590959],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0015883319,0.00016776728,0.00060138357,0.0005917253,0.00009727197,0.00023554129,0.0015490836,0.00007902506,0.0000036907102],"category_scores_gemma":[0.00491681,0.00017218725,0.000067043286,0.0008815523,0.000087512264,0.001078352,0.00079495186,0.00020688756,0.0000014814439],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015104559,0.00025603283,0.74107313,0.00025234927,0.00067198,0.000098159464,0.03499122,0.22026227,0.00042928598,0.0011019764,0.00018663886,0.0006618609],"study_design_scores_gemma":[0.0003666271,0.00015643216,0.73646307,0.00017523275,0.000049366306,0.0000061839173,0.0010615234,0.26083595,0.00017799102,0.00003493378,0.00040229142,0.0002703838],"about_ca_topic_score_codex":0.0005538892,"about_ca_topic_score_gemma":0.000028526412,"teacher_disagreement_score":0.4749487,"about_ca_system_score_codex":0.00007766574,"about_ca_system_score_gemma":0.000058898586,"threshold_uncertainty_score":0.7021593},"labels":[],"label_agreement":null},{"id":"W2727304061","doi":"10.1007/s10664-017-9531-3","title":"An empirical study of early access games on the Steam platform","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Educational Games and Gamification","field":"Psychology","cited_by":49,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Technische Universiteit Delft","keywords":"Computer science; Empirical research; Popularity; Game Developer; World Wide Web; Multimedia; Game design","score_opus":0.10240645122112182,"score_gpt":0.41855167410589367,"score_spread":0.31614522288477187,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2727304061","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.99613357,0.000039268372,0.0014118104,0.00094167376,0.0006404615,0.00032261733,0.000006954368,0.00010653394,0.00039711726],"genre_scores_gemma":[0.9989791,0.0000015174049,0.00013160452,0.00022727907,0.00028717503,0.000102008635,0.0000053970944,0.000029745017,0.00023617664],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99883187,0.00004171093,0.00028045627,0.0003196197,0.0002837283,0.0002426281],"domain_scores_gemma":[0.99805486,0.00051123364,0.00015159833,0.0011018149,0.00007120101,0.00010927053],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00027774737,0.0001652832,0.0001980375,0.00007247981,0.0001973936,0.00013709441,0.00091543514,0.0001012806,0.0002695186],"category_scores_gemma":[0.00045818655,0.00011705738,0.00006505372,0.00010608431,0.000048603462,0.00022571543,0.000081527585,0.00029430262,0.00006276413],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005128027,0.0011316445,0.9807123,0.0000089145,0.000077448,0.0000041117464,0.007515825,0.0006651629,0.00001574425,0.00036143523,0.0033334743,0.0061226417],"study_design_scores_gemma":[0.00028375996,0.0005112602,0.9938652,0.000014537789,0.000016403283,0.0000018612621,0.0008683683,0.00011756271,0.00003817147,0.000088495515,0.004054514,0.00013985316],"about_ca_topic_score_codex":0.00016734148,"about_ca_topic_score_gemma":0.000014249526,"teacher_disagreement_score":0.013152894,"about_ca_system_score_codex":0.00004051442,"about_ca_system_score_gemma":0.000024609846,"threshold_uncertainty_score":0.47734615},"labels":[],"label_agreement":null},{"id":"W2727610827","doi":"10.1007/s10664-017-9529-x","title":"Noise in Mylyn interaction traces and its impact on developers and recommendation systems","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Noise (video); Software; Code (set theory); Source code; Recommender system; Data science; Human–computer interaction; Information retrieval; Software engineering; Data mining; Artificial intelligence; Programming language; Image (mathematics)","score_opus":0.04160546049285644,"score_gpt":0.3472158973043576,"score_spread":0.30561043681150113,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2727610827","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9158277,0.00019746747,0.08264931,0.00041572424,0.00045646186,0.00017179058,0.000002502774,0.00026498197,0.000014050886],"genre_scores_gemma":[0.99686337,0.00004375012,0.0029409383,0.000017638253,0.000060749317,0.000029792176,0.0000023964901,0.000019592839,0.000021783417],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9989074,0.000028136905,0.00019466193,0.00037431341,0.00018200159,0.0003134792],"domain_scores_gemma":[0.99857455,0.00087927,0.00005203983,0.0002899754,0.000039652215,0.00016449725],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00038354762,0.000188522,0.00019802562,0.000247558,0.00013147315,0.0005107099,0.00034367127,0.00008993218,0.0000046626574],"category_scores_gemma":[0.0024556497,0.00016768994,0.000026127796,0.0001518232,0.000012982067,0.00089150696,0.00020581672,0.00033177645,0.000011409662],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000044097073,0.00008675684,0.8681904,0.00028750752,0.00006502094,0.00007813119,0.0015487624,0.071137615,0.0005472093,0.0001828166,0.00045806135,0.057373617],"study_design_scores_gemma":[0.00027400002,0.00007605549,0.799616,0.00014219429,0.0000016081186,0.00003030516,0.000008387822,0.19861881,0.00019686876,0.000005863926,0.0008224542,0.00020746094],"about_ca_topic_score_codex":0.000058801405,"about_ca_topic_score_gemma":0.0000028599409,"teacher_disagreement_score":0.1274812,"about_ca_system_score_codex":0.00019747652,"about_ca_system_score_gemma":0.000028536026,"threshold_uncertainty_score":0.6838197},"labels":[],"label_agreement":null},{"id":"W2738381526","doi":"10.1007/s10664-017-9533-1","title":"EnTagRec ++: An enhanced tag recommendation system for software information sites","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":81,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Ask price; Computer science; Set (abstract data type); Software; Recall; Precision and recall; Information retrieval; World Wide Web; Operating system","score_opus":0.03528399466237,"score_gpt":0.3128422166076763,"score_spread":0.2775582219453063,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2738381526","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08327109,0.000020718393,0.91234326,0.0002486821,0.0011672471,0.00047537053,0.000029635241,0.0024275135,0.000016490905],"genre_scores_gemma":[0.73108095,0.0000030275091,0.2681233,0.000069486196,0.00028496142,0.00023272497,0.00013173382,0.000040722596,0.000033059598],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979745,0.000029033628,0.00044979513,0.00047600127,0.00041507854,0.0006556055],"domain_scores_gemma":[0.99692136,0.001081145,0.00018585841,0.0012157037,0.0002780613,0.00031786502],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0006107065,0.00031386787,0.00031120898,0.00026069852,0.0005628496,0.0010951163,0.0016238932,0.00018947064,0.00001638814],"category_scores_gemma":[0.0069693318,0.00032582815,0.00012902553,0.0002216891,0.000026409654,0.004242302,0.00041442865,0.000303878,0.000089072266],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00015888653,0.0003991884,0.27490842,0.0038345903,0.00039859564,0.00003780098,0.0060339966,0.08503956,0.0036142478,0.0029935709,0.011393707,0.6111874],"study_design_scores_gemma":[0.0036827927,0.00089311553,0.40410197,0.0005587573,0.000049292034,0.00007787126,0.00012038978,0.4874371,0.034896724,0.00029877338,0.06505004,0.0028331894],"about_ca_topic_score_codex":0.000021087499,"about_ca_topic_score_gemma":0.0000030728663,"teacher_disagreement_score":0.64780986,"about_ca_system_score_codex":0.00039029418,"about_ca_system_score_gemma":0.000078874655,"threshold_uncertainty_score":0.9999418},"labels":[],"label_agreement":null},{"id":"W2755579216","doi":"10.1007/s10664-017-9547-8","title":"Inference of development activities from interaction with uninstrumented applications","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Documentation; Software engineering; Software development; Generalizability theory; Software; Event (particle physics); Set (abstract data type); Data science; Human–computer interaction; Programming language","score_opus":0.03065809452857621,"score_gpt":0.3057166064988471,"score_spread":0.27505851197027087,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2755579216","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.35861593,0.000018129778,0.64083177,0.000046946734,0.00008478012,0.000098786506,0.0000025182069,0.00028509076,0.000016057771],"genre_scores_gemma":[0.82002133,0.0000026724229,0.17979911,0.00000754813,0.00003678139,0.00008705775,0.0000058063965,0.000014930165,0.000024743813],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99888897,0.000009758797,0.00019820772,0.00030678115,0.00034433586,0.00025193844],"domain_scores_gemma":[0.9980744,0.00082076184,0.000103299964,0.0008147003,0.00007428827,0.000112532456],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00009704188,0.0001605831,0.0001854503,0.00012390397,0.00016814636,0.00018018096,0.0009873809,0.000059363763,0.000012321749],"category_scores_gemma":[0.00063578325,0.00014681932,0.000031319632,0.00016692189,0.000043886903,0.0006407005,0.00037267496,0.00023851602,0.000013135055],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018034107,0.00012457279,0.9194199,0.00010538942,0.00015838892,0.000014237557,0.00107378,0.014265409,0.0011135379,0.00039545732,0.00007226006,0.06323904],"study_design_scores_gemma":[0.00042237653,0.00007219061,0.9419943,0.00022701523,0.000007431962,0.0000062320123,0.000020742014,0.012348612,0.031298988,0.000058081067,0.013133704,0.00041029314],"about_ca_topic_score_codex":0.000049650393,"about_ca_topic_score_gemma":0.000006499766,"teacher_disagreement_score":0.46140543,"about_ca_system_score_codex":0.00011913314,"about_ca_system_score_gemma":0.00012136675,"threshold_uncertainty_score":0.59871185},"labels":[],"label_agreement":null},{"id":"W2757905449","doi":"10.1007/s10664-017-9545-x","title":"An exploratory study on assessing the energy impact of logging on Android applications","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Green IT and Sustainability","field":"Engineering","cited_by":49,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; University of Alberta","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada; Alberta Innovates - Technology Futures; Johns Hopkins University","keywords":"Android (operating system); Logging; Computer science; Exploratory research; Operating system; Forestry; Geography","score_opus":0.027996351793043654,"score_gpt":0.33390954012214097,"score_spread":0.3059131883290973,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2757905449","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9006576,0.00004614396,0.0984735,0.000017998595,0.00011134825,0.0002114987,0.000005438925,0.00037865347,0.00009785537],"genre_scores_gemma":[0.99947,0.0000021520943,0.00019759077,0.00001318096,0.00015086624,0.00010781488,0.000003819674,0.00004931438,0.0000052395662],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99896985,0.00003682205,0.00023664245,0.00023591932,0.00022194786,0.00029882023],"domain_scores_gemma":[0.99843955,0.00025861466,0.00005391705,0.0010734025,0.000050666116,0.0001238716],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00027856344,0.00022724127,0.00024301681,0.00009054694,0.00029416374,0.0001439982,0.00046743354,0.00007484163,0.0000090665635],"category_scores_gemma":[0.00020074274,0.00016821161,0.000117709096,0.00010974084,0.00004149809,0.00027248455,0.000052302377,0.0002776688,0.0000039002252],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009612633,0.0003379398,0.4270007,0.0000601557,0.00010479574,0.000013599565,0.0012165462,0.5619192,0.0005448166,0.000058961155,0.00017817246,0.008555499],"study_design_scores_gemma":[0.00032655307,0.00035256005,0.96003157,0.000035027122,0.000022804561,0.0000018328491,0.00046860598,0.036955014,0.0006742689,0.000109306595,0.00069563795,0.000326792],"about_ca_topic_score_codex":0.00002813277,"about_ca_topic_score_gemma":0.000005949967,"teacher_disagreement_score":0.5330309,"about_ca_system_score_codex":0.0002030229,"about_ca_system_score_gemma":0.000031390708,"threshold_uncertainty_score":0.685947},"labels":[],"label_agreement":null},{"id":"W2763684087","doi":"10.1007/s10664-017-9551-z","title":"Analyzing a decade of Linux system calls","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":31,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Linux kernel; Computer science; Operating system; System call; sysfs; Configfs; Kernel (algebra); GNU/Linux; Set (abstract data type); Source lines of code; Application programming interface; Software engineering; Programming language; Software","score_opus":0.03008028736372726,"score_gpt":0.30459034001024504,"score_spread":0.2745100526465178,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2763684087","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2011702,0.00023953254,0.79688865,0.00011899277,0.0004895875,0.00011470998,0.0000032494804,0.0009416571,0.000033444332],"genre_scores_gemma":[0.88699454,0.0000050314056,0.11274908,0.000009227003,0.00014451821,0.000020553029,0.000001025048,0.000034263106,0.0000417415],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99802285,0.000024060855,0.00039111727,0.00047538817,0.00050906296,0.00057750766],"domain_scores_gemma":[0.99674386,0.0010566823,0.00013370503,0.0016352849,0.00014850585,0.00028199123],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00051036105,0.00024072985,0.00039425847,0.0002468536,0.00020497791,0.0002730694,0.0023553993,0.00014960191,0.0000058388996],"category_scores_gemma":[0.005709514,0.00023736122,0.0001618747,0.00033559927,0.00004689403,0.0005080546,0.00081244257,0.00039075981,0.00004245695],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00000798194,0.00006971807,0.9518659,0.00074650266,0.00015320907,0.00023755748,0.00047751208,0.032683857,0.0016926163,0.0018366975,0.00072329334,0.0095051415],"study_design_scores_gemma":[0.00056945346,0.00009401853,0.86277187,0.000531327,0.000018064447,0.000064599495,0.00000801959,0.12247686,0.010125611,0.00003673609,0.0026582498,0.0006452103],"about_ca_topic_score_codex":0.000046961857,"about_ca_topic_score_gemma":0.0000011961505,"teacher_disagreement_score":0.68582433,"about_ca_system_score_codex":0.00018379946,"about_ca_system_score_gemma":0.000082350096,"threshold_uncertainty_score":0.967931},"labels":[],"label_agreement":null},{"id":"W2763776913","doi":"10.1007/s10664-017-9553-x","title":"Empirical study on the discrepancy between performance testing results from virtual and physical environments","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Computer science; Empirical research; Human–computer interaction; Statistics; Mathematics","score_opus":0.05186541589034044,"score_gpt":0.2904156404946555,"score_spread":0.23855022460431508,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2763776913","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9726922,0.000012942083,0.025638064,0.0006464387,0.00029843717,0.00034626073,0.000017608687,0.00030295676,0.00004510116],"genre_scores_gemma":[0.9969145,0.0000020451744,0.002395164,0.000075290285,0.00051023933,0.000040313316,0.0000037285238,0.000022983751,0.000035704987],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9977145,0.00007865388,0.0003974784,0.0007924361,0.00057568424,0.0004412191],"domain_scores_gemma":[0.99554974,0.0024089238,0.00016448204,0.0016662169,0.000022344642,0.00018829883],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005726657,0.00032522395,0.00037189393,0.000043901888,0.00083313824,0.00034319918,0.0012850206,0.00009798792,0.0000011977896],"category_scores_gemma":[0.0026310887,0.00020958578,0.00007699371,0.00014617307,0.000103761115,0.0005612588,0.0008974085,0.00052308914,0.00006718292],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011487026,0.00012928792,0.9871262,0.0000063616276,0.00003558017,0.000008731225,0.0019713857,0.0005078947,0.000013274228,0.000004386058,0.0000645593,0.01012088],"study_design_scores_gemma":[0.0005541903,0.0004320145,0.97756565,0.00007456738,0.00001564189,0.0000019806273,0.000039207298,0.02026502,0.00014755651,0.000047201676,0.00057584787,0.00028110907],"about_ca_topic_score_codex":0.000047460046,"about_ca_topic_score_gemma":8.0987445e-7,"teacher_disagreement_score":0.02422234,"about_ca_system_score_codex":0.00008039021,"about_ca_system_score_gemma":0.000028253971,"threshold_uncertainty_score":0.854666},"labels":[],"label_agreement":null},{"id":"W2765212458","doi":"10.1007/s10664-017-9558-5","title":"Understanding the factors for fast answers in technical Q&A websites","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":69,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Incentive; Dimension (graph theory); Order (exchange); Quality (philosophy); Computer science; Question answering; World Wide Web; Internet privacy; Business; Information retrieval; Mathematics","score_opus":0.15669093957551167,"score_gpt":0.32453475118650243,"score_spread":0.16784381161099077,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2765212458","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.052186772,0.00007387649,0.9456147,0.0008499928,0.0006275935,0.00019418812,0.0000032051032,0.0003575274,0.00009216549],"genre_scores_gemma":[0.97874045,0.00000234268,0.020939091,0.000039773124,0.00010337264,0.000037126178,0.0000020938344,0.000019424657,0.00011630275],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9988155,0.00001753534,0.00022140009,0.00032641098,0.0002099618,0.00040923458],"domain_scores_gemma":[0.99822235,0.00088946556,0.000070830196,0.0007051517,0.000018122135,0.00009407848],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00035122098,0.00017476878,0.00020781206,0.000096699165,0.00042964448,0.00039839532,0.0011943768,0.000116517665,0.000003018735],"category_scores_gemma":[0.0013390301,0.00012377826,0.00011069773,0.00015400976,0.000044602275,0.0003148542,0.0002258704,0.00023900841,0.000010758236],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007265584,0.000058125817,0.9771742,0.00007905673,0.000035218298,0.000010746442,0.003048903,0.0073314877,0.0009879448,0.005851203,0.0048515373,0.0005642991],"study_design_scores_gemma":[0.0015312782,0.00020278593,0.821027,0.00057204213,0.000018690484,0.000054186923,0.0004606776,0.08423481,0.000970676,0.0026946813,0.086571634,0.0016615828],"about_ca_topic_score_codex":0.000029990184,"about_ca_topic_score_gemma":0.000049278082,"teacher_disagreement_score":0.9265537,"about_ca_system_score_codex":0.00022838174,"about_ca_system_score_gemma":0.00002688951,"threshold_uncertainty_score":0.50475305},"labels":[],"label_agreement":null},{"id":"W2767795225","doi":"10.1007/s10664-017-9559-4","title":"Are tweets useful in the bug fixing process? An empirical study on Firefox and Chrome","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; World Wide Web; Social media; Process (computing); Empirical research; Security bug; Microblogging; Software; Internet privacy; Computer security","score_opus":0.06260857921942889,"score_gpt":0.36454485204685283,"score_spread":0.30193627282742397,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2767795225","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9867987,0.0000926638,0.010780568,0.0009587762,0.00025634264,0.00048135887,0.0000028742675,0.000618429,0.000010257442],"genre_scores_gemma":[0.9989322,0.000002386904,0.00042732584,0.0002464093,0.00020399629,0.00012434607,0.0000013451636,0.000046963767,0.000015058272],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.997077,0.00010676,0.00033798884,0.0008573271,0.00088366657,0.0007372953],"domain_scores_gemma":[0.99646175,0.0014078404,0.00011764126,0.0016690047,0.00007248557,0.0002712747],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009999641,0.00037387793,0.0003923956,0.000250878,0.00042429828,0.0008705517,0.0023928597,0.0001448561,0.0000056688314],"category_scores_gemma":[0.004522214,0.00028768115,0.000061076295,0.00045976462,0.00006234017,0.0008734513,0.00052965176,0.0008958021,0.000024022047],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00000582687,0.0002361498,0.99361575,0.000047526843,0.000011503029,0.00030035648,0.0027364613,0.0027752256,0.0000061393025,6.069437e-7,0.00012344886,0.00014102194],"study_design_scores_gemma":[0.00050731935,0.00030453765,0.9954608,0.00009613815,0.0000050242757,0.00003124673,0.00011804685,0.0026521236,0.000039225994,0.00005605809,0.00037356443,0.00035593056],"about_ca_topic_score_codex":0.000025151847,"about_ca_topic_score_gemma":0.000016155604,"teacher_disagreement_score":0.012133438,"about_ca_system_score_codex":0.00013146647,"about_ca_system_score_gemma":0.000052250547,"threshold_uncertainty_score":0.99995756},"labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low"},{"model":"gpt","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low"}],"label_agreement":"agree"},{"id":"W2786424616","doi":"10.1007/s10664-018-9595-8","title":"Studying software logging using topic models","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":95,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Logging; Computer science; Software; Software engineering; Operating system; Forestry; Geography","score_opus":0.0777747160628274,"score_gpt":0.31632667881580295,"score_spread":0.23855196275297555,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2786424616","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.26345015,0.00025438535,0.73268384,0.00007650625,0.00087029627,0.0001803098,0.0000018457758,0.0024655533,0.000017141108],"genre_scores_gemma":[0.6252622,0.0000036380486,0.3738894,0.00015090873,0.0005382026,0.000022490141,0.0000018200784,0.000067602305,0.00006372027],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9966944,0.000049304468,0.00044610328,0.0008622523,0.00080082525,0.0011471029],"domain_scores_gemma":[0.99704987,0.001229239,0.00006312567,0.0010519505,0.00021824561,0.00038757786],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005118187,0.00042134177,0.00040098032,0.0004252509,0.00030345394,0.00031547886,0.001483752,0.00019135338,0.00003722779],"category_scores_gemma":[0.002446872,0.0004387199,0.00015246426,0.0012021653,0.000070435526,0.0010697916,0.0009817422,0.00057238614,0.000099645236],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001816253,0.0002471808,0.3876337,0.00043449018,0.00027895125,0.00038795779,0.004282119,0.57202154,0.0012233742,0.0012693872,0.0022381775,0.029964939],"study_design_scores_gemma":[0.00067426916,0.00020977952,0.05016086,0.00025304616,0.00002132933,0.00013867297,0.000021560241,0.938502,0.0017560574,0.001162614,0.0058087194,0.0012911048],"about_ca_topic_score_codex":0.000035299057,"about_ca_topic_score_gemma":0.0000013859794,"teacher_disagreement_score":0.36648044,"about_ca_system_score_codex":0.0003814306,"about_ca_system_score_gemma":0.00013470015,"threshold_uncertainty_score":0.99980646},"labels":[],"label_agreement":null},{"id":"W2788314565","doi":"10.1007/s10664-018-9601-1","title":"App store mining is not enough for app improvement","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Spam and Phishing Detection","field":"Computer Science","cited_by":84,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto; University of Calgary","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada; Alberta Innovates - Technology Futures","keywords":"App store; Computer science; Mobile apps; Smartphone app; World Wide Web","score_opus":0.025491394227604578,"score_gpt":0.2673284083448976,"score_spread":0.241837014117293,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2788314565","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08635495,0.00004617598,0.910548,0.0005946341,0.0013871612,0.00017043593,0.00000454143,0.00086979347,0.000024317838],"genre_scores_gemma":[0.7604935,0.0000014560547,0.23676465,0.001262234,0.0011131671,0.00007261984,0.0000030242986,0.000030945463,0.0002584015],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9987914,0.0000065784834,0.00019612005,0.0004053984,0.00022771992,0.0003728042],"domain_scores_gemma":[0.9991916,0.00020899456,0.00004565119,0.00035072953,0.00008125326,0.00012177931],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00020256784,0.0001680099,0.0001539665,0.00008028747,0.00015612073,0.00012815133,0.00043325473,0.00009730754,0.000017272167],"category_scores_gemma":[0.00026955068,0.00016541028,0.00009136083,0.0002884888,0.000016972765,0.00026398245,0.00014823528,0.00011908699,0.000036721645],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00013105175,0.0003352782,0.01267989,0.0006119062,0.00034205202,0.000028003935,0.027526684,0.005494665,0.023792315,0.001998923,0.12248972,0.8045695],"study_design_scores_gemma":[0.00103528,0.0013164658,0.010227448,0.00010363039,0.000025891564,0.00001796729,0.00003143495,0.27451947,0.08418881,0.00034918595,0.62720037,0.0009840395],"about_ca_topic_score_codex":0.000012235963,"about_ca_topic_score_gemma":0.0000015886175,"teacher_disagreement_score":0.80358547,"about_ca_system_score_codex":0.00009785647,"about_ca_system_score_gemma":0.000028208751,"threshold_uncertainty_score":0.6745236},"labels":[],"label_agreement":null},{"id":"W2790537587","doi":"10.1007/s10664-018-9603-z","title":"Studying and detecting log-related issues","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":64,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Logging; Computer science; Statement (logic); Data science; Software; Scale (ratio); Root (linguistics); Open source; Data mining; Operating system","score_opus":0.019018094875514664,"score_gpt":0.27283382348902824,"score_spread":0.2538157286135136,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2790537587","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6337518,0.0004890784,0.36360052,0.0001910778,0.0006775502,0.00011272102,3.0578173e-7,0.0011459994,0.000030953364],"genre_scores_gemma":[0.9484857,0.000008895233,0.05110797,0.00008595525,0.00022168868,0.000010956312,3.9066904e-7,0.000016656493,0.00006180712],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99867386,0.000029265337,0.0002880434,0.0004225127,0.00021670248,0.00036960168],"domain_scores_gemma":[0.9990953,0.0002574419,0.00004651141,0.00038841646,0.00008094287,0.00013134503],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00042669938,0.00018065838,0.00021659432,0.00009731612,0.00020902994,0.0001152589,0.0003510012,0.0001158037,0.000012224899],"category_scores_gemma":[0.00067405426,0.00015404468,0.0000489221,0.0004904899,0.000052377138,0.00039384209,0.0003171944,0.00021882127,0.000072876435],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008522714,0.00008093378,0.8961291,0.0002545848,0.00011192741,0.000039103656,0.010815682,0.0017450915,0.00050025986,0.00019049265,0.0010338619,0.08909048],"study_design_scores_gemma":[0.0013019105,0.0007841918,0.7239574,0.0004506828,0.000032343978,0.0002792524,0.00018502162,0.22484434,0.006434192,0.0010297607,0.03905753,0.00164336],"about_ca_topic_score_codex":0.000024442099,"about_ca_topic_score_gemma":0.0000015547196,"teacher_disagreement_score":0.3147339,"about_ca_system_score_codex":0.00005345135,"about_ca_system_score_gemma":0.00002055082,"threshold_uncertainty_score":0.628176},"labels":[],"label_agreement":null},{"id":"W2792451387","doi":"10.1007/s10664-018-9600-2","title":"On the correctness of electronic documents: studying, finding, and localizing inconsistency bugs in PDF readers and files","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Engineering and Physical Sciences Research Council; Natural Sciences and Engineering Research Council of Canada; Microsoft Research","keywords":"Correctness; Computer science; Information retrieval; World Wide Web; Programming language","score_opus":0.015947982980012043,"score_gpt":0.2682681163052177,"score_spread":0.25232013332520564,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2792451387","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3614558,0.00014269722,0.63782024,0.0001176324,0.000083574836,0.000138945,5.452549e-7,0.0002098651,0.00003066506],"genre_scores_gemma":[0.98637545,0.000024330193,0.013359348,0.00014982486,0.000020703332,0.000030886058,3.4647246e-7,0.000011876578,0.000027236818],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9990344,0.000034410034,0.00020819918,0.0002916858,0.00015722607,0.00027405674],"domain_scores_gemma":[0.99894,0.0006779443,0.00006143233,0.00023512502,0.00003803401,0.000047457797],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00025943524,0.00014140084,0.00016617688,0.00015548534,0.000077866476,0.000042156487,0.0002439828,0.000058700403,0.000015520593],"category_scores_gemma":[0.00078338385,0.000114307935,0.000022210692,0.00040597207,0.00008544721,0.00021083544,0.00019583669,0.00023612472,0.0000039063434],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00025677745,0.0009554581,0.65984035,0.001264288,0.0004814835,0.00016520395,0.035194717,0.01438652,0.005560522,0.052011855,0.011343945,0.21853885],"study_design_scores_gemma":[0.004529833,0.007117333,0.40377453,0.004482636,0.000089398345,0.00043361288,0.0018007318,0.30602947,0.14732346,0.08212742,0.03755562,0.0047359634],"about_ca_topic_score_codex":0.00002646801,"about_ca_topic_score_gemma":0.00001969385,"teacher_disagreement_score":0.62491965,"about_ca_system_score_codex":0.000085421394,"about_ca_system_score_gemma":0.000022686736,"threshold_uncertainty_score":0.46613422},"labels":[],"label_agreement":null},{"id":"W2793175118","doi":"10.1007/s10664-018-9604-y","title":"Studying the consistency of star ratings and the complaints in 1 & 2-star user reviews for top free cross-platform Android and iOS apps","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Android (operating system); Computer science; Mobile apps; World Wide Web; Revenue; Star (game theory); App store; Download; User experience design; Consistency (knowledge bases); Human–computer interaction; Operating system; Artificial intelligence","score_opus":0.05466891975716319,"score_gpt":0.32305640697843746,"score_spread":0.26838748722127426,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2793175118","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6671609,0.0025257096,0.3287059,0.00034046682,0.00018382914,0.00093480916,0.000008436683,0.00013422589,0.0000057073257],"genre_scores_gemma":[0.87548465,0.00014735963,0.1236978,0.00021876543,0.00015460944,0.00020606485,0.0000027460837,0.00003822244,0.000049784783],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99819803,0.000058709313,0.0005300116,0.00041665003,0.00033338054,0.00046322803],"domain_scores_gemma":[0.99364233,0.0052459016,0.00009252973,0.00076308206,0.0001457359,0.00011044237],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0021029175,0.00023775328,0.00045627556,0.00011729375,0.00023147542,0.00019064212,0.00087319,0.00008292924,0.000006547239],"category_scores_gemma":[0.0098717,0.00014960703,0.00007735885,0.0005093376,0.00033567756,0.0002857153,0.0006853804,0.00034233846,0.0000036842446],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00031817492,0.00021966037,0.9233748,0.002445965,0.00027956654,0.00002443579,0.028993186,0.0029894419,0.00076391996,0.0065087164,0.006846989,0.027235165],"study_design_scores_gemma":[0.010077146,0.0006253661,0.735698,0.0013002312,0.00006006762,0.00016137015,0.00018506072,0.13059309,0.0009510088,0.002911281,0.11595922,0.0014781557],"about_ca_topic_score_codex":0.000048463284,"about_ca_topic_score_gemma":0.000035214834,"teacher_disagreement_score":0.20832373,"about_ca_system_score_codex":0.00005603553,"about_ca_system_score_gemma":0.000039093622,"threshold_uncertainty_score":0.9984686},"labels":[],"label_agreement":null},{"id":"W2793227253","doi":"10.1007/s10664-017-9592-3","title":"ProMeTA: a taxonomy for program metamodels in program reverse engineering","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Japan Society for the Promotion of Science","keywords":"Metamodeling; Taxonomy (biology); Computer science; Reuse; Software engineering; Program comprehension; Systems engineering; Orthogonality; Artificial intelligence; Engineering; Programming language; Software; Software system","score_opus":0.0615456604311363,"score_gpt":0.3297100329207495,"score_spread":0.2681643724896132,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2793227253","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.060714662,0.00022761556,0.92970574,0.00021566961,0.00067078887,0.0033833487,0.0000061074893,0.0050603827,0.000015685817],"genre_scores_gemma":[0.21023217,0.000004014928,0.782805,0.000056519002,0.00038660268,0.006340308,0.000008571548,0.00009507272,0.00007170754],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99647933,0.000029375975,0.0005775201,0.0009753951,0.00054267247,0.0013957135],"domain_scores_gemma":[0.99717396,0.001223357,0.000064567575,0.00089739286,0.0002412092,0.00039953046],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008171141,0.0004889889,0.0005491016,0.00068973453,0.0000893413,0.00029893167,0.0013230284,0.00023316349,0.000014186914],"category_scores_gemma":[0.0043379785,0.0004992426,0.00022831299,0.0022259285,0.000054290176,0.0007387496,0.00045094136,0.0005556318,0.000049229067],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00017504193,0.00421778,0.20692949,0.0035107478,0.0010046894,0.00048794172,0.0031232599,0.18003035,0.0018174254,0.0045965337,0.014209111,0.57989764],"study_design_scores_gemma":[0.0012826433,0.0010804357,0.023420863,0.00022797353,0.000021028347,0.000042641273,0.0000052740943,0.7049949,0.0017166379,0.0001226825,0.26592433,0.0011606001],"about_ca_topic_score_codex":0.000015199816,"about_ca_topic_score_gemma":0.0000037357106,"teacher_disagreement_score":0.578737,"about_ca_system_score_codex":0.00033828328,"about_ca_system_score_gemma":0.00014239909,"threshold_uncertainty_score":0.9997459},"labels":[],"label_agreement":null},{"id":"W2796115009","doi":"10.1007/s10664-018-9617-6","title":"Studying the consistency of star ratings and reviews of popular free hybrid Android and iOS apps","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Mobile and Web Applications","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"JavaScript; Android (operating system); Computer science; Codebase; Mobile apps; World Wide Web; App store; Web application; Source code; Operating system","score_opus":0.02501936444025835,"score_gpt":0.2575023618221422,"score_spread":0.23248299738188385,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2796115009","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49356547,0.00286974,0.5028134,0.00028220026,0.000052796076,0.0003236645,0.000004839434,0.00006291327,0.000024953524],"genre_scores_gemma":[0.9288677,0.00016053024,0.07074834,0.00011434121,0.00005236881,0.000037428454,8.219793e-7,0.000008155211,0.00001029197],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9992187,0.000025697625,0.00030024935,0.00020334199,0.00012422595,0.0001277705],"domain_scores_gemma":[0.9991544,0.00017690698,0.00009014024,0.00046435968,0.000056806693,0.00005736808],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00031989277,0.00009793554,0.00022641825,0.00003674128,0.00008107123,0.000023651239,0.00032814514,0.00002302396,0.0000041170097],"category_scores_gemma":[0.00046047568,0.00007046666,0.000034289984,0.0001998272,0.00008289429,0.0000957407,0.0002835197,0.000084551146,0.0000012754414],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003289525,0.00067596114,0.52226704,0.0032032675,0.0004202452,0.000029522416,0.02319421,0.00056832423,0.032038875,0.033955842,0.025038658,0.35857517],"study_design_scores_gemma":[0.004042016,0.0022525056,0.3786326,0.0018311233,0.00033310583,0.0003889493,0.00043692673,0.18070851,0.040902905,0.01225039,0.37548164,0.0027393235],"about_ca_topic_score_codex":0.000013541236,"about_ca_topic_score_gemma":0.0000019895333,"teacher_disagreement_score":0.43530226,"about_ca_system_score_codex":0.0000069495045,"about_ca_system_score_gemma":0.00001430343,"threshold_uncertainty_score":0.28735468},"labels":[],"label_agreement":null},{"id":"W2796164377","doi":"10.1007/s10664-018-9615-8","title":"An empirical study of Android Wear user complaints","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Wearable computer; Android (operating system); Wearable technology; Internet privacy; Mobile device; Computer science; Categorization; Empirical research; Complaint; World Wide Web; Human–computer interaction; Artificial intelligence","score_opus":0.020782005223002522,"score_gpt":0.3194921780268243,"score_spread":0.2987101728038218,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2796164377","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4356964,0.000008029429,0.5628458,0.000025097119,0.00017486652,0.00014859531,0.0000010851811,0.001089115,0.0000109894],"genre_scores_gemma":[0.81230843,7.302747e-7,0.18739556,0.0001092034,0.0001233406,0.000022233971,8.8548825e-7,0.000024926807,0.000014673565],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983179,0.00006165236,0.0003686321,0.00051211956,0.0003934809,0.00034620045],"domain_scores_gemma":[0.9985562,0.0001888634,0.00007975931,0.0008115786,0.00016626497,0.00019735083],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00022402206,0.00021748127,0.00029938505,0.00020419116,0.00010445073,0.000054740023,0.00083079067,0.000100277786,0.000028413238],"category_scores_gemma":[0.00029358306,0.00021082663,0.00005841868,0.00073963427,0.000052905958,0.0005401832,0.00028460607,0.00024818815,0.00002866187],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009393864,0.003325563,0.93521893,0.00010476589,0.00016132659,0.00016782578,0.012413535,0.0103554595,0.0071223504,0.00030082525,0.0037991332,0.026936334],"study_design_scores_gemma":[0.0020391163,0.008209524,0.79607373,0.0001079369,0.00003301326,0.00015197134,0.00022136874,0.098947585,0.060768273,0.00092014694,0.030841064,0.0016862331],"about_ca_topic_score_codex":0.000013320757,"about_ca_topic_score_gemma":0.00000770284,"teacher_disagreement_score":0.37661207,"about_ca_system_score_codex":0.00007694676,"about_ca_system_score_gemma":0.000024489655,"threshold_uncertainty_score":0.8597261},"labels":[],"label_agreement":null},{"id":"W2801496046","doi":"10.1007/s10664-018-9614-9","title":"Investigating whether and how software developers understand open source software licensing","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Systems, Applications & Products in Data Processing (Canada); University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"License; Context (archaeology); Open source software; Computer science; World Wide Web; Open source; Software; Source code; Software engineering; Software development; Data science; Knowledge management","score_opus":0.04796779462994141,"score_gpt":0.28552701820068405,"score_spread":0.23755922357074263,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2801496046","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09649286,0.00014088966,0.8975741,0.00277383,0.00044572062,0.00044628204,0.0000066094885,0.0020636285,0.000056079763],"genre_scores_gemma":[0.16921708,0.000003502408,0.82758653,0.0021933478,0.0003013122,0.000028929022,0.000009927056,0.00013210437,0.0005272765],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99676055,0.00009902815,0.0004576874,0.0011266465,0.0006002111,0.0009558786],"domain_scores_gemma":[0.9968863,0.0010614073,0.00019814335,0.00094603136,0.00042914055,0.00047897553],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00075442507,0.00055686437,0.00052307214,0.0003020545,0.0008196049,0.0020310734,0.0018696598,0.00027834377,0.000036481142],"category_scores_gemma":[0.005771109,0.00056701864,0.000083838015,0.0019463723,0.00029372037,0.0019884945,0.0024877796,0.00062502525,0.000065038024],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025423988,0.00022444461,0.71374357,0.0004938067,0.00057759054,0.00011037246,0.03701242,0.0061763646,0.0011216064,0.0033149666,0.043421045,0.19377838],"study_design_scores_gemma":[0.0068929936,0.0013629302,0.25538173,0.0035556986,0.00022446596,0.0011969381,0.0028654744,0.082366146,0.006336329,0.008742758,0.61966556,0.011408985],"about_ca_topic_score_codex":0.00004892262,"about_ca_topic_score_gemma":0.000025680702,"teacher_disagreement_score":0.57624453,"about_ca_system_score_codex":0.000295201,"about_ca_system_score_gemma":0.00028492906,"threshold_uncertainty_score":0.99967813},"labels":[],"label_agreement":null},{"id":"W2805828232","doi":"10.1007/s10664-018-9629-2","title":"What can Android mobile app developers do about the energy consumption of machine learning?","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Green IT and Sustainability","field":"Engineering","cited_by":49,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Alberta","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Machine learning; Artificial intelligence; Android (operating system); Energy consumption; Implementation; Facial recognition system; Multimedia; Feature extraction; Software engineering; Operating system; Engineering","score_opus":0.010641704860067385,"score_gpt":0.23646973270715824,"score_spread":0.22582802784709086,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2805828232","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9418951,0.0032487144,0.05326625,0.000035666246,0.00066220685,0.0001550472,0.0000069656994,0.0006849961,0.0000450201],"genre_scores_gemma":[0.99854565,0.00025285126,0.0007947812,0.00003890895,0.0001408936,0.000048998878,0.00001393212,0.00004919068,0.000114779075],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99889004,0.000030962576,0.00029476426,0.00020796475,0.00020810984,0.00036813074],"domain_scores_gemma":[0.9992586,0.0002222483,0.00003126569,0.00026849317,0.000107974214,0.000111455214],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00021724952,0.00022214459,0.00024508746,0.00008174524,0.000100892,0.00006526616,0.00021476703,0.00012123391,0.00015477541],"category_scores_gemma":[0.00022796127,0.00017673508,0.00008524151,0.00029492998,0.0001086219,0.00016281003,0.00006975102,0.0002798371,0.000011899829],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000049510367,0.000081021026,0.5245548,0.0007695589,0.00027140783,0.00002267845,0.008665725,0.39253944,0.0013076308,0.0001618655,0.0018874191,0.06968897],"study_design_scores_gemma":[0.001531358,0.00054727786,0.27467978,0.00040967655,0.00011541081,0.00007373177,0.0011258379,0.30067703,0.01825181,0.00019235133,0.4004335,0.0019622326],"about_ca_topic_score_codex":0.000044483306,"about_ca_topic_score_gemma":0.000038958467,"teacher_disagreement_score":0.3985461,"about_ca_system_score_codex":0.00014283664,"about_ca_system_score_gemma":0.00002787871,"threshold_uncertainty_score":0.72070473},"labels":[],"label_agreement":null},{"id":"W2810627707","doi":"10.1007/s10664-018-9634-5","title":"How do developers utilize source code from stack overflow?","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":129,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Source code; Copying; Codebase; Code reuse; Reuse; Code review; Programming language; Code (set theory); Stack (abstract data type); Software engineering; World Wide Web; Operating system; Static program analysis; Software; Software development; Engineering; Set (abstract data type)","score_opus":0.03258012208955109,"score_gpt":0.27706487837659804,"score_spread":0.24448475628704697,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2810627707","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.23133625,0.00031780673,0.76457226,0.00054820365,0.0008406875,0.00014496375,0.000013320156,0.0022068263,0.000019683055],"genre_scores_gemma":[0.7509447,0.000009914106,0.24738945,0.0002378718,0.00072349765,0.000035826346,0.000009809204,0.00009447217,0.00055446423],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.996817,0.00004482729,0.0003032953,0.0009373755,0.00088383164,0.0010136539],"domain_scores_gemma":[0.9960664,0.0020173742,0.00005008433,0.00116927,0.0002096768,0.00048722824],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003199541,0.0004252122,0.00037269207,0.00024091094,0.00016692663,0.00070499576,0.0017177847,0.00022448895,0.000086806744],"category_scores_gemma":[0.005428929,0.00042155126,0.00013706686,0.001171028,0.0000905628,0.00068468787,0.000814611,0.00056946074,0.00029806924],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007454663,0.00028128174,0.81464326,0.00021782018,0.0005738541,0.00037871193,0.007042255,0.02010233,0.0025289925,0.0007146338,0.055050023,0.09839229],"study_design_scores_gemma":[0.001216825,0.0002488083,0.37619677,0.00020126061,0.00001965729,0.00004228227,0.000045782264,0.09937797,0.008040308,0.00023285522,0.5126422,0.00173531],"about_ca_topic_score_codex":0.000033431144,"about_ca_topic_score_gemma":0.000004781396,"teacher_disagreement_score":0.51960844,"about_ca_system_score_codex":0.00029669012,"about_ca_system_score_gemma":0.00014621117,"threshold_uncertainty_score":0.99982363},"labels":[],"label_agreement":null},{"id":"W2883022143","doi":"10.1007/s10664-018-9640-7","title":"GreenScaler: training software energy models with automatic test generation","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Green IT and Sustainability","field":"Engineering","cited_by":57,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Energy consumption; Software; Computer science; Heuristics; Heuristic; Energy (signal processing); Reliability engineering; Embedded system; Software engineering; Engineering; Artificial intelligence; Operating system","score_opus":0.027419903582851166,"score_gpt":0.22244643836317063,"score_spread":0.19502653478031945,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2883022143","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.31540826,0.00012411915,0.68167436,0.000035704434,0.00022447703,0.000107067965,0.000008358882,0.002320431,0.000097207405],"genre_scores_gemma":[0.9481859,0.000004386654,0.050749768,0.0000894628,0.0006684251,0.00006478862,0.000027680644,0.00011982098,0.00008973913],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9982557,0.00001742287,0.0003622206,0.0003738427,0.00033134874,0.00065943657],"domain_scores_gemma":[0.99890006,0.00026672494,0.000030077104,0.00040031408,0.0001475091,0.00025530986],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00015232484,0.0003857305,0.0003568584,0.00014684249,0.00014541505,0.0000764388,0.00021284421,0.00018595094,0.000086408174],"category_scores_gemma":[0.0003486495,0.0003544639,0.00009083945,0.000487306,0.00007405629,0.00037146802,0.00004690324,0.00025275277,0.000019022793],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008607566,0.00009717884,0.05407498,0.00041001785,0.00016998622,0.0000847195,0.003409525,0.8292103,0.00061422744,0.00017287926,0.004882211,0.10686538],"study_design_scores_gemma":[0.000327124,0.00017057924,0.0113096405,0.0000627864,0.000029485975,0.000036417267,0.00004398392,0.98298347,0.0006205423,0.00022054165,0.0036066936,0.00058872683],"about_ca_topic_score_codex":0.000023177858,"about_ca_topic_score_gemma":0.00005999627,"teacher_disagreement_score":0.6327777,"about_ca_system_score_codex":0.00023680583,"about_ca_system_score_gemma":0.00005605214,"threshold_uncertainty_score":0.99989074},"labels":[],"label_agreement":null},{"id":"W2886347592","doi":"10.1007/s10664-018-9636-3","title":"An empirical study on the issue reports with questions raised during the issue resolving process","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Beihang University","keywords":"Computer science; Process (computing); Data science; Eclipse; Empirical research","score_opus":0.020315733376175562,"score_gpt":0.3259201812352616,"score_spread":0.3056044478590861,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2886347592","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.78446805,0.00005433773,0.2097725,0.00243463,0.0004819629,0.00081624975,0.0000015209741,0.0019339715,0.00003677963],"genre_scores_gemma":[0.99263924,0.0000011241353,0.005403257,0.00041951056,0.00095677376,0.00030556248,0.0000015485895,0.000093121576,0.00017983401],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99606216,0.00019329181,0.00049033645,0.0010305614,0.0012970266,0.0009266049],"domain_scores_gemma":[0.994655,0.0022942524,0.00010215922,0.0022683204,0.00033251935,0.00034776275],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012462725,0.00046531716,0.00034101246,0.00023382231,0.0008229113,0.0005514352,0.0018191771,0.00012982603,0.00008770775],"category_scores_gemma":[0.0046003996,0.0002730076,0.00008283887,0.0019148729,0.00015141041,0.0005372097,0.00037128362,0.0010245931,0.000107831605],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006344263,0.0007507473,0.9272142,0.00009521054,0.00017395652,0.00087613286,0.015719945,0.049404882,0.00028672582,0.000034771652,0.0046523423,0.00072766683],"study_design_scores_gemma":[0.00045749484,0.0013073367,0.96215516,0.00016035877,0.000020643563,0.00029635528,0.0003117158,0.021471547,0.0017185997,0.000048191003,0.011296851,0.0007557692],"about_ca_topic_score_codex":0.000026854457,"about_ca_topic_score_gemma":0.000008205568,"teacher_disagreement_score":0.20817122,"about_ca_system_score_codex":0.00020860501,"about_ca_system_score_gemma":0.00015654176,"threshold_uncertainty_score":0.9999722},"labels":[],"label_agreement":null},{"id":"W2888049099","doi":"10.1007/s10664-018-9643-4","title":"Preventing duplicate bug reports by continuously querying bug reports","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":46,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"Mitacs; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Data deduplication; Software bug; BitTorrent tracker; Search engine indexing; Security bug; Information retrieval; Software; Database; Artificial intelligence; Cloud computing; Programming language","score_opus":0.014794330250701704,"score_gpt":0.2740173627620474,"score_spread":0.2592230325113457,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2888049099","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.34296298,0.00048122572,0.6514064,0.00026357887,0.0013186573,0.00031753536,0.0000019672395,0.003190319,0.00005732326],"genre_scores_gemma":[0.9274199,0.000007716735,0.07069585,0.00018423526,0.0006259844,0.00011976999,0.00001403911,0.00012569728,0.000806783],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9952132,0.00005393223,0.0010096276,0.001389871,0.0009896115,0.0013437499],"domain_scores_gemma":[0.99605244,0.0011135146,0.00026174646,0.0017323113,0.00030551018,0.0005344505],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0013694625,0.00050769915,0.0005380542,0.0002911093,0.0002703097,0.0004254489,0.0008894708,0.00025225658,0.000055784363],"category_scores_gemma":[0.007395887,0.0005329254,0.00021458947,0.0012914675,0.00008458356,0.0007414448,0.00089722604,0.00070333545,0.00009016115],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011006779,0.00024099526,0.89390934,0.0003052698,0.00022380342,0.0038870443,0.0012349783,0.0043831556,0.010490741,0.000058347716,0.07383335,0.011421986],"study_design_scores_gemma":[0.0011723576,0.00075842167,0.3292599,0.001191394,0.00008515326,0.00795668,0.000028422062,0.053207703,0.070612915,0.0011312262,0.5293427,0.0052531185],"about_ca_topic_score_codex":0.000068204004,"about_ca_topic_score_gemma":0.000002100943,"teacher_disagreement_score":0.584457,"about_ca_system_score_codex":0.00026962024,"about_ca_system_score_gemma":0.00012785856,"threshold_uncertainty_score":0.9997122},"labels":[],"label_agreement":null},{"id":"W2897662303","doi":"10.1007/s10664-018-9656-z","title":"High-level software requirements and iteration changes: a predictive model","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Systems, Applications & Products in Data Processing (Canada); University of Victoria","funders":"","keywords":"Computer science; Software engineering; Reliability engineering; Engineering","score_opus":0.05761095084229382,"score_gpt":0.3005239212894088,"score_spread":0.242912970447115,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2897662303","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.12282977,0.00012401956,0.8743864,0.00030857974,0.000476547,0.00024754342,0.000021130269,0.0016018368,0.000004157588],"genre_scores_gemma":[0.5736981,0.000011456675,0.42529735,0.0002498865,0.00042829502,0.00009542896,0.0000127131025,0.00005437842,0.00015240398],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99760664,0.000029389279,0.00027958147,0.0007404235,0.00065534184,0.00068863423],"domain_scores_gemma":[0.9981291,0.00062445475,0.000051801206,0.00062338414,0.00023893628,0.0003322934],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003514484,0.0003347877,0.00028327806,0.0003062292,0.000183746,0.00024499354,0.00066545134,0.00017556656,0.000015316326],"category_scores_gemma":[0.002734497,0.00033817213,0.000050283783,0.00068243546,0.000071136004,0.00071295706,0.00060986483,0.00033830432,0.000046558354],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022639478,0.0007723271,0.43420875,0.0013579204,0.0009105712,0.00032867055,0.021248741,0.37223583,0.007162246,0.0040280176,0.03394394,0.12357657],"study_design_scores_gemma":[0.0008669805,0.00057326746,0.091463834,0.00021393732,0.00001795336,0.00004087868,0.000006874768,0.90018445,0.0031558345,0.00076132506,0.00187874,0.0008359416],"about_ca_topic_score_codex":0.000014047958,"about_ca_topic_score_gemma":0.000004597151,"teacher_disagreement_score":0.5279486,"about_ca_system_score_codex":0.00021765372,"about_ca_system_score_gemma":0.00007893251,"threshold_uncertainty_score":0.999907},"labels":[],"label_agreement":null},{"id":"W2899817180","doi":"10.1007/s10664-018-9665-y","title":"An empirical study of patch uplift in rapid release development pipelines","year":2018,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Geology; Channel (broadcasting); Lead (geology); Computer science; Paleontology; Telecommunications","score_opus":0.03303537718028712,"score_gpt":0.3249213934817224,"score_spread":0.2918860163014353,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2899817180","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.63532466,0.00006363533,0.3634701,0.000053104097,0.00026090245,0.00024329696,7.812225e-7,0.0005802842,0.0000032521673],"genre_scores_gemma":[0.9171636,0.000002396662,0.08244251,0.000063082676,0.00019350828,0.00006791969,0.0000032981297,0.00004806725,0.00001559709],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9968054,0.00009679923,0.00073558267,0.0007891082,0.00083032483,0.0007427476],"domain_scores_gemma":[0.99730825,0.0009847156,0.000067381035,0.0010263502,0.0002370926,0.00037620732],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007903177,0.00034782142,0.00045792945,0.0005577933,0.00008277837,0.00008621943,0.0013824899,0.00015523107,0.000035599285],"category_scores_gemma":[0.0023087736,0.0003384541,0.000062737185,0.0017924495,0.000046363304,0.0004381347,0.00046861297,0.0004669337,0.000054710614],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018577543,0.00089522783,0.9739526,0.00005134737,0.000026659167,0.000078975434,0.0051173647,0.0072645308,0.00013242864,0.0000043250466,0.0003684842,0.01208949],"study_design_scores_gemma":[0.0009151147,0.00082905334,0.93506384,0.00007244181,0.0000045176794,0.0000120460445,0.00007856073,0.05759485,0.0020699105,0.000014856973,0.0028352765,0.0005095267],"about_ca_topic_score_codex":0.00006956948,"about_ca_topic_score_gemma":0.000040478226,"teacher_disagreement_score":0.28183898,"about_ca_system_score_codex":0.000193946,"about_ca_system_score_gemma":0.00019706614,"threshold_uncertainty_score":0.9999067},"labels":[],"label_agreement":null},{"id":"W2907872047","doi":"10.1007/s10664-018-9677-7","title":"An empirical study of DLL injection bugs in the Firefox ecosystem","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada; Polytechnique Montréal","keywords":"Software bug; Software; Host (biology); Computer science; Operating system; Biology","score_opus":0.013934126707237858,"score_gpt":0.29050260713577125,"score_spread":0.2765684804285334,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2907872047","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.58172154,0.0000122852,0.4172077,0.000035763944,0.00018404357,0.00032843108,7.327533e-7,0.00049717235,0.00001230909],"genre_scores_gemma":[0.986906,8.678081e-7,0.0128547605,0.0000910864,0.000049941656,0.000069836235,0.0000010102874,0.000018193576,0.000008316828],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9984148,0.00013025034,0.0003857413,0.0004169553,0.0003885707,0.00026363987],"domain_scores_gemma":[0.9987004,0.00032934916,0.00008372392,0.00076411647,0.000060187303,0.000062264306],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004739884,0.00017836993,0.000258482,0.0002196207,0.00003974686,0.000052076863,0.0008228009,0.0000971856,0.0000067471847],"category_scores_gemma":[0.00016740459,0.00013874134,0.000056451092,0.0009912221,0.000006775102,0.00052175066,0.00013378127,0.00034754907,0.000015981306],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002336699,0.0010353936,0.8635916,0.000115967174,0.000025256102,0.000038697668,0.009238789,0.11600405,0.0007396837,0.00007645122,0.00022211966,0.008888625],"study_design_scores_gemma":[0.0010011388,0.0032924123,0.8639579,0.00009527574,0.000010340761,0.000101129524,0.0007151019,0.12250262,0.0028396007,0.00036067038,0.0044534607,0.0006703624],"about_ca_topic_score_codex":0.0000328416,"about_ca_topic_score_gemma":0.00003529001,"teacher_disagreement_score":0.40518442,"about_ca_system_score_codex":0.00012827912,"about_ca_system_score_gemma":0.000023654062,"threshold_uncertainty_score":0.56577075},"labels":[],"label_agreement":null},{"id":"W2911789761","doi":"10.1007/s10664-018-9671-0","title":"Automatic query reformulation for code search using crowdsourced knowledge","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":49,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Computer science; Information retrieval; Web search query; Java; Code (set theory); Query expansion; Search engine; Web query classification; XPath; Precision and recall; Natural language; Programming language; World Wide Web; Artificial intelligence; XML; Set (abstract data type)","score_opus":0.0489429157531969,"score_gpt":0.33474163569030263,"score_spread":0.2857987199371057,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2911789761","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46000692,0.0001012681,0.5379918,0.00004589828,0.00037769624,0.00036398403,0.0000033128138,0.0011017441,0.0000073220285],"genre_scores_gemma":[0.7616457,0.0000011984147,0.23783451,0.000033898432,0.00016481704,0.00004921729,0.000008650658,0.00007244794,0.0001895618],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977182,0.000038846352,0.00038096312,0.00058865926,0.0004703883,0.0008029219],"domain_scores_gemma":[0.9964126,0.002342806,0.000043038246,0.0007674495,0.00018661722,0.00024752857],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006851525,0.00028422123,0.00034155019,0.0003806327,0.000111745205,0.00019305928,0.0008666721,0.00018037207,0.000035950045],"category_scores_gemma":[0.0016491957,0.00028355015,0.000160757,0.00092464185,0.000016647295,0.0005499979,0.0003780221,0.00036523116,0.00015224589],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001872656,0.00016572347,0.22027807,0.0015305346,0.00013804635,0.000017587217,0.002220332,0.75005835,0.006509222,0.0008910924,0.0007799364,0.017392395],"study_design_scores_gemma":[0.00046227695,0.00007972159,0.048583258,0.00012039306,0.000005528124,0.00001788959,0.000005259923,0.94574136,0.0016052296,0.00004397348,0.002970816,0.00036428703],"about_ca_topic_score_codex":0.000010504246,"about_ca_topic_score_gemma":7.690562e-7,"teacher_disagreement_score":0.30163875,"about_ca_system_score_codex":0.000502596,"about_ca_system_score_gemma":0.0001638807,"threshold_uncertainty_score":0.9999617},"labels":[],"label_agreement":null},{"id":"W2914035201","doi":"10.1007/s10664-019-09684-y","title":"Towards prioritizing user-related issue reports of mobile applications","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Prioritization; Computer science; Android (operating system); World Wide Web; Empirical research; Mobile apps; Process (computing); Matching (statistics); Data science; Internet privacy; Engineering; Process management","score_opus":0.00985313260491852,"score_gpt":0.2752562160281317,"score_spread":0.26540308342321317,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2914035201","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.26893735,0.0005478863,0.72751224,0.00008653279,0.0006126953,0.00070722535,0.000002942196,0.0015042407,0.00008886349],"genre_scores_gemma":[0.87725246,0.000010650637,0.12183702,0.000032244367,0.00010089242,0.00023072322,0.000008037022,0.000057936988,0.00047005014],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99769497,0.000021531394,0.0005813214,0.0005902066,0.00060039153,0.00051158655],"domain_scores_gemma":[0.99758124,0.00078852935,0.00010398562,0.0011498843,0.00016276244,0.00021361471],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00044027876,0.00023900186,0.00037720625,0.0002682881,0.000044880526,0.000073088435,0.0007561847,0.00016831789,0.00008548388],"category_scores_gemma":[0.0008909609,0.00025108614,0.00013964115,0.0013045563,0.00002392574,0.00036195666,0.00043601464,0.00041217357,0.0001728538],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000072627963,0.00037327377,0.7370014,0.0013011115,0.00025694378,0.00026791368,0.001991295,0.2048502,0.008986604,0.0017510814,0.0026253997,0.040587507],"study_design_scores_gemma":[0.001121822,0.0005406137,0.41967422,0.00048317766,0.000037830985,0.00057536096,0.000045325916,0.05276448,0.01990912,0.00068133435,0.50204915,0.0021175472],"about_ca_topic_score_codex":0.000018319533,"about_ca_topic_score_gemma":1.821358e-7,"teacher_disagreement_score":0.6083151,"about_ca_system_score_codex":0.00014473365,"about_ca_system_score_gemma":0.0001198895,"threshold_uncertainty_score":0.99999416},"labels":[],"label_agreement":null},{"id":"W2914582798","doi":"10.1007/s10664-018-9679-5","title":"The impact of feature reduction techniques on defect prediction models","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Industrial Vision Systems and Defect Detection","field":"Engineering","cited_by":99,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Alberta","funders":"Japan Society for the Promotion of Science; Natural Sciences and Engineering Research Council of Canada","keywords":"Reduction (mathematics); Feature (linguistics); Computer science; Pattern recognition (psychology); Artificial intelligence; Data mining; Mathematics","score_opus":0.015841601350863765,"score_gpt":0.25694013528353327,"score_spread":0.2410985339326695,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2914582798","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95257866,0.00027284113,0.043523204,0.000011911919,0.0013894514,0.000494054,0.000021885382,0.0012150686,0.00049292645],"genre_scores_gemma":[0.9991565,0.000023094497,0.00032150198,0.0000017165326,0.00031744785,0.00003017338,0.0000066963744,0.000044764995,0.00009812917],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9991366,0.000022960521,0.00023598976,0.00016034066,0.00021812269,0.00022594113],"domain_scores_gemma":[0.99941957,0.0001565247,0.000040577983,0.00027239547,0.000051054943,0.000059870697],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023745756,0.00018405882,0.00020371628,0.00011807307,0.000059306734,0.000034111614,0.0000928402,0.00023321922,0.000010044647],"category_scores_gemma":[0.00009053442,0.00012449914,0.0002559493,0.00032686713,0.0000090701915,0.00014846127,0.000015050552,0.0003854626,0.00001858393],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000039505576,0.000011327257,0.0016384523,0.000037212936,0.00008112747,5.741479e-7,0.000073983894,0.97878677,0.006662637,0.000017892593,0.0057075634,0.006942984],"study_design_scores_gemma":[0.0014085592,0.00280703,0.042194027,0.00097309024,0.00007965819,0.00016429796,0.00009634214,0.81451374,0.110881574,0.0004520755,0.025140736,0.0012888818],"about_ca_topic_score_codex":0.000018750909,"about_ca_topic_score_gemma":2.1952773e-7,"teacher_disagreement_score":0.16427301,"about_ca_system_score_codex":0.00023985512,"about_ca_system_score_gemma":0.0000147229775,"threshold_uncertainty_score":0.5076928},"labels":[],"label_agreement":null},{"id":"W2920526824","doi":"10.1007/s10664-019-09695-9","title":"An empirical study of the long duration of continuous integration builds","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":82,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Duration (music); Computer science; Set (abstract data type); Empirical research; Software development; Cache; Software; Software engineering; Process management; Engineering; Statistics; Operating system; Programming language","score_opus":0.016944779324193943,"score_gpt":0.2973705753052279,"score_spread":0.280425795981034,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2920526824","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6099686,0.000024386154,0.38910192,0.00004963697,0.00032198857,0.00032466144,0.0000010655591,0.00020484999,0.0000028997792],"genre_scores_gemma":[0.99100786,7.491404e-7,0.008839797,0.000022457378,0.00005046264,0.000021930176,0.0000022291501,0.000024945388,0.000029572813],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9979908,0.00009961973,0.0005054274,0.0003893855,0.00072001666,0.000294764],"domain_scores_gemma":[0.9977026,0.0008527294,0.00011898172,0.001013509,0.00020919854,0.000102990685],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00047257723,0.00019425109,0.00032629946,0.00018031485,0.000039910366,0.000056685374,0.0011165756,0.000113240174,0.000015143071],"category_scores_gemma":[0.0015362619,0.00014740917,0.0000979603,0.0010486965,0.000024269886,0.00042848822,0.0002677703,0.00036173066,0.0000094875195],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000066317784,0.00027951848,0.9615797,0.000041762476,0.000023661787,0.0000030881574,0.0015713164,0.03270349,0.002013381,0.00003172963,0.000066666034,0.0016790788],"study_design_scores_gemma":[0.00040347135,0.0005050916,0.9385241,0.00005527132,0.000007447658,0.0000071289023,0.000056445908,0.05427423,0.0059232092,0.000015906906,0.000059833157,0.00016785864],"about_ca_topic_score_codex":0.00003744035,"about_ca_topic_score_gemma":0.0000069485827,"teacher_disagreement_score":0.38103926,"about_ca_system_score_codex":0.00009197221,"about_ca_system_score_gemma":0.00006817066,"threshold_uncertainty_score":0.60111713},"labels":[],"label_agreement":null},{"id":"W2920929439","doi":"10.1007/s10664-019-09690-0","title":"Extracting and studying the Logging-Code-Issue- Introducing changes in Java-based large-scale open source software systems","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":35,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Logging; Debugging; Java; Source code; Correctness; Code (set theory); Software; Code review; Database; Software engineering; Static program analysis; Software development; Programming language","score_opus":0.018345802225703055,"score_gpt":0.26551265940598784,"score_spread":0.24716685718028478,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2920929439","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46221194,0.0007761173,0.53343856,0.00079893786,0.0011858945,0.00090958754,0.0000034745112,0.00066389004,0.000011627102],"genre_scores_gemma":[0.9741946,0.000011000637,0.024632927,0.00041364436,0.00035280894,0.00015385401,0.000004603905,0.00005990713,0.00017669461],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99712163,0.00015024995,0.00052693667,0.00093258324,0.00046426727,0.00080434524],"domain_scores_gemma":[0.99704623,0.0015013606,0.00016275949,0.0010519819,0.00008166764,0.00015600014],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0023355375,0.00036667573,0.0005603786,0.00019706092,0.00028528913,0.000646113,0.001451949,0.00017807676,0.000017534892],"category_scores_gemma":[0.0008352607,0.00027484674,0.00006908485,0.00072760606,0.000030272518,0.00071918,0.0010936264,0.00064198696,0.000053704858],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011128683,0.000076644625,0.8419236,0.00055301725,0.000022793703,0.00001130627,0.003785328,0.14673261,0.00006215238,0.000009611953,0.00027373584,0.0065380763],"study_design_scores_gemma":[0.0017344878,0.00020517646,0.15765496,0.0011484189,0.000021168396,0.0000644603,0.0010246951,0.7782503,0.0003502602,0.000017269187,0.058465857,0.0010629578],"about_ca_topic_score_codex":0.00018729255,"about_ca_topic_score_gemma":0.00005012389,"teacher_disagreement_score":0.68426865,"about_ca_system_score_codex":0.00019876093,"about_ca_system_score_gemma":0.00008187284,"threshold_uncertainty_score":0.9999704},"labels":[],"label_agreement":null},{"id":"W2921594963","doi":"10.1007/s10664-019-09691-z","title":"Assessing and optimizing the performance impact of the just-in-time configuration parameters - a case study on PyPy","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Reliability engineering; Engineering","score_opus":0.03857829876151698,"score_gpt":0.31762281544977644,"score_spread":0.27904451668825947,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2921594963","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9443092,0.000018731827,0.054346547,0.00006328975,0.000088111636,0.00030148958,3.3024673e-7,0.00086412946,0.000008206014],"genre_scores_gemma":[0.9747898,5.8592025e-7,0.025110982,0.00005632575,0.000010820415,0.0000140304455,1.8730746e-7,0.000011881539,0.0000054168004],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9990417,0.000067883935,0.00021184793,0.00024487555,0.00022751144,0.00020619239],"domain_scores_gemma":[0.99827296,0.0011114659,0.00007682936,0.00046877694,0.000030147121,0.000039820778],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00046900133,0.00015028419,0.00017761947,0.000080324884,0.00008700611,0.0001822538,0.0003890549,0.000046313893,0.0000019744189],"category_scores_gemma":[0.00041703467,0.00008760655,0.000060855305,0.00038180602,0.000022245771,0.00025604726,0.00016436717,0.00026564233,0.0000033553224],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004739222,0.000093976276,0.88528943,0.00002981778,0.000023786308,0.00006369573,0.002827263,0.10625539,0.00016441752,0.000005297981,0.00011892341,0.0051232544],"study_design_scores_gemma":[0.0003556462,0.00044929126,0.5760097,0.00025118177,0.000011346584,0.00037207198,0.00007373088,0.421539,0.00066739327,0.000021245132,0.000008579959,0.00024085162],"about_ca_topic_score_codex":0.00010229798,"about_ca_topic_score_gemma":3.899007e-7,"teacher_disagreement_score":0.3152836,"about_ca_system_score_codex":0.00006319125,"about_ca_system_score_gemma":0.00003300023,"threshold_uncertainty_score":0.35724917},"labels":[],"label_agreement":null},{"id":"W2935734788","doi":"10.1007/s10664-019-09703-y","title":"iPerfDetector: Characterizing and detecting performance anti-patterns in iOS applications","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Green IT and Sustainability","field":"Engineering","cited_by":18,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Computer science; Workload; Mobile apps; Open source; Thread (computing); Empirical research; Performance improvement; Operating system; Software; World Wide Web; Software engineering; Engineering","score_opus":0.006439130811232374,"score_gpt":0.20538353318306649,"score_spread":0.1989444023718341,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2935734788","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9831235,0.00014691231,0.0156232985,0.000016903356,0.00014432361,0.00032940728,0.0000037322218,0.00058418,0.000027739201],"genre_scores_gemma":[0.9988944,0.000023729819,0.000840165,0.000020569021,0.00007646056,0.00008156362,0.000004138364,0.000046767065,0.000012230549],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99897295,0.000008603641,0.00025070878,0.00025514755,0.00011370901,0.000398853],"domain_scores_gemma":[0.99950725,0.00012955394,0.000017100163,0.00023094108,0.000019644996,0.00009549762],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00014612923,0.00019949694,0.00023819589,0.00014208491,0.000040344392,0.000038821883,0.000119237135,0.00010524697,0.000029609315],"category_scores_gemma":[0.000057854897,0.0002167459,0.00003940884,0.00027150038,0.000009615706,0.00021413607,0.000060339,0.00036320474,0.000026051073],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000015576129,0.0000076174783,0.9721139,0.00049938646,0.00000732367,0.0000033610877,0.00023768256,0.016719654,0.002502149,0.0000011180025,0.0000019370043,0.007904283],"study_design_scores_gemma":[0.00019615405,0.000018916255,0.91802657,0.00006573817,0.00000383843,0.000012611964,0.000041167637,0.077744044,0.0017805632,0.000003253678,0.0018237487,0.00028340475],"about_ca_topic_score_codex":0.000008148853,"about_ca_topic_score_gemma":0.0000037516018,"teacher_disagreement_score":0.061024394,"about_ca_system_score_codex":0.00014242498,"about_ca_system_score_gemma":0.00000921123,"threshold_uncertainty_score":0.88386416},"labels":[],"label_agreement":null},{"id":"W2941522337","doi":"10.1007/s10664-019-09711-y","title":"Characterizing industry-academia collaborations in software engineering: evidence from 101 projects","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":62,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"ITEA3; Estonian Research Competency Council; University of Calgary; Tartu Ülikool; Eesti Teadusagentuur; Tekes; Universidade do Minho; Norges Forskningsråd; ITEA; Strategic Research Council; Fundação para a Ciência e a Tecnologia; Åbo Akademi; Universidade Federal de Santa Catarina","keywords":"Context (archaeology); Relevance (law); Engineering; Empirical research; Engineering management; Business; Knowledge management; Marketing; Computer science; Political science; Geography","score_opus":0.046992848767424926,"score_gpt":0.2945148710774645,"score_spread":0.24752202231003956,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2941522337","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.474116,0.00057961233,0.52025825,0.0007226068,0.0009017498,0.00050692324,0.000015604555,0.0028917762,0.000007475404],"genre_scores_gemma":[0.5362819,0.000063510925,0.46252918,0.00045231188,0.00028734366,0.0001898865,0.000021534142,0.00009357416,0.00008079327],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.996932,0.00006171865,0.00064723525,0.00095064566,0.00058346376,0.00082488346],"domain_scores_gemma":[0.99581265,0.0025662167,0.00016287352,0.0009811198,0.000117554,0.00035960416],"candidate_categories":["metaepi_narrow","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.0005556786,0.0005238479,0.0005455241,0.00045370377,0.00007038322,0.0003659015,0.001357972,0.0009862276,0.000104969076],"category_scores_gemma":[0.006927787,0.0005659107,0.000117927244,0.0021335462,0.000015567277,0.0023775338,0.00052722276,0.0026114613,0.0001245586],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003067973,0.00014867159,0.88173497,0.00035081443,0.00010416963,0.00020094618,0.0029493517,0.102063574,0.0065091727,0.0004335084,0.0019201229,0.0035540347],"study_design_scores_gemma":[0.0010157259,0.00033777452,0.69883454,0.002671861,0.000039445596,0.00006973884,0.00004996161,0.20428921,0.008203299,0.00010949601,0.081652105,0.002726858],"about_ca_topic_score_codex":0.00010856313,"about_ca_topic_score_gemma":0.000004189284,"teacher_disagreement_score":0.18290043,"about_ca_system_score_codex":0.0003688305,"about_ca_system_score_gemma":0.00025998664,"threshold_uncertainty_score":0.9996896},"labels":[],"label_agreement":null},{"id":"W2944738881","doi":"10.1007/s10664-019-09704-x","title":"cregit: Token-level blame information in git version control repositories","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; University of Victoria","funders":"","keywords":"Blame; Commit; Security token; Computer science; Source code; Code (set theory); Computer security; Programming language; Psychology; Social psychology; Set (abstract data type); Database","score_opus":0.012995523476119583,"score_gpt":0.24247411488059042,"score_spread":0.22947859140447086,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2944738881","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.33698568,0.000079362726,0.6605103,0.00028435254,0.0010107206,0.00025727242,0.000005597129,0.0008280383,0.000038690825],"genre_scores_gemma":[0.97257787,0.0000040831433,0.026982717,0.0001474303,0.0001002637,0.000033321427,0.000009560059,0.000021668402,0.00012310687],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980318,0.000028105569,0.00043663537,0.00032975932,0.0006167126,0.0005569661],"domain_scores_gemma":[0.99785084,0.0011959469,0.000060184782,0.00060885143,0.00011342475,0.00017073035],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00040429638,0.0002301822,0.00027577844,0.00038269602,0.000045866083,0.00023909325,0.0007562831,0.00015987678,0.000027253573],"category_scores_gemma":[0.002389728,0.00023596155,0.000081773214,0.00086447597,0.000014333057,0.0017501507,0.00026476156,0.00045404033,0.0004628131],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002869029,0.00004910063,0.7998301,0.00018683459,0.000034486275,0.000048368314,0.0011154175,0.19263998,0.00034013775,0.0002785358,0.0026070036,0.0028414023],"study_design_scores_gemma":[0.0019728884,0.00021578486,0.76348656,0.00018903258,0.000005259017,0.00004230126,0.000021616317,0.1822825,0.0011249494,0.000054511853,0.04990393,0.0007006404],"about_ca_topic_score_codex":0.00003565646,"about_ca_topic_score_gemma":8.235966e-7,"teacher_disagreement_score":0.63559216,"about_ca_system_score_codex":0.0003175234,"about_ca_system_score_gemma":0.00008180086,"threshold_uncertainty_score":0.9622233},"labels":[],"label_agreement":null},{"id":"W2945530585","doi":"10.1007/s10664-019-09719-4","title":"Fostering good coding practices through individual feedback and gamification: an industrial case study","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Coding (social sciences); Variety (cybernetics); Computer science; Best practice; Code review; Software; Quality (philosophy); Software engineering; Knowledge management; Software quality; Data science; Software development; Management","score_opus":0.1720305715459125,"score_gpt":0.37070826398116774,"score_spread":0.19867769243525524,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2945530585","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.86110425,0.00012234715,0.13636559,0.00011655237,0.00064745406,0.0005954968,0.000003659246,0.0010280146,0.000016653126],"genre_scores_gemma":[0.9677744,0.0000043033306,0.031743947,0.000046123165,0.00028525066,0.0000575632,0.0000048986153,0.000050959945,0.00003258606],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99731725,0.00011247869,0.0004014002,0.00087931525,0.0006845658,0.0006049846],"domain_scores_gemma":[0.9964161,0.0021770068,0.00015808166,0.00087541,0.000088716006,0.00028470528],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00087235856,0.0003333396,0.00034106045,0.00020079123,0.00015791564,0.00069179665,0.00085448835,0.0001841045,0.000022877764],"category_scores_gemma":[0.0024079012,0.0003431807,0.00005020002,0.0008294105,0.00002361707,0.002146967,0.0014724473,0.0007565267,0.000043298387],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017406059,0.00039017593,0.9540211,0.00013310381,0.00017040093,0.0017299579,0.01133297,0.011325702,0.0001955544,0.00014794068,0.00015281652,0.020382904],"study_design_scores_gemma":[0.011125,0.0053503923,0.76828915,0.0005548931,0.00021924135,0.015264044,0.007378326,0.17166944,0.0014827979,0.00021863075,0.013156759,0.0052913115],"about_ca_topic_score_codex":0.00008920286,"about_ca_topic_score_gemma":0.000005512418,"teacher_disagreement_score":0.1857319,"about_ca_system_score_codex":0.000116054594,"about_ca_system_score_gemma":0.00008384765,"threshold_uncertainty_score":0.999902},"labels":[],"label_agreement":null},{"id":"W2945826489","doi":"10.1007/s10664-019-09788-5","title":"MSRBot: Using bots to answer questions from software repositories","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":39,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Software; Process (computing); Software development; Work (physics); Code (set theory); Software analytics; Team software process; Source code; Software peer review; Verification and validation","score_opus":0.04104006454653197,"score_gpt":0.29623689857983276,"score_spread":0.2551968340333008,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2945826489","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.15396798,0.0005128138,0.83971435,0.00096748996,0.0012667659,0.00023949893,0.000018221304,0.0033083034,0.0000045940974],"genre_scores_gemma":[0.29601112,0.0000035469657,0.7016112,0.0011442698,0.0010405811,0.00005149908,0.000010042897,0.00009018895,0.000037545527],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.997015,0.000046304336,0.00048536973,0.00094792916,0.00074984215,0.00075556873],"domain_scores_gemma":[0.9966645,0.0012291465,0.000053016563,0.00084253296,0.00019508287,0.0010156928],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00015735946,0.0004083345,0.00041268946,0.00019868842,0.00017662148,0.000391009,0.0013463319,0.00019254127,0.000029971143],"category_scores_gemma":[0.009156236,0.00044016485,0.0001500992,0.0016653427,0.000025629013,0.00060972513,0.00086952257,0.00059342035,0.0001593411],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003453331,0.0001929177,0.54694617,0.00023098127,0.00021598149,0.0005393698,0.004565518,0.41292045,0.0062121367,0.0002983669,0.016388396,0.011455161],"study_design_scores_gemma":[0.0013542286,0.00075710734,0.45013154,0.0010255086,0.000104266546,0.0001572518,0.000064218664,0.2563369,0.015157649,0.00056961534,0.26954463,0.0047970824],"about_ca_topic_score_codex":0.00013844247,"about_ca_topic_score_gemma":0.0000018185917,"teacher_disagreement_score":0.25315621,"about_ca_system_score_codex":0.00026750608,"about_ca_system_score_gemma":0.0002622548,"threshold_uncertainty_score":0.99980503},"labels":[],"label_agreement":null},{"id":"W2946357416","doi":"10.1007/s10664-019-09700-1","title":"An empirical study on the teams structures in social coding using GitHub projects","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Code review; Computer science; Team software process; Coding (social sciences); Project team; Software development; Process (computing); Software engineering; Software; Social network (sociolinguistics); Knowledge management; Software development process; World Wide Web; Software quality; Social media","score_opus":0.04951311858025997,"score_gpt":0.33549298622300366,"score_spread":0.2859798676427437,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2946357416","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8117486,0.0000060572906,0.18627124,0.0004099525,0.00033928905,0.00070514175,0.000002269651,0.00048331064,0.000034127723],"genre_scores_gemma":[0.9832113,9.373552e-8,0.015940497,0.000551587,0.00020781951,0.00003144726,0.0000023275293,0.00004140786,0.000013570743],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99762577,0.00015283117,0.0004142019,0.00064562436,0.00061001664,0.00055153936],"domain_scores_gemma":[0.9984138,0.00063165196,0.00008770026,0.00069616165,0.00008090067,0.00008980127],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00059569185,0.00030484016,0.0003249278,0.0003033763,0.0002586424,0.00035724675,0.0011846658,0.0001348242,0.000026312478],"category_scores_gemma":[0.000498252,0.00023480694,0.000076164964,0.0019146892,0.000028952729,0.0005333258,0.00032641747,0.0007213252,0.000037571423],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000076099846,0.000255942,0.9658611,0.0000137377265,0.000024494671,0.000025881996,0.0075611793,0.02407371,0.00025604077,0.001089618,0.00022492194,0.0006057789],"study_design_scores_gemma":[0.00066190603,0.00034532134,0.9205397,0.000048663966,0.000008210812,0.000015304722,0.0007587012,0.07583595,0.00017733085,0.0003800065,0.000649407,0.0005794917],"about_ca_topic_score_codex":0.000025021314,"about_ca_topic_score_gemma":0.0000040952064,"teacher_disagreement_score":0.17146264,"about_ca_system_score_codex":0.0003263724,"about_ca_system_score_gemma":0.00012704676,"threshold_uncertainty_score":0.9575149},"labels":[],"label_agreement":null},{"id":"W2946929989","doi":"10.1007/s10664-020-09853-4","title":"Automating system test case classification and prioritization for use case-driven testing in product lines","year":2020,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"H2020 European Research Council; Fonds National de la Recherche Luxembourg","keywords":"Computer science; Requirement prioritization; Test strategy; Context (archaeology); Domain engineering; Test case; Software product line; New product development; Risk-based testing; Software engineering; Software development; Systems engineering; Reliability engineering; Software; Engineering; Machine learning; Requirement; Component-based software engineering; Software construction","score_opus":0.18065782686550533,"score_gpt":0.34379375934561823,"score_spread":0.1631359324801129,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2946929989","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.10519456,0.00031198308,0.8882762,0.00015744455,0.0007024568,0.0010778789,0.00004001715,0.004238902,5.180662e-7],"genre_scores_gemma":[0.35704294,0.0000042527377,0.6423373,0.000014444454,0.00025237215,0.00026213698,0.000018291099,0.000065850756,0.0000024221488],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9970766,0.00010519489,0.00080429536,0.0012776252,0.00022699217,0.00050933566],"domain_scores_gemma":[0.9894244,0.0091549745,0.00028907935,0.0006842159,0.0002634804,0.00018385994],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00060464593,0.0005377589,0.0006633843,0.00031381135,0.00015103229,0.00039880173,0.00043547174,0.00031525723,1.3517509e-7],"category_scores_gemma":[0.042162094,0.00059479004,0.000086299195,0.00078622455,0.00002855277,0.0005749527,0.0009636826,0.0008243492,0.0000010651964],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000034515285,0.000031927346,0.05129757,0.0042251972,0.000029919986,0.004345784,0.000879021,0.9202151,0.0006256874,0.00012706428,0.00003216991,0.018187132],"study_design_scores_gemma":[0.00022472389,0.000052759246,0.015157182,0.0008691901,0.000026027681,0.004743444,0.00003606853,0.9779527,0.00015311112,0.00011975124,0.000040294675,0.0006247436],"about_ca_topic_score_codex":0.000040517498,"about_ca_topic_score_gemma":0.0000068509676,"teacher_disagreement_score":0.25184837,"about_ca_system_score_codex":0.00043174927,"about_ca_system_score_gemma":0.00012399812,"threshold_uncertainty_score":0.99965036},"labels":[],"label_agreement":null},{"id":"W2950236608","doi":"10.1007/s10664-019-09709-6","title":"A study of build inflation in 30 million CPAN builds on 13 Perl versions and 10 operating systems","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Polytechnique Montréal","funders":"","keywords":"Perl; Inflation (cosmology); Computer science; Operating system; Programming language; Physics; Astronomy","score_opus":0.021393192071352304,"score_gpt":0.28106298312999833,"score_spread":0.259669791058646,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2950236608","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95472777,0.00012845575,0.044000693,0.00003420308,0.00029552344,0.000504261,0.0000013876524,0.00029535897,0.000012365025],"genre_scores_gemma":[0.996511,0.0000028078105,0.0033214733,0.000010024082,0.000037940026,0.000037093767,0.0000018883613,0.000026038599,0.000051748375],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982754,0.00005947224,0.00035560847,0.0004575374,0.00050304807,0.00034895435],"domain_scores_gemma":[0.9979543,0.0013211126,0.000044775516,0.00047695206,0.00007625724,0.00012657362],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004233133,0.00020232538,0.00030123713,0.0004251356,0.00004855005,0.00009972361,0.0004083991,0.00010811724,0.000014913502],"category_scores_gemma":[0.0012914002,0.00019773992,0.000034812547,0.0008020266,0.000009875124,0.00029127044,0.00029348914,0.00036212002,0.00003689821],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000034288105,0.00009948769,0.57467824,0.00008359073,0.000013917633,0.000014439542,0.001396109,0.42290956,0.0002844516,0.000040763458,0.000042500276,0.000433511],"study_design_scores_gemma":[0.0009642425,0.0006763297,0.73247045,0.0002732,0.0000039901333,0.0000070790375,0.00015244688,0.2647152,0.0001411168,0.0000028142003,0.00029930222,0.00029380835],"about_ca_topic_score_codex":0.00009899455,"about_ca_topic_score_gemma":0.0000028146349,"teacher_disagreement_score":0.15819436,"about_ca_system_score_codex":0.00014915994,"about_ca_system_score_gemma":0.000035723515,"threshold_uncertainty_score":0.80636},"labels":[],"label_agreement":null},{"id":"W2954653546","doi":"10.1007/s10664-019-09718-5","title":"The inconsistency between theory and practice in managing inconsistency in requirements engineering","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":18,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Israel Science Foundation; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Consistency (knowledge bases); Perception; Identification (biology); Field (mathematics); Sight; Phenomenon; Psychology; Computer science; Management science; Knowledge management; Epistemology; Engineering; Artificial intelligence","score_opus":0.018747001289031427,"score_gpt":0.2946853165192847,"score_spread":0.27593831523025325,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2954653546","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.23367083,0.004073374,0.75690705,0.00186082,0.0008825084,0.0007439136,0.0000021373153,0.0014883223,0.00037105524],"genre_scores_gemma":[0.8702455,0.00017673688,0.1291344,0.00023506496,0.00006694117,0.00006242529,0.0000015675824,0.000046352816,0.0000309908],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99780786,0.00018069075,0.00054096937,0.000531825,0.00033830255,0.00060032564],"domain_scores_gemma":[0.9867577,0.012329406,0.000111036075,0.0006331375,0.000043379147,0.00012535752],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0030518833,0.00030095095,0.00032885972,0.00033217703,0.00007322389,0.00024764342,0.0007593279,0.00012202378,0.000003923135],"category_scores_gemma":[0.0073508928,0.00026593095,0.00005897858,0.0009078813,0.000023277087,0.0012916025,0.00057082006,0.00068245066,0.000016039181],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008146355,0.00012896859,0.8747517,0.00039894637,0.00016569416,0.0003871487,0.0031725625,0.029472949,0.00020597076,0.034008376,0.00016289092,0.057063356],"study_design_scores_gemma":[0.0021901126,0.00041020307,0.7963484,0.0014980496,0.0000643587,0.00034678337,0.0003394397,0.088069744,0.00030378462,0.013138823,0.094678156,0.0026121624],"about_ca_topic_score_codex":0.000051418006,"about_ca_topic_score_gemma":0.000005781634,"teacher_disagreement_score":0.6365747,"about_ca_system_score_codex":0.00017573794,"about_ca_system_score_gemma":0.000049870854,"threshold_uncertainty_score":0.9999793},"labels":[],"label_agreement":null},{"id":"W2955854775","doi":"10.1007/s10664-019-09733-6","title":"Identifying gameplay videos that exhibit bugs in computer games","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Digital Games and Media","field":"Social Sciences","cited_by":22,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Queen's University","funders":"","keywords":"Computer science; Metadata; Context (archaeology); Game Developer; Quality (philosophy); Classifier (UML); Multimedia; World Wide Web; Information retrieval; Human–computer interaction; Artificial intelligence; Game design","score_opus":0.0285645952500537,"score_gpt":0.2942501658445961,"score_spread":0.2656855705945424,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2955854775","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9668877,0.0003552158,0.02488929,0.00044598116,0.0017466546,0.00033306668,0.0000038353483,0.00050556223,0.0048326547],"genre_scores_gemma":[0.99139553,0.000031710057,0.0038962837,0.00041792594,0.0004126363,0.0000144590485,0.0000050486265,0.00003121488,0.0037951828],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99853206,0.00002897886,0.00019918391,0.00030384818,0.00041024832,0.0005256775],"domain_scores_gemma":[0.9991732,0.00038361782,0.00003689369,0.0001808762,0.000023534649,0.00020182529],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003150024,0.00015454619,0.0002524004,0.0001253109,0.000029664525,0.00017280459,0.00025359177,0.00012275083,0.00023731138],"category_scores_gemma":[0.00030367312,0.00015384846,0.000101298974,0.00034264696,0.000038840957,0.00040907905,0.000100514204,0.00023272999,0.00029638776],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007457043,0.00006357957,0.8983442,0.00008114099,0.000020318672,0.000051522547,0.010980324,0.004660797,0.000037384576,0.00037923566,0.0025429847,0.08283102],"study_design_scores_gemma":[0.00051996886,0.00004380701,0.15948242,0.00034561657,0.0000068888125,0.0000034581765,0.00044797646,0.0015404044,0.000117574054,0.00011562424,0.8368788,0.00049747946],"about_ca_topic_score_codex":0.00008469295,"about_ca_topic_score_gemma":0.000071651615,"teacher_disagreement_score":0.8343358,"about_ca_system_score_codex":0.00014783144,"about_ca_system_score_gemma":0.000054268705,"threshold_uncertainty_score":0.6273758},"labels":[],"label_agreement":null},{"id":"W2963079908","doi":"10.1007/s10664-019-09743-4","title":"CAPS: a supervised technique for classifying Stack Overflow posts concerning API issues","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":32,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan; Queen's University","funders":"","keywords":"Computer science; Documentation; Application programming interface; Task (project management); Conditional random field; Readability; Field (mathematics); Baseline (sea); Data science; World Wide Web; Artificial intelligence; Machine learning; Engineering; Programming language","score_opus":0.0362815258937799,"score_gpt":0.3089535097084091,"score_spread":0.2726719838146292,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2963079908","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1388863,0.0005875709,0.856402,0.00030578137,0.0006821617,0.0009951048,0.000011969684,0.002112942,0.000016217422],"genre_scores_gemma":[0.63723886,0.000015347367,0.36130652,0.00017661447,0.00029276081,0.00041315757,0.000018274452,0.00013014865,0.0004083585],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9970148,0.000034872824,0.0004216674,0.00084606867,0.000648731,0.0010338122],"domain_scores_gemma":[0.99602664,0.0024652053,0.000059132046,0.00094164396,0.00019674485,0.0003106311],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005976367,0.00041888384,0.00048525335,0.00031501698,0.00011051443,0.00028042882,0.0012809109,0.0002596469,0.000061486346],"category_scores_gemma":[0.0028012723,0.00043210137,0.00020940592,0.00078941195,0.000025943309,0.00067088613,0.00048820174,0.0005945592,0.00013764594],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007968644,0.000239183,0.77725595,0.0023575062,0.0003340075,0.00016229638,0.0031688863,0.122095175,0.07263812,0.0013390068,0.011372681,0.008957505],"study_design_scores_gemma":[0.0043957806,0.0013865028,0.1556188,0.0015641779,0.00004318863,0.00019300338,0.000083941566,0.58700234,0.09297669,0.0005418306,0.15185025,0.0043434887],"about_ca_topic_score_codex":0.000022481769,"about_ca_topic_score_gemma":7.0024436e-7,"teacher_disagreement_score":0.62163717,"about_ca_system_score_codex":0.0003459407,"about_ca_system_score_gemma":0.00013887841,"threshold_uncertainty_score":0.9998131},"labels":[],"label_agreement":null},{"id":"W2966112319","doi":"10.1007/s10664-019-09744-3","title":"Bounties on technical Q&A sites: a case study of Stack Overflow bounties","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":37,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Queen's University","funders":"","keywords":"Reputation; Stack (abstract data type); Computer science; Data science; Operations research; World Wide Web; Engineering; Sociology; Social science","score_opus":0.036897091978036166,"score_gpt":0.29205392285738857,"score_spread":0.2551568308793524,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2966112319","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9234815,0.00019225116,0.07462877,0.000043764096,0.00062651036,0.00031211445,0.0000037213065,0.0006553353,0.00005600961],"genre_scores_gemma":[0.9909761,0.000001719745,0.0084964065,0.00005171299,0.00010032454,0.000028599972,0.0000017145624,0.000028944467,0.0003144935],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99814534,0.000047803183,0.00041909056,0.00048465672,0.0005167782,0.00038635713],"domain_scores_gemma":[0.99837434,0.00049648335,0.000088284265,0.00085760164,0.0000680535,0.000115215844],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00029130155,0.00026222915,0.00042476755,0.0002208936,0.00008634429,0.000143652,0.00055441674,0.00012879563,0.000017039796],"category_scores_gemma":[0.00029613863,0.00023001285,0.00011219976,0.00052345864,0.000021962036,0.00026683678,0.00027352554,0.00031433388,0.000100894445],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000044882458,0.0017325735,0.91799694,0.00056454935,0.00022504157,0.0010626157,0.023300393,0.039204944,0.002614147,0.0010360059,0.011071104,0.001146774],"study_design_scores_gemma":[0.01783111,0.023530073,0.4286807,0.0040295385,0.0002705977,0.013454734,0.013893386,0.18824431,0.0075214067,0.0006159544,0.2888736,0.013054591],"about_ca_topic_score_codex":0.00015446029,"about_ca_topic_score_gemma":0.000034963134,"teacher_disagreement_score":0.48931625,"about_ca_system_score_codex":0.00010865308,"about_ca_system_score_gemma":0.000037314832,"threshold_uncertainty_score":0.9379652},"labels":[],"label_agreement":null},{"id":"W2967780468","doi":"10.1007/s10664-019-09736-3","title":"The impact of context metrics on just-in-time defect prediction","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":57,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Japan Society for the Promotion of Science; Natural Sciences and Engineering Research Council of Canada","keywords":"Source lines of code; Context (archaeology); Metric (unit); Computer science; Measure (data warehouse); Machine learning; Software metric; Software; Data mining; Artificial intelligence; Statistics; Software quality; Mathematics; Software development; Programming language; Engineering; Operations management","score_opus":0.020471050761877507,"score_gpt":0.29267956045619137,"score_spread":0.27220850969431387,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2967780468","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.86480504,0.00039576067,0.13348983,0.000052138355,0.00040842168,0.0003104192,0.0000064913997,0.0005016779,0.00003021909],"genre_scores_gemma":[0.99736875,0.000012080928,0.0023901951,0.000016364595,0.00005185621,0.0000210025,0.000002245707,0.000030001604,0.000107511194],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980774,0.00004893344,0.00032802805,0.00036497632,0.00065355754,0.0005271297],"domain_scores_gemma":[0.993427,0.005577719,0.00005104967,0.0007196696,0.00008446842,0.00014013368],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00074926723,0.0002142181,0.00027757688,0.00038615303,0.00004411215,0.00009928853,0.0010083107,0.00011690465,0.000021054118],"category_scores_gemma":[0.004344278,0.00015382648,0.0002442067,0.0016501067,0.000020107238,0.00021716584,0.00023135233,0.0004663,0.00019017699],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017667873,0.000062195875,0.7642879,0.000029541457,0.00005570652,0.000009013283,0.00014858357,0.22933908,0.00017035594,0.00007752859,0.0009349811,0.004867441],"study_design_scores_gemma":[0.00046162962,0.00044236935,0.79811937,0.000077580604,0.0000032642351,0.000008737044,0.0000031157692,0.19940251,0.000450569,0.000023388173,0.00081033376,0.00019713935],"about_ca_topic_score_codex":0.000034036497,"about_ca_topic_score_gemma":4.1303196e-7,"teacher_disagreement_score":0.1325637,"about_ca_system_score_codex":0.00039294807,"about_ca_system_score_gemma":0.00008307324,"threshold_uncertainty_score":0.6272862},"labels":[],"label_agreement":null},{"id":"W2969694848","doi":"10.1007/s10664-019-09760-3","title":"Empirical study of android repackaged applications","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec à Chicoutimi; Concordia University","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Malware; Android (operating system); Computer science; Popularity; Mobile malware; Upload; Download; Computer security; World Wide Web; Empirical research; Publication; Internet privacy; Operating system; Advertising","score_opus":0.014366851444132947,"score_gpt":0.28824754942819064,"score_spread":0.2738806979840577,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2969694848","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.29164547,0.000032973487,0.7064019,0.000039635102,0.00010026189,0.00048385496,0.0000011440745,0.0012544824,0.000040304985],"genre_scores_gemma":[0.8767215,0.0000019878983,0.12291483,0.0000701845,0.000038341128,0.00014871599,0.0000012109961,0.00002201704,0.000081237515],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99858326,0.00003343973,0.00035115686,0.00046578888,0.00031866468,0.0002476799],"domain_scores_gemma":[0.99851245,0.00031381715,0.00008869709,0.00088312203,0.00009472724,0.000107196036],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00017999324,0.00016998812,0.0002723895,0.00017507568,0.000046987774,0.000030769443,0.00060431566,0.00008235822,0.00001895561],"category_scores_gemma":[0.00020795625,0.00016813868,0.00007510555,0.000862877,0.0000127747735,0.000294886,0.00028524097,0.0002630622,0.000047279027],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000031849093,0.0019153194,0.9267113,0.00026035478,0.00014902426,0.000041670144,0.0033725465,0.03247862,0.0066654277,0.0005629157,0.0020144305,0.025796503],"study_design_scores_gemma":[0.0037190178,0.0037394213,0.7591798,0.00015640198,0.00006155073,0.00016698736,0.00035723374,0.044344947,0.049226698,0.0031346832,0.1331692,0.0027441098],"about_ca_topic_score_codex":0.0000047533485,"about_ca_topic_score_gemma":0.0000010117508,"teacher_disagreement_score":0.585076,"about_ca_system_score_codex":0.00007540386,"about_ca_system_score_gemma":0.000025912976,"threshold_uncertainty_score":0.68564963},"labels":[],"label_agreement":null},{"id":"W2971660263","doi":"10.1007/s10664-019-09771-0","title":"Why reinventing the wheels? An empirical study on library reuse and re-implementation","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Digital and Traditional Archives Management","field":"Arts and Humanities","cited_by":47,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Reuse; Empirical research; Computer science; Engineering; Engineering management; Mathematics; Statistics; Waste management","score_opus":0.044619323073125015,"score_gpt":0.2730376624820939,"score_spread":0.22841833940896886,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2971660263","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.98862994,0.000014259974,0.0001836838,0.0026689712,0.0002567111,0.0004122234,0.000019633373,0.00029224923,0.007522339],"genre_scores_gemma":[0.9955533,0.0000017460313,0.00024076359,0.0028268243,0.0003339439,0.000034722438,0.000056605193,0.000031503874,0.0009205534],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9990535,0.00003926066,0.0002004415,0.00026853086,0.00023439094,0.00020387913],"domain_scores_gemma":[0.9993618,0.00024237808,0.000037526963,0.0002679497,0.000010342659,0.00007997203],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00009995579,0.00015329222,0.00012528485,0.00007481866,0.00017138413,0.0003893889,0.00021626696,0.000010860537,0.00044671228],"category_scores_gemma":[0.000037333142,0.00010689623,0.000050226336,0.00005212503,0.000033552158,0.0006584431,0.00018490799,0.00015866387,0.0000623748],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00017088096,0.0020622234,0.6350383,0.0003121098,0.0005563295,0.00010908974,0.115818284,0.003963846,0.000013982812,0.062050007,0.15913543,0.020769471],"study_design_scores_gemma":[0.00057012076,0.0009517932,0.45872122,0.000053222924,0.00003336595,0.0000019900976,0.004121512,0.0007385788,0.000011377579,0.003728113,0.5307197,0.00034897315],"about_ca_topic_score_codex":0.000016879832,"about_ca_topic_score_gemma":0.000010725312,"teacher_disagreement_score":0.3715843,"about_ca_system_score_codex":0.000013269,"about_ca_system_score_gemma":0.000006727147,"threshold_uncertainty_score":0.4891186},"labels":[],"label_agreement":null},{"id":"W2983752450","doi":"10.1007/s10664-019-09784-9","title":"An experimental scrutiny of visual design modelling: VCL up against UML+OCL","year":2019,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Visualization and Analytics","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Computer science; Unified Modeling Language; Visual modeling; Comprehension; Notation; Software engineering; Programming language; Software; Mathematics","score_opus":0.04332768921336913,"score_gpt":0.3190907181283654,"score_spread":0.27576302891499627,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2983752450","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.112167425,0.000074829215,0.8869577,0.00001966959,0.00030635504,0.000101513564,0.000003489614,0.0003457574,0.000023225686],"genre_scores_gemma":[0.8812183,0.000004647366,0.1184016,0.00018418909,0.00006196895,0.0000037550062,0.000020861327,0.000023688412,0.00008096311],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9987023,0.000034800418,0.000294838,0.00035536778,0.0003336121,0.00027907683],"domain_scores_gemma":[0.9991958,0.000097141405,0.0000627945,0.00041348135,0.000058724032,0.00017206391],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00018924916,0.00017573024,0.0002232057,0.00012361213,0.000038644477,0.00009662232,0.0005948724,0.000078296915,0.000036217883],"category_scores_gemma":[0.000055332115,0.00017502489,0.00007028312,0.00040111493,0.000015920083,0.0005164545,0.0001535487,0.000118331205,0.00006395774],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000061812902,0.00016211985,0.0030099335,0.00003077604,0.000018750896,0.0000055163796,0.0006646753,0.98837566,0.0047406075,0.0011390607,0.0003944028,0.0014522866],"study_design_scores_gemma":[0.00024590164,0.0001354892,0.00014295221,0.000027809427,0.000002814278,0.0000018647047,0.000021304897,0.97882414,0.01879715,0.00001657391,0.0015639064,0.00022009038],"about_ca_topic_score_codex":0.000003019139,"about_ca_topic_score_gemma":4.180457e-8,"teacher_disagreement_score":0.7690509,"about_ca_system_score_codex":0.00004841543,"about_ca_system_score_gemma":0.0000482726,"threshold_uncertainty_score":0.7137308},"labels":[],"label_agreement":null},{"id":"W2988589793","doi":"10.1007/s10664-019-09776-9","title":"Guest Editorial: Special Issue on Software Engineering for Mobile Applications","year":2019,"lang":"en","type":"editorial","venue":"Empirical Software Engineering","topic":"Green IT and Sustainability","field":"Engineering","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Software engineering; Systems engineering; Engineering","score_opus":0.005986996702665983,"score_gpt":0.25187963848650285,"score_spread":0.24589264178383688,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2988589793","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00013050955,0.00037598785,0.061781425,0.00001097498,0.930323,0.0028273254,0.0010125819,0.0034726353,0.00006554936],"genre_scores_gemma":[0.000054954737,0.00005023405,0.0059999987,0.000013644665,0.9876925,0.0033218837,0.0014407255,0.00065207,0.00077401934],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99521565,0.000023171993,0.0009813429,0.0011567753,0.0012260334,0.0013970362],"domain_scores_gemma":[0.99345785,0.00418337,0.000120263634,0.0012629918,0.00051793136,0.0004575988],"candidate_categories":["metaepi_narrow","research_integrity"],"consensus_categories":["metaepi_narrow","research_integrity"],"category_scores_codex":[0.00045603685,0.0012989517,0.0013641774,0.0004374818,0.0001572896,0.00021866134,0.00095762545,0.0024243614,0.00011140572],"category_scores_gemma":[0.0049873726,0.0014593857,0.0006293838,0.0005817475,0.00003993024,0.0002351722,0.00017640828,0.002422338,0.00041080327],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000026156706,0.0000618471,0.00014433135,0.0022282188,0.0001288368,0.0000052248774,0.00009156125,0.15571897,0.000004541155,0.000004398553,0.84085554,0.0007303727],"study_design_scores_gemma":[0.0006908081,0.00022351443,0.000057146877,0.00020192783,0.00010657255,0.0000011390293,0.000012975185,0.002595708,0.00006947207,0.000015248265,0.9945612,0.0014642896],"about_ca_topic_score_codex":0.000011030129,"about_ca_topic_score_gemma":0.000004168026,"teacher_disagreement_score":0.15370566,"about_ca_system_score_codex":0.0014871415,"about_ca_system_score_gemma":0.00025363578,"threshold_uncertainty_score":0.9999762},"labels":[],"label_agreement":null},{"id":"W3000116190","doi":"10.1007/s10664-019-09790-x","title":"What should your run-time configuration framework do to help developers?","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Université du Québec à Chicoutimi; Queen's University","funders":"","keywords":"Software engineering; Configuration Management (ITSM); Computer science; Software configuration management; Software deployment; Debugging; Domain (mathematical analysis); Flexibility (engineering); Requirements engineering; Systems engineering; Code refactoring; Software system; Software; Engineering; Software construction; Operating system","score_opus":0.103734205417861,"score_gpt":0.33745975405508927,"score_spread":0.23372554863722828,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3000116190","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.005627914,0.00049932144,0.9809837,0.007619653,0.0012813169,0.00028354072,0.0000028626923,0.0036936987,0.000008024487],"genre_scores_gemma":[0.025763253,0.000028419194,0.9690979,0.00450876,0.0004071078,0.000055195353,0.000005433322,0.00006952023,0.00006440939],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9975287,0.00006606748,0.00042263538,0.00081277196,0.0005075733,0.0006623062],"domain_scores_gemma":[0.9971034,0.0015710568,0.00006026307,0.0005817727,0.00012492927,0.0005585964],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003320691,0.00042141363,0.00046101309,0.00014381635,0.00010557586,0.00050917483,0.0011624843,0.00025729026,0.00005368536],"category_scores_gemma":[0.009827473,0.00042609623,0.000119542055,0.0013134078,0.000020578133,0.0013785532,0.0005121237,0.0006231599,0.000498073],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001798851,0.000025877414,0.0007776617,0.000091113514,0.000056770397,0.00008848205,0.006117245,0.9508638,0.0024349631,0.0012243526,0.0032123167,0.035089456],"study_design_scores_gemma":[0.0025873529,0.0022651083,0.043992016,0.0027458565,0.0001320473,0.00023319272,0.0011018508,0.28762186,0.081444055,0.01544338,0.5498446,0.012588677],"about_ca_topic_score_codex":0.0000010863704,"about_ca_topic_score_gemma":7.764495e-8,"teacher_disagreement_score":0.6632419,"about_ca_system_score_codex":0.00015572277,"about_ca_system_score_gemma":0.000068148656,"threshold_uncertainty_score":0.9998191},"labels":[],"label_agreement":null},{"id":"W3001783472","doi":"10.1007/s10664-020-09807-w","title":"Ammonia: an approach for deriving project-specific bug patterns","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Java; Software bug; Software regression; Security bug; Software development; Source code; Limiting; Pointer (user interface); Software maintenance","score_opus":0.0743664019156529,"score_gpt":0.30118913813212417,"score_spread":0.22682273621647125,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3001783472","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07932287,0.00019156351,0.9165338,0.00030326375,0.00024490754,0.00055844843,0.00001427511,0.0028250827,0.000005758966],"genre_scores_gemma":[0.497615,0.000007714687,0.50135255,0.00021579328,0.0004899678,0.0001931844,0.000026137272,0.000086580534,0.000013074007],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99721617,0.000036722904,0.00035594922,0.00096729264,0.0005469983,0.00087685726],"domain_scores_gemma":[0.9977812,0.0009069751,0.000044594402,0.00066709286,0.000109916626,0.0004902042],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00029787506,0.00036737774,0.0003686232,0.0001780068,0.00012905727,0.0003452415,0.0014730268,0.00016494583,0.0000070524775],"category_scores_gemma":[0.0015033237,0.00037545842,0.00015892263,0.00090078387,0.000019540506,0.000669119,0.00042993212,0.0005055483,0.000017677075],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000067809226,0.00057099765,0.6201463,0.0018147354,0.0002292334,0.00017598076,0.008812224,0.31685603,0.0025068019,0.0006929485,0.012317166,0.035809804],"study_design_scores_gemma":[0.0009967644,0.00045140382,0.123379625,0.000059969978,0.000011371955,0.000042988722,0.00004841281,0.83411497,0.001767881,0.00002155796,0.037836753,0.0012682778],"about_ca_topic_score_codex":0.000009308594,"about_ca_topic_score_gemma":3.4826692e-7,"teacher_disagreement_score":0.51725894,"about_ca_system_score_codex":0.00013562068,"about_ca_system_score_gemma":0.00008141144,"threshold_uncertainty_score":0.99986976},"labels":[],"label_agreement":null},{"id":"W3004570974","doi":"10.1007/s10664-019-09781-y","title":"How bugs are born: a model to identify how bugs are introduced in software components","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":74,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; University of Waterloo","funders":"H2020 Industrial Leadership; Ministerio de Asuntos Económicos y Transformación Digital, Gobierno de España; Nederlandse Organisatie voor Wetenschappelijk Onderzoek","keywords":"Software bug; Software regression; Computer science; False positive paradox; Source lines of code; Software; Source code; Open source; Debugging; Software maintenance; Snapshot (computer storage); Code (set theory); Security bug; Data mining; Software development; Software quality; Programming language; Machine learning; Database; Operating system; Set (abstract data type); Software security assurance","score_opus":0.062116288198003355,"score_gpt":0.2983208692382943,"score_spread":0.23620458104029096,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3004570974","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.30517566,0.00023562602,0.6740202,0.017350918,0.0004331913,0.000482423,0.000029056868,0.002272167,8.021084e-7],"genre_scores_gemma":[0.86170757,0.000008995316,0.13617913,0.0011978733,0.0004453474,0.00018075915,0.000018669249,0.00015426968,0.00010737476],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9945916,0.000089343266,0.0005273791,0.0017208928,0.0014800726,0.001590737],"domain_scores_gemma":[0.9959326,0.0011062747,0.00014252873,0.0012708894,0.00027428538,0.001273462],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00043710048,0.0007660761,0.0009336406,0.00064313976,0.00014112373,0.0010112706,0.00266517,0.00033254336,0.0000068696913],"category_scores_gemma":[0.013511785,0.0008349529,0.00025620527,0.002949003,0.000040517974,0.0011481189,0.0016987114,0.0013672451,0.00012151567],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007407378,0.00029252307,0.36557826,0.0009722452,0.00015567697,0.0010481703,0.0035951645,0.5887282,0.006323423,0.000042664302,0.027234396,0.0059552505],"study_design_scores_gemma":[0.0014000132,0.00017867582,0.57824713,0.00048895914,0.00001741664,0.000035717494,0.00007240051,0.40853205,0.002270009,0.0000810058,0.00687061,0.0018059806],"about_ca_topic_score_codex":0.00001697437,"about_ca_topic_score_gemma":0.000006936054,"teacher_disagreement_score":0.5565319,"about_ca_system_score_codex":0.0005524658,"about_ca_system_score_gemma":0.00012219872,"threshold_uncertainty_score":0.99941015},"labels":[],"label_agreement":null},{"id":"W3011992028","doi":"10.1007/s10664-019-09796-5","title":"An exploratory study of smart contracts in the Ethereum blockchain platform","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Blockchain Technology Applications and Security","field":"Computer Science","cited_by":241,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; Queen's University","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada; Microsoft","keywords":"Blockchain; Smart contract; Computer science; Computer security","score_opus":0.03360663077197832,"score_gpt":0.26758092421985136,"score_spread":0.23397429344787304,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3011992028","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7949066,0.00006522352,0.20350721,0.0009444395,0.000031112126,0.00024192715,9.624584e-7,0.00029750477,0.0000050179965],"genre_scores_gemma":[0.99495095,0.0000010967601,0.0044374964,0.0004967001,0.000029534762,0.00007463584,5.1752596e-7,0.000008733611,3.464717e-7],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99902695,0.000041584,0.00025232421,0.0002822917,0.00019549095,0.00020135028],"domain_scores_gemma":[0.9990709,0.00025874155,0.000046999354,0.000517589,0.000029145576,0.00007664689],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00035901967,0.00012088834,0.00018387647,0.000065770066,0.000048012254,0.000025279543,0.0010405132,0.000098154575,0.0000028130123],"category_scores_gemma":[0.00019172933,0.000094987336,0.000029849647,0.0006933764,0.000020684527,0.00009803758,0.00012165983,0.00038475948,0.000004596657],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000083638064,0.0076053236,0.43977502,0.00018706957,0.00019889446,0.00036767224,0.36519367,0.11482962,0.0025308675,0.027800016,0.0016047717,0.039823417],"study_design_scores_gemma":[0.002932456,0.0032293177,0.18468753,0.00004960935,0.000028629407,0.00002594456,0.007807211,0.78586656,0.002866615,0.0023963542,0.009005351,0.0011044057],"about_ca_topic_score_codex":0.000013311306,"about_ca_topic_score_gemma":0.000040986597,"teacher_disagreement_score":0.67103696,"about_ca_system_score_codex":0.000018750196,"about_ca_system_score_gemma":0.000022331255,"threshold_uncertainty_score":0.3873471},"labels":[],"label_agreement":null},{"id":"W3013163251","doi":"10.1007/s10664-019-09783-w","title":"Building the perfect game – an empirical study of game modifications","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Digital Games and Media","field":"Social Sciences","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Queen's University","funders":"","keywords":"Nexus (standard); Game Developer; Video game development; Game design; Empirical research; Game mechanics; Expectancy theory; Video game; Game design document","score_opus":0.06416992483423542,"score_gpt":0.3518687718725928,"score_spread":0.28769884703835735,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3013163251","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9780183,0.000096098964,0.018322486,0.0021226304,0.00018216083,0.00037159462,0.00000577041,0.00033940442,0.00054156326],"genre_scores_gemma":[0.9979661,0.0000066362873,0.0010965548,0.00034914297,0.00039613422,0.000032175863,0.0000019599609,0.00002376673,0.00012754255],"study_design_codex":"qualitative","study_design_gemma":"not_applicable","domain_scores_codex":[0.9985545,0.00009563302,0.00026563523,0.0002811683,0.00046918695,0.00033384934],"domain_scores_gemma":[0.99880046,0.0004929586,0.000054369655,0.00022644007,0.00007023829,0.00035553632],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00030179173,0.00014058575,0.00022486334,0.000041155137,0.00008173901,0.0000875713,0.00044547295,0.000074349686,0.000046100984],"category_scores_gemma":[0.002238974,0.000108729146,0.000093271,0.0006243519,0.000089710025,0.0002186154,0.000083903054,0.0002545589,0.00001503102],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006715673,0.002048092,0.3437551,0.0001129492,0.00022929581,0.00004547494,0.4835863,0.089212745,0.0009191225,0.0017750909,0.0036883447,0.07456029],"study_design_scores_gemma":[0.00083089696,0.0010511222,0.27074197,0.00004407983,0.00010227781,0.0000026338105,0.013339429,0.013082726,0.00011500836,0.000109458895,0.6999124,0.0006679485],"about_ca_topic_score_codex":0.00006344262,"about_ca_topic_score_gemma":0.000032013475,"teacher_disagreement_score":0.6962241,"about_ca_system_score_codex":0.000054385022,"about_ca_system_score_gemma":0.00008313686,"threshold_uncertainty_score":0.4433846},"labels":[],"label_agreement":null},{"id":"W3016381677","doi":"10.1007/s10664-020-09818-7","title":"How software engineering research aligns with design science: a review","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":69,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Lunds Universitet","keywords":"Design science; Novelty; Software; Software design; Set (abstract data type); Design science research; Social software engineering","score_opus":0.13199070938243,"score_gpt":0.34603854400817013,"score_spread":0.21404783462574015,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3016381677","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0005321363,0.005341699,0.98017544,0.008201665,0.00021712974,0.0006750276,0.000003431834,0.0048444304,0.00000905789],"genre_scores_gemma":[0.055217072,0.0006445114,0.9421637,0.001292299,0.00030927156,0.00022054954,0.0000037669608,0.00011130835,0.000037512233],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.995357,0.000096237265,0.00038241147,0.0011604594,0.0017087894,0.0012951227],"domain_scores_gemma":[0.9955274,0.0020146002,0.00009881588,0.0010301592,0.0004808681,0.0008481624],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0020786417,0.00050469366,0.0005555315,0.0004016044,0.0003027814,0.0008898627,0.0025783796,0.00015439901,0.000016870803],"category_scores_gemma":[0.012705798,0.00043866065,0.00012499538,0.0052243476,0.00011582141,0.0020281607,0.00082624605,0.0012132863,0.000040929484],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0002833659,0.0009442094,0.037179064,0.025694791,0.0011135502,0.0051506315,0.00987165,0.5108272,0.008574741,0.015458884,0.2768185,0.1080834],"study_design_scores_gemma":[0.001252669,0.002672586,0.0061245756,0.006422673,0.00012926706,0.00065071863,0.00003719061,0.36349788,0.014362622,0.00024874345,0.60026836,0.004332704],"about_ca_topic_score_codex":0.000006415452,"about_ca_topic_score_gemma":1.5081993e-7,"teacher_disagreement_score":0.32344985,"about_ca_system_score_codex":0.00024467966,"about_ca_system_score_gemma":0.0003753397,"threshold_uncertainty_score":0.9998065},"labels":[],"label_agreement":null},{"id":"W3016717485","doi":"10.1007/s10664-020-09814-x","title":"Using machine learning to assist with the selection of security controls during security assessment","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Information and Cyber Security","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Operationalization; Computer science; Security information and event management; Security controls; Computer security model; Computer security; Context (archaeology); Security service; Security testing; Cloud computing security; Security engineering; Security through obscurity; Security domain; Standard of Good Practice; Security convergence; Information security; Risk analysis (engineering); Software security assurance; Control (management); Artificial intelligence; Business; Cloud computing; Network security policy","score_opus":0.01568697951433107,"score_gpt":0.26038288969559326,"score_spread":0.2446959101812622,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3016717485","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.43882298,0.000020094272,0.55961573,0.0010320742,0.00004738045,0.0001482185,0.0000024531719,0.0002680629,0.00004302372],"genre_scores_gemma":[0.97674555,8.01728e-7,0.022654254,0.0004984791,0.000073021605,0.0000112216785,0.0000022156628,0.00001215695,0.0000023120776],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9987693,0.00006086892,0.0002635363,0.00023613819,0.00039282383,0.00027732528],"domain_scores_gemma":[0.9993198,0.000121953955,0.000101874546,0.00016590834,0.00010907107,0.00018138657],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023433918,0.00017282409,0.00022885438,0.00005955998,0.00019446708,0.00011658824,0.0003741349,0.000054093824,0.000017195842],"category_scores_gemma":[0.00017328396,0.00012964322,0.00008360157,0.0007287817,0.000015004396,0.00038611548,0.00020472692,0.00048308572,0.0000047058465],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000093975556,0.00013436493,0.2797683,0.00042933584,0.00022161062,0.000023484803,0.027907895,0.68559515,0.002757788,0.0020290215,0.00015596215,0.00088310585],"study_design_scores_gemma":[0.00045912206,0.00013828487,0.03861531,0.000032183474,0.000014485394,0.00001922441,0.000063442734,0.9551188,0.002672199,0.00001444398,0.0026101705,0.00024236515],"about_ca_topic_score_codex":0.00003232461,"about_ca_topic_score_gemma":0.000009071061,"teacher_disagreement_score":0.53792256,"about_ca_system_score_codex":0.000107239575,"about_ca_system_score_gemma":0.000059766306,"threshold_uncertainty_score":0.52866966},"labels":[],"label_agreement":null},{"id":"W3017078585","doi":"10.1007/s10664-020-09806-x","title":"Preface to the special issue on program comprehension","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Intelligent Tutoring Systems and Adaptive Learning","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Program comprehension; Computer science; Comprehension; Programming language; Software","score_opus":0.03621576006486358,"score_gpt":0.27980399103285897,"score_spread":0.2435882309679954,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3017078585","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.020021267,0.000042831576,0.96706855,0.0093363635,0.0014478362,0.0005036823,9.912644e-7,0.0013059293,0.0002725361],"genre_scores_gemma":[0.7166,0.000003930274,0.2271561,0.010352489,0.041827045,0.0001935119,0.000005295432,0.0001258742,0.0037357742],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9988218,0.000033380813,0.0001728829,0.00035147878,0.00032175911,0.0002987236],"domain_scores_gemma":[0.99930215,0.0001759,0.000028118935,0.00025782257,0.000041046307,0.00019493613],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00010460185,0.00016001344,0.00015273533,0.000032825646,0.000118507814,0.00015713737,0.000593186,0.000046596797,0.00002844268],"category_scores_gemma":[0.00036357602,0.00011358871,0.000070129485,0.00039602714,0.0000056173003,0.0000941984,0.0002383762,0.0003108525,0.00082717394],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020470998,0.00008848798,0.003995409,0.000067038134,0.000036853817,0.000044542536,0.006436817,0.7793066,0.00017041803,0.0045476155,0.065801494,0.13948427],"study_design_scores_gemma":[0.00005395217,0.0002583627,0.005831048,0.00004516975,0.0000014958681,0.0000019641961,0.000010947802,0.035998613,0.00036438886,0.0000025528127,0.9572782,0.00015329427],"about_ca_topic_score_codex":0.0000043022255,"about_ca_topic_score_gemma":3.0073596e-7,"teacher_disagreement_score":0.8914767,"about_ca_system_score_codex":0.000046269724,"about_ca_system_score_gemma":0.000013836355,"threshold_uncertainty_score":0.9999508},"labels":[],"label_agreement":null},{"id":"W3018447383","doi":"10.1007/s10664-020-09819-6","title":"What do Programmers Discuss about Deep Learning Frameworks","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":89,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Deep learning; Workflow; Computer science; Latent Dirichlet allocation; Leverage (statistics); Artificial intelligence; Topic model; Data science; Profiling (computer programming); World Wide Web; Machine learning","score_opus":0.01947834175930554,"score_gpt":0.2802885285859658,"score_spread":0.2608101868266603,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3018447383","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.032515768,0.002891416,0.9568699,0.0026851085,0.0009356673,0.0002666907,5.9139865e-7,0.003827915,0.000006925092],"genre_scores_gemma":[0.83747447,0.00022917612,0.16059017,0.00077054533,0.0006138339,0.00012227499,0.000009727327,0.00013155692,0.00005828019],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9968327,0.000059575093,0.0003779858,0.000868712,0.00081376877,0.0010472534],"domain_scores_gemma":[0.9970175,0.0014640602,0.000056071625,0.00055739295,0.00010444477,0.0008005175],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00030971138,0.00040307074,0.00038386593,0.00015461659,0.00014893942,0.0011914984,0.001438423,0.000371329,0.000055092438],"category_scores_gemma":[0.006076903,0.00038088573,0.00019772595,0.0015929206,0.00004358851,0.001342746,0.00073610776,0.0022700147,0.0002126407],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000016830598,0.00009720041,0.24402243,0.0003409267,0.00015150291,0.0003239774,0.007526493,0.4202347,0.00012801317,0.00034559166,0.0010042716,0.32580805],"study_design_scores_gemma":[0.0010356191,0.00057420926,0.1271799,0.00064060546,0.00002869936,0.000063210005,0.00030898562,0.68023235,0.00073071977,0.00017040322,0.18674555,0.0022897047],"about_ca_topic_score_codex":0.0000032528465,"about_ca_topic_score_gemma":2.797342e-7,"teacher_disagreement_score":0.80495864,"about_ca_system_score_codex":0.00014201936,"about_ca_system_score_gemma":0.000051668252,"threshold_uncertainty_score":0.9998643},"labels":[],"label_agreement":null},{"id":"W3021573492","doi":"10.1007/s10664-021-09956-6","title":"On systematically building a controlled natural language for functional requirements","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Fonds National de la Recherche Luxembourg","keywords":"Computer science; Domain (mathematical analysis); Natural language; Vagueness; Context (archaeology); Ambiguity; Flexibility (engineering); Quality (philosophy); Popularity; Grammar; Natural language processing; Artificial intelligence; Programming language; Linguistics; Psychology","score_opus":0.034998506852739225,"score_gpt":0.31997698353604714,"score_spread":0.2849784766833079,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3021573492","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.13576126,0.0014065377,0.8558551,0.00023930571,0.0037567562,0.0013160878,0.00001527759,0.001643858,0.0000057888365],"genre_scores_gemma":[0.7813567,0.0000034614093,0.21638049,0.00021240448,0.0005336285,0.0011538909,0.00007682083,0.00010728409,0.00017529649],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99567014,0.000093862094,0.0008022106,0.0012414829,0.0012848572,0.000907453],"domain_scores_gemma":[0.99005336,0.0078558205,0.0001623384,0.0012571348,0.0003584652,0.0003128822],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009590141,0.0006533408,0.0011417822,0.00043286293,0.00012610972,0.00073395896,0.0013838689,0.00041430947,0.000023552459],"category_scores_gemma":[0.020795206,0.0006140214,0.0006468409,0.00045905265,0.000021232961,0.00023651478,0.001454059,0.0013871788,0.000019056686],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00044467355,0.0006790278,0.004770703,0.012284867,0.0026763754,0.00078792794,0.0019318725,0.9555535,0.0029092538,0.005445916,0.008805268,0.0037106192],"study_design_scores_gemma":[0.0060726553,0.00024828676,0.011307671,0.0042577926,0.000113924885,0.00006825857,0.000026899073,0.9738753,0.0012771174,0.0004454569,0.0003643706,0.0019422641],"about_ca_topic_score_codex":0.000006046631,"about_ca_topic_score_gemma":6.681983e-7,"teacher_disagreement_score":0.6455955,"about_ca_system_score_codex":0.00067838543,"about_ca_system_score_gemma":0.0003066837,"threshold_uncertainty_score":0.9996311},"labels":[],"label_agreement":null},{"id":"W3024356476","doi":"10.1007/s10664-020-09878-9","title":"On the time-based conclusion stability of cross-project defect prediction models","year":2020,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Stability (learning theory); Computer science; Software; Predictive modelling; Product (mathematics); Limit (mathematics); Data mining; Time limit; Analytics; Data science; Empirical research; Econometrics; Reliability engineering; Statistics; Machine learning; Mathematics; Engineering; Systems engineering","score_opus":0.07151071104906312,"score_gpt":0.31989840498568256,"score_spread":0.24838769393661944,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3024356476","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.23546879,0.0001486282,0.760733,0.00041631548,0.00056365685,0.0008056509,0.00010869775,0.001736491,0.00001875912],"genre_scores_gemma":[0.9682839,0.0000056754875,0.031052439,0.00014242547,0.00016539496,0.00020241592,0.0000508444,0.00008748394,0.000009437477],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99601305,0.00019507825,0.00067465537,0.0011793951,0.0013383095,0.0005995297],"domain_scores_gemma":[0.98989934,0.0075680935,0.00016756522,0.0018376024,0.00031163392,0.00021578041],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0011924896,0.00056261994,0.0006346086,0.00026143072,0.00013528955,0.0002657011,0.002273106,0.00046390755,0.000041293642],"category_scores_gemma":[0.010183464,0.00045144864,0.0005248895,0.000885552,0.000106847794,0.00023358596,0.0025144634,0.0018902249,0.000035803554],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000343278,0.000083534804,0.010164823,0.0005089681,0.00007981791,0.000014677232,0.00036616751,0.98660755,0.00025267745,0.00033786832,0.0013126411,0.00023696496],"study_design_scores_gemma":[0.00028858925,0.00020116735,0.015604639,0.00031075723,0.000016884598,0.0000021995845,0.0000011223333,0.97902626,0.0029114033,0.0010695038,0.00017031554,0.00039715294],"about_ca_topic_score_codex":0.000044565753,"about_ca_topic_score_gemma":3.6076045e-7,"teacher_disagreement_score":0.7328151,"about_ca_system_score_codex":0.00048244742,"about_ca_system_score_gemma":0.00053857197,"threshold_uncertainty_score":0.9997937},"labels":[],"label_agreement":null},{"id":"W3028200206","doi":"10.1007/s10664-020-09837-4","title":"Do code review measures explain the incidence of post-release defects?","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Operationalization; Code (set theory); Computer science; Replication (statistics); Code review; Replicate; Variable (mathematics); Empirical research; Contrast (vision); Econometrics; Statistics; Software; Software quality; Artificial intelligence; Mathematics; Software development; Programming language","score_opus":0.042557336039330304,"score_gpt":0.29470602258305645,"score_spread":0.25214868654372613,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3028200206","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.05263807,0.021171913,0.9197892,0.004700792,0.00021184598,0.00042753475,0.000010391858,0.0010422248,0.00000801839],"genre_scores_gemma":[0.9706662,0.00065509183,0.025533052,0.0028373394,0.00015712892,0.000082118386,0.0000034530767,0.00005374529,0.000011843282],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99752015,0.00009960772,0.00044114797,0.0005055402,0.00093241385,0.0005011156],"domain_scores_gemma":[0.9957696,0.0027293689,0.00007951804,0.0008147147,0.00024309423,0.0003636764],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0007110306,0.00027399257,0.00040143926,0.00008233248,0.00007849156,0.00008265262,0.0018977325,0.0000835216,0.000032392763],"category_scores_gemma":[0.023099653,0.00020875807,0.00019286867,0.0013585875,0.000049472404,0.00031111552,0.00061282935,0.00056639087,0.00008041334],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011904931,0.00046508166,0.43751034,0.01812414,0.0006595632,0.001053008,0.011079352,0.30010518,0.011447469,0.0031920646,0.1037074,0.11253736],"study_design_scores_gemma":[0.0028865978,0.0021847517,0.48077342,0.012702562,0.0002825493,0.0004623859,0.00011566399,0.19060156,0.030278977,0.00041532842,0.2738061,0.0054901303],"about_ca_topic_score_codex":0.000011720933,"about_ca_topic_score_gemma":9.1062986e-7,"teacher_disagreement_score":0.9180282,"about_ca_system_score_codex":0.000074531825,"about_ca_system_score_gemma":0.00012993642,"threshold_uncertainty_score":0.9851292},"labels":[],"label_agreement":null},{"id":"W3030475425","doi":"10.1007/s10664-020-09858-z","title":"The who, what, how of software engineering research: a socio-technical framework","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":90,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Social software engineering; Software peer review; Software development; Empirical research; Beneficiary; Personal software process; Software","score_opus":0.08362109294565391,"score_gpt":0.3358897120986377,"score_spread":0.2522686191529838,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3030475425","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0050639827,0.003128952,0.9764602,0.010920592,0.0007459611,0.0003450937,0.0000047026942,0.0033261147,0.000004379253],"genre_scores_gemma":[0.25836688,0.00068078033,0.73956597,0.00046282535,0.0006086036,0.00014838023,0.00000385509,0.00012453404,0.00003813852],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99647576,0.00012508813,0.0005225668,0.0007314382,0.001127786,0.0010173855],"domain_scores_gemma":[0.98838097,0.009636621,0.00013869653,0.0010893553,0.00027678514,0.00047756344],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0014109433,0.00042594215,0.000514979,0.00018024603,0.00030126292,0.00092249166,0.002546756,0.00040396414,0.000011675275],"category_scores_gemma":[0.022925375,0.00035611907,0.0002780344,0.0019587085,0.00011077169,0.0014512758,0.0011933433,0.002024962,0.00002162034],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00039491602,0.0009604874,0.079078354,0.005299305,0.0017760714,0.0013629735,0.024502816,0.24411151,0.00552921,0.14786272,0.2555035,0.23361814],"study_design_scores_gemma":[0.0007955385,0.0013333211,0.022813216,0.0018991844,0.00007115351,0.00014946036,0.00025156356,0.17820682,0.0058515146,0.004000217,0.78191274,0.002715291],"about_ca_topic_score_codex":0.000005986829,"about_ca_topic_score_gemma":4.5192098e-7,"teacher_disagreement_score":0.5264092,"about_ca_system_score_codex":0.00014898078,"about_ca_system_score_gemma":0.000116868665,"threshold_uncertainty_score":0.9998891},"labels":[],"label_agreement":null},{"id":"W3036270494","doi":"10.1007/s10664-021-09951-x","title":"Lags in the release, adoption, and propagation of npm vulnerability fixes","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Information and Cyber Security","field":"Computer Science","cited_by":56,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Japan Society for the Promotion of Science","keywords":"Vulnerability (computing); Empirical research; Vulnerability assessment; Software release life cycle; Software","score_opus":0.014491196281586089,"score_gpt":0.246677101234731,"score_spread":0.2321859049531449,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3036270494","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.47345817,0.00011873161,0.52532023,0.0007952777,0.000072037205,0.00006966795,0.0000011774162,0.00007355165,0.000091144786],"genre_scores_gemma":[0.9861366,0.0000068132713,0.013574349,0.00024649955,0.000016282173,0.000008130864,0.0000035409632,0.0000020347807,0.0000057224593],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9994193,0.000041741354,0.00017938907,0.00010921815,0.00015485915,0.00009550833],"domain_scores_gemma":[0.9995222,0.00016862995,0.000027431397,0.00019362582,0.00006156186,0.000026532152],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00027815672,0.00005664182,0.000078195284,0.000031624648,0.000032312702,0.000042833748,0.00014665959,0.00003585887,0.000005835728],"category_scores_gemma":[0.00039150243,0.000043806347,0.00002236883,0.00035814245,0.000012728054,0.0002842973,0.00006509606,0.00012657107,0.0000023017951],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000023162962,0.00072734413,0.65665895,0.001325342,0.000048192156,0.0000844536,0.054196674,0.0417157,0.0007663313,0.103423506,0.00354672,0.13748361],"study_design_scores_gemma":[0.00034841968,0.00004011228,0.8918744,0.00005534598,0.0000037515524,0.000038206083,0.000124995,0.09294318,0.0029818527,0.0011380963,0.010243132,0.00020855632],"about_ca_topic_score_codex":0.0000063315324,"about_ca_topic_score_gemma":0.0000050241956,"teacher_disagreement_score":0.51267844,"about_ca_system_score_codex":0.00001845714,"about_ca_system_score_gemma":0.00003080484,"threshold_uncertainty_score":0.1786371},"labels":[],"label_agreement":null},{"id":"W3048902333","doi":"10.1007/s10664-020-09822-x","title":"A study of the performance of general compressors on log files","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":30,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Compression ratio; Gas compressor; Lossless compression; Data compression; Computer science; Compression (physics); Volume (thermodynamics); Database; Real-time computing; Algorithm; Engineering; Physics","score_opus":0.026760541705675916,"score_gpt":0.2510870800200216,"score_spread":0.2243265383143457,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3048902333","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9837969,0.000035112767,0.01529492,0.0001581983,0.00028728068,0.0002441958,0.0000027594017,0.00016814818,0.000012467918],"genre_scores_gemma":[0.9969822,0.0000018980958,0.002794544,0.00012363607,0.00006551499,0.000013580095,3.7913884e-7,0.00000925441,0.000008980315],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9987916,0.000041314317,0.0003485644,0.0002550972,0.00038102924,0.0001824008],"domain_scores_gemma":[0.9990904,0.00018928446,0.000097402466,0.0004905441,0.000058366204,0.000074004856],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00012694026,0.00014171636,0.00027160053,0.00003617816,0.000052475283,0.000012143782,0.0008328629,0.000053361287,0.000005638911],"category_scores_gemma":[0.00022955023,0.00009158057,0.000092334834,0.00045885786,0.00002785239,0.00012106818,0.00030136868,0.00019492903,0.000006226423],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014965171,0.00021882833,0.7052243,0.00022149214,0.000034124634,0.0000022744084,0.0036804616,0.28889942,0.00020970065,0.00003256744,0.00048646436,0.00097543024],"study_design_scores_gemma":[0.0005774781,0.0009920516,0.8215243,0.00010797263,0.00001076541,0.0000034553107,0.00003720873,0.16725029,0.008628861,0.0000044424737,0.00064618787,0.0002169627],"about_ca_topic_score_codex":0.0000121828025,"about_ca_topic_score_gemma":3.521828e-7,"teacher_disagreement_score":0.12164914,"about_ca_system_score_codex":0.000022663482,"about_ca_system_score_gemma":0.000028932618,"threshold_uncertainty_score":0.37345475},"labels":[],"label_agreement":null},{"id":"W3080471220","doi":"10.1007/s10664-020-09840-9","title":"An empirical study of the characteristics of popular Minecraft mods","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Artificial Intelligence in Games","field":"Computer Science","cited_by":23,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Queen's University","funders":"","keywords":"Popularity; Remuneration; Context (archaeology); Mod; Empirical research; Empirical evidence","score_opus":0.0470913351693498,"score_gpt":0.312429986715034,"score_spread":0.2653386515456842,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3080471220","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6133212,0.000015664107,0.385793,0.000424193,0.00017516401,0.00013760389,0.000002982251,0.00012799536,0.000002207675],"genre_scores_gemma":[0.984865,7.174652e-7,0.014758085,0.00023958631,0.00010276303,0.000008078787,8.923723e-7,0.000018350778,0.000006491261],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983628,0.00007299167,0.0005400612,0.0003395809,0.0004460528,0.00023853031],"domain_scores_gemma":[0.99880093,0.00015418557,0.00012923023,0.0006421675,0.00010616994,0.000167311],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00015139657,0.0001745629,0.0003371467,0.000048410137,0.00004499111,0.0000330233,0.0013547742,0.000077341174,0.000010710467],"category_scores_gemma":[0.0010144686,0.00013448171,0.000102132064,0.00070448476,0.000041771276,0.00020411186,0.0003746084,0.00024207054,0.0000059191525],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010752077,0.0005112683,0.9598757,0.000063603606,0.000037103422,0.000015810241,0.0110601,0.0209024,0.0018949773,0.0001094388,0.00021068803,0.005308159],"study_design_scores_gemma":[0.00017599844,0.0009935267,0.5639244,0.000048402297,0.000028907572,0.0000052838245,0.0002692059,0.4232955,0.009141679,0.00011333591,0.0016008733,0.00040290307],"about_ca_topic_score_codex":0.000016290911,"about_ca_topic_score_gemma":0.000002002117,"teacher_disagreement_score":0.4023931,"about_ca_system_score_codex":0.000026067162,"about_ca_system_score_gemma":0.000040132472,"threshold_uncertainty_score":0.54840046},"labels":[],"label_agreement":null},{"id":"W3081943439","doi":"10.1007/s10664-020-09863-2","title":"CROKAGE: effective solution recommendation for programming tasks by leveraging crowd knowledge","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":21,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Computer science; Leverage (statistics); Code (set theory); Information retrieval; Task (project management); Relevance (law); Programming language; Artificial intelligence","score_opus":0.03546418867217926,"score_gpt":0.3080167954479544,"score_spread":0.27255260677577514,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3081943439","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.011046211,0.00036638117,0.983021,0.0019456176,0.00050218846,0.0008578616,0.000014071986,0.002240756,0.0000058728074],"genre_scores_gemma":[0.70134324,0.000005006192,0.29693666,0.00032823332,0.0004638433,0.0006796911,0.00009683302,0.00009956496,0.000046933357],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9978516,0.000054311433,0.00032854528,0.0007440279,0.0002671276,0.00075438473],"domain_scores_gemma":[0.996856,0.0022697872,0.00005812565,0.0002874682,0.0001560589,0.00037258794],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004746476,0.00031646257,0.00030170282,0.00013829287,0.00019457539,0.0002771814,0.0006547721,0.00015080317,0.000014963596],"category_scores_gemma":[0.0050083506,0.00034687502,0.00014768876,0.00091439084,0.00002123617,0.00060399115,0.00034933502,0.0004763439,0.000048304384],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007019431,0.00035446603,0.03840925,0.0015128321,0.00028191853,0.000025867143,0.0073632835,0.022167502,0.008529781,0.00019234687,0.09479993,0.82629263],"study_design_scores_gemma":[0.0014247413,0.0004889605,0.012180125,0.00011820893,0.000022494949,0.000014471834,0.000015798088,0.4538182,0.010730839,0.000039440016,0.52009064,0.0010560453],"about_ca_topic_score_codex":0.000008051482,"about_ca_topic_score_gemma":4.0519214e-7,"teacher_disagreement_score":0.82523656,"about_ca_system_score_codex":0.00032377258,"about_ca_system_score_gemma":0.00006271742,"threshold_uncertainty_score":0.9998983},"labels":[],"label_agreement":null},{"id":"W3082966820","doi":"10.1007/s10664-021-10005-5","title":"Understanding peer review of software engineering papers","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Novelty; Quality (philosophy); Computer science; Technical peer review; Peer review; Software technical review; Psychology; Medical education; Software; Engineering ethics; Software quality; Software development; Engineering; Medicine; Political science; Social psychology","score_opus":0.10291582984899653,"score_gpt":0.3172505489139484,"score_spread":0.21433471906495186,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3082966820","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.004221763,0.021297008,0.9671869,0.0009962153,0.0026138767,0.0006569602,0.000042597123,0.0029491999,0.00003545551],"genre_scores_gemma":[0.18600495,0.0064496403,0.80344176,0.0009238784,0.00095376413,0.0005051087,0.00047564416,0.0006742362,0.00057100167],"study_design_codex":"simulation_or_modeling","study_design_gemma":"systematic_review","domain_scores_codex":[0.99249893,0.00010004719,0.0013150823,0.0016480123,0.0030938925,0.0013440405],"domain_scores_gemma":[0.9919074,0.0037404017,0.00029910478,0.0025076636,0.00091831316,0.00062711],"candidate_categories":["metaresearch","metaepi_narrow","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.0017161806,0.0010729575,0.0016480639,0.00067208754,0.000099805016,0.00039787032,0.0029482555,0.000760555,0.00014686864],"category_scores_gemma":[0.03981585,0.001187156,0.0008390548,0.0017977128,0.000058170077,0.000438545,0.004130129,0.0027997934,0.000020381474],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001653861,0.0004723098,0.10265647,0.1548562,0.0019416186,0.0015997174,0.0021667983,0.69430304,0.000656747,0.0011283854,0.036954403,0.0032477928],"study_design_scores_gemma":[0.0042876573,0.00083378976,0.225594,0.3545582,0.0011355015,0.0011243582,0.00020000667,0.2254487,0.008421434,0.00067471806,0.15684381,0.020877851],"about_ca_topic_score_codex":0.000022838607,"about_ca_topic_score_gemma":0.0000012268567,"teacher_disagreement_score":0.4688543,"about_ca_system_score_codex":0.0013702348,"about_ca_system_score_gemma":0.0006825131,"threshold_uncertainty_score":0.9995008},"labels":[],"label_agreement":null},{"id":"W3084301791","doi":"10.1007/s10664-020-09864-1","title":"Automated demarcation of requirements in textual specifications: a machine learning-based approach","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":38,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"H2020 European Research Council; Natural Sciences and Engineering Research Council of Canada; Fonds National de la Recherche Luxembourg; European Commission","keywords":"Computer science; Software requirements specification; Requirements analysis; Formal specification; Markup language; System requirements specification; Simple (philosophy); Identifier; Task (project management); Requirements engineering; Variety (cybernetics); Software engineering; Enforcement; Artificial intelligence; Programming language; XML; Systems engineering; Engineering; Software; World Wide Web","score_opus":0.07854626844973958,"score_gpt":0.29834454275323014,"score_spread":0.21979827430349055,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3084301791","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.038468182,0.00021243813,0.9583296,0.00048562966,0.000051423583,0.00025044975,0.0000038146497,0.0021828008,0.00001563914],"genre_scores_gemma":[0.8554055,0.0000038655326,0.14436573,0.00007199433,0.00003735818,0.00005018683,0.000029703671,0.000028539402,0.000007081735],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99810874,0.00006604752,0.0004486494,0.0004712906,0.0005267445,0.0003785059],"domain_scores_gemma":[0.99847865,0.00079466373,0.00007408461,0.00036399698,0.0000901025,0.00019848782],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003524045,0.00019953554,0.00027316203,0.00028567013,0.00003668119,0.000050507122,0.0007699582,0.000099607634,0.000014416171],"category_scores_gemma":[0.004875594,0.00021187922,0.00007013707,0.0018585639,0.0000247518,0.00024909418,0.00018556068,0.0004273621,0.000026106223],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009632417,0.00012161963,0.25740123,0.00018011474,0.000016319036,0.000009894819,0.00046025886,0.7401083,0.0005458983,0.00010992965,0.0002431986,0.000793603],"study_design_scores_gemma":[0.00039247068,0.00007774652,0.12684986,0.000035698664,0.0000023651237,0.0000018772372,0.0000053558597,0.8707291,0.0007340831,0.000004831146,0.0009845165,0.00018208206],"about_ca_topic_score_codex":0.000016939546,"about_ca_topic_score_gemma":5.416901e-7,"teacher_disagreement_score":0.8169373,"about_ca_system_score_codex":0.00014730229,"about_ca_system_score_gemma":0.000093730516,"threshold_uncertainty_score":0.8640184},"labels":[],"label_agreement":null},{"id":"W3084421431","doi":"10.1007/s10664-020-09852-5","title":"Code cloning in smart contracts: a case study on verified contracts from the Ethereum blockchain platform","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":60,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; Queen's University","funders":"Japan Society for the Promotion of Science; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Smart contract; clone (Java method); Blockchain; Cryptocurrency; Code (set theory); Source code; Backward compatibility; Cloning (programming); Computer science; Order (exchange); Computer security; Business; Operating system; Programming language; Finance","score_opus":0.05194985740488874,"score_gpt":0.2930333489933576,"score_spread":0.24108349158846884,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3084421431","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.50095606,0.000058352103,0.4967632,0.00066820945,0.00014645711,0.0003702822,0.000006811245,0.0010240441,0.0000065877084],"genre_scores_gemma":[0.9717741,0.0000016764739,0.02602714,0.001932175,0.00013091337,0.00009376157,0.0000014149756,0.00003544126,0.0000033714925],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9982099,0.00006791125,0.00041832408,0.00056322064,0.00032473385,0.00041592665],"domain_scores_gemma":[0.9971827,0.0019880699,0.00009337935,0.0004946658,0.000043471882,0.00019770394],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00032149808,0.00028549045,0.00033445517,0.0000702322,0.00012016006,0.00012492725,0.0006179621,0.00012453782,0.000007936126],"category_scores_gemma":[0.0018779988,0.00023441251,0.00007302482,0.00056677556,0.00001962234,0.00022074862,0.00022321231,0.00085725007,0.000016808934],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00079590286,0.0024589382,0.15854685,0.00010326016,0.0006675537,0.06577554,0.116753764,0.5360863,0.010349857,0.0010275935,0.0031888066,0.10424563],"study_design_scores_gemma":[0.009263211,0.0036423865,0.13475738,0.0005467158,0.00008120931,0.0011680023,0.00448853,0.77153516,0.029800907,0.0011836459,0.039937068,0.0035957769],"about_ca_topic_score_codex":0.00027399813,"about_ca_topic_score_gemma":0.00010552303,"teacher_disagreement_score":0.47081804,"about_ca_system_score_codex":0.0001448006,"about_ca_system_score_gemma":0.000037748076,"threshold_uncertainty_score":0.95590645},"labels":[],"label_agreement":null},{"id":"W3091970108","doi":"10.1007/s10664-020-09851-6","title":"Publish or perish, but do not forget your software artifacts","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":42,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"Deutsche Forschungsgemeinschaft; Deutscher Akademischer Austauschdienst","keywords":"Artifact (error); Publish or perish; Computer science; Publication; Context (archaeology); Replication (statistics); Data science; Software; Empirical research; Open science; Software engineering; World Wide Web; Publishing; Artificial intelligence; Political science","score_opus":0.06981769528065791,"score_gpt":0.2983438529525711,"score_spread":0.2285261576719132,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3091970108","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.15515994,0.0001941859,0.8345214,0.003568618,0.0007709985,0.00034481447,0.000024231016,0.005401933,0.000013875503],"genre_scores_gemma":[0.67203087,0.000013979663,0.32400873,0.002231437,0.0008999843,0.00009690906,0.000021857652,0.00019830067,0.00049790146],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9953931,0.000057505764,0.0006233194,0.0012421485,0.001275584,0.001408344],"domain_scores_gemma":[0.9952504,0.0020608488,0.00008791624,0.0010498505,0.00026148424,0.0012894731],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00042250045,0.0006036046,0.0006102833,0.00027840433,0.00018300248,0.0009791852,0.0023675861,0.00030919892,0.00015295245],"category_scores_gemma":[0.024072984,0.0005535996,0.0002589301,0.0019578403,0.000048477203,0.0015329523,0.0012528055,0.0010950324,0.000372231],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00030084766,0.0005469616,0.60007864,0.0018816149,0.0006195218,0.0032997916,0.011492202,0.13944611,0.0027516864,0.00058936555,0.16073532,0.07825793],"study_design_scores_gemma":[0.003839673,0.0013150082,0.3571498,0.000474565,0.00007566138,0.00039026112,0.00019267552,0.17514504,0.0143807065,0.00019047078,0.4409731,0.0058730156],"about_ca_topic_score_codex":0.00001381516,"about_ca_topic_score_gemma":0.0000015802368,"teacher_disagreement_score":0.5168709,"about_ca_system_score_codex":0.00022320065,"about_ca_system_score_gemma":0.00026432308,"threshold_uncertainty_score":0.99969155},"labels":[],"label_agreement":null},{"id":"W3092232764","doi":"10.1007/s10664-020-09916-6","title":"What makes a popular academic AI repository?","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":17,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"License; Publishing; Set (abstract data type); Software; Empirical research; Inclusion (mineral)","score_opus":0.10157218383589905,"score_gpt":0.39141405928048384,"score_spread":0.2898418754445848,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3092232764","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.50336325,0.0033086645,0.4743167,0.0053178403,0.01252566,0.00014956729,0.000008375945,0.00081491435,0.00019504229],"genre_scores_gemma":[0.94587857,0.000061444196,0.02436164,0.0037288645,0.0011241826,0.000022935852,0.00004557776,0.000058754395,0.024718031],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9966963,0.000084412546,0.00060736254,0.0008923992,0.0013294857,0.00039006933],"domain_scores_gemma":[0.99762,0.0007879256,0.00007811757,0.0011034112,0.00019750593,0.00021305625],"candidate_categories":["metaresearch","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0015130036,0.00016889824,0.00025731506,0.00018464691,0.00017019971,0.0012477082,0.00078857294,0.000103715836,0.0001515596],"category_scores_gemma":[0.0088716205,0.0001423702,0.0001490109,0.0013500502,0.00002826646,0.0007349015,0.0006561973,0.00038092578,0.00033536396],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015458145,0.00014520103,0.26351017,0.000057833924,0.00009224902,0.0012211745,0.0016290791,0.07919599,0.0013292842,0.00071984367,0.5393986,0.11268508],"study_design_scores_gemma":[0.00024177619,0.00002185707,0.056862142,0.00014831603,0.000020577585,0.000110672336,0.00046646,0.027502503,0.0015536535,0.0018293267,0.9107945,0.00044822248],"about_ca_topic_score_codex":0.000003890807,"about_ca_topic_score_gemma":0.0000010809523,"teacher_disagreement_score":0.44995505,"about_ca_system_score_codex":0.000064484135,"about_ca_system_score_gemma":0.000052595697,"threshold_uncertainty_score":0.9997891},"labels":[],"label_agreement":null},{"id":"W3096162789","doi":"10.1007/s10664-020-09874-z","title":"A feature location approach for mapping application features extracted from crowd-based screencasts to source code","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Concordia University","funders":"","keywords":"Computer science; Source code; Codebase; Set (abstract data type); Program comprehension; Software; Workflow; Code (set theory); Information retrieval; Multimedia; World Wide Web; Human–computer interaction; Database; Software system; Programming language","score_opus":0.03464601524693783,"score_gpt":0.2754876655325143,"score_spread":0.24084165028557647,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3096162789","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.011674613,0.00025196414,0.9798812,0.0046437574,0.00012189131,0.0009545099,0.00003453668,0.0024351191,0.00000240086],"genre_scores_gemma":[0.37360057,5.191186e-7,0.62442523,0.0009870076,0.00036958963,0.000390178,0.00012871216,0.000064881584,0.000033326844],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9975409,0.000031301173,0.00026355454,0.00097452634,0.0005719211,0.00061782857],"domain_scores_gemma":[0.99708545,0.0014031727,0.00006396186,0.0006491372,0.00023452505,0.0005637685],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00023476566,0.00035333858,0.00033242727,0.00020972727,0.00012960227,0.00026042483,0.0011670359,0.00024249626,0.0000030597196],"category_scores_gemma":[0.0041127163,0.00037608648,0.00012321993,0.0018022272,0.000016056354,0.00028190704,0.00023400677,0.00052413053,0.000027465934],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000052482,0.000089877416,0.021017853,0.00033139772,0.000063367064,0.000006886034,0.0012118338,0.9361033,0.0062461556,0.000051150677,0.017360201,0.017465526],"study_design_scores_gemma":[0.0004954257,0.00008756268,0.10847461,0.000058910297,0.000009325399,0.0000035065157,0.000012499956,0.8477927,0.0026664098,0.000009812014,0.039878182,0.00051104557],"about_ca_topic_score_codex":0.000017302642,"about_ca_topic_score_gemma":9.870547e-7,"teacher_disagreement_score":0.36192596,"about_ca_system_score_codex":0.0001916232,"about_ca_system_score_gemma":0.000111173154,"threshold_uncertainty_score":0.9998691},"labels":[],"label_agreement":null},{"id":"W3118322826","doi":"10.1007/s10664-020-09921-9","title":"ID-correspondence: a measure for detecting evolutionary coupling","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Identifier; False positive paradox; Computer science; Similarity (geometry); Measure (data warehouse); Artificial intelligence; Coupling (piping); Data mining; Natural language processing; Machine learning; Programming language; Engineering","score_opus":0.032689246102338025,"score_gpt":0.29433416246353283,"score_spread":0.2616449163611948,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3118322826","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08055264,0.0015252469,0.9146821,0.00027913618,0.0009969594,0.00021057819,0.0000069253542,0.0017414173,0.0000049810837],"genre_scores_gemma":[0.6145052,0.0000065475456,0.3844753,0.00010978984,0.00042040943,0.00018225901,0.0000124732,0.000075951924,0.00021209517],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99743557,0.000023985083,0.00034099884,0.0007208653,0.0006782405,0.0008003131],"domain_scores_gemma":[0.9944706,0.0041423207,0.000043432177,0.0006833845,0.00037777197,0.00028251417],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005794277,0.00027787333,0.00029117052,0.00020724384,0.00021857109,0.0002041139,0.00077508687,0.00018086647,0.000024421788],"category_scores_gemma":[0.016026486,0.000308707,0.00017499684,0.0012175018,0.000019740215,0.00042915653,0.00040657193,0.0004994496,0.00003979545],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000055217897,0.00023140496,0.19634174,0.0007001081,0.00024300914,0.0006318436,0.0012372129,0.76039267,0.0117613925,0.0008857865,0.010207619,0.017311983],"study_design_scores_gemma":[0.0009773918,0.00013422128,0.08503464,0.0003086375,0.000020927268,0.00032249268,0.000028341346,0.8698115,0.010242132,0.00033217666,0.031687655,0.001099895],"about_ca_topic_score_codex":0.000003932977,"about_ca_topic_score_gemma":0.0000010909221,"teacher_disagreement_score":0.53395253,"about_ca_system_score_codex":0.000313225,"about_ca_system_score_gemma":0.00032617655,"threshold_uncertainty_score":0.9999365},"labels":[],"label_agreement":null},{"id":"W3119029787","doi":"10.1007/s10664-020-09893-w","title":"Demystifying the challenges and benefits of analyzing user-reported logs in bug reports","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Manitoba; Concordia University","funders":"","keywords":"Debugging; Computer science; Software bug; Security bug; Process (computing); Software engineering; World Wide Web; Data science; Software; Programming language; Computer security","score_opus":0.03840174201774539,"score_gpt":0.26340921822509084,"score_spread":0.22500747620734546,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3119029787","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9057561,0.014393839,0.078319415,0.00081649685,0.00037357525,0.00011181599,5.100559e-7,0.00021235482,0.000015867872],"genre_scores_gemma":[0.9886401,0.00036296365,0.010868802,0.000039566225,0.0000507428,0.0000151220975,0.0000010998308,0.000011790475,0.000009784892],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9984164,0.000045868288,0.00055211515,0.0004454886,0.00025661543,0.00028351034],"domain_scores_gemma":[0.9985342,0.0005309871,0.0001272027,0.0006194586,0.00010953374,0.00007864412],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007220297,0.00015831068,0.00031199207,0.000093709925,0.00005847064,0.00004008003,0.0002194976,0.00010726217,0.0000023941725],"category_scores_gemma":[0.0012779839,0.000118990465,0.00007069768,0.000553222,0.000027303215,0.00025492773,0.00029415629,0.00023351141,0.0000010135502],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000023737946,0.00005954798,0.94089204,0.00035960102,0.00005437132,0.00029697342,0.0021571824,0.031368915,0.000104161976,0.0003192146,0.00003229833,0.024353297],"study_design_scores_gemma":[0.00015681342,0.000022502336,0.99039143,0.0002987082,0.000011776827,0.00046441503,0.000046218753,0.005713766,0.0014509717,0.00026360285,0.0009557644,0.00022401447],"about_ca_topic_score_codex":0.000021205362,"about_ca_topic_score_gemma":0.000018997369,"teacher_disagreement_score":0.082884,"about_ca_system_score_codex":0.000046903086,"about_ca_system_score_gemma":0.000066431305,"threshold_uncertainty_score":0.48522902},"labels":[],"label_agreement":null},{"id":"W3119686444","doi":"10.1007/s10664-021-10024-2","title":"A pragmatic approach for hyper-parameter tuning in search-based test case generation","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Metric (unit); Computer science; Heuristic; Domain (mathematical analysis); Fine-tuning; Class (philosophy); Test case; Parameter space; Performance metric; Mathematical optimization; Machine learning; Artificial intelligence; Mathematics; Statistics; Engineering","score_opus":0.06445494654789755,"score_gpt":0.31047606952561757,"score_spread":0.24602112297772002,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3119686444","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19326378,0.00043533547,0.8038024,0.0001511561,0.00044791755,0.00097039266,0.000022149627,0.00090439135,0.00000250882],"genre_scores_gemma":[0.43723512,0.0000030364968,0.56148326,0.000068659225,0.00021614022,0.0007309055,0.0001621339,0.00008470425,0.000016061495],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99602854,0.00012727454,0.00069243344,0.0014087127,0.0007257101,0.0010173558],"domain_scores_gemma":[0.99132174,0.0066214483,0.0000850461,0.0013684768,0.00026664473,0.00033665812],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001151533,0.0006032107,0.0007210156,0.0007401615,0.00011154279,0.0008039682,0.0011344131,0.0005431077,0.0000089358955],"category_scores_gemma":[0.0152959945,0.0006659634,0.00029509282,0.0010575011,0.000035351877,0.00031938255,0.0012231993,0.0017587885,0.0000046368923],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000027066762,0.00019818655,0.01829114,0.0013062299,0.000035553945,0.0007850033,0.0005943097,0.9758859,0.0002936911,0.000008718611,0.00017738607,0.0024211546],"study_design_scores_gemma":[0.0005149324,0.00006858459,0.002486774,0.00028886384,0.000014143657,0.00027713456,0.00001341634,0.9948089,0.0006935905,0.000011091976,0.00013613558,0.0006864156],"about_ca_topic_score_codex":0.00007104901,"about_ca_topic_score_gemma":0.000008229496,"teacher_disagreement_score":0.24397132,"about_ca_system_score_codex":0.00069562986,"about_ca_system_score_gemma":0.00056859426,"threshold_uncertainty_score":0.99957913},"labels":[],"label_agreement":null},{"id":"W3119769340","doi":"10.1007/s10664-020-09917-5","title":"An exploratory study on the introduction and removal of different types of technical debt in deep learning frameworks","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Technical debt; Debt; Deep learning; Exploratory research; Quality (philosophy); Field (mathematics)","score_opus":0.018401164916278124,"score_gpt":0.2802219691050121,"score_spread":0.261820804188734,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3119769340","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.69905984,0.00020361258,0.30020908,0.00017399449,0.00011025506,0.00009774831,1.9350898e-7,0.00014384955,0.000001417217],"genre_scores_gemma":[0.9897897,0.000010474074,0.01005917,0.000013532957,0.000082100756,0.000021965661,0.0000010377211,0.000018372939,0.0000036215279],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99862367,0.0001415627,0.00027282338,0.00035818038,0.0003794992,0.000224252],"domain_scores_gemma":[0.9976185,0.0016720922,0.00003916859,0.00050746807,0.00008691233,0.00007586972],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00046600695,0.0001427889,0.00025144592,0.00015656397,0.000033686694,0.0000358693,0.0003597004,0.00012885603,0.000007198003],"category_scores_gemma":[0.004478796,0.00011251809,0.000038000253,0.00069512735,0.00003203165,0.00013506683,0.0002434662,0.000907081,0.0000010632554],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000033561417,0.0010952053,0.75644964,0.00014085832,0.00007239396,0.00023527975,0.00592004,0.20962074,0.010305325,0.0014527474,0.0000464684,0.01462777],"study_design_scores_gemma":[0.00033607872,0.00060326926,0.9496832,0.00011523418,0.000008833403,0.00004995941,0.00033124108,0.04133167,0.0069565265,0.00018206009,0.0001399996,0.000261935],"about_ca_topic_score_codex":0.000002243522,"about_ca_topic_score_gemma":0.0000051795264,"teacher_disagreement_score":0.29072985,"about_ca_system_score_codex":0.000069014015,"about_ca_system_score_gemma":0.000026333004,"threshold_uncertainty_score":0.5361861},"labels":[],"label_agreement":null},{"id":"W3121596715","doi":"10.1007/s10664-017-9521-5","title":"Do developers update their library dependencies?","year":2017,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":335,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Japan Society for the Promotion of Science","keywords":"Reuse; Dependency (UML); Computer science; Workload; Exploit; Software; World Wide Web; Software engineering; Data science; Computer security; Engineering","score_opus":0.03368879006535396,"score_gpt":0.27991691801105534,"score_spread":0.24622812794570137,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3121596715","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.21668161,0.00045470783,0.777249,0.0013577598,0.0010100744,0.00017537629,0.000007910457,0.0029037553,0.00015983084],"genre_scores_gemma":[0.8369367,0.00004444676,0.16216742,0.00023415107,0.00023668182,0.000043431166,0.0000052616842,0.00007974629,0.00025217998],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.997673,0.000025676165,0.00028270847,0.0006737908,0.00052283605,0.00082203076],"domain_scores_gemma":[0.9967326,0.00082471996,0.000073098774,0.0019059692,0.00006116453,0.00040247722],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00032825684,0.0003475395,0.00030134912,0.00020871687,0.00042646914,0.0013960644,0.0038015046,0.0001689659,0.00008755517],"category_scores_gemma":[0.0036537985,0.00031180592,0.0001342911,0.0003464165,0.000056929657,0.002070071,0.0018528024,0.0005723782,0.0002911898],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008786748,0.000063160944,0.93910104,0.000115517425,0.00011915199,0.0003765228,0.0006863118,0.0071769566,0.00015157694,0.0016504657,0.012605481,0.03794502],"study_design_scores_gemma":[0.00041684235,0.000046346864,0.8815826,0.000110096706,0.0000047262242,0.00007426986,0.000007898125,0.013692452,0.0035353405,0.0004270641,0.099232346,0.0008700054],"about_ca_topic_score_codex":0.000005230858,"about_ca_topic_score_gemma":6.6753535e-7,"teacher_disagreement_score":0.62025505,"about_ca_system_score_codex":0.00009624911,"about_ca_system_score_gemma":0.0001634188,"threshold_uncertainty_score":0.9999334},"labels":[],"label_agreement":null},{"id":"W3123086775","doi":"10.1007/s10664-021-09982-4","title":"Can Offline Testing of Deep Neural Networks Replace Their Online Testing?","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"H2020 European Research Council; Canada Research Chairs; Ministry of Education; National Research Foundation of Korea; Fonds National de la Recherche Luxembourg; National Research Foundation; Natural Sciences and Engineering Research Council of Canada; European Commission","keywords":"Computer science; Online and offline; Test strategy; Manual testing; Orthogonal array testing; White-box testing; Context (archaeology); Black-box testing; Exploit; Model-based testing; Machine learning; Artificial intelligence; Test case; Computer security; Operating system; Software","score_opus":0.03600581497936919,"score_gpt":0.27307237304567505,"score_spread":0.23706655806630586,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3123086775","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.11019496,0.0007748669,0.8853611,0.00036108086,0.0016483072,0.00022945594,0.000012803371,0.0014016767,0.000015768608],"genre_scores_gemma":[0.50988925,0.0000025799072,0.48920652,0.00013935515,0.0005971879,0.000014409093,0.00006449842,0.00007599666,0.000010210311],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99618363,0.00017513825,0.0010017881,0.001276819,0.00055236195,0.00081028277],"domain_scores_gemma":[0.99346703,0.0036597454,0.00055616716,0.0015359082,0.0004973536,0.0002837669],"candidate_categories":["metaresearch","metaepi_narrow","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.00058153394,0.00073243066,0.0010242402,0.00022442869,0.00013641667,0.00022917507,0.0020289295,0.0005134956,0.000011255699],"category_scores_gemma":[0.016020365,0.00073760055,0.0002700669,0.0014902491,0.00005888531,0.00020977657,0.0049253483,0.0030115577,6.539402e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000032976482,0.000064674234,0.052532364,0.0002136478,0.000056481214,0.000104396604,0.00034633686,0.92391664,0.000035088167,0.000009757418,0.000018074114,0.022699248],"study_design_scores_gemma":[0.00021506209,0.00006277485,0.022934655,0.0004361886,0.000030680807,0.0000675534,0.000018687431,0.97539455,0.000056681674,0.00006698829,0.00006459967,0.00065155677],"about_ca_topic_score_codex":0.0001880452,"about_ca_topic_score_gemma":0.000024171104,"teacher_disagreement_score":0.3996943,"about_ca_system_score_codex":0.00023808396,"about_ca_system_score_gemma":0.0002080832,"threshold_uncertainty_score":0.9995075},"labels":[],"label_agreement":null},{"id":"W3123088576","doi":"10.1007/s10664-020-09900-0","title":"Investigating design anti-pattern and design pattern mutations and their change- and fault-proneness","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Computer Research Institute of Montréal; Polytechnique Montréal","funders":"","keywords":"Software design pattern; Structural pattern; Software design; Computer science; Design pattern; Software evolution; Software quality; Software; Software development; Software engineering; Software construction; Programming language","score_opus":0.09046305438679865,"score_gpt":0.2872581248634783,"score_spread":0.19679507047667966,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3123088576","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.28225225,0.0013441229,0.71518075,0.00045749752,0.000112896356,0.00019843914,0.0000038406406,0.0004498941,2.9730265e-7],"genre_scores_gemma":[0.866912,0.00006394124,0.13251781,0.00024214327,0.00010318292,0.000100012374,0.0000050962403,0.000047644393,0.00000816315],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9982318,0.0001128608,0.0002418454,0.0006563077,0.0002508579,0.0005063264],"domain_scores_gemma":[0.9963823,0.0027842156,0.000038201637,0.00035459027,0.00009672813,0.00034394304],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.000424442,0.00030363398,0.00030319486,0.00016271044,0.00016987069,0.00037260551,0.00027293593,0.00012320445,0.0000041438707],"category_scores_gemma":[0.0018981193,0.000291549,0.000030442841,0.00052656565,0.00006954762,0.00049607776,0.00054621644,0.0003487967,0.0000035936296],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000015435724,0.000058959005,0.72022855,0.0005325508,0.000109790744,0.00025003953,0.008224299,0.026457407,0.0033802318,0.000026315836,0.00014752461,0.24058278],"study_design_scores_gemma":[0.00045852052,0.00008125293,0.49510875,0.0002480244,0.0000113023025,0.0002954906,0.00005010016,0.49615085,0.0065697385,0.00021019595,0.00021375237,0.0006020092],"about_ca_topic_score_codex":0.000024066754,"about_ca_topic_score_gemma":0.0000013812349,"teacher_disagreement_score":0.58465976,"about_ca_system_score_codex":0.000046799938,"about_ca_system_score_gemma":0.00006160336,"threshold_uncertainty_score":0.9999537},"labels":[],"label_agreement":null},{"id":"W3129150043","doi":"10.1007/s10664-020-09902-y","title":"On the Removal of Feature Toggles","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"","keywords":"Python (programming language); Feature (linguistics); Computer science; Source code; Cyclomatic complexity; Software; Data science; Software engineering; Programming language","score_opus":0.024005836158592978,"score_gpt":0.27051804039179844,"score_spread":0.24651220423320547,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3129150043","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19969119,0.0012489241,0.79430795,0.0031905062,0.000565761,0.000110622816,0.0000043000596,0.0007155007,0.00016527563],"genre_scores_gemma":[0.8462153,0.000014910434,0.15204236,0.0005251983,0.0001865686,0.000027062155,0.000004554532,0.00004982993,0.0009342268],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99851954,0.000043988104,0.00017344234,0.00034494337,0.00055066246,0.00036744497],"domain_scores_gemma":[0.9949041,0.0039969673,0.000028970791,0.0008266402,0.00012882818,0.00011450289],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00026914716,0.00017277407,0.00019983396,0.000088605106,0.0000611681,0.000077076744,0.0008107423,0.000101638216,0.000041549414],"category_scores_gemma":[0.007912821,0.00012900421,0.00012420423,0.000948319,0.000025018779,0.000114942995,0.00034844992,0.00046939,0.00003700053],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000060660757,0.00096705515,0.10912358,0.0010124368,0.0007867107,0.007941659,0.0041550556,0.30660534,0.014482484,0.29487327,0.18806638,0.07192537],"study_design_scores_gemma":[0.0016042285,0.0005441995,0.40284845,0.0010485757,0.000039593517,0.0021063956,0.00006797056,0.12482657,0.113718726,0.0071799997,0.3435898,0.002425477],"about_ca_topic_score_codex":0.000001992782,"about_ca_topic_score_gemma":3.09542e-7,"teacher_disagreement_score":0.6465241,"about_ca_system_score_codex":0.00006716131,"about_ca_system_score_gemma":0.000092980314,"threshold_uncertainty_score":0.9472958},"labels":[],"label_agreement":null},{"id":"W3130317758","doi":"10.1007/s10664-022-10170-1","title":"Optimal priority assignment for real-time systems: a coevolution-based approach","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Real-Time Systems Scheduling","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"European Research Council; Natural Sciences and Engineering Research Council of Canada; European Commission; Université du Luxembourg","keywords":"Coevolution; Computer science; Real-time computing; Biology","score_opus":0.01832058550148069,"score_gpt":0.24808763269531386,"score_spread":0.22976704719383317,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3130317758","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.02004353,0.0001985398,0.97653884,0.000109802655,0.00070171966,0.000961532,0.000022875853,0.0013533593,0.00006982269],"genre_scores_gemma":[0.510075,5.6675594e-7,0.48832518,0.000039647046,0.0002045472,0.0011226182,0.000035970195,0.000051117422,0.00014539922],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9971501,0.00014976307,0.00052098103,0.00074377033,0.0007704872,0.00066487055],"domain_scores_gemma":[0.99807155,0.00073628343,0.00014797834,0.0007074129,0.00009005465,0.00024672766],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0011424893,0.00030093433,0.0004321278,0.0001820768,0.00039156745,0.00026800964,0.0009808241,0.00009510026,0.000014228093],"category_scores_gemma":[0.000331659,0.00032146278,0.00019894534,0.0006344632,0.000017166747,0.00035555311,0.0004020407,0.00032032182,0.00001845042],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009447601,0.000105740764,0.0011588023,0.00014798014,0.000033132816,0.000009175381,0.00022035028,0.99588364,0.0005837253,0.0003646215,0.0012877389,0.00019563352],"study_design_scores_gemma":[0.00051368645,0.00015938228,0.0010657546,0.00002627622,0.000012278238,0.000032670298,0.000028341894,0.992204,0.00007390445,0.000011162117,0.0054853167,0.00038720752],"about_ca_topic_score_codex":0.000049971473,"about_ca_topic_score_gemma":7.088641e-8,"teacher_disagreement_score":0.49003142,"about_ca_system_score_codex":0.00091921486,"about_ca_system_score_gemma":0.0002505371,"threshold_uncertainty_score":0.99992377},"labels":[],"label_agreement":null},{"id":"W3134065202","doi":"10.1007/s10664-020-09904-w","title":"Helping or not helping? Why and how trivial packages impact the npm ecosystem","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Publication; Computer science; JavaScript; Documentation; World Wide Web; Overhead (engineering); Publishing; Dependency (UML); Software engineering; Operating system; Business; Advertising","score_opus":0.03123396730809244,"score_gpt":0.2809393432019691,"score_spread":0.24970537589387667,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3134065202","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.17984155,0.00024049832,0.810628,0.007757902,0.00055211846,0.00019553084,0.000018787265,0.00075562525,0.000010027229],"genre_scores_gemma":[0.94763917,0.000010533571,0.05026085,0.0013703041,0.00039652167,0.000041451105,0.0000076541055,0.000048414095,0.00022511854],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9980165,0.00010620668,0.00029677307,0.00057192903,0.00044205046,0.00056657393],"domain_scores_gemma":[0.9973149,0.001454904,0.00009384637,0.0007523074,0.00018942723,0.00019463668],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00049322826,0.00031624056,0.00035934962,0.00012898142,0.00031574685,0.0009549832,0.00071734574,0.00013111814,0.00003385797],"category_scores_gemma":[0.002615432,0.00021902149,0.00014221694,0.0014621554,0.000030167817,0.000650103,0.0006466437,0.00044111273,0.000021600452],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00023542058,0.0009102067,0.53424233,0.0022679819,0.0026646107,0.0044940594,0.02446955,0.084483005,0.014671871,0.009985225,0.14734437,0.17423138],"study_design_scores_gemma":[0.0034747904,0.00069641456,0.39451078,0.0012770299,0.00014606222,0.003118971,0.00054438907,0.17229663,0.018482897,0.0004081144,0.40100527,0.0040386715],"about_ca_topic_score_codex":0.00000939763,"about_ca_topic_score_gemma":0.000016093341,"teacher_disagreement_score":0.7677976,"about_ca_system_score_codex":0.0001627881,"about_ca_system_score_gemma":0.00018705154,"threshold_uncertainty_score":0.9208925},"labels":[],"label_agreement":null},{"id":"W3135032580","doi":"10.1007/s10664-020-09913-9","title":"Software product-line evaluation in the large","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"Saab; Vetenskapsrådet; Deutsche Forschungsgemeinschaft; Deutscher Akademischer Austauschdienst","keywords":"Domain engineering; Software product line; Software engineering; Product (mathematics); Domain (mathematical analysis); New product development; Software development; Product engineering; Systems engineering; Computer science; Process management; Software; Engineering; Risk analysis (engineering); Product design; Software construction; Business; Marketing","score_opus":0.07412629567025701,"score_gpt":0.34508034406743116,"score_spread":0.27095404839717413,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3135032580","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.01938204,0.0020427955,0.9753402,0.0010033024,0.0008328577,0.00028375455,0.000004120418,0.0010981433,0.000012805488],"genre_scores_gemma":[0.14745465,0.000023719625,0.8514238,0.00058058376,0.0002487252,0.0001563844,0.000022552276,0.000037091417,0.000052497773],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99723107,0.0003082116,0.000389821,0.0006832488,0.0007510889,0.00063657487],"domain_scores_gemma":[0.9960898,0.002481279,0.000056403536,0.0010676275,0.00022075257,0.00008414575],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0022472222,0.00028436322,0.0003017797,0.00014579491,0.000099274206,0.00012977888,0.0009723552,0.00010556767,0.000022141794],"category_scores_gemma":[0.023475714,0.00023511544,0.0001070279,0.0015834807,0.000018429655,0.00050258456,0.00037430192,0.000580352,0.000027939257],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004371956,0.00019369008,0.016401982,0.00010883255,0.00003100831,0.0002599955,0.0020376092,0.93810284,0.00025554642,0.0010780768,0.0011740745,0.040351946],"study_design_scores_gemma":[0.0045106704,0.00033631665,0.5058439,0.0005850985,0.00011977986,0.00095066323,0.00035998336,0.3158521,0.015190411,0.030894544,0.1213017,0.0040548523],"about_ca_topic_score_codex":0.0000021522546,"about_ca_topic_score_gemma":0.0000056419653,"teacher_disagreement_score":0.6222508,"about_ca_system_score_codex":0.00020795311,"about_ca_system_score_gemma":0.00017328293,"threshold_uncertainty_score":0.98475},"labels":[],"label_agreement":null},{"id":"W3135374914","doi":"10.1007/s10664-020-09892-x","title":"variED: an editor for collaborative, real-time feature modeling","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"Deutsche Forschungsgemeinschaft; Deutscher Akademischer Austauschdienst","keywords":"Computer science; Feature model; Usability; Feature (linguistics); Software engineering; Software product line; Merge (version control); Software; Artifact (error); Human–computer interaction; Software development; Artificial intelligence; Information retrieval; Programming language","score_opus":0.040571527766392786,"score_gpt":0.3121451311731008,"score_spread":0.271573603406708,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3135374914","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0033802877,0.0003258968,0.9895776,0.0004050939,0.003010292,0.00022151704,0.00002538725,0.0030405761,0.000013382637],"genre_scores_gemma":[0.0017106843,0.000026022697,0.99585176,0.00011594043,0.0018227078,0.00010401695,0.000041385843,0.00007267569,0.00025481835],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978475,0.00008493866,0.00029653753,0.0007977695,0.00033587293,0.0006373939],"domain_scores_gemma":[0.99654096,0.0018756139,0.000055334825,0.00081253593,0.00044192068,0.0002736232],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004165357,0.00036250183,0.00044561783,0.00011605669,0.0001593322,0.0002016403,0.00071944925,0.00026249106,0.000007994229],"category_scores_gemma":[0.005127067,0.00037868836,0.00012789687,0.00094540213,0.00001829435,0.0008607668,0.00029037526,0.00035533978,0.00001231583],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008458158,0.000041535448,0.000087384105,0.000060386003,0.000035231526,0.000036093614,0.0003691102,0.9903228,0.0030247343,0.00070492056,0.0036206366,0.0016886916],"study_design_scores_gemma":[0.0005383919,0.00016120805,0.00037919448,0.00007106029,0.000020845164,0.00003401275,0.000034247958,0.9668662,0.0037440162,0.0031306385,0.024100123,0.0009200922],"about_ca_topic_score_codex":0.0000017414503,"about_ca_topic_score_gemma":6.7141144e-7,"teacher_disagreement_score":0.023456648,"about_ca_system_score_codex":0.0001562439,"about_ca_system_score_gemma":0.00021166417,"threshold_uncertainty_score":0.9998665},"labels":[],"label_agreement":null},{"id":"W3135659312","doi":"10.1007/s10664-020-09918-4","title":"Wikifying software artifacts","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Wikis in Education and Collaboration","field":"Social Sciences","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Leverage (statistics); Software; Precision and recall; Recall; Domain (mathematical analysis); Data mining; Artificial intelligence; Machine learning; Information retrieval; Software engineering; Programming language","score_opus":0.03677555461118149,"score_gpt":0.34255519330012735,"score_spread":0.30577963868894587,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3135659312","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6658956,0.0013345465,0.30976936,0.010178177,0.00530402,0.00032183863,0.00000997332,0.0025134205,0.004673054],"genre_scores_gemma":[0.9349603,0.000060927538,0.05774019,0.001481128,0.0013477381,0.000042359068,0.00003155328,0.00004093284,0.0042948676],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9988463,0.00006672996,0.0001761749,0.00022826204,0.00035486533,0.00032766897],"domain_scores_gemma":[0.99904364,0.0003269376,0.00003307857,0.00017282597,0.0002198055,0.00020369471],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00024248842,0.0001047681,0.00012219467,0.000052738756,0.0003232239,0.00015276897,0.0001259097,0.00011545908,0.0007405031],"category_scores_gemma":[0.0055386466,0.00012045939,0.00005748063,0.00078787684,0.00003434901,0.0002486738,0.000037480255,0.00017283687,0.00015764056],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000019853205,0.00061949494,0.7150834,0.00020509871,0.00015143653,0.00020212046,0.062326077,0.021693636,0.0009736756,0.014471614,0.08932918,0.0949244],"study_design_scores_gemma":[0.00013653268,0.000009275961,0.04180263,0.00004302193,0.000011364395,0.0000033419926,0.0014226137,0.00011007493,0.0008895327,0.00043273883,0.95484275,0.00029611742],"about_ca_topic_score_codex":0.000035347533,"about_ca_topic_score_gemma":0.00009598657,"teacher_disagreement_score":0.86551356,"about_ca_system_score_codex":0.00019790899,"about_ca_system_score_gemma":0.00047044075,"threshold_uncertainty_score":0.81079894},"labels":[],"label_agreement":null},{"id":"W3137668562","doi":"10.1007/s10664-020-09929-1","title":"Release synchronization in software ecosystems","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Ecosystem; Context (archaeology); Synchronization (alternating current); Project team; Computer science; Knowledge management; Ecology; Telecommunications; Geography","score_opus":0.015028767916721806,"score_gpt":0.25037503824282314,"score_spread":0.23534627032610134,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3137668562","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059201494,0.00040049994,0.9377848,0.0006278353,0.00061484106,0.00014878355,0.0000057899783,0.0011797112,0.000036227175],"genre_scores_gemma":[0.68066967,0.0000088153165,0.3182804,0.00050345296,0.00017376743,0.00005909436,0.000049803915,0.00005341424,0.00020157956],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99797755,0.00006358847,0.00050064205,0.0005916394,0.00038538934,0.0004811567],"domain_scores_gemma":[0.99832183,0.0004861716,0.00007224892,0.000729661,0.00023220664,0.00015789461],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00025757262,0.00023422287,0.00028329546,0.00024075297,0.000093178016,0.00023367522,0.0005935466,0.00015404195,0.000053752832],"category_scores_gemma":[0.0032617247,0.00026735873,0.00008106125,0.0027054932,0.00001325931,0.00067757315,0.00036729098,0.00037235898,0.00014712989],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004779279,0.00038319,0.74027187,0.00026396592,0.00006722378,0.0012008175,0.0015509849,0.21603951,0.00024390825,0.0039525284,0.009055483,0.026965717],"study_design_scores_gemma":[0.0026304452,0.00015139146,0.2682121,0.0008778855,0.000030399215,0.0006416562,0.00009575959,0.5327619,0.004033465,0.00195851,0.18556671,0.0030398173],"about_ca_topic_score_codex":0.000021071444,"about_ca_topic_score_gemma":0.000028820523,"teacher_disagreement_score":0.6214682,"about_ca_system_score_codex":0.0003371067,"about_ca_system_score_gemma":0.00024253235,"threshold_uncertainty_score":0.9999779},"labels":[],"label_agreement":null},{"id":"W3139738689","doi":"10.1007/s10664-021-10028-y","title":"An exploratory study on the repeatedly shared external links on Stack Overflow","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Queen's University","funders":"","keywords":"Computer science; Stack (abstract data type); Point (geometry); World Wide Web; Operating system","score_opus":0.05440002308055616,"score_gpt":0.29818777125471374,"score_spread":0.24378774817415758,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3139738689","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.64333373,0.0003104389,0.35137,0.0002936302,0.002824609,0.0005274331,0.000014329101,0.0012941007,0.000031748365],"genre_scores_gemma":[0.9916154,0.000007551888,0.006533143,0.000533445,0.00085451803,0.00026200557,0.000020089778,0.00008814635,0.00008566494],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99602807,0.000355167,0.0005814879,0.0014279495,0.0010353451,0.0005719616],"domain_scores_gemma":[0.99597937,0.00068584044,0.00016121338,0.002743555,0.00013218871,0.0002978568],"candidate_categories":["metaepi_narrow","scholarly_communication","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.000808654,0.00062567176,0.00059014163,0.00019948423,0.00021228533,0.0012052689,0.0022335805,0.00054198614,0.000026372814],"category_scores_gemma":[0.000619069,0.00047926191,0.00025316968,0.00039671594,0.000017846582,0.0002883556,0.0011180734,0.0027233548,0.000068406174],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009210194,0.004200627,0.15015128,0.00049285695,0.00085449,0.0027634832,0.11193382,0.70734364,0.0012269089,0.0005415698,0.013404069,0.0069951625],"study_design_scores_gemma":[0.0031402947,0.006171983,0.39272246,0.01096462,0.00014583353,0.00012737274,0.004098247,0.5543502,0.0064820074,0.0003272712,0.013144692,0.008325038],"about_ca_topic_score_codex":0.000029414532,"about_ca_topic_score_gemma":0.000006608829,"teacher_disagreement_score":0.3482817,"about_ca_system_score_codex":0.00031866293,"about_ca_system_score_gemma":0.00016258356,"threshold_uncertainty_score":0.99983156},"labels":[],"label_agreement":null},{"id":"W3147362533","doi":"10.1007/s10664-021-09944-w","title":"Revisiting the VCCFinder approach for the identification of vulnerability-contributing commits","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec à Montréal","funders":"Fonds National de la Recherche Luxembourg; European Commission","keywords":"Commit; Computer science; Identification (biology); Vulnerability (computing); Replicate; Artificial intelligence; Machine learning; Replication (statistics); Software; Set (abstract data type); Software deployment; Data science; Software engineering; Computer security; Programming language; Database","score_opus":0.03679435711364231,"score_gpt":0.3062734892830032,"score_spread":0.26947913216936087,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3147362533","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0221562,0.0014173784,0.97382563,0.0016221863,0.00029393297,0.00034510266,0.000007873459,0.00032631346,0.0000053768786],"genre_scores_gemma":[0.92702097,0.000009736954,0.07230441,0.00009087668,0.00032199427,0.00015622537,0.000011594973,0.000028236176,0.00005595777],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99797004,0.00009664859,0.00054119254,0.00041928331,0.00046752594,0.00050531223],"domain_scores_gemma":[0.987686,0.010754251,0.00010610746,0.0010291211,0.000352231,0.000072318464],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0025549715,0.00017252378,0.00023633875,0.00005479828,0.0003298402,0.00020137854,0.0011237971,0.00008295929,0.0000037918403],"category_scores_gemma":[0.01932131,0.000117420066,0.00017331776,0.0007969289,0.000044267294,0.00019065135,0.00045905387,0.0004402239,0.0000028091447],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013290309,0.00013723089,0.1903456,0.0010002261,0.00033463156,0.000012950165,0.0022753666,0.71304864,0.0047160084,0.004848188,0.001226254,0.082041614],"study_design_scores_gemma":[0.0003915092,0.000018673925,0.23079439,0.00006635732,0.000025667536,0.00003921447,0.00008939335,0.74611235,0.015415678,0.00022061999,0.0064964043,0.0003297637],"about_ca_topic_score_codex":0.0000050781946,"about_ca_topic_score_gemma":1.881656e-7,"teacher_disagreement_score":0.9048648,"about_ca_system_score_codex":0.0000937538,"about_ca_system_score_gemma":0.000083871484,"threshold_uncertainty_score":0.98893934},"labels":[],"label_agreement":null},{"id":"W3152918650","doi":"10.1007/s10664-023-10314-x","title":"Evaluating pre-trained models for user feedback analysis in software engineering: a study on classification of app-reviews","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia, Okanagan Campus; University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Machine learning; Artificial intelligence; F1 score; Macro; Set (abstract data type); Task (project management); Binary classification; Support vector machine; Data mining; Engineering","score_opus":0.15107932163450608,"score_gpt":0.40137155726348567,"score_spread":0.2502922356289796,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3152918650","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39833373,0.00010553124,0.59895813,0.0000570318,0.00016911297,0.0011875121,0.000011027163,0.0011771664,7.6200143e-7],"genre_scores_gemma":[0.85802376,0.000012178333,0.14040406,0.000022587627,0.00009277896,0.0012133574,0.000042390424,0.000094483985,0.00009441034],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9958406,0.000114052506,0.0010784331,0.0010543858,0.0010031634,0.00090934575],"domain_scores_gemma":[0.9932325,0.004889186,0.00016610813,0.0012573322,0.00021301369,0.0002418632],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0025761563,0.0004885182,0.00095676346,0.0018413303,0.00007305111,0.000117351796,0.0012919693,0.0001962051,0.000007067131],"category_scores_gemma":[0.0129050445,0.00048835913,0.00040385642,0.0074360543,0.00001887842,0.00045975865,0.00033849516,0.0004916502,0.000028172972],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018803876,0.00020744128,0.10994871,0.00023437466,0.00018236668,0.000007519881,0.0013471094,0.8845833,0.0003143036,0.00008745297,0.00022063272,0.0028479944],"study_design_scores_gemma":[0.0005524695,0.00024670747,0.40958425,0.00009286507,0.000049061284,7.547147e-7,0.000017280667,0.58882785,0.000085171676,0.000032247583,0.00020428949,0.00030705123],"about_ca_topic_score_codex":0.00001813176,"about_ca_topic_score_gemma":0.000005851226,"teacher_disagreement_score":0.45969003,"about_ca_system_score_codex":0.00032396102,"about_ca_system_score_gemma":0.00009707417,"threshold_uncertainty_score":0.9997568},"labels":[],"label_agreement":null},{"id":"W3157341372","doi":"10.1007/s10664-021-09977-1","title":"Individual differences limit predicting well-being and productivity using software repositories: a longitudinal industrial study","year":2021,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Mental Health Research Topics","field":"Psychology","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Oulun Yliopisto; Academy of Finland; KAUTE-Säätiö","keywords":"Productivity; Commit; Software; Computer science; Variance (accounting); Software development; Data science; Software metric; Software engineering; Work (physics); Software sizing; Personal software process; Software quality; Software construction; Engineering; Database; Business","score_opus":0.1754318038953424,"score_gpt":0.39816209944449493,"score_spread":0.22273029554915252,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3157341372","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.98003834,0.0012192823,0.0081875585,0.00009343826,0.007930376,0.0018156382,0.000039459293,0.00064823794,0.000027682028],"genre_scores_gemma":[0.989648,0.000011549645,0.0060252463,0.00002370331,0.0036124121,0.00030977878,0.00006386561,0.00012711593,0.00017836798],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9939856,0.0006027107,0.0010330149,0.001939535,0.0012417719,0.0011973456],"domain_scores_gemma":[0.9965388,0.0013546662,0.0003418325,0.0010063594,0.00018880423,0.0005695492],"candidate_categories":["metaepi_narrow","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.0013461242,0.0007666122,0.001117532,0.0003478986,0.0004583028,0.0005540271,0.0006076369,0.00091448973,0.00015631347],"category_scores_gemma":[0.004273266,0.0007976205,0.00016659855,0.000568667,0.000117267555,0.00022989858,0.0028701567,0.0041193343,0.000004934286],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007138579,0.0005338582,0.9829057,0.00065519864,0.0005145601,0.0005326398,0.010052559,0.0007980246,0.0000045518045,0.0000028098148,0.00011674068,0.0038120123],"study_design_scores_gemma":[0.001086299,0.0005431404,0.99277866,0.0011012094,0.00027786664,0.00016433568,0.0020167194,0.00075976027,0.000047750415,0.000037544676,0.00027936665,0.0009073426],"about_ca_topic_score_codex":0.0010089002,"about_ca_topic_score_gemma":0.0000534134,"teacher_disagreement_score":0.009873006,"about_ca_system_score_codex":0.0005186577,"about_ca_system_score_gemma":0.00043929322,"threshold_uncertainty_score":0.99944746},"labels":[],"label_agreement":null},{"id":"W3157562347","doi":"10.1007/s10664-020-09926-4","title":"The nature of build changes","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Dependency (UML); Java; Plug-in; Software evolution; Software maintenance; Software engineering; Software; Legacy system; Software system; Programming language; Software construction","score_opus":0.015523738207438768,"score_gpt":0.28122853624808597,"score_spread":0.2657047980406472,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3157562347","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.1145447,0.0066554747,0.8725674,0.003655913,0.0013947842,0.00012529688,0.0000043751493,0.0010183166,0.000033743552],"genre_scores_gemma":[0.90915596,0.00012636642,0.08971984,0.00024338678,0.00027060596,0.00003895099,0.000003404663,0.00004540464,0.0003960492],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99850386,0.00003114606,0.00018562531,0.0003238371,0.0005101588,0.00044538546],"domain_scores_gemma":[0.9958517,0.0030142535,0.00003246222,0.00077717326,0.00019141013,0.00013302313],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00030854432,0.0001569713,0.00018340592,0.00007612924,0.0000963471,0.000110173445,0.00095233653,0.00015696751,0.000010108611],"category_scores_gemma":[0.006270567,0.00012233476,0.0000869118,0.0009905755,0.000030025887,0.0001191677,0.00051558076,0.0006127556,0.000013831359],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003178929,0.00045843786,0.69863665,0.0011125529,0.0007622643,0.0012648194,0.0025402226,0.06747495,0.015289637,0.030000728,0.05026024,0.13216768],"study_design_scores_gemma":[0.0006577487,0.00016787952,0.44898662,0.0002513387,0.000017177801,0.00021823846,0.000030405645,0.027678909,0.09937501,0.0007121814,0.42090237,0.0010021246],"about_ca_topic_score_codex":0.0000023606199,"about_ca_topic_score_gemma":0.0000023663938,"teacher_disagreement_score":0.7946113,"about_ca_system_score_codex":0.000060782866,"about_ca_system_score_gemma":0.00009527302,"threshold_uncertainty_score":0.75069076},"labels":[],"label_agreement":null},{"id":"W3159076496","doi":"10.1007/s10664-020-09910-y","title":"Promises and challenges of microservices: an exploratory study","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":95,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Microservices; Computer science; Software deployment; Scalability; Reuse; Best practice; Software development; Process (computing); Software engineering; Code reuse; Software; Knowledge management; Engineering; Cloud computing; Database; Management","score_opus":0.04185144518438682,"score_gpt":0.2711256357909792,"score_spread":0.22927419060659238,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3159076496","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.96382326,0.0044394103,0.030845614,0.00013898627,0.00023682197,0.0001542126,0.000001347482,0.0003545736,0.0000057620473],"genre_scores_gemma":[0.98264736,0.00013716279,0.017071875,0.000039087932,0.00005695205,0.000028829767,9.937503e-7,0.000012848441,0.0000048723045],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.998859,0.00006325552,0.0002609225,0.00041256877,0.00020672464,0.0001975225],"domain_scores_gemma":[0.99897754,0.00016738597,0.000045409255,0.0005593191,0.0001280776,0.00012228166],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003010401,0.00014570529,0.00026022704,0.00005355816,0.000050573028,0.000047297835,0.00031369054,0.00006757558,0.000003654732],"category_scores_gemma":[0.00017226677,0.00012712892,0.00003923872,0.00026375664,0.000017823213,0.00050657353,0.0002732749,0.00011218316,0.0000036343183],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013694165,0.0015088556,0.9178703,0.0022440876,0.00013532596,0.00013148894,0.0437936,0.0017958808,0.0013200837,0.00024934654,0.000050202336,0.03088713],"study_design_scores_gemma":[0.0013167496,0.0008328237,0.97322595,0.0002772186,0.000030394702,0.000074473355,0.0033177335,0.0070722043,0.008656147,0.00019538075,0.0041910126,0.0008098825],"about_ca_topic_score_codex":0.0000045238385,"about_ca_topic_score_gemma":0.000014761531,"teacher_disagreement_score":0.055355676,"about_ca_system_score_codex":0.000021702435,"about_ca_system_score_gemma":0.000064462954,"threshold_uncertainty_score":0.5184167},"labels":[],"label_agreement":null},{"id":"W3163691159","doi":"10.1007/s10664-021-09954-8","title":"On using Stack Overflow comment-edit pairs to recommend code maintenance changes","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Commit; Code (set theory); Source code; Code review; Code smell; Data mining; Programming language; Point (geometry); Interface (matter); Static program analysis; Set (abstract data type); Software; Database; Software development; Software quality; Operating system","score_opus":0.05172537609133042,"score_gpt":0.3103960695023846,"score_spread":0.25867069341105414,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3163691159","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.083575815,0.00015357864,0.90456706,0.008683088,0.001467301,0.00022534234,0.000027637147,0.0012792094,0.000020959786],"genre_scores_gemma":[0.40109494,0.000041787414,0.58505017,0.011833523,0.00086142827,0.00014829233,0.000031864303,0.00023939453,0.00069863925],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9969857,0.000069034184,0.00031015842,0.00084762473,0.0007412783,0.001046257],"domain_scores_gemma":[0.9957552,0.0024088165,0.000043272674,0.0010651646,0.00018531558,0.0005422173],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004294866,0.0003846782,0.0003934594,0.00027032685,0.00013823288,0.00025666397,0.0009489559,0.00014510711,0.000085004125],"category_scores_gemma":[0.005248722,0.00040598825,0.000115822484,0.001521758,0.000018264462,0.00024216532,0.0008058378,0.0006023632,0.0001230358],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000055435794,0.0005732759,0.06170691,0.00047490365,0.00028904225,0.0017748357,0.0019009504,0.63813573,0.0028411506,0.0020531327,0.26929325,0.020901382],"study_design_scores_gemma":[0.0019609393,0.0007884565,0.0542052,0.0013074764,0.000028715613,0.0002507354,0.000059275753,0.3153614,0.020641305,0.00040878006,0.6021369,0.0028507952],"about_ca_topic_score_codex":0.000013672595,"about_ca_topic_score_gemma":0.000007792567,"teacher_disagreement_score":0.33284363,"about_ca_system_score_codex":0.0005973795,"about_ca_system_score_gemma":0.000108564454,"threshold_uncertainty_score":0.9998392},"labels":[],"label_agreement":null},{"id":"W3166692829","doi":"10.1007/s10664-021-09979-z","title":"Studying backers and hunters in bounty issue addressing process of open source projects","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Manitoba; Concordia University; Huawei Technologies (Canada)","funders":"","keywords":"Value (mathematics); Process (computing); Business; Computer science","score_opus":0.05329574829042888,"score_gpt":0.33616590050921935,"score_spread":0.28287015221879047,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3166692829","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5404119,0.00020645223,0.45803183,0.0005347229,0.00015865249,0.00031765038,0.0000025961508,0.00023123447,0.000104966166],"genre_scores_gemma":[0.87978816,0.000003089258,0.11973871,0.00027267393,0.000039452636,0.00004297038,0.0000045399684,0.000031537,0.00007887055],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99833214,0.00005032373,0.0004228937,0.00052842934,0.00030342914,0.0003628106],"domain_scores_gemma":[0.99887234,0.00035070826,0.00009891415,0.00041405478,0.00016709724,0.000096860545],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00039169734,0.00019430123,0.00034296283,0.00019569436,0.000094412106,0.0004260707,0.00086429174,0.00008332999,0.000011295352],"category_scores_gemma":[0.0012922089,0.00020893013,0.00003465688,0.0018587218,0.000038839902,0.00097630924,0.0011010182,0.00030419778,0.0000040128875],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001929196,0.00044771403,0.85933894,0.00096518436,0.00012342382,0.00022489109,0.03709278,0.06258941,0.0010533748,0.00036582144,0.0010762925,0.036702864],"study_design_scores_gemma":[0.00799258,0.00054596685,0.6533783,0.006932154,0.00009493425,0.0004960147,0.014192729,0.21413112,0.044799786,0.00084265525,0.051618133,0.0049756016],"about_ca_topic_score_codex":0.000034645294,"about_ca_topic_score_gemma":0.0000101332225,"teacher_disagreement_score":0.33937627,"about_ca_system_score_codex":0.0000871853,"about_ca_system_score_gemma":0.00020327869,"threshold_uncertainty_score":0.8519923},"labels":[],"label_agreement":null},{"id":"W3168361240","doi":"10.1007/s10664-021-10111-4","title":"PRINS: scalable model inference for component-based system logs","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada; Fonds National de la Recherche Luxembourg","keywords":"Computer science; Scalability; Component (thermodynamics); Inference; Data mining; Software; Component-based software engineering; Software system; Process (computing); Implementation; Software engineering; Machine learning; Artificial intelligence; Database; Programming language","score_opus":0.024218276058589044,"score_gpt":0.26245946344478,"score_spread":0.23824118738619096,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3168361240","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.078187846,0.000111601395,0.9185867,0.0001826127,0.00074583007,0.00048215018,0.000023488672,0.0016626713,0.000017136977],"genre_scores_gemma":[0.85714424,5.514617e-7,0.14189106,0.00021968201,0.000059592716,0.0005773597,0.000015149374,0.000026495327,0.00006587579],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979761,0.000043730168,0.00041049536,0.0005530903,0.00048773995,0.000528829],"domain_scores_gemma":[0.9984079,0.00054418383,0.000086515385,0.0006963549,0.00009817499,0.0001668758],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005887353,0.00023848121,0.00033116472,0.0001413635,0.00040024467,0.00008738963,0.0009980495,0.00007800073,0.0000085576985],"category_scores_gemma":[0.00025901245,0.00022751573,0.00016839812,0.000526773,0.000019620982,0.0002700476,0.0004632587,0.00030201118,0.00001722847],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010036037,0.000073224866,0.016739842,0.00038025554,0.000013125614,0.0000046008754,0.0001913855,0.98030883,0.000050641618,0.0008864209,0.0006225886,0.00071907084],"study_design_scores_gemma":[0.00047348905,0.00010086347,0.0019282878,0.00004836346,0.000006549776,0.00000976041,0.000016257152,0.9901599,0.00021759809,0.00006813185,0.006665257,0.00030549208],"about_ca_topic_score_codex":0.0000143372245,"about_ca_topic_score_gemma":4.071857e-7,"teacher_disagreement_score":0.77895635,"about_ca_system_score_codex":0.00048827572,"about_ca_system_score_gemma":0.00020465624,"threshold_uncertainty_score":0.92778224},"labels":[],"label_agreement":null},{"id":"W3176971075","doi":"10.1007/s10664-021-09994-0","title":"MLASP: Machine learning assisted capacity planning","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Top Hat (Canada)","funders":"","keywords":"Computer science; Capacity planning; Artificial intelligence; Operating system","score_opus":0.03252904706006558,"score_gpt":0.26101616788215914,"score_spread":0.22848712082209355,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3176971075","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2915414,0.00064591697,0.7052264,0.00022050798,0.00073279254,0.0000634524,0.0000019104102,0.0014800455,0.00008754812],"genre_scores_gemma":[0.8935647,0.000008114336,0.10576287,0.00020538826,0.00017908175,0.000018620649,0.000013127218,0.000027539858,0.00022057617],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99814326,0.00008087593,0.00035620874,0.00054223655,0.00038849507,0.00048890914],"domain_scores_gemma":[0.9986124,0.00045178004,0.000065766944,0.0005307401,0.00013173134,0.0002075992],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00036028906,0.00024386145,0.00032232451,0.000089722234,0.0002065527,0.00015462053,0.00044996958,0.00015887115,0.000036551322],"category_scores_gemma":[0.0014325618,0.00022525204,0.00014983193,0.00079608033,0.000020895866,0.00040549453,0.0003251487,0.0006134029,0.00007674175],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004397941,0.00010050545,0.92055994,0.00022764843,0.00007032736,0.00025349812,0.0010997831,0.06840389,0.00062919606,0.00013153352,0.0006446962,0.00787457],"study_design_scores_gemma":[0.0007864112,0.0001145394,0.6312494,0.00030992436,0.00002143661,0.0005158804,0.000027314076,0.26304558,0.0049626958,0.00012883547,0.097783715,0.001054215],"about_ca_topic_score_codex":0.000017003535,"about_ca_topic_score_gemma":0.0000016161296,"teacher_disagreement_score":0.6020233,"about_ca_system_score_codex":0.00013854756,"about_ca_system_score_gemma":0.00008681638,"threshold_uncertainty_score":0.91855115},"labels":[],"label_agreement":null},{"id":"W3176989815","doi":"10.1007/s10664-021-10066-6","title":"Test case selection and prioritization using machine learning: a systematic literature review","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":158,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Mitacs; Huawei Technologies; Canada Research Chairs","keywords":"Regression testing; Computer science; Machine learning; Prioritization; Artificial intelligence; Feature selection; Process (computing); Software; Test case; Test (biology); Software engineering; Regression analysis; Software development; Engineering; Management science; Software construction","score_opus":0.019469488494802994,"score_gpt":0.28500089700866355,"score_spread":0.26553140851386053,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3176989815","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.025345538,0.10207016,0.8708385,0.0001737702,0.00017776217,0.0003307455,0.000002990254,0.001059249,0.0000012439626],"genre_scores_gemma":[0.525873,0.010307354,0.46211225,0.0005257826,0.00041597884,0.00015593033,0.00005051575,0.0001890332,0.0003701526],"study_design_codex":"systematic_review","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99842465,0.00010006376,0.000332643,0.0004576697,0.00033997957,0.00034502384],"domain_scores_gemma":[0.9975602,0.0016290557,0.000055470162,0.00030498355,0.00025958527,0.00019071055],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.00041749678,0.00022715364,0.00036402376,0.000156809,0.00014118411,0.00029711242,0.00018054116,0.000109451714,0.0000068608447],"category_scores_gemma":[0.015280339,0.00022322133,0.0000673894,0.0018478818,0.000007971352,0.00040388838,0.00022581842,0.00053013244,0.000005628417],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004342449,0.00030313025,0.43335825,0.45772877,0.00026585624,0.024347449,0.0019142657,0.07470246,0.0042832787,0.00029733582,0.0005284228,0.00226644],"study_design_scores_gemma":[0.00030380773,0.00008468128,0.0031825318,0.043899357,0.00006338166,0.037656058,0.000005007431,0.91261595,0.00074113597,0.000013989019,0.000829468,0.0006046354],"about_ca_topic_score_codex":0.0000047094313,"about_ca_topic_score_gemma":0.000001945197,"teacher_disagreement_score":0.8379135,"about_ca_system_score_codex":0.00015187578,"about_ca_system_score_gemma":0.000070205104,"threshold_uncertainty_score":0.9930144},"labels":[],"label_agreement":null},{"id":"W3178956269","doi":"10.1007/s10664-021-10004-6","title":"Evaluating the impact of falsely detected performance bug-inducing changes in JIT models","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Polytechnique Montréal; Concordia University","funders":"","keywords":"Leverage (statistics); Software bug; Computer science; Software; Software quality; Empirical research; Software quality assurance; Quality (philosophy); Capability Maturity Model; Source code; Software engineering; Software development; Operating system; Artificial intelligence","score_opus":0.09454756759896381,"score_gpt":0.3660511802990926,"score_spread":0.2715036127001288,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3178956269","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7388585,0.00041321246,0.26002377,0.000106310115,0.00013295957,0.00013264551,0.0000020284886,0.00032515518,0.000005386173],"genre_scores_gemma":[0.96337706,0.000022375398,0.0363963,0.000018796278,0.000073520794,0.000049438768,0.0000024792066,0.000035773228,0.000024246994],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977514,0.00008817369,0.00035473864,0.0004877521,0.00064551213,0.0006723888],"domain_scores_gemma":[0.9966437,0.0021563082,0.00006390115,0.0007829524,0.0002174389,0.00013564911],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009117905,0.00025641275,0.0003265537,0.00024874826,0.00007799259,0.000098871526,0.0008806887,0.0001183155,0.00001784961],"category_scores_gemma":[0.004146754,0.0002027383,0.00012932396,0.0019495529,0.000024275389,0.00039048775,0.0005087447,0.0005994975,0.000007512048],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000045706192,0.000033296732,0.10132488,0.00007102894,0.00003069766,0.000024283703,0.00081761274,0.8833954,0.0045157876,0.000011173557,0.000022507737,0.009748791],"study_design_scores_gemma":[0.00020659922,0.00013094832,0.32233068,0.00013434264,0.0000029077062,0.000029977362,0.000006314343,0.66976947,0.0071501518,0.000046037836,0.000007651085,0.00018495061],"about_ca_topic_score_codex":0.00006984235,"about_ca_topic_score_gemma":0.000008422672,"teacher_disagreement_score":0.22451852,"about_ca_system_score_codex":0.00031013097,"about_ca_system_score_gemma":0.0002614035,"threshold_uncertainty_score":0.82674277},"labels":[],"label_agreement":null},{"id":"W3179948295","doi":"10.1007/s10664-021-10006-4","title":"Task estimation for software company employees based on computer interaction logs","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Business Process Modeling and Analysis","field":"Business, Management and Accounting","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure","funders":"","keywords":"Task (project management); Computer science; Software; Estimation; Software engineering; Engineering; Operating system; Systems engineering","score_opus":0.02722979127888976,"score_gpt":0.2638586404644538,"score_spread":0.23662884918556404,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3179948295","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06650223,0.00004200255,0.93126655,0.00076237414,0.00056802906,0.00010700707,0.0000058848973,0.0007315018,0.000014405337],"genre_scores_gemma":[0.9097536,0.0000012214301,0.08599347,0.0024444978,0.0012363589,0.00005813243,0.00039706807,0.00006847699,0.000047170062],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99869525,0.0000059302574,0.0003131205,0.00042248087,0.00026126098,0.0003019801],"domain_scores_gemma":[0.998976,0.00030542945,0.000107586435,0.00022881254,0.00035657323,0.000025582627],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00013610373,0.0002550443,0.00030768043,0.00025485357,0.0001740746,0.00033057405,0.00014510221,0.000104567065,0.000062537205],"category_scores_gemma":[0.000838304,0.00025151434,0.00019003513,0.0006088131,0.000012872834,0.00054730923,0.00006849064,0.0001910042,0.00007325012],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000022434364,0.000131062,0.011398401,0.00040617664,0.000032942953,0.000008215981,0.000017476563,0.9723668,0.000022291855,0.00005601116,0.0025446878,0.01299349],"study_design_scores_gemma":[0.00042818254,0.000013789836,0.004683814,0.00022292204,0.00008053557,0.0000014695953,0.000008656483,0.9779408,0.000068773705,0.00020906082,0.016024802,0.00031720207],"about_ca_topic_score_codex":0.000023707262,"about_ca_topic_score_gemma":0.0000046251,"teacher_disagreement_score":0.8452731,"about_ca_system_score_codex":0.000064864704,"about_ca_system_score_gemma":0.000029548142,"threshold_uncertainty_score":0.9999937},"labels":[],"label_agreement":null},{"id":"W3180512116","doi":"10.1007/s10664-021-09969-1","title":"The secret life of test smells - an empirical study on test smell evolution and maintenance","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":45,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Code smell; Test (biology); Code refactoring; Empirical research; Computer science; Reliability engineering; Engineering; Software; Software quality; Software development; Statistics","score_opus":0.02457635823879043,"score_gpt":0.29435137768838804,"score_spread":0.2697750194495976,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3180512116","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6378664,0.00044414657,0.35987666,0.00061442977,0.00031902656,0.00027775337,0.000010337729,0.00057858863,0.000012683523],"genre_scores_gemma":[0.9905614,0.00001959529,0.008975967,0.00011335673,0.00014120231,0.000039711722,0.000003131909,0.000044492084,0.000101102094],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99729836,0.000100283105,0.0004525641,0.00072748755,0.00076510845,0.0006561994],"domain_scores_gemma":[0.98822814,0.009930656,0.00006783718,0.001090883,0.00025930663,0.0004231556],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007439303,0.00030709634,0.00035438038,0.00013749131,0.00019197127,0.00021437171,0.0008595215,0.00012666038,0.000006963271],"category_scores_gemma":[0.023483694,0.00024582166,0.00008453949,0.001156752,0.000077198296,0.00029172856,0.0005323696,0.000611037,0.000020411911],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000063982593,0.0005139294,0.9906872,0.000043914442,0.00003260735,0.00009662886,0.0005274643,0.0055641443,0.00045810224,0.00009962605,0.00097082066,0.0009991917],"study_design_scores_gemma":[0.00045317007,0.00070761825,0.96227145,0.000062981104,0.000008862659,0.000033905777,0.000090969166,0.03372103,0.00055732566,0.00008404954,0.0017034004,0.00030524214],"about_ca_topic_score_codex":0.000013337316,"about_ca_topic_score_gemma":0.0000059765025,"teacher_disagreement_score":0.35269508,"about_ca_system_score_codex":0.0001721793,"about_ca_system_score_gemma":0.00022028733,"threshold_uncertainty_score":0.9999994},"labels":[],"label_agreement":null},{"id":"W3185088259","doi":"10.1007/s10664-021-09992-2","title":"Perceived diversity in software engineering: a systematic literature review","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":136,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; University of British Columbia, Okanagan Campus; Kelowna General Hospital; University of British Columbia","funders":"","keywords":"Diversity (politics); Gender diversity; Diversity training; Cultural diversity; Knowledge management; Computer science; Psychology; Engineering; Political science; Management","score_opus":0.023105166104566845,"score_gpt":0.2595688079767733,"score_spread":0.23646364187220645,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3185088259","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.019852206,0.032092575,0.9433278,0.0011995303,0.0007437178,0.0007805031,0.000016886692,0.0019690176,0.00001774927],"genre_scores_gemma":[0.1990599,0.0030030585,0.7901362,0.0060075237,0.000345243,0.00039533785,0.00014025922,0.00019442309,0.0007180438],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.997235,0.00010137746,0.0006975649,0.00073732185,0.0006143326,0.0006144254],"domain_scores_gemma":[0.9973913,0.00084972195,0.00010643687,0.0010649714,0.00035837933,0.00022921135],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00049315044,0.00040932436,0.000774449,0.00030088465,0.00015437586,0.00026493202,0.0011584248,0.00020071187,0.00003709001],"category_scores_gemma":[0.008309484,0.0004178934,0.0002246945,0.003878955,0.000015029012,0.00076762226,0.0014786143,0.00074819435,0.00007901601],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012182514,0.0011596082,0.45487806,0.4312982,0.0006836665,0.011143935,0.016766295,0.05829002,0.00047120417,0.0049000387,0.01828294,0.0021138692],"study_design_scores_gemma":[0.0023967791,0.00017927377,0.5358446,0.39035398,0.0002610283,0.0021640016,0.00008616557,0.0478363,0.00030746122,0.0003178801,0.015319704,0.004932828],"about_ca_topic_score_codex":0.000004672394,"about_ca_topic_score_gemma":0.0000031221336,"teacher_disagreement_score":0.1792077,"about_ca_system_score_codex":0.0003199359,"about_ca_system_score_gemma":0.000118607335,"threshold_uncertainty_score":0.99982727},"labels":[],"label_agreement":null},{"id":"W3185295980","doi":"10.1007/s10664-021-10099-x","title":"Clones in deep learning code: what, where, and why?","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Deep learning; Computer science; Artificial intelligence; Timeline; Source lines of code; Context (archaeology); Python (programming language); Dependability; Software development; Java; Software evolution; Software system; Software engineering; Machine learning; Software quality; Code (set theory); Software; Programming language; Software construction; Biology","score_opus":0.017447204121601945,"score_gpt":0.26496860410912637,"score_spread":0.24752139998752443,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3185295980","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.29896238,0.006352085,0.6917799,0.00077329564,0.0006305519,0.00018551928,0.0000013373891,0.0013091475,0.000005771686],"genre_scores_gemma":[0.9516197,0.00025479405,0.04732112,0.00026579303,0.00010507178,0.00020800803,0.000005694144,0.000077527824,0.00014228352],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978762,0.000092685856,0.00027152878,0.00056466233,0.00057165674,0.00062321976],"domain_scores_gemma":[0.9978053,0.0015631688,0.000032427626,0.00036803496,0.000033474236,0.00019757226],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006112858,0.00022429088,0.00026529396,0.00036893252,0.00018686701,0.00029991264,0.0007422165,0.00006750785,0.000056355377],"category_scores_gemma":[0.0013298963,0.0002581751,0.000054402553,0.0011992296,0.000022303624,0.0007628214,0.0011922464,0.0010405721,0.000012052645],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007747304,0.000066741595,0.5060139,0.000102696795,0.00002288428,0.00028162307,0.0017603545,0.46489665,0.00009256024,0.00022619653,0.0006413293,0.025887325],"study_design_scores_gemma":[0.00096446136,0.00029590327,0.38735902,0.00012049616,0.0000058569885,0.00024508726,0.00020778777,0.44982138,0.00011465551,0.00022194324,0.15963115,0.001012235],"about_ca_topic_score_codex":0.00002969155,"about_ca_topic_score_gemma":0.000008961553,"teacher_disagreement_score":0.65265733,"about_ca_system_score_codex":0.00023654832,"about_ca_system_score_gemma":0.000037755955,"threshold_uncertainty_score":0.99998707},"labels":[],"label_agreement":null},{"id":"W3192457683","doi":"10.1007/s10664-021-09980-6","title":"An empirical study of same-day releases of popular packages in the npm ecosystem","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Alberta; Huawei Technologies (Canada)","funders":"","keywords":"Computer science; Reuse; Schedule; Exploratory research; Software engineering; Operating system; Engineering","score_opus":0.030237698177021738,"score_gpt":0.3166212142887117,"score_spread":0.28638351611169,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3192457683","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8281608,0.00040283738,0.17052068,0.00017386927,0.00020392606,0.00026003108,0.0000072402527,0.0002679767,0.0000026446319],"genre_scores_gemma":[0.988879,0.0000059752247,0.010896174,0.00004810404,0.00007511509,0.000046467765,0.000005492347,0.000034154506,0.00000951855],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99688864,0.00041005222,0.0006604079,0.00057407294,0.0009685873,0.00049826613],"domain_scores_gemma":[0.9958042,0.0023825432,0.00008543748,0.0013795719,0.00018886005,0.00015937185],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010801117,0.00026443074,0.00052898756,0.00028687017,0.000047844027,0.00008229791,0.0014700757,0.00012598341,0.000012267345],"category_scores_gemma":[0.004799764,0.00021559407,0.00012530407,0.0019769343,0.000022515322,0.00031333207,0.00032792255,0.00049078895,0.0000058067485],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004851409,0.0009353172,0.9477472,0.00016538291,0.0000427859,0.00031735734,0.0051968666,0.043829445,0.0003511543,0.000045195782,0.00025965882,0.0011048353],"study_design_scores_gemma":[0.0006149946,0.00055540726,0.97792,0.00014428228,0.000014935232,0.000052469542,0.0006938087,0.01702499,0.0017971391,0.000045294048,0.00079200766,0.00034468336],"about_ca_topic_score_codex":0.0000439787,"about_ca_topic_score_gemma":0.000041149444,"teacher_disagreement_score":0.1607182,"about_ca_system_score_codex":0.00009731655,"about_ca_system_score_gemma":0.00012068008,"threshold_uncertainty_score":0.87916714},"labels":[],"label_agreement":null},{"id":"W3193822405","doi":"10.1007/s10664-021-10014-4","title":"An empirical study of Q&A websites for game developers","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Latent Dirichlet allocation; Game Developer; Game testing; Computer science; Video game development; Empirical research; World Wide Web; Game design document; Software development; Software; Game design; Topic model; Data science; Knowledge management; Multimedia; Artificial intelligence","score_opus":0.05243780047764452,"score_gpt":0.3366985135992673,"score_spread":0.28426071312162277,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3193822405","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46238667,0.0001466471,0.5365849,0.000104022896,0.0003262162,0.00013535743,0.0000026078753,0.00030396308,0.000009617418],"genre_scores_gemma":[0.85912156,0.0000016097651,0.14037171,0.00010703525,0.00014485959,0.000058141395,0.000008596442,0.000028192104,0.00015830463],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982798,0.00006527676,0.00040424897,0.00053361506,0.00033466824,0.00038240274],"domain_scores_gemma":[0.998421,0.00051558483,0.000060194554,0.00062545884,0.00018249142,0.00019523976],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00028342343,0.00020904935,0.00037409444,0.0001178251,0.000077981604,0.00010673531,0.00058918505,0.00010895269,0.000012459502],"category_scores_gemma":[0.00072467234,0.00019540297,0.0001046877,0.00066768803,0.000012665367,0.00024838664,0.0001603289,0.00015954704,0.000018905406],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000019032654,0.0016337209,0.9287329,0.0002350416,0.00020790167,0.00007012822,0.022472175,0.031765595,0.004601689,0.00016026056,0.005950088,0.0041514817],"study_design_scores_gemma":[0.0046951156,0.0023448677,0.5599155,0.00038762024,0.00008324894,0.00024787852,0.001695286,0.1691123,0.012459049,0.00023291261,0.24594551,0.002880733],"about_ca_topic_score_codex":0.000012725915,"about_ca_topic_score_gemma":0.000016607419,"teacher_disagreement_score":0.3967349,"about_ca_system_score_codex":0.00007044478,"about_ca_system_score_gemma":0.0001115354,"threshold_uncertainty_score":0.7968302},"labels":[],"label_agreement":null},{"id":"W3194114348","doi":"10.1007/s10664-021-09986-0","title":"Empirical evaluation of tools for hairy requirements engineering tasks","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":45,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Empirical research; Systems engineering; Software engineering; Engineering; Mathematics","score_opus":0.13428343853465402,"score_gpt":0.37559869310442306,"score_spread":0.24131525456976904,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3194114348","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19189514,0.00071004225,0.8051142,0.00027835413,0.0008399896,0.00041144507,0.000013749381,0.00072480517,0.000012277014],"genre_scores_gemma":[0.7328849,0.000009165295,0.26629856,0.000098761586,0.00025728028,0.00027839313,0.000044471715,0.00007848632,0.00004999486],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99632156,0.00007179335,0.0006275795,0.00074141496,0.0014739328,0.0007637376],"domain_scores_gemma":[0.99414605,0.0036581224,0.000083692714,0.00094455207,0.0008853114,0.00028229406],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0015726966,0.00034001487,0.0004632274,0.00027903958,0.00007271355,0.00020605333,0.000823898,0.00021129794,0.000043097083],"category_scores_gemma":[0.026033618,0.00037546948,0.00025459623,0.0012128293,0.000021508911,0.0008130212,0.00044364386,0.00035912744,0.000016972963],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000028407534,0.0004913096,0.12318887,0.0011270695,0.00049172435,0.00009766027,0.0011182469,0.79285717,0.013930474,0.00077118125,0.007303443,0.05859444],"study_design_scores_gemma":[0.0020642276,0.00019952797,0.19679713,0.0003294218,0.000088902365,0.00006189213,0.000016933225,0.7388078,0.04005982,0.00025756643,0.020305743,0.0010110008],"about_ca_topic_score_codex":0.000002739798,"about_ca_topic_score_gemma":6.662875e-7,"teacher_disagreement_score":0.54098976,"about_ca_system_score_codex":0.00041436352,"about_ca_system_score_gemma":0.00041879388,"threshold_uncertainty_score":0.9998697},"labels":[],"label_agreement":null},{"id":"W3195715707","doi":"10.1007/s10664-022-10136-3","title":"The sense of logging in the Linux kernel","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"Conselho Nacional de Desenvolvimento Científico e Tecnológico; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada; Mitacs; Coordenação de Aperfeiçoamento de Pessoal de Nível Superior","keywords":"Logging; Computer science; Linux kernel; Debugging; Operating system; Source code; Java; Consistency (knowledge bases); Database; Forestry","score_opus":0.013708953952654526,"score_gpt":0.24621354922894564,"score_spread":0.2325045952762911,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3195715707","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.76637673,0.000752535,0.22977354,0.0015509339,0.00094634783,0.00026832664,0.0000026135676,0.0002860765,0.000042900803],"genre_scores_gemma":[0.9973368,0.00000808171,0.002285144,0.00021894932,0.000056288885,0.00006177438,7.69797e-7,0.000008025953,0.00002415546],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99866444,0.000118770615,0.0003025662,0.00020846953,0.00041925843,0.00028647293],"domain_scores_gemma":[0.9980312,0.0012753629,0.000057770143,0.00057904393,0.000024175164,0.000032451135],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0013544054,0.00010681916,0.00014083224,0.000057638576,0.00026044075,0.0000452616,0.00087647286,0.000031098476,0.000005112525],"category_scores_gemma":[0.000386179,0.000062887295,0.00008527304,0.00056402414,0.000028616789,0.00010792558,0.00040138388,0.0003944529,0.0000053723306],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000028383567,0.00026314586,0.49589342,0.00024714397,0.0000537321,0.0001495401,0.016436962,0.45511496,0.00010335156,0.0025244951,0.005776513,0.02340836],"study_design_scores_gemma":[0.0008489182,0.00030286322,0.50716037,0.000078188656,0.000012388345,0.00031225255,0.00072167744,0.30380985,0.00032378524,0.0014557309,0.18428999,0.0006839857],"about_ca_topic_score_codex":0.000029763147,"about_ca_topic_score_gemma":0.0000017027373,"teacher_disagreement_score":0.23096009,"about_ca_system_score_codex":0.000085061576,"about_ca_system_score_gemma":0.000049002738,"threshold_uncertainty_score":0.25644693},"labels":[],"label_agreement":null},{"id":"W3198375663","doi":"10.1007/s10664-021-10021-5","title":"An empirical study of IoT topics in IoT developer discussions on Stack Overflow","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":50,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Trent University; Concordia University; University of Calgary","funders":"","keywords":"Computer science; Internet of Things; Popularity; World Wide Web; Categorization; Software; Download; Data science; Multimedia; Artificial intelligence","score_opus":0.025234064105648206,"score_gpt":0.2996113619742789,"score_spread":0.2743772978686307,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3198375663","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8505424,0.000039310995,0.14839548,0.00036168232,0.00030530326,0.000115624454,0.0000016322832,0.00021252254,0.000026008285],"genre_scores_gemma":[0.9654544,0.0000012344337,0.034089822,0.00021828071,0.000105371386,0.0000113190035,0.0000025731524,0.000024451838,0.00009253029],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980526,0.00012078398,0.00042577123,0.00057679135,0.0004201832,0.00040387042],"domain_scores_gemma":[0.9985819,0.00029573616,0.000046235436,0.0007930936,0.00008565756,0.00019738656],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00024362518,0.00022928308,0.0003554248,0.00015411047,0.00008444519,0.000095706986,0.00043945678,0.000117591735,0.000018824716],"category_scores_gemma":[0.00061488093,0.0001947534,0.00007226732,0.0009347929,0.000015704243,0.00010123659,0.00024412641,0.00041723848,0.000008792152],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014888741,0.0021058237,0.65968394,0.00006263921,0.000043731572,0.00050802226,0.009303259,0.30537826,0.00069297955,0.00013845212,0.00066073536,0.021407263],"study_design_scores_gemma":[0.0009801503,0.0005315225,0.89800704,0.00021776889,0.00001196791,0.000029356934,0.00044585654,0.091459796,0.0028804322,0.00006718507,0.004748992,0.00061991217],"about_ca_topic_score_codex":0.000018540006,"about_ca_topic_score_gemma":0.000038219358,"teacher_disagreement_score":0.23832309,"about_ca_system_score_codex":0.00011673688,"about_ca_system_score_gemma":0.00011427639,"threshold_uncertainty_score":0.7941813},"labels":[],"label_agreement":null},{"id":"W3200568639","doi":"10.1007/s10664-021-10025-1","title":"A study of how Docker Compose is used to compose multi-component systems","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":29,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Component (thermodynamics); Computer science; Container (type theory); Software deployment; Operating system; Component-based software engineering; Software; Leverage (statistics); Web application; Software engineering; AKA; Database; World Wide Web; Software system; Engineering","score_opus":0.07160242864086612,"score_gpt":0.3200402561237651,"score_spread":0.24843782748289894,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3200568639","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.51182896,0.00022462274,0.48605332,0.0003096069,0.00056446326,0.00039450618,0.00000775188,0.0006158747,9.1214906e-7],"genre_scores_gemma":[0.9348178,0.0000021334688,0.06471321,0.000077781726,0.00009296052,0.000100309466,0.0000054387074,0.000062617175,0.00012774064],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99660563,0.00012364332,0.00051910913,0.00087839115,0.0011256608,0.0007475874],"domain_scores_gemma":[0.9961826,0.0014760519,0.000074081756,0.0013723284,0.00036714642,0.00052778865],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00039547795,0.00040445555,0.0006915497,0.00036899454,0.00009262652,0.0003080824,0.0012267523,0.00012815041,0.000011069527],"category_scores_gemma":[0.0014843505,0.00041430452,0.00015345152,0.0017300212,0.000019095924,0.00028917808,0.0010727849,0.0004778804,0.00005763608],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025792626,0.0034611758,0.6040625,0.000631141,0.00053749967,0.0017719473,0.019083586,0.35216194,0.0113203,0.000090211775,0.0057163946,0.0011375298],"study_design_scores_gemma":[0.00496042,0.0010646859,0.7343587,0.00058433105,0.000055350047,0.0002659513,0.0006734622,0.22473866,0.015380248,0.0000067745304,0.015860023,0.0020513735],"about_ca_topic_score_codex":0.000054526212,"about_ca_topic_score_gemma":0.0000056673616,"teacher_disagreement_score":0.42298886,"about_ca_system_score_codex":0.00022813783,"about_ca_system_score_gemma":0.00010575072,"threshold_uncertainty_score":0.9998309},"labels":[],"label_agreement":null},{"id":"W3202264790","doi":"10.1007/s10664-021-10016-2","title":"Rotten green tests in Java, Pharo and Python","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure","funders":"","keywords":"Python (programming language); Java; Computer science; Programming language; Unit testing; Test (biology); Software; Biology; Ecology","score_opus":0.02234213422314712,"score_gpt":0.284636052570599,"score_spread":0.26229391834745186,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3202264790","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5171917,0.0013563054,0.4788109,0.0010219608,0.00033300082,0.00014020328,0.0000028164955,0.0011169313,0.00002619029],"genre_scores_gemma":[0.8797066,0.000034665063,0.11940037,0.00026194265,0.00015092154,0.00005175068,0.000005844661,0.000060903934,0.00032695808],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981206,0.000038505543,0.0002632236,0.00059087184,0.0003982885,0.0005884987],"domain_scores_gemma":[0.9977119,0.0014088108,0.000020428139,0.00052409887,0.0000759944,0.00025879336],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00029560272,0.00022494429,0.00026653064,0.00022917749,0.000043420798,0.00014457882,0.0004917932,0.00012211502,0.000020461754],"category_scores_gemma":[0.0030136707,0.000242061,0.000055099605,0.0011667063,0.000019966506,0.00038209837,0.0005946724,0.00046500715,0.000037445945],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000029186176,0.000077866665,0.9692427,0.00014847095,0.000026105545,0.0010016114,0.0005603389,0.006321461,0.0011002715,0.00015934119,0.00067107467,0.020687826],"study_design_scores_gemma":[0.00051031186,0.000041601576,0.95423824,0.000106992266,0.000003312822,0.000154256,0.0000048545717,0.033186324,0.0016589472,0.00018351714,0.009465955,0.0004456697],"about_ca_topic_score_codex":0.00004475958,"about_ca_topic_score_gemma":0.000011722294,"teacher_disagreement_score":0.36251497,"about_ca_system_score_codex":0.00013098186,"about_ca_system_score_gemma":0.000088707886,"threshold_uncertainty_score":0.9870961},"labels":[],"label_agreement":null},{"id":"W3204686189","doi":"10.1007/s10664-021-10032-2","title":"How are project-specific forums utilized? A study of participation, content, and sentiment in the Eclipse ecosystem","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Japan Society for the Promotion of Science","keywords":"Eclipse; Popularity; World Wide Web; Computer science; Bridge (graph theory); Sentiment analysis; Knowledge management; Data science; Public relations; Political science; Artificial intelligence","score_opus":0.09961188966302743,"score_gpt":0.31488797726991474,"score_spread":0.21527608760688732,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3204686189","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8571513,0.0005283762,0.14090396,0.00046935372,0.00022068557,0.0005501672,0.0000039990255,0.00016958144,0.000002544656],"genre_scores_gemma":[0.99523157,0.000015237018,0.0043936586,0.000027938266,0.00008246249,0.00019279221,0.0000022162621,0.00001878833,0.000035332934],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980344,0.00014666021,0.00034932562,0.0004698036,0.00057330343,0.00042648177],"domain_scores_gemma":[0.997735,0.0013068038,0.00006873959,0.00065396656,0.0001499397,0.00008553748],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000547877,0.00019753662,0.00032201383,0.00017739544,0.000059782793,0.00024817642,0.00050208793,0.00006327042,0.0000030881238],"category_scores_gemma":[0.0014684747,0.00015700374,0.00005810516,0.0010815858,0.000014407829,0.0002642759,0.0003198278,0.0002804111,0.0000022515596],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010854824,0.0007644825,0.9813867,0.00023407742,0.00007479888,0.00034625403,0.009213388,0.0042631878,0.00019158814,0.00025208996,0.001070985,0.0021915545],"study_design_scores_gemma":[0.001728243,0.00027805197,0.9632555,0.00019012245,0.000014341982,0.00006730339,0.0027045994,0.02112972,0.0009124011,0.000028315137,0.009285018,0.0004063336],"about_ca_topic_score_codex":0.000009920147,"about_ca_topic_score_gemma":0.000033439188,"teacher_disagreement_score":0.13808025,"about_ca_system_score_codex":0.00008347738,"about_ca_system_score_gemma":0.000045842855,"threshold_uncertainty_score":0.6402427},"labels":[],"label_agreement":null},{"id":"W3206397610","doi":"10.1007/s10664-021-10083-5","title":"A fine-grained data set and analysis of tangling in bug fixing commits","year":2022,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Trent University; University of Saskatchewan; University of British Columbia, Okanagan Campus; Kelowna General Hospital; University of British Columbia; University of Ottawa","funders":"Horizon 2020 Framework Programme; Technische Universität Clausthal; Deutsche Forschungsgemeinschaft","keywords":"Context (archaeology); Computer science; Software bug; Set (abstract data type); Code (set theory); Software; Source lines of code; Data mining; Programming language; Biology","score_opus":0.07673165410839632,"score_gpt":0.34358802997785276,"score_spread":0.26685637586945643,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3206397610","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.43716666,0.0015036713,0.559519,0.00020002092,0.00039404057,0.00028061267,0.00028570532,0.0006455846,0.000004708373],"genre_scores_gemma":[0.9005946,0.000044739227,0.09835825,0.00003887497,0.00008441571,0.00010021468,0.0006705888,0.00007854923,0.00002975645],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9961899,0.00011777743,0.00077870156,0.0013561716,0.00088530674,0.0006721644],"domain_scores_gemma":[0.9927702,0.0040525096,0.00016455143,0.0026987807,0.00008698328,0.00022697922],"candidate_categories":["metaepi_narrow","open_science"],"consensus_categories":[],"category_scores_codex":[0.0014812486,0.00046759838,0.0010727186,0.0019193875,0.000071543436,0.00017550147,0.0035523872,0.00026545092,0.000046928104],"category_scores_gemma":[0.006420252,0.0005358075,0.00018927144,0.0037924354,0.000038958577,0.0002816796,0.01319303,0.0017713642,0.0000014416925],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004175589,0.000038514143,0.5387885,0.00032270997,0.00042174826,0.00010242815,0.00078600127,0.45839927,0.000014129388,0.00001812622,0.00024344344,0.0008609879],"study_design_scores_gemma":[0.00022597668,0.00002734392,0.3619174,0.00015215114,0.000113444185,0.000004878833,0.000010250638,0.635909,0.000020618376,0.000051399937,0.0010891062,0.0004783697],"about_ca_topic_score_codex":0.00019011031,"about_ca_topic_score_gemma":0.00004278309,"teacher_disagreement_score":0.46342793,"about_ca_system_score_codex":0.00028419483,"about_ca_system_score_gemma":0.0001785826,"threshold_uncertainty_score":0.99970937},"labels":[],"label_agreement":null},{"id":"W3208323417","doi":"10.1007/s10664-021-10045-x","title":"How do i refactor this? An empirical study on refactoring trends and topics in Stack Overflow","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":64,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"","keywords":"Code refactoring; Computer science; Software engineering; Unit testing; Empirical research; Implementation; Software; Programming language","score_opus":0.04809255609342547,"score_gpt":0.3290085982567614,"score_spread":0.2809160421633359,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3208323417","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.93496555,0.0002615024,0.06207125,0.0010804415,0.000587225,0.00015393342,0.0000051395496,0.00085194985,0.000023011591],"genre_scores_gemma":[0.9822619,0.000011412268,0.016781567,0.000096638905,0.00028048392,0.000037860238,0.000006135808,0.000060483177,0.000463527],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9968198,0.0001315854,0.00034458877,0.0010816507,0.0008383324,0.0007840655],"domain_scores_gemma":[0.99714637,0.0011354723,0.000039665756,0.0011320531,0.000100374054,0.00044608128],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00041527575,0.00040778602,0.00046433244,0.00048530183,0.000090057525,0.00068038923,0.00068695995,0.00020661541,0.000026834676],"category_scores_gemma":[0.0030053325,0.0004020835,0.00008521849,0.001589853,0.000021882126,0.00077545736,0.0005539307,0.0009633711,0.0000090124195],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009577157,0.00037880582,0.9817031,0.00004150998,0.00003326242,0.00062909856,0.0021374691,0.002350326,0.00012057599,0.000050627,0.0002733618,0.012272296],"study_design_scores_gemma":[0.0006888092,0.0004226411,0.98372406,0.00006276588,0.000005337918,0.000026108088,0.00006630971,0.005417284,0.00046385668,0.000024621126,0.008595739,0.00050249184],"about_ca_topic_score_codex":0.000010954666,"about_ca_topic_score_gemma":0.000010401813,"teacher_disagreement_score":0.047296345,"about_ca_system_score_codex":0.0003645536,"about_ca_system_score_gemma":0.000096823795,"threshold_uncertainty_score":0.9998431},"labels":[],"label_agreement":null},{"id":"W4200448782","doi":"10.1007/s10664-021-10076-4","title":"Using code reviews to automatically configure static analysis tools","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Leverage (statistics); Code review; Static program analysis; Java; Code (set theory); Source code; Precision and recall; Context (archaeology); Information retrieval; Software engineering; Statement (logic); Programming language; Artificial intelligence; Software; Software development","score_opus":0.09646917508994118,"score_gpt":0.3629266213116876,"score_spread":0.26645744622174644,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4200448782","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.05229883,0.00059391465,0.94478464,0.00055805885,0.00028273405,0.00026974504,0.000013743219,0.0011829857,0.000015325551],"genre_scores_gemma":[0.087735,0.00002068298,0.91096294,0.00083910866,0.00011748433,0.00006851611,0.000018371646,0.00005500846,0.00018290867],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99695295,0.00011234458,0.0006315529,0.0007782438,0.00072060665,0.00080428633],"domain_scores_gemma":[0.9956821,0.002226671,0.000056872814,0.001194577,0.00024042996,0.0005993216],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00066976814,0.00034221602,0.0007353207,0.0004490739,0.000096102776,0.00047367212,0.00095526996,0.00012921305,0.0001472377],"category_scores_gemma":[0.014718988,0.00034392052,0.0003123057,0.0049791792,0.000017013403,0.00041065013,0.0005278935,0.00042229483,0.00020612267],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004874769,0.00017151801,0.073488176,0.00053285726,0.0008298051,0.0007623399,0.0012522317,0.8994227,0.0031037568,0.0007863692,0.0055086883,0.014136703],"study_design_scores_gemma":[0.00039937842,0.000089262634,0.15028116,0.00033734637,0.00025258,0.00009715138,0.00001215635,0.76800656,0.002674414,0.000104408005,0.07642516,0.0013204173],"about_ca_topic_score_codex":0.000006894655,"about_ca_topic_score_gemma":0.0000035641613,"teacher_disagreement_score":0.13141611,"about_ca_system_score_codex":0.00029027174,"about_ca_system_score_gemma":0.00019909909,"threshold_uncertainty_score":0.9999013},"labels":[],"label_agreement":null},{"id":"W4200462088","doi":"10.1007/s10664-021-10080-8","title":"A study of gender in user reviews on the Google Play Store","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Digital Marketing and Social Media","field":"Social Sciences","cited_by":19,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"App store; Android (operating system); World Wide Web; Internet privacy; Mobile apps; Computer science; Social media; Advertising; Business","score_opus":0.0853712131898278,"score_gpt":0.3420812369426887,"score_spread":0.2567100237528609,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4200462088","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.99522376,0.0003583529,0.000103525905,0.00034964745,0.00030931045,0.00025804964,0.0000017217853,0.00006751036,0.0033280954],"genre_scores_gemma":[0.9989788,0.000039174633,0.00016831968,0.00016948754,0.0001456911,0.000035863046,9.37043e-7,0.0000121105495,0.00044965243],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99879134,0.00026251317,0.00022505855,0.0001654069,0.00032754862,0.00022812052],"domain_scores_gemma":[0.9984417,0.0012469428,0.000039764418,0.00015988555,0.000039049053,0.000072640185],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0009147655,0.00009189239,0.00021488464,0.000034643846,0.00008924694,0.000034411976,0.00016479114,0.00006308732,0.00007362298],"category_scores_gemma":[0.009759831,0.000069616275,0.00006583366,0.0005254365,0.0000318822,0.00005755348,0.00004771958,0.00021087275,0.00001560081],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020040154,0.0012622861,0.76041377,0.000089208115,0.000063352076,0.00013350164,0.21222407,0.0014285996,0.000020069105,0.0022921695,0.0130568,0.008996142],"study_design_scores_gemma":[0.00061625976,0.00015490322,0.52209806,0.00033154673,0.00002526558,9.95648e-7,0.022155793,0.000026150528,0.00004330424,0.0003317728,0.45375904,0.00045688893],"about_ca_topic_score_codex":0.00008590243,"about_ca_topic_score_gemma":0.00028070467,"teacher_disagreement_score":0.44070226,"about_ca_system_score_codex":0.00008671232,"about_ca_system_score_gemma":0.000069448884,"threshold_uncertainty_score":0.9985814},"labels":[],"label_agreement":null},{"id":"W4214741785","doi":"10.1007/s10664-021-10070-w","title":"TraceSim: An Alignment Method for Computing Stack Trace Similarity","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada; Telefonaktiebolaget LM Ericsson; Western Canada Research Grid; Compute Canada","keywords":"Computer science; Data deduplication; Crash; Data mining; TRACE (psycholinguistics); Software; Task (project management); Subroutine; Flexibility (engineering); Benchmark (surveying); Similarity (geometry); Information retrieval; Artificial intelligence; Machine learning; Database; Programming language; Engineering","score_opus":0.04467345933549554,"score_gpt":0.34524277005109066,"score_spread":0.3005693107155951,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4214741785","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.056403596,0.00021935804,0.9396067,0.0005901467,0.00070482685,0.00050869037,0.000027313516,0.0019345345,0.0000047876842],"genre_scores_gemma":[0.28710338,0.000001318836,0.71207654,0.00026922903,0.00019041804,0.00019845118,0.000018678751,0.00007218883,0.000069807735],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99667567,0.00015975085,0.0004256399,0.00088947365,0.0009130204,0.000936461],"domain_scores_gemma":[0.995086,0.003536529,0.00006994839,0.00082341244,0.00009230051,0.0003917988],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001858493,0.0003334258,0.0003760479,0.00024313059,0.0004593968,0.00020421649,0.0016557653,0.000092125905,0.000041300467],"category_scores_gemma":[0.001564014,0.00038179898,0.00017724323,0.0009229234,0.000015289961,0.00042221593,0.00082987564,0.0007059509,0.000006077321],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015470556,0.00025215856,0.014880481,0.00012587423,0.00005639212,0.00007600976,0.0013006639,0.96036816,0.00026922996,0.00051344716,0.002986797,0.019155346],"study_design_scores_gemma":[0.0006275609,0.00041371924,0.025482327,0.0000138011565,0.000012036144,0.00007321222,0.000037710648,0.9081931,0.001126854,0.00021812551,0.063169666,0.0006318879],"about_ca_topic_score_codex":0.000015392283,"about_ca_topic_score_gemma":0.0000011145598,"teacher_disagreement_score":0.23069978,"about_ca_system_score_codex":0.0004929421,"about_ca_system_score_gemma":0.00011943954,"threshold_uncertainty_score":0.9998634},"labels":[],"label_agreement":null},{"id":"W4214871480","doi":"10.1007/s10664-021-10078-2","title":"Reuse and maintenance practices among divergent forks in three software ecosystems","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":31,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Polytechnique Montréal","funders":"Vetenskapsrådet; Vlaamse regering; Fonds De La Recherche Scientifique - FNRS; Fonds Wetenschappelijk Onderzoek; Canada Research Chairs","keywords":"Computer science; Software engineering; Software development; Software construction; Software bug; Software; Social software engineering; Software evolution; Software analytics; Software peer review; Software system; Code reuse; Software maintenance; Operating system","score_opus":0.02947488547754301,"score_gpt":0.27531612184347026,"score_spread":0.24584123636592725,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4214871480","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.551243,0.00073105097,0.44580734,0.0003696859,0.0006255427,0.00031638425,0.000009110056,0.0008940714,0.0000038035712],"genre_scores_gemma":[0.9418012,0.000025945468,0.057453223,0.00009021946,0.00009926432,0.0003736003,0.000007102087,0.000063760526,0.00008567947],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9969613,0.00009073731,0.0004397749,0.00082468067,0.0008226731,0.00086084154],"domain_scores_gemma":[0.99605495,0.0023295225,0.000168209,0.0010838119,0.00006541538,0.0002980882],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010276693,0.00032132483,0.00036473802,0.0004053564,0.00023789766,0.0002017309,0.0018239259,0.000104850886,0.000048603804],"category_scores_gemma":[0.010563063,0.000340557,0.0000883043,0.0012592265,0.00003393582,0.00075262436,0.0031041033,0.0010608069,0.000012924049],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009004022,0.00006169804,0.9347528,0.000101063815,0.000026250038,0.0002606456,0.000582234,0.060988303,0.000017245597,0.00009305452,0.0016754358,0.0014322351],"study_design_scores_gemma":[0.00070401514,0.00017129967,0.82312053,0.00011145946,0.0000072861408,0.00012682128,0.000040218332,0.15586935,0.000043992037,0.00033083768,0.01878409,0.00069008983],"about_ca_topic_score_codex":0.0002285829,"about_ca_topic_score_gemma":0.00007235178,"teacher_disagreement_score":0.39055818,"about_ca_system_score_codex":0.0005063232,"about_ca_system_score_gemma":0.000089637295,"threshold_uncertainty_score":0.99990463},"labels":[],"label_agreement":null},{"id":"W4220690511","doi":"10.1007/s10664-021-10109-y","title":"Why secret detection tools are not enough: It’s not just about false positives - An industrial case study","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"National Security Agency; North Carolina State University; National Science Foundation","keywords":"False positive paradox; Computer security; Computer science; True positive rate; Data science; Artificial intelligence","score_opus":0.08079733134382909,"score_gpt":0.315110970198614,"score_spread":0.2343136388547849,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4220690511","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39722323,0.00001359759,0.5990293,0.0003204612,0.0007152741,0.0004874711,0.00003160465,0.002176764,0.0000022647246],"genre_scores_gemma":[0.9803837,0.0000014376626,0.01719886,0.001538216,0.0003308848,0.00044904926,0.000007431146,0.000060499384,0.000029917557],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9971676,0.0002500678,0.0004925961,0.000886214,0.00067117985,0.0005323357],"domain_scores_gemma":[0.99819374,0.0004884761,0.00017779169,0.00076045207,0.000113877926,0.0002656318],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005284437,0.00037775325,0.00037968913,0.00035070238,0.0006728069,0.00029746213,0.00075998425,0.00014619567,0.000042457013],"category_scores_gemma":[0.00080436375,0.00043064248,0.00012535481,0.0010842435,0.000025531795,0.0013358645,0.0008318364,0.0010939386,0.0000056416884],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0008076903,0.0050013494,0.02132387,0.0002127243,0.00058868935,0.04092084,0.040017463,0.42305607,0.018663531,0.0002643226,0.007480518,0.44166294],"study_design_scores_gemma":[0.011866348,0.02011757,0.07237997,0.0003146056,0.00038772137,0.030272285,0.023869667,0.19886218,0.2818412,0.0003340313,0.3483109,0.01144356],"about_ca_topic_score_codex":0.00019120226,"about_ca_topic_score_gemma":0.00006049818,"teacher_disagreement_score":0.58316046,"about_ca_system_score_codex":0.0005036097,"about_ca_system_score_gemma":0.00005562223,"threshold_uncertainty_score":0.9998145},"labels":[],"label_agreement":null},{"id":"W4224436806","doi":"10.1007/s10664-022-10133-6","title":"Revisiting reopened bugs in open source software systems","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Queen's University","funders":"","keywords":"Software bug; Rework; Documentation; Pipeline (software); Software; Open source; Computer science; Software engineering; Engineering; Computer security; Operating system; Embedded system","score_opus":0.03242953496458023,"score_gpt":0.29263426994624303,"score_spread":0.2602047349816628,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4224436806","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.115305394,0.001185633,0.87893796,0.00045889532,0.0009959936,0.0008007521,0.000010685785,0.0022714084,0.000033274562],"genre_scores_gemma":[0.885883,0.000010889614,0.11123257,0.00035993027,0.00034467547,0.001014275,0.000026343407,0.00018818115,0.00094014267],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99589044,0.00022117705,0.0006706078,0.0010381764,0.0010752631,0.0011043584],"domain_scores_gemma":[0.9956003,0.0026665148,0.00010597931,0.0012134153,0.00008562275,0.0003281656],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0019618664,0.00038989115,0.0005758023,0.0005369998,0.0003561173,0.0007539394,0.0044134166,0.00010688606,0.000083633044],"category_scores_gemma":[0.0052525382,0.00044712058,0.000119754266,0.0025828066,0.000020093416,0.00087481394,0.0060637468,0.0012343661,0.000060959977],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012477801,0.00007116953,0.29183578,0.00016446385,0.000031035146,0.00031196926,0.0006805236,0.6979424,0.000054645287,0.00024590484,0.002135429,0.006514226],"study_design_scores_gemma":[0.002963814,0.0004051102,0.30061495,0.0006647922,0.000018321443,0.0006508674,0.00025229165,0.39895478,0.00020972785,0.00017435505,0.292062,0.0030289716],"about_ca_topic_score_codex":0.00024919573,"about_ca_topic_score_gemma":0.0000015015798,"teacher_disagreement_score":0.7705776,"about_ca_system_score_codex":0.0008950108,"about_ca_system_score_gemma":0.00020398405,"threshold_uncertainty_score":0.99979806},"labels":[],"label_agreement":null},{"id":"W4225714444","doi":"10.1007/s10664-022-10125-6","title":"Tracking bad updates in mobile apps: a search-based approach","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Thompson Rivers University; Queen's University; École de Technologie Supérieure; Université du Québec à Montréal","funders":"","keywords":"Android (operating system); Computer science; Benchmark (surveying); Genetic programming; Machine learning; Sorting; Mobile apps; Artificial intelligence; World Wide Web","score_opus":0.0278380218906817,"score_gpt":0.2811678476185983,"score_spread":0.2533298257279166,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4225714444","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19636445,0.0007116129,0.80066746,0.00013006167,0.00025668732,0.00038255786,0.000007784425,0.0014607185,0.000018653933],"genre_scores_gemma":[0.8643171,0.0000036165372,0.13468617,0.00013831173,0.00007199349,0.00066314894,0.000021877277,0.00006569698,0.0000320527],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99671143,0.000108038104,0.00038411113,0.0007862397,0.0010376093,0.00097258814],"domain_scores_gemma":[0.9973378,0.0015541748,0.000032003267,0.00076749315,0.00005367381,0.0002548171],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010790462,0.0003032467,0.0003389636,0.00068400416,0.00018764703,0.00018224791,0.0016894536,0.00008566244,0.00008300369],"category_scores_gemma":[0.00095988193,0.0003409381,0.00013518927,0.002485667,0.000031813684,0.00036160855,0.00094849,0.0012060521,0.000027386339],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000006231563,0.00018027192,0.07722432,0.0000653056,0.000011926838,0.00010128809,0.0005168917,0.91665393,0.00009805784,0.000094899166,0.0003238703,0.004723028],"study_design_scores_gemma":[0.0009229572,0.00022522613,0.07186915,0.00004533051,0.000003935281,0.00005871221,0.000062386076,0.8994974,0.0008401312,0.000047764315,0.025622815,0.000804181],"about_ca_topic_score_codex":0.000026345579,"about_ca_topic_score_gemma":6.4492855e-7,"teacher_disagreement_score":0.66795266,"about_ca_system_score_codex":0.00058293197,"about_ca_system_score_gemma":0.0001788181,"threshold_uncertainty_score":0.9999043},"labels":[],"label_agreement":null},{"id":"W4226394169","doi":"10.1007/s10664-022-10139-0","title":"Studying logging practice in test code","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Concordia University","funders":"","keywords":"Logging; Test (biology); Production (economics); Computer science; Software; Database; Operating system; Forestry; Geography; Ecology","score_opus":0.01950508164430709,"score_gpt":0.27719995302169476,"score_spread":0.25769487137738767,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226394169","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.33445755,0.0010030759,0.65862995,0.001833936,0.0016055299,0.0004533499,0.0000071284767,0.0018504489,0.00015899933],"genre_scores_gemma":[0.959817,0.000006887085,0.039344195,0.00056259235,0.000080309444,0.00012834529,0.0000022284482,0.000021016369,0.000037419097],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9981444,0.00008486849,0.0003707774,0.0004618157,0.000491548,0.00044661813],"domain_scores_gemma":[0.99717677,0.002110013,0.00007112747,0.00050448836,0.000039483435,0.00009811018],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010853892,0.00017180655,0.0002281637,0.00018442521,0.0002514053,0.0000753466,0.00075694866,0.00004640194,0.000021902626],"category_scores_gemma":[0.0029205484,0.0001738645,0.00006523935,0.0009851548,0.000012906215,0.00060363906,0.00078708335,0.0005977347,0.000027427312],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000079511665,0.00039098784,0.7084484,0.00009894468,0.00002005447,0.00023767122,0.0049854256,0.27473986,0.00006230006,0.00008482939,0.00090199197,0.010021573],"study_design_scores_gemma":[0.0015902513,0.00049962354,0.3022946,0.000120263,0.000018022216,0.00048637963,0.0006626359,0.2896693,0.00019098332,0.00016675894,0.40292904,0.0013721224],"about_ca_topic_score_codex":0.00003914573,"about_ca_topic_score_gemma":0.0000018756899,"teacher_disagreement_score":0.6253594,"about_ca_system_score_codex":0.00035372283,"about_ca_system_score_gemma":0.000087605375,"threshold_uncertainty_score":0.70899886},"labels":[],"label_agreement":null},{"id":"W4235836287","doi":"10.1007/s10664-006-9035-z","title":"In this issue","year":2007,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"","keywords":"Computer science","score_opus":0.016852074762018047,"score_gpt":0.29734895763221014,"score_spread":0.2804968828701921,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4235836287","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8787827,0.0005959036,0.11414928,0.00039858342,0.0010813137,0.00030472776,0.000012452518,0.0023379084,0.0023371428],"genre_scores_gemma":[0.9644427,0.0000027061508,0.032223698,0.00065718597,0.0007454752,0.000016137647,0.000009405207,0.00020595502,0.0016967886],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.9983619,0.000012918264,0.00033470304,0.0002956891,0.00031101174,0.0006838157],"domain_scores_gemma":[0.9990264,0.00044546585,0.000028589264,0.00025752973,0.000029651368,0.00021234722],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.00064722134,0.00021213252,0.00022776604,0.00032558,0.000027495684,0.000021759362,0.00020541331,0.00014292289,0.0014888465],"category_scores_gemma":[0.0013997732,0.00022117682,0.00006939056,0.0009956445,0.000018851812,0.00021487057,0.00007724259,0.0004139197,0.0063875173],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000822308,0.00030065156,0.8065998,0.00012038664,0.000049179795,0.0011586222,0.0022406743,0.022211727,0.0063417535,0.0001455593,0.15679675,0.0039526606],"study_design_scores_gemma":[0.00051186536,0.000034446548,0.2509416,0.00005112133,0.0000065754725,0.000030059506,0.000014169331,0.0012224645,0.004553057,0.000044153847,0.74211067,0.00047980412],"about_ca_topic_score_codex":0.00005956072,"about_ca_topic_score_gemma":0.000021453056,"teacher_disagreement_score":0.5853139,"about_ca_system_score_codex":0.00033200023,"about_ca_system_score_gemma":0.000018849638,"threshold_uncertainty_score":0.9994239},"labels":[],"label_agreement":null},{"id":"W4235970556","doi":"10.1007/s10664-021-10009-1","title":"Conclusion stability for natural language based mining of design discussions","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Artifact (error); Code refactoring; Computer science; Relevance (law); Task (project management); Context (archaeology); Documentation; Stability (learning theory); Software; Machine learning; Artificial intelligence; Data mining; Software engineering; Engineering; Programming language; Systems engineering","score_opus":0.04044770805115119,"score_gpt":0.3104229520862787,"score_spread":0.2699752440351275,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4235970556","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.15467247,0.000469575,0.84347236,0.00028718723,0.00037590967,0.00020372366,0.000010477954,0.0005065912,0.0000016736259],"genre_scores_gemma":[0.58887476,5.8151625e-7,0.4109587,0.000032554166,0.00004048151,0.000032880092,0.000011893511,0.000019497687,0.000028666687],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99833095,0.000068252564,0.00030277407,0.00044692884,0.00040678692,0.00044433223],"domain_scores_gemma":[0.99381304,0.0051657264,0.000042729393,0.0006088945,0.0002002684,0.00016934672],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.00050394144,0.00018955012,0.00028552636,0.000116147035,0.00008472801,0.000056486915,0.0005340473,0.00009398717,0.00003063654],"category_scores_gemma":[0.009141325,0.00016685524,0.00015636467,0.00071619445,0.000026000787,0.00019636103,0.00035325243,0.00023078876,0.0000025292197],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00016613526,0.0010899486,0.19901507,0.0023711273,0.0003251683,0.00053745863,0.011218359,0.5244576,0.1704056,0.0006019215,0.0099108275,0.0799008],"study_design_scores_gemma":[0.0010032615,0.0001804506,0.042115077,0.00019032019,0.000015195361,0.000016239801,0.00007743159,0.776938,0.17735875,0.000040168772,0.0015170032,0.00054808776],"about_ca_topic_score_codex":0.0000031382845,"about_ca_topic_score_gemma":8.8129207e-7,"teacher_disagreement_score":0.43420225,"about_ca_system_score_codex":0.00012518067,"about_ca_system_score_gemma":0.0001973307,"threshold_uncertainty_score":0.9992051},"labels":[],"label_agreement":null},{"id":"W4240721973","doi":"10.1007/s10664-020-09811-0","title":"Guest Editorial: Special Issue on Predictive Models and Data Analytics in Software Engineering","year":2020,"lang":"en","type":"editorial","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Computer science; Predictive analytics; Data science; Analytics; Data analysis; Software analytics; Software engineering; Software; Data mining; Software development; Software development process; Programming language","score_opus":0.027203129250548533,"score_gpt":0.2815026032040564,"score_spread":0.2542994739535079,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4240721973","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00007284116,0.00030969147,0.20402072,0.00011750664,0.79335195,0.00047295733,0.00038996732,0.0012510178,0.000013348795],"genre_scores_gemma":[0.00022312772,0.00015502547,0.01676185,0.000048136826,0.98209697,0.000059142698,0.000475947,0.00014407194,0.00003574797],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9937125,0.00007833219,0.0010788934,0.0022261564,0.0019416665,0.0009624743],"domain_scores_gemma":[0.99333143,0.0034493683,0.00023065537,0.002194431,0.0002630763,0.00053103146],"candidate_categories":["metaresearch","metaepi_narrow","research_integrity"],"consensus_categories":["research_integrity"],"category_scores_codex":[0.00091813883,0.0010026441,0.0013522663,0.00046155942,0.00011656186,0.00040101825,0.0030480528,0.0016769334,0.000007729027],"category_scores_gemma":[0.0123402495,0.0009843626,0.00016464731,0.0011302471,0.00005594362,0.0013068209,0.0023041442,0.0030084408,0.00006168904],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000032692224,0.0000625998,0.0012585995,0.0005792424,0.00007815974,0.00007205563,0.00041735187,0.07322224,3.4756667e-7,0.000003948282,0.9237615,0.0005112521],"study_design_scores_gemma":[0.0005695735,0.0002228634,0.0004571742,0.00061877153,0.00004065027,0.0000035244086,0.0000051899046,0.17455557,0.0000057519987,0.000030313297,0.82262003,0.00087056065],"about_ca_topic_score_codex":0.000055993092,"about_ca_topic_score_gemma":0.000009057481,"teacher_disagreement_score":0.188745,"about_ca_system_score_codex":0.00057960406,"about_ca_system_score_gemma":0.0004825123,"threshold_uncertainty_score":0.9996191},"labels":[],"label_agreement":null},{"id":"W4250654813","doi":"10.1007/s10664-007-9041-9","title":"In this issue","year":2007,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"","keywords":"Computer science","score_opus":0.018849793462910394,"score_gpt":0.30064744145276495,"score_spread":0.28179764798985457,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4250654813","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08003991,0.00026628296,0.91686827,0.0005289781,0.0008181773,0.00012623664,5.704736e-7,0.0011919956,0.00015957774],"genre_scores_gemma":[0.73372227,0.0000104617675,0.26372367,0.000659499,0.00054329645,0.000028778917,0.0000020732118,0.0000699016,0.0012400436],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99793106,0.0000148602485,0.00032058524,0.00044074038,0.00047964262,0.00081310724],"domain_scores_gemma":[0.99722254,0.001896485,0.000021546532,0.0005532151,0.00004827827,0.000257906],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000887265,0.00020816764,0.00021438891,0.0004582774,0.000036738766,0.000080693964,0.0009083214,0.00013919851,0.00016049133],"category_scores_gemma":[0.0030113095,0.00021642969,0.00006881349,0.0014729903,0.000016626165,0.00048505078,0.00031884125,0.00052065315,0.0004192839],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017677827,0.0002518259,0.9044839,0.00019912118,0.000046995894,0.0019458962,0.0024642758,0.025495522,0.001099129,0.0014808577,0.03900425,0.023510553],"study_design_scores_gemma":[0.0006866908,0.00010402672,0.6306284,0.00009503536,0.000002672239,0.00009426015,0.0000072186444,0.019214518,0.0059217596,0.00021981618,0.34213746,0.00088812545],"about_ca_topic_score_codex":0.000035985697,"about_ca_topic_score_gemma":0.0000040567065,"teacher_disagreement_score":0.65368235,"about_ca_system_score_codex":0.00024416126,"about_ca_system_score_gemma":0.000041975316,"threshold_uncertainty_score":0.8825746},"labels":[],"label_agreement":null},{"id":"W4252966483","doi":"10.1007/s10664-021-10000-w","title":"FACER: An API usage-based code-example recommender for opportunistic reuse","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Snippet; Java; Application programming interface; Code reuse; Android (operating system); Source code; Cluster analysis; Reuse; Information retrieval; Code (set theory); Software; World Wide Web; Data mining; Programming language; Operating system; Artificial intelligence; Set (abstract data type)","score_opus":0.11146040033542123,"score_gpt":0.3362907313551917,"score_spread":0.22483033101977049,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4252966483","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.027354693,0.00014259323,0.9680461,0.0013124798,0.00072113686,0.0002935704,0.000046696954,0.0020747827,0.000007926183],"genre_scores_gemma":[0.22548552,0.00001096924,0.77233815,0.0009116557,0.0002912916,0.00026373108,0.00017552428,0.00014519466,0.0003779771],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9970983,0.00007431887,0.0004177605,0.0009469128,0.00054691196,0.00091574434],"domain_scores_gemma":[0.99343413,0.003712774,0.000052418633,0.0019054435,0.00028708184,0.00060815347],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006358133,0.0003617225,0.0003913965,0.00021084788,0.00025929426,0.0002713485,0.0015572164,0.00019302274,0.00008930646],"category_scores_gemma":[0.008402136,0.00039609033,0.00017830812,0.0008552255,0.000027603955,0.00046272544,0.00073358277,0.00044567132,0.000030218434],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00013415728,0.0025012747,0.27742654,0.0025189926,0.0007041895,0.0031860112,0.0044800043,0.4913901,0.0064609987,0.006024988,0.13315412,0.0720186],"study_design_scores_gemma":[0.0016846354,0.0003815428,0.022562487,0.00013158935,0.000029871897,0.000106970816,0.000019985262,0.6417835,0.0043750596,0.00057357905,0.32691935,0.0014314876],"about_ca_topic_score_codex":0.000021813363,"about_ca_topic_score_gemma":0.0000059284584,"teacher_disagreement_score":0.25486407,"about_ca_system_score_codex":0.00025500404,"about_ca_system_score_gemma":0.0004137048,"threshold_uncertainty_score":0.9999505},"labels":[],"label_agreement":null},{"id":"W4281476990","doi":"10.1007/s10664-022-10181-y","title":"Understanding and supporting the design systems practice","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Design education; Design brief; Computer science; Design knowledge; Systems design; Knowledge management; Design technology; User experience design; User interface; Systems engineering; Design review (U.S. government); Human–computer interaction; Process management; Engineering; Software engineering","score_opus":0.11921303592575368,"score_gpt":0.30020229743708404,"score_spread":0.18098926151133038,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4281476990","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0013790918,0.00019634054,0.9936474,0.0032896218,0.0005244188,0.00027941103,0.0000020677496,0.0006301672,0.00005149873],"genre_scores_gemma":[0.86101276,0.0000020710972,0.1378883,0.0007921902,0.00008089201,0.000120213335,0.0000019224296,0.000027892806,0.000073781164],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998437,0.00016862495,0.00028614126,0.0003112683,0.00045011556,0.00034686172],"domain_scores_gemma":[0.9964177,0.0029604677,0.00011810877,0.00037583706,0.000050414932,0.00007745252],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0016708444,0.00013762536,0.0001345785,0.00010268359,0.00075180514,0.000373878,0.0006331195,0.000030512716,0.000011743839],"category_scores_gemma":[0.0021720862,0.000117716634,0.000031245258,0.0010210492,0.000022931297,0.00051870826,0.0007583662,0.00046168425,0.000005833878],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007765938,0.00006259368,0.00986426,0.000042588847,0.00009166877,0.00013209738,0.0053038886,0.91762114,0.000059766182,0.05471089,0.010853553,0.0012498151],"study_design_scores_gemma":[0.0004474257,0.00021126153,0.0029336952,0.000035825797,0.000036551784,0.001190312,0.0026052631,0.79614127,0.000033953263,0.0015174092,0.19409506,0.0007519734],"about_ca_topic_score_codex":0.000013883712,"about_ca_topic_score_gemma":9.609176e-8,"teacher_disagreement_score":0.8596336,"about_ca_system_score_codex":0.0003117316,"about_ca_system_score_gemma":0.00007226461,"threshold_uncertainty_score":0.5782353},"labels":[],"label_agreement":null},{"id":"W4281617421","doi":"10.1007/s10664-021-10108-z","title":"Evolving software system families in space and time with feature revisions","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"Natural Sciences and Engineering Research Council of Canada; Bundesministerium für Digitalisierung und Wirtschaftsstandort; Österreichische Forschungsförderungsgesellschaft; Fundação Carlos Chagas Filho de Amparo à Pesquisa do Estado do Rio de Janeiro; Conselho Nacional de Desenvolvimento Científico e Tecnológico; Austrian Science Fund; Österreichische Nationalstiftung für Forschung, Technologie und Entwicklung","keywords":"Feature (linguistics); Correctness; Software; Computer science; Precision and recall; Feature vector; Software system; Feature model; Data mining; Space (punctuation); Artificial intelligence; Algorithm; Programming language","score_opus":0.00914075863851467,"score_gpt":0.23168815875263116,"score_spread":0.2225474001141165,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4281617421","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.18939449,0.0030136092,0.80215114,0.0008254088,0.00034642944,0.00058664003,0.000020116684,0.003593511,0.00006863952],"genre_scores_gemma":[0.8271012,0.000011746857,0.1722081,0.0000691647,0.00007579786,0.00019027306,0.000009213801,0.000080245576,0.0002542775],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9974806,0.000081280275,0.00024561145,0.00071236194,0.000781683,0.0006984634],"domain_scores_gemma":[0.99710524,0.0018591403,0.000046655958,0.0006678801,0.0000687698,0.00025231417],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005428516,0.00032654364,0.00039504655,0.00050047936,0.00026363778,0.000187498,0.00084442075,0.00009047267,0.000036028036],"category_scores_gemma":[0.0016872643,0.0003108511,0.000061346975,0.0018340832,0.00003065836,0.00041152778,0.0011435648,0.0009746645,0.000016930504],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000034390618,0.0001278039,0.6471444,0.00078964763,0.0000970797,0.0013575455,0.0026754928,0.3319969,0.00029189722,0.0011789964,0.009024988,0.0052809147],"study_design_scores_gemma":[0.0020469117,0.0007753239,0.57850325,0.001221047,0.000031187523,0.0013755888,0.0003673539,0.38110232,0.0002091135,0.00007674158,0.031924453,0.0023666807],"about_ca_topic_score_codex":0.000024190498,"about_ca_topic_score_gemma":9.161057e-7,"teacher_disagreement_score":0.6377067,"about_ca_system_score_codex":0.0005450548,"about_ca_system_score_gemma":0.0001088708,"threshold_uncertainty_score":0.9999344},"labels":[],"label_agreement":null},{"id":"W4281627483","doi":"10.1007/s10664-022-10156-z","title":"A qualitative study of developers’ discussions of their problems and joys during the early COVID-19 months","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Trent University; University of Calgary","funders":"Natural Sciences and Engineering Research Council of Canada; University of Calgary","keywords":"Notice; Preparedness; Documentation; Coronavirus disease 2019 (COVID-19); Loneliness; Work (physics); Software; Psychology; Medical education; Computer science; Public relations; World Wide Web; Engineering; Management; Political science; Medicine; Social psychology","score_opus":0.05312803563303627,"score_gpt":0.33345928796693347,"score_spread":0.2803312523338972,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4281627483","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.82474357,0.00020991037,0.17388196,0.0003035,0.000094555704,0.00048991863,0.00001521149,0.00025978463,0.0000015789869],"genre_scores_gemma":[0.99570346,0.000003580713,0.0039292476,0.000022526781,0.000014904065,0.0002555168,0.0000011587058,0.000027182003,0.00004242975],"study_design_codex":"qualitative","study_design_gemma":"observational","domain_scores_codex":[0.9978886,0.0002632798,0.00040895134,0.00040210452,0.00067381246,0.00036324354],"domain_scores_gemma":[0.996046,0.0030324433,0.000092708564,0.0005510436,0.000070226655,0.00020756951],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010035124,0.00020419223,0.00031603462,0.00023577327,0.0003061994,0.000041059586,0.0010035937,0.00003627904,0.000015917005],"category_scores_gemma":[0.0023216815,0.00014053402,0.00006789781,0.0012525596,0.000060418155,0.00017661559,0.0014614881,0.00047186995,7.42913e-7],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002077862,0.00036761118,0.2166982,0.0002846804,0.00016959802,0.000027842467,0.5706556,0.21052507,0.0004810429,0.00015275393,0.00009868493,0.0005181954],"study_design_scores_gemma":[0.0022052338,0.0012420635,0.94412804,0.00008485729,0.00002379564,0.000064171116,0.03386729,0.015083899,0.00048194596,0.00035532215,0.0016529193,0.0008104558],"about_ca_topic_score_codex":0.00017038704,"about_ca_topic_score_gemma":0.000008404431,"teacher_disagreement_score":0.72742987,"about_ca_system_score_codex":0.00018620765,"about_ca_system_score_gemma":0.00009611824,"threshold_uncertainty_score":0.5730811},"labels":[],"label_agreement":null},{"id":"W4281811500","doi":"10.1007/s10664-022-10153-2","title":"Works for Me! Cannot Reproduce – A Large Scale Empirical Study of Non-reproducible Bugs","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Dalhousie University","funders":"","keywords":"Software bug; Computer science; Empirical research; Software; Debugging; Eclipse; Software regression; Software engineering; Data science; Software development; Software quality; Programming language; Statistics","score_opus":0.030991629615728825,"score_gpt":0.32073289073010847,"score_spread":0.28974126111437964,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4281811500","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.53508997,0.00021626447,0.4621003,0.00032522046,0.0006807381,0.00083746813,0.000016649425,0.00072968245,0.000003695502],"genre_scores_gemma":[0.94651747,0.0000021031908,0.051705565,0.00011503837,0.0002589962,0.0010571451,0.000012935061,0.000097736964,0.00023304168],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99522,0.000102723076,0.00069533236,0.001669237,0.001222535,0.0010901439],"domain_scores_gemma":[0.9952249,0.0018064238,0.00012041913,0.002301301,0.00023886815,0.00030808424],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0022089465,0.00039188075,0.0006281553,0.00048970326,0.0003619581,0.0001182216,0.0019550524,0.00010605157,0.000045766687],"category_scores_gemma":[0.004290054,0.00042303302,0.00021674298,0.0025817414,0.000025443613,0.00035909767,0.0019507521,0.00088190223,0.000010371411],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000096479314,0.002618855,0.6198634,0.00026932653,0.00020189004,0.00008492649,0.012508541,0.3434747,0.00068999705,0.000016931419,0.016860196,0.0033147687],"study_design_scores_gemma":[0.0068199537,0.0047102193,0.6002888,0.00017368191,0.00009818013,0.0001497943,0.0014079986,0.32255554,0.0046882327,0.00015212924,0.056358743,0.0025966908],"about_ca_topic_score_codex":0.000047973244,"about_ca_topic_score_gemma":0.0000059911977,"teacher_disagreement_score":0.41142747,"about_ca_system_score_codex":0.00036691586,"about_ca_system_score_gemma":0.00021238797,"threshold_uncertainty_score":0.99982214},"labels":[],"label_agreement":null},{"id":"W4283068924","doi":"10.1007/s10664-022-10167-w","title":"A mixed-methods analysis of micro-collaborative coding practices in OpenStack","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Fonds De La Recherche Scientifique - FNRS","keywords":"Commit; Thematic analysis; Computer science; Data science; Coding (social sciences); Knowledge management; Empirical research; Qualitative research; World Wide Web; Database","score_opus":0.04646674099401797,"score_gpt":0.3676935198609035,"score_spread":0.3212267788668855,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4283068924","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.22876704,0.00012863155,0.76998985,0.00040526333,0.00026506442,0.00017577404,0.000022023014,0.0001978268,0.000048554823],"genre_scores_gemma":[0.41205242,0.0000021597145,0.5876517,0.0001450801,0.0000135667215,0.00007667068,0.000012582976,0.000014926041,0.00003086715],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99791336,0.00030867857,0.0005618963,0.0004829507,0.00040051495,0.0003326101],"domain_scores_gemma":[0.99671835,0.0020857647,0.00041160264,0.00054695684,0.00016375509,0.00007354442],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0015136407,0.00017987004,0.0004657896,0.0010656354,0.00014514418,0.00010823184,0.0010537728,0.000052353753,0.00007880626],"category_scores_gemma":[0.0033135572,0.0002025617,0.00011972284,0.013741945,0.000024896155,0.00055850775,0.0010402425,0.0004684101,0.0000035896153],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017111794,0.00032684507,0.31977984,0.00005195608,0.00077132246,0.0000763601,0.009282733,0.650961,0.003387675,0.0037043458,0.0009990684,0.010641751],"study_design_scores_gemma":[0.0010931551,0.00030327786,0.47648838,0.00006405036,0.00035121947,0.000032803182,0.0018111345,0.45209372,0.0061622863,0.00036835845,0.059835467,0.0013961481],"about_ca_topic_score_codex":0.00005898534,"about_ca_topic_score_gemma":0.00001330355,"teacher_disagreement_score":0.19886726,"about_ca_system_score_codex":0.0002597857,"about_ca_system_score_gemma":0.00015069157,"threshold_uncertainty_score":0.8260226},"labels":[],"label_agreement":null},{"id":"W4284882237","doi":"10.1007/s10664-022-10119-4","title":"FIXME: synchronize with database! An empirical study of data access self-admitted technical debt","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Technical debt; Code refactoring; Computer science; Commit; Maintainability; Database; Data access; Software development; Software; Data science; Software engineering; Operating system","score_opus":0.06157596337977576,"score_gpt":0.35714592806012,"score_spread":0.29556996468034424,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4284882237","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.45768958,0.000109143795,0.53926265,0.0001254595,0.00018573206,0.0005203523,0.000056324858,0.0020474093,0.0000033073804],"genre_scores_gemma":[0.82882607,0.000003178353,0.17056493,0.00009061266,0.00010833827,0.00021028257,0.00009713772,0.000090530884,0.000008913026],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9947391,0.00023328363,0.0006314885,0.0015403073,0.0019641921,0.00089158607],"domain_scores_gemma":[0.9929691,0.002153449,0.000118706936,0.0041146544,0.00014940176,0.00049470545],"candidate_categories":["metaepi_narrow","open_science"],"consensus_categories":["open_science"],"category_scores_codex":[0.0011683452,0.00045904916,0.00059619604,0.0004909823,0.00028654025,0.00023596475,0.008166237,0.00010472166,0.000110651796],"category_scores_gemma":[0.0018908377,0.00043155262,0.00006361503,0.0031499413,0.000045099703,0.0019424178,0.008878092,0.001332231,0.000009238185],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000073466945,0.0035098195,0.9159962,0.00018769542,0.00022996053,0.0007140819,0.001250629,0.0727176,0.00013795422,0.000046103403,0.00323483,0.0019016777],"study_design_scores_gemma":[0.0025222262,0.0036751577,0.7151698,0.00009551175,0.00010202001,0.00049260684,0.00015530195,0.26807645,0.00018898604,0.0000404904,0.007863525,0.0016179223],"about_ca_topic_score_codex":0.00007549027,"about_ca_topic_score_gemma":0.000015029653,"teacher_disagreement_score":0.3711365,"about_ca_system_score_codex":0.00040991392,"about_ca_system_score_gemma":0.0004011756,"threshold_uncertainty_score":0.9998136},"labels":[],"label_agreement":null},{"id":"W4286697220","doi":"10.1007/s10664-022-10157-y","title":"Predicting sensitive information leakage in IoT applications using flows-aware machine learning approach","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Toronto Metropolitan University","funders":"","keywords":"Computer science; Source code; Process (computing); Information leakage; Control flow; Machine learning; Code (set theory); Data mining; Taint checking; Vulnerability (computing); Artificial intelligence; Statement (logic); Computer security; Software; Programming language","score_opus":0.013390852009675411,"score_gpt":0.24466050194561964,"score_spread":0.23126964993594423,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4286697220","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.037611466,0.000045569497,0.95980704,0.00003415406,0.00009741936,0.0003797641,0.000009468371,0.0019891101,0.000025989186],"genre_scores_gemma":[0.6712236,0.0000019509089,0.3283052,0.00009565723,0.000044610497,0.00026928502,0.000029792544,0.000021662489,0.000008236479],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9985558,0.00007419469,0.000359349,0.00032191718,0.0003513218,0.00033745178],"domain_scores_gemma":[0.9992866,0.00018203043,0.000113347036,0.00027765892,0.000058245623,0.00008210603],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003497783,0.0001863999,0.00019102922,0.0003990379,0.0003550621,0.000087703,0.00037919465,0.00006216554,0.0000063289003],"category_scores_gemma":[0.00024383208,0.00022488738,0.000060886563,0.0013225845,0.000010698166,0.00083659164,0.00061707856,0.00086227857,0.0000038163967],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000030098854,0.00002504477,0.0058655315,0.000030925163,0.000005793268,0.000006027031,0.000734631,0.98417306,0.00018924614,0.00007583587,0.0000052829278,0.008885629],"study_design_scores_gemma":[0.00015276452,0.000036518042,0.0016127431,0.000011615897,0.0000029214461,0.00007678872,0.00010196112,0.9905229,0.0005715364,0.000049462982,0.0066199047,0.00024088731],"about_ca_topic_score_codex":0.000049997037,"about_ca_topic_score_gemma":0.0000018676002,"teacher_disagreement_score":0.63361216,"about_ca_system_score_codex":0.0005149764,"about_ca_system_score_gemma":0.00004325961,"threshold_uncertainty_score":0.91706413},"labels":[],"label_agreement":null},{"id":"W4288079797","doi":"10.1007/s10664-020-09875-y","title":"Pandemic programming","year":2020,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Occupational Health and Safety Research","field":"Health Professions","cited_by":234,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"New York Institute of Technology; Dalhousie University","funders":"Natural Sciences and Engineering Research Council of Canada; Oulun Yliopisto; Dalhousie University; University of Adelaide","keywords":"Structural equation modeling; Productivity; Confirmatory factor analysis; Notice; Pandemic; Psychology; Applied psychology; Preparedness; Human factors and ergonomics; Poison control; Computer science; Coronavirus disease 2019 (COVID-19); Medicine; Environmental health; Political science; Economics; Management; Economic growth","score_opus":0.16444328769583408,"score_gpt":0.4689126539423454,"score_spread":0.30446936624651133,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4288079797","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6869334,0.0008008236,0.2896228,0.014339817,0.0011942339,0.0024295268,0.000037722722,0.0035726088,0.0010690523],"genre_scores_gemma":[0.97383106,0.000026508562,0.019178446,0.00478512,0.0015398251,0.00033159653,0.00004281739,0.00005783553,0.00020678189],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983725,0.000081266306,0.0003520275,0.00023501247,0.00030601822,0.0006531872],"domain_scores_gemma":[0.9978056,0.0013687235,0.000038779115,0.000116157826,0.00008423669,0.00058648194],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00030146894,0.00011910339,0.00019208094,0.000041593936,0.00028407024,0.0000062407757,0.00015013973,0.00015159757,0.00044959888],"category_scores_gemma":[0.0034452085,0.000106489366,0.00005519183,0.00038949522,0.000013473988,0.00007747,0.00010704597,0.0009308369,0.00093923206],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004094437,0.000010483984,0.9811835,0.0003669524,0.0000073861443,0.0000069198823,0.0006556424,0.0003518433,0.000017780809,0.00004168698,0.0032998966,0.0140169365],"study_design_scores_gemma":[0.0004329698,0.00009409067,0.53247523,0.00006878919,0.000003872986,0.0000010542067,0.000058208632,0.0037257236,0.0000037022476,0.000016733375,0.46295953,0.00016010368],"about_ca_topic_score_codex":0.0000181271,"about_ca_topic_score_gemma":0.0000022858526,"teacher_disagreement_score":0.4596596,"about_ca_system_score_codex":0.0001232532,"about_ca_system_score_gemma":0.00022545319,"threshold_uncertainty_score":0.99983865},"labels":[],"label_agreement":null},{"id":"W4288723601","doi":"10.1007/s10664-022-10158-x","title":"GBGallery : A benchmark and framework for game testing","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Benchmark (surveying); Software engineering; Video game development; Test strategy; Software development; Game testing; Software; Database; Game Developer; Game design; Artificial intelligence; Programming language; Game design document","score_opus":0.04408833245119895,"score_gpt":0.28456053815274496,"score_spread":0.240472205701546,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4288723601","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.017909167,0.00032365578,0.9699004,0.00037500713,0.00031921756,0.0001900908,0.0000052699133,0.010965148,0.000012002123],"genre_scores_gemma":[0.27993697,9.603775e-7,0.7193131,0.00041507935,0.00008661759,0.00020793857,0.0000023278349,0.000023429506,0.000013604414],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9986637,0.000027298325,0.00021325311,0.00044744366,0.0002520745,0.00039622444],"domain_scores_gemma":[0.9954759,0.003915411,0.000057557747,0.00037189797,0.000049107042,0.00013014475],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004130433,0.00018079043,0.00019845286,0.0001290258,0.00027360665,0.000113031565,0.00053265033,0.000060267324,0.000008476762],"category_scores_gemma":[0.0059794514,0.00019577536,0.00006174139,0.00059061876,0.00001673447,0.00013868198,0.0006909408,0.0003821112,0.0000012392709],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000398163,0.00039281492,0.60476315,0.0005772759,0.0001464876,0.00021382161,0.0045756237,0.053589415,0.00019834955,0.0187963,0.0653415,0.25136545],"study_design_scores_gemma":[0.0005824194,0.0009323744,0.07590459,0.00025242678,0.000026161973,0.00041265468,0.000014686433,0.6742082,0.00014052514,0.19222225,0.053763453,0.0015402604],"about_ca_topic_score_codex":0.000008707528,"about_ca_topic_score_gemma":1.0253888e-7,"teacher_disagreement_score":0.62061876,"about_ca_system_score_codex":0.00008557643,"about_ca_system_score_gemma":0.000047447476,"threshold_uncertainty_score":0.7983487},"labels":[],"label_agreement":null},{"id":"W4289976167","doi":"10.1007/s10664-022-10168-9","title":"SSPCatcher: Learning to catch security patches","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":27,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec à Montréal","funders":"Institut des Sciences Mathématiques, Université du Québec à Montréal","keywords":"Computer science; Benchmarking; Computer security; Vulnerability (computing); Code (set theory); Safeguard; Source code; Programming language; Business","score_opus":0.011691828823452129,"score_gpt":0.2559203562742536,"score_spread":0.24422852745080145,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4289976167","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.076229595,0.0000867726,0.91841525,0.0007973111,0.00034989076,0.00016653615,0.000002501007,0.0039306185,0.000021510867],"genre_scores_gemma":[0.84742606,0.0000023500963,0.1514947,0.0005440044,0.00007754957,0.0002491214,0.0000021427568,0.000034625646,0.00016943809],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.99841654,0.000059038222,0.00022143661,0.0004891162,0.00039810795,0.00041573567],"domain_scores_gemma":[0.9990927,0.00019729022,0.000046128338,0.00041794605,0.000046457353,0.00019949117],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00029797966,0.00018979092,0.00019222834,0.00017236237,0.0002878785,0.00006536932,0.0007774342,0.000048589354,0.000043826545],"category_scores_gemma":[0.00054956495,0.0002221338,0.00008415735,0.0010357518,0.0000068597788,0.00025348127,0.0011270582,0.0007106576,0.000033753473],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000029181101,0.00028358647,0.05597947,0.00011372391,0.000070552174,0.0002580671,0.017035296,0.8066592,0.0038559583,0.0008305211,0.015999245,0.0988852],"study_design_scores_gemma":[0.0003954373,0.0007215574,0.011407197,0.00003260099,0.000010189959,0.0001705771,0.0003698558,0.059523217,0.025894122,0.0034296762,0.89640164,0.0016439002],"about_ca_topic_score_codex":0.00003105629,"about_ca_topic_score_gemma":0.0000014630452,"teacher_disagreement_score":0.88040245,"about_ca_system_score_codex":0.00032158647,"about_ca_system_score_gemma":0.000038550465,"threshold_uncertainty_score":0.90583533},"labels":[],"label_agreement":null},{"id":"W4289978463","doi":"10.1007/s10664-022-10172-z","title":"Correction to: Can Offline Testing of Deep Neural Networks Replace Their Online Testing?","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Computer science; Artificial neural network; Deep neural networks; Artificial intelligence; Online and offline; Machine learning; Information retrieval; Data mining; Data science; Operating system","score_opus":0.024391967307452477,"score_gpt":0.2589459812873127,"score_spread":0.23455401397986025,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4289978463","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09758713,0.000063945125,0.898588,0.0002882823,0.002405311,0.00014807592,0.0000045980664,0.0008967158,0.000017965489],"genre_scores_gemma":[0.7462247,1.3098024e-7,0.25309935,0.00029494628,0.00027602396,0.000020164676,0.000010478054,0.000036237994,0.0000379697],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99813235,0.000108410844,0.0004250459,0.0005039689,0.00037126846,0.00045895283],"domain_scores_gemma":[0.9964635,0.0025957553,0.00016485045,0.00048046515,0.00012843945,0.00016701252],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00045909543,0.00024342281,0.00030411425,0.00017360004,0.00027461722,0.00004476778,0.0008436788,0.000047012894,0.000015611344],"category_scores_gemma":[0.008893302,0.00025459199,0.00007493827,0.0021943345,0.000016234402,0.00014686173,0.0011933977,0.00094372575,7.913206e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000006570829,0.000041122177,0.060131896,0.000010374013,0.000009057035,0.000016288794,0.00029642574,0.90438795,0.000084009785,0.0000074374207,0.00020095176,0.034807935],"study_design_scores_gemma":[0.0001621095,0.00022250993,0.028194956,0.000025742112,0.000006162208,0.00007903717,0.00002777124,0.9703556,0.00004089443,0.000012922764,0.0006182452,0.00025410662],"about_ca_topic_score_codex":0.0001372187,"about_ca_topic_score_gemma":0.000013046308,"teacher_disagreement_score":0.6486376,"about_ca_system_score_codex":0.00022592796,"about_ca_system_score_gemma":0.00004709311,"threshold_uncertainty_score":0.99999064},"labels":[],"label_agreement":null},{"id":"W4292291355","doi":"10.1007/s10664-022-10193-8","title":"Revisiting the debate: Are code metrics useful for measuring maintenance effort?","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":21,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"National Science Foundation of Sri Lanka; Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Software maintenance; Code (set theory); Context (archaeology); Source code; Java; Software metric; Code review; Granularity; Software engineering; Data science; Software; Data mining; Software quality; Software development; Programming language; Set (abstract data type)","score_opus":0.05036001561640007,"score_gpt":0.2812554912041638,"score_spread":0.23089547558776374,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4292291355","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.05134663,0.0009921885,0.9430584,0.0016874828,0.00086890324,0.0005055216,0.000025795462,0.0015083042,0.000006824882],"genre_scores_gemma":[0.90002066,0.000014500833,0.09777731,0.00046162406,0.0005569324,0.00080924964,0.000009202268,0.00012282656,0.00022768915],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9967305,0.000066872715,0.00043811675,0.00069923344,0.0010435814,0.0010216727],"domain_scores_gemma":[0.993872,0.0046587726,0.0001218897,0.00094754674,0.00018283816,0.00021698906],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0018751207,0.0003197871,0.00035662396,0.0003109351,0.00070442446,0.00026072792,0.0022382361,0.00007027645,0.00001916068],"category_scores_gemma":[0.009829804,0.000277147,0.00023038223,0.0021739649,0.000027159263,0.00029675636,0.001341121,0.00087287853,0.0000130868775],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024043002,0.000059028982,0.3633584,0.00040083381,0.00014490985,0.00018012553,0.0007184285,0.6098366,0.000113731134,0.0009944591,0.008132557,0.016036902],"study_design_scores_gemma":[0.0010715835,0.00015322235,0.2497959,0.00018045334,0.00003236451,0.00027417383,0.0000871819,0.44313172,0.0008262068,0.00030684788,0.30288282,0.0012575234],"about_ca_topic_score_codex":0.000007414489,"about_ca_topic_score_gemma":6.2414404e-7,"teacher_disagreement_score":0.84867406,"about_ca_system_score_codex":0.0005956405,"about_ca_system_score_gemma":0.0000941486,"threshold_uncertainty_score":0.99996805},"labels":[],"label_agreement":null},{"id":"W4296481997","doi":"10.1007/s10664-022-10187-6","title":"Sources of software development task friction","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Personal Information Management and User Behavior","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Task (project management); Computer science; Work (physics); Software; Forcing (mathematics); Software development; Human–computer interaction; Software engineering; Field (mathematics); Development environment; Systems engineering; Engineering","score_opus":0.1488232714857706,"score_gpt":0.3677832485603151,"score_spread":0.21895997707454448,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4296481997","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.84229684,0.00008507358,0.15676877,0.00010288199,0.00037108053,0.00010483838,0.000014102883,0.00020793547,0.000048472826],"genre_scores_gemma":[0.9791258,0.000001032082,0.019370832,0.00019945916,0.00004744052,0.00006237313,0.000020944291,0.000015573494,0.0011565258],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99745506,0.00003795892,0.0006051595,0.00023105457,0.0014479345,0.00022284003],"domain_scores_gemma":[0.9989809,0.0004391594,0.00016047955,0.00023496861,0.00009561789,0.00008887135],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.000939592,0.00013066076,0.0002056513,0.00037674155,0.00026313827,0.00006859928,0.0005262375,0.00003254995,0.0010913381],"category_scores_gemma":[0.0009893734,0.00011680544,0.00010565461,0.0009315445,0.000014954964,0.00029480745,0.00040772924,0.00018694447,0.000093022674],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000023832432,0.00011693569,0.8110575,0.000025828926,0.000037558995,0.0000120931145,0.0046156105,0.12297597,0.000091386435,0.00007430949,0.021622552,0.039346404],"study_design_scores_gemma":[0.00023765609,0.000053368487,0.42561635,0.0000075743537,0.000011219768,0.000007152439,0.000607685,0.002453643,0.0003589192,0.00009245931,0.570303,0.00025096562],"about_ca_topic_score_codex":0.0000050855456,"about_ca_topic_score_gemma":8.6867664e-7,"teacher_disagreement_score":0.5486804,"about_ca_system_score_codex":0.00011389132,"about_ca_system_score_gemma":0.000047590795,"threshold_uncertainty_score":0.9998218},"labels":[],"label_agreement":null},{"id":"W4297903144","doi":"10.1007/s10664-022-10221-7","title":"On the usage and development of deep learning compilers: an empirical study on TVM","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Parallel Computing and Optimization Techniques","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Compiler; Computer science; Deep learning; Artificial intelligence; Software engineering; Programming language","score_opus":0.035874689591281164,"score_gpt":0.291733287783201,"score_spread":0.25585859819191986,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4297903144","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39575717,0.000017801807,0.6035596,0.00008912552,0.00005411333,0.00011986585,2.719624e-7,0.000380836,0.000021183703],"genre_scores_gemma":[0.9149977,5.16492e-7,0.08466762,0.00025342326,0.000015299194,0.000038697857,0.0000020980476,0.000013394697,0.000011235259],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9986237,0.00018976821,0.00024885294,0.00033053316,0.00040220187,0.0002048946],"domain_scores_gemma":[0.99885863,0.00068175787,0.00006487297,0.0002873622,0.000027122238,0.00008023728],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0006225129,0.0001491909,0.00017239901,0.00012223086,0.00040252294,0.000059656155,0.00053601974,0.000027649723,0.000017624434],"category_scores_gemma":[0.00023818144,0.00011937096,0.000028691004,0.00035676867,0.000015614158,0.000071722294,0.0004631085,0.00043350284,0.0000028539723],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014068414,0.00048589328,0.0519694,0.000009524586,0.000031132447,0.000013991452,0.013052354,0.92129105,0.000015539632,0.00024920516,0.00028607447,0.0125817815],"study_design_scores_gemma":[0.0004618884,0.0017622868,0.1664804,0.00002687423,0.000005465957,0.0000107894575,0.00051025086,0.8241179,0.0003033708,0.00011091896,0.005751077,0.00045880355],"about_ca_topic_score_codex":0.0000015523246,"about_ca_topic_score_gemma":2.9574534e-7,"teacher_disagreement_score":0.51924056,"about_ca_system_score_codex":0.00007447127,"about_ca_system_score_gemma":0.000027925154,"threshold_uncertainty_score":0.48678064},"labels":[],"label_agreement":null},{"id":"W4300960622","doi":"10.1007/s10664-022-10223-5","title":"A controlled experiment of different code representations for learning-based program repair","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Computer science; Code (set theory); Source code; Representation (politics); Artificial intelligence; Point (geometry); Syntax; Programming language; Perspective (graphical); Process (computing); Natural language processing; Set (abstract data type)","score_opus":0.03017434424875222,"score_gpt":0.33110164950214477,"score_spread":0.30092730525339256,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4300960622","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.12513198,0.0002498413,0.86840266,0.0002737334,0.00037261055,0.0015029935,0.0000123139735,0.004048437,0.0000054334105],"genre_scores_gemma":[0.836029,0.0000011172424,0.1587044,0.000038490347,0.00005283027,0.004955577,0.000023284158,0.000047061094,0.00014822515],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99774414,0.000096723845,0.0004592634,0.0005090194,0.00069575483,0.0004950995],"domain_scores_gemma":[0.9955553,0.0034799327,0.00009556532,0.00057670375,0.00012215614,0.00017038042],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00046193565,0.00022448448,0.00044342058,0.00026848624,0.00020805563,0.00005347774,0.0007002062,0.00004961792,0.000041575255],"category_scores_gemma":[0.0036844797,0.00022096616,0.0003754682,0.00061909476,0.000025170102,0.00009448664,0.00041038584,0.00039118688,0.0000017708597],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00013029091,0.0006919187,0.05214154,0.00012290393,0.00014209708,0.000017610952,0.0005531971,0.9415842,0.0004051982,0.00021793757,0.0021599033,0.0018331497],"study_design_scores_gemma":[0.0035399462,0.0010572528,0.016829614,0.000023411229,0.000017247112,0.0000058498754,0.000028236604,0.96370596,0.002591805,0.00003383371,0.01182937,0.00033750446],"about_ca_topic_score_codex":0.0000073969777,"about_ca_topic_score_gemma":4.334389e-7,"teacher_disagreement_score":0.710897,"about_ca_system_score_codex":0.00025099562,"about_ca_system_score_gemma":0.00010445005,"threshold_uncertainty_score":0.9010739},"labels":[],"label_agreement":null},{"id":"W4309382233","doi":"10.1007/s10664-022-10244-0","title":"Developer discussion topics on the adoption and barriers of low code software development platforms","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Agile software development; Computer science; Software development; Personalization; Software; Software development process; Software engineering; World Wide Web; Data science; Engineering","score_opus":0.02297163341640662,"score_gpt":0.2529683407359211,"score_spread":0.2299967073195145,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4309382233","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.461475,0.00008338749,0.5368379,0.000664422,0.0003207857,0.00020578239,0.0000062299328,0.00040357772,0.0000028994896],"genre_scores_gemma":[0.9010598,0.000007278252,0.09811879,0.0002867807,0.00006546421,0.00018445886,0.000010452835,0.000047123078,0.00021985822],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980641,0.000029824541,0.00031569737,0.0003973049,0.00077522045,0.00041785144],"domain_scores_gemma":[0.9983463,0.00089139614,0.000053544183,0.00045404004,0.000055714987,0.00019898395],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005617972,0.00022218758,0.0002093318,0.00017686743,0.00035817208,0.00006246607,0.0007895258,0.00006556595,0.000060327908],"category_scores_gemma":[0.0022105235,0.00014114977,0.00005626586,0.0006454943,0.000034067703,0.00018837646,0.0009340012,0.0005394362,0.0000072574708],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007437978,0.00022832266,0.52332366,0.0005806622,0.00020946773,0.00010785246,0.013913706,0.3495318,0.00037125318,0.0030749554,0.0055389954,0.10304495],"study_design_scores_gemma":[0.001394567,0.0006112215,0.76629543,0.0004310427,0.000016668044,0.00011662776,0.00040959846,0.055324946,0.008923988,0.0005839845,0.16400109,0.0018908491],"about_ca_topic_score_codex":0.0000020039672,"about_ca_topic_score_gemma":6.668921e-7,"teacher_disagreement_score":0.43958476,"about_ca_system_score_codex":0.00027199474,"about_ca_system_score_gemma":0.00017678573,"threshold_uncertainty_score":0.57559204},"labels":[],"label_agreement":null},{"id":"W4309896433","doi":"10.1007/s10664-022-10242-2","title":"Automatic prediction of rejected edits in Stack Overflow","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary; University of Saskatchewan","funders":"Natural Sciences and Engineering Research Council of Canada; Canada First Research Excellence Fund; Global Institute for Water Security, University of Saskatchewan","keywords":"Computer science; Quality (philosophy); World Wide Web; Information retrieval; Software; Precision and recall; Artificial intelligence; Programming language","score_opus":0.0211596723309598,"score_gpt":0.2590476734414302,"score_spread":0.2378880011104704,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4309896433","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6489607,0.00015123164,0.34867114,0.00009534373,0.0008044802,0.0001954544,0.00001850692,0.0010962285,0.0000069035536],"genre_scores_gemma":[0.9647613,0.0000029405753,0.034934886,0.00002906788,0.00007862505,0.00011572805,0.0000108812665,0.000032109067,0.000034450666],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99787915,0.000068550056,0.0004067474,0.00038014795,0.00081389985,0.000451486],"domain_scores_gemma":[0.9980262,0.0012000666,0.000052899322,0.00054745434,0.00005394492,0.00011943477],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005308778,0.00017084033,0.00025835593,0.0004828691,0.00006468678,0.00003291663,0.00080388336,0.00006208589,0.00010598069],"category_scores_gemma":[0.0025359013,0.00019592048,0.000072796225,0.0019801883,0.000014669879,0.0002801796,0.00064081233,0.00057279714,0.000008423865],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000075005455,0.00020674222,0.5424304,0.00021591329,0.00003644719,0.00009473741,0.0011185087,0.44661486,0.00065403845,0.00011922853,0.003880177,0.0046214205],"study_design_scores_gemma":[0.0003774985,0.00014160568,0.5606921,0.000040016785,0.0000023938237,0.00002206347,0.00000815824,0.43594018,0.00037622408,0.000047888883,0.0021804618,0.00017141699],"about_ca_topic_score_codex":0.000020955742,"about_ca_topic_score_gemma":7.276503e-7,"teacher_disagreement_score":0.3158006,"about_ca_system_score_codex":0.00040283348,"about_ca_system_score_gemma":0.00011352434,"threshold_uncertainty_score":0.79894054},"labels":[],"label_agreement":null},{"id":"W4312118684","doi":"10.1007/s10664-022-10246-y","title":"How programmers find online learning resources","year":2022,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Personalization; Resource (disambiguation); World Wide Web; Online learning; Knowledge management","score_opus":0.020907684315663436,"score_gpt":0.251643855097324,"score_spread":0.2307361707816606,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4312118684","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.28682074,0.00014771857,0.70660466,0.0038814358,0.00039243032,0.00019099153,0.000008251738,0.0019315417,0.000022245635],"genre_scores_gemma":[0.74917585,0.0000014452362,0.24786778,0.00054222124,0.00020106997,0.00013924835,0.0000449754,0.00006085002,0.001966575],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9980003,0.000077861034,0.0002493994,0.00050685444,0.00062667276,0.0005389064],"domain_scores_gemma":[0.9988446,0.00038131647,0.00009358608,0.00045715488,0.00007396166,0.00014937369],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00033446154,0.00022240778,0.00021558149,0.0002466844,0.0005201503,0.00039656746,0.0011360787,0.00005570048,0.000048304875],"category_scores_gemma":[0.0010510052,0.00024324007,0.00010618315,0.001953377,0.00002663187,0.00041029626,0.0011176652,0.0009633902,0.000012393897],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014230713,0.00059178355,0.32096943,0.00008970622,0.00016727624,0.00026634315,0.008753995,0.47596818,0.00038286683,0.0015142747,0.008714339,0.1825676],"study_design_scores_gemma":[0.00031569932,0.00021701546,0.038436893,0.000014584965,0.000007896487,0.00007061495,0.00028514964,0.040541776,0.00005212624,0.00009085851,0.9194314,0.00053601514],"about_ca_topic_score_codex":0.000010408454,"about_ca_topic_score_gemma":8.8884104e-7,"teacher_disagreement_score":0.910717,"about_ca_system_score_codex":0.00019288334,"about_ca_system_score_gemma":0.000048217116,"threshold_uncertainty_score":0.9919042},"labels":[],"label_agreement":null},{"id":"W4319079817","doi":"10.1007/s10664-022-10283-7","title":"What makes Ethereum blockchain transactions be processed fast or slow? An empirical study","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Blockchain Technology Applications and Security","field":"Computer Science","cited_by":18,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Queen's University","funders":"","keywords":"Blockchain; Computer science; Database transaction; Smart contract; Transaction processing; Quality of service; Transaction data; Block (permutation group theory); Distributed transaction; Quality (philosophy); Computer security; Database; Computer network","score_opus":0.0366201365717742,"score_gpt":0.3119537041319782,"score_spread":0.275333567560204,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4319079817","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46041998,0.000049466988,0.5316494,0.0034852836,0.00027354682,0.0003786518,0.000004360375,0.003736001,0.0000032810628],"genre_scores_gemma":[0.9809334,0.000020507809,0.017635925,0.00042268456,0.00009388951,0.00048731777,0.000007623601,0.000050214596,0.00034842873],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99770033,0.00006405146,0.00039167868,0.0008283512,0.0004030791,0.00061248924],"domain_scores_gemma":[0.99822307,0.0004062325,0.000054541015,0.00097044656,0.00009612679,0.00024958645],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00044558285,0.00032125498,0.00033965788,0.00034040166,0.0003260796,0.0003236277,0.0011856351,0.00028440735,0.000042114334],"category_scores_gemma":[0.00018249097,0.00027962643,0.000096642616,0.002790078,0.00005580139,0.00051993167,0.00015019678,0.00062188605,0.00006038315],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00023084368,0.014985767,0.13349585,0.0009205943,0.0012826812,0.0017551525,0.14287871,0.29705197,0.0018351165,0.002016428,0.015929209,0.38761768],"study_design_scores_gemma":[0.0016657017,0.0012605104,0.057480335,0.000102409445,0.000068422196,0.00012190863,0.005324951,0.9005259,0.0014957386,0.0012497641,0.028990308,0.0017140509],"about_ca_topic_score_codex":0.000013009281,"about_ca_topic_score_gemma":0.000062209736,"teacher_disagreement_score":0.6034739,"about_ca_system_score_codex":0.00007830774,"about_ca_system_score_gemma":0.000098602664,"threshold_uncertainty_score":0.9999656},"labels":[],"label_agreement":null},{"id":"W4319083508","doi":"10.1007/s10664-022-10276-6","title":"An empirical study of text-based machine learning models for vulnerability detection","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":34,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Manitoba","funders":"","keywords":"Computer science; Machine learning; Vulnerability (computing); Artificial intelligence; Context (archaeology); Empirical research; Source code; Function (biology); Construct (python library); Vulnerability assessment; Code (set theory); Data science; Computer security; Geography; Programming language","score_opus":0.059584817812828254,"score_gpt":0.3385498813244283,"score_spread":0.27896506351160005,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4319083508","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46428123,0.000020002633,0.53321487,0.000043592863,0.0001643452,0.00036281513,0.0000039942283,0.0019085902,5.398801e-7],"genre_scores_gemma":[0.9705984,9.66892e-7,0.028935948,0.00001955082,0.00008921457,0.00025843293,0.000015017208,0.00006674042,0.000015726731],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99721426,0.00014327742,0.0004893462,0.0007793269,0.0007030893,0.00067071477],"domain_scores_gemma":[0.9950135,0.0036066188,0.000067930225,0.00081900746,0.0002099592,0.00028299485],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012916204,0.00030366427,0.00041581548,0.00053752196,0.00018096823,0.00010129824,0.00089414086,0.00015766632,0.0000056872277],"category_scores_gemma":[0.00420384,0.0003041868,0.00014737764,0.0020676884,0.000025108038,0.00047618683,0.0002249824,0.00058764423,0.000011084379],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002075827,0.00026562138,0.15544547,0.00009332769,0.000023284987,0.000008296414,0.0005768446,0.8385668,0.0004713528,0.000004666596,0.00003429801,0.0044892696],"study_design_scores_gemma":[0.0007227359,0.00095461286,0.11086089,0.000016213711,0.000008365895,0.0000023918892,0.000022156912,0.8852826,0.0014958571,0.00009546398,0.00025649692,0.00028222444],"about_ca_topic_score_codex":0.000049450005,"about_ca_topic_score_gemma":0.0000113255865,"teacher_disagreement_score":0.5063172,"about_ca_system_score_codex":0.00017318212,"about_ca_system_score_gemma":0.00007742558,"threshold_uncertainty_score":0.99994105},"labels":[],"label_agreement":null},{"id":"W4319228909","doi":"10.1007/s10664-023-10290-2","title":"Applying declarative analysis to industrial automotive software product line models","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Software Engineering Methodologies","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo; University of Toronto","funders":"General Motors of Canada; Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Automotive industry; Software product line; Software engineering; Computer science; Software; Milestone; Product (mathematics); Programming language; Software development; Engineering","score_opus":0.1475329072300927,"score_gpt":0.34279995238639305,"score_spread":0.19526704515630033,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4319228909","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.013363274,0.00012910571,0.97436893,0.00044826063,0.0009508656,0.00082064164,0.000024188605,0.009888727,0.000006002344],"genre_scores_gemma":[0.05199992,0.000010497053,0.94617265,0.00028792964,0.00042063542,0.0007733606,0.000038265647,0.00009023185,0.00020649856],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99608433,0.00013836927,0.00060833484,0.0013083796,0.00074819266,0.0011123919],"domain_scores_gemma":[0.9949213,0.0030979766,0.00011026998,0.0011263778,0.00028443657,0.00045964192],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001031321,0.0005768208,0.000835253,0.0013908215,0.00020550616,0.00018976854,0.0013991391,0.00024023023,0.000008805082],"category_scores_gemma":[0.015367712,0.0005775847,0.0003249698,0.010368728,0.000034918776,0.0009124732,0.0010851286,0.0007757434,0.0001262664],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007690776,0.00002300351,0.002182854,0.000018818528,0.00031706368,0.000058780315,0.0009045689,0.9799122,0.00006502103,0.00012130638,0.00094124675,0.015447423],"study_design_scores_gemma":[0.0011623875,0.00043488093,0.020402377,0.00019328947,0.0003847988,0.000030776697,0.000108566564,0.94712514,0.0069575235,0.007977585,0.0119103715,0.0033122783],"about_ca_topic_score_codex":0.000011310222,"about_ca_topic_score_gemma":0.0000025327317,"teacher_disagreement_score":0.038636643,"about_ca_system_score_codex":0.000359356,"about_ca_system_score_gemma":0.00012927974,"threshold_uncertainty_score":0.9996676},"labels":[],"label_agreement":null},{"id":"W4319459392","doi":"10.1007/s10664-022-10270-y","title":"Assessing the exposure of software changes","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Deliverable; Computer science; Ranking (information retrieval); Software; Source code; Task (project management); Code (set theory); Set (abstract data type); Software engineering; Reliability engineering; Data mining; Systems engineering; Engineering; Machine learning; Operating system; Programming language","score_opus":0.050832639175595495,"score_gpt":0.3292309896640568,"score_spread":0.2783983504884613,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4319459392","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.304438,0.00037395724,0.69024575,0.0010362096,0.0006793432,0.00017783104,0.0000039127517,0.003038458,0.0000065640934],"genre_scores_gemma":[0.9344052,0.000027867149,0.06482815,0.00013551551,0.0002711224,0.00009237159,0.000008750244,0.000074221156,0.00015681815],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9976913,0.00005613997,0.00030622756,0.00045092087,0.00078383397,0.00071160885],"domain_scores_gemma":[0.99439,0.0043164343,0.00006608803,0.0009273404,0.00013289094,0.00016725274],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008170631,0.00025833157,0.00029703963,0.00034300494,0.00014392077,0.0002352464,0.0015144008,0.0001354267,0.000016450482],"category_scores_gemma":[0.006404017,0.0002006878,0.00012366132,0.0026243627,0.000052679217,0.0004491071,0.00072962727,0.00046402036,0.000086504224],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000005261877,0.00010943451,0.76630324,0.0007176657,0.00020956404,0.0002604603,0.0034921684,0.1377815,0.0023970462,0.00047068283,0.01535077,0.07290219],"study_design_scores_gemma":[0.000399778,0.00014966709,0.9408216,0.00023449823,0.0000145462745,0.00004715462,0.00006894533,0.036205012,0.0051258574,0.00020512159,0.016047854,0.0006799612],"about_ca_topic_score_codex":0.000008823455,"about_ca_topic_score_gemma":0.0000010766835,"teacher_disagreement_score":0.62996715,"about_ca_system_score_codex":0.00008046658,"about_ca_system_score_gemma":0.00008428729,"threshold_uncertainty_score":0.81838113},"labels":[],"label_agreement":null},{"id":"W4320039274","doi":"10.1007/s10664-022-10262-y","title":"Towards understanding quality challenges of the federated learning for neural networks: a first look from the lens of robustness","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Privacy-Preserving Technologies in Data","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; University of Alberta; University of Calgary","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Robustness (evolution); Computer science; Machine learning; Artificial intelligence; Artificial neural network; Deep learning; Data mining; Deep neural networks; Process (computing)","score_opus":0.1321526516824001,"score_gpt":0.30885220074616543,"score_spread":0.17669954906376534,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4320039274","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06699679,0.00051392394,0.9127201,0.018105604,0.00043479924,0.00021056113,0.0000149639045,0.0009987006,0.000004585446],"genre_scores_gemma":[0.97924525,0.00008672894,0.020507406,0.00003562415,0.0000635182,0.000028289465,0.000008047055,0.000022482473,0.0000026541477],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998506,0.000087344066,0.0003601663,0.0003580509,0.00031390294,0.0003745155],"domain_scores_gemma":[0.99457246,0.0032474285,0.00015702732,0.001928018,0.00006338903,0.000031678028],"candidate_categories":["metaresearch","open_science"],"consensus_categories":["open_science"],"category_scores_codex":[0.00070689217,0.00017457295,0.00028623766,0.000055242013,0.00023370936,0.000063774365,0.0068540312,0.00014013765,0.0000020619268],"category_scores_gemma":[0.025811492,0.00011729024,0.00013256555,0.00084395637,0.000087522836,0.00022562349,0.013131669,0.00039313186,4.662367e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010417277,0.00002166088,0.012504787,0.00013768024,0.00008294819,0.000002005891,0.0004505349,0.97741604,0.000035744826,0.0011966154,0.0046765134,0.0034650327],"study_design_scores_gemma":[0.00018487762,0.00003058714,0.031139765,0.0001239185,0.000008427663,8.6860877e-7,0.00016568297,0.9642054,0.0001998284,0.0035616243,0.00023328402,0.0001457657],"about_ca_topic_score_codex":0.000051098887,"about_ca_topic_score_gemma":0.000033361976,"teacher_disagreement_score":0.91224843,"about_ca_system_score_codex":0.00010824232,"about_ca_system_score_gemma":0.000034096505,"threshold_uncertainty_score":0.99851936},"labels":[],"label_agreement":null},{"id":"W4320882938","doi":"10.1007/s10664-022-10271-x","title":"Refactoring practices in the context of data-intensive systems","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec – Nature et technologies; Natural Sciences and Engineering Research Council of Canada","keywords":"Code refactoring; Computer science; Data access; Context (archaeology); Maintainability; Database; Software engineering; Software; Programming language","score_opus":0.16027050351680053,"score_gpt":0.3800548519220058,"score_spread":0.21978434840520525,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4320882938","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6203074,0.0010403085,0.3732488,0.0014462819,0.0015092591,0.00056050566,0.000027106267,0.001840361,0.000020007677],"genre_scores_gemma":[0.9951123,0.000028576014,0.004554164,0.0000712671,0.00012564848,0.000041559855,0.000016257234,0.000025661551,0.000024542016],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9978617,0.00008834242,0.00037577542,0.0004812245,0.000695383,0.0004975727],"domain_scores_gemma":[0.99038017,0.007827806,0.00013192532,0.0014097723,0.00016494526,0.00008540061],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0015421484,0.00017811674,0.00027798186,0.0003255376,0.000044258293,0.00014829225,0.0026880577,0.000087641085,0.000002076143],"category_scores_gemma":[0.020186497,0.00013762238,0.00004615837,0.002045355,0.000024963148,0.00082892604,0.00084199075,0.00049043575,0.000057618254],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002494992,0.0001523428,0.8171542,0.001138858,0.0002171477,0.0010908124,0.015947977,0.120519504,0.0009381273,0.0019714008,0.02606054,0.014784112],"study_design_scores_gemma":[0.0006993068,0.00016117067,0.47689748,0.0004524116,0.000016239324,0.00012001785,0.0017395368,0.46005067,0.00047290983,0.000060149436,0.05859949,0.0007306314],"about_ca_topic_score_codex":0.00016526942,"about_ca_topic_score_gemma":0.000004576747,"teacher_disagreement_score":0.37480494,"about_ca_system_score_codex":0.000078784564,"about_ca_system_score_gemma":0.00007037201,"threshold_uncertainty_score":0.9880669},"labels":[],"label_agreement":null},{"id":"W4321093054","doi":"10.1007/s10664-022-10267-7","title":"Vulnerability management in Linux distributions","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Security and Verification in Computing","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Vulnerability (computing); Vulnerability management; Computer science; Revenue; Software; Computer security; Linux kernel; Secure coding; Open source; Upstream (networking); Vulnerability assessment; Operating system; Business; Software security assurance; Telecommunications; Accounting; Information security","score_opus":0.037604282998133204,"score_gpt":0.30874347030439714,"score_spread":0.27113918730626396,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4321093054","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.115291916,0.000026381394,0.8822191,0.0006946233,0.00037974888,0.00009361474,0.0000021925384,0.0012282496,0.000064180174],"genre_scores_gemma":[0.93824476,0.000008094835,0.061493836,0.00008043456,0.00006552161,0.000043650944,0.000013224371,0.000007828374,0.000042640873],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99899036,0.000024650886,0.0002139831,0.00030889243,0.00016818382,0.00029394688],"domain_scores_gemma":[0.99922824,0.0002952197,0.000019127681,0.0003634793,0.000020933967,0.000072970564],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003584366,0.00009428723,0.00010037258,0.00014208352,0.000080549806,0.000068088055,0.00044575383,0.000048575796,0.000008079214],"category_scores_gemma":[0.0002931904,0.000102300175,0.000046391022,0.0015960647,0.000010665273,0.00015158515,0.0002862357,0.00018880914,0.00012880687],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007349639,0.0004537321,0.27415764,0.00040963225,0.00008948837,0.0003272883,0.003453212,0.46423155,0.0000718148,0.12976421,0.013923831,0.11311023],"study_design_scores_gemma":[0.00018196317,0.000010854001,0.5056603,0.000030660958,0.0000018498519,0.000003532489,0.0000115316,0.46872875,0.00009947048,0.002251256,0.022802634,0.00021718713],"about_ca_topic_score_codex":0.0000046841737,"about_ca_topic_score_gemma":9.4711334e-7,"teacher_disagreement_score":0.82295287,"about_ca_system_score_codex":0.000104633335,"about_ca_system_score_gemma":0.00001378234,"threshold_uncertainty_score":0.417168},"labels":[],"label_agreement":null},{"id":"W4321787049","doi":"10.1007/s10664-022-10272-w","title":"Semantically-enhanced topic recommendation systems for software projects","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Metadata; Recommender system; Software; Information retrieval; World Wide Web; Software engineering; Data science; Programming language","score_opus":0.050022828276608984,"score_gpt":0.3143586557869092,"score_spread":0.2643358275103002,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4321787049","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.055141248,0.00009515486,0.9356018,0.00057689444,0.0017335081,0.00076848286,0.000012073665,0.006062954,0.000007913521],"genre_scores_gemma":[0.76207006,0.00004961091,0.23366036,0.00020465735,0.0009495391,0.0016110435,0.00016719871,0.0001994828,0.0010880646],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99738055,0.000041546635,0.00042560793,0.00072933757,0.00047043228,0.00095250073],"domain_scores_gemma":[0.9952188,0.0036401993,0.000056014673,0.0006197763,0.00020491217,0.00026025504],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006755839,0.00031266312,0.00035868224,0.0004339781,0.00020263872,0.00040386897,0.00085037766,0.00019219011,0.000011904284],"category_scores_gemma":[0.008461006,0.0003160505,0.00014380638,0.0017083146,0.000022613225,0.00047206867,0.00037861735,0.00034765637,0.00018878642],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008777827,0.0005038369,0.18832728,0.006749321,0.0006954838,0.0002494555,0.0059414143,0.52276844,0.0035474622,0.004452484,0.14149873,0.12517832],"study_design_scores_gemma":[0.0018709049,0.00051244395,0.1146839,0.00047333952,0.000029792032,0.000050445105,0.000051955525,0.7572559,0.0036304505,0.00046564318,0.11904717,0.0019280198],"about_ca_topic_score_codex":0.000009882524,"about_ca_topic_score_gemma":6.6824475e-7,"teacher_disagreement_score":0.7069288,"about_ca_system_score_codex":0.00021937341,"about_ca_system_score_gemma":0.00012824309,"threshold_uncertainty_score":0.99992913},"labels":[],"label_agreement":null},{"id":"W4323925049","doi":"10.1007/s10664-022-10277-5","title":"Registered reports in software engineering","year":2023,"lang":"en","type":"editorial","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Software engineering; Computer science; Engineering; Systems engineering","score_opus":0.027683895632145253,"score_gpt":0.2998760716857518,"score_spread":0.27219217605360657,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4323925049","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.001034013,0.0008124347,0.3279935,0.00009780284,0.6608652,0.0005109036,0.0000421967,0.008639998,0.000003955327],"genre_scores_gemma":[0.0023283693,0.0003326848,0.2531723,0.00007917738,0.7373201,0.0014018442,0.00076625485,0.0019346945,0.0026645444],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9900621,0.000081693695,0.0017899063,0.0026078888,0.0030079004,0.00245047],"domain_scores_gemma":[0.97617763,0.018932827,0.00032619893,0.0034293148,0.0003786323,0.00075542246],"candidate_categories":["metaresearch","metaepi_narrow","research_integrity"],"consensus_categories":["metaepi_narrow","research_integrity"],"category_scores_codex":[0.0020047626,0.0013771163,0.0015809903,0.0019466423,0.00011021748,0.0006538834,0.0030875937,0.0020569265,0.000021354148],"category_scores_gemma":[0.1465379,0.0015752374,0.0004991703,0.003821583,0.0000459256,0.0007652278,0.0020826967,0.004505586,0.00027872765],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000071303793,0.0000649799,0.0112050995,0.0010434014,0.00010729417,0.0051641446,0.00023917055,0.059695236,0.000014918477,0.000007693468,0.92175776,0.00069320004],"study_design_scores_gemma":[0.00060232077,0.00012527723,0.016365727,0.0018096165,0.000025580746,0.000077397366,0.0000028612105,0.012547206,0.00007207957,0.000046358207,0.9659832,0.0023423815],"about_ca_topic_score_codex":0.00013643192,"about_ca_topic_score_gemma":0.00001906115,"teacher_disagreement_score":0.14453314,"about_ca_system_score_codex":0.0015586412,"about_ca_system_score_gemma":0.00074666797,"threshold_uncertainty_score":0.99989796},"labels":[],"label_agreement":null},{"id":"W4324374545","doi":"10.1007/s10664-022-10260-0","title":"Evaluating ensemble imputation in software effort estimation","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"","keywords":"Imputation (statistics); Computer science; Missing data; Statistics; Data mining; Regression; Mathematics; Machine learning","score_opus":0.054878849205720864,"score_gpt":0.3653485525550296,"score_spread":0.3104697033493088,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4324374545","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.40228146,0.000049494527,0.5942717,0.0001286237,0.0003346319,0.00019449319,0.0000011548209,0.0027354318,0.000002979969],"genre_scores_gemma":[0.6401286,0.00000499726,0.35944113,0.000049890332,0.000085469495,0.00013729004,0.00002981282,0.000055904533,0.00006686425],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99720025,0.00004370179,0.0004599646,0.0006275141,0.0008452049,0.00082336506],"domain_scores_gemma":[0.9966041,0.0024933738,0.000053537064,0.00054915395,0.00010314331,0.00019668086],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012018225,0.00027544398,0.0002807828,0.0007754887,0.0000972728,0.00018109615,0.0006960167,0.00015987235,0.000011320672],"category_scores_gemma":[0.00977081,0.00030241493,0.00009235963,0.0032230837,0.000015181941,0.00067851134,0.00040072232,0.0004584561,0.0003563159],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000027990375,0.00001972154,0.119501024,0.00009206953,0.000009941994,0.00006550624,0.00045326335,0.8373654,0.00018419491,0.00005513284,0.00044517897,0.041805774],"study_design_scores_gemma":[0.00028518567,0.000057738864,0.331961,0.00007939721,0.0000023034863,0.000013113846,0.000003702219,0.6663236,0.00039437794,0.00045676986,0.00016202079,0.00026080062],"about_ca_topic_score_codex":0.000020799116,"about_ca_topic_score_gemma":0.0000023609261,"teacher_disagreement_score":0.23784718,"about_ca_system_score_codex":0.00034316428,"about_ca_system_score_gemma":0.000121887635,"threshold_uncertainty_score":0.9999428},"labels":[],"label_agreement":null},{"id":"W4360948905","doi":"10.1007/s10664-022-10278-4","title":"Empirical analysis of security vulnerabilities in Python packages","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":72,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec à Montréal; Concordia University","funders":"","keywords":"Python (programming language); Computer science; Secure coding; Software security assurance; Software engineering; Security bug; Vulnerability management; Software; Software development; Software bug; Computer security; Ecosystem; Reusability; Programming language; Vulnerability assessment; Information security; Ecology","score_opus":0.02247784916848785,"score_gpt":0.3112819751728626,"score_spread":0.2888041260043748,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4360948905","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.40340132,0.000045744964,0.59426403,0.00017784214,0.00010677413,0.000078455945,0.0000062997833,0.0019063468,0.000013206025],"genre_scores_gemma":[0.9566135,0.00001703142,0.043172177,0.000063951586,0.00002878809,0.000048187907,0.000007488195,0.000019639097,0.000029261038],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983647,0.00006441235,0.000423045,0.00044031875,0.0003324565,0.00037506933],"domain_scores_gemma":[0.99827945,0.00095482747,0.00006449285,0.0005346041,0.000068435285,0.00009820632],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00043436873,0.00018487156,0.00043679966,0.0011522945,0.00003669563,0.000033129723,0.00047702604,0.00012427464,0.000016068934],"category_scores_gemma":[0.0013533139,0.00018691251,0.00019011692,0.006206218,0.0000353628,0.00033169254,0.00031016173,0.00030253542,0.000010265586],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012068047,0.00014694716,0.64885,0.00018603502,0.00025730926,0.00010642979,0.005330871,0.3355256,0.00068347395,0.000561448,0.0015135346,0.0068262415],"study_design_scores_gemma":[0.00023950399,0.000132764,0.6120567,0.00005783767,0.000054117092,0.000005497299,0.00008499203,0.36535573,0.012660118,0.004420194,0.004331416,0.0006011626],"about_ca_topic_score_codex":0.000027037526,"about_ca_topic_score_gemma":0.00001543889,"teacher_disagreement_score":0.55321217,"about_ca_system_score_codex":0.00012555077,"about_ca_system_score_gemma":0.000027230659,"threshold_uncertainty_score":0.7622071},"labels":[],"label_agreement":null},{"id":"W4362584426","doi":"10.1007/s10664-022-10282-8","title":"Towards a change taxonomy for machine learning pipelines","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; York University; Ansys (Canada); Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Metadata; Replicate; Pipeline (software); Taxonomy (biology); Implementation; Data science; Fork (system call); Source code; Information retrieval; World Wide Web; Software engineering; Programming language; Ecology","score_opus":0.10899159167165356,"score_gpt":0.316721378863043,"score_spread":0.20772978719138946,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4362584426","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.017213577,0.00039237697,0.97370934,0.00093419704,0.0007699691,0.00062283553,0.000012491853,0.006337363,0.0000078157855],"genre_scores_gemma":[0.53806365,0.00009906943,0.45363796,0.00044867522,0.0019625125,0.0043466175,0.00012238063,0.00025771506,0.0010613983],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99769634,0.000026167258,0.0002951347,0.00058898045,0.000442971,0.00095038186],"domain_scores_gemma":[0.99710137,0.0019750427,0.00004015173,0.0004734985,0.00012435851,0.00028559077],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006436497,0.00030156664,0.00031288902,0.0004710841,0.00014799206,0.00016908153,0.0009382021,0.00013642827,0.000015133762],"category_scores_gemma":[0.006345161,0.00030060147,0.00017460412,0.0017199927,0.000017720678,0.00042599178,0.00053613074,0.00043298,0.00017600323],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000042893236,0.00017928316,0.5485391,0.0013374715,0.00026393557,0.00032376748,0.0025833035,0.16155982,0.00054394314,0.0014604245,0.02679804,0.25636804],"study_design_scores_gemma":[0.00051198,0.00013760051,0.066999234,0.000066319335,0.000006814629,0.000018173017,0.000005791951,0.59105355,0.00045543115,0.000105559695,0.34007567,0.0005638945],"about_ca_topic_score_codex":0.000027920583,"about_ca_topic_score_gemma":0.0000014754388,"teacher_disagreement_score":0.52085006,"about_ca_system_score_codex":0.00012319438,"about_ca_system_score_gemma":0.000058407364,"threshold_uncertainty_score":0.9999446},"labels":[],"label_agreement":null},{"id":"W4362606928","doi":"10.1007/s10664-023-10291-1","title":"Bugs in machine learning-based systems: a faultload benchmark","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; Polytechnique Montréal","funders":"","keywords":"Benchmark (surveying); Computer science; Software portability; Debugging; Software bug; Software quality; Usability; Software engineering; Software; Relevance (law); Benchmarking; Machine learning; Software development; Operating system","score_opus":0.017192324944400338,"score_gpt":0.26297108381207424,"score_spread":0.2457787588676739,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4362606928","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.045049958,0.00025037344,0.95028144,0.00048720159,0.00081800926,0.00018848383,0.0000023846976,0.0028802599,0.000041880154],"genre_scores_gemma":[0.97500914,0.000006373434,0.024359908,0.00011101738,0.00015437268,0.0000656977,0.00002956101,0.00005313547,0.00021078589],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977197,0.00013191396,0.00040013625,0.00058175763,0.0004856059,0.0006808754],"domain_scores_gemma":[0.99820846,0.0010521453,0.00007873627,0.00044415751,0.000044155786,0.00017231492],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00070397754,0.00029058306,0.00035161115,0.0005229392,0.000120627206,0.00015367384,0.00084342086,0.00015695728,0.000019877707],"category_scores_gemma":[0.0025048403,0.0002947534,0.00010029457,0.0023579604,0.000020261816,0.00031303495,0.0003927077,0.0008900735,0.00017937884],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000035272642,0.000016303822,0.10670644,0.0000617139,0.0000078552575,0.00014201835,0.00020925423,0.8916973,0.000023076756,0.00011677886,0.00019761997,0.0008181003],"study_design_scores_gemma":[0.00043361203,0.000046168636,0.035583455,0.00010188586,0.0000031664415,0.0000075340376,0.000009821753,0.9504763,0.000024598621,0.00002678515,0.012956804,0.0003298963],"about_ca_topic_score_codex":0.00012223779,"about_ca_topic_score_gemma":0.0000056142217,"teacher_disagreement_score":0.9299592,"about_ca_system_score_codex":0.00019493396,"about_ca_system_score_gemma":0.00007151637,"threshold_uncertainty_score":0.99995047},"labels":[],"label_agreement":null},{"id":"W4366428239","doi":"10.1007/s10664-022-10279-3","title":"Introduction to the special issue on program comprehension","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Teaching and Learning Programming","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Program comprehension; Computer science; Comprehension; Programming language; Software","score_opus":0.02009561219030038,"score_gpt":0.2880236548705743,"score_spread":0.2679280426802739,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4366428239","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09988373,0.00003206428,0.81747186,0.0625978,0.007636388,0.0006887044,9.019853e-7,0.011629896,0.000058637008],"genre_scores_gemma":[0.25193474,0.0000067371343,0.61620605,0.0022375265,0.12501645,0.00040372,0.00004401793,0.0001623075,0.0039884737],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99878836,0.000048954425,0.00013711522,0.00036837626,0.00030038608,0.0003568369],"domain_scores_gemma":[0.999205,0.00022439189,0.000021293725,0.00041424925,0.000025103145,0.00010994991],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00035698857,0.00013413068,0.00011283966,0.00013295731,0.00020864572,0.00019217863,0.0004656197,0.000050642968,0.000018114555],"category_scores_gemma":[0.0006768729,0.000099589786,0.00005840999,0.0010738756,0.000008916055,0.00009263279,0.0002065921,0.00040309972,0.0013621333],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000029918608,0.000028604418,0.00063444173,0.000007867807,0.0000061081964,0.000006423578,0.0010251077,0.16873899,0.0000071526583,0.0001168789,0.14158162,0.6878438],"study_design_scores_gemma":[0.000055335953,0.00014959268,0.024267256,0.000016418659,0.000001713687,0.0000042452707,0.000009999337,0.02255828,0.000028894847,0.000005338886,0.95277596,0.00012698561],"about_ca_topic_score_codex":0.0000052331748,"about_ca_topic_score_gemma":7.729036e-7,"teacher_disagreement_score":0.8111943,"about_ca_system_score_codex":0.000051112456,"about_ca_system_score_gemma":0.000010145911,"threshold_uncertainty_score":0.9994154},"labels":[],"label_agreement":null},{"id":"W4366990909","doi":"10.1007/s10664-023-10292-0","title":"Ranking code clones to support maintenance activities","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University","funders":"","keywords":"clone (Java method); Code refactoring; Software maintenance; Commit; Computer science; Java; Code (set theory); Software evolution; Code reuse; Programming language; Software quality; Software system; Software development; Software; Database; Biology; Software construction; Genetics; Gene","score_opus":0.036875034545710464,"score_gpt":0.3080833253675743,"score_spread":0.2712082908218639,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4366990909","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.23774174,0.000029376653,0.75493324,0.0008701523,0.0008758429,0.00019688092,0.000008996386,0.00532072,0.000023077244],"genre_scores_gemma":[0.91661894,0.000013918847,0.081241,0.00041096986,0.0003072181,0.00017867835,0.000010571834,0.00010189958,0.001116798],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9971921,0.000026707386,0.0002994001,0.00065317756,0.0007204309,0.0011081767],"domain_scores_gemma":[0.9968102,0.0019607286,0.000028924143,0.0007405601,0.000071572926,0.00038799745],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005906024,0.0003015956,0.00032177664,0.0005555145,0.0001263229,0.00022985699,0.0012555426,0.00011388476,0.000030144163],"category_scores_gemma":[0.0036146038,0.00030786998,0.00012293448,0.0024326332,0.0000232733,0.00047073455,0.0008215052,0.00040782368,0.0007267226],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000404166,0.00013179847,0.3259587,0.0005540971,0.00020169743,0.001584798,0.005674681,0.44985732,0.0043324013,0.0015900199,0.15954453,0.050529562],"study_design_scores_gemma":[0.001148454,0.00051310856,0.5863791,0.00038679174,0.000014447737,0.00018408021,0.00006530244,0.10192947,0.011524204,0.0003634168,0.29512691,0.002364685],"about_ca_topic_score_codex":0.0000068165905,"about_ca_topic_score_gemma":0.0000022253619,"teacher_disagreement_score":0.67887723,"about_ca_system_score_codex":0.0001980304,"about_ca_system_score_gemma":0.000086322165,"threshold_uncertainty_score":0.99993736},"labels":[],"label_agreement":null},{"id":"W4377139078","doi":"10.1007/s10664-023-10300-3","title":"Learning to Predict Code Review Completion Time In Modern Code Review","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Code review; Computer science; Context (archaeology); Coding (social sciences); Code (set theory); Process (computing); Best practice; Quality (philosophy); Software engineering; Software quality; Software development; Software","score_opus":0.03732351305167277,"score_gpt":0.3175656528412004,"score_spread":0.28024213978952767,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4377139078","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.019920243,0.02642496,0.9392123,0.0057001417,0.00038530433,0.0013557817,0.00001844396,0.0069545335,0.000028294744],"genre_scores_gemma":[0.33024386,0.15724534,0.4605459,0.03082517,0.0023456067,0.006235922,0.0011440213,0.0019104934,0.009503715],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9967684,0.00012032615,0.0005911176,0.0007495865,0.0008831168,0.0008874669],"domain_scores_gemma":[0.9971321,0.0015823268,0.000052961077,0.00073727797,0.00011743376,0.00037787974],"candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0016181535,0.0003325176,0.00064631173,0.00041982002,0.00007602699,0.000076675875,0.001152269,0.00010329314,0.000052109597],"category_scores_gemma":[0.010448774,0.00034757666,0.0001373789,0.0035853528,0.000014864136,0.00029950566,0.0007023069,0.00075726636,0.0013008277],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009871421,0.0001236713,0.09819517,0.018355535,0.00008262094,0.00051184127,0.00085082965,0.63788897,0.0005510251,0.00004736949,0.18696111,0.056421977],"study_design_scores_gemma":[0.00043083573,0.00021557063,0.14810039,0.027604733,0.00002189878,0.0000620368,0.0000021356343,0.6504895,0.00006320106,0.00005254851,0.17187881,0.0010783589],"about_ca_topic_score_codex":0.000007059699,"about_ca_topic_score_gemma":0.0000016180834,"teacher_disagreement_score":0.47866642,"about_ca_system_score_codex":0.0002945855,"about_ca_system_score_gemma":0.000075815595,"threshold_uncertainty_score":0.9998976},"labels":[],"label_agreement":null},{"id":"W4377140397","doi":"10.1007/s10664-023-10313-y","title":"Relationship between diversity of collaborative group members’ race and ethnicity and the frequency of their collaborative contributions in GitHub","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia; University of Waterloo","funders":"","keywords":"Ethnic group; Diversity (politics); Race (biology); Empirical research; Cultural diversity; Collaborative learning; Computer science; Knowledge management; Sociology; Gender studies; Mathematics","score_opus":0.029149707248459115,"score_gpt":0.28394597866501314,"score_spread":0.25479627141655403,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4377140397","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.803874,0.00022368817,0.19373263,0.0014067469,0.000055885113,0.0003781274,0.00016252046,0.00015014395,0.0000162691],"genre_scores_gemma":[0.9883027,0.000012644831,0.011592189,0.000025790432,0.000015297846,0.000022196196,0.0000128362935,0.000007640574,0.000008690674],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9987442,0.00018124751,0.00037835466,0.00025790717,0.0002316202,0.00020667871],"domain_scores_gemma":[0.99312574,0.00605164,0.00015420635,0.0002637679,0.0003393673,0.000065299115],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010027587,0.00014517819,0.00037087762,0.00021738108,0.00022243163,0.000035414527,0.00034130883,0.00010796024,0.0000018836831],"category_scores_gemma":[0.0039088316,0.000115963485,0.000034854813,0.0044454955,0.00026436397,0.00038167133,0.0006965361,0.00029873004,0.0000011915774],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000063736165,0.000014299593,0.9719951,0.000025873382,0.00003065806,0.000002023718,0.01141238,0.00092823076,0.000021304018,0.015330911,0.000098243574,0.00013458615],"study_design_scores_gemma":[0.00084496965,0.000036243742,0.9917688,0.00004336214,0.000012844323,0.0000010172083,0.00036789785,0.00205677,0.0001321713,0.0045188954,0.00008950066,0.00012755668],"about_ca_topic_score_codex":0.00010492614,"about_ca_topic_score_gemma":0.000041765143,"teacher_disagreement_score":0.18442872,"about_ca_system_score_codex":0.00006733256,"about_ca_system_score_gemma":0.000076158045,"threshold_uncertainty_score":0.47288534},"labels":[],"label_agreement":null},{"id":"W4378072131","doi":"10.1007/s10664-023-10287-x","title":"Rubbing salt in the wound? A large-scale investigation into the effects of refactoring on security","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"Natural Sciences and Engineering Research Council of Canada; Ministero dell’Istruzione, dell’Università e della Ricerca; Università degli Studi di Salerno; Schweizerischer Nationalfonds zur Förderung der Wissenschaftlichen Forschung; National Science Foundation","keywords":"Code refactoring; Computer science; Technical debt; Source code; Software engineering; Software; Code (set theory); Source lines of code; Software development; Computer security; Programming language; Set (abstract data type)","score_opus":0.017211188368017438,"score_gpt":0.2799320263002199,"score_spread":0.26272083793220247,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378072131","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9078957,0.00012931324,0.08963843,0.0007295521,0.0005313822,0.0003208953,8.859742e-7,0.00074917736,0.0000046712858],"genre_scores_gemma":[0.9965361,0.000013664154,0.0030016117,0.00017327513,0.000120334764,0.000109117165,0.0000036169172,0.000028368755,0.000013933514],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9978916,0.0001295098,0.0002940606,0.00037790183,0.0007507701,0.0005561519],"domain_scores_gemma":[0.99118876,0.007891758,0.000046349352,0.0007310732,0.000046001067,0.00009603991],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0013552114,0.00020558185,0.00021181785,0.00030448448,0.00012816736,0.00011186556,0.0012195389,0.000107035,0.0000014337073],"category_scores_gemma":[0.0067621013,0.00014040136,0.00008739368,0.0026265197,0.00003382858,0.00024327257,0.00036317154,0.00072510995,0.000031326257],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001839034,0.0002079646,0.70437884,0.0017295468,0.000087250846,0.00023648336,0.16525334,0.11275876,0.0035368758,0.0024932332,0.0034280766,0.0058712293],"study_design_scores_gemma":[0.00060483784,0.00021819273,0.87923527,0.0005444887,0.000008146007,0.00000882698,0.00018121004,0.10548409,0.008140718,0.0018890846,0.003223157,0.00046197983],"about_ca_topic_score_codex":0.000053426167,"about_ca_topic_score_gemma":0.000012418657,"teacher_disagreement_score":0.17485642,"about_ca_system_score_codex":0.00016141911,"about_ca_system_score_gemma":0.000051611398,"threshold_uncertainty_score":0.80953556},"labels":[],"label_agreement":null},{"id":"W4378190425","doi":"10.1007/s10664-023-10325-8","title":"18 million links in commit messages: purpose, evolution, and decay","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Wikis in Education and Collaboration","field":"Social Sciences","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Precursory Research for Embryonic Science and Technology; Japan Society for the Promotion of Science; McGill University","keywords":"Commit; Computer science; Database","score_opus":0.03138405792739127,"score_gpt":0.3356004717254505,"score_spread":0.3042164137980592,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378190425","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.963207,0.00078158494,0.023190591,0.008381096,0.0013656052,0.00038461195,0.000005854055,0.0010891777,0.001594501],"genre_scores_gemma":[0.9976329,0.00011406735,0.0009608848,0.00012869803,0.0002424091,0.000042484975,0.00001095054,0.000012891052,0.00085466745],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99915385,0.00006065979,0.00015909679,0.0001647227,0.0002208687,0.00024077116],"domain_scores_gemma":[0.9994201,0.00029313038,0.000022504631,0.00009191728,0.00006406473,0.00010827065],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00047658745,0.00007902636,0.00009980261,0.00018457836,0.00019240344,0.00006541122,0.00010120257,0.0001712426,0.000075465025],"category_scores_gemma":[0.0010234966,0.000086372784,0.000019818872,0.0015421534,0.00004647076,0.00017125685,0.00003147033,0.0002077043,0.00005170354],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008243583,0.00007774189,0.90333766,0.000053147687,0.000013313695,0.000011321819,0.019871032,0.007472419,0.00007057827,0.0070602577,0.053935215,0.008089067],"study_design_scores_gemma":[0.00021581196,0.000016260396,0.6009762,0.000051352145,0.0000048177276,5.923283e-7,0.0020690984,0.0013358658,0.00003266885,0.0007717556,0.39430913,0.00021644661],"about_ca_topic_score_codex":0.00015578697,"about_ca_topic_score_gemma":0.00033716526,"teacher_disagreement_score":0.3403739,"about_ca_system_score_codex":0.00019029956,"about_ca_system_score_gemma":0.00011241247,"threshold_uncertainty_score":0.352218},"labels":[],"label_agreement":null},{"id":"W4378474030","doi":"10.1007/s10664-023-10317-8","title":"Using the uniqueness of global identifiers to determine the provenance of Python software source code","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Identifier; Python (programming language); Computer science; Source code; Software; Open source; Source lines of code; Database; Programming language; Operating system","score_opus":0.05727007110122434,"score_gpt":0.33277357167196664,"score_spread":0.2755035005707423,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378474030","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.391875,0.00010236963,0.60653496,0.00027619736,0.00032639562,0.00027066818,0.000014032423,0.00059931027,0.0000010879356],"genre_scores_gemma":[0.93402505,0.000005732264,0.065610416,0.00008984856,0.0000922676,0.000060860442,0.00000230321,0.00005201456,0.000061512306],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9975078,0.000078311234,0.0004857901,0.00045552937,0.00081758894,0.000654984],"domain_scores_gemma":[0.99606526,0.0023109617,0.00011217272,0.0011327161,0.00021254474,0.00016635976],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009006191,0.0002642915,0.00034812946,0.00014053837,0.00013387192,0.00008707017,0.0020820925,0.00010738918,0.0000034344264],"category_scores_gemma":[0.005694861,0.00018493066,0.00016078439,0.0035916793,0.0000979251,0.00021496887,0.0010185533,0.00030822048,0.000012731999],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017563792,0.00003658905,0.1754474,0.0003003596,0.000070451024,0.00003204285,0.0011952225,0.80986255,0.0020597095,0.00024165619,0.0010258725,0.009710601],"study_design_scores_gemma":[0.0007027884,0.00032384848,0.55959076,0.00084936904,0.00005349954,0.0001256068,0.00012367088,0.40312886,0.023643082,0.00065919396,0.009665016,0.0011343166],"about_ca_topic_score_codex":0.00006861271,"about_ca_topic_score_gemma":0.000006025543,"teacher_disagreement_score":0.54215,"about_ca_system_score_codex":0.00019391127,"about_ca_system_score_gemma":0.00015483971,"threshold_uncertainty_score":0.75412536},"labels":[],"label_agreement":null},{"id":"W4384039146","doi":"10.1007/s10664-023-10342-7","title":"BTLink : automatic link recovery between issues and commits based on pre-trained BERT model","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Science Foundation of Jiangsu Province; National Natural Science Foundation of China","keywords":"Commit; Computer science; Traceability; Identifier; Software; Software engineering; Classifier (UML); Data mining; Machine learning; Artificial intelligence; Database; Programming language","score_opus":0.035326283152714626,"score_gpt":0.3095404327340525,"score_spread":0.2742141495813379,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4384039146","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2118092,0.0000966363,0.7795288,0.0025482294,0.00023099064,0.00027470806,0.000014062361,0.0054856176,0.000011770884],"genre_scores_gemma":[0.8188981,0.000018685074,0.17979936,0.00029635688,0.0003423572,0.00011324184,0.000035674024,0.0001190601,0.00037714935],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9973046,0.00005226497,0.000390774,0.0006855703,0.00076031627,0.00080644246],"domain_scores_gemma":[0.9941942,0.0044707768,0.000043266013,0.0008368078,0.00007072308,0.00038420424],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006940857,0.00037858298,0.000434845,0.00055762957,0.00013506923,0.00023984503,0.00089022215,0.00022445105,0.000010635874],"category_scores_gemma":[0.004029626,0.0003775264,0.00011786945,0.0014229452,0.000034432665,0.00034738396,0.00043444743,0.0005986484,0.00011083138],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000059738663,0.000033540535,0.07411813,0.00028939135,0.000052914802,0.000058036163,0.0004299364,0.8894247,0.000047682595,0.00005483059,0.0046731,0.03081177],"study_design_scores_gemma":[0.0002746191,0.00012267898,0.20422724,0.00014594961,0.0000066815,0.0000024782153,9.992957e-7,0.793617,0.00014042307,0.00023179383,0.00088764774,0.00034250534],"about_ca_topic_score_codex":0.000010432096,"about_ca_topic_score_gemma":4.985867e-7,"teacher_disagreement_score":0.6070889,"about_ca_system_score_codex":0.00014774942,"about_ca_system_score_gemma":0.00009431694,"threshold_uncertainty_score":0.9998677},"labels":[],"label_agreement":null},{"id":"W4384408554","doi":"10.1007/s10664-023-10345-4","title":"Towards a taxonomy of Roxygen documentation in R packages","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"National Research Council Canada; Natural Sciences and Engineering Research Council of Canada; University of Saskatchewan","keywords":"Documentation; Software documentation; Computer science; Taxonomy (biology); Software engineering; Card sorting; Software; World Wide Web; Reuse; Software development; Programming language; Engineering; Software development process; Task (project management); Systems engineering; Ecology","score_opus":0.16462003362110883,"score_gpt":0.3920678767072386,"score_spread":0.22744784308612975,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4384408554","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.75517225,0.00004192736,0.24278913,0.00043108806,0.000772408,0.00019716137,0.00001428702,0.00027569913,0.00030602902],"genre_scores_gemma":[0.98866725,0.0000032702171,0.010616098,0.000046863624,0.000049858954,0.000027775412,0.00001747031,0.000008769254,0.0005626289],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981084,0.000039444414,0.0004797038,0.0003866094,0.0007505744,0.00023525607],"domain_scores_gemma":[0.99855536,0.0008169197,0.00007198048,0.00043415467,0.000054842316,0.00006675652],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0022735302,0.00009179551,0.00018653246,0.0005791363,0.000030388375,0.00012697217,0.00046964653,0.00003342122,0.0001571178],"category_scores_gemma":[0.0048031835,0.00007634322,0.000065146334,0.002564183,0.000018695939,0.00022130838,0.0003249549,0.00007989993,0.0002949721],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001213625,0.00007109946,0.47849596,0.000047177447,0.0000242808,0.000045515604,0.0012423096,0.20574932,0.00019074521,0.0002649045,0.11152291,0.20233366],"study_design_scores_gemma":[0.00029095353,0.000028210074,0.85660774,0.00003910478,0.0000039952893,0.0000010737066,0.00031934644,0.040483356,0.0003798923,0.0013164699,0.10035285,0.00017699432],"about_ca_topic_score_codex":0.000041673153,"about_ca_topic_score_gemma":0.0000065675395,"teacher_disagreement_score":0.3781118,"about_ca_system_score_codex":0.000043932087,"about_ca_system_score_gemma":0.000023860486,"threshold_uncertainty_score":0.5750206},"labels":[],"label_agreement":null},{"id":"W4385740122","doi":"10.1007/s10664-023-10309-8","title":"What have we learned? A conceptual framework on New Zealand software professionals and companies’ response to COVID-19","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Supply Chain Resilience and Risk Management","field":"Business, Management and Accounting","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Workaround; Thematic analysis; Software; Software development; Work (physics); Personal software process; Software peer review; Exploratory research; Software review; Knowledge management; Engineering; Business; Computer science; Qualitative research; Software construction; Sociology","score_opus":0.051342450730401896,"score_gpt":0.33055182627901813,"score_spread":0.27920937554861625,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4385740122","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.86959624,0.0005218088,0.045719374,0.07838892,0.0019739443,0.0011619595,0.000010256384,0.0026122853,0.0000152073835],"genre_scores_gemma":[0.84381527,0.000552301,0.015328073,0.104643084,0.0058593173,0.00049599505,0.00017291289,0.00043143527,0.028701587],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9977877,0.00003278378,0.00032392328,0.0006314278,0.00057424756,0.0006499186],"domain_scores_gemma":[0.9972101,0.002118225,0.0000808661,0.00035002863,0.000048983497,0.00019180788],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005971172,0.00035517328,0.00036095892,0.0005607345,0.00028757966,0.00051403826,0.0003415123,0.00017698565,0.00030331122],"category_scores_gemma":[0.0067423484,0.0003052952,0.00009166546,0.0010549433,0.00006091734,0.0006572028,0.00054620596,0.00039667415,0.00075035077],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0011744567,0.00015585648,0.16708471,0.000483344,0.00011589243,0.0003911065,0.007596183,0.16338514,0.000036204117,0.0011636703,0.6440413,0.01437211],"study_design_scores_gemma":[0.0007451512,0.000091724265,0.060766425,0.00087674346,0.00003198627,0.000003450901,0.0043926765,0.0033509322,0.000010987624,0.0021294765,0.92689705,0.00070340466],"about_ca_topic_score_codex":0.000110943816,"about_ca_topic_score_gemma":0.000018730845,"teacher_disagreement_score":0.28285572,"about_ca_system_score_codex":0.00011861286,"about_ca_system_score_gemma":0.000078793346,"threshold_uncertainty_score":0.9999399},"labels":[],"label_agreement":null},{"id":"W4385952926","doi":"10.1007/s10664-023-10323-w","title":"XSnare: application-specific client-side cross-site scripting protection","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Web Application Security Vulnerabilities","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Cross-site scripting; Exploit; Computer science; Scripting language; Client-side; Context (archaeology); Computer security; Overhead (engineering); World Wide Web; Database; Web page; Web application security; Operating system; Web development","score_opus":0.03303138290703469,"score_gpt":0.2844141654592613,"score_spread":0.2513827825522266,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4385952926","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19148806,0.00008918803,0.8032235,0.0006845313,0.00032821306,0.0003958833,0.0000053825775,0.0037365097,0.000048746988],"genre_scores_gemma":[0.953398,0.000012458087,0.04485638,0.00020585535,0.0003949932,0.0007945431,0.00003029684,0.000053572465,0.00025393226],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.99785286,0.00003453683,0.000428118,0.0007108441,0.00046360865,0.00051005854],"domain_scores_gemma":[0.9984777,0.00030389117,0.0000861619,0.00082442927,0.00013821134,0.00016962526],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0005275284,0.0002263273,0.00020048037,0.00024037347,0.00027348657,0.0003692375,0.00069714664,0.00012986791,0.000012873261],"category_scores_gemma":[0.00040270024,0.00024652504,0.000102366655,0.0018623085,0.00004273904,0.00057433185,0.00036240317,0.00036759887,0.0011487252],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000035685982,0.00039488898,0.3032826,0.0008419884,0.00012301475,0.00006544213,0.012464878,0.3715861,0.037018657,0.05757759,0.021137614,0.19547155],"study_design_scores_gemma":[0.00042350154,0.000049032315,0.24037516,0.000045429406,0.0000037543484,0.00002610581,0.00003351937,0.30013737,0.004600766,0.0022616256,0.45126122,0.000782523],"about_ca_topic_score_codex":0.000018492101,"about_ca_topic_score_gemma":0.0000015955162,"teacher_disagreement_score":0.7619099,"about_ca_system_score_codex":0.00018141519,"about_ca_system_score_gemma":0.00003404374,"threshold_uncertainty_score":0.9999987},"labels":[],"label_agreement":null},{"id":"W4386140507","doi":"10.1007/s10664-023-10363-2","title":"A comparison of reinforcement learning frameworks for software testing tasks","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Regression testing; Software performance testing; Software engineering; System integration testing; Software reliability testing; Software development; Adaptation (eye); Context (archaeology); Test strategy; Reinforcement learning; Software inspection; Manual testing; Software testing; Software construction; Machine learning; Software; Software quality; Programming language","score_opus":0.06973939181903052,"score_gpt":0.35039542706618215,"score_spread":0.28065603524715166,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4386140507","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.014462404,0.000107738066,0.95816946,0.00011609215,0.00030068605,0.00026869407,0.0000018636539,0.0265605,0.000012583083],"genre_scores_gemma":[0.44839302,0.000001263909,0.5512787,0.00006548847,0.00007747795,0.000091636626,0.000012845375,0.00003470445,0.00004486924],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99792284,0.000027709151,0.0005696762,0.00046463145,0.00040300633,0.0006121518],"domain_scores_gemma":[0.9940146,0.0049387002,0.00018759698,0.0004905515,0.00021803974,0.00015048088],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005765295,0.00026913846,0.0004463816,0.00030291115,0.00019096614,0.00009405167,0.0007281156,0.0002742961,0.000003558172],"category_scores_gemma":[0.017922437,0.00027577413,0.00014598809,0.0015287899,0.000029038352,0.00019724634,0.00042629775,0.00067096297,0.000015821432],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000065271793,0.0000453128,0.34595644,0.00029851033,0.00003686968,0.000009685733,0.0009760349,0.58968806,0.00011650944,0.00019016516,0.0131597705,0.049516086],"study_design_scores_gemma":[0.00024785503,0.0003686287,0.016040094,0.00043067063,0.000014320244,0.0000076289916,0.000008849095,0.9749947,0.0012233724,0.0023331763,0.0038736844,0.00045702554],"about_ca_topic_score_codex":0.000014645267,"about_ca_topic_score_gemma":2.3152698e-7,"teacher_disagreement_score":0.4339306,"about_ca_system_score_codex":0.000079355625,"about_ca_system_score_gemma":0.0000638294,"threshold_uncertainty_score":0.9999694},"labels":[],"label_agreement":null},{"id":"W4386245573","doi":"10.1007/s10664-023-10348-1","title":"On practitioners’ concerns when adopting service mesh frameworks","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Service (business); Documentation; Data science; Microservices; Service provider; Domain (mathematical analysis); Computer security; Cloud computing; Business","score_opus":0.028533688769403535,"score_gpt":0.2867786734879754,"score_spread":0.2582449847185719,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4386245573","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.17340778,0.00004329537,0.8173346,0.0029560672,0.0015731755,0.00022303934,0.000004509306,0.004366006,0.00009153808],"genre_scores_gemma":[0.90802807,0.000016865526,0.08745925,0.0034645873,0.00063448545,0.000115364084,0.000025333047,0.00006150642,0.0001945248],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9981247,0.000039484457,0.00032039138,0.00051190896,0.00048039827,0.000523076],"domain_scores_gemma":[0.9977385,0.001221769,0.00007768594,0.0006539534,0.00012146714,0.00018664118],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00045346393,0.00023588417,0.0002481809,0.00015832136,0.0001735274,0.0001483878,0.00064559386,0.0003174652,0.00004223075],"category_scores_gemma":[0.0012367567,0.00021160037,0.00010231649,0.001228295,0.000014665697,0.0005323697,0.00028155482,0.0007641031,0.0008848084],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000050739316,0.00033776407,0.15157372,0.0013192237,0.0002927583,0.00035372312,0.017959923,0.6531162,0.00015628595,0.008349165,0.12676162,0.03972886],"study_design_scores_gemma":[0.0014323104,0.00041150258,0.1389798,0.0012790951,0.00003545232,0.00007576459,0.00020604455,0.70220035,0.0006158377,0.004379385,0.14815019,0.0022342673],"about_ca_topic_score_codex":0.00002147087,"about_ca_topic_score_gemma":0.0000015250774,"teacher_disagreement_score":0.73462033,"about_ca_system_score_codex":0.00013939592,"about_ca_system_score_gemma":0.00006361696,"threshold_uncertainty_score":0.9998931},"labels":[],"label_agreement":null},{"id":"W4386715810","doi":"10.1007/s10664-023-10373-0","title":"Investigating developers’ perception on software testability and its effects","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa; Dalhousie University","funders":"","keywords":"Testability; Code smell; Software engineering; Computer science; Empirical research; Software quality; Software development; Software; Engineering; Reliability engineering; Programming language","score_opus":0.03623200916931937,"score_gpt":0.2923139179714733,"score_spread":0.256081908802154,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4386715810","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7424552,0.00008087375,0.25230923,0.000273611,0.00035679326,0.00026737794,0.0000028038853,0.0042509483,0.000003190613],"genre_scores_gemma":[0.89555234,0.000014248554,0.10379482,0.00022489604,0.00013821071,0.0001212517,0.000010711555,0.00006949865,0.00007404224],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9973735,0.00006839338,0.00029395532,0.0007988783,0.0006828805,0.0007823948],"domain_scores_gemma":[0.99447894,0.0044346806,0.000035124584,0.0005164697,0.000111997004,0.00042278823],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00068524346,0.00034614405,0.00030969593,0.00038197782,0.0001985905,0.00018598854,0.00062324724,0.00017721891,0.0000088934985],"category_scores_gemma":[0.024068933,0.00034302654,0.00007329649,0.0018624067,0.00003526339,0.00044454046,0.00059057184,0.0005891629,0.00030630428],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000006374602,0.00012995527,0.89413255,0.0014867601,0.00008133575,0.00018833815,0.0036880407,0.052130796,0.0034770407,0.00054516795,0.0024426938,0.04169092],"study_design_scores_gemma":[0.00025666069,0.00014880994,0.92532617,0.00018886563,0.0000042089346,0.000018017608,0.000007649423,0.071765795,0.00091768015,0.0001224282,0.00078671833,0.0004570082],"about_ca_topic_score_codex":0.000004877584,"about_ca_topic_score_gemma":4.4758707e-7,"teacher_disagreement_score":0.15309715,"about_ca_system_score_codex":0.00024464485,"about_ca_system_score_gemma":0.00008401519,"threshold_uncertainty_score":0.9999022},"labels":[],"label_agreement":null},{"id":"W4386781823","doi":"10.1007/s10664-023-10347-2","title":"A study of documentation for software architecture","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; University of Victoria","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Documentation; Software documentation; Computer science; Internal documentation; Context (archaeology); World Wide Web; Software engineering; Software; Software system; Information retrieval; Programming language; Software construction","score_opus":0.0365751899518581,"score_gpt":0.329109199373997,"score_spread":0.2925340094221389,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4386781823","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38331786,0.000030814375,0.61414987,0.00008287748,0.0002816273,0.00048389722,0.000005641344,0.0016465323,8.4752276e-7],"genre_scores_gemma":[0.8274964,0.0000031408058,0.17167288,0.000035366644,0.00014170136,0.0004390661,0.00001966079,0.00006978574,0.000121984194],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99798375,0.00003121793,0.00036049736,0.0004959764,0.00058250205,0.0005460336],"domain_scores_gemma":[0.9960341,0.00302805,0.000056040913,0.00059718534,0.00012697882,0.00015765907],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00043187026,0.00022868723,0.00029772703,0.000504912,0.0000822615,0.0000896812,0.00083520846,0.000089996756,0.0000075328435],"category_scores_gemma":[0.004306775,0.00022543318,0.00011622862,0.001830761,0.000014547491,0.00027291258,0.00034732543,0.00024403894,0.0000314481],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003161276,0.00040093652,0.27755678,0.00053422066,0.00020789022,0.000091685055,0.008216061,0.68276495,0.0005168204,0.00014373002,0.0055955737,0.023939759],"study_design_scores_gemma":[0.0048569315,0.0025779742,0.9311477,0.00023831696,0.000056921544,0.000048313013,0.00035688322,0.04435869,0.0035788042,0.0019584035,0.009161323,0.00165972],"about_ca_topic_score_codex":0.000011814188,"about_ca_topic_score_gemma":0.0000021369613,"teacher_disagreement_score":0.6535909,"about_ca_system_score_codex":0.00008428335,"about_ca_system_score_gemma":0.00005204963,"threshold_uncertainty_score":0.9192898},"labels":[],"label_agreement":null},{"id":"W4386982649","doi":"10.1007/s10664-023-10380-1","title":"Is GitHub’s Copilot as bad as humans at introducing vulnerabilities in code?","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":101,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Vulnerability (computing); Code (set theory); Process (computing); Computer security; Perspective (graphical); Secure coding; Software; Software engineering; Software security assurance; Artificial intelligence; Operating system; Information security; Programming language","score_opus":0.03474644801521013,"score_gpt":0.3154596742478399,"score_spread":0.28071322623262973,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4386982649","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.90116787,0.0002737354,0.09115834,0.0020919421,0.0009352678,0.00031507146,0.000010179828,0.0039760475,0.000071557035],"genre_scores_gemma":[0.9809093,0.000040227722,0.014695872,0.0005557049,0.00037613767,0.00019344322,0.000018504079,0.00012966481,0.0030811701],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99620795,0.00006611568,0.0005365604,0.0010444755,0.0009449642,0.0011999552],"domain_scores_gemma":[0.99529153,0.0030855204,0.000045622615,0.0011176829,0.0001024623,0.0003571924],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0009188286,0.0004059611,0.0004711092,0.0007471671,0.00017509334,0.00021904364,0.0012794675,0.0001956342,0.00022250746],"category_scores_gemma":[0.008167374,0.000430545,0.00014484939,0.0024663033,0.000056584246,0.00050536817,0.0011923014,0.00076359,0.001956843],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027974555,0.00013253897,0.70241296,0.00041673047,0.00009869167,0.0006664553,0.0106657725,0.24272318,0.00093661365,0.0011395542,0.03754087,0.0032386573],"study_design_scores_gemma":[0.001648072,0.0006143534,0.7704953,0.0004618157,0.000013413136,0.00012814204,0.00010282081,0.1269248,0.009269198,0.0017489212,0.08645829,0.0021349038],"about_ca_topic_score_codex":0.00016052452,"about_ca_topic_score_gemma":0.000022226233,"teacher_disagreement_score":0.11579839,"about_ca_system_score_codex":0.00070651126,"about_ca_system_score_gemma":0.00015489692,"threshold_uncertainty_score":0.9998146},"labels":[],"label_agreement":null},{"id":"W4387435361","doi":"10.1007/s10664-023-10364-1","title":"On the effectiveness of log representation for log-based anomaly detection","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Representation (politics); Computer science; Workflow; Web log analysis software; Context (archaeology); Data mining; Anomaly detection; Feature (linguistics); External Data Representation; Heuristic; Log-log plot; Software; Binary logarithm; Artificial intelligence; Database; Mathematics; World Wide Web; Programming language; Web service","score_opus":0.025893510235824126,"score_gpt":0.2845307926426779,"score_spread":0.25863728240685374,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4387435361","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.43131155,0.0000076104184,0.5676087,0.00007224674,0.00034949582,0.0002716269,0.0000017775966,0.0003735286,0.0000034609056],"genre_scores_gemma":[0.99723613,6.6530487e-7,0.0024687583,0.000035699835,0.000046657144,0.00018867984,0.000004688917,0.000012050717,0.0000066380353],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9990077,0.00009934689,0.00020069885,0.00027682187,0.00020963913,0.00020581868],"domain_scores_gemma":[0.99395496,0.005454477,0.000051461957,0.00041628207,0.00008462039,0.000038210317],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010625997,0.00011317228,0.00016502994,0.0001158298,0.00008917017,0.000027118991,0.00029629451,0.000078015146,0.0000018308956],"category_scores_gemma":[0.002138769,0.000078727724,0.000120705685,0.00085275917,0.000020102163,0.00011966806,0.000052893836,0.000103085535,0.000020002994],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021389962,0.00011884241,0.1682466,0.0012924969,0.00007175375,0.000006194713,0.000304404,0.810657,0.005210523,0.0011656713,0.0005874189,0.012125202],"study_design_scores_gemma":[0.0005079249,0.00031320986,0.49903268,0.00015351344,0.000008280121,0.0000021759677,0.00000566659,0.4225384,0.07566311,0.0012465242,0.00032851595,0.00019999627],"about_ca_topic_score_codex":0.000011288314,"about_ca_topic_score_gemma":8.144629e-7,"teacher_disagreement_score":0.5659246,"about_ca_system_score_codex":0.000067776156,"about_ca_system_score_gemma":0.00003017155,"threshold_uncertainty_score":0.32104233},"labels":[],"label_agreement":null},{"id":"W4387739952","doi":"10.1007/s10664-023-10382-z","title":"Studying the characteristics of AIOps projects on GitHub","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Baseline (sea); Sample (material); Context (archaeology); Data science; Set (abstract data type); Quality (philosophy); Software engineering; Software; Open source; Anomaly detection; Data mining","score_opus":0.05649726322564533,"score_gpt":0.29934722376021994,"score_spread":0.2428499605345746,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4387739952","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.71938914,0.00004407885,0.27675548,0.00053374196,0.000800606,0.0003159458,0.0000073285396,0.002135778,0.000017876806],"genre_scores_gemma":[0.9931253,0.00001009835,0.0063445466,0.00008696625,0.00017781432,0.000070033726,0.0000042730217,0.000043993663,0.000136972],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9979677,0.00003987382,0.00032054415,0.0003883962,0.0007165851,0.0005668797],"domain_scores_gemma":[0.9956033,0.0033241622,0.000056091227,0.00079592573,0.00009586444,0.00012465575],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00063602533,0.00021933464,0.0002659443,0.00030482063,0.00010538904,0.00009893594,0.0011804348,0.00008309639,0.0000061010796],"category_scores_gemma":[0.005399967,0.00016506799,0.000100680685,0.0019763939,0.00003380949,0.00015908838,0.00051172695,0.00044827047,0.0001436313],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003176175,0.00030180358,0.85136616,0.0007159897,0.00027096053,0.0003868886,0.007123851,0.07122626,0.0016886074,0.002124474,0.011489209,0.05327402],"study_design_scores_gemma":[0.00025049885,0.00018782739,0.9472635,0.00014003045,0.000006517438,0.000013272397,0.000022827417,0.04410337,0.0016794231,0.000049883587,0.005929194,0.00035370307],"about_ca_topic_score_codex":0.0000054460397,"about_ca_topic_score_gemma":1.9372803e-7,"teacher_disagreement_score":0.27373615,"about_ca_system_score_codex":0.00008283102,"about_ca_system_score_gemma":0.0000678981,"threshold_uncertainty_score":0.6731277},"labels":[],"label_agreement":null},{"id":"W4388568365","doi":"10.1007/s10664-023-10403-x","title":"On the coordination of vulnerability fixes","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Network Security and Intrusion Detection","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Vulnerability (computing); Computer security; Secure coding; Computer science; Software; Order (exchange); Internet privacy; Business; Software security assurance; Information security","score_opus":0.02498656131703465,"score_gpt":0.2621775284910033,"score_spread":0.23719096717396865,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388568365","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5735465,0.000015238331,0.42403075,0.0014594891,0.0003648992,0.0000686314,8.0746315e-7,0.00047806866,0.000035565485],"genre_scores_gemma":[0.99853265,0.0000033271156,0.0012302356,0.00012191439,0.000052868323,0.000012499673,0.0000010491868,0.000004781856,0.00004068992],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.999364,0.00003491418,0.0001303264,0.00014753174,0.00019085727,0.00013232326],"domain_scores_gemma":[0.99866337,0.0009928058,0.00002733158,0.0002553468,0.00003033338,0.000030790008],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00035876036,0.0000657476,0.00007853102,0.000065896194,0.000076942466,0.000024952868,0.00027139983,0.00004241283,0.000023888175],"category_scores_gemma":[0.0008800008,0.00004695518,0.000048693917,0.0006632487,0.000014120558,0.00010653821,0.00010325119,0.00015055014,0.00004659436],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000035421577,0.00028660064,0.01606001,0.00020088321,0.00008995661,0.000020713156,0.003943545,0.5782914,0.0030568065,0.12973681,0.11929496,0.14898287],"study_design_scores_gemma":[0.00017345755,0.00019247574,0.086304784,0.000061474064,0.0000027152194,0.0000033185656,0.0000104380615,0.86935055,0.009738274,0.014389799,0.019540217,0.0002325097],"about_ca_topic_score_codex":0.0000043026903,"about_ca_topic_score_gemma":6.428038e-7,"teacher_disagreement_score":0.4249861,"about_ca_system_score_codex":0.000025056059,"about_ca_system_score_gemma":0.000008317243,"threshold_uncertainty_score":0.19147767},"labels":[],"label_agreement":null},{"id":"W4388568366","doi":"10.1007/s10664-023-10394-9","title":"An empirical comparison of ethnic and gender diversity of DevOps and non-DevOps contributions to open-source projects","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"DevOps; Diversity (politics); Ethnic group; Empirical research; Open source software; Cultural diversity; Engineering; Computer science; Political science; Software; Software deployment; Software engineering; Statistics","score_opus":0.08380178275201619,"score_gpt":0.3824720258389828,"score_spread":0.29867024308696666,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388568366","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5153491,0.000038492642,0.4837247,0.00028274962,0.000063239444,0.00027558493,0.000024986219,0.00023388147,0.0000072408525],"genre_scores_gemma":[0.93532646,0.0000029233347,0.064428985,0.0001408218,0.000027918146,0.000022176378,0.000013549236,0.000019280857,0.000017907245],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99823964,0.00006062365,0.00044199725,0.0005023313,0.00036603736,0.000389391],"domain_scores_gemma":[0.9981517,0.0006675565,0.000108340566,0.00050322484,0.00028945864,0.0002797046],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00057929487,0.00020512029,0.00046632005,0.0003214514,0.00023368762,0.00009586565,0.00093699625,0.00013856363,0.000004629309],"category_scores_gemma":[0.0011510266,0.00021104624,0.000044139186,0.0026581984,0.00007450191,0.0004981692,0.002931136,0.00027334128,0.000008218575],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013202001,0.00017211106,0.96816665,0.00011418015,0.000059868504,0.000007371983,0.014212781,0.009305428,0.0007775824,0.0003507073,0.0023674294,0.004452684],"study_design_scores_gemma":[0.0005215636,0.00023728186,0.9495266,0.00005364368,0.0000214226,0.000008610082,0.00019979099,0.04681509,0.0010569762,0.00022248988,0.0010094738,0.00032702522],"about_ca_topic_score_codex":0.00008483845,"about_ca_topic_score_gemma":0.000011443489,"teacher_disagreement_score":0.4199773,"about_ca_system_score_codex":0.00007189802,"about_ca_system_score_gemma":0.0000948582,"threshold_uncertainty_score":0.8606216},"labels":[],"label_agreement":null},{"id":"W4389141459","doi":"10.1007/s10664-023-10389-6","title":"Silent bugs in deep learning frameworks: an empirical study of Keras and TensorFlow","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":38,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec; Consortium de Recherche et d’innovation en Aérospatiale au Québec; Canadian Institute for Advanced Research","keywords":"Software bug; Computer science; Relevance (law); Debugging; Artificial intelligence; Empirical research; Deep learning; Machine learning; Software; Programming language","score_opus":0.02542377729338102,"score_gpt":0.3172849195316602,"score_spread":0.2918611422382792,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389141459","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.79406023,0.00014582863,0.20389059,0.00011790655,0.00019452136,0.00025457173,6.557017e-7,0.0013341261,0.0000015675912],"genre_scores_gemma":[0.98282135,0.000019318319,0.016882451,0.00003880433,0.00008467044,0.000070611204,0.0000037350494,0.00005545393,0.00002359024],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99717736,0.0001312888,0.00048748724,0.0007404437,0.0007193772,0.00074401643],"domain_scores_gemma":[0.99628747,0.0026391477,0.000049421025,0.0006112237,0.00009158458,0.00032115396],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00079667824,0.00030290274,0.00045484578,0.0007151951,0.000081786384,0.0001248895,0.0007526302,0.00027906726,0.000010442047],"category_scores_gemma":[0.0048218826,0.00030578213,0.00006132034,0.0025380475,0.00003277393,0.00041467548,0.0006485448,0.001401623,0.000025767113],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000005796861,0.00017816131,0.7727258,0.00005062195,0.000019532472,0.00012671562,0.004600234,0.21717091,0.000046673435,0.0000066883385,0.00005444929,0.0050144135],"study_design_scores_gemma":[0.00043509481,0.0003959333,0.73616993,0.00005428299,0.0000037917946,0.000010354941,0.00020043881,0.26213282,0.000039528873,0.00004142063,0.00024734903,0.00026902292],"about_ca_topic_score_codex":0.00004442613,"about_ca_topic_score_gemma":0.000010708204,"teacher_disagreement_score":0.18876113,"about_ca_system_score_codex":0.00011630524,"about_ca_system_score_gemma":0.00003861107,"threshold_uncertainty_score":0.99993944},"labels":[],"label_agreement":null},{"id":"W4389141545","doi":"10.1007/s10664-023-10399-4","title":"Unreproducible builds: time to fix, causes, and correlation with external ecosystem factors","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Software; Process (computing); Reproducibility; Cyclomatic complexity; Software engineering; Data science; Operating system; Statistics; Mathematics","score_opus":0.019738238128314236,"score_gpt":0.25718784017076846,"score_spread":0.23744960204245422,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389141545","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5063151,0.000054549997,0.49118158,0.00016106095,0.00017964271,0.00018247057,0.000004932148,0.0019170463,0.0000035671872],"genre_scores_gemma":[0.9504387,0.0000046497207,0.04818923,0.00005118646,0.00019389714,0.00006954865,0.000012180192,0.000085145555,0.00095546985],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99788207,0.000025976087,0.00022963002,0.000727298,0.00054469606,0.00059030973],"domain_scores_gemma":[0.9976515,0.0012234506,0.000030387759,0.0006449248,0.00007816755,0.00037153298],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00039547577,0.0002591191,0.0002528773,0.00041713667,0.00009809965,0.00020675485,0.0004985857,0.00009354213,0.00002814341],"category_scores_gemma":[0.0018169186,0.0002272259,0.00004040503,0.0016848793,0.000011138134,0.00037717534,0.00039194062,0.0002817857,0.00057799293],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010811542,0.000028692608,0.5569205,0.0001457908,0.000053844567,0.00014492696,0.0010256382,0.43428355,0.0007202265,0.000049819777,0.004334965,0.0022811887],"study_design_scores_gemma":[0.00033700216,0.00024243978,0.833124,0.0002565295,0.00000881479,0.00007010263,0.000008561996,0.15191422,0.0012666521,0.000022527369,0.012104712,0.0006444582],"about_ca_topic_score_codex":0.000016082307,"about_ca_topic_score_gemma":0.0000018445544,"teacher_disagreement_score":0.44412357,"about_ca_system_score_codex":0.00014521199,"about_ca_system_score_gemma":0.000053897344,"threshold_uncertainty_score":0.92660034},"labels":[],"label_agreement":null},{"id":"W4389195248","doi":"10.1007/s10664-023-10398-5","title":"A fly in the ointment: an empirical study on the characteristics of Ethereum smart contract code weaknesses","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Blockchain Technology Applications and Security","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Rannís","keywords":"Computer science; Strengths and weaknesses; Smart contract; Generality; Categorization; Code (set theory); Source code; Coding (social sciences); Data science; Artificial intelligence; Data mining; Database; Programming language; Database transaction; Mathematics","score_opus":0.03717403338258266,"score_gpt":0.30452667689705554,"score_spread":0.26735264351447285,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389195248","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9408048,0.000022086786,0.054013915,0.0041562985,0.00009809674,0.00041406878,0.0000091246775,0.00046993175,0.000011696716],"genre_scores_gemma":[0.9986434,0.0000060592315,0.00073947804,0.00036448587,0.000034758,0.00018956602,0.0000037502587,0.000013082169,0.000005445799],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9986391,0.00012446281,0.00032021265,0.00032864377,0.00028843837,0.0002991319],"domain_scores_gemma":[0.9974348,0.0015279466,0.00006657978,0.0008843501,0.00003906097,0.000047297017],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009259325,0.00016427724,0.00023100042,0.0001283488,0.000112799025,0.000056038898,0.0012089532,0.00011119285,0.000006236161],"category_scores_gemma":[0.00058937387,0.000100410245,0.000054681703,0.0011802163,0.000051847026,0.000073934236,0.00018241511,0.00048513804,0.000026546211],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027051323,0.0028209954,0.9376024,0.000060509945,0.000108554625,0.00014595441,0.021884533,0.0015293522,0.00024268229,0.01684461,0.0017466735,0.016986683],"study_design_scores_gemma":[0.0002237303,0.0003645561,0.9634312,0.00002311948,0.0000076068304,0.00000737502,0.00033635256,0.031413313,0.000110869645,0.0007823816,0.0031362923,0.00016323697],"about_ca_topic_score_codex":0.000020845542,"about_ca_topic_score_gemma":0.000020045238,"teacher_disagreement_score":0.057838593,"about_ca_system_score_codex":0.000033931155,"about_ca_system_score_gemma":0.000032075834,"threshold_uncertainty_score":0.4094611},"labels":[],"label_agreement":null},{"id":"W4389337865","doi":"10.1007/s10664-023-10400-0","title":"Bug characterization in machine learning-based systems","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":30,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; Polytechnique Montréal","funders":"","keywords":"Computer science; Software bug; Software; Component (thermodynamics); Software system; Task (project management); Process (computing); Software engineering; Software development; Focus (optics); Software maintenance; Corrective maintenance; Machine learning; Reliability engineering; Operating system; Systems engineering; Engineering; Preventive maintenance","score_opus":0.024405773824118494,"score_gpt":0.2681081263815844,"score_spread":0.2437023525574659,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389337865","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2658264,0.00008593258,0.7295107,0.00024177588,0.00056085497,0.00017570339,0.000006239956,0.0035901158,0.0000022452107],"genre_scores_gemma":[0.9960705,0.000011326628,0.0032403658,0.00004593772,0.00011796119,0.00011648533,0.0001125114,0.000060602393,0.00022429164],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979972,0.00006134751,0.00031662182,0.0004666106,0.00051216537,0.00064603],"domain_scores_gemma":[0.9980649,0.001244058,0.000038248767,0.0004167254,0.000057817815,0.00017825703],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00057052926,0.00022568334,0.00025466183,0.0007104835,0.000062231935,0.00018018542,0.0006534284,0.00012848277,0.000009783492],"category_scores_gemma":[0.002826751,0.00023731065,0.00006381487,0.0028062302,0.000011949776,0.00030342804,0.0002245215,0.00053100876,0.0002576011],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000023529242,0.00002121031,0.4431038,0.00009710897,0.0000065925633,0.00010058147,0.00013213733,0.5551315,0.00071004377,0.000025921989,0.00008390458,0.0005848475],"study_design_scores_gemma":[0.00020258706,0.000030595475,0.36641493,0.00005753236,8.4316036e-7,0.000004478646,0.0000012261036,0.62568957,0.00020293712,0.0000025172262,0.0071990206,0.0001937924],"about_ca_topic_score_codex":0.000029207751,"about_ca_topic_score_gemma":0.0000012663953,"teacher_disagreement_score":0.7302441,"about_ca_system_score_codex":0.00019036417,"about_ca_system_score_gemma":0.000063945234,"threshold_uncertainty_score":0.96772474},"labels":[],"label_agreement":null},{"id":"W4389685461","doi":"10.1007/s10664-023-10409-5","title":"Detection and evaluation of bias-inducing features in machine learning","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada; Consortium de Recherche et d’innovation en Aérospatiale au Québec","keywords":"Computer science; Machine learning; Identification (biology); Context (archaeology); Artificial intelligence; Harm; Feature (linguistics); Outcome (game theory); Data mining; Psychology","score_opus":0.06541519331576667,"score_gpt":0.314828704639924,"score_spread":0.24941351132415734,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389685461","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.78428334,0.00021157162,0.21483956,0.00010348933,0.00014057015,0.00009767535,2.657974e-7,0.00031073717,0.000012788109],"genre_scores_gemma":[0.9958068,0.00001339494,0.004091706,0.00001451201,0.00002826056,0.000016656688,0.0000014386138,0.000010397866,0.000016804044],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998922,0.000080674356,0.0001990834,0.00024083417,0.0003398687,0.00021757573],"domain_scores_gemma":[0.9992569,0.00043229994,0.00004235404,0.00014629854,0.00007379539,0.000048352944],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0012762022,0.00009475324,0.00012626483,0.00031322066,0.00005351029,0.00004360809,0.000153604,0.000063221334,0.0000039722468],"category_scores_gemma":[0.00265992,0.000096444186,0.000026988058,0.0011786816,0.000010168757,0.0002744215,0.00012773807,0.00022849758,0.000010107464],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000003612331,0.000016423093,0.06790005,0.00005118032,0.000008666393,0.000011186441,0.0023198707,0.78294444,0.007442107,0.00012337632,0.000018718743,0.13916038],"study_design_scores_gemma":[0.00007785369,0.000035553807,0.13414863,0.000040690196,0.0000034747582,0.0000049575715,0.00004046899,0.84376657,0.021332059,0.0002966005,0.00015134625,0.00010179097],"about_ca_topic_score_codex":0.00013640411,"about_ca_topic_score_gemma":0.00006239272,"teacher_disagreement_score":0.21152349,"about_ca_system_score_codex":0.00008251705,"about_ca_system_score_gemma":0.000023409735,"threshold_uncertainty_score":0.393288},"labels":[],"label_agreement":null},{"id":"W4389804693","doi":"10.1007/s10664-023-10376-x","title":"On the intuitive comprehensibility of contribution links in goal models: an experimental study","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Cognitive and psychological constructs research","field":"Psychology","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Psychology","score_opus":0.138767554850462,"score_gpt":0.42106434379206314,"score_spread":0.28229678894160115,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4389804693","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.99570245,0.00006334635,0.0027945768,0.00010675019,0.00019023138,0.00063358253,0.000027955912,0.00016811448,0.0003130007],"genre_scores_gemma":[0.9996382,0.0000010737175,0.00003038951,0.00011442643,0.000042033906,0.00012136252,0.0000175683,0.0000142343415,0.000020681846],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9983781,0.00034145048,0.00028852222,0.0003765072,0.00026795434,0.00034744487],"domain_scores_gemma":[0.99760157,0.0019000191,0.000030869145,0.00029367383,0.00008684862,0.00008702977],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0006252785,0.00014680371,0.00023918036,0.00014122414,0.00005160319,0.000014850921,0.00020441142,0.00015631031,0.0005793179],"category_scores_gemma":[0.0006627213,0.00010624537,0.00006324026,0.0006714037,0.00008266323,0.00006142069,0.00009857019,0.0007040755,0.000083631196],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0031603542,0.013682778,0.8685585,0.00005628699,0.0004623296,0.00067198015,0.041040413,0.017136186,0.0033150723,0.023542292,0.001982894,0.026390908],"study_design_scores_gemma":[0.0016088226,0.002171287,0.9839682,0.000036832087,0.0000066518223,0.0000034525472,0.003456295,0.005786825,0.00047002628,0.0021605787,0.00010478572,0.00022621543],"about_ca_topic_score_codex":0.000051236813,"about_ca_topic_score_gemma":0.000007772198,"teacher_disagreement_score":0.115409724,"about_ca_system_score_codex":0.000063498104,"about_ca_system_score_gemma":0.000012919941,"threshold_uncertainty_score":0.6343125},"labels":[],"label_agreement":null},{"id":"W4390144898","doi":"10.1007/s10664-023-10431-7","title":"A multi-objective effort-aware approach for early code review prediction and prioritization","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Code review; Code (set theory); Machine learning; Sorting; Prioritization; Empirical research; Source code; Task (project management); Artificial intelligence; Software; Static program analysis; Software engineering; Software development; Management science; Algorithm; Engineering; Systems engineering; Programming language","score_opus":0.039604791623415604,"score_gpt":0.3068962005751302,"score_spread":0.26729140895171455,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4390144898","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.017851993,0.0014170029,0.97642434,0.00013948721,0.00025261685,0.00091847876,0.000033388238,0.0029609345,0.0000017381116],"genre_scores_gemma":[0.29250836,0.0013417391,0.70280915,0.00027233825,0.00043140168,0.0018622385,0.00023601453,0.00019326604,0.00034549707],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9982046,0.00002305704,0.00028712265,0.0006021303,0.00037428533,0.00050879637],"domain_scores_gemma":[0.99837327,0.0008503727,0.000039792823,0.00038066687,0.00015542022,0.00020045886],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005409,0.0002305267,0.00028644578,0.0002445939,0.00012317936,0.00010357312,0.00038714428,0.00012803021,0.000001496701],"category_scores_gemma":[0.00262638,0.0002345416,0.00008771264,0.0013124223,0.000021406799,0.00040382126,0.00024920618,0.00026488118,0.000012796132],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025689646,0.00022972222,0.8498963,0.011352596,0.00027811155,0.00005488804,0.0022346291,0.096705094,0.0003612161,0.0001972234,0.016201513,0.022463001],"study_design_scores_gemma":[0.00043757458,0.00009516062,0.44762573,0.0003960829,0.000013994328,0.000016534985,0.000003798795,0.5491959,0.000086497355,0.000017646136,0.0018366146,0.000274484],"about_ca_topic_score_codex":0.00000533897,"about_ca_topic_score_gemma":2.875078e-7,"teacher_disagreement_score":0.4524908,"about_ca_system_score_codex":0.00012579668,"about_ca_system_score_gemma":0.000049919676,"threshold_uncertainty_score":0.95643294},"labels":[],"label_agreement":null},{"id":"W4390411712","doi":"10.1007/s10664-023-10421-9","title":"Using knowledge units of programming languages to recommend reviewers for pull requests: an empirical study","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Baseline (sea); Operationalization; Code (set theory); Java; Task (project management); Recommender system; Artificial intelligence; Range (aeronautics); Code review; Information retrieval; Machine learning; Programming language; Natural language processing; Set (abstract data type); Software; Software quality; Software development; Engineering","score_opus":0.15121060831279526,"score_gpt":0.43717184664025377,"score_spread":0.2859612383274585,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4390411712","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.47445935,0.0002828169,0.5218954,0.00015722183,0.00036687305,0.0010614664,0.000009432938,0.0017643487,0.0000030945819],"genre_scores_gemma":[0.56470644,0.000012369543,0.43402317,0.00009719787,0.00039083976,0.0004674545,0.000028943072,0.00016302566,0.00011058535],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99717444,0.000121440105,0.0005739108,0.00074853754,0.00050089153,0.0008807536],"domain_scores_gemma":[0.9959971,0.0021623853,0.00006808074,0.0009115085,0.0003990992,0.0004618835],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00162014,0.00034193162,0.00053794717,0.0006769534,0.000119823904,0.0001366377,0.0011936274,0.00012803667,0.00000527019],"category_scores_gemma":[0.009665093,0.00034119285,0.000112748006,0.004591008,0.000021026803,0.00040852034,0.00059168984,0.00034305616,0.000027557944],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005960845,0.0013621753,0.697902,0.0021294532,0.00034547542,0.0002386032,0.021872958,0.10415528,0.0013213329,0.00013404267,0.0118459845,0.15863308],"study_design_scores_gemma":[0.0038210663,0.006560785,0.33069357,0.0022535424,0.00018205614,0.00010720594,0.0017382604,0.3884089,0.0046562916,0.00014816155,0.25692582,0.00450432],"about_ca_topic_score_codex":0.00002489637,"about_ca_topic_score_gemma":0.000007713755,"teacher_disagreement_score":0.36720842,"about_ca_system_score_codex":0.00024232383,"about_ca_system_score_gemma":0.00015333774,"threshold_uncertainty_score":0.99990404},"labels":[],"label_agreement":null},{"id":"W4390502905","doi":"10.1007/s10664-023-10362-3","title":"What is an app store? The software engineering perspective","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; University of Waterloo","funders":"European Commission","keywords":"App store; Computer science; World Wide Web; Mobile apps; Android (operating system); Software; Download; Operating system","score_opus":0.0255173147224049,"score_gpt":0.30159338532327223,"score_spread":0.27607607060086736,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4390502905","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.038880304,0.010148619,0.93549573,0.0022298922,0.0041029383,0.0004151686,0.000011840115,0.008708646,0.000006860855],"genre_scores_gemma":[0.7999589,0.0004074872,0.19340084,0.001248626,0.0027228124,0.00051403395,0.000024730753,0.0005697563,0.0011528257],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99592185,0.000057108973,0.0004271032,0.0012914686,0.0011124242,0.0011900185],"domain_scores_gemma":[0.9944205,0.0032822078,0.00003224234,0.0015313483,0.00021906948,0.00051463104],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00075602066,0.0006162223,0.00040774638,0.00048709055,0.00021733232,0.002215052,0.0022432152,0.00025909295,0.000085476066],"category_scores_gemma":[0.0029187503,0.00050269,0.00029430774,0.002105472,0.00006213362,0.0029483496,0.0007265692,0.0014100503,0.00034731955],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005530885,0.0007265051,0.058432918,0.002619698,0.0021824727,0.003370414,0.09405138,0.6385192,0.0015478967,0.02863812,0.07222659,0.097629465],"study_design_scores_gemma":[0.0005573312,0.00041094332,0.055984244,0.0011157481,0.00007202716,0.0004921827,0.0005062137,0.72338504,0.0020986593,0.0008688233,0.21181487,0.0026939495],"about_ca_topic_score_codex":0.00003756531,"about_ca_topic_score_gemma":0.0000019470112,"teacher_disagreement_score":0.7610786,"about_ca_system_score_codex":0.00075033365,"about_ca_system_score_gemma":0.00020669009,"threshold_uncertainty_score":0.99974245},"labels":[],"label_agreement":null},{"id":"W4390749519","doi":"10.1007/s10664-023-10424-6","title":"Diversity in issue assignment: humans vs bots","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia, Okanagan Campus; University of British Columbia; University of Waterloo","funders":"","keywords":"Diversity (politics); Open source; Process (computing); Open source software; Computer science; Ethnic group; Race (biology); White (mutation); Data science; World Wide Web; Knowledge management; Software; Political science; Sociology; Law","score_opus":0.02814452508204056,"score_gpt":0.2804815817334202,"score_spread":0.2523370566513796,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4390749519","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07028796,0.00021757212,0.92513406,0.0015055739,0.00088246283,0.00013833895,0.000003736293,0.0016574779,0.00017281578],"genre_scores_gemma":[0.95584846,0.000003975781,0.04306289,0.0004124519,0.00017453727,0.000020333135,0.0000042378833,0.000026268206,0.00044684028],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99847746,0.000023255032,0.00025706305,0.00047870065,0.00036380265,0.0003996976],"domain_scores_gemma":[0.9990909,0.00035423046,0.00001903813,0.00039426042,0.00003289061,0.000108662294],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00029251736,0.00018747845,0.00018471028,0.00030867854,0.00021340713,0.00021343786,0.000784164,0.000100914745,0.00008725599],"category_scores_gemma":[0.00028158393,0.00019589002,0.00007236345,0.0015622778,0.000019098075,0.0006700939,0.001514119,0.00041113602,0.00029236331],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009719112,0.00035993083,0.79926395,0.0005015983,0.00017427509,0.0015873506,0.024233527,0.077554636,0.00020501437,0.024974784,0.03953672,0.031598505],"study_design_scores_gemma":[0.0005471579,0.00014898312,0.4332486,0.000498534,0.000019699575,0.000039870116,0.000034425626,0.12442954,0.00041650413,0.002399221,0.43691078,0.0013066776],"about_ca_topic_score_codex":0.000037384827,"about_ca_topic_score_gemma":0.000005042222,"teacher_disagreement_score":0.8855605,"about_ca_system_score_codex":0.00026991518,"about_ca_system_score_gemma":0.00004685072,"threshold_uncertainty_score":0.7988163},"labels":[],"label_agreement":null},{"id":"W4391745357","doi":"10.1007/s10664-023-10437-1","title":"A study of common bug fix patterns in Rust","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Rust (programming language); Computer science; Compiler; Programming language; Software engineering","score_opus":0.03218305884190426,"score_gpt":0.3177540607064256,"score_spread":0.28557100186452133,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4391745357","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.71506125,0.0003365367,0.282981,0.000072917515,0.00040950405,0.00020445105,0.0000031871155,0.00092664466,0.0000044980475],"genre_scores_gemma":[0.9947239,0.0000056278996,0.0050505293,0.000015666514,0.00007047048,0.00006153783,0.000001972236,0.000038999653,0.00003127464],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9980941,0.000041259318,0.0003949539,0.00048319736,0.00052534934,0.000461176],"domain_scores_gemma":[0.99752223,0.0017016234,0.000018772354,0.00058812706,0.000033732438,0.00013548849],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00038343214,0.00021764878,0.00031258154,0.0005294609,0.000022590122,0.00012962571,0.00084967224,0.00008897385,0.000017949415],"category_scores_gemma":[0.00072171766,0.00021253643,0.000075584816,0.001478211,0.000010922339,0.0003049536,0.000451832,0.0005466135,0.00003171161],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000026291316,0.00033405816,0.9454165,0.00028223186,0.000043602475,0.0006764429,0.0031017454,0.044625748,0.000038498503,0.000071002185,0.00035636735,0.0050511365],"study_design_scores_gemma":[0.00040785733,0.00029805125,0.9468004,0.00033814568,0.0000063635125,0.00003172875,0.000057546014,0.049546134,0.00025850345,0.000034974655,0.001849087,0.000371225],"about_ca_topic_score_codex":0.00013736502,"about_ca_topic_score_gemma":0.00002326178,"teacher_disagreement_score":0.27966267,"about_ca_system_score_codex":0.000151599,"about_ca_system_score_gemma":0.000049512084,"threshold_uncertainty_score":0.8666984},"labels":[],"label_agreement":null},{"id":"W4391835605","doi":"10.1007/s10664-024-10449-5","title":"Quantifying and characterizing clones of self-admitted technical debt in build systems","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Precursory Research for Embryonic Science and Technology; Japan Science and Technology Agency; Japan Society for the Promotion of Science","keywords":"Artifact (error); Technical debt; Computer science; Context (archaeology); Maintainability; Reliability (semiconductor); Code (set theory); Scale (ratio); Data science; Data mining; Software; Software engineering; Software development; Artificial intelligence; Programming language; Biology; Set (abstract data type); Geography","score_opus":0.029443787547574637,"score_gpt":0.29859674966661726,"score_spread":0.2691529621190426,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4391835605","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5985848,0.0033272882,0.395175,0.00016025917,0.0005629373,0.0001841546,0.0000025997758,0.0019996932,0.0000032824676],"genre_scores_gemma":[0.95514137,0.00006336455,0.04458539,0.000014864654,0.00009208801,0.000048155176,0.0000020560635,0.00004500174,0.000007740903],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99800944,0.00003830022,0.00046571973,0.0005519405,0.000419323,0.00051525026],"domain_scores_gemma":[0.9972329,0.0021125,0.000030274961,0.00040404356,0.00004797196,0.00017232052],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00060648075,0.00024603136,0.00037572611,0.0005792045,0.00003386423,0.00023911108,0.0005316393,0.00017453088,0.0000024381588],"category_scores_gemma":[0.0012899362,0.00023948426,0.00007181911,0.0013899589,0.000023721856,0.00045248098,0.0003898511,0.000563495,0.000011492971],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009830938,0.00013848612,0.9514954,0.0037601858,0.00011619196,0.0006758624,0.0013446548,0.01166968,0.020537801,0.004409812,0.0002785143,0.0055635865],"study_design_scores_gemma":[0.00028064798,0.00010482566,0.64891875,0.0014329179,0.000011719945,0.0002568405,0.000010003252,0.3416263,0.0018202337,0.00003126507,0.00491172,0.0005948101],"about_ca_topic_score_codex":0.000031623687,"about_ca_topic_score_gemma":0.00000153524,"teacher_disagreement_score":0.35655656,"about_ca_system_score_codex":0.00015551613,"about_ca_system_score_gemma":0.00007386294,"threshold_uncertainty_score":0.9765885},"labels":[],"label_agreement":null},{"id":"W4391915512","doi":"10.1007/s10664-024-10443-x","title":"Studying the impact of risk assessment analytics on risk awareness and code review performance","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); University of Waterloo","funders":"","keywords":"Analytics; Computer science; Data science; Code (set theory); Risk analysis (engineering); Business; Programming language","score_opus":0.04283266006473341,"score_gpt":0.3663655825837216,"score_spread":0.3235329225189882,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4391915512","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5085651,0.007979798,0.48238948,0.000094326984,0.00022601358,0.00020917668,0.000028108976,0.0005030975,0.0000048730362],"genre_scores_gemma":[0.98461133,0.005523089,0.009672006,0.00002399821,0.000085257925,0.00003523786,0.0000025749794,0.000036779667,0.000009710259],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9981062,0.000088312474,0.0003570903,0.0004543928,0.00058247166,0.0004115693],"domain_scores_gemma":[0.9943905,0.00459697,0.000065089625,0.0007031382,0.000092958086,0.00015138347],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0012818141,0.00027200417,0.0003552908,0.00019368359,0.00014358907,0.00018029482,0.00071386824,0.00006458027,0.000009758146],"category_scores_gemma":[0.0028298288,0.00017610482,0.00017105363,0.0011999719,0.000038158476,0.0002713583,0.00034858365,0.0008328125,0.000010940907],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000025919069,0.000041725576,0.7245479,0.0009797936,0.00018428614,0.000027139371,0.00030429623,0.25265047,0.000007316076,0.000047715694,0.0010795306,0.020127205],"study_design_scores_gemma":[0.0000749311,0.00012830335,0.5936566,0.0010494678,0.000029243423,0.000014731825,0.000001989805,0.4043375,0.000033867473,0.00001738391,0.00048954925,0.00016642292],"about_ca_topic_score_codex":0.000043026488,"about_ca_topic_score_gemma":8.696767e-7,"teacher_disagreement_score":0.4760462,"about_ca_system_score_codex":0.00022877957,"about_ca_system_score_gemma":0.0001592507,"threshold_uncertainty_score":0.7181346},"labels":[],"label_agreement":null},{"id":"W4391998095","doi":"10.1007/s10664-023-10433-5","title":"Evaluating the impact of flaky simulators on testing autonomous driving systems","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Test (biology); Replication (statistics); Key (lock); Scope (computer science); Code coverage; Simulation; Machine learning; Operating system; Statistics; Software; Mathematics; Programming language","score_opus":0.08166117695653764,"score_gpt":0.382740428437695,"score_spread":0.3010792514811574,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4391998095","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3181773,0.0006337332,0.6648792,0.000049973656,0.0005592858,0.00019127864,0.0000019979407,0.015460415,0.000046781537],"genre_scores_gemma":[0.90906656,7.105418e-7,0.090658106,0.00001594235,0.00017778913,0.00002912883,7.75879e-7,0.000033444,0.00001753143],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99819064,0.000070631984,0.00041922738,0.00045634957,0.00043615143,0.00042700703],"domain_scores_gemma":[0.9920613,0.0070006237,0.000081150174,0.00064229657,0.000102960774,0.00011164112],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008557593,0.00027611415,0.0002797238,0.00021323175,0.00013094707,0.00034516162,0.0007429481,0.00009477434,0.0000022762263],"category_scores_gemma":[0.0067758556,0.00018407048,0.00018954511,0.0011339703,0.000027594046,0.00021823228,0.00019068106,0.00043522337,0.000013235176],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000014108847,0.000023411914,0.026586583,0.000102277474,0.00005286303,0.00002832965,0.00045597032,0.9353039,0.00025324032,0.00023218877,0.0010775281,0.035882268],"study_design_scores_gemma":[0.000047957477,0.00037547632,0.023873597,0.00071153574,0.000009390677,0.000044428944,0.0000011926463,0.9740264,0.000107077634,0.0005256711,0.00005931027,0.00021799764],"about_ca_topic_score_codex":0.00011701354,"about_ca_topic_score_gemma":9.881979e-8,"teacher_disagreement_score":0.5908893,"about_ca_system_score_codex":0.00026049785,"about_ca_system_score_gemma":0.000175335,"threshold_uncertainty_score":0.81118214},"labels":[],"label_agreement":null},{"id":"W4392224127","doi":"10.1007/s10664-024-10474-4","title":"An empirical study of challenges in machine learning asset management","year":2024,"lang":"en","type":"preprint","venue":"Empirical Software Engineering","topic":"Big Data and Business Intelligence","field":"Business, Management and Accounting","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Asset management; Empirical research; Asset (computer security); Business; Computer science; Knowledge management; Finance; Computer security; Mathematics; Statistics","score_opus":0.11135924812227603,"score_gpt":0.3433271423296989,"score_spread":0.23196789420742286,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4392224127","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9861467,0.0039635175,0.0047231335,0.0005474896,0.0017963368,0.00097415404,0.000023362953,0.0011644547,0.00066084426],"genre_scores_gemma":[0.99733096,0.0002522293,0.0009855615,0.00008193812,0.000817923,0.00013913053,0.00020541357,0.00013397647,0.000052844505],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99682486,0.000033915756,0.0008253201,0.001128885,0.00066122337,0.0005258233],"domain_scores_gemma":[0.9987329,0.00012709541,0.00020532541,0.00078445126,0.00011029801,0.000039906143],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006869369,0.00062259816,0.0007818856,0.0010933714,0.000056626784,0.00030267716,0.00094118924,0.00035370994,0.00014030229],"category_scores_gemma":[0.000287761,0.00058891665,0.00014809587,0.00090603216,0.000029319666,0.00037763594,0.002951605,0.0017846696,0.00011062183],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007943157,0.0023991189,0.72576547,0.011420591,0.00035966048,0.0006190408,0.0009969969,0.22306366,0.0000124489425,0.00033294453,0.0011453902,0.033805277],"study_design_scores_gemma":[0.0011226285,0.00022165461,0.6223978,0.0031782577,0.0006181779,0.00001244262,0.0014238465,0.27289662,0.000023599825,0.0030496905,0.09237478,0.00268052],"about_ca_topic_score_codex":0.000339766,"about_ca_topic_score_gemma":0.00021484047,"teacher_disagreement_score":0.10336766,"about_ca_system_score_codex":0.000109315464,"about_ca_system_score_gemma":0.000023323375,"threshold_uncertainty_score":0.9996562},"labels":[],"label_agreement":null},{"id":"W4396610513","doi":"10.1007/s10664-024-10462-8","title":"Patterns of multi-container composition for service orchestration with Docker Compose","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Service-Oriented Architecture and Web Services","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Orchestration; Container (type theory); Service composition; Computer science; Service (business); Operating system; Business; Engineering; Marketing","score_opus":0.018755243539334786,"score_gpt":0.2640178082306203,"score_spread":0.24526256469128555,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4396610513","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.23152962,0.00013255667,0.7667828,0.000698844,0.00018607895,0.00020493104,0.000014707275,0.00044618754,0.0000042716806],"genre_scores_gemma":[0.849904,0.0000016475548,0.14932927,0.00052849145,0.000105295345,0.00004698406,0.0000526407,0.000023897683,0.000007761259],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99900365,0.000016326356,0.00022100142,0.00033247564,0.0001949641,0.00023160814],"domain_scores_gemma":[0.99920213,0.00030974878,0.000036324782,0.00025001809,0.00011683214,0.00008492191],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00009548736,0.00017904305,0.00017719992,0.000121374665,0.000046210465,0.00010309542,0.00033175037,0.000066749686,0.0000033642166],"category_scores_gemma":[0.0000061222763,0.00014125193,0.00006387747,0.00045345785,0.000004610055,0.00033150095,0.00007686062,0.00014667117,0.0000057043208],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0003805855,0.0012595776,0.12968886,0.015796982,0.0011858662,0.0002741564,0.039686672,0.67417806,0.05764505,0.025398744,0.00042277493,0.054082688],"study_design_scores_gemma":[0.0011071654,0.00036927094,0.045405544,0.0008332584,0.000059967562,0.00006266549,0.000056308036,0.9255232,0.018320085,0.00019980631,0.007465419,0.0005972814],"about_ca_topic_score_codex":0.00007323885,"about_ca_topic_score_gemma":0.00005544241,"teacher_disagreement_score":0.6183744,"about_ca_system_score_codex":0.000034018773,"about_ca_system_score_gemma":0.00003847507,"threshold_uncertainty_score":0.5760087},"labels":[],"label_agreement":null},{"id":"W4399259577","doi":"10.1007/s10664-024-10464-6","title":"Towards graph-anonymization of software analytics data: empirical study on JIT defect prediction","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Analytics; Data mining; Empirical research; Software; Graph; Data science; Statistics; Theoretical computer science; Mathematics; Programming language","score_opus":0.06589437020513635,"score_gpt":0.3439454061040775,"score_spread":0.27805103589894115,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399259577","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14177115,0.000577612,0.8526489,0.00016672918,0.0012466375,0.00044959545,0.00008178692,0.003047864,0.000009696723],"genre_scores_gemma":[0.9522227,0.000021321672,0.047023725,0.000056052497,0.0003574948,0.00006075718,0.000094863935,0.00010786487,0.00005519349],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9958278,0.00010807798,0.0006848006,0.0012991922,0.001416858,0.0006632582],"domain_scores_gemma":[0.9949925,0.0024803851,0.00006136293,0.00197608,0.00018405645,0.00030561062],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010990278,0.00045264827,0.00050639024,0.0009186955,0.00009946365,0.00032339958,0.0018617325,0.0002266685,0.000023809367],"category_scores_gemma":[0.0062293573,0.00042404284,0.00023376833,0.003267763,0.00004527239,0.0008622362,0.0009741728,0.0008166108,0.000060875922],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025400386,0.00068862754,0.7455744,0.0005460212,0.00061584817,0.00029435754,0.0015061751,0.22768952,0.00006553968,0.0001376036,0.010276212,0.012580308],"study_design_scores_gemma":[0.0006518567,0.0011775162,0.6509946,0.00039066424,0.00013704742,0.000047221736,0.000031098396,0.33878884,0.0004210951,0.00020669431,0.0063649095,0.00078844785],"about_ca_topic_score_codex":0.000016989203,"about_ca_topic_score_gemma":0.0000020156613,"teacher_disagreement_score":0.81045157,"about_ca_system_score_codex":0.00028445965,"about_ca_system_score_gemma":0.00025176725,"threshold_uncertainty_score":0.9998211},"labels":[],"label_agreement":null},{"id":"W4399333905","doi":"10.1007/s10664-024-10485-1","title":"A large-scale exploratory study on the proxy pattern in Ethereum","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Spam and Phishing Detection","field":"Computer Science","cited_by":14,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Proxy (statistics); Scale (ratio); Computer science; Geography; Cartography; Machine learning","score_opus":0.026902270609266733,"score_gpt":0.2671268061620694,"score_spread":0.24022453555280268,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399333905","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49313056,0.00011355,0.50457156,0.0007084056,0.0006654152,0.00016936743,8.9071585e-7,0.0006263424,0.000013882252],"genre_scores_gemma":[0.9989941,0.0000011286245,0.00043545727,0.00028122892,0.00014293585,0.000097293465,3.7544336e-7,0.000019151183,0.00002831808],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99899226,0.00006046482,0.00014537209,0.0003232976,0.00023733673,0.00024127291],"domain_scores_gemma":[0.99920917,0.0003826464,0.0000107434125,0.00033588664,0.000011531715,0.00005000187],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00048595236,0.00012861654,0.00010156112,0.000108837616,0.00005218493,0.00019834512,0.00037758477,0.000052725907,0.000011771219],"category_scores_gemma":[0.00013840581,0.000090275906,0.000046907528,0.000604619,0.0000051222255,0.00020742518,0.00012798345,0.00042206558,0.00010545821],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000023677652,0.0018184733,0.6289285,0.00033004436,0.00020047018,0.0010280747,0.25952566,0.036203165,0.0005333613,0.0010784639,0.012439529,0.057890594],"study_design_scores_gemma":[0.00063208514,0.00079122355,0.25676203,0.0005216246,0.000014542189,0.00001952625,0.001289098,0.6893366,0.0009867528,0.000765078,0.047983333,0.0008980868],"about_ca_topic_score_codex":0.0000069888515,"about_ca_topic_score_gemma":0.000017933176,"teacher_disagreement_score":0.65313345,"about_ca_system_score_codex":0.00007662622,"about_ca_system_score_gemma":0.000021498165,"threshold_uncertainty_score":0.36813444},"labels":[],"label_agreement":null},{"id":"W4399334258","doi":"10.1007/s10664-024-10493-1","title":"Towards understanding barriers and mitigation strategies of software engineers with non-traditional educational and occupational backgrounds","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Disability Education and Employment","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia, Okanagan Campus; University of British Columbia; University of Waterloo","funders":"","keywords":"Engineering; Software; Computer science; Knowledge management; Software engineering; Data science; Engineering management","score_opus":0.055875200237333236,"score_gpt":0.3262491438988426,"score_spread":0.27037394366150935,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399334258","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7883314,0.0003633764,0.20567209,0.0040510637,0.000567231,0.00022492722,0.000056800127,0.00022364952,0.000509478],"genre_scores_gemma":[0.9892288,0.000021604259,0.010338001,0.000047481353,0.00020260928,0.000030495603,0.00005353164,0.000015745296,0.00006173726],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.99899215,0.000022037153,0.0001784467,0.00023282996,0.00038203498,0.00019252735],"domain_scores_gemma":[0.99907446,0.0005509076,0.000023743569,0.00006211993,0.000060266353,0.00022851578],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00025377073,0.00012816284,0.00012372238,0.00012702525,0.00017798982,0.00019930527,0.000063577805,0.00007474533,0.000266378],"category_scores_gemma":[0.0003731508,0.00012381197,0.00003411413,0.00031236076,0.00025254238,0.00047511773,0.000015272773,0.00012292832,0.0000020344808],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005070456,0.0001611845,0.40388897,0.0012531184,0.00031950918,0.000005120197,0.05439143,0.012267244,0.00006850442,0.5187119,0.006361348,0.002520923],"study_design_scores_gemma":[0.0006607349,0.0002408864,0.9021627,0.00078717066,0.000120671546,0.000024887224,0.046101432,0.004471891,0.000070444905,0.03310008,0.0113414135,0.00091771694],"about_ca_topic_score_codex":0.00011414057,"about_ca_topic_score_gemma":0.00004393996,"teacher_disagreement_score":0.4982737,"about_ca_system_score_codex":0.00032299737,"about_ca_system_score_gemma":0.00092039676,"threshold_uncertainty_score":0.5048906},"labels":[],"label_agreement":null},{"id":"W4399362212","doi":"10.1007/s10664-024-10448-6","title":"VulNet: Towards improving vulnerability management in the Maven ecosystem","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; Concordia University","funders":"Indian Institute of Technology Gandhinagar","keywords":"Vulnerability (computing); Ecosystem; Environmental resource management; Environmental science; Business; Computer science; Ecology; Computer security; Biology","score_opus":0.01655092419775144,"score_gpt":0.2847365155113277,"score_spread":0.2681855913135763,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399362212","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.011656821,0.00030438197,0.9839826,0.0005052598,0.0005401231,0.00033761378,0.0000027301767,0.0025170548,0.00015338037],"genre_scores_gemma":[0.8511752,0.0000071904237,0.14828016,0.00015957963,0.000099015255,0.00021170842,0.0000011578309,0.00002196375,0.000044018518],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99848884,0.00006168565,0.00029069584,0.0004966492,0.0003176681,0.00034444258],"domain_scores_gemma":[0.99904555,0.00028062618,0.000022112312,0.00057382527,0.000020259473,0.000057614037],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00072840584,0.0001887472,0.00015542527,0.00019516825,0.00006205574,0.00024969145,0.0008130928,0.00007306413,0.000008255354],"category_scores_gemma":[0.00016511943,0.00014420162,0.000087248765,0.0009091447,0.0000093747385,0.00043070858,0.0003057747,0.00042683794,0.000026392023],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000068538748,0.00010882065,0.0018584015,0.0014546026,0.00007052168,0.00094701356,0.0022004293,0.0306453,0.00022850075,0.011700858,0.0016765794,0.9491021],"study_design_scores_gemma":[0.00039574492,0.00021882865,0.039312422,0.0005005558,0.000028212336,0.00032579002,0.000106620406,0.6872747,0.0052853455,0.009802249,0.2553804,0.001369157],"about_ca_topic_score_codex":0.00001373672,"about_ca_topic_score_gemma":0.0000058510186,"teacher_disagreement_score":0.947733,"about_ca_system_score_codex":0.0002768786,"about_ca_system_score_gemma":0.000022210872,"threshold_uncertainty_score":0.58803713},"labels":[],"label_agreement":null},{"id":"W4399362314","doi":"10.1007/s10664-024-10487-z","title":"Characterizing and classifying developer forum posts with their intentions","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Mitacs","keywords":"Computer science; Knowledge management","score_opus":0.025739535136037356,"score_gpt":0.2690836191399585,"score_spread":0.24334408400392113,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399362314","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.15474555,0.0009353578,0.8399348,0.0015195159,0.0004068373,0.00012909203,0.0000035733515,0.002305971,0.000019259125],"genre_scores_gemma":[0.9199287,0.000023751132,0.0796047,0.00015525556,0.00009266616,0.00005432482,0.0000042842976,0.00006019091,0.00007611195],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9982931,0.000016255779,0.00020440172,0.00054132886,0.0003648794,0.00058004487],"domain_scores_gemma":[0.9983345,0.0010160364,0.000014524338,0.00033862554,0.000071026334,0.00022526238],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023056196,0.00026256283,0.00020165915,0.00029600406,0.00011351429,0.00057280425,0.00043417586,0.000092723814,0.000009823888],"category_scores_gemma":[0.00065424124,0.00020469018,0.00005801045,0.0008043709,0.000032211716,0.00074872206,0.0004035495,0.0005036855,0.000028097016],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000030730745,0.00016380097,0.7321415,0.00200001,0.0008514443,0.0015835435,0.008694709,0.007118403,0.015249514,0.007389344,0.003920813,0.22085619],"study_design_scores_gemma":[0.00052508275,0.00030277908,0.63998115,0.001904758,0.000024983132,0.0010974181,0.00010303879,0.23774926,0.004012702,0.00021566432,0.11235884,0.0017243194],"about_ca_topic_score_codex":0.0000040635637,"about_ca_topic_score_gemma":0.0000015183184,"teacher_disagreement_score":0.76518315,"about_ca_system_score_codex":0.00013356144,"about_ca_system_score_gemma":0.00010718763,"threshold_uncertainty_score":0.8347023},"labels":[],"label_agreement":null},{"id":"W4399615587","doi":"10.1007/s10664-024-10450-y","title":"How far are we with automated machine learning? characterization and challenges of AutoML toolkits","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University; University of Calgary","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Characterization (materials science); Computer science; Artificial intelligence; Machine learning; Nanotechnology; Materials science","score_opus":0.02592931027558718,"score_gpt":0.24558559859600892,"score_spread":0.21965628832042175,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399615587","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.029161748,0.0049816626,0.952388,0.009592379,0.00013919537,0.00009105576,0.000011382222,0.0036266427,0.000007954043],"genre_scores_gemma":[0.97888863,0.0011996294,0.019524593,0.000030119882,0.00006344818,0.000015651602,0.000059715898,0.000032273423,0.00018594065],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99906987,0.00004085178,0.00014113325,0.00036692285,0.0002054617,0.00017578522],"domain_scores_gemma":[0.9994222,0.00016882237,0.00006421043,0.00022419477,0.000041217576,0.00007934346],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00014771099,0.00016235912,0.00018268128,0.00015249483,0.00005001102,0.0002205696,0.00020132218,0.00007431487,0.0000024743806],"category_scores_gemma":[0.00022984171,0.0001303305,0.00002600879,0.000376779,0.000017871294,0.00047103205,0.000099865494,0.00025887118,0.00000381],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003161925,0.00018132785,0.12231929,0.005054217,0.00028682328,0.00017547865,0.005417316,0.019307535,0.007794795,0.016526071,0.0003328328,0.8225727],"study_design_scores_gemma":[0.00009175236,0.00008491086,0.15071067,0.00020599514,0.000008148154,0.000026438775,0.000008380929,0.768507,0.00013766754,0.0000075117105,0.08004882,0.00016268887],"about_ca_topic_score_codex":0.0000033228703,"about_ca_topic_score_gemma":0.0000013760347,"teacher_disagreement_score":0.9497269,"about_ca_system_score_codex":0.000024623087,"about_ca_system_score_gemma":0.000022126032,"threshold_uncertainty_score":0.5314724},"labels":[],"label_agreement":null},{"id":"W4399644038","doi":"10.1007/s10664-024-10457-5","title":"Utilization of pre-trained language models for adapter-based knowledge transfer in software engineering","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Okanagan University College; University of British Columbia, Okanagan Campus; University of British Columbia","funders":"","keywords":"Adapter (computing); Computer science; Code (set theory); Automatic summarization; Downstream (manufacturing); Software; Artificial intelligence; Natural language processing; Source code; Programming language; Engineering; Computer hardware","score_opus":0.046160511992757766,"score_gpt":0.31951923156039336,"score_spread":0.2733587195676356,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399644038","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.106163085,0.002512527,0.88807905,0.000063060455,0.0005025719,0.0005354747,0.00003483802,0.0021063937,0.0000029860912],"genre_scores_gemma":[0.8534494,0.000008366714,0.14599869,0.000017632212,0.000111156834,0.00023451779,0.000028039454,0.00010600573,0.00004616359],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9975589,0.000028704926,0.0005634526,0.00069564104,0.0004332408,0.00072009314],"domain_scores_gemma":[0.99582523,0.0032899333,0.0000188753,0.0005468912,0.00011605152,0.00020300511],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00054653216,0.0003926278,0.00044455315,0.00083306624,0.000033947217,0.00013187008,0.0007822468,0.0002299025,0.000012667709],"category_scores_gemma":[0.002096102,0.0004095064,0.0002357558,0.0016462278,0.000022721637,0.00059106946,0.00012004556,0.00042947545,0.000006674304],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000023560939,0.00012766657,0.0029655634,0.0021758096,0.00005640932,0.00004977635,0.0032115635,0.9782438,0.0010495558,0.0011455927,0.00017782788,0.010772833],"study_design_scores_gemma":[0.0005266158,0.00011853114,0.006877706,0.00063747953,0.000011852358,0.0000068428535,0.0000070687147,0.98337543,0.006595522,0.000057902755,0.0013501871,0.00043488864],"about_ca_topic_score_codex":0.000016549318,"about_ca_topic_score_gemma":0.00000594873,"teacher_disagreement_score":0.7472863,"about_ca_system_score_codex":0.00026070734,"about_ca_system_score_gemma":0.00021463569,"threshold_uncertainty_score":0.99983567},"labels":[],"label_agreement":null},{"id":"W4399686980","doi":"10.1007/s10664-024-10500-5","title":"Common challenges of deep reinforcement learning applications development: an empirical study","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec; Canadian Institute for Advanced Research","keywords":"Computer science; Taxonomy (biology); Leverage (statistics); Popularity; Reinforcement learning; Artificial intelligence; Data science; Software engineering","score_opus":0.039303453045082565,"score_gpt":0.3277676583422513,"score_spread":0.28846420529716876,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399686980","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.112002686,0.0019291072,0.8830235,0.00012081217,0.00019732224,0.00044708757,3.3073295e-7,0.0022480276,0.000031099913],"genre_scores_gemma":[0.9631806,0.00002390165,0.036247168,0.00001611213,0.00011648335,0.00031008807,0.000007683204,0.000056204382,0.000041734325],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9974092,0.000069372225,0.0005384495,0.0006826479,0.000757054,0.00054322253],"domain_scores_gemma":[0.99750566,0.0013790433,0.000040612285,0.0006885441,0.00009693089,0.00028918043],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006953875,0.00030031818,0.00034770623,0.00043664963,0.00012310628,0.00016701248,0.0010225632,0.00012039017,0.000020951444],"category_scores_gemma":[0.00049391034,0.00029563624,0.00008712632,0.0010543308,0.000025984049,0.00047310075,0.00051124784,0.00065389107,0.000070491944],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009076185,0.0006328684,0.24842922,0.00086040644,0.00029851327,0.00014080352,0.018196091,0.5643942,0.00010766435,0.0010120033,0.00010651373,0.16581264],"study_design_scores_gemma":[0.0005427838,0.00090201705,0.36678782,0.00023871592,0.00003202899,0.000049225215,0.00030481015,0.5819047,0.0010265097,0.00013383786,0.046932653,0.0011448817],"about_ca_topic_score_codex":0.000006231698,"about_ca_topic_score_gemma":0.00000429308,"teacher_disagreement_score":0.85117793,"about_ca_system_score_codex":0.00021367487,"about_ca_system_score_gemma":0.0001132977,"threshold_uncertainty_score":0.9999496},"labels":[],"label_agreement":null},{"id":"W4399703649","doi":"10.1007/s10664-024-10492-2","title":"Post deployment recycling of machine learning models","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Stream Mining Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Software deployment; Reuse; Computer science; Artificial intelligence; Machine learning; Baseline (sea); Artificial neural network; Inference; Random forest; Logistic regression; Predictive modelling; Engineering","score_opus":0.03166989974971166,"score_gpt":0.27371393124768234,"score_spread":0.24204403149797069,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399703649","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.020522302,0.0010870321,0.9744176,0.00022842082,0.00023911154,0.000072625524,0.00001127076,0.0033641553,0.000057446472],"genre_scores_gemma":[0.5782191,0.00002720494,0.4215511,0.000045223278,0.000044925782,0.000012701419,0.000011751912,0.00003154784,0.000056402543],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9988047,0.000021984817,0.0002780303,0.00036659974,0.00026282298,0.0002658519],"domain_scores_gemma":[0.99908817,0.00039342625,0.00003123344,0.00035669925,0.000040148727,0.000090310845],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002752404,0.00016528528,0.00019054126,0.00019651187,0.00003594228,0.00011436127,0.0005886576,0.000069533984,0.00000856791],"category_scores_gemma":[0.0003666236,0.00015564988,0.00008597223,0.00044159841,0.00001271565,0.00051158306,0.00042729173,0.00033990922,0.0000134156135],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000022000766,0.00028588172,0.026839249,0.0015054108,0.00040463966,0.0006675712,0.006248757,0.48513597,0.01222318,0.028912915,0.005304981,0.43244946],"study_design_scores_gemma":[0.000054935463,0.000102905666,0.00041593137,0.00028185316,0.000008594384,0.000026960306,0.0000027669198,0.9836341,0.0053941202,0.00076370465,0.009080454,0.00023366768],"about_ca_topic_score_codex":0.00003709805,"about_ca_topic_score_gemma":8.5894743e-7,"teacher_disagreement_score":0.5576968,"about_ca_system_score_codex":0.00007159151,"about_ca_system_score_gemma":0.000036702022,"threshold_uncertainty_score":0.63472176},"labels":[],"label_agreement":null},{"id":"W4399774065","doi":"10.1007/s10664-024-10452-w","title":"A literature review and existing challenges on software logging practices","year":2024,"lang":"en","type":"review","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; École de Technologie Supérieure","funders":"","keywords":"Logging; Computer science; Software; Software engineering; Data science; Forestry; Geography; Operating system","score_opus":0.10665188973831913,"score_gpt":0.3761438582854184,"score_spread":0.2694919685470993,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399774065","genre_codex":"review","genre_gemma":"review","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"review","genre_consensus":"review","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0000010262498,0.9819801,0.013248723,0.00041950293,0.0011713916,0.0008372068,0.000019786869,0.0022800774,0.0000421988],"genre_scores_gemma":[0.0000015825032,0.977493,0.02116375,0.0002865664,0.00052238116,0.00026364016,0.000024819059,0.00011043573,0.00013382174],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99610853,0.00018161158,0.000951102,0.0015589372,0.000520115,0.0006797073],"domain_scores_gemma":[0.99615526,0.0016740003,0.0005631315,0.0012192057,0.00009979099,0.00028862222],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010901683,0.0009982319,0.0022368347,0.00039478176,0.00015674991,0.00050382514,0.00108951,0.0006084721,0.0000051587695],"category_scores_gemma":[0.004520068,0.00069382123,0.0005416244,0.0011238526,0.00003531148,0.00062310486,0.0007811487,0.0019101714,0.00018266412],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[4.5408453e-7,0.000022943283,0.000013843241,0.35141537,0.00012685738,0.00016724797,0.00019847171,0.000008025454,3.5940937e-9,0.00010535336,0.0018223723,0.64611906],"study_design_scores_gemma":[0.000039489823,0.000062224455,0.000015011616,0.21301669,0.00047752814,0.00047553275,0.0000016086478,0.000113220965,5.1887703e-8,0.000029132958,0.78514755,0.00062193506],"about_ca_topic_score_codex":0.0000017843765,"about_ca_topic_score_gemma":3.7672604e-7,"teacher_disagreement_score":0.7833252,"about_ca_system_score_codex":0.00023206635,"about_ca_system_score_gemma":0.00018382889,"threshold_uncertainty_score":0.9995513},"labels":[],"label_agreement":null},{"id":"W4399871949","doi":"10.1007/s10664-024-10501-4","title":"Systematic Evaluation of Deep Learning Models for Log-based Failure Prediction","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"HORIZON EUROPE Framework Programme; Canada Research Chairs; Natural Sciences and Engineering Research Council of Canada; European Commission; Science Foundation Ireland; Université du Luxembourg","keywords":"Computer science; Machine learning; Artificial intelligence; Artificial neural network; Algorithm; Convolutional neural network; Deep learning; Encoder; Predictive modelling; Data mining","score_opus":0.030171465703690212,"score_gpt":0.2758273760452596,"score_spread":0.24565591034156936,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4399871949","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.022065183,0.001525577,0.97379744,0.000078443445,0.0007155329,0.0007536022,0.0000037031896,0.001054915,0.000005572862],"genre_scores_gemma":[0.95527124,0.000001966541,0.044259064,0.000011853528,0.00009265574,0.00032299513,0.000010782924,0.000020322086,0.000009108267],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99827784,0.0000859886,0.000468572,0.00034968852,0.00058896415,0.00022895371],"domain_scores_gemma":[0.9984323,0.00083967857,0.00006552543,0.0003170718,0.0002786901,0.00006675566],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0018023187,0.00015830077,0.00028449262,0.00017321423,0.000066053835,0.00008610819,0.0002689055,0.0001300262,0.0000047408375],"category_scores_gemma":[0.0011412364,0.00012956306,0.00016657049,0.00049042783,0.000012228601,0.00048245306,0.00004438586,0.00017555112,0.000008132311],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000019079287,0.00001541261,0.0012895821,0.012673359,0.00003877226,6.2198814e-7,0.0004280924,0.98316485,0.000031728287,0.00014177646,0.000059324593,0.0021545484],"study_design_scores_gemma":[0.00020347371,0.000078226265,0.0005767031,0.001976764,0.00006426674,0.0000050968924,0.00001233481,0.996222,0.00024192035,0.00038590754,0.00010651037,0.00012675082],"about_ca_topic_score_codex":0.000002378913,"about_ca_topic_score_gemma":6.9231453e-7,"teacher_disagreement_score":0.9332061,"about_ca_system_score_codex":0.00020286838,"about_ca_system_score_gemma":0.0001039763,"threshold_uncertainty_score":0.52834284},"labels":[],"label_agreement":null},{"id":"W4400289352","doi":"10.1007/s10664-024-10476-2","title":"Design smells in multi-language systems and bug-proneness: a survival analysis","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Natural language processing; Programming language; Linguistics; Artificial intelligence","score_opus":0.050847855513788974,"score_gpt":0.3200556375862844,"score_spread":0.26920778207249546,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4400289352","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.11004531,0.006223967,0.88136864,0.00006705015,0.0005669858,0.00023323372,0.0000037466327,0.0014895659,0.0000015174624],"genre_scores_gemma":[0.91128856,0.000033112035,0.0883409,0.000012243627,0.00007701665,0.000088140136,0.000004276125,0.00004743233,0.00010831729],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977258,0.00009680409,0.00034743652,0.00073348713,0.00047399182,0.00062246906],"domain_scores_gemma":[0.99632347,0.0028689166,0.00001858288,0.00051478547,0.000039198127,0.00023506487],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009691268,0.00028953288,0.0004291118,0.0010182445,0.000039978473,0.0005327909,0.0005880914,0.00014664848,0.000007762651],"category_scores_gemma":[0.0011636441,0.0002767815,0.0001025886,0.0033984582,0.00002327578,0.00040222952,0.0003142194,0.00046857007,0.00003649373],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000048691127,0.00006149836,0.19745246,0.00049451226,0.00037443027,0.0009416079,0.0025301,0.79418504,0.0004329667,0.00013189782,0.00011346356,0.0032771272],"study_design_scores_gemma":[0.00018255263,0.000028613033,0.10960326,0.00011718596,0.000029340428,0.00001853138,0.000019847654,0.889207,0.0001208523,0.0000053984495,0.00033587107,0.0003315539],"about_ca_topic_score_codex":0.00012837481,"about_ca_topic_score_gemma":0.000005541731,"teacher_disagreement_score":0.80124325,"about_ca_system_score_codex":0.00021088372,"about_ca_system_score_gemma":0.000075308664,"threshold_uncertainty_score":0.9999684},"labels":[],"label_agreement":null},{"id":"W4400289394","doi":"10.1007/s10664-024-10499-9","title":"What causes exceptions in machine learning applications? Mining machine learning-related stack traces on Stack Overflow","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Stream Mining Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Stack (abstract data type); Computer science; Call stack; Machine learning; Artificial intelligence; Operating system","score_opus":0.021904454347741253,"score_gpt":0.28656119998284973,"score_spread":0.2646567456351085,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4400289394","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14999975,0.0049264748,0.8350209,0.00094565656,0.0006464844,0.00048558722,0.00003850258,0.007865048,0.0000715794],"genre_scores_gemma":[0.8932597,0.00070701994,0.10422555,0.00012877496,0.00015329677,0.0002840103,0.00024189043,0.00013329569,0.00086647586],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9975033,0.00010684123,0.0005227478,0.0008673525,0.00043533096,0.0005644743],"domain_scores_gemma":[0.9979958,0.0011492078,0.00007684889,0.0005626863,0.00004060688,0.00017485234],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004923074,0.00039957123,0.00035121027,0.0005957116,0.00016649815,0.000850452,0.0007776772,0.0001896265,0.00007296541],"category_scores_gemma":[0.00061985286,0.00039937164,0.00011427543,0.0015504134,0.000039690884,0.0014502684,0.00037680226,0.0012783944,0.00013305213],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003159789,0.000579338,0.21710026,0.000656291,0.00028124778,0.00061016844,0.010074915,0.51542705,0.0009364777,0.0035246075,0.0040311837,0.24674684],"study_design_scores_gemma":[0.00039588954,0.00042076505,0.014186811,0.0012443493,0.000027224412,0.000067455694,0.00009072227,0.8289036,0.0007845036,0.00036153893,0.15248862,0.0010285336],"about_ca_topic_score_codex":0.000047158308,"about_ca_topic_score_gemma":0.000021994656,"teacher_disagreement_score":0.7432599,"about_ca_system_score_codex":0.00026236023,"about_ca_system_score_gemma":0.0000622278,"threshold_uncertainty_score":0.9998458},"labels":[],"label_agreement":null},{"id":"W4400608520","doi":"10.1007/s10664-024-10488-y","title":"An empirical study on cross-component dependent changes: A case study on the components of OpenStack","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec","funders":"","keywords":"Component (thermodynamics); Computer science; Physics","score_opus":0.0628725410747167,"score_gpt":0.35741987604312486,"score_spread":0.2945473349684082,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4400608520","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9573742,0.00006975222,0.038656194,0.0002914412,0.0013143073,0.0014226827,0.00001034271,0.00084993336,0.000011158846],"genre_scores_gemma":[0.9990065,0.0000010363716,0.00040951036,0.00016126983,0.00017716415,0.00018203723,0.000002769692,0.00004203601,0.00001763806],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99666053,0.00026084692,0.0006048532,0.0009547261,0.0010174746,0.00050154765],"domain_scores_gemma":[0.9968921,0.001200098,0.00007443628,0.0014992908,0.000104136125,0.0002299538],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0013688115,0.0004216619,0.00047427634,0.00029794124,0.0002724291,0.00040274556,0.001179512,0.000117316325,0.000015183713],"category_scores_gemma":[0.00022500085,0.00026685087,0.00014428837,0.0009239677,0.000049632752,0.00031654173,0.00041238294,0.00063484366,0.000094024144],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005721477,0.004971937,0.94338995,0.0001863978,0.00026472379,0.0047253887,0.022375103,0.02117923,0.00012977642,0.000044642336,0.00028431637,0.0023912995],"study_design_scores_gemma":[0.001280163,0.0062769772,0.90966684,0.00034203214,0.00005634335,0.00061634736,0.002200037,0.076951616,0.0006075175,0.000022870967,0.0011399742,0.0008393047],"about_ca_topic_score_codex":0.00014197578,"about_ca_topic_score_gemma":0.000019581672,"teacher_disagreement_score":0.055772383,"about_ca_system_score_codex":0.00022198244,"about_ca_system_score_gemma":0.000050089195,"threshold_uncertainty_score":0.99997836},"labels":[],"label_agreement":null},{"id":"W4400832950","doi":"10.1007/s10664-024-10497-x","title":"Does using Bazel help speed up continuous integration builds?","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Heuristics; Duration (music); Artifact (error); Scheduling (production processes); Service (business); Software engineering; Operating system; Engineering; Operations management; Artificial intelligence","score_opus":0.019175855886186262,"score_gpt":0.27795946949845585,"score_spread":0.2587836136122696,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4400832950","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.26701552,0.00038412792,0.7257091,0.00024771664,0.004689927,0.00015126691,0.0000029546345,0.0017872329,0.000012160075],"genre_scores_gemma":[0.945301,0.000011641762,0.053645376,0.00010705892,0.0005800593,0.000011244625,0.0000050562594,0.000034585897,0.00030399646],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99828804,0.000031840522,0.00038050633,0.00054437347,0.00034512824,0.00041010132],"domain_scores_gemma":[0.9989474,0.000318232,0.000035344066,0.00047121182,0.00008354365,0.0001443039],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00037916537,0.00025594636,0.0002801847,0.0001740403,0.00009787745,0.00038687524,0.00046605326,0.00016069117,0.00002403177],"category_scores_gemma":[0.00039600214,0.00015773835,0.00015762683,0.0007300353,0.000026293043,0.000770587,0.00018285325,0.00034547763,0.00009792947],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000061191175,0.0004255266,0.37455395,0.0042916574,0.0006555514,0.00079316023,0.017489767,0.12649779,0.02072193,0.0050120465,0.017853396,0.43164402],"study_design_scores_gemma":[0.00026047387,0.000085092855,0.013568369,0.0006898187,0.000025849413,0.00009159819,0.00004473857,0.9488584,0.005960482,0.00044861773,0.029249953,0.00071661186],"about_ca_topic_score_codex":0.00004058309,"about_ca_topic_score_gemma":0.0000016907835,"teacher_disagreement_score":0.8223606,"about_ca_system_score_codex":0.00021019812,"about_ca_system_score_gemma":0.00008774896,"threshold_uncertainty_score":0.6432383},"labels":[],"label_agreement":null},{"id":"W4400988474","doi":"10.1007/s10664-024-10518-9","title":"The impact of concept drift and data leakage on log level prediction models","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Stream Mining Techniques","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Queen's University","funders":"","keywords":"Interpretability; Debugging; Computer science; Context (archaeology); Statement (logic); Machine learning; Data mining; Data science; Deep learning; Artificial intelligence; Programming language","score_opus":0.09240412971559399,"score_gpt":0.34139983353286124,"score_spread":0.24899570381726727,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4400988474","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.010845771,0.00056360004,0.98690444,0.00014318239,0.00017046835,0.00008760177,0.00028897033,0.00096443493,0.000031510568],"genre_scores_gemma":[0.9320875,0.00003446692,0.067686155,0.000016725902,0.00007246355,0.00000836233,0.000045952824,0.000018835646,0.000029553434],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9990585,0.000017928694,0.00017601222,0.00036390492,0.00018941719,0.0001942612],"domain_scores_gemma":[0.99837816,0.0006335396,0.000022803846,0.0008789687,0.000019586843,0.00006696616],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00026646676,0.00013169547,0.0001213203,0.00006718607,0.000052390278,0.00017596813,0.000932861,0.000059787126,0.0000016768375],"category_scores_gemma":[0.00031854206,0.000087974106,0.000039278173,0.00024361371,0.000040705832,0.0006184513,0.000649567,0.00020916128,0.0000020455566],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000026724321,0.00016961175,0.0078076213,0.0002169083,0.0004730168,0.000121320474,0.0034904536,0.18115133,0.00072387443,0.023217656,0.26482785,0.5177736],"study_design_scores_gemma":[0.00005291123,0.00015620908,0.0065028393,0.00011784222,0.0000056371323,0.000016041577,0.0000023762698,0.98922265,0.00051135663,0.001045895,0.002255382,0.00011087339],"about_ca_topic_score_codex":0.000033544013,"about_ca_topic_score_gemma":6.4849223e-7,"teacher_disagreement_score":0.9212417,"about_ca_system_score_codex":0.000049848146,"about_ca_system_score_gemma":0.00004788441,"threshold_uncertainty_score":0.358748},"labels":[],"label_agreement":null},{"id":"W4401109317","doi":"10.1007/s10664-024-10523-y","title":"Dependabot and security pull requests: large empirical study","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure","funders":"Natural Sciences and Engineering Research Council of Canada; European Commission","keywords":"Computer science; Empirical research; Computer security; Statistics; Mathematics","score_opus":0.014386359565617011,"score_gpt":0.30830945252597364,"score_spread":0.2939230929603566,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4401109317","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.10663223,0.0007781047,0.88563424,0.00034478985,0.00048442744,0.00027453294,0.0000072113917,0.005814823,0.000029631967],"genre_scores_gemma":[0.94147104,0.000019619793,0.057961196,0.00018853595,0.00015606606,0.0001175918,0.0000015095403,0.000042202355,0.000042250205],"study_design_codex":"observational","study_design_gemma":"not_applicable","domain_scores_codex":[0.99807554,0.000055758745,0.00031001875,0.0007482794,0.00034536512,0.00046504746],"domain_scores_gemma":[0.9987238,0.000500018,0.000025706635,0.00047703178,0.000052943473,0.00022051288],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00042083158,0.00027440625,0.00026537842,0.00022833899,0.000109059736,0.00030177453,0.0004420911,0.0001280799,0.000019348494],"category_scores_gemma":[0.0006978492,0.00025826358,0.00007468319,0.0008048368,0.000019336469,0.00074384094,0.0005444323,0.00059283944,0.00003426756],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000089137524,0.0034125468,0.70487833,0.0022257357,0.00087041745,0.009683413,0.029872721,0.003455195,0.0013102023,0.016613465,0.043953545,0.18363532],"study_design_scores_gemma":[0.0013218875,0.0016147266,0.1478923,0.0005650761,0.00010745393,0.0010865,0.00021384,0.2626015,0.008514871,0.019633088,0.5528632,0.003585552],"about_ca_topic_score_codex":0.000007899427,"about_ca_topic_score_gemma":0.0000074180284,"teacher_disagreement_score":0.8348388,"about_ca_system_score_codex":0.00014909443,"about_ca_system_score_gemma":0.000043683434,"threshold_uncertainty_score":0.99998695},"labels":[],"label_agreement":null},{"id":"W4401253331","doi":"10.1007/s10664-024-10514-z","title":"IRJIT: A simple, online, information retrieval approach for just-in-time software defect prediction","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University; University of Alberta","funders":"","keywords":"Computer science; Simple (philosophy); Information retrieval; Data mining; Software; Machine learning; Artificial intelligence; Programming language","score_opus":0.02499642994780278,"score_gpt":0.28696472697798037,"score_spread":0.2619682970301776,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4401253331","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.045241818,0.00047847224,0.9482628,0.00010222743,0.0006285324,0.00071551895,0.00014476464,0.0044186264,0.000007213327],"genre_scores_gemma":[0.31473863,0.000028146282,0.6827137,0.00022802256,0.0008216176,0.0003530711,0.0007839494,0.00016603083,0.00016684858],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.997047,0.000035539273,0.0006024919,0.00065226376,0.00080749445,0.000855217],"domain_scores_gemma":[0.9965793,0.0023857523,0.000040401912,0.00058920245,0.00014039307,0.000264958],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00081766065,0.00038297486,0.00036301717,0.0008273606,0.00009051833,0.00057038636,0.00095873454,0.0002955585,0.000013382432],"category_scores_gemma":[0.0063201804,0.00038710216,0.00027474685,0.0021434468,0.000023988881,0.0017957322,0.00033866995,0.0007125728,0.00007696715],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000111686095,0.0003632691,0.099419154,0.003732862,0.0002811455,0.000096911754,0.0023087338,0.838023,0.00024037118,0.00042131395,0.02095475,0.034046795],"study_design_scores_gemma":[0.0005673195,0.00018249331,0.05321984,0.0001662634,0.000018417311,0.000055616052,0.000008284934,0.9243411,0.0001670078,0.00012060232,0.020625543,0.0005275522],"about_ca_topic_score_codex":0.00000827854,"about_ca_topic_score_gemma":4.2148497e-7,"teacher_disagreement_score":0.26949683,"about_ca_system_score_codex":0.00051777175,"about_ca_system_score_gemma":0.00018208538,"threshold_uncertainty_score":0.9998581},"labels":[],"label_agreement":null},{"id":"W4401667316","doi":"10.1007/s10664-024-10533-w","title":"Impact of log parsing on deep learning-based anomaly detection","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Fonds National de la Recherche Luxembourg; Science Foundation Ireland","keywords":"Anomaly detection; Parsing; Anomaly (physics); Computer science; Artificial intelligence; Natural language processing; Deep learning; Physics","score_opus":0.012061831520582988,"score_gpt":0.2753947141403206,"score_spread":0.2633328826197376,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4401667316","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.43452424,0.00021420534,0.56393677,0.000022335875,0.00044319523,0.0000628017,6.1002777e-7,0.0007876225,0.000008214613],"genre_scores_gemma":[0.9946241,0.0000028106008,0.005200387,0.0000152261055,0.00011231728,0.000011684653,0.0000018603614,0.000021036298,0.000010625721],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9987398,0.000038949893,0.00027709373,0.00037063012,0.00026940458,0.0003041251],"domain_scores_gemma":[0.99895036,0.0005293405,0.000039287148,0.00032208176,0.00005353698,0.00010536384],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00028379718,0.00019050343,0.00022069227,0.00022229122,0.00006714499,0.00009113236,0.00026247278,0.00012318783,0.000013566571],"category_scores_gemma":[0.00039648268,0.00014957947,0.00024784452,0.00076061505,0.00001997786,0.000254921,0.000052837364,0.00035605195,0.00005016213],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00001380749,0.000057753692,0.14621435,0.00028314724,0.00004973699,0.000022810373,0.00028550308,0.7976631,0.00046209714,0.000012507338,0.000051202413,0.054884028],"study_design_scores_gemma":[0.00013543048,0.00052483333,0.21189313,0.00020591718,0.000007790163,0.000015518195,0.0000014868775,0.783646,0.0024727562,0.000024965557,0.0008548245,0.00021735902],"about_ca_topic_score_codex":0.000028260836,"about_ca_topic_score_gemma":0.0000011191312,"teacher_disagreement_score":0.56009984,"about_ca_system_score_codex":0.00025682413,"about_ca_system_score_gemma":0.00007373441,"threshold_uncertainty_score":0.6099674},"labels":[],"label_agreement":null},{"id":"W4401995539","doi":"10.1007/s10664-024-10535-8","title":"Data-access performance anti-patterns in data-intensive systems","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec – Nature et technologies; Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; NoSQL; Data access; Data quality; Database; Component (thermodynamics); Scalability; Engineering","score_opus":0.09091578223945451,"score_gpt":0.3403247534093067,"score_spread":0.24940897116985217,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4401995539","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39832562,0.0017612363,0.5938849,0.00035238618,0.0038282492,0.00028235326,0.00013533987,0.001418351,0.000011553332],"genre_scores_gemma":[0.9958489,0.00011432441,0.0032235072,0.00013382455,0.00038635317,0.00003165285,0.00017719642,0.00003724021,0.00004700204],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99715054,0.000043695025,0.00057240104,0.0012008458,0.0004596701,0.0005728659],"domain_scores_gemma":[0.9964064,0.00042967324,0.0000470892,0.0028673762,0.00010152429,0.0001479806],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00086826825,0.00030086088,0.0004224534,0.0002704525,0.00006774556,0.00071101985,0.004385156,0.00014677175,0.000010564478],"category_scores_gemma":[0.0005365773,0.00024818646,0.000047549718,0.0010507926,0.000024671344,0.003738209,0.0029488294,0.00051376515,0.0001438243],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000048633515,0.00006027499,0.9078114,0.0029205696,0.000092383976,0.0005047434,0.00058661395,0.06618725,0.000016533228,0.000058034882,0.012228692,0.009528659],"study_design_scores_gemma":[0.00011337729,0.000020729478,0.12793054,0.0006506494,0.000008913394,0.00012407327,0.000013944161,0.8366795,0.00002430323,0.000003542109,0.034092277,0.00033814387],"about_ca_topic_score_codex":0.00011481632,"about_ca_topic_score_gemma":0.000009613326,"teacher_disagreement_score":0.7798808,"about_ca_system_score_codex":0.00014805111,"about_ca_system_score_gemma":0.00014095199,"threshold_uncertainty_score":0.999997},"labels":[],"label_agreement":null},{"id":"W4402050968","doi":"10.1007/s10664-024-10477-1","title":"On combining commit grouping and build skip prediction to reduce redundant continuous integration activity","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Commit; Computer science; Database","score_opus":0.021112466439620347,"score_gpt":0.28530239328041973,"score_spread":0.2641899268407994,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402050968","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.21643277,0.00011884489,0.76722264,0.0006548041,0.0006082682,0.00015359507,0.0000034848065,0.014774957,0.00003064267],"genre_scores_gemma":[0.8669764,0.0000037587165,0.13260134,0.00020132714,0.0001120936,0.000043489486,0.0000033048802,0.000028677832,0.000029647279],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99861926,0.00004041895,0.00021651176,0.0005482761,0.00026482227,0.00031068508],"domain_scores_gemma":[0.9982319,0.0011634437,0.000029867053,0.00035015485,0.00005221976,0.00017241115],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00040918562,0.00022727426,0.00021856398,0.0002598206,0.00013608647,0.0004275401,0.0002835311,0.00010680062,0.0000018165259],"category_scores_gemma":[0.0018866383,0.00021603792,0.00005028531,0.0006476004,0.000019301391,0.00036657145,0.00020944397,0.00045334283,0.000012506724],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000068908426,0.00029184108,0.021161497,0.00046682297,0.00014117893,0.0002917821,0.0060163615,0.0122079095,0.008170389,0.012833222,0.08313455,0.85521555],"study_design_scores_gemma":[0.00037700948,0.0011871778,0.0632141,0.0034379254,0.000032587224,0.0002500824,0.000010134882,0.9047509,0.009865593,0.009819921,0.005994017,0.0010605445],"about_ca_topic_score_codex":0.000045162196,"about_ca_topic_score_gemma":9.587746e-7,"teacher_disagreement_score":0.892543,"about_ca_system_score_codex":0.00016171468,"about_ca_system_score_gemma":0.000033387256,"threshold_uncertainty_score":0.8809771},"labels":[],"label_agreement":null},{"id":"W4402136389","doi":"10.1007/s10664-024-10528-7","title":"Consensus task interaction trace recommender to guide developers’ software navigation","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université de Montréal; Concordia University; Polytechnique Montréal","funders":"","keywords":"TRACE (psycholinguistics); Recommender system; Computer science; Task (project management); Software; World Wide Web; Human–computer interaction; Software engineering; Data science; Engineering; Programming language; Systems engineering","score_opus":0.036349133622559676,"score_gpt":0.32891704916676656,"score_spread":0.2925679155442069,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402136389","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.052301757,0.00052472885,0.9371352,0.0023311928,0.002486274,0.00030847138,0.000010398119,0.004877973,0.000023969696],"genre_scores_gemma":[0.39291456,0.000011157602,0.60519236,0.00043864053,0.0004148858,0.0001633549,0.000026911533,0.00013770725,0.00070041954],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9971571,0.00005385943,0.0005089015,0.0008998078,0.0006270838,0.0007532681],"domain_scores_gemma":[0.99573994,0.0029871801,0.00002958805,0.0006313734,0.00017189744,0.0004399992],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00059574086,0.00038537834,0.00029323497,0.0005083413,0.000119397475,0.0005460621,0.0008081369,0.00018542036,0.000046402733],"category_scores_gemma":[0.004590492,0.00039404244,0.00014133683,0.0018181234,0.000018887986,0.000525048,0.0004117959,0.0007662778,0.000649775],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000038188322,0.00018791782,0.031594567,0.0011824206,0.00045594206,0.0015178388,0.009180169,0.41547355,0.0030830773,0.00061718526,0.28011274,0.2565564],"study_design_scores_gemma":[0.00036059454,0.00019761959,0.023685498,0.0010514796,0.00002276327,0.00054261007,0.000054898057,0.16481596,0.0029092408,0.00016456682,0.80474144,0.0014532999],"about_ca_topic_score_codex":0.000031101365,"about_ca_topic_score_gemma":0.0000021016006,"teacher_disagreement_score":0.5246287,"about_ca_system_score_codex":0.00083422975,"about_ca_system_score_gemma":0.00021000707,"threshold_uncertainty_score":0.99985117},"labels":[],"label_agreement":null},{"id":"W4402237118","doi":"10.1007/s10664-024-10527-8","title":"An empirical study of token-based micro commits","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Radiation Effects in Electronics","field":"Engineering","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Japan Society for the Promotion of Science; Natural Sciences and Engineering Research Council of Canada","keywords":"Security token; Empirical research; Computer science; Mathematics; Statistics; Computer security","score_opus":0.011498103106017855,"score_gpt":0.28642940074989637,"score_spread":0.2749312976438785,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402237118","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8585826,0.0010944129,0.13678361,0.00003622617,0.0007036518,0.00028408028,0.0000121473395,0.0024735192,0.000029746756],"genre_scores_gemma":[0.9954952,0.000003574436,0.0040531354,0.0000523009,0.0001889944,0.00004953794,0.000016909344,0.0001296209,0.000010698574],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9984579,0.000050827457,0.00040398154,0.0003174259,0.00032433157,0.00044550354],"domain_scores_gemma":[0.9987028,0.0006343799,0.000017649247,0.00042716626,0.000037782404,0.00018020956],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00023703086,0.0003004239,0.00033585422,0.0002779995,0.00003808928,0.00006917354,0.0002981535,0.00015605548,0.000051071078],"category_scores_gemma":[0.0001575225,0.00030746483,0.00010139788,0.0007612747,0.000018463323,0.0001820515,0.00002330821,0.00053130265,0.000033958157],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000048839756,0.00017451137,0.020669663,0.00023876666,0.000085025946,0.000032029217,0.000586572,0.97327083,0.0008921067,0.000004464706,0.0018818069,0.0021593175],"study_design_scores_gemma":[0.00038170535,0.00044277817,0.028002823,0.000068183275,0.000045722747,0.0000071143368,0.000026047,0.95787793,0.0029271252,0.000010533475,0.009818852,0.00039119058],"about_ca_topic_score_codex":0.000007567217,"about_ca_topic_score_gemma":0.0000069165058,"teacher_disagreement_score":0.13691263,"about_ca_system_score_codex":0.00025842054,"about_ca_system_score_gemma":0.000055069846,"threshold_uncertainty_score":0.9999378},"labels":[],"label_agreement":null},{"id":"W4402460553","doi":"10.1007/s10664-024-10536-7","title":"Quality issues in machine learning software systems","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Software quality; Quality (philosophy); Software engineering; Software; Software development; Operating system","score_opus":0.04127975929546383,"score_gpt":0.3288721992370755,"score_spread":0.28759243994161166,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402460553","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.039526228,0.008408898,0.94716465,0.00058844907,0.0012090023,0.00017051201,0.0000033574008,0.0028855617,0.00004334557],"genre_scores_gemma":[0.9464851,0.000081469734,0.051937096,0.00010369704,0.00030808238,0.00006983627,0.000009740812,0.00006422296,0.00094077474],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99767184,0.00012103039,0.00053643616,0.00063818623,0.00043086975,0.00060161],"domain_scores_gemma":[0.9982638,0.001049557,0.0000370722,0.00042776906,0.000057517314,0.00016433545],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008984094,0.00026795318,0.00033805342,0.0003060975,0.000090160414,0.00054447865,0.0006957834,0.0001397366,0.000027661921],"category_scores_gemma":[0.0019475752,0.00026049907,0.000106784064,0.001279749,0.00002226869,0.00075463945,0.0003044161,0.0006934678,0.00031760873],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000089139785,0.00011718368,0.18421172,0.0010532324,0.000066855384,0.00094632775,0.0050369664,0.7523727,0.00047235907,0.019435294,0.0016828073,0.034595694],"study_design_scores_gemma":[0.00008031623,0.00007946778,0.008508105,0.0004067263,0.000004291565,0.0000380241,0.000068309375,0.864954,0.0009978769,0.00079935533,0.123400934,0.00066261814],"about_ca_topic_score_codex":0.0005549881,"about_ca_topic_score_gemma":0.000027773458,"teacher_disagreement_score":0.9069589,"about_ca_system_score_codex":0.00024914808,"about_ca_system_score_gemma":0.00006160173,"threshold_uncertainty_score":0.99998474},"labels":[],"label_agreement":null},{"id":"W4402810568","doi":"10.1007/s10664-024-10540-x","title":"An empirical study on developers’ shared conversations with ChatGPT in GitHub pull requests and issues","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Online Learning and Analytics","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Kingston Health Sciences Centre; Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Empirical research; World Wide Web","score_opus":0.022466646346923743,"score_gpt":0.3286231495348124,"score_spread":0.3061565031878887,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402810568","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8303592,0.00023867827,0.16259153,0.0054576285,0.00014816229,0.00016956615,0.0000033828828,0.0010165841,0.000015223781],"genre_scores_gemma":[0.9665133,0.0000058565433,0.033072166,0.00017362667,0.000092969065,0.000014373266,0.0000066978446,0.000025833233,0.000095226256],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9986868,0.000048463986,0.00019675697,0.00052026456,0.0002692335,0.00027848702],"domain_scores_gemma":[0.9992749,0.0002289652,0.000016181166,0.00028783316,0.000036966663,0.00015514591],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023006868,0.0001955537,0.00020486263,0.00025608888,0.000058240304,0.00034155676,0.00028279948,0.00006900248,0.0000065607364],"category_scores_gemma":[0.00019105831,0.00015856915,0.00002415208,0.0008067499,0.000019479425,0.0004137968,0.00008213975,0.00039770006,0.000024574354],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010352816,0.00036522496,0.94871074,0.00007463278,0.00006850626,0.00064429804,0.018122692,0.025846647,0.00002272456,0.00033452813,0.0006142805,0.005185381],"study_design_scores_gemma":[0.0003656194,0.00078217365,0.6056481,0.00023705626,0.000013550885,0.000027783191,0.00022510617,0.3858392,0.000024103088,0.00009545854,0.0063148425,0.0004270187],"about_ca_topic_score_codex":0.000023787414,"about_ca_topic_score_gemma":0.000017395254,"teacher_disagreement_score":0.35999256,"about_ca_system_score_codex":0.00008168244,"about_ca_system_score_gemma":0.00007779609,"threshold_uncertainty_score":0.64662623},"labels":[],"label_agreement":null},{"id":"W4402931849","doi":"10.1007/s10664-024-10538-5","title":"Examining ownership models in software teams","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Business; Software; Computer science; Process management; Software engineering; Knowledge management; Operating system","score_opus":0.05269670071759766,"score_gpt":0.28812360806570175,"score_spread":0.2354269073481041,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4402931849","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.016424429,0.0023407193,0.97197974,0.00031596932,0.0009180288,0.00015834502,0.0000030755866,0.007780146,0.00007956704],"genre_scores_gemma":[0.51202327,0.000046542533,0.48731774,0.00016003918,0.00018426534,0.00007829492,0.0000050748654,0.00006950089,0.00011525473],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977827,0.000041611904,0.00040267291,0.0007275629,0.00039765664,0.0006477831],"domain_scores_gemma":[0.99739,0.0017448342,0.000033976223,0.0006013289,0.00003325925,0.00019662082],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00064518827,0.00035314448,0.00030712527,0.00048278036,0.000052599582,0.00045620385,0.0009109874,0.00021665852,0.000021944075],"category_scores_gemma":[0.0010571134,0.00033285143,0.000107460764,0.0013981248,0.000017316193,0.0016962818,0.00034510132,0.0007947568,0.000056423152],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015937636,0.00018020296,0.05122218,0.0011461062,0.00015927202,0.002120705,0.0049478607,0.71510106,0.00020613187,0.016364537,0.009804049,0.19873197],"study_design_scores_gemma":[0.00025788302,0.00015712525,0.009856659,0.0008785205,0.000017741277,0.0001757913,0.00002193823,0.91979986,0.00042826007,0.0063608335,0.06078843,0.0012569709],"about_ca_topic_score_codex":0.00003354309,"about_ca_topic_score_gemma":0.0000022355919,"teacher_disagreement_score":0.49559885,"about_ca_system_score_codex":0.00024316667,"about_ca_system_score_gemma":0.00009576549,"threshold_uncertainty_score":0.9999124},"labels":[],"label_agreement":null},{"id":"W4403036283","doi":"10.1007/s10664-024-10548-3","title":"An empirical study on the effectiveness of large language models for SATD identification and classification","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Manitoba; Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Identification (biology); Computer science; Empirical research; Natural language processing; Artificial intelligence; Data science; Statistics; Mathematics; Biology","score_opus":0.027547982238886545,"score_gpt":0.3451586090035,"score_spread":0.31761062676461344,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403036283","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2642289,0.00057833845,0.73388577,0.00016217369,0.000087492575,0.00041138896,0.000006774099,0.0006370893,0.000002040754],"genre_scores_gemma":[0.9673632,0.0000014090176,0.032366883,0.00004117642,0.000039603136,0.00015610507,0.000007054527,0.000018778459,0.0000057914485],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9989317,0.00012563643,0.00019068092,0.00038138512,0.0002089054,0.00016168114],"domain_scores_gemma":[0.9979991,0.0014624557,0.00003263625,0.00039958267,0.000059805283,0.000046441863],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0013093932,0.00012906225,0.00014118852,0.00011688711,0.00006698204,0.00017985537,0.00040015587,0.000069379465,6.9190344e-7],"category_scores_gemma":[0.00046916405,0.00008937769,0.00004429621,0.0003815466,0.000015268744,0.00039979065,0.00007634996,0.00017983641,0.0000014881106],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0004466712,0.0042461306,0.07825341,0.0067844368,0.0006479669,0.0001369919,0.07012993,0.013022685,0.28295717,0.39800042,0.0026620908,0.14271209],"study_design_scores_gemma":[0.00024202214,0.00039617042,0.07359385,0.00024888042,0.000027798638,0.000006732423,0.00015634214,0.8916962,0.020806782,0.012395501,0.00013538805,0.00029433743],"about_ca_topic_score_codex":0.0000027420394,"about_ca_topic_score_gemma":6.034356e-7,"teacher_disagreement_score":0.8786735,"about_ca_system_score_codex":0.000062318606,"about_ca_system_score_gemma":0.00002603913,"threshold_uncertainty_score":0.36447164},"labels":[],"label_agreement":null},{"id":"W4403255280","doi":"10.1007/s10664-024-10441-z","title":"Evaluating few-shot and contrastive learning methods for code clone detection","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Okanagan University College; University of British Columbia, Okanagan Campus; University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; clone (Java method); Artificial intelligence; Code (set theory); Shot (pellet); Programming language; Biology; DNA; Genetics","score_opus":0.09238815332234587,"score_gpt":0.4441704756515731,"score_spread":0.35178232232922724,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403255280","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.041494917,0.0023924694,0.9522672,0.00016197297,0.0008139101,0.00033385263,0.0000030372398,0.0025297778,0.0000028617753],"genre_scores_gemma":[0.39555782,0.000014585778,0.6038764,0.000024416231,0.00021005057,0.0001732244,0.0000031752552,0.000057794132,0.000082545805],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99808913,0.000098224875,0.0002802058,0.0006706298,0.0003013687,0.00056043157],"domain_scores_gemma":[0.9894595,0.009957649,0.000026106987,0.00023404657,0.000117184376,0.0002055117],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0017209566,0.00025125867,0.0002712153,0.00027088186,0.00016206614,0.0003979222,0.0003139579,0.00013746916,0.000006441904],"category_scores_gemma":[0.012485165,0.00025126917,0.00010143864,0.00072048034,0.000026641257,0.00039488636,0.00023553232,0.0006276869,0.000010913518],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000021863578,0.000025916945,0.0039901976,0.00067701837,0.00017851799,0.000027770024,0.0014342848,0.1416033,0.026793776,0.00039512207,0.00013959766,0.82471263],"study_design_scores_gemma":[0.00026050277,0.00026087367,0.007532994,0.00013377442,0.000018229497,0.000053195676,0.000007917377,0.97237974,0.006766586,0.00018711212,0.012069369,0.00032970207],"about_ca_topic_score_codex":0.00000573772,"about_ca_topic_score_gemma":5.7771075e-7,"teacher_disagreement_score":0.83077645,"about_ca_system_score_codex":0.00019475662,"about_ca_system_score_gemma":0.000067075394,"threshold_uncertainty_score":0.999994},"labels":[],"label_agreement":null},{"id":"W4403359935","doi":"10.1007/s10664-024-10547-4","title":"Extracting microservices from monolithic systems using deep reinforcement learning","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université Laval","funders":"","keywords":"Microservices; Reinforcement learning; Computer science; Artificial intelligence; Operating system","score_opus":0.021718347657193302,"score_gpt":0.271314858023251,"score_spread":0.2495965103660577,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403359935","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.30007577,0.004169207,0.6919038,0.000041552787,0.00194564,0.00013679902,6.783002e-7,0.0017107775,0.000015716463],"genre_scores_gemma":[0.97126037,0.000025773132,0.028086532,0.000043854372,0.00047623107,0.0000255454,0.0000075089865,0.000038727296,0.00003548405],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978648,0.000053534237,0.00052687584,0.0006232683,0.0004078203,0.0005237164],"domain_scores_gemma":[0.99852943,0.00070177304,0.00007538455,0.00045806455,0.00006929142,0.00016605158],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00048127578,0.00029138054,0.00030876914,0.00016225391,0.00020271682,0.0007102436,0.00056650577,0.00018661159,0.00001498264],"category_scores_gemma":[0.00024361094,0.0002566736,0.00013730163,0.0006339871,0.000014770492,0.0009648227,0.00029093318,0.00060464913,0.00011784326],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000021922367,0.000011502105,0.036194377,0.00045155207,0.00006777834,0.00005342453,0.0017463984,0.9572186,0.0007926454,0.00008044112,0.000027518854,0.0033535645],"study_design_scores_gemma":[0.00009090299,0.00002663263,0.0038410942,0.0005497777,0.000015497417,0.000036970156,0.00005805597,0.9872126,0.00039995194,0.00003938784,0.0073899147,0.0003392411],"about_ca_topic_score_codex":0.0004204083,"about_ca_topic_score_gemma":0.000001788497,"teacher_disagreement_score":0.67118454,"about_ca_system_score_codex":0.0003026008,"about_ca_system_score_gemma":0.0000800186,"threshold_uncertainty_score":0.99998856},"labels":[],"label_agreement":null},{"id":"W4403539235","doi":"10.1007/s10664-024-10570-5","title":"Bridging the language gap: an empirical study of bindings for open source machine learning libraries across software package ecosystems","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Bridging (networking); Computer science; Open source software; Open source; Empirical research; Software engineering; Software; Programming language","score_opus":0.03953274196089684,"score_gpt":0.34172572270005686,"score_spread":0.30219298073916,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403539235","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.46065587,0.00091460336,0.535108,0.00017951969,0.0004246845,0.0007811706,0.000028613893,0.0019063734,0.0000012337964],"genre_scores_gemma":[0.9743163,0.0000046519026,0.02430666,0.000070513895,0.00039888363,0.00032814854,0.000038673606,0.00018721263,0.0003489706],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9961512,0.00018628428,0.00068773556,0.0010777949,0.00082207646,0.0010748823],"domain_scores_gemma":[0.9930904,0.0052441834,0.000102590304,0.0011165714,0.00011684838,0.00032940359],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0018504391,0.00050466775,0.0006430388,0.0002952784,0.00046428794,0.002183208,0.0033964824,0.00018388913,0.000019528625],"category_scores_gemma":[0.0047499686,0.00039639845,0.000209805,0.001696249,0.00006298391,0.0015286499,0.0022361346,0.001070485,0.000023121946],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000058820304,0.00051859504,0.7996554,0.0012664233,0.0004488527,0.0002745165,0.070960395,0.09002834,0.00041945765,0.00009640654,0.003022096,0.03325073],"study_design_scores_gemma":[0.0017376367,0.001719085,0.056941945,0.0006631568,0.00006690773,0.00016737317,0.0020946872,0.90567493,0.0019980895,0.00007941144,0.027333044,0.0015237355],"about_ca_topic_score_codex":0.00023418476,"about_ca_topic_score_gemma":0.000037310554,"teacher_disagreement_score":0.8156466,"about_ca_system_score_codex":0.0001769465,"about_ca_system_score_gemma":0.00014694854,"threshold_uncertainty_score":0.9998488},"labels":[],"label_agreement":null},{"id":"W4403547350","doi":"10.1007/s10664-024-10566-1","title":"Correction to: Examining ownership models in software teams","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"York University","funders":"","keywords":"Computer science; Software; Data science; Software engineering; Programming language","score_opus":0.037156770907130494,"score_gpt":0.28165430634688066,"score_spread":0.24449753543975017,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403547350","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.041556355,0.00019608023,0.95031375,0.00053051667,0.0035749215,0.00023886393,0.0000033192077,0.0034580657,0.0001281374],"genre_scores_gemma":[0.74007744,0.0000021472326,0.25840518,0.0004682514,0.00022851523,0.00010924529,0.00000840496,0.00006715394,0.00063365477],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99768704,0.000039041308,0.00045930748,0.0007676525,0.00043608583,0.0006108601],"domain_scores_gemma":[0.99786264,0.0012713048,0.000030007335,0.0005438531,0.00007969839,0.0002124994],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005171616,0.0002983924,0.0002685554,0.00070650026,0.00008677205,0.00049377483,0.0007729808,0.00013522439,0.000022012442],"category_scores_gemma":[0.0019651034,0.00030067295,0.00007900761,0.0035827777,0.000015710739,0.00118727,0.00038523012,0.0006900036,0.0002141316],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000052621826,0.00007955159,0.04330573,0.00013119113,0.000041881158,0.00022434597,0.0055975276,0.81890035,0.0001409511,0.0020756142,0.019484235,0.110013366],"study_design_scores_gemma":[0.00026517376,0.00016434483,0.04208797,0.0011011953,0.000011243146,0.00010620682,0.00007854825,0.9252672,0.00035388273,0.0017923227,0.027636256,0.0011356663],"about_ca_topic_score_codex":0.000054436445,"about_ca_topic_score_gemma":0.000014619661,"teacher_disagreement_score":0.6985211,"about_ca_system_score_codex":0.00048153143,"about_ca_system_score_gemma":0.00015316522,"threshold_uncertainty_score":0.99994457},"labels":[],"label_agreement":null},{"id":"W4403651448","doi":"10.1007/s10664-024-10568-z","title":"The downside of functional constructs: a quantitative and qualitative analysis of their fix-inducing effects","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Ministero dell'Università e della Ricerca","keywords":"Qualitative analysis; Quantitative analysis (chemistry); Downside risk; Computer science; Qualitative research; Chemistry; Economics; Sociology; Chromatography; Financial economics","score_opus":0.03409354701278354,"score_gpt":0.3381684211001117,"score_spread":0.30407487408732814,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403651448","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38960436,0.0024198839,0.60732776,0.00010385329,0.00023311477,0.000088156354,0.00001122458,0.00020837411,0.0000032936173],"genre_scores_gemma":[0.96915305,0.000025698155,0.030731047,0.000010524668,0.000022568405,0.000023021325,0.0000034880147,0.000016570937,0.000014029491],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99847794,0.000107929984,0.00035357004,0.00036133992,0.00041002358,0.00028919734],"domain_scores_gemma":[0.9713819,0.027989853,0.000052262592,0.00033163815,0.00014806436,0.00009627835],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008575548,0.00018287374,0.0003799021,0.0004909195,0.000071656774,0.00009781189,0.00032405424,0.00006970931,0.000004500922],"category_scores_gemma":[0.005228233,0.00012870494,0.00016424121,0.00254067,0.00013105893,0.00023008979,0.00022071204,0.00029646093,0.0000022698332],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0001385187,0.00031033668,0.1354889,0.005325518,0.023710156,0.000222342,0.16433923,0.42855245,0.022663185,0.14010054,0.002194063,0.07695477],"study_design_scores_gemma":[0.00038306243,0.00041600983,0.3144654,0.00060289836,0.0003013857,0.000026822236,0.0008237519,0.6694965,0.01032633,0.0017507772,0.0008298232,0.00057722646],"about_ca_topic_score_codex":0.000022640603,"about_ca_topic_score_gemma":0.0000028394766,"teacher_disagreement_score":0.5795487,"about_ca_system_score_codex":0.000065515786,"about_ca_system_score_gemma":0.00009065699,"threshold_uncertainty_score":0.6259061},"labels":[],"label_agreement":null},{"id":"W4403887388","doi":"10.1007/s10664-024-10549-2","title":"Towards effectively testing machine translation systems from white-box perspectives","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; University of Waterloo","funders":"","keywords":"Computer science; Translation (biology); Machine translation; White box; Systems engineering; Artificial intelligence; Engineering; Software engineering","score_opus":0.02217214222265518,"score_gpt":0.27888827832438084,"score_spread":0.25671613610172567,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403887388","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0047033667,0.048169672,0.939902,0.00034202044,0.000503942,0.00017486073,0.00001367687,0.0061416733,0.000048786245],"genre_scores_gemma":[0.5284211,0.000003910083,0.47131407,0.000017271714,0.00017529954,0.000024280418,0.0000048722745,0.000024766188,0.000014431037],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998576,0.000044010638,0.00021779066,0.00055584474,0.00031716013,0.00028918835],"domain_scores_gemma":[0.9987878,0.0007505735,0.000029634648,0.00025596708,0.00007753926,0.00009849368],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023899328,0.00024122825,0.00022252744,0.00018630718,0.00007331328,0.0005232217,0.00048000622,0.00012411189,0.0000059112126],"category_scores_gemma":[0.00069484214,0.00020766347,0.00008158614,0.00089992705,0.000016061233,0.00064532587,0.000118810996,0.00044708932,0.000016018634],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025761577,0.00018458991,0.023748662,0.0018412711,0.0004003766,0.00093346264,0.02710804,0.02066796,0.01732184,0.008618684,0.0015169267,0.8976324],"study_design_scores_gemma":[0.00014889862,0.00009628874,0.005722585,0.00076418597,0.00002502429,0.000053274547,0.000028615863,0.9868129,0.0021545794,0.0018517044,0.0017596252,0.0005823542],"about_ca_topic_score_codex":0.00013923967,"about_ca_topic_score_gemma":0.0000012163791,"teacher_disagreement_score":0.9661449,"about_ca_system_score_codex":0.00023358208,"about_ca_system_score_gemma":0.0000637554,"threshold_uncertainty_score":0.84682703},"labels":[],"label_agreement":null},{"id":"W4403946944","doi":"10.1007/s10664-024-10560-7","title":"A qualitative study on refactorings induced by code review","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Computer science; Code (set theory); Programming language; Qualitative research; Software engineering; Sociology","score_opus":0.0852812258416174,"score_gpt":0.41490363149144455,"score_spread":0.32962240564982714,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403946944","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.30011636,0.011416169,0.67704153,0.0019626727,0.0015014333,0.0010066458,0.000023974562,0.0068980046,0.000033181255],"genre_scores_gemma":[0.9931138,0.00012674014,0.0056123314,0.0003917619,0.00015347298,0.00025641636,0.000008134398,0.000099591256,0.00023775961],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9970685,0.00012795789,0.00040763288,0.00085823477,0.0009273917,0.00061028334],"domain_scores_gemma":[0.9951478,0.003683724,0.000027880531,0.0007688346,0.00007740426,0.0002943586],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0010677681,0.00036812384,0.0004043005,0.00025778433,0.000066883826,0.00026203267,0.0009952897,0.00010155333,0.000026100843],"category_scores_gemma":[0.004186178,0.00032988336,0.00013204507,0.0016223767,0.000013370811,0.00041515008,0.00031995232,0.00088803424,0.0003133417],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006769383,0.0038894927,0.028513566,0.018314565,0.0026920545,0.0033955658,0.25915933,0.0046703923,0.008142936,0.0043765465,0.58874446,0.07803338],"study_design_scores_gemma":[0.0035659794,0.00898038,0.094175495,0.031186039,0.0002744251,0.00020085849,0.0019105335,0.03548494,0.008475264,0.0007760573,0.8048749,0.010095116],"about_ca_topic_score_codex":0.000012805603,"about_ca_topic_score_gemma":3.3809826e-7,"teacher_disagreement_score":0.69299746,"about_ca_system_score_codex":0.00034074567,"about_ca_system_score_gemma":0.00008422404,"threshold_uncertainty_score":0.9999153},"labels":[],"label_agreement":null},{"id":"W4404199042","doi":"10.1007/s10664-024-10579-w","title":"Towards enhancing the reproducibility of deep learning bugs: an empirical study","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Dalhousie University","funders":"","keywords":"Reproducibility; Empirical research; Computer science; Data science; Artificial intelligence; Statistics; Mathematics","score_opus":0.02200891701765622,"score_gpt":0.32076736995582494,"score_spread":0.2987584529381687,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4404199042","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.33699337,0.000362176,0.66019773,0.0003568415,0.00068318704,0.00020926568,3.7773387e-7,0.0011678517,0.000029214312],"genre_scores_gemma":[0.9516475,0.000002438131,0.04778844,0.000063025815,0.00038367667,0.000029157856,0.0000020805428,0.00004624359,0.000037405964],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99624777,0.00039266158,0.0006420213,0.0015466661,0.00068012805,0.00049076247],"domain_scores_gemma":[0.9965321,0.0012479076,0.00008202429,0.0018726982,0.00010536895,0.000159937],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004156473,0.00030947232,0.000401739,0.00016722118,0.00021875568,0.00025650897,0.0012779164,0.000116500116,0.000034688044],"category_scores_gemma":[0.008217238,0.00023047089,0.00016620525,0.0013485348,0.00005541886,0.0006687381,0.00085032976,0.0012699306,0.000019996753],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012748067,0.00024947076,0.18526019,0.00017923916,0.00012616797,0.000120849254,0.022731422,0.7102042,0.00029724973,0.00027399216,0.000059848004,0.08048464],"study_design_scores_gemma":[0.00020873152,0.0004474057,0.16933587,0.000086796026,0.000044388154,0.00003529135,0.0004595938,0.82494247,0.00064768794,0.00031277293,0.003012308,0.00046669657],"about_ca_topic_score_codex":0.00005873099,"about_ca_topic_score_gemma":0.000010811915,"teacher_disagreement_score":0.6146542,"about_ca_system_score_codex":0.00016190822,"about_ca_system_score_gemma":0.00011255031,"threshold_uncertainty_score":0.9837395},"labels":[],"label_agreement":null},{"id":"W4404406203","doi":"10.1007/s10664-024-10564-3","title":"Can search-based testing with pareto optimization effectively cover failure-revealing test inputs?","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; HORIZON EUROPE Reforming and enhancing the European Research and Innovation system; Technische Universität München; European Commission","keywords":"Cover (algebra); Pareto principle; Reliability engineering; Computer science; Engineering; Mathematical optimization; Mathematics; Mechanical engineering","score_opus":0.02105501579744261,"score_gpt":0.25535485620899456,"score_spread":0.23429984041155194,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4404406203","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0073423116,0.00012425256,0.96389335,0.00089756557,0.00015640548,0.00024344794,0.000008736138,0.0273138,0.000020147896],"genre_scores_gemma":[0.50223076,4.2856695e-7,0.4973808,0.00019124176,0.00008827004,0.000043048265,0.0000075211474,0.00004749534,0.000010475882],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979359,0.000051619278,0.0002743934,0.0006991413,0.00047843385,0.0005604955],"domain_scores_gemma":[0.99458957,0.0045320923,0.000042539166,0.00044581422,0.0001737889,0.00021620825],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00046048392,0.00034731237,0.00027279026,0.00030457298,0.00016379617,0.0005566832,0.00051436323,0.00012799974,0.000005486373],"category_scores_gemma":[0.003716754,0.00030074728,0.00007247796,0.0017888668,0.000043765172,0.00036013316,0.00017302678,0.00050930964,0.000017335931],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000028386903,0.00003612791,0.075687945,0.00023744257,0.00002318834,0.00016845696,0.00021039518,0.9183418,0.00008690227,0.0000359758,0.001760019,0.0034088911],"study_design_scores_gemma":[0.00016640047,0.00026221055,0.0048872125,0.0009815374,0.000015676287,0.00006682674,7.7028665e-7,0.99125594,0.0013018444,0.00013114183,0.00046228114,0.00046814798],"about_ca_topic_score_codex":0.000066041015,"about_ca_topic_score_gemma":0.0000022503655,"teacher_disagreement_score":0.49488842,"about_ca_system_score_codex":0.00027150623,"about_ca_system_score_gemma":0.000249433,"threshold_uncertainty_score":0.99994445},"labels":[],"label_agreement":null},{"id":"W4404860113","doi":"10.1007/s10664-024-10589-8","title":"Contrasting test selection, prioritization, and batch testing at scale","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada; Concordia University","keywords":"Prioritization; Selection (genetic algorithm); Scale (ratio); Test (biology); Computer science; Regression testing; Reliability engineering; Engineering; Machine learning; Biology; Geography; Management science; Cartography; Operating system; Software","score_opus":0.016660816534096935,"score_gpt":0.2537672231562379,"score_spread":0.23710640662214097,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4404860113","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.043348573,0.0006820197,0.9247217,0.00028658434,0.0002671727,0.00010915838,0.0000023682023,0.030534482,0.000047935635],"genre_scores_gemma":[0.5154388,0.000004572682,0.48409325,0.000115907744,0.00019028597,0.000023278826,0.0000022334684,0.00003442233,0.00009726094],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9986075,0.000016994296,0.00026447728,0.0005208901,0.0002209009,0.00036920185],"domain_scores_gemma":[0.9961893,0.0033158185,0.0000303802,0.00019340136,0.000118573036,0.00015251075],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00030727903,0.00020419822,0.00017694583,0.00015260198,0.00022343115,0.00037971485,0.00023659348,0.00009642054,0.0000043863693],"category_scores_gemma":[0.0056741484,0.00020713145,0.00003743677,0.0010801813,0.000026157899,0.0003449387,0.0002575862,0.00025193885,0.000016592112],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[7.019847e-7,0.000025287765,0.95486224,0.0002186542,0.000016156102,0.00004476535,0.00042505437,0.0011452684,0.0009108341,0.0001620083,0.009716628,0.03247243],"study_design_scores_gemma":[0.00011732141,0.0001045481,0.12222808,0.00054038747,0.000015067568,0.00051819626,0.0000011876617,0.8676903,0.0010444006,0.0020727196,0.0051730135,0.0004947563],"about_ca_topic_score_codex":0.00001469796,"about_ca_topic_score_gemma":0.0000015158278,"teacher_disagreement_score":0.8665451,"about_ca_system_score_codex":0.00014542507,"about_ca_system_score_gemma":0.000058954018,"threshold_uncertainty_score":0.84465754},"labels":[],"label_agreement":null},{"id":"W4405259287","doi":"10.1007/s10664-024-10597-8","title":"Harnessing pre-trained generalist agents for software engineering tasks","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Artificial intelligence; Reinforcement learning; Software engineering; Generalizability theory; Scheduling (production processes); Machine learning; Software; Domain (mathematical analysis); Human–computer interaction; Engineering; Operations management","score_opus":0.03556022016790315,"score_gpt":0.3148634202315637,"score_spread":0.27930320006366055,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4405259287","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.045434702,0.0019703456,0.9407122,0.00040139447,0.0028371832,0.0005890769,0.000043410488,0.008008216,0.0000034792163],"genre_scores_gemma":[0.32485867,0.000023145314,0.6715576,0.00023729251,0.0013881844,0.0006050072,0.00008436262,0.00035011908,0.00089560304],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99577016,0.00003170504,0.0006363588,0.0012727298,0.00083346793,0.0014555646],"domain_scores_gemma":[0.99494165,0.003332436,0.00004773644,0.0009669708,0.00016005925,0.00055115897],"candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00068019296,0.0006729741,0.0005399033,0.00065938395,0.00019986788,0.0011132003,0.0015161345,0.0003171391,0.000039508737],"category_scores_gemma":[0.0063889353,0.00069222174,0.00039767622,0.0016129764,0.000045094803,0.0010355453,0.0005418199,0.00077864894,0.00007466984],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000033638495,0.0002148305,0.0389718,0.004677472,0.0006932817,0.0008248025,0.0025344167,0.8669565,0.003382016,0.0039875777,0.03841651,0.039307196],"study_design_scores_gemma":[0.00057724415,0.0001354187,0.036995046,0.00068434107,0.000039577382,0.00012118598,0.0000035473113,0.79940635,0.0018541267,0.00013207056,0.15869781,0.0013532854],"about_ca_topic_score_codex":0.000013809323,"about_ca_topic_score_gemma":8.4050214e-7,"teacher_disagreement_score":0.27942398,"about_ca_system_score_codex":0.0005131269,"about_ca_system_score_gemma":0.00021171889,"threshold_uncertainty_score":0.99992377},"labels":[],"label_agreement":null},{"id":"W4406172285","doi":"10.1007/s10664-024-10555-4","title":"Lightweight dynamic build batching algorithms for continuous integration","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Process (computing); Batch processing; Quality (philosophy); Software; Quality assurance; Distributed computing; Industrial engineering; Real-time computing; Software engineering; Engineering; Operating system; Operations management","score_opus":0.014474937432760255,"score_gpt":0.3036872083430948,"score_spread":0.2892122709103346,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4406172285","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.019822989,0.00052246795,0.9748395,0.0009682216,0.0013821636,0.00042741798,0.000006726134,0.002007445,0.000023078628],"genre_scores_gemma":[0.41251105,0.000012982906,0.58582145,0.000256961,0.00015689497,0.0002806955,0.000021674756,0.000055649743,0.0008826329],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979912,0.000024903931,0.00037560926,0.0006267416,0.00033901562,0.00064249674],"domain_scores_gemma":[0.9963999,0.0026258521,0.000040985236,0.0006072464,0.00016838401,0.00015766882],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00043506868,0.00029991166,0.00033916827,0.00042328163,0.00014710466,0.0002684975,0.0010021445,0.00017652265,0.000006184513],"category_scores_gemma":[0.003922426,0.00029149023,0.00016783971,0.0010147034,0.000019884132,0.00041254604,0.00030501076,0.0004385975,0.000018823988],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000813027,0.0006422429,0.116215356,0.0014353874,0.00076262007,0.00017855862,0.002880317,0.10872462,0.008348334,0.023429941,0.034629826,0.70267147],"study_design_scores_gemma":[0.0009709833,0.0001560878,0.051872224,0.0003794776,0.00002327315,0.0000199287,0.000012617229,0.87998265,0.005315335,0.0017170525,0.058735706,0.00081467966],"about_ca_topic_score_codex":0.000010543106,"about_ca_topic_score_gemma":0.0000021736994,"teacher_disagreement_score":0.771258,"about_ca_system_score_codex":0.0003163811,"about_ca_system_score_gemma":0.00010611702,"threshold_uncertainty_score":0.99995375},"labels":[],"label_agreement":null},{"id":"W4406405723","doi":"10.1007/s10664-024-10611-z","title":"Negativity in self-admitted technical debt: how sentiment influences prioritization","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Auditing, Earnings Management, Governance","field":"Business, Management and Accounting","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Negativity effect; Prioritization; Debt; Computer science; Psychology; Cognitive psychology; Business; Process management; Finance","score_opus":0.007187406690226814,"score_gpt":0.2287443263012959,"score_spread":0.22155691961106908,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4406405723","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.45935392,0.000086770015,0.5363791,0.0017442998,0.00030838727,0.00035157788,9.138467e-7,0.0011270167,0.0006480272],"genre_scores_gemma":[0.9930682,0.000008260337,0.005216395,0.0010448231,0.00026933625,0.00006252318,0.000012095835,0.000030577175,0.00028776453],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9984809,0.000011846392,0.00030394964,0.0004330979,0.00034956308,0.00042060355],"domain_scores_gemma":[0.9984523,0.0001657213,0.0010611919,0.00022512353,0.00007691945,0.000018717034],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003103162,0.0002525162,0.00026764444,0.0003604011,0.000104666935,0.00028907298,0.0002977193,0.000118351076,0.00003665891],"category_scores_gemma":[0.0092589,0.00026883002,0.000072617644,0.0016330562,0.0000243758,0.0008385884,0.0003833403,0.0003521191,0.000055276356],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007700048,0.00010518055,0.97650343,0.00035293604,0.000030867093,0.000025298674,0.000032290023,0.008669114,0.00023841235,0.0014855902,0.003012091,0.0095370915],"study_design_scores_gemma":[0.00043577416,0.0000057767793,0.8601969,0.00025258595,0.000032469077,6.5168115e-7,0.000013328202,0.010853433,0.00009401835,0.0003439347,0.12744865,0.00032251555],"about_ca_topic_score_codex":0.00007246222,"about_ca_topic_score_gemma":0.000035095454,"teacher_disagreement_score":0.5337143,"about_ca_system_score_codex":0.00022849123,"about_ca_system_score_gemma":0.00002769813,"threshold_uncertainty_score":0.9999764},"labels":[],"label_agreement":null},{"id":"W4407277384","doi":"10.1007/s10664-024-10609-7","title":"UPC sentinel: An accurate approach for detecting upgradeability proxy contracts in Ethereum","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Proxy (statistics); Computer science; Engineering; Machine learning","score_opus":0.03488539790364532,"score_gpt":0.326717984410252,"score_spread":0.2918325865066067,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4407277384","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25678518,0.00012231898,0.7407638,0.00020579394,0.00025997125,0.00061174034,0.00000232986,0.0012314784,0.000017406212],"genre_scores_gemma":[0.8020984,0.0000016216986,0.19726804,0.00010989728,0.00008897339,0.00033467417,0.0000071378977,0.000038992883,0.00005227392],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9972674,0.00008079836,0.00050429377,0.00088498223,0.00034278285,0.00091973017],"domain_scores_gemma":[0.99584365,0.0029065288,0.000053964985,0.0008367139,0.00014083009,0.00021831064],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012923139,0.00033204298,0.00040973927,0.00040929022,0.00010912304,0.00025363162,0.001115085,0.0002303836,0.0000026154976],"category_scores_gemma":[0.009714092,0.00033899234,0.00012929009,0.0014982541,0.000028985056,0.0006106863,0.0003290249,0.00068722107,0.0000034390782],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000050668878,0.00052545074,0.68499047,0.0009196601,0.00009130552,0.000032040985,0.0010068003,0.29542226,0.0012857335,0.00077450316,0.00033337096,0.014567753],"study_design_scores_gemma":[0.00080114446,0.000057993566,0.2391988,0.00008377989,0.000006292325,0.0000074878817,0.00001568502,0.7557353,0.0018057311,0.00033197316,0.001516839,0.00043899682],"about_ca_topic_score_codex":0.00002599632,"about_ca_topic_score_gemma":0.000002746795,"teacher_disagreement_score":0.5453132,"about_ca_system_score_codex":0.00029500652,"about_ca_system_score_gemma":0.00015359996,"threshold_uncertainty_score":0.9999062},"labels":[],"label_agreement":null},{"id":"W4407551092","doi":"10.1007/s10664-025-10614-4","title":"Bugs in large language models generated code: an empirical study","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":44,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec; Natural Sciences and Engineering Research Council of Canada; Consortium de Recherche et d’innovation en Aérospatiale au Québec; Canadian Institute for Advanced Research","keywords":"Computer science; Programming language; Empirical research; Code (set theory); Statistics; Mathematics","score_opus":0.019862348010958434,"score_gpt":0.32677663977065224,"score_spread":0.3069142917596938,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4407551092","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.31674367,0.000104716004,0.6805146,0.000102711936,0.00017139796,0.00026298064,0.000004037655,0.0020712067,0.000024683091],"genre_scores_gemma":[0.8850526,0.0000029386683,0.11414263,0.0004938691,0.000043534987,0.00011809715,0.0000053184353,0.000026028047,0.00011498042],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99809754,0.000090390015,0.00039026063,0.0006570683,0.00027099752,0.00049374474],"domain_scores_gemma":[0.99887294,0.00017452029,0.00003904894,0.0007041023,0.00007333372,0.00013608509],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00037530225,0.00025908416,0.0003153931,0.0004232114,0.000082371866,0.00010571527,0.0007464796,0.00014358768,0.00000901813],"category_scores_gemma":[0.00032468775,0.0002644567,0.000059684335,0.0016321202,0.000011947224,0.0007223745,0.000408114,0.000470113,0.0000077998375],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000075671625,0.0040735756,0.38574767,0.00016709165,0.00014689661,0.0011448221,0.015458582,0.5474647,0.003199092,0.0023363275,0.0035875705,0.036597997],"study_design_scores_gemma":[0.0011183675,0.00026721822,0.06742168,0.0000584455,0.000009697252,0.0000123231175,0.00017504841,0.9209314,0.0050443946,0.0013687665,0.0029184115,0.00067424186],"about_ca_topic_score_codex":0.000026855741,"about_ca_topic_score_gemma":0.000058749367,"teacher_disagreement_score":0.56830895,"about_ca_system_score_codex":0.0002216932,"about_ca_system_score_gemma":0.00006635619,"threshold_uncertainty_score":0.99998075},"labels":[],"label_agreement":null},{"id":"W4407986251","doi":"10.1007/s10664-025-10619-z","title":"An empirical study on LLM-based classification of requirements-related provisions in food-safety regulations","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Technology and Data Analysis","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Ontario Centre of Innovation","keywords":"Food safety; Empirical research; Business; Computer science; Food science; Chemistry; Statistics; Mathematics","score_opus":0.032420337496137465,"score_gpt":0.33388914271590625,"score_spread":0.30146880521976877,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4407986251","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38448808,0.000015168134,0.61385065,0.0009525428,0.00007129531,0.00021041305,0.0000062346357,0.0003817513,0.000023890912],"genre_scores_gemma":[0.984479,4.9083184e-7,0.015332881,0.000085452324,0.000005564064,0.00004080568,0.000029746914,0.000007848167,0.000018198634],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9984426,0.00009217099,0.00052271987,0.00047855574,0.00024516098,0.00021877718],"domain_scores_gemma":[0.9984913,0.00034425533,0.000077182325,0.0009614989,0.000064209045,0.00006158406],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00037616352,0.00015217546,0.00025189217,0.0006684231,0.00009189205,0.000028062774,0.0006792341,0.0001672407,0.0000071528316],"category_scores_gemma":[0.00064447854,0.00014571824,0.00006961461,0.0023924778,0.00003318385,0.000262034,0.00011795946,0.00029503155,0.000005775739],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000022611743,0.0022866244,0.90734214,0.000022700347,0.00009219996,0.0000074704462,0.0004429429,0.076483935,0.00044447638,0.005535738,0.00030060965,0.007018585],"study_design_scores_gemma":[0.0004624904,0.0003516178,0.79755354,0.00004303358,0.000015419284,2.331796e-7,0.0000316771,0.20036972,0.00023165811,0.00040150102,0.00041690317,0.00012218213],"about_ca_topic_score_codex":0.000008518087,"about_ca_topic_score_gemma":0.000018707082,"teacher_disagreement_score":0.59999096,"about_ca_system_score_codex":0.00013792946,"about_ca_system_score_gemma":0.00009503092,"threshold_uncertainty_score":0.5942218},"labels":[],"label_agreement":null},{"id":"W4407986476","doi":"10.1007/s10664-025-10618-0","title":"Evaluating interactive documentation for programmers","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Fonds de recherche du Québec – Nature et technologies; Natural Sciences and Engineering Research Council of Canada","keywords":"Documentation; Computer science; World Wide Web; Software engineering; Programming language","score_opus":0.04750618481790624,"score_gpt":0.41323817976843863,"score_spread":0.3657319949505324,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4407986476","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.046227597,0.0001325639,0.95065737,0.0004972745,0.00076336187,0.00054623367,0.0000017438205,0.00115608,0.00001775961],"genre_scores_gemma":[0.40563363,0.0000023429625,0.59318155,0.00019215359,0.000088810324,0.00057939114,0.000010744059,0.000027306703,0.000284084],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99849176,0.00002611233,0.00025838715,0.0004496453,0.0003059309,0.00046816448],"domain_scores_gemma":[0.99617934,0.0031611703,0.000035885852,0.00035659262,0.0001650233,0.0001020006],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00048439612,0.00018267224,0.00018097121,0.0002806352,0.00009707376,0.0002444249,0.0005816056,0.00007684704,0.000008679946],"category_scores_gemma":[0.0064599286,0.00018894902,0.000088026645,0.0008658777,0.000014078603,0.0005133756,0.0002286604,0.00023997702,0.000015537402],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000101026846,0.0003225047,0.15609053,0.0010559555,0.0005492997,0.00002635933,0.0026377533,0.22065902,0.0041197594,0.00587531,0.011416832,0.5971457],"study_design_scores_gemma":[0.002311099,0.00065287197,0.12177661,0.00044966652,0.000044465436,0.0000129177,0.000051832,0.8232046,0.010981331,0.0021498487,0.037341356,0.0010234239],"about_ca_topic_score_codex":0.000007656922,"about_ca_topic_score_gemma":4.583866e-7,"teacher_disagreement_score":0.60254556,"about_ca_system_score_codex":0.00029384572,"about_ca_system_score_gemma":0.00010294838,"threshold_uncertainty_score":0.7733605},"labels":[],"label_agreement":null},{"id":"W4408054408","doi":"10.1007/s10664-024-10610-0","title":"Assessing the adoption of security policies by developers in terraform across different cloud providers","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Information and Cyber Security","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Fonds de recherche du Québec; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Cloud computing; Business; Computer security; Cloud service provider; Cloud computing security; Internet privacy; Computer science","score_opus":0.014692344055413297,"score_gpt":0.29999987601119277,"score_spread":0.28530753195577946,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408054408","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.59666187,0.000066616645,0.4019008,0.0008216047,0.0002514121,0.00010102422,0.000001964013,0.00012236026,0.00007235837],"genre_scores_gemma":[0.99766743,0.000005348843,0.0017975996,0.00047661146,0.000015152851,0.00001613174,0.000004840378,0.0000041532057,0.000012751516],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99904287,0.000024580566,0.00031807824,0.00014155806,0.00019500812,0.00027793218],"domain_scores_gemma":[0.99948424,0.00015875408,0.000056401943,0.00021929666,0.000042468306,0.000038847138],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00025349704,0.0001279578,0.00015385762,0.00007230935,0.00009534397,0.00019786673,0.00045357726,0.00007220096,0.0000017568296],"category_scores_gemma":[0.0002053259,0.00009306913,0.000052275638,0.00053364184,0.00003252769,0.0006555859,0.00023303066,0.00023791578,0.0000019048082],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027339183,0.0006386563,0.6191683,0.001415593,0.00017991771,0.000010135451,0.22436145,0.02110812,0.00076306594,0.026976952,0.015947519,0.08940297],"study_design_scores_gemma":[0.0009293887,0.00004450001,0.8167591,0.0003592827,0.0000086386635,0.0000069979255,0.0025281569,0.14586392,0.005553548,0.0010226119,0.026311958,0.000611902],"about_ca_topic_score_codex":0.000053172967,"about_ca_topic_score_gemma":0.000018732046,"teacher_disagreement_score":0.40100557,"about_ca_system_score_codex":0.00014098405,"about_ca_system_score_gemma":0.000053112377,"threshold_uncertainty_score":0.3795249},"labels":[],"label_agreement":null},{"id":"W4408069583","doi":"10.1007/s10664-025-10616-2","title":"WIA-SZZ: Work item aware SZZ","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Work (physics); Engineering; Mechanical engineering","score_opus":0.0177152079228471,"score_gpt":0.2842851256222398,"score_spread":0.2665699176993927,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408069583","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059616785,0.0007379374,0.9332075,0.0012436738,0.0012088205,0.00023621635,0.000002324702,0.0036700068,0.00007672357],"genre_scores_gemma":[0.8944761,0.000019197747,0.102202766,0.00080951845,0.00030369175,0.00015542675,0.000009644085,0.00008752992,0.0019361094],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99717146,0.000041978048,0.0004186433,0.00078916416,0.0006179488,0.0009607868],"domain_scores_gemma":[0.9952059,0.003167103,0.00003895078,0.0011327058,0.00013901161,0.00031632345],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003915579,0.00039573765,0.00039098866,0.0005047129,0.00014949305,0.0002994074,0.0017268572,0.00023174469,0.00004171531],"category_scores_gemma":[0.004297173,0.00040538405,0.00018544168,0.0030501673,0.0000370465,0.0004186676,0.0007969146,0.00075218,0.00019678967],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017540258,0.00016858615,0.85250366,0.000379372,0.00020454297,0.000251341,0.000509457,0.05991035,0.00011243578,0.0016976361,0.059068978,0.025176099],"study_design_scores_gemma":[0.0008617357,0.00009467158,0.75283414,0.00059779186,0.00002190053,0.000029106854,0.00001129789,0.04844889,0.0015732126,0.00040445922,0.19383363,0.0012891442],"about_ca_topic_score_codex":0.000010146745,"about_ca_topic_score_gemma":6.988068e-7,"teacher_disagreement_score":0.8348593,"about_ca_system_score_codex":0.0003337487,"about_ca_system_score_gemma":0.0001616844,"threshold_uncertainty_score":0.9998398},"labels":[],"label_agreement":null},{"id":"W4408089088","doi":"10.1007/s10664-025-10634-0","title":"Fixer-level supervised contrastive learning for bug assignment","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"National Natural Science Foundation of China","keywords":"Computer science; Artificial intelligence; Natural language processing; Machine learning","score_opus":0.03540607240363587,"score_gpt":0.3021084142661999,"score_spread":0.26670234186256403,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408089088","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.024095705,0.00029468126,0.9720847,0.00064473803,0.0007475187,0.00046085575,0.000007047391,0.0016351098,0.000029620689],"genre_scores_gemma":[0.8336421,0.0000075155335,0.16446385,0.0002406847,0.00014646885,0.00035294108,0.000013532289,0.00004888239,0.0010840246],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977872,0.00003930305,0.00034202015,0.00062946207,0.00041384122,0.0007882148],"domain_scores_gemma":[0.9944142,0.0047151647,0.000033786215,0.00044925284,0.00017316632,0.0002143913],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00048051655,0.0003079392,0.00035290478,0.00030976054,0.00017722034,0.00018847131,0.000902181,0.00015887037,0.000018924298],"category_scores_gemma":[0.0072672167,0.00031578582,0.00016892883,0.0008602953,0.000026164515,0.00027666506,0.00038112173,0.00051486713,0.0000268734],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000068293266,0.00029885126,0.45616218,0.0007415258,0.00050660287,0.00007797925,0.0011911296,0.46129757,0.0028338174,0.0036262479,0.01878338,0.054412406],"study_design_scores_gemma":[0.0023905234,0.00031019753,0.40113702,0.00035558073,0.00003274269,0.000011613469,0.000024879004,0.5120049,0.0075535984,0.0005181789,0.074580066,0.0010806966],"about_ca_topic_score_codex":0.000007853665,"about_ca_topic_score_gemma":5.8314555e-7,"teacher_disagreement_score":0.8095464,"about_ca_system_score_codex":0.00033359413,"about_ca_system_score_gemma":0.00015578083,"threshold_uncertainty_score":0.9999294},"labels":[],"label_agreement":null},{"id":"W4408204750","doi":"10.1007/s10664-025-10631-3","title":"Towards semantic versioning of open pre-trained language model releases on hugging face","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Face recognition and analysis","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Queen's University","funders":"","keywords":"Software versioning; Computer science; Face (sociological concept); Natural language processing; Artificial intelligence; World Wide Web; Information retrieval; Linguistics; Programming language; Software","score_opus":0.01979562248878209,"score_gpt":0.3129141107030443,"score_spread":0.29311848821426223,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408204750","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.10399237,0.000099838406,0.8945722,0.0005347233,0.000074867436,0.000099096236,0.0000042593497,0.00029894905,0.0003236969],"genre_scores_gemma":[0.9309401,0.00000737257,0.0680889,0.00036293003,0.000011429272,0.000009981058,0.0000053587937,0.0000093988965,0.000564567],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9990667,0.000021231868,0.00020842465,0.0002919036,0.0001998351,0.00021190158],"domain_scores_gemma":[0.9993372,0.00020074043,0.000042380096,0.00030346026,0.000041342188,0.000074895084],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001478327,0.00013564953,0.00024092148,0.00020725511,0.0000686399,0.00011557067,0.000733683,0.000053443215,0.000017810657],"category_scores_gemma":[0.00054431043,0.00012732185,0.00010680604,0.0006895301,0.000011079787,0.0002536874,0.00041969065,0.00014642318,0.000009706787],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014526075,0.00012832152,0.0017919557,0.00018652104,0.000113451846,0.000021177353,0.0015775362,0.9558013,0.0019513072,0.0012956309,0.0015943502,0.035523918],"study_design_scores_gemma":[0.00027635886,0.000026609829,0.0020130007,0.00030908658,0.000018820478,0.0000010279917,0.000050952454,0.9897111,0.007020039,0.00014638364,0.00026097873,0.00016565018],"about_ca_topic_score_codex":0.00003659803,"about_ca_topic_score_gemma":0.0000015135756,"teacher_disagreement_score":0.8269477,"about_ca_system_score_codex":0.000057655045,"about_ca_system_score_gemma":0.000069221926,"threshold_uncertainty_score":0.5192035},"labels":[],"label_agreement":null},{"id":"W4408463271","doi":"10.1007/s10664-025-10632-2","title":"Empathy, self-determination and motivation: moderating diversity for enhanced performance in software development teams","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Gender Diversity and Inequality","field":"Social Sciences","cited_by":4,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Empathy; Diversity (politics); Psychology; Applied psychology; Knowledge management; Computer science; Social psychology; Sociology","score_opus":0.04744867423182276,"score_gpt":0.284422745266576,"score_spread":0.23697407103475326,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408463271","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.750537,0.000024670657,0.24874103,0.00010528131,0.00012581513,0.00020196321,0.000001760722,0.00017869136,0.00008380736],"genre_scores_gemma":[0.9585072,0.000010769034,0.04120733,0.00008860376,0.000034618595,0.000019582018,0.000006407425,0.0000041826374,0.00012135045],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9991485,0.000030997377,0.00016939199,0.00022064529,0.00018066062,0.00024980013],"domain_scores_gemma":[0.9994253,0.00034088906,0.0000337321,0.00006301349,0.000077940815,0.000059123595],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004696467,0.000100567326,0.00013381339,0.00010040177,0.0007603619,0.00004657407,0.00012236966,0.00010263304,0.000006536871],"category_scores_gemma":[0.0010022538,0.00011727375,0.0000271654,0.00025577634,0.000023177481,0.0003111171,0.00017750025,0.000114569746,0.0000012463424],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010999776,0.000053011438,0.9299462,0.00022954251,0.000014110837,7.2383244e-7,0.05386909,0.003474562,0.000010834379,0.000091262234,0.00004383987,0.012255864],"study_design_scores_gemma":[0.0006056631,0.000027671444,0.98494875,0.0001318501,0.000012005357,1.4962836e-7,0.0020954583,0.008062252,0.00084930426,0.00016368231,0.0028214601,0.0002817488],"about_ca_topic_score_codex":0.000050607603,"about_ca_topic_score_gemma":0.00007588472,"teacher_disagreement_score":0.20797017,"about_ca_system_score_codex":0.00027248877,"about_ca_system_score_gemma":0.00011474358,"threshold_uncertainty_score":0.5848166},"labels":[],"label_agreement":null},{"id":"W4409168587","doi":"10.1007/s10664-024-10607-9","title":"How far are app secrets from being stolen? a case study on android","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Fonds de recherche du Québec – Nature et technologies; Natural Sciences and Engineering Research Council of Canada; City University of Hong Kong","keywords":"Android (operating system); Computer science; Mobile apps; Computer security; Operating system; Internet privacy; World Wide Web","score_opus":0.014935987453394393,"score_gpt":0.27402721746446645,"score_spread":0.25909123001107204,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409168587","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.30351838,0.00008777826,0.69298327,0.00022630846,0.00031017754,0.00026556288,0.000004165533,0.0025897329,0.000014585797],"genre_scores_gemma":[0.93007904,0.0000022791328,0.06929904,0.00031694456,0.00006950821,0.00010025177,0.0000014693785,0.000025661477,0.00010579792],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.99841034,0.000046515175,0.00023078508,0.00067749893,0.0002846397,0.0003502334],"domain_scores_gemma":[0.9985375,0.00045680942,0.000069212074,0.0007365675,0.00007098953,0.00012892748],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00012244482,0.000290207,0.00031230337,0.00027273144,0.00016018453,0.0002356565,0.00052488106,0.000114492505,0.0000026373505],"category_scores_gemma":[0.0005711126,0.0002929547,0.00008886697,0.0007614475,0.000013914357,0.00039975563,0.00037536238,0.00045602216,0.000006377555],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00014215214,0.002966902,0.33655855,0.0004734287,0.001125556,0.06269472,0.016259704,0.04711243,0.0038849288,0.0022805992,0.006203076,0.52029794],"study_design_scores_gemma":[0.010384058,0.0046193986,0.38645393,0.0027325721,0.00039151165,0.0032839277,0.0077197077,0.28309676,0.11011715,0.019588212,0.16128945,0.010323298],"about_ca_topic_score_codex":0.000038760634,"about_ca_topic_score_gemma":0.000013178006,"teacher_disagreement_score":0.6265606,"about_ca_system_score_codex":0.00018956345,"about_ca_system_score_gemma":0.000026555896,"threshold_uncertainty_score":0.99995226},"labels":[],"label_agreement":null},{"id":"W4409405452","doi":"10.1007/s10664-025-10656-8","title":"Logging requirement for continuous auditing of responsible machine learning-based applications","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Audit; Logging; Computer science; Business; Accounting; Forestry; Geography","score_opus":0.015888302818632512,"score_gpt":0.29317120968724425,"score_spread":0.27728290686861173,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409405452","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0017101318,0.00021103752,0.9958072,0.0008127785,0.0002207186,0.00041212316,0.000003911425,0.00073307933,0.00008904434],"genre_scores_gemma":[0.5337496,0.0000015258197,0.4656082,0.0001641894,0.00006774092,0.00017112038,0.0000128959155,0.00002264249,0.00020212252],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9985346,0.000059277594,0.00042009732,0.0004074744,0.00021712405,0.00036142804],"domain_scores_gemma":[0.99746495,0.0018000836,0.00015663731,0.00039510947,0.000113988404,0.00006925536],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00065484864,0.00018853358,0.00030924063,0.00028414314,0.0001993529,0.000054937984,0.0006073925,0.00008834333,0.0000073323467],"category_scores_gemma":[0.0026058173,0.00020181511,0.00012739885,0.00069339684,0.000028966979,0.00013036845,0.0002494657,0.00033411328,0.0000029482394],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018709135,0.000042809723,0.019658932,0.00023725655,0.000034395252,0.000002488217,0.000100137455,0.96455437,0.0005482498,0.005320205,0.00016908394,0.009313358],"study_design_scores_gemma":[0.0006348061,0.00007424141,0.0022677071,0.00014540013,0.000020934782,0.0000012888115,0.000009818369,0.94050163,0.0025198746,0.0005765163,0.053010866,0.0002369244],"about_ca_topic_score_codex":0.000010650476,"about_ca_topic_score_gemma":0.0000011195174,"teacher_disagreement_score":0.5320394,"about_ca_system_score_codex":0.0001292658,"about_ca_system_score_gemma":0.00015522471,"threshold_uncertainty_score":0.82297814},"labels":[],"label_agreement":null},{"id":"W4409405484","doi":"10.1007/s10664-025-10651-z","title":"Predicting the understandability of computational notebooks through code metrics analysis","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Vector Institute","funders":"","keywords":"Computer science; Programming language; Code (set theory)","score_opus":0.03729922534070134,"score_gpt":0.3189262083606235,"score_spread":0.2816269830199222,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409405484","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09295937,0.00025737245,0.9052759,0.00049139385,0.00023398617,0.00016185713,0.0000119579445,0.0005632672,0.000044899767],"genre_scores_gemma":[0.87856245,0.0000030604037,0.12123819,0.0000998882,0.000025780846,0.000017887292,0.000004812607,0.000011327025,0.000036615715],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998014,0.00006291727,0.0004427471,0.00042611983,0.00067913934,0.0003750873],"domain_scores_gemma":[0.9909049,0.008033009,0.00006672711,0.0007002655,0.00022084815,0.00007422304],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000753136,0.00018944354,0.00034031502,0.00043420098,0.00013576273,0.0000989037,0.0010690347,0.00010137827,0.000008595207],"category_scores_gemma":[0.006151878,0.00015334848,0.0002319919,0.005106668,0.00007741563,0.00020418709,0.0004668231,0.00040722173,0.0000027556562],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000025693862,0.000027500231,0.38934094,0.00006206812,0.0003026762,0.0000024090534,0.00039695276,0.6075402,0.00000549165,0.0015786779,0.0001686567,0.00057185016],"study_design_scores_gemma":[0.00021477645,0.000029240713,0.39274177,0.00003479729,0.00009423342,0.0000019359238,0.000021021606,0.60278815,0.00036988492,0.0024973983,0.0010241896,0.00018259625],"about_ca_topic_score_codex":0.00004545998,"about_ca_topic_score_gemma":0.0000048744937,"teacher_disagreement_score":0.78560305,"about_ca_system_score_codex":0.00024657339,"about_ca_system_score_gemma":0.00016070611,"threshold_uncertainty_score":0.73648167},"labels":[],"label_agreement":null},{"id":"W4409405816","doi":"10.1007/s10664-025-10648-8","title":"Opportunities and security risks of technical leverage: A replication study on the NPM ecosystem","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Information and Cyber Security","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary; Concordia University","funders":"","keywords":"Leverage (statistics); Replication (statistics); Business; Computer science; Computer security; Medicine","score_opus":0.07617329618644243,"score_gpt":0.3188694552498021,"score_spread":0.24269615906335967,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409405816","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6652716,0.00005374974,0.33090544,0.0020393985,0.00015498996,0.00040644995,0.0000056769086,0.0003844241,0.00077830435],"genre_scores_gemma":[0.99891967,0.0000041543863,0.00070549466,0.00029121066,0.0000117483105,0.000041931027,0.0000010859708,0.000003320587,0.00002137446],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.9991822,0.000048374008,0.0002684457,0.00019366114,0.00018714282,0.00012015511],"domain_scores_gemma":[0.99889004,0.0003594091,0.00005656516,0.0005916349,0.00006046716,0.00004188187],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005573302,0.00010002185,0.00013909592,0.00008872346,0.00008662996,0.0000639284,0.00037002322,0.000053110918,0.0000034980578],"category_scores_gemma":[0.0004269413,0.000073966185,0.000036968024,0.00028374762,0.000014445913,0.00016511606,0.00021327288,0.00019524005,0.000002586202],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006881718,0.0017147713,0.22739579,0.00096273504,0.00035555958,0.00004531858,0.035246804,0.004345884,0.0001754337,0.6680485,0.020706262,0.040934134],"study_design_scores_gemma":[0.00089307205,0.00041257305,0.8691528,0.00025885348,0.00003418528,0.000025309404,0.0010191284,0.08051565,0.0017867372,0.0022453982,0.043101463,0.0005548442],"about_ca_topic_score_codex":0.000008653802,"about_ca_topic_score_gemma":0.000003412501,"teacher_disagreement_score":0.6658031,"about_ca_system_score_codex":0.00004444864,"about_ca_system_score_gemma":0.000035313522,"threshold_uncertainty_score":0.30162537},"labels":[],"label_agreement":null},{"id":"W4409441986","doi":"10.1007/s10664-025-10655-9","title":"Predicting long time contributors with knowledge units of programming languages: an empirical study","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Computer science; Empirical research; Programming language; Natural language processing; Data science; Artificial intelligence; Statistics; Mathematics","score_opus":0.017723677763842167,"score_gpt":0.3199314896089665,"score_spread":0.30220781184512435,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409441986","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.59133625,0.0002451727,0.40643704,0.00003747397,0.00011001339,0.00044175636,0.0000029283956,0.0013734795,0.000015866473],"genre_scores_gemma":[0.975531,9.1762337e-7,0.024005808,0.000023117856,0.00009296311,0.00009884936,0.000009874134,0.000050003964,0.00018750674],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9973222,0.00012733812,0.0004945934,0.000706501,0.0005837436,0.0007656083],"domain_scores_gemma":[0.9960057,0.002324192,0.00007401529,0.0008769675,0.00042255162,0.00029655098],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007894702,0.000371083,0.00052959303,0.0005264287,0.000126241,0.00017144941,0.0011964559,0.00015407536,0.000009383289],"category_scores_gemma":[0.004101091,0.0003282413,0.00006980447,0.0034402858,0.000054088705,0.00047762317,0.00053771643,0.0006004456,0.000014688852],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000151216345,0.00040403384,0.9838823,0.00012955954,0.00012874449,0.00009131531,0.0019052059,0.007397753,0.000035168538,0.00001646802,0.000102579805,0.005891766],"study_design_scores_gemma":[0.0013981846,0.0010415263,0.9392359,0.00040607617,0.000051081995,0.00002670825,0.00021560538,0.05432653,0.0010457532,0.0000057303478,0.0016601577,0.00058672857],"about_ca_topic_score_codex":0.000031188432,"about_ca_topic_score_gemma":0.000013167584,"teacher_disagreement_score":0.38419467,"about_ca_system_score_codex":0.00021160569,"about_ca_system_score_gemma":0.00031364226,"threshold_uncertainty_score":0.999917},"labels":[],"label_agreement":null},{"id":"W4409608564","doi":"10.1007/s10664-025-10633-1","title":"Impact of extensions on browser performance: An empirical study on google chrome","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Queen's University; University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Empirical research; World Wide Web; Computer science; Business; Mathematics","score_opus":0.03426352058869829,"score_gpt":0.36338248516094646,"score_spread":0.32911896457224815,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409608564","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8929907,0.000040707982,0.10504573,0.00009898664,0.00040996043,0.00037836697,0.0000052092037,0.0009920008,0.000038310052],"genre_scores_gemma":[0.9953534,0.0000033215706,0.0042259204,0.000106339656,0.00008906873,0.000059821326,0.000004371854,0.00004459788,0.00011316145],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9971841,0.00008113162,0.00045915754,0.0007707784,0.00078899454,0.0007158655],"domain_scores_gemma":[0.99627304,0.0017716867,0.00005013528,0.0014102447,0.00016694065,0.00032794513],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00046302305,0.00040743264,0.00049825996,0.0007368722,0.0001290096,0.00010187212,0.0012142083,0.00016652234,0.000033074997],"category_scores_gemma":[0.00204744,0.00034455745,0.0002179184,0.0018986467,0.000037459195,0.00036391045,0.00040671494,0.0007523548,0.00007010771],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000033661054,0.0010170925,0.87610954,0.000043257092,0.00009755798,0.000036982994,0.0004620917,0.118344784,0.00020384717,0.00001997681,0.0014748726,0.0021563105],"study_design_scores_gemma":[0.0005663706,0.0024760943,0.96493584,0.00012576653,0.00000955594,0.0000062446575,0.000009302647,0.030871028,0.0003957877,0.00001430515,0.00028057452,0.0003091372],"about_ca_topic_score_codex":0.000021004036,"about_ca_topic_score_gemma":6.965975e-7,"teacher_disagreement_score":0.10236267,"about_ca_system_score_codex":0.0004158051,"about_ca_system_score_gemma":0.00021649335,"threshold_uncertainty_score":0.99990064},"labels":[],"label_agreement":null},{"id":"W4409833316","doi":"10.1007/s10664-025-10654-w","title":"Towards understanding code review practices for infrastructure-as-code: An empirical study on OpenStack projects","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"École de Technologie Supérieure; Université du Québec à Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Code (set theory); Empirical research; Operating system; Software engineering; Programming language","score_opus":0.09559856838119425,"score_gpt":0.4004029999918837,"score_spread":0.3048044316106895,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4409833316","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09911789,0.0009178966,0.8912778,0.002842282,0.0014593304,0.002868889,0.000013402954,0.0012825857,0.00021992753],"genre_scores_gemma":[0.94533527,0.00018540803,0.048466653,0.0048188847,0.00028186134,0.0006481884,0.000019011131,0.000068279194,0.00017646811],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99693626,0.00015056608,0.0006738908,0.0010331158,0.00059804186,0.00060812203],"domain_scores_gemma":[0.9972184,0.001008851,0.00025337367,0.0011155256,0.00018441894,0.00021944009],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0013514746,0.00043052135,0.0006602097,0.00022780774,0.00029984076,0.00029166535,0.0012091895,0.00020000918,0.000011032896],"category_scores_gemma":[0.0047203177,0.0003428565,0.00017284352,0.0012323324,0.000036142494,0.0009113851,0.00037065882,0.00052898325,0.000018771661],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00039936783,0.0030707316,0.82278246,0.014464953,0.00089671026,0.00016967767,0.010859432,0.021783555,0.000073752424,0.004281345,0.08277517,0.038442876],"study_design_scores_gemma":[0.008733464,0.010238461,0.4411017,0.0128558045,0.00069571997,0.00019264771,0.0025030056,0.14512287,0.0012178972,0.0074483766,0.3644784,0.0054116338],"about_ca_topic_score_codex":0.000024974826,"about_ca_topic_score_gemma":0.000011140877,"teacher_disagreement_score":0.84621733,"about_ca_system_score_codex":0.0006050221,"about_ca_system_score_gemma":0.00049365347,"threshold_uncertainty_score":0.99990237},"labels":[],"label_agreement":null},{"id":"W4410602514","doi":"10.1007/s10664-025-10665-7","title":"Correction to: Utilization of pre-trained language models for adapter-based knowledge transfer in software engineering","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Okanagan University College; University of British Columbia, Okanagan Campus; University of British Columbia","funders":"","keywords":"Adapter (computing); Computer science; Software engineering; Natural language processing; Artificial intelligence; Operating system","score_opus":0.02983421994874074,"score_gpt":0.293690710703116,"score_spread":0.26385649075437523,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4410602514","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.067205705,0.00022117209,0.93057054,0.00006929537,0.00094640016,0.00042590982,0.000005998198,0.00054326036,0.00001173572],"genre_scores_gemma":[0.7937878,0.0000010471991,0.2058821,0.00006657743,0.00003957022,0.0001081059,0.0000075187263,0.000021165284,0.000086115775],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9987082,0.00001740087,0.00039662817,0.0004026331,0.00015042408,0.00032472424],"domain_scores_gemma":[0.9988333,0.00064073014,0.000020168123,0.00033427466,0.000087049906,0.00008448827],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002251684,0.00020220623,0.00027814243,0.0004191953,0.000030422541,0.00003532133,0.00038250693,0.00010885598,0.0000027217138],"category_scores_gemma":[0.00090862805,0.0002239857,0.000105199164,0.0008399015,0.0000056714625,0.00023679623,0.00006716061,0.00016718644,9.067697e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000015031298,0.000054584314,0.0014050931,0.0002279061,0.000010281263,0.0000014181774,0.001368091,0.984389,0.00053017755,0.0005120112,0.000103492486,0.011382923],"study_design_scores_gemma":[0.00045715418,0.00005046453,0.004661008,0.00037090285,0.0000072500084,6.983379e-7,0.000009315225,0.98796904,0.0057722447,0.000044704706,0.0004581145,0.00019911549],"about_ca_topic_score_codex":0.000026250938,"about_ca_topic_score_gemma":0.000026138607,"teacher_disagreement_score":0.7265821,"about_ca_system_score_codex":0.00016009765,"about_ca_system_score_gemma":0.000102456266,"threshold_uncertainty_score":0.9133872},"labels":[],"label_agreement":null},{"id":"W4411278626","doi":"10.1007/s10664-025-10682-6","title":"On the need to monitor continuous integration practices","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science","score_opus":0.0242274045556198,"score_gpt":0.31203987733416977,"score_spread":0.28781247277855,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4411278626","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.028772961,0.00010021141,0.9561245,0.011812114,0.00086693675,0.00028170846,0.0000012102664,0.001839817,0.00020053463],"genre_scores_gemma":[0.77163494,0.000008479271,0.22467212,0.003025445,0.00013431956,0.00016236886,0.0000016088018,0.000021861679,0.0003388408],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9988538,0.000049572547,0.00022272154,0.00034085126,0.00024783544,0.0002852224],"domain_scores_gemma":[0.9953841,0.0037387998,0.00009798544,0.0006076029,0.00007963911,0.00009190092],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0004537435,0.0002042144,0.0001723087,0.0001929279,0.00012229441,0.00034022372,0.00089020754,0.00009409721,0.000012554745],"category_scores_gemma":[0.009841264,0.00014705659,0.00007088646,0.0009573756,0.000009499948,0.00038303598,0.00022424354,0.00041872307,0.00005588044],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00014231562,0.0005196428,0.015348458,0.00018891484,0.00038199322,0.00012320845,0.0029072275,0.0778196,0.002926937,0.30419794,0.38091445,0.2145293],"study_design_scores_gemma":[0.0006628006,0.0008944604,0.053863656,0.0009799997,0.0000766597,0.000039705064,0.00012221708,0.17022373,0.029529482,0.009017164,0.7327732,0.0018169368],"about_ca_topic_score_codex":0.000027138769,"about_ca_topic_score_gemma":0.0000010832872,"teacher_disagreement_score":0.742862,"about_ca_system_score_codex":0.00008993883,"about_ca_system_score_gemma":0.00002903405,"threshold_uncertainty_score":0.9984993},"labels":[],"label_agreement":null},{"id":"W4411440374","doi":"10.1007/s10664-025-10661-x","title":"Supporting multi-dimensional unit test classification","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Computer science; Unit testing; Test (biology); Unit (ring theory); Artificial intelligence; Pattern recognition (psychology); Geology; Psychology; Operating system; Mathematics education","score_opus":0.04423106979949689,"score_gpt":0.34067254859068613,"score_spread":0.29644147879118926,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4411440374","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.010679972,0.00011379149,0.9713554,0.00066116627,0.00030435182,0.000111676614,0.0000018549114,0.016716829,0.000054992186],"genre_scores_gemma":[0.42888668,9.3318033e-7,0.57018584,0.0005349859,0.00003781327,0.000030100662,0.000006478042,0.0000144247615,0.00030272183],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9986003,0.000022832188,0.0003376119,0.00043011803,0.00021717726,0.00039193063],"domain_scores_gemma":[0.997534,0.0016681949,0.000070394686,0.0005045161,0.00011633687,0.00010658502],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003653619,0.00019419072,0.00018494151,0.00022844576,0.00014296798,0.00011563777,0.00058684975,0.00011527851,0.0000069315183],"category_scores_gemma":[0.005936855,0.00019009103,0.000074742224,0.0008629489,0.000022604347,0.0002135735,0.00028784713,0.0002866406,0.000030965035],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000023528257,0.00022935972,0.93297845,0.00009577847,0.000029858746,0.000039177885,0.00017161352,0.0035578487,0.0008075076,0.0013368527,0.034575224,0.026175985],"study_design_scores_gemma":[0.0002728696,0.000042710206,0.34154877,0.00019414064,0.000010351791,0.000020941377,0.0000018627693,0.64514107,0.0011290426,0.0020588492,0.009193354,0.00038604878],"about_ca_topic_score_codex":0.00001242957,"about_ca_topic_score_gemma":6.606442e-7,"teacher_disagreement_score":0.6415832,"about_ca_system_score_codex":0.0000822188,"about_ca_system_score_gemma":0.00009998502,"threshold_uncertainty_score":0.7751687},"labels":[],"label_agreement":null},{"id":"W4411538180","doi":"10.1007/s10664-025-10669-3","title":"A comprehensive study of machine learning techniques for log-based anomaly detection","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Alliance de recherche numérique du Canada; Natural Sciences and Engineering Research Council of Canada; Mitacs; Canada Research Chairs","keywords":"Anomaly detection; Anomaly (physics); Computer science; Machine learning; Artificial intelligence; Data mining; Physics","score_opus":0.01575046496258971,"score_gpt":0.27854004777971597,"score_spread":0.2627895828171263,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4411538180","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.35166603,0.00009999712,0.6469004,0.000033706863,0.00019860921,0.0004342333,0.0000010929765,0.00066290086,0.000003020166],"genre_scores_gemma":[0.952826,0.0000012312895,0.046910103,0.000050067076,0.000028951821,0.00015178836,0.0000020049586,0.000011056932,0.000018795092],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99887466,0.00004624543,0.00035113524,0.00033963314,0.00016683494,0.00022146518],"domain_scores_gemma":[0.99870366,0.00064350443,0.000076532066,0.00034960906,0.0001823143,0.00004439701],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00019726377,0.0001676588,0.00030266857,0.00022733485,0.00011303368,0.000032529377,0.0003444327,0.00009702189,0.0000012846058],"category_scores_gemma":[0.00040931982,0.00014891766,0.000103991006,0.0006511037,0.000015917825,0.00013783813,0.00011951938,0.0002048066,0.0000011954273],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010415597,0.0008220663,0.82607216,0.0015939418,0.0001646259,0.00000766473,0.0010509198,0.099810265,0.0037879855,0.00003663397,0.000105776664,0.06644382],"study_design_scores_gemma":[0.0019214832,0.002199841,0.23596811,0.00027334152,0.00004890871,0.000005555664,0.000081561164,0.6871225,0.053726003,0.00009273681,0.01797976,0.00058017106],"about_ca_topic_score_codex":0.000058983776,"about_ca_topic_score_gemma":0.000007167594,"teacher_disagreement_score":0.60115993,"about_ca_system_score_codex":0.000085965876,"about_ca_system_score_gemma":0.000045986908,"threshold_uncertainty_score":0.6072686},"labels":[],"label_agreement":null},{"id":"W4412152791","doi":"10.1007/s10664-025-10629-x","title":"Guiding principles for mixed methods research in software engineering","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"Natural Sciences and Engineering Research Council of Canada; Monash University; University of Victoria","keywords":"Computer science; Software engineering; Systems engineering; Engineering","score_opus":0.15558234056282055,"score_gpt":0.44330466850829364,"score_spread":0.2877223279454731,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4412152791","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.007192907,0.0013264733,0.9857756,0.00068759284,0.0014001392,0.00060715736,0.000004584833,0.0029724876,0.000033059903],"genre_scores_gemma":[0.039305408,0.000040531606,0.9596441,0.000105959254,0.0001494885,0.00054728054,0.0000065975178,0.00006538838,0.00013527402],"study_design_codex":"simulation_or_modeling","study_design_gemma":"not_applicable","domain_scores_codex":[0.9966766,0.0001612313,0.0007142025,0.0008900531,0.00043040214,0.0011274634],"domain_scores_gemma":[0.9878512,0.010706748,0.00007429941,0.00095464295,0.00021897593,0.0001941484],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.004388855,0.00040938577,0.00051803625,0.0012474215,0.0001810296,0.00033407236,0.0016142805,0.00032501743,0.0000055145974],"category_scores_gemma":[0.016593937,0.00043664503,0.0001918947,0.002744485,0.000027865266,0.00068718917,0.0009113242,0.00074381114,0.000007123912],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000092408256,0.00051592774,0.075299285,0.0036709507,0.00042248517,0.0002159841,0.0017192882,0.5760681,0.0031166296,0.089417845,0.01793153,0.23152955],"study_design_scores_gemma":[0.0007290631,0.00015517026,0.026094371,0.00096579094,0.00002106993,0.000025083666,0.00002141285,0.42459852,0.0110770315,0.0025687388,0.5326872,0.0010565642],"about_ca_topic_score_codex":0.00002884998,"about_ca_topic_score_gemma":0.0000030647525,"teacher_disagreement_score":0.51475567,"about_ca_system_score_codex":0.00043847153,"about_ca_system_score_gemma":0.00016572923,"threshold_uncertainty_score":0.99980855},"labels":[],"label_agreement":null},{"id":"W4412825323","doi":"10.1007/s10664-025-10705-2","title":"An Empirical Investigation on the Challenges in Scientific Workflow Systems Development","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Workflow; Computer science; Systems engineering; Data science; Empirical research; Software engineering; Engineering; Database; Mathematics","score_opus":0.24292159394730398,"score_gpt":0.3787638175672227,"score_spread":0.13584222361991874,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4412825323","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.89798343,0.00068981125,0.09137697,0.004891791,0.003963945,0.0003937016,0.0000034375141,0.00034002517,0.00035689145],"genre_scores_gemma":[0.9950467,0.0000025549714,0.003824239,0.00031687302,0.00007803885,0.000053334785,0.000009495845,0.000011057509,0.0006577556],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9963683,0.00024052405,0.00068486435,0.0010133518,0.0012930724,0.00039987714],"domain_scores_gemma":[0.9956666,0.0027447746,0.00007703342,0.0012498084,0.00012210039,0.00013963658],"candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.007899741,0.00019474828,0.00023935465,0.00075617747,0.00033707567,0.0011760484,0.001256522,0.0000825198,0.000025084717],"category_scores_gemma":[0.0065720524,0.00012632333,0.0000535801,0.0025668896,0.00008416171,0.00022146315,0.00030867528,0.0002743975,0.00018697685],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024651907,0.00032077418,0.32537672,0.000073785755,0.000042151707,0.000029388673,0.005487001,0.40649003,0.000055649063,0.005476751,0.0892111,0.167412],"study_design_scores_gemma":[0.00024417066,0.00003807347,0.5101786,0.0003663612,0.000005810102,0.0000016429767,0.0007306241,0.1709346,0.00018321842,0.0025663092,0.31439245,0.00035810255],"about_ca_topic_score_codex":0.000005845271,"about_ca_topic_score_gemma":0.000022108561,"teacher_disagreement_score":0.23555543,"about_ca_system_score_codex":0.00018539389,"about_ca_system_score_gemma":0.00013348086,"threshold_uncertainty_score":0.9998608},"labels":[],"label_agreement":null},{"id":"W4413087881","doi":"10.1007/s10664-025-10685-3","title":"On state reverting in solidity smart contracts: Developer practices, fault categorization, and tool evaluation","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Blockchain Technology Applications and Security","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Solidity; Categorization; Computer science; State (computer science); Fault (geology); Mean reversion; Engineering; Artificial intelligence; Economics; Econometrics; Programming language","score_opus":0.019389849230910663,"score_gpt":0.29830760325301664,"score_spread":0.278917754022106,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4413087881","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.34772786,0.00013434203,0.6501659,0.0013925433,0.000089537956,0.00022206304,8.29171e-7,0.00022928305,0.000037625734],"genre_scores_gemma":[0.97221273,0.000017144104,0.027164469,0.00044644505,0.000009926151,0.000106434425,0.0000048946345,0.0000062201857,0.00003172835],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9989381,0.00004755003,0.00027583423,0.00035990865,0.0001703789,0.00020822829],"domain_scores_gemma":[0.99890727,0.00054891576,0.000099630735,0.0002868343,0.00012340589,0.00003397105],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00080463354,0.000118910895,0.00013979687,0.00016938783,0.00009598679,0.00007369815,0.00024232332,0.000113235496,0.0000043869018],"category_scores_gemma":[0.0044949143,0.00011880435,0.000015057086,0.00071198517,0.000016332991,0.0002499057,0.00012907131,0.00030099374,0.0000049280884],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003942341,0.00063038676,0.3769287,0.00026244432,0.00012736325,0.000034418154,0.002592032,0.11135078,0.0006858305,0.09673923,0.0034930396,0.40711635],"study_design_scores_gemma":[0.0007297754,0.00003350986,0.29897648,0.000102551654,0.000012574798,0.000006846326,0.000011265669,0.66639787,0.0010125701,0.017729105,0.014640315,0.00034717188],"about_ca_topic_score_codex":0.00003965681,"about_ca_topic_score_gemma":0.000028561484,"teacher_disagreement_score":0.6244849,"about_ca_system_score_codex":0.00014895025,"about_ca_system_score_gemma":0.00011730735,"threshold_uncertainty_score":0.53811574},"labels":[],"label_agreement":null},{"id":"W4413219439","doi":"10.1007/s10664-025-10693-3","title":"Adversarial attack classification and robustness testing for large language models for code","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Robustness (evolution); Adversarial system; Artificial intelligence; Programming language; Biology","score_opus":0.04674644636670435,"score_gpt":0.3271197308610341,"score_spread":0.28037328449432974,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4413219439","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0145582175,0.00013838796,0.9830019,0.0005006216,0.0005887534,0.00049184315,0.000019084075,0.00066875626,0.000032423413],"genre_scores_gemma":[0.3774274,8.075747e-7,0.62197536,0.00011787367,0.00018826158,0.00013549939,0.000017062535,0.000024468436,0.00011328671],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9986454,0.000024472281,0.00026910013,0.0005019841,0.00013956362,0.0004194555],"domain_scores_gemma":[0.9975361,0.0018775623,0.00007200529,0.00031058263,0.000121274825,0.00008244884],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00044991562,0.00019851688,0.0002436438,0.00013432228,0.00023173411,0.0001254875,0.00043786573,0.0001378217,0.0000010639819],"category_scores_gemma":[0.0032272753,0.00020817184,0.00007344179,0.00042008638,0.000014184492,0.0004318284,0.00025131198,0.00020723212,6.1151104e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020251295,0.00002899746,0.0046961554,0.00020991685,0.00002995373,0.0000019138688,0.00036992412,0.9757539,0.00014457799,0.010620769,0.0006617925,0.0074618184],"study_design_scores_gemma":[0.0009230642,0.000036688227,0.0030402506,0.00006712246,0.000020111724,0.0000021120693,0.000031721866,0.99251103,0.000047680685,0.0004298856,0.0026716026,0.00021870933],"about_ca_topic_score_codex":0.0000039502092,"about_ca_topic_score_gemma":0.0000023879236,"teacher_disagreement_score":0.36286917,"about_ca_system_score_codex":0.00010040199,"about_ca_system_score_gemma":0.00007174403,"threshold_uncertainty_score":0.84890014},"labels":[],"label_agreement":null},{"id":"W4413308446","doi":"10.1007/s10664-025-10707-0","title":"Towards understanding the challenges of bug localization in deep learning systems","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Dalhousie University","funders":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Deep learning; Data science; Artificial intelligence; Software engineering","score_opus":0.042013341184440375,"score_gpt":0.2910416336842223,"score_spread":0.24902829249978192,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4413308446","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0069762324,0.004434514,0.98685324,0.00047879934,0.0004861508,0.000182084,2.9352185e-7,0.00049637584,0.00009229438],"genre_scores_gemma":[0.9967422,0.00010434454,0.0030182768,0.000022694285,0.000036113557,0.0000361399,7.925364e-7,0.00001811265,0.00002134989],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99848706,0.00008308285,0.00033576033,0.00031728958,0.0003915507,0.00038523859],"domain_scores_gemma":[0.9976309,0.001816022,0.000043752425,0.00038332507,0.00006550797,0.000060478065],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00071965,0.00016789166,0.0002478986,0.00038717408,0.00007483425,0.00008829025,0.0007211612,0.00011355498,0.0000020188966],"category_scores_gemma":[0.0030070124,0.00014074649,0.00005933852,0.0013897321,0.00003050839,0.0001950677,0.000299999,0.00043140684,0.0000033974359],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000025611666,0.000021282529,0.062277798,0.00027953027,0.000026455105,0.0000099179715,0.001054961,0.910623,0.000015668358,0.023329616,0.000048639766,0.0023105773],"study_design_scores_gemma":[0.0002996636,0.00004705014,0.075742595,0.00044697814,0.0000054487355,0.000006373176,0.00027159127,0.92043227,0.00014021547,0.0008431447,0.0015313176,0.00023334657],"about_ca_topic_score_codex":0.00003254796,"about_ca_topic_score_gemma":0.0000043754253,"teacher_disagreement_score":0.98976594,"about_ca_system_score_codex":0.00039411604,"about_ca_system_score_gemma":0.000067575456,"threshold_uncertainty_score":0.5739475},"labels":[],"label_agreement":null},{"id":"W4414474108","doi":"10.1007/s10664-025-10667-5","title":"Correction to: Assessing the adoption of security policies by developers in terraform across different cloud providers","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Information and Cyber Security","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Cloud computing; Cloud service provider; Security policy; Cloud computing security","score_opus":0.010958199298789092,"score_gpt":0.2911385402859514,"score_spread":0.28018034098716227,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4414474108","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.49017125,0.000025024405,0.50786227,0.000693328,0.00093675527,0.00011793589,0.0000014200649,0.000120984536,0.00007100654],"genre_scores_gemma":[0.99787766,0.0000027976416,0.0014471374,0.00058913697,0.000019498437,0.000021964126,0.00000423625,0.0000041483017,0.000033431024],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9991047,0.00002372042,0.00029076968,0.00014530028,0.00018349309,0.00025201787],"domain_scores_gemma":[0.9994831,0.00018283399,0.00004779995,0.00018955427,0.00005020947,0.000046489073],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00023423218,0.00011973909,0.0001386421,0.00009048705,0.00010051278,0.00015986199,0.00034952554,0.000055876448,0.0000014454238],"category_scores_gemma":[0.00035761032,0.00009104824,0.000045135977,0.00068502745,0.000018568422,0.00045832986,0.00019727056,0.00022912986,0.0000027097965],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000039627033,0.00052894343,0.49116564,0.00071929186,0.00012036001,0.0000051249726,0.22974595,0.06258469,0.0010649579,0.005905452,0.07573773,0.13238221],"study_design_scores_gemma":[0.00051380956,0.000056923105,0.7638164,0.00038200023,0.0000059557524,0.0000061678975,0.0018223557,0.20266058,0.0063139577,0.00024486872,0.02372528,0.00045166537],"about_ca_topic_score_codex":0.000103211605,"about_ca_topic_score_gemma":0.000051017378,"teacher_disagreement_score":0.5077064,"about_ca_system_score_codex":0.00019772236,"about_ca_system_score_gemma":0.000049437203,"threshold_uncertainty_score":0.37128395},"labels":[],"label_agreement":null},{"id":"W4414589104","doi":"10.1007/s10664-025-10717-y","title":"Towards understanding the impact of data bugs on deep learning models in software engineering","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal; Dalhousie University","funders":"","keywords":"Deep learning; Overfitting; Generalizability theory; Data pre-processing; Software quality; Leverage (statistics); Preprocessor; Metric (unit); Software bug; Software","score_opus":0.09277203969085854,"score_gpt":0.34203059390168095,"score_spread":0.2492585542108224,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4414589104","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.060646873,0.0006010526,0.9369383,0.00015986795,0.00030640114,0.00029368713,0.00000945866,0.0010105858,0.00003376151],"genre_scores_gemma":[0.94847757,0.00003122827,0.05124502,0.00003484089,0.00006734216,0.00003684526,0.000016002725,0.0000612369,0.000029899818],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9968323,0.000078406374,0.0005893702,0.00084247644,0.0007165012,0.0009409618],"domain_scores_gemma":[0.9933135,0.0046577933,0.00007791749,0.0016818689,0.00008579233,0.0001831439],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001131837,0.00046252928,0.0005322708,0.0008392753,0.00012374621,0.00020027057,0.0030994776,0.00021162351,0.000011518874],"category_scores_gemma":[0.007969806,0.0003758424,0.00020045851,0.002696023,0.000045037516,0.0008935893,0.0015926023,0.0013382537,0.000008170665],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011418748,0.000051215386,0.046579424,0.000106667656,0.000089440444,0.000027184566,0.00044517822,0.9478329,0.00003557518,0.0012376072,0.00018583478,0.0033975905],"study_design_scores_gemma":[0.00042619967,0.00010538259,0.09214449,0.0003675938,0.000008953152,0.000009013921,0.000025282436,0.90559983,0.00009920617,0.00068020175,0.00016047446,0.00037338978],"about_ca_topic_score_codex":0.0001279906,"about_ca_topic_score_gemma":0.00000414792,"teacher_disagreement_score":0.88783073,"about_ca_system_score_codex":0.0010833091,"about_ca_system_score_gemma":0.0002510791,"threshold_uncertainty_score":0.99986935},"labels":[],"label_agreement":null},{"id":"W4414622738","doi":"10.1007/s10664-025-10731-0","title":"DeepCodeProbe: Evaluating Code Representation Quality in Models Trained on Code","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Polytechnique Montréal","funders":"Fonds de Recherche du Québec - Santé; Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research","keywords":"Representation (politics); Software quality; Code (set theory); Quality (philosophy); Code smell; Source code; Software; Static program analysis","score_opus":0.11928940518778033,"score_gpt":0.4194135589370064,"score_spread":0.3001241537492261,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4414622738","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.24599624,0.00012261485,0.751425,0.00061242894,0.0003258441,0.00031728068,0.000005283383,0.0011065936,0.00008871525],"genre_scores_gemma":[0.88093495,0.000007008774,0.11832409,0.00027362426,0.000060340182,0.00017056172,0.000009481593,0.000036517882,0.00018341222],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99682957,0.00016657822,0.00064098695,0.0008870212,0.00078751944,0.0006883411],"domain_scores_gemma":[0.99443835,0.0042560385,0.000059873568,0.0009540233,0.00012972901,0.00016197341],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0015244586,0.00030264916,0.00039920135,0.0005759531,0.00009017318,0.00016331064,0.0009931377,0.00017748383,0.000009456379],"category_scores_gemma":[0.009862482,0.00032371693,0.00012034108,0.002037353,0.00002492132,0.000496674,0.00033987837,0.0006770328,0.000022090766],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000022705488,0.00009600682,0.059450883,0.00010195319,0.000025296145,0.000024840758,0.00054476137,0.92693764,0.00032526313,0.0033178146,0.00030163614,0.008851197],"study_design_scores_gemma":[0.00081189175,0.000070992726,0.19462289,0.0001895208,0.000004159376,0.000003082802,0.000010802924,0.7998773,0.0010339486,0.002805253,0.00022379555,0.000346379],"about_ca_topic_score_codex":0.000044474353,"about_ca_topic_score_gemma":0.000013698823,"teacher_disagreement_score":0.6349387,"about_ca_system_score_codex":0.0004908887,"about_ca_system_score_gemma":0.00016006206,"threshold_uncertainty_score":0.9999215},"labels":[],"label_agreement":null},{"id":"W4415991171","doi":"10.1007/s10664-025-10732-z","title":"MonoEmbed: Enhancing LLM representations for monolith to microservices decomposition through contrastive learning","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université Laval","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Microservices; Flexibility (engineering); Representation (politics); Decomposition; Adaptation (eye); Rank (graph theory); Benchmark (surveying)","score_opus":0.011455719920588168,"score_gpt":0.32757723681534184,"score_spread":0.31612151689475365,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4415991171","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2351282,0.00015116441,0.7626957,0.00042710517,0.00052869925,0.0004204834,0.000004278265,0.0006206096,0.000023722874],"genre_scores_gemma":[0.76715225,0.000005460251,0.23185724,0.0004956452,0.00010558503,0.00026623835,0.000011247572,0.000015554337,0.00009075827],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.99849546,0.000034240402,0.00038531315,0.0005262367,0.00015575066,0.00040297414],"domain_scores_gemma":[0.99842054,0.000909233,0.000060725317,0.00034925537,0.00016212284,0.00009814694],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00025363202,0.00019202549,0.00027539124,0.00013587762,0.0002314019,0.0001593915,0.0004349218,0.00010553803,0.0000031945042],"category_scores_gemma":[0.0005965588,0.00019198422,0.000119145676,0.0007213475,0.0000121220855,0.000523696,0.00019695534,0.0001986269,0.000019395877],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010076372,0.00031811214,0.21245253,0.0010524241,0.0003027043,0.000014863678,0.0146254115,0.73761904,0.008281263,0.0019739568,0.0037745435,0.019484363],"study_design_scores_gemma":[0.0026637863,0.0007182948,0.45745483,0.0014522852,0.000104805025,0.000026782634,0.0005640723,0.3971105,0.073632896,0.0031988132,0.06124411,0.0018288248],"about_ca_topic_score_codex":0.00006588205,"about_ca_topic_score_gemma":0.000015555705,"teacher_disagreement_score":0.5320241,"about_ca_system_score_codex":0.00017450705,"about_ca_system_score_gemma":0.000071077935,"threshold_uncertainty_score":0.78288895},"labels":[],"label_agreement":null},{"id":"W4416240214","doi":"10.1007/s10664-025-10700-7","title":"Fuzzing-based mutation testing of C/C++ software in cyber-physical systems","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"Kangwon National University; European Space Agency","keywords":"Fuzz testing; Symbolic execution; Code coverage; Process (computing); Software testing; Mutation testing; Mutation; Software; Software bug; Test case","score_opus":0.02328434506771092,"score_gpt":0.2833576778545125,"score_spread":0.26007333278680156,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4416240214","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.12074714,0.00023157442,0.87084526,0.00007979422,0.0002898045,0.0001828661,0.0000024058659,0.0075912983,0.000029836048],"genre_scores_gemma":[0.67071134,3.0207838e-7,0.3291052,0.000070125905,0.000036041038,0.000048205795,0.0000030119988,0.000016387374,0.000009416397],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9983263,0.000057830384,0.00045916915,0.00045219003,0.00031484297,0.0003896737],"domain_scores_gemma":[0.9958228,0.003323096,0.0001098148,0.00050660153,0.00015766881,0.00008001168],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00037942608,0.00024195816,0.0003925974,0.0004729523,0.000059941332,0.000088160436,0.0006105587,0.000122066034,5.8977054e-7],"category_scores_gemma":[0.0070249545,0.0002481102,0.000088858964,0.001918226,0.000031539563,0.00021059338,0.00018416856,0.00031453627,0.000004022447],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008600548,0.00026576238,0.56466424,0.00076805404,0.000028615044,0.000099241915,0.0004261025,0.403086,0.00019697065,0.00073638477,0.0015382898,0.028181756],"study_design_scores_gemma":[0.00042487183,0.00011697904,0.085944705,0.0011888229,0.000012577245,0.000012902667,0.000003599929,0.9071342,0.0015484256,0.0028699667,0.00034171564,0.0004012304],"about_ca_topic_score_codex":0.00014232291,"about_ca_topic_score_gemma":0.0000012802349,"teacher_disagreement_score":0.5499642,"about_ca_system_score_codex":0.00016521175,"about_ca_system_score_gemma":0.00015957636,"threshold_uncertainty_score":0.99999714},"labels":[],"label_agreement":null},{"id":"W4416248181","doi":"10.1007/s10664-025-10761-8","title":"SBEST: Spectrum-based fault localization without fault-triggering tests","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Alberta; Concordia University","funders":"","keywords":"Stack (abstract data type); Software bug; Context (archaeology); TRACE (psycholinguistics); Call stack; Fault (geology); Software regression; Software; Debugging","score_opus":0.01565781770040737,"score_gpt":0.28534965561632986,"score_spread":0.2696918379159225,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4416248181","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.013589659,0.00020604886,0.9603709,0.00056990894,0.00045643613,0.00019573471,0.0000015757662,0.024522327,0.00008738725],"genre_scores_gemma":[0.73704773,0.00000308053,0.26188353,0.0008266711,0.00008129121,0.000030872347,0.0000068340096,0.00003345054,0.00008655217],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99805874,0.0000370401,0.00039593826,0.0006189129,0.00035830037,0.0005310681],"domain_scores_gemma":[0.99815595,0.0007701451,0.00007497206,0.0007457039,0.00009728661,0.00015594765],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00029888723,0.00032995266,0.00032596022,0.00039154294,0.00017841152,0.00023509545,0.00081173645,0.00017303712,0.0000067100996],"category_scores_gemma":[0.0024440452,0.0003300446,0.000114387934,0.001494956,0.000033240103,0.00027776693,0.00025416428,0.00034571218,0.000024605813],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011476716,0.00018452399,0.56948876,0.0003021368,0.000053713356,0.00008842591,0.00022660481,0.39394453,0.0001181643,0.0011133985,0.015944825,0.018523455],"study_design_scores_gemma":[0.00035547354,0.00007117067,0.026351627,0.0004587669,0.000015530873,0.000015519441,7.002429e-7,0.95582885,0.0026805731,0.0031061973,0.010577179,0.0005384132],"about_ca_topic_score_codex":0.000047419497,"about_ca_topic_score_gemma":0.0000034864138,"teacher_disagreement_score":0.72345805,"about_ca_system_score_codex":0.00021393452,"about_ca_system_score_gemma":0.00014476497,"threshold_uncertainty_score":0.9999152},"labels":[],"label_agreement":null},{"id":"W4416271974","doi":"10.1007/s10664-025-10744-9","title":"An efficient model maintenance approach for MLOps","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Data Stream Mining Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Reuse; Pipeline (software); Inference; Data modeling; Time series; Computation; Statistical model; Maintenance actions","score_opus":0.01994939686437478,"score_gpt":0.29261028354318575,"score_spread":0.27266088667881094,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4416271974","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0031494163,0.00004105196,0.9940296,0.00015053165,0.00012731341,0.00025915907,0.000017053784,0.0020871786,0.0001387113],"genre_scores_gemma":[0.16772678,9.506901e-7,0.83174115,0.0002475632,0.000024438274,0.00014217975,0.000017081265,0.000014531775,0.00008533893],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998745,0.000010997872,0.00020160676,0.0005081435,0.00014952963,0.0003846845],"domain_scores_gemma":[0.9988965,0.0001586385,0.00002854838,0.0007522923,0.00006432209,0.000099711026],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00025801803,0.00017538667,0.00018492526,0.00013827905,0.0000745554,0.00011850308,0.0011482029,0.000088028006,5.8450564e-7],"category_scores_gemma":[0.0004041522,0.00016663491,0.00006843908,0.00043045497,0.000017708999,0.0001978384,0.00026158616,0.00014745313,0.0000016602344],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000006364645,0.00016280293,0.0009962629,0.00009291981,0.00001765431,0.000002762433,0.00020504932,0.9541528,0.00016159954,0.019158907,0.012534634,0.012508228],"study_design_scores_gemma":[0.00013433053,0.000036768284,0.0005777282,0.0000323246,0.000004295238,0.0000022348647,0.0000024237233,0.99500626,0.0007466138,0.0006825785,0.0025880672,0.00018638012],"about_ca_topic_score_codex":0.0000025785375,"about_ca_topic_score_gemma":1.2237625e-7,"teacher_disagreement_score":0.16457736,"about_ca_system_score_codex":0.000104010214,"about_ca_system_score_gemma":0.00006307128,"threshold_uncertainty_score":0.67951745},"labels":[],"label_agreement":null},{"id":"W4417025955","doi":"10.1007/s10664-025-10779-y","title":"Immutable in principle, upgradeable by design: exploratory study of smart contract upgradeability","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Blockchain Technology Applications and Security","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Upgrade; Empirical research; Smart contract; Vulnerability (computing); Exploratory research; Exploit","score_opus":0.01826608134195462,"score_gpt":0.2740175608584428,"score_spread":0.2557514795164882,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4417025955","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.47589955,0.0003713998,0.5228258,0.00016969266,0.000063230465,0.00036480502,0.0000013207747,0.00029736088,0.0000068815766],"genre_scores_gemma":[0.9799041,0.000007228866,0.019753357,0.000058895617,0.000004987062,0.0002376878,9.342757e-7,0.000008565594,0.000024256165],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99853647,0.00006988822,0.0004427994,0.0004635723,0.00016342552,0.0003238211],"domain_scores_gemma":[0.99857146,0.00047703562,0.0000612672,0.000777271,0.000055828226,0.000057153346],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00057360675,0.00017261616,0.0003346514,0.00021062247,0.00006538054,0.000028068105,0.0007750192,0.00015579526,0.0000044733442],"category_scores_gemma":[0.0003904773,0.00017629676,0.000045303896,0.0013567818,0.00003389642,0.00017601861,0.0002979684,0.0003790826,0.0000029179662],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000034614015,0.0046735904,0.9481219,0.00020418342,0.00013014654,0.00002364851,0.0025460336,0.025120093,0.0013757668,0.008385388,0.0025312004,0.006853435],"study_design_scores_gemma":[0.0057122996,0.0012792052,0.52658314,0.0002670043,0.00005748577,0.000010482932,0.00041372274,0.3844734,0.039097264,0.010106405,0.030295819,0.0017037523],"about_ca_topic_score_codex":0.000068249225,"about_ca_topic_score_gemma":0.000019890933,"teacher_disagreement_score":0.50400454,"about_ca_system_score_codex":0.00012508978,"about_ca_system_score_gemma":0.00007998276,"threshold_uncertainty_score":0.7189173},"labels":[],"label_agreement":null},{"id":"W4417026560","doi":"10.1007/s10664-025-10772-5","title":"Automatically Detecting Checked-In Secrets in Android Apps: How Far Are We?","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Advanced Malware Detection Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University; McGill University","funders":"Natural Sciences and Engineering Research Council of Canada; Fonds de recherche du Québec","keywords":"Android (operating system); Obfuscation; Cloud computing; Mobile device; Empirical research; Humanoid robot","score_opus":0.01545296850898984,"score_gpt":0.2740933592147346,"score_spread":0.25864039070574474,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4417026560","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.03958645,0.00037618788,0.95575553,0.0012436376,0.00023460535,0.0002458997,9.4523e-7,0.0025083327,0.000048379672],"genre_scores_gemma":[0.71684295,0.000026858364,0.28269514,0.00021911459,0.000027092614,0.000104551495,5.7515945e-7,0.000022479868,0.000061224135],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9982943,0.000044221022,0.00036996583,0.00054558757,0.00023524508,0.0005107075],"domain_scores_gemma":[0.9988573,0.00046575814,0.00007147007,0.0004616056,0.00004684608,0.0000969755],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003160642,0.00025629252,0.0003522143,0.0005666331,0.00005364509,0.0001191202,0.0006170396,0.00018241529,0.0000036826566],"category_scores_gemma":[0.0016544802,0.0002761375,0.000077510565,0.0017446423,0.000019160878,0.00047762887,0.00035305574,0.0005801294,0.0000050988456],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005692465,0.00059197046,0.2570586,0.002066787,0.00011382544,0.0015106351,0.0032685678,0.14781705,0.009410609,0.0044276696,0.001614124,0.57206327],"study_design_scores_gemma":[0.0012836183,0.00012880369,0.19358,0.0018363834,0.000008724407,0.00006730477,0.00010484598,0.7412504,0.034502357,0.0073837107,0.018471336,0.0013824881],"about_ca_topic_score_codex":0.000007357057,"about_ca_topic_score_gemma":0.000027263746,"teacher_disagreement_score":0.6772565,"about_ca_system_score_codex":0.00035236258,"about_ca_system_score_gemma":0.000050535702,"threshold_uncertainty_score":0.99996907},"labels":[],"label_agreement":null},{"id":"W4417026595","doi":"10.1007/s10664-025-10740-z","title":"Empirical studies of parameter efficient methods for large language models of code and knowledge transfer to R","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Topic Modeling","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Okanagan University College; University of British Columbia, Okanagan Campus; University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Automatic summarization; Empirical research; Code (set theory); Knowledge transfer; Language model; Resource (disambiguation)","score_opus":0.06950578563380058,"score_gpt":0.4078640021553274,"score_spread":0.33835821652152687,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4417026595","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2014153,0.002281768,0.7955889,0.00020029556,0.0001759043,0.00022175408,0.0000085848515,0.000098923585,0.000008529418],"genre_scores_gemma":[0.46529078,0.0000048229676,0.5345202,0.00010116912,0.00001329128,0.000035857032,4.2653642e-7,0.000007983052,0.000025463782],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9988676,0.000049734695,0.00036589763,0.00034324345,0.000103868966,0.00026965537],"domain_scores_gemma":[0.99774605,0.0017289285,0.00001475921,0.0003171864,0.00011284491,0.000080242535],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00058620615,0.0001505181,0.00041575608,0.00017615233,0.00003353435,0.000013753392,0.00030706683,0.000073968564,8.260014e-7],"category_scores_gemma":[0.00094711327,0.0001320522,0.00010097123,0.0003970787,0.000017833685,0.000062256266,0.0002598746,0.00010178445,3.1265344e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006551703,0.00055006595,0.0057432177,0.002891763,0.0005134849,0.00000542242,0.06091888,0.80420893,0.0063230903,0.022056036,0.00075997115,0.09596361],"study_design_scores_gemma":[0.00040520105,0.000070475566,0.00067901576,0.00014500551,0.000025153195,0.000001006739,0.00007010571,0.98605514,0.010446073,0.0006254177,0.0013178692,0.0001595178],"about_ca_topic_score_codex":0.0000021841392,"about_ca_topic_score_gemma":0.000001698936,"teacher_disagreement_score":0.26387548,"about_ca_system_score_codex":0.000040440766,"about_ca_system_score_gemma":0.00004174186,"threshold_uncertainty_score":0.5384933},"labels":[],"label_agreement":null},{"id":"W4417031931","doi":"10.1007/s10664-025-10765-4","title":"Maintaining shared understanding of non-functional requirements in small companies using continuous software engineering","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Techniques and Practices","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria; Microsoft (Canada); SST Wireless (Canada)","funders":"Science Foundation Ireland","keywords":"Maintainability; Rework; Personal software process; Social software engineering; Software development; Software Engineering Process Group; Context (archaeology); Software peer review; Software requirements; Software","score_opus":0.07692965972545587,"score_gpt":0.3050413437707852,"score_spread":0.2281116840453293,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4417031931","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08920708,0.00028797024,0.90823954,0.000079007885,0.0007922446,0.00024851848,0.000005286532,0.001116461,0.00002391174],"genre_scores_gemma":[0.5501785,0.000009111739,0.44960698,0.000065764216,0.0000555943,0.000025526993,0.0000072751163,0.000034513323,0.000016737431],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99769586,0.000035214805,0.0007208737,0.0005754881,0.00032262524,0.0006499253],"domain_scores_gemma":[0.99768126,0.001439555,0.0001596374,0.0005171925,0.00007640598,0.00012595962],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006136873,0.0004013434,0.00057487865,0.00079232885,0.00010161348,0.00016994737,0.0007475706,0.00020741393,0.000013635216],"category_scores_gemma":[0.0018978233,0.00045473408,0.00015474697,0.0014282232,0.000029716246,0.00067689666,0.0004966326,0.000545738,0.000002224117],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003668954,0.0001341855,0.23290284,0.00048341375,0.00019366544,0.00009772144,0.000646169,0.75760037,0.0014687849,0.004910018,0.00036121777,0.0011649303],"study_design_scores_gemma":[0.0012224736,0.00013515989,0.104235604,0.0020514429,0.000050651313,0.00004262565,0.000090694644,0.8861827,0.0020084414,0.0010234457,0.0019158265,0.0010408927],"about_ca_topic_score_codex":0.00007461983,"about_ca_topic_score_gemma":0.0000054095512,"teacher_disagreement_score":0.46097142,"about_ca_system_score_codex":0.0006698257,"about_ca_system_score_gemma":0.000134137,"threshold_uncertainty_score":0.99979043},"labels":[],"label_agreement":null},{"id":"W7109084896","doi":"10.1007/s10664-025-10768-1","title":"Output format biases in the evaluation of large language models for code translation","year":2025,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Huawei Technologies (Canada); Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada; Vector Institute","keywords":"Executable; Source code; Disk formatting; Code (set theory); Code review; Translation (biology); Empirical research; Software","score_opus":0.07234957541314808,"score_gpt":0.3707188625070804,"score_spread":0.29836928709393234,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W7109084896","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.025658293,0.0031291018,0.9701245,0.00040020584,0.000063838655,0.0003453087,0.000009061304,0.00025171548,0.000018005243],"genre_scores_gemma":[0.75568146,0.000002204044,0.24406298,0.00015307387,0.000011480431,0.000071295784,0.000008725264,0.0000042823913,0.0000044774993],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9991794,0.000039320454,0.00019687013,0.00014372153,0.00026770146,0.00017296683],"domain_scores_gemma":[0.99910074,0.0005500224,0.000035534827,0.0002145579,0.00008527847,0.000013863918],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0011098525,0.00008374727,0.00011209483,0.00014717497,0.000032217766,0.000038036967,0.00041971824,0.000060511706,7.8290003e-7],"category_scores_gemma":[0.00077054184,0.00006258894,0.00005051588,0.0004524664,0.0000057922543,0.00035220658,0.000041613803,0.00010506612,2.1678504e-7],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004152625,0.00035017374,0.002786603,0.00096260547,0.000059327245,0.000007367986,0.037609763,0.37932032,0.0014392572,0.02281322,0.0032726526,0.5513372],"study_design_scores_gemma":[0.00030348502,0.000015440208,0.00047042387,0.0001132704,0.000011430521,9.926662e-7,0.000032690306,0.9889132,0.0032166392,0.00651692,0.00032963353,0.000075877935],"about_ca_topic_score_codex":0.0000070763735,"about_ca_topic_score_gemma":0.00000938769,"teacher_disagreement_score":0.7300232,"about_ca_system_score_codex":0.00006446775,"about_ca_system_score_gemma":0.00005478014,"threshold_uncertainty_score":0.25523028},"labels":[],"label_agreement":null}]}