{"meta":{"query_hash":"987367f4bff6","filters":{"venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)"},"cohort_total":8,"direct_labels_cover":0,"predictions_cover":8,"exported":8,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/987367f4bff6","api":"https://metacan.xera.ac/api/v1/cohort?venue=2021+IEEE+21st+International+Conference+on+Software+Quality%2C+Reliability+and+Security+%28QRS%29"},"results":[{"id":"W3196967454","doi":"10.1109/qrs54544.2021.00116","title":"The Challenge of Reproducible ML: An Empirical Study on The Impact of Bugs","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Determinism; Inference; Software bug; Reproducibility; Software; Machine learning; Artificial intelligence; Face (sociological concept); Causal inference; Software engineering; Programming language; Statistics","score_opus":0.11430068119320197,"score_gpt":0.4201996151009927,"score_spread":0.3058989339077907,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196967454","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"reproducibility","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":"reproducibility","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9469404,0.0024936148,0.040500395,0.0021624432,0.00017160957,0.00054653414,0.0017315102,0.0016882122,0.0037653395],"genre_scores_gemma":[0.9807115,0.00038210032,0.015360561,0.00033248938,0.000091236674,0.00025995992,0.0019325388,0.0004305322,0.00049906166],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9004913,0.054970365,0.00985989,0.011897173,0.021272432,0.0015087833],"domain_scores_gemma":[0.24360158,0.62681305,0.039339755,0.0681784,0.020006444,0.002060784],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.06669086,0.00105985,0.0009475901,0.002818651,0.0020014446,0.0035977727,0.0027442898,0.0022023888,0.001651537],"category_scores_gemma":[0.4104068,0.0008801651,0.0012583062,0.0030986539,0.0055384915,0.0067142574,0.004182488,0.0041125105,0.0006767687],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0027283102,0.0032882458,0.6389276,0.003917778,0.0011675616,0.0015321263,0.01401108,0.046495136,0.010174689,0.012946138,0.022561513,0.24224983],"study_design_scores_gemma":[0.0010023041,0.009343557,0.5457035,0.002373867,0.0012635244,0.005699039,0.01212409,0.2760196,0.046479806,0.044059463,0.055241752,0.00068945024],"about_ca_topic_score_codex":0.0020439734,"about_ca_topic_score_gemma":0.00211706,"teacher_disagreement_score":0.93330914,"about_ca_system_score_codex":0.0020068313,"about_ca_system_score_gemma":0.0023194079,"threshold_uncertainty_score":0.35269922},"labels":[],"label_agreement":null},{"id":"W3212787291","doi":"10.1109/qrs54544.2021.00037","title":"On Assessing The Safety of Reinforcement Learning algorithms Using Formal Methods","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Adversarial system; Probabilistic logic; Reinforcement learning; Computer science; Artificial intelligence; Computer security; Multi-agent system; Autonomous agent; Grid; Machine learning; Mathematics","score_opus":0.06773412524589584,"score_gpt":0.41051923656668443,"score_spread":0.3427851113207886,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3212787291","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.016733414,0.00025918082,0.9807196,0.00042797273,0.000028354427,0.000089950314,0.000037378082,0.00034837765,0.0013557252],"genre_scores_gemma":[0.6192753,0.0006417008,0.37820643,0.00026578706,0.00008968855,0.00042056185,0.00015739298,0.00021389345,0.0007292076],"study_design_codex":"simulation_or_modeling","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.98241043,0.011149671,0.0008155123,0.001219123,0.003677322,0.0007280483],"domain_scores_gemma":[0.7167638,0.25812137,0.009750491,0.00799838,0.006019915,0.0013460586],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.024737876,0.0018899264,0.0011406074,0.0023225027,0.0010527009,0.0028816208,0.0019298034,0.0019918869,0.0023700893],"category_scores_gemma":[0.13489774,0.000672757,0.0015501162,0.00090998545,0.005616733,0.0042417,0.0029376165,0.003175537,0.00024429918],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012828594,0.00010596007,0.0036671087,0.00021069321,0.00007547016,0.00004307006,0.00019371633,0.8959075,0.0011905472,0.067933485,0.00023878107,0.030305317],"study_design_scores_gemma":[0.00001836989,0.000078674646,0.00016898081,0.000052145326,0.000010829484,0.000018083676,0.000021584705,0.9593529,0.0007525616,0.03924681,0.00026784567,0.000011264169],"about_ca_topic_score_codex":0.0042536943,"about_ca_topic_score_gemma":0.0021880225,"teacher_disagreement_score":0.024737876,"about_ca_system_score_codex":0.0032425898,"about_ca_system_score_gemma":0.0045526377,"threshold_uncertainty_score":0.13082796},"labels":[],"label_agreement":null},{"id":"W4225775465","doi":"10.1109/qrs54544.2021.00106","title":"Analyzing the Impact of Cyberattacks on Industrial Control Systems using Timed Automata","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Smart Grid Security and Resilience","field":"Engineering","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Exploit; Computer science; Industrial control system; Spoofing attack; Automaton; Computer security; Model checking; Critical infrastructure; Vulnerability (computing); Ransomware; Control (management); Malware; Theoretical computer science; Artificial intelligence","score_opus":0.057983503388506816,"score_gpt":0.33548058703050765,"score_spread":0.2774970836420008,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4225775465","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38060355,0.00015324382,0.6138385,0.00012561439,0.000029247007,0.00007904889,0.00017242042,0.00092988694,0.00406848],"genre_scores_gemma":[0.9701671,0.00008988773,0.029150149,0.000011567779,0.000005692886,0.000045823155,0.000056675963,0.00003041972,0.00044271073],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99935585,0.00021059916,0.00004254189,0.00009088422,0.00021952396,0.0000805294],"domain_scores_gemma":[0.99609214,0.0029465898,0.00038785234,0.0002607558,0.00025402743,0.00005869129],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009243843,0.000622299,0.00031940572,0.00070278737,0.00033969764,0.0009777867,0.00059828145,0.00041321054,0.0010955981],"category_scores_gemma":[0.0042605363,0.00026810708,0.0007328546,0.0003699016,0.0010435679,0.0009933988,0.0005269593,0.0006373971,0.00008361707],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004873653,0.00003617238,0.003908341,0.00003335093,0.000027695449,0.00009094108,0.000074509844,0.9697762,0.0047077793,0.015500835,0.00006208046,0.0057334257],"study_design_scores_gemma":[0.0000027085127,0.000019449322,0.0002586964,0.000002432391,0.000007971907,0.000008638178,0.0000126251525,0.99523735,0.0015480054,0.0028041129,0.0000951818,0.0000027341016],"about_ca_topic_score_codex":0.012694259,"about_ca_topic_score_gemma":0.009024845,"teacher_disagreement_score":0.012694259,"about_ca_system_score_codex":0.0010061124,"about_ca_system_score_gemma":0.0011228824,"threshold_uncertainty_score":0.025240779},"labels":[],"label_agreement":null},{"id":"W4225807728","doi":"10.1109/qrs54544.2021.00063","title":"Predictors of Software Metric Correlation: A Non-parametric Analysis","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Cyclomatic complexity; Metric (unit); Code (set theory); Correlation; Computer science; Code refactoring; Source lines of code; Code coverage; Software quality; Software metric; Software; Statistics; Programming language; Mathematics; Software development; Set (abstract data type); Engineering","score_opus":0.034565574273561835,"score_gpt":0.32378986542613564,"score_spread":0.2892242911525738,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4225807728","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9708711,0.00040497154,0.023862585,0.0003679276,0.00010262306,0.00022189741,0.0010268439,0.0003278496,0.0028141926],"genre_scores_gemma":[0.9957419,0.000045818164,0.0027718102,0.000036910566,0.000030952957,0.00034200348,0.0005130956,0.000070847236,0.0004465757],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.96325815,0.023326924,0.001953811,0.0044629104,0.005112452,0.0018857365],"domain_scores_gemma":[0.56594366,0.3984325,0.012228957,0.013988676,0.0065383115,0.0028678516],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.028606111,0.0011577124,0.0014141579,0.0038670443,0.0008249706,0.0023556869,0.0022443207,0.0017548358,0.009391672],"category_scores_gemma":[0.1380795,0.00048180055,0.0027235511,0.0036885836,0.002806222,0.0037436222,0.0032414785,0.003846492,0.0018475701],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0026505394,0.0008028257,0.93505883,0.0003133212,0.002298481,0.0005783679,0.0038017374,0.0050962428,0.0018500024,0.0025388203,0.0028767013,0.042134315],"study_design_scores_gemma":[0.00016534881,0.005200429,0.86371,0.0001565176,0.00074310025,0.0015281994,0.0049908664,0.11031739,0.0020724004,0.006896439,0.0039717373,0.00024752226],"about_ca_topic_score_codex":0.0010942208,"about_ca_topic_score_gemma":0.00046729392,"teacher_disagreement_score":0.028606111,"about_ca_system_score_codex":0.00053679675,"about_ca_system_score_gemma":0.0009701438,"threshold_uncertainty_score":0.15128541},"labels":[],"label_agreement":null},{"id":"W4226069469","doi":"10.1109/qrs54544.2021.00012","title":"Analyzing Structural Security Posture to Evaluate System Design Decisions","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Leverage (statistics); Computer science; Computer security; Software security assurance; Security testing; Computer security model; Identification (biology); Resource (disambiguation); Security information and event management; Secure coding; Security service; Security through obscurity; Software; Cloud computing security; Risk analysis (engineering); Information security; Artificial intelligence; Cloud computing; Business","score_opus":0.07041286786716698,"score_gpt":0.3641071605009214,"score_spread":0.2936942926337544,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226069469","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.63999563,0.00037602105,0.33983865,0.00064680073,0.000059431655,0.00067786244,0.00082958053,0.00200625,0.015569696],"genre_scores_gemma":[0.8468289,0.000097608914,0.15160152,0.000049576964,0.0000128003285,0.00025348656,0.0004908075,0.00008861765,0.000576576],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99043244,0.003776015,0.00093607395,0.000621371,0.0037142453,0.00051990617],"domain_scores_gemma":[0.95486385,0.020681422,0.009416785,0.003744851,0.010370939,0.0009221224],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.010902034,0.0016185767,0.00087663805,0.009062931,0.0008065822,0.0031882247,0.00071825157,0.0011800252,0.0019173758],"category_scores_gemma":[0.047153745,0.00044349633,0.0007907893,0.0033834518,0.0012486962,0.0033264884,0.0014209545,0.001104646,0.00046503972],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0007267113,0.0011080327,0.23928697,0.00075673335,0.00052063115,0.0005851386,0.006030521,0.19379483,0.05991667,0.04653995,0.003270259,0.4474636],"study_design_scores_gemma":[0.00008756667,0.002469682,0.112897426,0.00034200333,0.00027214846,0.00028292893,0.0038531707,0.8068075,0.033195574,0.0345635,0.0050110333,0.00021742206],"about_ca_topic_score_codex":0.0033122716,"about_ca_topic_score_gemma":0.0052286186,"teacher_disagreement_score":0.010902034,"about_ca_system_score_codex":0.002620702,"about_ca_system_score_gemma":0.0018704614,"threshold_uncertainty_score":0.05765617},"labels":[],"label_agreement":null},{"id":"W4226205863","doi":"10.1109/qrs54544.2021.00076","title":"Vulnerability Analysis of Similar Code","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"New York Institute of Technology","funders":"","keywords":"Computer science; Secure coding; Vulnerability (computing); Software security assurance; Code (set theory); Computer security; Code review; Static program analysis; Domain (mathematical analysis); Software; Scripting language; Source code; Vulnerability assessment; Security bug; Application security; Software engineering; Information security; Programming language; Software development; Security service; Mathematics","score_opus":0.060345565260924376,"score_gpt":0.3613439035872422,"score_spread":0.30099833832631784,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226205863","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9770074,0.0008988419,0.014568059,0.00010048763,0.00002080208,0.00012524973,0.004723755,0.0007402314,0.0018151513],"genre_scores_gemma":[0.968457,0.00031201585,0.018559966,0.000055917943,0.000016922633,0.00012489021,0.011434686,0.00016390478,0.00087466126],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99619484,0.0004405602,0.00038801203,0.0011429752,0.0015223039,0.00031130714],"domain_scores_gemma":[0.9836083,0.006496385,0.0044019157,0.0023237506,0.002629205,0.0005404951],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0017562163,0.0005574684,0.0005310334,0.01084073,0.0007026822,0.0008326156,0.0005660453,0.0006371642,0.00076588034],"category_scores_gemma":[0.015771972,0.00022279735,0.00092790165,0.0051127565,0.0007145038,0.0016457448,0.0015346468,0.00055014057,0.00033560168],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00039825888,0.00024639472,0.8178826,0.00087649166,0.00065287447,0.0016280371,0.0021684773,0.009976147,0.01966852,0.0037449547,0.005439931,0.13731723],"study_design_scores_gemma":[0.00003642501,0.0003821667,0.8486122,0.00021681043,0.00035560655,0.0052194814,0.0019406048,0.100133985,0.016026417,0.009109963,0.017839137,0.0001272298],"about_ca_topic_score_codex":0.004464858,"about_ca_topic_score_gemma":0.004423911,"teacher_disagreement_score":0.01084073,"about_ca_system_score_codex":0.0006273366,"about_ca_system_score_gemma":0.0007081648,"threshold_uncertainty_score":0.009287894},"labels":[],"label_agreement":null},{"id":"W4226207655","doi":"10.1109/qrs54544.2021.00118","title":"Understanding the Resilience of Neural Network Ensembles against Faulty Training Data","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Resilience (materials science); Machine learning; Artificial intelligence; Artificial neural network; Training set; Ensemble learning; Entropy (arrow of time); Training (meteorology); Ensemble forecasting; Data mining","score_opus":0.21383400830963548,"score_gpt":0.3701135125969131,"score_spread":0.15627950428727763,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226207655","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7650633,0.0008904764,0.22843662,0.0014953224,0.00012335765,0.00006528535,0.00022843614,0.0006576907,0.003039623],"genre_scores_gemma":[0.9931665,0.00012758796,0.006163506,0.00008485196,0.000024032975,0.000022895974,0.00009099858,0.000027568609,0.0002919816],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99897075,0.00032210094,0.00006913585,0.00020514421,0.00024710284,0.00018567998],"domain_scores_gemma":[0.9868471,0.008194512,0.0014686632,0.0019337821,0.0011537942,0.00040208382],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0038745422,0.0007348652,0.00059284386,0.0010132655,0.0005416111,0.0009133366,0.0008083643,0.0011357128,0.0008754389],"category_scores_gemma":[0.031365916,0.00037758247,0.00045926255,0.00040111263,0.0012116025,0.0026729577,0.001504632,0.0014578681,0.00019933608],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010355347,0.00003245485,0.00836568,0.000039089377,0.000063856,0.000079451056,0.00009855825,0.96977353,0.004004749,0.002623571,0.000332222,0.01448332],"study_design_scores_gemma":[0.0000042268057,0.00009639929,0.002738444,0.000020442061,0.000022601116,0.000054776996,0.000062126855,0.98455703,0.0039505563,0.008213253,0.0002667853,0.000013530847],"about_ca_topic_score_codex":0.0031126754,"about_ca_topic_score_gemma":0.0020215567,"teacher_disagreement_score":0.0038745422,"about_ca_system_score_codex":0.0009369771,"about_ca_system_score_gemma":0.0005655292,"threshold_uncertainty_score":0.020490766},"labels":[],"label_agreement":null},{"id":"W4226466002","doi":"10.1109/qrs54544.2021.00093","title":"MINTS: Unsupervised Temporal Specifications Miner","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Correctness; Computer science; Debugging; Task (project management); Finite-state machine; Scalability; Frame (networking); Software; State (computer science); Trie; Software bug; Software system; Artificial intelligence; Programming language; Data structure; Engineering; Database; Systems engineering","score_opus":0.10583280632417456,"score_gpt":0.34160529800087,"score_spread":0.2357724916766954,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226466002","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.015949756,0.00045549485,0.9462593,0.0003698213,0.00007815745,0.00025313886,0.00740912,0.027345328,0.001879897],"genre_scores_gemma":[0.17979737,0.00033031646,0.7839303,0.00038183975,0.00005944866,0.0006453912,0.02884185,0.0020130074,0.0040004845],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99885786,0.00024216599,0.00012434658,0.00038594048,0.00031332002,0.0000764204],"domain_scores_gemma":[0.99676263,0.0019364712,0.00022627943,0.0005612992,0.00043284774,0.00008043925],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0012350332,0.0010578255,0.0006226752,0.0016372615,0.00052356825,0.0011055293,0.0023333502,0.0011231262,0.004875531],"category_scores_gemma":[0.008825713,0.0007936501,0.0017598062,0.0012187585,0.0006062924,0.002005195,0.001717199,0.0015164193,0.0022906784],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0007922871,0.0003329186,0.012530496,0.0013483153,0.00048196394,0.000893466,0.0005652016,0.24362981,0.018460877,0.039686743,0.06480631,0.61647165],"study_design_scores_gemma":[0.00005965423,0.000070949456,0.0005817048,0.000051993247,0.000035798254,0.00031685916,0.0000845832,0.93472177,0.0090290215,0.039113827,0.015911816,0.000022078948],"about_ca_topic_score_codex":0.0034104509,"about_ca_topic_score_gemma":0.013291674,"teacher_disagreement_score":0.004875531,"about_ca_system_score_codex":0.00075681176,"about_ca_system_score_gemma":0.0028263074,"threshold_uncertainty_score":0.016310334},"labels":[],"label_agreement":null}]}