{"meta":{"query_hash":"987367f4bff6","filters":{"venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)"},"cohort_total":8,"direct_labels_cover":0,"predictions_cover":8,"exported":8,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/987367f4bff6","api":"https://metacan.xera.ac/api/v1/cohort?venue=2021+IEEE+21st+International+Conference+on+Software+Quality%2C+Reliability+and+Security+%28QRS%29"},"results":[{"id":"W3196967454","doi":"10.1109/qrs54544.2021.00116","title":"The Challenge of Reproducible ML: An Empirical Study on The Impact of Bugs","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Determinism; Inference; Software bug; Reproducibility; Software; Machine learning; Artificial intelligence; Face (sociological concept); Causal inference; Software engineering; Programming language; Statistics","score_opus":0.11430068119320197,"score_gpt":0.4201996151009927,"score_spread":0.3058989339077907,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196967454","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9764558,0.000059228256,0.0039467206,0.01578588,0.00042145964,0.0003392248,0.00013824206,0.000058144928,0.002795313],"genre_scores_gemma":[0.99914926,0.00018975008,0.0002789745,0.000104916006,0.000085133586,0.000025706746,0.000039788654,0.000007313386,0.00011916431],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.9956048,0.0016934852,0.0006928632,0.00093746214,0.0008682088,0.00020318285],"domain_scores_gemma":[0.99469453,0.0016019575,0.00039119265,0.002193066,0.0010185991,0.00010063599],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0042963573,0.00020692778,0.00032957303,0.00006241138,0.0002549404,0.00023834196,0.0012607386,0.000098723045,0.00014041018],"category_scores_gemma":[0.0041578556,0.00012762083,0.00016132378,0.00029092852,0.00025482714,0.0002703684,0.0003252844,0.0005532941,0.000011311241],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00043992128,0.007763709,0.2064069,0.00011805059,0.00030659093,0.000010402311,0.025873406,0.00036685434,0.0004438902,0.7220569,0.0008686363,0.035344686],"study_design_scores_gemma":[0.0014603047,0.003968172,0.7161769,0.00022337379,0.000039210932,0.000014152122,0.008127679,0.041501656,0.0019217816,0.22352718,0.0023011477,0.0007384323],"about_ca_topic_score_codex":0.0007344012,"about_ca_topic_score_gemma":0.00020653602,"teacher_disagreement_score":0.50977,"about_ca_system_score_codex":0.00006738812,"about_ca_system_score_gemma":0.00032618412,"threshold_uncertainty_score":0.5204227},"labels":[],"label_agreement":null},{"id":"W3212787291","doi":"10.1109/qrs54544.2021.00037","title":"On Assessing The Safety of Reinforcement Learning algorithms Using Formal Methods","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Adversarial system; Probabilistic logic; Reinforcement learning; Computer science; Artificial intelligence; Computer security; Multi-agent system; Autonomous agent; Grid; Machine learning; Mathematics","score_opus":0.06773412524589584,"score_gpt":0.41051923656668443,"score_spread":0.3427851113207886,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3212787291","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.070139624,0.000032094107,0.9236363,0.0014437939,0.0013476382,0.00019177698,0.000013812672,0.00008018511,0.0031147655],"genre_scores_gemma":[0.87486565,0.0000911314,0.12427531,0.0003943564,0.00015310272,0.000010423493,0.000032257914,0.0000148568015,0.00016288378],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9946704,0.002061088,0.0009475261,0.0008202532,0.0011073061,0.00039343786],"domain_scores_gemma":[0.9950376,0.0024982572,0.00057690026,0.00079079706,0.00097376277,0.00012267314],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0050358684,0.0003166614,0.00048746332,0.00011708439,0.000608278,0.00047032343,0.0010220992,0.0001884002,0.0002918755],"category_scores_gemma":[0.004493813,0.0002696184,0.0002091481,0.00037325823,0.00027532646,0.0009351127,0.00084799353,0.0011788992,0.0000067257292],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011005426,0.0002679261,0.0033430369,0.00015858245,0.00014163474,0.000017239949,0.0046639806,0.17426969,0.00054470287,0.73810625,0.000016140686,0.07836075],"study_design_scores_gemma":[0.0007923249,0.00016642272,0.0023374392,0.0003276605,0.000028364972,0.000024478299,0.0016029411,0.9389766,0.0025437716,0.051725626,0.0010150423,0.00045931723],"about_ca_topic_score_codex":0.00024801394,"about_ca_topic_score_gemma":0.000014917453,"teacher_disagreement_score":0.80472606,"about_ca_system_score_codex":0.000232891,"about_ca_system_score_gemma":0.00041023013,"threshold_uncertainty_score":0.9999756},"labels":[],"label_agreement":null},{"id":"W4225775465","doi":"10.1109/qrs54544.2021.00106","title":"Analyzing the Impact of Cyberattacks on Industrial Control Systems using Timed Automata","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Smart Grid Security and Resilience","field":"Engineering","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Exploit; Computer science; Industrial control system; Spoofing attack; Automaton; Computer security; Model checking; Critical infrastructure; Vulnerability (computing); Ransomware; Control (management); Malware; Theoretical computer science; Artificial intelligence","score_opus":0.057983503388506816,"score_gpt":0.33548058703050765,"score_spread":0.2774970836420008,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4225775465","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9919842,0.00023659974,0.0027084565,0.00033405406,0.002157403,0.00037035174,0.00083787524,0.0001213233,0.0012497648],"genre_scores_gemma":[0.99902344,0.000268984,0.00006563296,0.00004731373,0.0004492633,0.000018827639,0.000060767285,0.000021586413,0.000044186458],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.996952,0.0006273448,0.0008357504,0.00054488366,0.00068664574,0.0003533691],"domain_scores_gemma":[0.9973027,0.0010414955,0.00021048739,0.00065885455,0.000620674,0.0001657956],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012891988,0.0003565427,0.0006146236,0.00012497559,0.00020226749,0.00028019975,0.00049260614,0.00032653354,0.00032768428],"category_scores_gemma":[0.0011386451,0.00027826393,0.00029066685,0.00031495898,0.00028951842,0.00023960481,0.000089762936,0.0008431909,0.0000178282],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0017353883,0.0028778506,0.14690807,0.0012770047,0.0030934664,0.00013970422,0.009290051,0.7196962,0.010899285,0.091006525,0.0043659625,0.008710484],"study_design_scores_gemma":[0.0046660253,0.00054128026,0.019668031,0.0014238876,0.00016778005,0.00007999068,0.0021601282,0.95754075,0.0053284704,0.0062178066,0.00082425703,0.0013815679],"about_ca_topic_score_codex":0.0007933878,"about_ca_topic_score_gemma":0.000075102376,"teacher_disagreement_score":0.23784459,"about_ca_system_score_codex":0.00026598762,"about_ca_system_score_gemma":0.00030777158,"threshold_uncertainty_score":0.999967},"labels":[],"label_agreement":null},{"id":"W4225807728","doi":"10.1109/qrs54544.2021.00063","title":"Predictors of Software Metric Correlation: A Non-parametric Analysis","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Cyclomatic complexity; Metric (unit); Code (set theory); Correlation; Computer science; Code refactoring; Source lines of code; Code coverage; Software quality; Software metric; Software; Statistics; Programming language; Mathematics; Software development; Set (abstract data type); Engineering","score_opus":0.034565574273561835,"score_gpt":0.32378986542613564,"score_spread":0.2892242911525738,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4225807728","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44776836,0.00019690098,0.5484849,0.0006628959,0.0016047556,0.00029470303,0.00028531605,0.00023984669,0.00046228565],"genre_scores_gemma":[0.9792657,0.0003219086,0.019604458,0.000116758354,0.00012981467,0.000046848047,0.00014667094,0.000019192874,0.00034867914],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.994462,0.00053890294,0.0011027298,0.0013604733,0.0020596306,0.00047624018],"domain_scores_gemma":[0.99074787,0.0045131687,0.0003798471,0.0014979496,0.0025176012,0.00034358716],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0020637622,0.00039894925,0.00083643885,0.0012767735,0.00016979079,0.00039652936,0.0014704284,0.00032792505,0.0005523082],"category_scores_gemma":[0.0142199695,0.0004176641,0.0004495476,0.005286693,0.00025516094,0.00064707565,0.0006530428,0.0008008901,0.00004846042],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009384055,0.0012513063,0.9289014,0.00041020082,0.0014196191,0.000060065082,0.0021842073,0.009417396,0.0000667761,0.045492824,0.00050659693,0.010195806],"study_design_scores_gemma":[0.0022335937,0.00060176104,0.77436626,0.0003270089,0.00041407664,0.00004355197,0.0005918821,0.18763442,0.0030899046,0.027372994,0.0016523,0.0016722485],"about_ca_topic_score_codex":0.0003311195,"about_ca_topic_score_gemma":0.00007642589,"teacher_disagreement_score":0.5314973,"about_ca_system_score_codex":0.0002728913,"about_ca_system_score_gemma":0.000570382,"threshold_uncertainty_score":0.9998275},"labels":[],"label_agreement":null},{"id":"W4226069469","doi":"10.1109/qrs54544.2021.00012","title":"Analyzing Structural Security Posture to Evaluate System Design Decisions","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Carleton University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Leverage (statistics); Computer science; Computer security; Software security assurance; Security testing; Computer security model; Identification (biology); Resource (disambiguation); Security information and event management; Secure coding; Security service; Security through obscurity; Software; Cloud computing security; Risk analysis (engineering); Information security; Artificial intelligence; Cloud computing; Business","score_opus":0.07041286786716698,"score_gpt":0.3641071605009214,"score_spread":0.2936942926337544,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226069469","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.60473675,0.000154493,0.38768926,0.0034616631,0.0023099082,0.00057431374,0.00025721523,0.00046439978,0.00035198906],"genre_scores_gemma":[0.9713441,0.00009804448,0.027768143,0.00030169537,0.00022658678,0.00006870931,0.000057076235,0.000025970978,0.00010968789],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9935381,0.0012768738,0.00091520365,0.0016908584,0.0019114633,0.00066750543],"domain_scores_gemma":[0.9911456,0.0039040006,0.00018887738,0.0015762708,0.0025853554,0.00059988146],"candidate_categories":["metaresearch","metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0032788066,0.00048739105,0.0006546951,0.00031870385,0.00039904224,0.0010718982,0.0018460538,0.00030275676,0.00025286607],"category_scores_gemma":[0.011013911,0.00047949495,0.00023913535,0.0009231057,0.00014355946,0.0006523737,0.0010228414,0.0009929362,0.00013467767],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00077345973,0.001183837,0.08302734,0.0011223472,0.0008809125,0.00070682826,0.020869875,0.019233625,0.0027200987,0.81965595,0.0035814296,0.046244293],"study_design_scores_gemma":[0.004985053,0.0013510628,0.15029174,0.003138156,0.00015804867,0.0006048137,0.0044297306,0.56066364,0.012542643,0.2527576,0.004005407,0.0050721057],"about_ca_topic_score_codex":0.00023981299,"about_ca_topic_score_gemma":0.00011041704,"teacher_disagreement_score":0.56689835,"about_ca_system_score_codex":0.00055197254,"about_ca_system_score_gemma":0.00066271465,"threshold_uncertainty_score":0.9999651},"labels":[],"label_agreement":null},{"id":"W4226205863","doi":"10.1109/qrs54544.2021.00076","title":"Vulnerability Analysis of Similar Code","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Engineering Research","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"New York Institute of Technology","funders":"","keywords":"Computer science; Secure coding; Vulnerability (computing); Software security assurance; Code (set theory); Computer security; Code review; Static program analysis; Domain (mathematical analysis); Software; Scripting language; Source code; Vulnerability assessment; Security bug; Application security; Software engineering; Information security; Programming language; Software development; Security service; Mathematics","score_opus":0.060345565260924376,"score_gpt":0.3613439035872422,"score_spread":0.30099833832631784,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226205863","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8105078,0.00012708448,0.18197146,0.0038432607,0.0011490809,0.0002782122,0.00077801954,0.00026708934,0.0010780239],"genre_scores_gemma":[0.98985684,0.00023932634,0.009191988,0.0002687005,0.00007735227,0.000033270266,0.000120891295,0.00001395305,0.0001976668],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.99488324,0.00079894724,0.00097116694,0.0013565675,0.0015596951,0.00043037834],"domain_scores_gemma":[0.9923891,0.0032225258,0.00024457354,0.001711118,0.0021475514,0.00028514073],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0027145941,0.0003411519,0.0008067535,0.0003942791,0.00015110518,0.00032217853,0.0014200491,0.00025099836,0.0008414883],"category_scores_gemma":[0.008373417,0.00035489444,0.00040131505,0.0015022834,0.00033213067,0.00047982103,0.00065329025,0.00071376556,0.000024507159],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022952688,0.0031315975,0.49637464,0.0008107368,0.0025807947,0.000108140135,0.0066157645,0.008281243,0.0014897177,0.46040365,0.00074981165,0.019224405],"study_design_scores_gemma":[0.0024190075,0.00054472015,0.5289764,0.00037859354,0.00045013276,0.00003116604,0.00110757,0.29463494,0.016058693,0.14740798,0.005687853,0.0023029656],"about_ca_topic_score_codex":0.00030191772,"about_ca_topic_score_gemma":0.00032126709,"teacher_disagreement_score":0.31299564,"about_ca_system_score_codex":0.00021051579,"about_ca_system_score_gemma":0.0004377986,"threshold_uncertainty_score":0.9999795},"labels":[],"label_agreement":null},{"id":"W4226207655","doi":"10.1109/qrs54544.2021.00118","title":"Understanding the Resilience of Neural Network Ensembles against Faulty Training Data","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Resilience (materials science); Machine learning; Artificial intelligence; Artificial neural network; Training set; Ensemble learning; Entropy (arrow of time); Training (meteorology); Ensemble forecasting; Data mining","score_opus":0.21383400830963548,"score_gpt":0.3701135125969131,"score_spread":0.15627950428727763,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226207655","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.15805149,0.00013802854,0.8268261,0.007924349,0.0024940236,0.0002800206,0.00020785745,0.00015063096,0.0039274963],"genre_scores_gemma":[0.987369,0.00021582126,0.011271614,0.0006608759,0.00028187566,0.000007805702,0.00010530332,0.000012783786,0.000074945194],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9953601,0.0011873402,0.00079368503,0.0011570269,0.0010827096,0.00041918526],"domain_scores_gemma":[0.99491453,0.0022725898,0.00043370383,0.0016957717,0.0005479175,0.00013545908],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0031518573,0.00029505533,0.00045782462,0.00006685985,0.00048624523,0.00043283618,0.0025902495,0.00016455346,0.00009706607],"category_scores_gemma":[0.003878123,0.0002543586,0.00012183142,0.0004611015,0.00048907625,0.00083982205,0.0015583977,0.0008652725,0.0000053578906],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011197565,0.00022694147,0.01488284,0.0001496199,0.00012214646,0.00004566487,0.009562735,0.06821425,0.00019424796,0.8874332,0.00049430644,0.018562103],"study_design_scores_gemma":[0.00080730114,0.000103621554,0.0050466238,0.00039628512,0.00002955314,0.000032926542,0.0070699356,0.7965906,0.00017318196,0.18788879,0.0012145466,0.0006465896],"about_ca_topic_score_codex":0.00013234025,"about_ca_topic_score_gemma":0.00028575657,"teacher_disagreement_score":0.82931745,"about_ca_system_score_codex":0.00014111551,"about_ca_system_score_gemma":0.00041958856,"threshold_uncertainty_score":0.9999909},"labels":[],"label_agreement":null},{"id":"W4226466002","doi":"10.1109/qrs54544.2021.00093","title":"MINTS: Unsupervised Temporal Specifications Miner","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Software Testing and Debugging Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Correctness; Computer science; Debugging; Task (project management); Finite-state machine; Scalability; Frame (networking); Software; State (computer science); Trie; Software bug; Software system; Artificial intelligence; Programming language; Data structure; Engineering; Database; Systems engineering","score_opus":0.10583280632417456,"score_gpt":0.34160529800087,"score_spread":0.2357724916766954,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4226466002","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.40136823,0.0004151687,0.52641046,0.03800658,0.004964688,0.00098605,0.00078117015,0.008023242,0.019044405],"genre_scores_gemma":[0.918654,0.00023520958,0.07834878,0.0013177736,0.000257179,0.00007159985,0.00018018586,0.000021283171,0.0009139547],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9961213,0.00052292855,0.0007968613,0.0012755813,0.0009010931,0.0003822579],"domain_scores_gemma":[0.99555457,0.0010368119,0.00023693804,0.0014185075,0.0014956987,0.00025748266],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0012454136,0.00036439172,0.0004521011,0.00016626061,0.00027574168,0.00064305397,0.0012392136,0.00023888022,0.0005097736],"category_scores_gemma":[0.0030449005,0.00036953454,0.00019815241,0.00046324808,0.00027157555,0.0005498744,0.00045026492,0.0005499446,0.00010317806],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011431859,0.002703737,0.11658213,0.00031804232,0.00018768155,0.00014084284,0.0056962627,0.0000424138,0.00073562935,0.8077672,0.023597732,0.04211399],"study_design_scores_gemma":[0.0015968403,0.00033127778,0.029756969,0.0006098012,0.000041615225,0.00014020501,0.0006032791,0.024352323,0.0046452438,0.905537,0.030546082,0.001839338],"about_ca_topic_score_codex":0.00041341467,"about_ca_topic_score_gemma":0.00012913396,"teacher_disagreement_score":0.5172858,"about_ca_system_score_codex":0.00015010883,"about_ca_system_score_gemma":0.00039474486,"threshold_uncertainty_score":0.99987566},"labels":[],"label_agreement":null}]}