{"id":"W4362606928","doi":"10.1007/s10664-023-10291-1","title":"Bugs in machine learning-based systems: a faultload benchmark","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"ca_institutions":"York University; Polytechnique Montréal","funders":"","keywords":"Benchmark (surveying); Computer science; Software portability; Debugging; Software bug; Software quality; Usability; Software engineering; Software; Relevance (law); Benchmarking; Machine learning; Software development; Operating system","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007039775,0.0002905831,0.0003516111,0.0005229392,0.0001206272,0.0001536738,0.0008434209,0.0001569573,0.00001987771],"category_scores_gemma":[0.00250484,0.0002947534,0.0001002946,0.00235796,0.00002026182,0.000313035,0.0003927077,0.0008900735,0.0001793788],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000194934,"about_ca_system_score_gemma":0.00007151637,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001222378,"about_ca_topic_score_gemma":0.000005614222,"domain_scores_codex":[0.9977197,0.000131914,0.0004001363,0.0005817576,0.0004856059,0.0006808754],"domain_scores_gemma":[0.9982085,0.001052145,0.00007873627,0.0004441575,0.00004415579,0.0001723149],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003527264,0.00001630382,0.1067064,0.0000617139,0.000007855258,0.0001420184,0.0002092542,0.8916973,0.00002307676,0.0001167789,0.00019762,0.0008181003],"study_design_scores_gemma":[0.000433612,0.00004616864,0.03558346,0.0001018859,0.000003166442,0.000007534038,0.000009821753,0.9504763,0.00002459862,0.00002678515,0.0129568,0.0003298963],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04504996,0.0002503734,0.9502814,0.0004872016,0.0008180093,0.0001884838,0.000002384698,0.00288026,0.00004188015],"genre_scores_gemma":[0.9750091,0.000006373434,0.02435991,0.0001110174,0.0001543727,0.0000656977,0.00002956101,0.00005313547,0.0002107859],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9299592,"threshold_uncertainty_score":0.9999505,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01719232494440034,"score_gpt":0.2629710838120742,"score_spread":0.2457787588676739,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}