{"id":"W3196967454","doi":"10.1109/qrs54544.2021.00116","title":"The Challenge of Reproducible ML: An Empirical Study on The Impact of Bugs","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Determinism; Inference; Software bug; Reproducibility; Software; Machine learning; Artificial intelligence; Face (sociological concept); Causal inference; Software engineering; Programming language; Statistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.06669086,0.00105985,0.0009475901,0.002818651,0.002001445,0.003597773,0.00274429,0.002202389,0.001651537],"category_scores_gemma":[0.4104068,0.0008801651,0.001258306,0.003098654,0.005538492,0.006714257,0.004182488,0.00411251,0.0006767687],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002006831,"about_ca_system_score_gemma":0.002319408,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002043973,"about_ca_topic_score_gemma":0.00211706,"domain_scores_codex":[0.9004913,0.05497037,0.00985989,0.01189717,0.02127243,0.001508783],"domain_scores_gemma":[0.2436016,0.6268131,0.03933975,0.0681784,0.02000644,0.002060784],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00272831,0.003288246,0.6389276,0.003917778,0.001167562,0.001532126,0.01401108,0.04649514,0.01017469,0.01294614,0.02256151,0.2422498],"study_design_scores_gemma":[0.001002304,0.009343557,0.5457035,0.002373867,0.001263524,0.005699039,0.01212409,0.2760196,0.04647981,0.04405946,0.05524175,0.0006894502],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9469404,0.002493615,0.04050039,0.002162443,0.0001716096,0.0005465341,0.00173151,0.001688212,0.003765339],"genre_scores_gemma":[0.9807115,0.0003821003,0.01536056,0.0003324894,0.00009123667,0.0002599599,0.001932539,0.0004305322,0.0004990617],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9333091,"threshold_uncertainty_score":0.3526992,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.114300681193202,"score_gpt":0.4201996151009927,"score_spread":0.3058989339077907,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}