{"id":"W3196967454","doi":"10.1109/qrs54544.2021.00116","title":"The Challenge of Reproducible ML: An Empirical Study on The Impact of Bugs","year":2021,"lang":"en","type":"article","venue":"2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"Polytechnique Montréal","funders":"","keywords":"Computer science; Determinism; Inference; Software bug; Reproducibility; Software; Machine learning; Artificial intelligence; Face (sociological concept); Causal inference; Software engineering; Programming language; Statistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004296357,0.0002069278,0.000329573,0.00006241138,0.0002549404,0.000238342,0.001260739,0.00009872305,0.0001404102],"category_scores_gemma":[0.004157856,0.0001276208,0.0001613238,0.0002909285,0.0002548271,0.0002703684,0.0003252844,0.0005532941,0.00001131124],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006738812,"about_ca_system_score_gemma":0.0003261841,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007344012,"about_ca_topic_score_gemma":0.000206536,"domain_scores_codex":[0.9956048,0.001693485,0.0006928632,0.0009374621,0.0008682088,0.0002031828],"domain_scores_gemma":[0.9946945,0.001601957,0.0003911926,0.002193066,0.001018599,0.000100636],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","study_design_scores_codex":[0.0004399213,0.007763709,0.2064069,0.0001180506,0.0003065909,0.00001040231,0.02587341,0.0003668543,0.0004438902,0.7220569,0.0008686363,0.03534469],"study_design_scores_gemma":[0.001460305,0.003968172,0.7161769,0.0002233738,0.00003921093,0.00001415212,0.008127679,0.04150166,0.001921782,0.2235272,0.002301148,0.0007384323],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9764558,0.00005922826,0.003946721,0.01578588,0.0004214596,0.0003392248,0.0001382421,0.00005814493,0.002795313],"genre_scores_gemma":[0.9991493,0.0001897501,0.0002789745,0.000104916,0.00008513359,0.00002570675,0.00003978865,0.000007313386,0.0001191643],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.50977,"threshold_uncertainty_score":0.5204227,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.114300681193202,"score_gpt":0.4201996151009927,"score_spread":0.3058989339077907,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}