{"id":"W4400934506","doi":"10.1145/3680463","title":"An Empirical Study of Testing Machine Learning in the Wild","year":2024,"lang":"en","type":"article","venue":"ACM Transactions on Software Engineering and Methodology","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; York University; Polytechnique Montréal","funders":"","keywords":"Computer science; White-box testing; Regression testing; Oracle; Test strategy; Workflow; Quality assurance; Machine learning; Software reliability testing; Manual testing; Software quality; Empirical research; Non-regression testing; Software performance testing; Software engineering; Artificial intelligence; Software testing; Software; Software system; Software construction; Software development; Database; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001819359,0.0001320767,0.0002031112,0.0002851225,0.0001020214,0.00004758339,0.0005174559,0.00007182019,0.000003173845],"category_scores_gemma":[0.00182242,0.0001029555,0.00003041954,0.0007248658,0.00002298292,0.0001795619,0.00002074633,0.000866452,8.961425e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001691945,"about_ca_system_score_gemma":0.00002445197,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001251521,"about_ca_topic_score_gemma":0.0000168657,"domain_scores_codex":[0.9983522,0.0007828331,0.0002127751,0.0003322057,0.0001376881,0.0001823383],"domain_scores_gemma":[0.9927317,0.006790328,0.00002616424,0.0003952949,0.00002027098,0.00003624104],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000006558609,0.0000900351,0.006564916,0.00003320345,0.00002241308,0.00003502746,0.007643343,0.8109229,0.0001436511,0.000151376,7.030007e-7,0.1743859],"study_design_scores_gemma":[0.000327604,0.001113866,0.01420931,0.00005817844,0.00003362363,0.0001265069,0.0006632074,0.9825988,0.00006292856,0.0003399006,0.0002880776,0.0001779673],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1676306,0.000144251,0.8314117,0.0001549549,0.0002629766,0.00009541404,5.62385e-7,0.0002963232,0.000003245156],"genre_scores_gemma":[0.5721211,0.000004274119,0.4278005,0.00002603228,0.00001892639,0.00001321424,3.536942e-7,0.000009347805,0.000006196258],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.4044906,"threshold_uncertainty_score":0.4198405,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1074213531816717,"score_gpt":0.370455849240389,"score_spread":0.2630344960587173,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}