{"id":"W4400934506","doi":"10.1145/3680463","title":"An Empirical Study of Testing Machine Learning in the Wild","year":2024,"lang":"en","type":"article","venue":"ACM Transactions on Software Engineering and Methodology","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; York University; Polytechnique Montréal","funders":"","keywords":"Computer science; White-box testing; Regression testing; Oracle; Test strategy; Workflow; Quality assurance; Machine learning; Software reliability testing; Manual testing; Software quality; Empirical research; Non-regression testing; Software performance testing; Software engineering; Artificial intelligence; Software testing; Software; Software system; Software construction; Software development; Database; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04144661,0.0006240197,0.000637649,0.003764091,0.001312481,0.002928789,0.003052307,0.002028254,0.001749724],"category_scores_gemma":[0.2779034,0.0005819688,0.0006022694,0.003287403,0.006281191,0.006003357,0.002410691,0.002693112,0.0005065002],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002113375,"about_ca_system_score_gemma":0.001347041,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00278028,"about_ca_topic_score_gemma":0.00352821,"domain_scores_codex":[0.9191273,0.05422352,0.005041789,0.007251419,0.01261154,0.001744486],"domain_scores_gemma":[0.3444975,0.5579932,0.04166053,0.02975416,0.02252448,0.00357018],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0009310237,0.002655725,0.8257572,0.0009823916,0.0003511031,0.001285652,0.02523285,0.006494014,0.003006945,0.007651851,0.006281456,0.1193698],"study_design_scores_gemma":[0.0004261322,0.005110093,0.7277496,0.001901024,0.0003429111,0.003700802,0.04635414,0.137086,0.0148515,0.02536765,0.03673925,0.0003708078],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9877316,0.000347329,0.008864909,0.000633034,0.00002820898,0.0001947398,0.0004544414,0.0001128106,0.001632924],"genre_scores_gemma":[0.9904575,0.00009381988,0.007901549,0.0002529354,0.00002785567,0.0002373133,0.0006642945,0.0000744976,0.0002901366],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9585534,"threshold_uncertainty_score":0.2191933,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1074213531816717,"score_gpt":0.370455849240389,"score_spread":0.2630344960587173,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}