{"id":"W4226137778","doi":"10.1109/tse.2022.3201209","title":"Flakify: A Black-Box, Language Model-Based Predictor for Flaky Tests","year":2022,"lang":"en","type":"article","venue":"IEEE Transactions on Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada; Mitacs; Canada Research Chairs; Western Canada Research Grid; Compute Canada","keywords":"Computer science; Debugging; Code coverage; Code (set theory); Set (abstract data type); Source code; White-box testing; Programming language; Software; Overhead (engineering); Black box; Test (biology); Machine learning; Artificial intelligence; Software development; Software construction","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0003637219,0.0003157754,0.0002584973,0.0005749749,0.0003326743,0.0001078081,0.001062895,0.00008419794,0.00004864874],"category_scores_gemma":[0.0001928573,0.0003763529,0.0002283105,0.0008831326,0.00002737217,0.0002842188,0.00001667911,0.0006556029,0.0000202773],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004042498,"about_ca_system_score_gemma":0.0002182465,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008133142,"about_ca_topic_score_gemma":0.000002290183,"domain_scores_codex":[0.99772,0.00003092653,0.0002877488,0.0006157823,0.0006844972,0.0006610624],"domain_scores_gemma":[0.9975183,0.001239119,0.00004135358,0.0008935883,0.00008429398,0.000223341],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001951599,0.0001104738,0.00003470486,0.00009179872,0.00003732978,0.00001962381,0.0005371762,0.9944057,0.001262673,0.00007815384,0.0003563368,0.003046443],"study_design_scores_gemma":[0.0007840076,0.0002240602,0.0001198014,0.00003215539,0.00001520836,0.0000132341,0.00001762621,0.9863634,0.01111075,0.00001961468,0.0009075382,0.0003925785],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02758115,0.00006694455,0.9677375,0.0000905668,0.001028588,0.0005997883,0.0001872871,0.002704164,0.000004006464],"genre_scores_gemma":[0.787229,0.000001626101,0.2111709,0.00009261142,0.00006244829,0.001024038,0.000009407239,0.00008939544,0.0003206378],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7596478,"threshold_uncertainty_score":0.9998689,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01371496591471183,"score_gpt":0.248779662007871,"score_spread":0.2350646960931591,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}