{"id":"W4401047029","doi":"10.1016/j.jss.2024.112159","title":"EvaluateXAI: A framework to evaluate the reliability and consistency of rule-based XAI techniques for software analytics tasks","year":2024,"lang":"en","type":"article","venue":"Journal of Systems and Software","topic":"Software Engineering Research","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Consistency (knowledge bases); Computer science; Analytics; Reliability (semiconductor); Data mining; Software; Reliability engineering; Artificial intelligence; Engineering; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004570485,0.0001903214,0.00047652,0.0003043791,0.0001296013,0.0003723715,0.0005763333,0.0001414981,0.000002609437],"category_scores_gemma":[0.007364826,0.000120581,0.0001585535,0.0005273788,0.0001156546,0.0002286562,0.0001568679,0.0003925057,0.000001061629],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001013637,"about_ca_system_score_gemma":0.000459023,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002880564,"about_ca_topic_score_gemma":0.000001031514,"domain_scores_codex":[0.9977468,0.0001642974,0.0007275951,0.0003149209,0.0007724125,0.0002739489],"domain_scores_gemma":[0.9927135,0.005276339,0.0001569334,0.0005565973,0.001099798,0.0001968016],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0008462284,0.0006966126,0.2024264,0.0345961,0.002260121,0.0005331429,0.01515651,0.05754972,0.001810917,0.02529636,0.03408361,0.6247443],"study_design_scores_gemma":[0.005123546,0.01408611,0.1112568,0.04696508,0.001692155,0.003171162,0.001563176,0.6320532,0.006634647,0.1095389,0.06425761,0.003657625],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08180678,0.007578806,0.9082361,0.001020328,0.000553616,0.0006426029,0.00002975207,0.0001298511,0.000002140005],"genre_scores_gemma":[0.7792181,0.00007268514,0.2203535,0.00007905453,0.0001613947,0.00005123639,6.563677e-7,0.00002495033,0.00003840066],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.6974114,"threshold_uncertainty_score":0.8816917,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03221025691242322,"score_gpt":0.3260664120274342,"score_spread":0.293856155115011,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}