{"id":"W3209014031","doi":"10.1109/iccvw54120.2021.00462","title":"SVEA: A Small-scale Benchmark for Validating the Usability of Post-hoc Explainable AI Solutions in Image and Signal Recognition","year":2021,"lang":"en","type":"article","venue":"","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Usability; Computer science; Benchmark (surveying); Scale (ratio); SIGNAL (programming language); Artificial intelligence; Computer vision; Image (mathematics); Post hoc; Speech recognition; Pattern recognition (psychology); Human–computer interaction; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001407974,0.0001152293,0.0001753487,0.00007154659,0.0003077872,0.0001749172,0.0003451569,0.00005827543,0.0001299405],"category_scores_gemma":[0.0005134019,0.00009696157,0.00007467723,0.0005699146,0.0001314943,0.0007528663,0.0003296007,0.0001346423,0.000006963169],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005939602,"about_ca_system_score_gemma":0.0001753642,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005035978,"about_ca_topic_score_gemma":0.002378022,"domain_scores_codex":[0.9984223,0.0001838663,0.0004259943,0.0004270883,0.0001517496,0.0003890107],"domain_scores_gemma":[0.9979925,0.0008396231,0.00009024788,0.0004089812,0.0006094241,0.00005921473],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0002302163,0.001878798,0.005463608,0.0006473775,0.00008055547,0.00006059028,0.02415592,0.001314584,0.4771856,0.1115039,0.001941923,0.3755369],"study_design_scores_gemma":[0.0003891357,0.0004437859,0.001861687,0.0001030191,0.00002027481,0.00003709801,0.01265217,0.1973677,0.6258143,0.1605944,0.0003406768,0.0003758181],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4518314,0.000115863,0.5407245,0.00387875,0.00009066115,0.0004802713,0.00001608199,0.00004384453,0.002818718],"genre_scores_gemma":[0.8851453,0.00001523682,0.1139372,0.0004659606,0.00003530821,0.0001336063,0.00001557785,0.000007799228,0.0002440067],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.4333139,"threshold_uncertainty_score":0.3953978,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04936387042831403,"score_gpt":0.2791279378312783,"score_spread":0.2297640674029643,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}