{"id":"W3209014031","doi":"10.1109/iccvw54120.2021.00462","title":"SVEA: A Small-scale Benchmark for Validating the Usability of Post-hoc Explainable AI Solutions in Image and Signal Recognition","year":2021,"lang":"en","type":"article","venue":"","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Usability; Computer science; Benchmark (surveying); Scale (ratio); SIGNAL (programming language); Artificial intelligence; Computer vision; Image (mathematics); Post hoc; Speech recognition; Pattern recognition (psychology); Human–computer interaction; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004989821,0.002323976,0.0005903443,0.003329544,0.0009576287,0.002139939,0.00375457,0.002386388,0.005149454],"category_scores_gemma":[0.03360838,0.0003345828,0.001533116,0.002249641,0.001192534,0.002927112,0.002382129,0.002345057,0.001818925],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001993192,"about_ca_system_score_gemma":0.00151037,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006563155,"about_ca_topic_score_gemma":0.01393297,"domain_scores_codex":[0.9949132,0.002163341,0.0004547641,0.001016625,0.001225076,0.0002269625],"domain_scores_gemma":[0.9801186,0.01391703,0.000752999,0.003308462,0.00151439,0.0003885445],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.00138543,0.002068899,0.01702851,0.00848409,0.0009930613,0.0007937766,0.001165785,0.1640173,0.01461738,0.02659979,0.1721289,0.5907171],"study_design_scores_gemma":[0.0005779676,0.001538827,0.0201271,0.0009949121,0.0002606677,0.001061806,0.001434293,0.7616841,0.03651959,0.04458536,0.1310209,0.0001944549],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3875875,0.01869106,0.4296448,0.00455876,0.001462737,0.002831541,0.06466899,0.0427494,0.04780518],"genre_scores_gemma":[0.4381492,0.00272331,0.4090851,0.0008328119,0.000176868,0.001755366,0.1392263,0.001942177,0.006108826],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.006563155,"threshold_uncertainty_score":0.026389,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04936387042831403,"score_gpt":0.2791279378312783,"score_spread":0.2297640674029643,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}