{"id":"W3048030566","doi":"10.1145/3385732","title":"Automatic Detection of Usability Problem Encounters in Think-aloud Sessions","year":2020,"lang":"en","type":"article","venue":"ACM Transactions on Interactive Intelligent Systems","topic":"Software Engineering Research","field":"Computer Science","cited_by":32,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Usability; Think aloud protocol; Computer science; Human–computer interaction; Workflow; Usability lab; Usability inspection; Cognitive walkthrough; Pluralistic walkthrough; Usability engineering; Test (biology); Automation; Usability goals; Engineering; Database","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004413463,0.0002260609,0.0003510985,0.0003828036,0.0000795861,0.00009238387,0.001138676,0.0001054277,0.00005872666],"category_scores_gemma":[0.0007555846,0.0002106836,0.0001527195,0.001061329,0.00005816581,0.0006463496,0.00004118338,0.0006536943,0.0001153507],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004683583,"about_ca_system_score_gemma":0.00009082248,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004866051,"about_ca_topic_score_gemma":0.00004022396,"domain_scores_codex":[0.9976228,0.0002833147,0.0007087031,0.0005476594,0.0005287415,0.0003087186],"domain_scores_gemma":[0.996501,0.00217936,0.0001598197,0.0007987602,0.0002003866,0.0001606566],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005107837,0.003275793,0.005619736,0.002949412,0.0008762718,0.00009598779,0.1539627,0.4055805,0.09782233,0.0009128849,0.0002073203,0.3281863],"study_design_scores_gemma":[0.0005867891,0.001197484,0.004903093,0.001344869,0.00002483717,0.00004998199,0.007291684,0.7494932,0.233754,0.0004737149,0.0002887147,0.0005916465],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2317817,0.00003329768,0.766141,0.000383082,0.0007130732,0.000639996,0.000009743902,0.0002562287,0.00004186741],"genre_scores_gemma":[0.9962303,0.00001283559,0.003541987,0.00004156555,0.00002807158,0.00009782177,0.000001088643,0.00002023639,0.00002607863],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7644486,"threshold_uncertainty_score":0.859143,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03056204144816713,"score_gpt":0.2917840176591617,"score_spread":0.2612219762109945,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}