{"id":"W4220771431","doi":"10.2196/36610","title":"Ensuring Interrater Reliability When Evaluating Voice Assistants. Comment on “Evaluating Voice Assistants’ Responses to COVID-19 Vaccination in Portuguese: Quality Assessment”","year":2022,"lang":"en","type":"article","venue":"JMIR Human Factors","topic":"AI in Service Interactions","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Inter-rater reliability; Coronavirus disease 2019 (COVID-19); Reliability (semiconductor); Portuguese; Quality (philosophy); Severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2); 2019-20 coronavirus outbreak; Psychology; Applied psychology; Medicine; Audiology; Computer science; Speech recognition; Linguistics; Virology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","sts","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.009554721,0.0004791516,0.0005399476,0.0008197334,0.00147188,0.0004686076,0.001953358,0.0001125255,0.001091397],"category_scores_gemma":[0.002366196,0.0005018669,0.0001836662,0.001050753,0.00003435282,0.001116871,0.001917143,0.0012574,0.00004466909],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006815291,"about_ca_system_score_gemma":0.0005630986,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002087966,"about_ca_topic_score_gemma":0.001269409,"domain_scores_codex":[0.9891837,0.004682755,0.001589626,0.00146441,0.002350343,0.0007291595],"domain_scores_gemma":[0.9934512,0.003147282,0.0008860815,0.001754248,0.0003779356,0.0003832038],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001216922,0.004588483,0.7733459,0.0006357349,0.0002415359,0.0001198588,0.1253192,0.0229254,0.03379471,0.007138329,0.01676048,0.01391335],"study_design_scores_gemma":[0.0009881967,0.0010405,0.9734532,0.00009218149,0.00001663336,0.000005210293,0.005989039,0.010937,0.0004125223,0.002300791,0.004102847,0.0006618612],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9792985,0.00000892362,0.00690185,0.009624474,0.001113069,0.001439918,0.00009303986,0.0003825248,0.001137686],"genre_scores_gemma":[0.9886404,4.478607e-7,0.004427665,0.005603509,0.00005826139,0.0005625256,0.00008037563,0.00004450591,0.0005823366],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2001073,"threshold_uncertainty_score":0.999828,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2205183446838135,"score_gpt":0.4874524549307537,"score_spread":0.2669341102469402,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}