{"id":"W4220771431","doi":"10.2196/36610","title":"Ensuring Interrater Reliability When Evaluating Voice Assistants. Comment on “Evaluating Voice Assistants’ Responses to COVID-19 Vaccination in Portuguese: Quality Assessment”","year":2022,"lang":"en","type":"article","venue":"JMIR Human Factors","topic":"AI in Service Interactions","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Inter-rater reliability; Coronavirus disease 2019 (COVID-19); Reliability (semiconductor); Portuguese; Quality (philosophy); Severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2); 2019-20 coronavirus outbreak; Psychology; Applied psychology; Medicine; Audiology; Computer science; Speech recognition; Linguistics; Virology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.220676,0.0009640583,0.001022637,0.002875767,0.01075029,0.00520368,0.006746542,0.02192754,0.006452096],"category_scores_gemma":[0.6771256,0.001388383,0.002007821,0.002907132,0.01483685,0.008721998,0.006817801,0.0241123,0.004489879],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01289255,"about_ca_system_score_gemma":0.02338692,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.04443128,"about_ca_topic_score_gemma":0.05831364,"domain_scores_codex":[0.6844389,0.1620228,0.04179273,0.01398726,0.08827942,0.009478869],"domain_scores_gemma":[0.226296,0.5240097,0.03010401,0.01580106,0.1934399,0.01034941],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001015113,0.00002718163,0.004028998,0.0009595289,0.00005341298,0.0007084008,0.02552772,0.000180313,0.001460984,0.00656054,0.9431843,0.01720688],"study_design_scores_gemma":[0.0001440763,0.0002309449,0.01274576,0.00720491,0.0001465064,0.002036936,0.03176585,0.001818158,0.006632883,0.01605706,0.9208285,0.0003883104],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.002078474,0.0006643648,0.006564294,0.9739624,0.01085145,0.000312123,0.0003205809,0.0003618871,0.004884456],"genre_scores_gemma":[0.0384498,0.0005065683,0.00994162,0.9410151,0.004465185,0.001572032,0.0001481489,0.000256291,0.00364508],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.779324,"threshold_uncertainty_score":0.9610452,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2205183446838135,"score_gpt":0.4874524549307537,"score_spread":0.2669341102469402,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}