{"id":"W4408022558","doi":"10.2196/63058","title":"Advancing Clinical Chatbot Validation Using AI-Powered Evaluation With a New 3-Bot Evaluation System: Instrument Validation Study","year":2025,"lang":"en","type":"article","venue":"JMIR Nursing","topic":"AI in Service Interactions","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Chatbot; Health care; Test (biology); Computer science; Economic shortage; Automation; Artificial intelligence; Engineering","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04412152,0.00102171,0.0006185598,0.001115436,0.0008431495,0.001574649,0.00156476,0.001265789,0.002955512],"category_scores_gemma":[0.08108528,0.0005469577,0.0008173784,0.0005571327,0.001599636,0.002133542,0.002126253,0.001190949,0.001266952],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00206996,"about_ca_system_score_gemma":0.003215524,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001787235,"about_ca_topic_score_gemma":0.002709294,"domain_scores_codex":[0.9752972,0.01672746,0.001752215,0.001914544,0.003431428,0.000877123],"domain_scores_gemma":[0.9022262,0.06110313,0.004808042,0.008335175,0.02167036,0.001857082],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.007560478,0.02234757,0.2441789,0.003884973,0.0007902759,0.00079389,0.02420496,0.01589719,0.06092879,0.007601006,0.01721341,0.5945986],"study_design_scores_gemma":[0.006841779,0.0599147,0.4542392,0.002341815,0.001450585,0.002399773,0.01094825,0.2991273,0.08164753,0.009684758,0.07070669,0.0006976917],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8351511,0.0002453856,0.1368627,0.0004170891,0.00019104,0.01822279,0.0006344354,0.002091802,0.006183756],"genre_scores_gemma":[0.7756741,0.0001637145,0.1957566,0.0008232356,0.00006828262,0.02279174,0.001249065,0.0002769363,0.003196388],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9558785,"threshold_uncertainty_score":0.2333397,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08517371719177196,"score_gpt":0.4622386755596723,"score_spread":0.3770649583679003,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}