{"id":"W4388014118","doi":"10.1002/art.42737","title":"Doctor Versus Artificial Intelligence: Patient and Physician Evaluation of Large Language Model Responses to Rheumatology Patient Questions in a <scp>Cross‐Sectional</scp> Study","year":2023,"lang":"en","type":"article","venue":"Arthritis & Rheumatology","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":63,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Medicine; Readability; Rheumatology; Internal medicine; Likert scale; Chatbot; Cross-sectional study; Family medicine; Artificial intelligence; Pathology; Psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009703934,0.0002112592,0.0003203178,0.0006531678,0.0005278229,0.001012911,0.0003463036,0.0006108822,0.001784922],"category_scores_gemma":[0.02759805,0.000232939,0.0003328603,0.000557796,0.0006469687,0.0006621255,0.0007279204,0.0005797253,0.0003656313],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009278521,"about_ca_system_score_gemma":0.0009464338,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008857761,"about_ca_topic_score_gemma":0.01059613,"domain_scores_codex":[0.9927209,0.004862185,0.0004944493,0.000584318,0.001057453,0.000280674],"domain_scores_gemma":[0.96652,0.01860819,0.008665795,0.001030068,0.003135117,0.002040893],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0004522069,0.0002947189,0.9933451,0.00003380756,0.0000561438,0.0000639039,0.002815727,0.00007538234,0.0002568004,0.00001290337,0.0002162759,0.002377049],"study_design_scores_gemma":[0.00008817965,0.001818066,0.9925262,0.00001801899,0.00004653575,0.0002627213,0.003651912,0.0008882359,0.0003206627,0.00001966342,0.0003431727,0.00001663572],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9995692,0.00002213612,0.000070768,0.00004313833,0.000002893262,0.0000239806,0.00006168699,0.000002615041,0.0002035278],"genre_scores_gemma":[0.9995402,0.00001541573,0.0001448832,0.0001059386,0.000006623548,0.0000263332,0.00009711017,0.000001785334,0.00006176726],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.009703934,"threshold_uncertainty_score":0.0513199,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1205783328331111,"score_gpt":0.4459697096145198,"score_spread":0.3253913767814086,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}