{"id":"W4323354733","doi":"10.1101/2023.03.02.23286705","title":"AI chatbots not yet ready for clinical use","year":2023,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Machine Learning in Healthcare","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"St. Thomas Hospital","funders":"King's College London","keywords":"Transformer; Generative grammar; Computer science; Artificial intelligence; Language model; Data science; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01122561,0.001158739,0.000941302,0.0007820365,0.0007121711,0.002110532,0.002446466,0.002553759,0.03033021],"category_scores_gemma":[0.04635672,0.0004776876,0.000825914,0.0004574877,0.001168993,0.003228644,0.003055805,0.003069058,0.01110108],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001347446,"about_ca_system_score_gemma":0.001429516,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001565542,"about_ca_topic_score_gemma":0.001459873,"domain_scores_codex":[0.9935854,0.00441316,0.0002512279,0.0007533103,0.0007351122,0.0002618612],"domain_scores_gemma":[0.9639891,0.0274089,0.0007449409,0.003780965,0.002353384,0.001722819],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.005944188,0.00136733,0.008701161,0.002779537,0.0004381521,0.001097144,0.003042452,0.06127106,0.01973439,0.03145769,0.139535,0.724632],"study_design_scores_gemma":[0.0009634404,0.002765659,0.005190335,0.0008238409,0.0001838777,0.001013017,0.001549166,0.7180192,0.02675544,0.1023938,0.1400633,0.0002789745],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1366756,0.003971573,0.6894049,0.0204534,0.003474543,0.001922041,0.006036984,0.103881,0.03417997],"genre_scores_gemma":[0.7461192,0.0007828244,0.2170317,0.004481135,0.0006949764,0.001470566,0.007467605,0.00176492,0.02018704],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.03033021,"threshold_uncertainty_score":0.1014647,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3724696250322858,"score_gpt":0.4850980656928995,"score_spread":0.1126284406606138,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}