{"id":"W4323354733","doi":"10.1101/2023.03.02.23286705","title":"AI chatbots not yet ready for clinical use","year":2023,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Machine Learning in Healthcare","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"St. Thomas Hospital","funders":"King's College London","keywords":"Transformer; Generative grammar; Computer science; Artificial intelligence; Language model; Data science; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.003289976,0.0003562111,0.000661311,0.0002029151,0.0001543401,0.0003918216,0.002699129,0.0006066724,0.00001113315],"category_scores_gemma":[0.00427659,0.0003427695,0.0003241917,0.0002347905,0.00007084405,0.0001851041,0.003011563,0.001988019,0.0004300692],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006551254,"about_ca_system_score_gemma":0.0004045136,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004045222,"about_ca_topic_score_gemma":0.0001026157,"domain_scores_codex":[0.99573,0.0005983153,0.0009733295,0.001553373,0.0005390167,0.0006059377],"domain_scores_gemma":[0.9936043,0.002637187,0.0004449315,0.002682254,0.0003419293,0.0002893668],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00007056793,0.0001772975,0.8220937,0.001579602,0.0001631193,0.0001973711,0.001094871,0.001511641,0.00001494575,0.04835759,0.07269595,0.05204331],"study_design_scores_gemma":[0.0004778566,0.0002488102,0.549741,0.0003938551,0.00002876732,0.00000993592,0.000004354642,0.2657224,0.00003510368,0.01115202,0.1714302,0.0007558305],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2749305,0.0002406822,0.5337903,0.1381581,0.04377189,0.00380419,0.0003306358,0.00471711,0.0002565073],"genre_scores_gemma":[0.6498525,0.001044671,0.2921939,0.02849498,0.005297059,0.001682376,0.000548973,0.0004816121,0.02040391],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.374922,"threshold_uncertainty_score":0.9999024,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3724696250322858,"score_gpt":0.4850980656928995,"score_spread":0.1126284406606138,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}