{"id":"W4394951047","doi":"10.1101/2024.04.17.24305928","title":"Assessing GPT-4’s Diagnostic Accuracy with Darker Skin Tones: Underperformance and Implications","year":2024,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Cutaneous Melanoma Detection and Management","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Medical diagnosis; Diagnostic accuracy; Medicine; Skin lesion; Dermatology; Univariate; Artificial intelligence; Computer science; Pathology; Multivariate statistics; Radiology; Machine learning","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02738349,0.0008455523,0.0006510985,0.001229546,0.0007633155,0.002600745,0.001458303,0.001386174,0.003472887],"category_scores_gemma":[0.2141466,0.00030554,0.000980429,0.0009096499,0.001465785,0.002455295,0.002367457,0.001724584,0.001589658],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001089899,"about_ca_system_score_gemma":0.000915999,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004646206,"about_ca_topic_score_gemma":0.004393131,"domain_scores_codex":[0.9730842,0.01741669,0.002073873,0.003521446,0.002873429,0.001030422],"domain_scores_gemma":[0.7780067,0.1858905,0.01303548,0.008349562,0.01226029,0.002457458],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.003024496,0.0002710989,0.8665752,0.001181295,0.0005968714,0.0005763495,0.003708254,0.004354626,0.003945468,0.0007426612,0.01244126,0.1025823],"study_design_scores_gemma":[0.0003330666,0.001798452,0.8193415,0.001296735,0.001007939,0.003482018,0.01337014,0.1125722,0.01508615,0.01177108,0.01967805,0.0002626825],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9722127,0.002633824,0.009334041,0.004756626,0.0004938763,0.0002166482,0.003309065,0.0007412009,0.006302173],"genre_scores_gemma":[0.9926823,0.0001865522,0.004520047,0.000486754,0.0001099664,0.00008865073,0.001415634,0.000079546,0.0004306423],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02738349,"threshold_uncertainty_score":0.1448195,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02680971138463495,"score_gpt":0.3154956831577606,"score_spread":0.2886859717731256,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}