{"id":"W4396894318","doi":"10.1038/s41746-024-01118-4","title":"Shortcut learning in medical AI hinders generalization: method for estimating AI model generalization without external data","year":2024,"lang":"en","type":"article","venue":"npj Digital Medicine","topic":"Machine Learning in Healthcare","field":"Computer Science","cited_by":65,"is_retracted":false,"has_abstract":true,"ca_institutions":"Princess Margaret Cancer Centre; Vector Institute; University Health Network; Toronto General Hospital; University of Toronto; Ted Rogers Centre for Heart Research","funders":"University of Toronto; Natural Sciences and Engineering Research Council of Canada; University Health Network; Canadian Institute for Advanced Research","keywords":"Generalization; Computer science; Artificial intelligence; Machine learning; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01407528,0.00184889,0.001581447,0.001521825,0.0007590519,0.001800699,0.003509683,0.003552423,0.002539052],"category_scores_gemma":[0.06603866,0.0007898554,0.001188724,0.0009866139,0.002081961,0.002337163,0.003600026,0.004658244,0.0007116273],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001753795,"about_ca_system_score_gemma":0.001979867,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005160414,"about_ca_topic_score_gemma":0.004636075,"domain_scores_codex":[0.9960235,0.001690358,0.0002730467,0.0009832812,0.0007836266,0.0002460993],"domain_scores_gemma":[0.9659258,0.02398908,0.002013389,0.00355373,0.003674694,0.0008433519],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0006633541,0.0003064026,0.01864457,0.0003545032,0.0004548404,0.0004083323,0.0003637901,0.6824563,0.005912156,0.01338774,0.009183505,0.2678644],"study_design_scores_gemma":[0.00001938503,0.00006394344,0.0008129691,0.00003129715,0.00001387689,0.00004831213,0.00001061989,0.9896421,0.001272134,0.00769685,0.0003781312,0.00001042553],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.04973098,0.0009121696,0.944509,0.0008269579,0.0001100108,0.0001678152,0.0002940477,0.002416437,0.001032512],"genre_scores_gemma":[0.7361838,0.0003550765,0.2567501,0.001117355,0.0002052735,0.0004696361,0.001546379,0.0005298569,0.002842442],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.01407528,"threshold_uncertainty_score":0.0744381,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06358721130492692,"score_gpt":0.4232754399995051,"score_spread":0.3596882286945782,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}