{"id":"W4353015179","doi":"10.1038/s41592-023-02150-0","title":"Understanding metric-related pitfalls in image analysis validation","year":2024,"lang":"en","type":"review","venue":"Nature Methods","topic":"Radiomics and Machine Learning in Medical Imaging","field":"Medicine","cited_by":175,"is_retracted":false,"has_abstract":false,"ca_institutions":"Sunnybrook Health Science Centre; Vector Institute; Princess Margaret Cancer Centre; Mila - Quebec Artificial Intelligence Institute; University of Toronto; University Health Network; McGill University","funders":"National Cancer Institute; Centre For Medical Engineering, King’s College London; Engineering and Physical Sciences Research Council; Natural Sciences and Engineering Research Council of Canada; Sydäntutkimussäätiö; National Institute of General Medical Sciences; Bundesministerium für Bildung und Forschung; Nederlandse Organisatie voor Wetenschappelijk Onderzoek; Cancer Research UK; Wellcome Trust; Innosuisse - Schweizerische Agentur für Innovationsförderung; Alzheimer's Society; NIH Clinical Center; Agence Nationale de la Recherche; Schweizerischer Nationalfonds zur Förderung der Wissenschaftlichen Forschung; Oulun Yliopisto; Silicon Valley Community Foundation; European Commission; National Institutes of Health; National Institute of Neurological Disorders and Stroke; Canadian Institute for Advanced Research; Royal Academy of Engineering; Novo Nordisk Fonden; Foundation for Cardiovascular Research; National Science Foundation","keywords":"Computer science; Data science; Multidisciplinary approach; Strengths and weaknesses; Bridging (networking); Metric (unit); Process (computing); Key (lock); Information retrieval; Artificial intelligence; Data mining; Psychology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01758649,0.001223252,0.002618315,0.00514716,0.0004996217,0.003899689,0.003471487,0.002967017,0.002541933],"category_scores_gemma":[0.04691631,0.0008217942,0.001014724,0.004316036,0.002897536,0.003697192,0.00169604,0.003183033,0.001845023],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001432196,"about_ca_system_score_gemma":0.003079217,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002846106,"about_ca_topic_score_gemma":0.003772659,"domain_scores_codex":[0.9933877,0.002385197,0.0007488271,0.0009411875,0.00243839,0.00009873974],"domain_scores_gemma":[0.9374735,0.05042943,0.001953018,0.00162156,0.008239551,0.0002829192],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0000391854,0.00002698469,0.0009239175,0.007833893,0.0002456983,0.00007495063,0.0001243561,0.002037919,0.0008905791,0.02147114,0.0143533,0.9519781],"study_design_scores_gemma":[0.000059606,0.0002599697,0.007049087,0.01653493,0.0009603247,0.003441872,0.0003694366,0.02595224,0.007724621,0.1612806,0.7760563,0.0003109016],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0005566195,0.956789,0.03795576,0.002086092,0.0005258812,0.00002541483,0.00009560435,0.000123879,0.001841728],"genre_scores_gemma":[0.01846688,0.8981065,0.0750922,0.002720114,0.002687447,0.0001217034,0.000519295,0.0001975077,0.002088308],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9824135,"threshold_uncertainty_score":0.09300733,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.104001752503866,"score_gpt":0.5138637104579991,"score_spread":0.409861957954133,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}