{"id":"W4353015179","doi":"10.1038/s41592-023-02150-0","title":"Understanding metric-related pitfalls in image analysis validation","year":2024,"lang":"en","type":"review","venue":"Nature Methods","topic":"Radiomics and Machine Learning in Medical Imaging","field":"Medicine","cited_by":175,"is_retracted":false,"has_abstract":false,"ca_institutions":"Sunnybrook Health Science Centre; Vector Institute; Princess Margaret Cancer Centre; Mila - Quebec Artificial Intelligence Institute; University of Toronto; University Health Network; McGill University","funders":"National Cancer Institute; Centre For Medical Engineering, King’s College London; Engineering and Physical Sciences Research Council; Natural Sciences and Engineering Research Council of Canada; Sydäntutkimussäätiö; National Institute of General Medical Sciences; Bundesministerium für Bildung und Forschung; Nederlandse Organisatie voor Wetenschappelijk Onderzoek; Cancer Research UK; Wellcome Trust; Innosuisse - Schweizerische Agentur für Innovationsförderung; Alzheimer's Society; NIH Clinical Center; Agence Nationale de la Recherche; Schweizerischer Nationalfonds zur Förderung der Wissenschaftlichen Forschung; Oulun Yliopisto; Silicon Valley Community Foundation; European Commission; National Institutes of Health; National Institute of Neurological Disorders and Stroke; Canadian Institute for Advanced Research; Royal Academy of Engineering; Novo Nordisk Fonden; Foundation for Cardiovascular Research; National Science Foundation","keywords":"Computer science; Data science; Multidisciplinary approach; Strengths and weaknesses; Bridging (networking); Metric (unit); Process (computing); Key (lock); Information retrieval; Artificial intelligence; Data mining; Psychology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","research_integrity"],"consensus_categories":["research_integrity"],"category_scores_codex":[0.005831076,0.0004917638,0.002978103,0.004719186,0.00005169485,0.0001224069,0.0002465522,0.001450061,0.0001904423],"category_scores_gemma":[0.00399896,0.0003458612,0.001383962,0.0101567,0.00007470165,0.00007134234,0.0001079179,0.006985373,0.00004043831],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001057578,"about_ca_system_score_gemma":0.0002655437,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002030095,"about_ca_topic_score_gemma":0.000001501655,"domain_scores_codex":[0.9959958,0.001345264,0.0009877256,0.000781908,0.0005052117,0.000384045],"domain_scores_gemma":[0.9976439,0.001254962,0.0003287626,0.0005243668,0.0000568685,0.0001911059],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000005329237,0.00005097131,0.00003212319,0.01224919,0.004059251,0.0003765656,0.00009223287,0.00001261063,0.000007554092,0.002391539,0.001601748,0.9791209],"study_design_scores_gemma":[0.0003747074,0.00003530136,0.00001928493,0.01013728,0.04467811,0.0001344259,0.00003069389,0.00569394,0.000003865122,0.001954436,0.9364929,0.0004450981],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[9.955234e-7,0.8725703,0.1200321,0.0003698906,0.0009342761,0.0004944394,0.000007912083,0.0001270277,0.005463077],"genre_scores_gemma":[0.0000220781,0.8435957,0.1545174,0.0001144297,0.0001917789,0.000021708,0.0003597731,0.00009423847,0.00108298],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9786758,"threshold_uncertainty_score":0.9998993,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.104001752503866,"score_gpt":0.5138637104579991,"score_spread":0.409861957954133,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}