{"id":"W3138114698","doi":"10.1126/scitranslmed.abb1655","title":"Reproducibility in machine learning for health research: Still a ways to go","year":2021,"lang":"en","type":"review","venue":"Science Translational Medicine","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":261,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; TD Bank Group; University of Toronto","funders":"National Institute of Mental Health; National Institutes of Health; Natural Sciences and Engineering Research Council of Canada; Mitacs; Canadian Institute for Advanced Research","keywords":"Machine learning; Computer science; Reproducibility; Artificial intelligence; Data science; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04902624,0.0003300075,0.001394694,0.001691283,0.0006014879,0.0001410013,0.002553443,0.0000924406,0.00005275819],"category_scores_gemma":[0.01597384,0.0002644586,0.0001625317,0.01009595,0.0008563627,0.0005983405,0.0002796921,0.00100942,0.00006892021],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005772401,"about_ca_system_score_gemma":0.004956218,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003748395,"about_ca_topic_score_gemma":0.0006504644,"domain_scores_codex":[0.9908908,0.0006256737,0.001391121,0.003621255,0.002272446,0.001198701],"domain_scores_gemma":[0.9935359,0.002513828,0.0002373739,0.002217048,0.0009748687,0.0005209937],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000004555542,0.00005952394,0.000009893853,0.001460347,0.000004939489,0.000009797032,0.002907041,0.0003770521,0.000008468051,0.07206228,0.0002893494,0.9228067],"study_design_scores_gemma":[0.0001185345,0.0003945936,0.00002391082,0.008518963,0.000009345213,0.00002079529,0.0000682988,0.008790153,0.00001718879,0.01086529,0.9709039,0.0002690091],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.000001870435,0.7220171,0.2599781,0.0152752,0.0004654147,0.001799566,0.00001094807,0.00006015976,0.0003915733],"genre_scores_gemma":[0.0004187706,0.9434252,0.05408718,0.000641585,0.0004769167,0.0004882917,0.00007730649,0.00003733327,0.0003474414],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9706146,"threshold_uncertainty_score":0.9999807,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5508278289633713,"score_gpt":0.532768470116231,"score_spread":0.01805935884714038,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}