{"id":"W3138114698","doi":"10.1126/scitranslmed.abb1655","title":"Reproducibility in machine learning for health research: Still a ways to go","year":2021,"lang":"en","type":"review","venue":"Science Translational Medicine","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":261,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; TD Bank Group; University of Toronto","funders":"National Institute of Mental Health; National Institutes of Health; Natural Sciences and Engineering Research Council of Canada; Mitacs; Canadian Institute for Advanced Research","keywords":"Machine learning; Computer science; Reproducibility; Artificial intelligence; Data science; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2013797,0.001363433,0.005982207,0.008566329,0.001615294,0.01036608,0.005461784,0.005649289,0.006552823],"category_scores_gemma":[0.3689412,0.001147749,0.004829325,0.009699056,0.007489639,0.01883062,0.004571622,0.009080673,0.002735195],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006651934,"about_ca_system_score_gemma":0.02685587,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004762426,"about_ca_topic_score_gemma":0.007062876,"domain_scores_codex":[0.8817101,0.06656253,0.01829412,0.005220602,0.02674728,0.001465404],"domain_scores_gemma":[0.3279195,0.5557236,0.02520296,0.02905215,0.05899969,0.003102021],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0002287417,0.00005836411,0.001918492,0.08457303,0.002244961,0.000181574,0.0008914325,0.0005196096,0.0005177053,0.03949217,0.09395212,0.7754217],"study_design_scores_gemma":[0.0001386374,0.000238986,0.003469699,0.1505424,0.002695818,0.000658909,0.0007371071,0.0005363527,0.0009330449,0.0822224,0.7576461,0.0001805998],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0001932523,0.951173,0.003624201,0.04004217,0.003426473,0.00007000411,0.0001435174,0.00008437941,0.001242967],"genre_scores_gemma":[0.01095184,0.940016,0.0135655,0.02678519,0.006352803,0.0004745653,0.0006157579,0.0001836737,0.001054693],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.7986203,"threshold_uncertainty_score":0.984841,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5508278289633713,"score_gpt":0.532768470116231,"score_spread":0.01805935884714038,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}