{"id":"W2973196970","doi":"10.1186/s13059-019-1794-0","title":"Reproducible biomedical benchmarking in the cloud: lessons from crowd-sourced data challenges","year":2019,"lang":"en","type":"letter","venue":"Genome biology","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":31,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; Ontario Institute for Cancer Research","funders":"National Institute of General Medical Sciences; National Institutes of Health; National Cancer Institute; Instituto Tecnológico de Costa Rica; University of California, Santa Cruz","keywords":"Benchmarking; Cloud computing; Data science; Computer science; Software; Virtualization; Key (lock); Data sharing; Operating system","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.05263668,0.0005959069,0.001004398,0.0009510299,0.00323922,0.006913827,0.003083521,0.01685206,0.004328122],"category_scores_gemma":[0.1831374,0.0005543396,0.001199285,0.001409287,0.00779921,0.01106587,0.005680698,0.02326979,0.003547567],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.005336932,"about_ca_system_score_gemma":0.00805595,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004019818,"about_ca_topic_score_gemma":0.005060075,"domain_scores_codex":[0.9659984,0.01888885,0.002539498,0.002934256,0.007978383,0.001660641],"domain_scores_gemma":[0.7369424,0.2016777,0.006521629,0.01520086,0.03037938,0.009277947],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001304016,0.00006828721,0.002259697,0.0003775814,0.0000692831,0.001183686,0.0007309213,0.001671598,0.00046,0.05159497,0.8187107,0.1227429],"study_design_scores_gemma":[0.0001327104,0.00009907866,0.001523752,0.001357705,0.00003693581,0.001327019,0.001403411,0.006420625,0.0009377149,0.353682,0.6329495,0.0001295284],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.000665891,0.003495325,0.005336125,0.9832485,0.005417984,0.00001916452,0.00009234464,0.00008482793,0.001639771],"genre_scores_gemma":[0.04672427,0.009386314,0.01901686,0.874608,0.04670728,0.0002840897,0.0003554395,0.0003170003,0.002600689],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.9473633,"threshold_uncertainty_score":0.2783727,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4717548244814443,"score_gpt":0.4421544353624705,"score_spread":0.02960038911897384,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}