{"id":"W2973196970","doi":"10.1186/s13059-019-1794-0","title":"Reproducible biomedical benchmarking in the cloud: lessons from crowd-sourced data challenges","year":2019,"lang":"en","type":"letter","venue":"Genome biology","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":31,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; Ontario Institute for Cancer Research","funders":"National Institute of General Medical Sciences; National Institutes of Health; National Cancer Institute; Instituto Tecnológico de Costa Rica; University of California, Santa Cruz","keywords":"Benchmarking; Cloud computing; Data science; Computer science; Software; Virtualization; Key (lock); Data sharing; Operating system","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["open_science","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01949045,0.0003538553,0.0007440003,0.0007637991,0.0001988711,0.0004553983,0.01268148,0.0006947804,0.000825143],"category_scores_gemma":[0.00348238,0.0002115735,0.000121877,0.0009135136,0.0004196193,0.0001079664,0.005589627,0.001383201,0.00189036],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005307842,"about_ca_system_score_gemma":0.0001397174,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005479194,"about_ca_topic_score_gemma":0.0001411282,"domain_scores_codex":[0.9906509,0.001701143,0.001093814,0.004480888,0.001265715,0.0008075279],"domain_scores_gemma":[0.981373,0.005104787,0.000542382,0.01284885,0.00008355793,0.00004743837],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000005767322,0.00003015796,0.000119959,0.000009330674,0.00003766984,0.0000818507,0.0006619169,0.000006124397,0.00004781052,0.0001088224,0.946786,0.05210463],"study_design_scores_gemma":[0.0001827413,0.00005477586,0.002617175,0.00002358647,0.00002188097,0.000009025645,0.0004355637,0.000394333,5.260145e-7,0.005153407,0.9908396,0.0002674106],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.003289771,0.005168705,0.001337468,0.9727071,0.01108596,0.0005075765,0.002077351,0.00005797798,0.003768107],"genre_scores_gemma":[0.03581195,0.001605121,0.003840016,0.8302422,0.06191861,0.00008143927,0.05579362,0.0001186986,0.01058834],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.1424649,"threshold_uncertainty_score":0.9988868,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4717548244814443,"score_gpt":0.4421544353624705,"score_spread":0.02960038911897384,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}