{"id":"W1555759181","doi":"10.1126/science.aaa9375","title":"The reusable holdout: Preserving validity in adaptive data analysis","year":2015,"lang":"en","type":"article","venue":"Science","topic":"Privacy-Preserving Technologies in Data","field":"Computer Science","cited_by":319,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada; Alfred P. Sloan Foundation; National Science Foundation","keywords":"Spurious relationship; Computer science; Data mining; Reuse; Set (abstract data type); Data set; Process (computing); Data validation; Data science; Data analysis; Machine learning; Artificial intelligence; Database","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","open_science"],"consensus_categories":["open_science"],"category_scores_codex":[0.009251626,0.0001274069,0.0001737374,0.0003768941,0.0004142286,0.0008151449,0.1481637,0.00005176873,0.00000293535],"category_scores_gemma":[0.07104897,0.00009088143,0.00002829555,0.009039273,0.0008723687,0.004209153,0.2843893,0.0002867257,0.00004308584],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002043223,"about_ca_system_score_gemma":0.0005458128,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004245198,"about_ca_topic_score_gemma":0.0006543613,"domain_scores_codex":[0.996671,0.0001396548,0.0002666267,0.001140664,0.001116947,0.0006650697],"domain_scores_gemma":[0.9695064,0.000408261,0.0001395817,0.02959096,0.0002055839,0.000149203],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005694888,0.0003903959,0.2115961,0.00001988006,0.000204811,0.0001857527,0.001997474,0.004533678,0.001365294,0.0722517,0.6413673,0.06603064],"study_design_scores_gemma":[0.00009241112,0.00002805894,0.01089436,0.000008243817,0.000009935201,0.00000256706,0.000140003,0.7992459,0.0005444916,0.1854455,0.00345161,0.000136946],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05136115,0.0009306584,0.8948042,0.03925518,0.001364485,0.0005170902,0.00006730941,0.001006713,0.01069316],"genre_scores_gemma":[0.7370363,0.00004283747,0.2626646,0.00008966339,0.00002383983,0.00001237318,0.000004945041,0.000004701925,0.000120688],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7947122,"threshold_uncertainty_score":0.936776,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2519054428928074,"score_gpt":0.3576285230451907,"score_spread":0.1057230801523834,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}