{"id":"W1555759181","doi":"10.1126/science.aaa9375","title":"The reusable holdout: Preserving validity in adaptive data analysis","year":2015,"lang":"en","type":"article","venue":"Science","topic":"Privacy-Preserving Technologies in Data","field":"Computer Science","cited_by":319,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada; Alfred P. Sloan Foundation; National Science Foundation","keywords":"Spurious relationship; Computer science; Data mining; Reuse; Set (abstract data type); Data set; Process (computing); Data validation; Data science; Data analysis; Machine learning; Artificial intelligence; Database","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.08489823,0.001370594,0.002540037,0.002462661,0.003887621,0.008637605,0.007289757,0.004665397,0.001483451],"category_scores_gemma":[0.2596953,0.002040327,0.003708961,0.004067209,0.02159293,0.01951098,0.01920521,0.009892177,0.000772733],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002280036,"about_ca_system_score_gemma":0.006867436,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001846898,"about_ca_topic_score_gemma":0.001149183,"domain_scores_codex":[0.9043669,0.05780161,0.006307125,0.01242362,0.01685828,0.00224243],"domain_scores_gemma":[0.5639771,0.2284321,0.01618866,0.1788091,0.009704446,0.002888537],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.001195262,0.0003223451,0.01580575,0.0005449078,0.0005892274,0.001379153,0.007379299,0.07371246,0.01104076,0.6501496,0.004322907,0.2335584],"study_design_scores_gemma":[0.00009524445,0.0001641642,0.000719903,0.000128699,0.0001197753,0.0004690989,0.0002899286,0.1190881,0.009153663,0.8640919,0.005600617,0.00007894107],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.009802093,0.0002959645,0.9867975,0.001637856,0.00004800148,0.0001197835,0.00009591239,0.0004415906,0.0007612863],"genre_scores_gemma":[0.3286494,0.0006352473,0.665951,0.001629256,0.00033815,0.000679246,0.0003387022,0.0004567926,0.00132219],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.9151018,"threshold_uncertainty_score":0.4489902,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2519054428928074,"score_gpt":0.3576285230451907,"score_spread":0.1057230801523834,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}