{"id":"W4403532992","doi":"10.1016/j.knosys.2024.112477","title":"Backtest overfitting in the machine learning era: A comparison of out-of-sample testing methods in a synthetic controlled environment","year":2024,"lang":"en","type":"article","venue":"Knowledge-Based Systems","topic":"Statistical Methods and Inference","field":"Mathematics","cited_by":8,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Toronto; York University","funders":"","keywords":"Overfitting; Sample (material); Artificial intelligence; Computer science; Machine learning; Artificial neural network","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.008040308,0.0002349664,0.001126996,0.0002356102,0.00005299604,0.00005275231,0.0002500195,0.0001002803,0.00004030123],"category_scores_gemma":[0.01875774,0.0001589945,0.0001246065,0.0004616644,0.00008523381,0.00002865927,0.00005279217,0.0005619174,0.00001045309],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001080368,"about_ca_system_score_gemma":0.0001072069,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003443646,"about_ca_topic_score_gemma":0.00006715968,"domain_scores_codex":[0.9942212,0.003416608,0.00145391,0.0003253867,0.0002790779,0.0003038736],"domain_scores_gemma":[0.9282026,0.07103918,0.0003451509,0.0003082207,0.00006147383,0.00004338973],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0007326368,0.005251128,0.21393,0.0290834,0.0004752855,0.00008985923,0.05597883,0.02079699,0.05908189,0.2987156,0.0001572597,0.3157072],"study_design_scores_gemma":[0.001519853,0.0002506128,0.0004458756,0.003079019,0.00008894214,0.000003195461,0.001207316,0.9859477,0.001097652,0.005847303,0.0003263433,0.0001861844],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04693802,0.005882147,0.9414631,0.00004251847,0.000543365,0.001313603,0.00003718528,0.00005912265,0.003720954],"genre_scores_gemma":[0.7654461,0.000002123205,0.2343184,0.000002383398,0.00004050541,0.0001373583,0.000002106743,0.00002540126,0.00002560175],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9651507,"threshold_uncertainty_score":0.9895077,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1717677044474834,"score_gpt":0.437279618389988,"score_spread":0.2655119139425046,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}