{"id":"W2986321504","doi":"10.1002/sta4.270","title":"Testing prediction algorithms as null hypotheses: Application to assessing the performance of deep neural networks","year":2020,"lang":"en","type":"article","venue":"Stat","topic":"Forecasting Techniques and Applications","field":"Decision Sciences","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Artificial neural network; Computer science; Algorithm; Machine learning; Bayesian probability; Artificial intelligence; Posterior predictive distribution; Predictive value; Set (abstract data type); Bayesian linear regression; Bayesian inference","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.09167732,0.002118352,0.001811141,0.0046585,0.001343153,0.003487789,0.00275104,0.003656261,0.001344711],"category_scores_gemma":[0.3032582,0.0006202763,0.001812464,0.003543704,0.005062164,0.005380351,0.003351798,0.003448133,0.0002961188],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001928221,"about_ca_system_score_gemma":0.002380693,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002110338,"about_ca_topic_score_gemma":0.001086837,"domain_scores_codex":[0.9563219,0.02927074,0.002461853,0.003801073,0.007491719,0.0006527475],"domain_scores_gemma":[0.5232214,0.4457664,0.01129659,0.01005389,0.008252004,0.00140977],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002078522,0.0006008523,0.09712192,0.0004945478,0.001555457,0.0004006937,0.0009628459,0.5742,0.003369639,0.05921882,0.002123519,0.2578731],"study_design_scores_gemma":[0.0001076412,0.0005551313,0.008380242,0.00008643798,0.00008537808,0.0001457624,0.0001914383,0.9348413,0.004326533,0.0507144,0.0005051776,0.00006047046],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3182779,0.00136068,0.6736641,0.001203016,0.0001149536,0.0002527252,0.0004039404,0.0008183586,0.003904336],"genre_scores_gemma":[0.891855,0.0002616407,0.1064568,0.0001760094,0.00008861896,0.0002459371,0.0004984814,0.0001121513,0.0003052614],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.09167732,"threshold_uncertainty_score":0.4848418,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1237782181359969,"score_gpt":0.3795740915048949,"score_spread":0.255795873368898,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}