{"id":"W2986321504","doi":"10.1002/sta4.270","title":"Testing prediction algorithms as null hypotheses: Application to assessing the performance of deep neural networks","year":2020,"lang":"en","type":"article","venue":"Stat","topic":"Forecasting Techniques and Applications","field":"Decision Sciences","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Artificial neural network; Computer science; Algorithm; Machine learning; Bayesian probability; Artificial intelligence; Posterior predictive distribution; Predictive value; Set (abstract data type); Bayesian linear regression; Bayesian inference","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007661847,0.00008516942,0.0001199752,0.00004095762,0.0002468509,0.0001479705,0.0005243209,0.00003566036,0.00001999303],"category_scores_gemma":[0.001075893,0.00005533138,0.00003741861,0.001147397,0.00006733683,0.0002013164,0.0001285014,0.0001313098,0.00003312158],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001702363,"about_ca_system_score_gemma":0.00001796708,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003316467,"about_ca_topic_score_gemma":0.000002448427,"domain_scores_codex":[0.9986348,0.00005052265,0.0003992547,0.0002919238,0.0004651012,0.0001583935],"domain_scores_gemma":[0.9983584,0.0007395097,0.0002233155,0.0003522432,0.0002434656,0.00008305347],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000008788189,0.00001731751,0.01391803,0.000005115005,0.000003187582,2.550643e-7,0.0005504778,0.07946551,0.002487134,0.0003114635,0.001041397,0.9021913],"study_design_scores_gemma":[0.00003880244,0.0001281599,0.01236298,0.000008239148,0.000005439562,0.000003986339,0.0003060199,0.9830521,0.0006772236,0.001133563,0.002227102,0.00005632885],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3706658,0.0000199671,0.6259232,0.001508382,0.00004478381,0.0003565174,0.000008298922,0.0001122463,0.001360848],"genre_scores_gemma":[0.9693588,0.000001825313,0.02984415,0.0005352796,0.0001514891,0.00007520529,0.00000379663,0.00001037857,0.00001905917],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9035866,"threshold_uncertainty_score":0.2256348,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1237782181359969,"score_gpt":0.3795740915048949,"score_spread":0.255795873368898,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}