{"id":"W3135261467","doi":"10.48550/arxiv.2103.03098","title":"Accounting for Variance in Machine Learning Benchmarks","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Adversarial Robustness in Machine Learning","field":"Computer Science","cited_by":40,"is_retracted":false,"has_abstract":true,"ca_institutions":"Polytechnique Montréal; Canadian Institute for Advanced Research; McGill University; École de Technologie Supérieure; Université de Montréal","funders":"","keywords":"Variance (accounting); Computer science; Artificial intelligence; Accounting; Machine learning; Economics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0008812775,0.0004046051,0.0005292816,0.0003733166,0.0002802136,0.0003258072,0.002192947,0.0004122559,0.00006389898],"category_scores_gemma":[0.0005688853,0.0005319102,0.0002546647,0.0009289124,0.00006095347,0.0007917798,0.003640926,0.001913186,0.000007269372],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003498794,"about_ca_system_score_gemma":0.0003301367,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004786931,"about_ca_topic_score_gemma":0.0002048995,"domain_scores_codex":[0.9970194,0.0003171301,0.0003307708,0.001618053,0.0001264725,0.0005882386],"domain_scores_gemma":[0.9977759,0.0004924925,0.0004476411,0.0009801767,0.0001972102,0.0001065565],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0000207804,0.00004464414,0.02151557,0.0001343153,0.00004242744,0.0003068566,0.0004133006,0.9351125,0.00002054125,0.04124818,0.00001735282,0.001123567],"study_design_scores_gemma":[0.0007165902,0.00002534097,0.002045021,0.0002427803,0.00003231882,0.000005074598,0.0001333349,0.986378,0.00002179725,0.009306605,0.0005619152,0.000531226],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08683636,0.0001746165,0.9098408,0.000156603,0.0009315446,0.0003473697,0.000004364752,0.0001922385,0.00151608],"genre_scores_gemma":[0.9639696,0.00009240833,0.03492375,0.00008081791,0.0001445918,0.00000370818,0.00009219255,0.00003350688,0.0006594267],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8771333,"threshold_uncertainty_score":0.9997132,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04317978687297177,"score_gpt":0.2012533850704508,"score_spread":0.1580735981974791,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}