{"id":"W2955229766","doi":"10.1145/3331184.3331356","title":"Quantifying Bias and Variance of System Rankings","year":2019,"lang":"en","type":"article","venue":"","topic":"Bayesian Modeling and Causal Inference","field":"Computer Science","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Variance (accounting); Pooling; Statistics; Estimator; Conflation; Mathematics; Bias of an estimator; Rank (graph theory); Econometrics; Computer science; Minimum-variance unbiased estimator; Artificial intelligence; Combinatorics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003236629,0.00005915228,0.0001245169,0.00003808891,0.00002368549,0.00005896711,0.0002750157,0.0000364199,0.000004994529],"category_scores_gemma":[0.000007479297,0.00004803282,0.00001901729,0.0001363622,0.00001384926,0.0002021147,0.00009655449,0.00005458325,0.00003927279],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000006094253,"about_ca_system_score_gemma":0.00002241188,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006892183,"about_ca_topic_score_gemma":0.00000165909,"domain_scores_codex":[0.9994,0.00002855405,0.0001443172,0.0001998874,0.0001181276,0.000109125],"domain_scores_gemma":[0.9995229,0.00006246303,0.00005823126,0.0002757136,0.00004894548,0.00003181124],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000001799239,0.000006219812,0.004104527,0.0001534259,0.000004536448,0.000001164067,0.000268485,0.00006782553,0.003459986,0.9853178,0.00001540515,0.006598865],"study_design_scores_gemma":[0.0003117853,0.00006680044,0.002525681,0.0002580058,0.000004286637,0.0000308091,0.0001022936,0.9891567,0.005492293,0.001736971,0.0001363775,0.0001780604],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1754191,0.00009214243,0.8179772,0.00007593477,0.0001421804,0.00004910822,2.4206e-7,0.00008944708,0.006154685],"genre_scores_gemma":[0.9507631,0.000004221062,0.04894856,0.00006120698,0.000006883422,9.951641e-7,1.199135e-7,0.000002598907,0.0002123316],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9890888,"threshold_uncertainty_score":0.1958722,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06170573545308153,"score_gpt":0.2608923500520292,"score_spread":0.1991866145989477,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}