{"id":"W4386562232","doi":"10.1080/00031305.2023.2257237","title":"Statistical Challenges in Online Controlled Experiments: A Review of A/B Testing Methodology","year":2023,"lang":"en","type":"review","venue":"The American Statistician","topic":"Statistical Methods in Clinical Trials","field":"Mathematics","cited_by":44,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Data science; The Internet; Computer science; Scale (ratio); Big data; World Wide Web; Marketing; Business; Data mining; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2376337,0.002765192,0.009017028,0.00816489,0.001217089,0.006134271,0.006604015,0.005828395,0.004004066],"category_scores_gemma":[0.4288664,0.002024274,0.004824724,0.009971178,0.01034141,0.005930586,0.003354816,0.007347941,0.00187582],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006343113,"about_ca_system_score_gemma":0.01577773,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005117554,"about_ca_topic_score_gemma":0.003848233,"domain_scores_codex":[0.7238119,0.2207592,0.0211826,0.008485605,0.02499579,0.0007649355],"domain_scores_gemma":[0.2200382,0.7503327,0.01046779,0.006687,0.01178603,0.0006883964],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0004194576,0.0002253452,0.001964547,0.0813158,0.002169199,0.0001956308,0.0007875938,0.003304373,0.0003797272,0.06064793,0.01395125,0.8346391],"study_design_scores_gemma":[0.000831942,0.002470557,0.006293668,0.2045143,0.004630624,0.001447871,0.0009875175,0.01465361,0.002378648,0.3180416,0.4432197,0.0005300578],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0003244503,0.9022124,0.08520949,0.007406957,0.0009780881,0.0009526067,0.0002430995,0.0001464717,0.002526448],"genre_scores_gemma":[0.02086589,0.8209647,0.1392005,0.007518382,0.002919544,0.007274942,0.0003009262,0.0002630416,0.0006920903],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.7623662,"threshold_uncertainty_score":0.9401333,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9449757609267309,"score_gpt":0.7096567412494859,"score_spread":0.235319019677245,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}