{"id":"W4386562232","doi":"10.1080/00031305.2023.2257237","title":"Statistical Challenges in Online Controlled Experiments: A Review of A/B Testing Methodology","year":2023,"lang":"en","type":"review","venue":"The American Statistician","topic":"Statistical Methods in Clinical Trials","field":"Mathematics","cited_by":44,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Data science; The Internet; Computer science; Scale (ratio); Big data; World Wide Web; Marketing; Business; Data mining; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","metaepi_broad"],"consensus_categories":[],"category_scores_codex":[0.01621939,0.0008837228,0.01484955,0.0003561169,0.00006813382,0.00001798054,0.001253209,0.0002053413,0.0002150135],"category_scores_gemma":[0.5453734,0.0005533636,0.0005939066,0.00163007,0.001626952,0.00002381242,0.0004297123,0.001243969,0.00007921427],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001856553,"about_ca_system_score_gemma":0.0005422517,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002239077,"about_ca_topic_score_gemma":0.00006504169,"domain_scores_codex":[0.9696804,0.0218228,0.00579107,0.0009955093,0.0008417498,0.00086841],"domain_scores_gemma":[0.5802326,0.4143905,0.00394371,0.001065724,0.0002143029,0.0001531469],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00009556975,0.0002442285,4.792701e-7,0.0940394,0.0005066429,0.00009889769,0.00005465731,6.520546e-8,2.083753e-7,0.07922193,0.0009111266,0.8248268],"study_design_scores_gemma":[0.002151677,0.001300287,0.00003586386,0.2674146,0.006674326,0.00005607423,0.0006784787,0.00006328825,2.979798e-7,0.6704901,0.05002152,0.00111348],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[5.186626e-7,0.9472945,0.04454283,0.0002187574,0.0003551216,0.00362268,0.003222942,0.0001125046,0.0006301327],"genre_scores_gemma":[9.608961e-8,0.5375277,0.4614201,0.0001664506,0.0001685159,0.0005331427,0.00002594204,0.0001265825,0.00003149514],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.8237133,"threshold_uncertainty_score":0.999776,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9449757609267309,"score_gpt":0.7096567412494859,"score_spread":0.235319019677245,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}