{"id":"W4291009573","doi":"10.31222/osf.io/a9vhr","title":"We Need to Talk about Mechanical Turk: What 22,989 Hypothesis Tests Tell Us about Publication Bias and p-Hacking in Online Experiments","year":2022,"lang":"en","type":"preprint","venue":"","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"Wilfrid Laurier University; University of Ottawa","funders":"Canada Research Chairs","keywords":"Credibility; Hacker; Set (abstract data type); Trustworthiness; Deception; Psychology; Marketing; Data science; Internet privacy; Computer science; Social psychology; Political science; Business; Computer security","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.174712,0.001006599,0.002933115,0.008819171,0.003186395,0.01261964,0.002676092,0.006424257,0.01267432],"category_scores_gemma":[0.6657302,0.001128481,0.003126093,0.01241266,0.013168,0.01964316,0.003299611,0.004707742,0.004788782],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002377658,"about_ca_system_score_gemma":0.004024999,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001745932,"about_ca_topic_score_gemma":0.001323456,"domain_scores_codex":[0.8389105,0.1009089,0.01595632,0.01743053,0.02341208,0.003381623],"domain_scores_gemma":[0.1044097,0.7681406,0.0526698,0.05732631,0.01542912,0.002024555],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00348018,0.0003572619,0.4117179,0.009608328,0.007137628,0.001487926,0.01147673,0.003449394,0.00288792,0.06077801,0.08882163,0.3987971],"study_design_scores_gemma":[0.0008370671,0.001005,0.3055034,0.008232325,0.003072828,0.002453858,0.008130664,0.008807417,0.006934132,0.4514233,0.2028155,0.0007844911],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.3817274,0.1043745,0.1189788,0.2757111,0.01109969,0.001157095,0.02887621,0.002619466,0.07545581],"genre_scores_gemma":[0.9138233,0.01282522,0.02325734,0.03171089,0.007687787,0.001067861,0.004577291,0.001098739,0.003951611],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9935758,"threshold_uncertainty_score":0.9239762,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0625741574800946,"score_gpt":0.2973621482351371,"score_spread":0.2347879907550425,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}