{"id":"W3044177754","doi":"10.1145/3360311","title":"Threats of a replication crisis in empirical computer science","year":2020,"lang":"en","type":"article","venue":"Communications of the ACM","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":153,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Replication (statistics); Computer science; Empirical research; Biology; Virology; Epistemology; Philosophy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"theoretical_or_conceptual","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reproducibility","study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","open_science"],"consensus_categories":["open_science"],"category_scores_codex":[0.004969656,0.00004751322,0.0001414617,0.0001717135,0.0001684585,0.00007269508,0.0466121,0.00001566552,0.00001228618],"category_scores_gemma":[0.03970119,0.00003110394,0.00006310783,0.003367757,0.0006185626,0.0001675452,0.03950883,0.00009104406,0.00002735842],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002017941,"about_ca_system_score_gemma":0.00008120152,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002682378,"about_ca_topic_score_gemma":0.00001640395,"domain_scores_codex":[0.9979695,0.0001983475,0.0005553489,0.0004407115,0.0007406866,0.00009542393],"domain_scores_gemma":[0.9454848,0.0009277249,0.0003137244,0.05289485,0.0003340354,0.00004491196],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"observational","study_design_scores_codex":[0.00003633139,0.0006956136,0.2469696,0.00002273187,0.00002149991,2.569647e-7,0.01362308,0.004076044,0.004555173,0.02011324,0.4895969,0.2202895],"study_design_scores_gemma":[0.0003135656,0.00005449406,0.4701027,0.00004854015,0.00001450281,0.00000122277,0.001341088,0.1534911,0.002782529,0.363605,0.008119355,0.0001258204],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"commentary","genre_gemma":"empirical","genre_scores_codex":[0.4489119,0.0003083417,0.006903906,0.5398703,0.0002397655,0.0003817548,0.00002157849,0.00003270893,0.003329712],"genre_scores_gemma":[0.9281175,0.00001303491,0.07116816,0.0006545411,0.000007711981,0.000003438714,0.00000167411,0.000001877854,0.00003207436],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5392157,"threshold_uncertainty_score":0.9683878,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5130321443891269,"score_gpt":0.5065862083389177,"score_spread":0.006445936050209267,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}