{"id":"W3044177754","doi":"10.1145/3360311","title":"Threats of a replication crisis in empirical computer science","year":2020,"lang":"en","type":"article","venue":"Communications of the ACM","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":153,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Replication (statistics); Computer science; Empirical research; Biology; Virology; Epistemology; Philosophy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"theoretical_or_conceptual","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reproducibility","study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.6676168,0.001623873,0.004629797,0.00655891,0.008131582,0.01289364,0.007907079,0.01893243,0.01142495],"category_scores_gemma":[0.9058226,0.002716526,0.002830043,0.00859548,0.03658274,0.03091656,0.01615675,0.0246004,0.004267756],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.007937277,"about_ca_system_score_gemma":0.02699501,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004951091,"about_ca_topic_score_gemma":0.003284375,"domain_scores_codex":[0.3649932,0.4850619,0.02800175,0.03932892,0.07702129,0.005592878],"domain_scores_gemma":[0.03548659,0.7647032,0.03150794,0.1217424,0.04208179,0.004478162],"domain_codex":"methods","domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.003990127,0.0006617086,0.04158396,0.003309017,0.002067803,0.001234387,0.01410025,0.00355179,0.001041541,0.6796193,0.1149246,0.1339155],"study_design_scores_gemma":[0.001767783,0.0004920686,0.006535345,0.002428613,0.000333174,0.0005820689,0.002408135,0.006935512,0.0008880477,0.9051128,0.07235245,0.0001639194],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"commentary","genre_gemma":"empirical","genre_scores_codex":[0.03248055,0.01249193,0.11191,0.7625963,0.01139214,0.002615051,0.001584463,0.001078837,0.06385082],"genre_scores_gemma":[0.6921596,0.003173115,0.09330513,0.1800671,0.01520522,0.008048299,0.00105632,0.0007686869,0.006216616],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.3323832,"threshold_uncertainty_score":0.4098877,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5130321443891269,"score_gpt":0.5065862083389177,"score_spread":0.006445936050209267,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}