{"id":"W3016194403","doi":"10.1007/978-3-030-45442-5_6","title":"Reproducibility is a Process, Not an Achievement: The Replicability of IR Reproducibility Experiments","year":2020,"lang":"en","type":"book-chapter","venue":"Lecture notes in computer science","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":13,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Reproducibility; Replicate; Computer science; Process (computing); Set (abstract data type); Artificial intelligence; Data science; Mathematics; Statistics; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1916332,0.001044263,0.002361961,0.003313707,0.002655804,0.008245848,0.004679139,0.00428429,0.00653789],"category_scores_gemma":[0.4648263,0.001204773,0.002113434,0.003135971,0.02012991,0.01660269,0.006346672,0.008540066,0.001891175],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002439417,"about_ca_system_score_gemma":0.003869459,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0009711764,"about_ca_topic_score_gemma":0.0006778228,"domain_scores_codex":[0.8244514,0.1304378,0.005843493,0.01493233,0.02317746,0.001157632],"domain_scores_gemma":[0.3547744,0.4161572,0.01682663,0.189309,0.02096434,0.001968433],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005506115,0.0001649734,0.007560115,0.001654435,0.0007108728,0.0003362317,0.004486986,0.003797924,0.006863878,0.7233144,0.01855612,0.2320035],"study_design_scores_gemma":[0.0001354881,0.0004219136,0.00463049,0.0009201599,0.0003665023,0.0009279855,0.000616258,0.009357964,0.01050351,0.9192483,0.05270194,0.0001695138],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04243134,0.02978616,0.7734051,0.03286627,0.007431553,0.0009231419,0.0006630018,0.002305246,0.1101882],"genre_scores_gemma":[0.5732619,0.007256632,0.3858958,0.009401515,0.004410189,0.002597786,0.0008733261,0.002315705,0.01398714],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8083668,"threshold_uncertainty_score":0.9968602,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1470045001085528,"score_gpt":0.3877759823654807,"score_spread":0.2407714822569278,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}