{"id":"W4415381631","doi":"10.1145/3736731.3746136","title":"Experience with Reproducibility and Consistency in Writing an Academic Paper","year":2025,"lang":"","type":"article","venue":"","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"Critical Systems Labs; University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Consistency (knowledge bases); Reproducibility; Reliability (semiconductor); Measure (data warehouse)","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reporting","study_design":"design_other","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","scholarly_communication"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.02991581,0.0002609114,0.0004359261,0.0005328204,0.0005154239,0.001046255,0.001319771,0.0001151916,0.0004904654],"category_scores_gemma":[0.009432159,0.0001893519,0.00003866233,0.003070105,0.001010358,0.001543957,0.001562208,0.000437389,0.00003700783],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006042947,"about_ca_system_score_gemma":0.0002578484,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004733151,"about_ca_topic_score_gemma":0.0005864047,"domain_scores_codex":[0.9904428,0.0005127206,0.001835531,0.005566626,0.001069028,0.0005732872],"domain_scores_gemma":[0.9916441,0.001275703,0.0002200342,0.006403398,0.0002627356,0.000193983],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00009630172,0.000186148,0.5544764,0.00004088223,0.000009753287,0.00002229039,0.005743971,0.0001007612,0.0002370163,0.01311229,0.002381234,0.423593],"study_design_scores_gemma":[0.001257592,0.0002023744,0.826324,0.0006799562,0.00003283154,0.0000173289,0.09553938,0.03735132,0.0003444858,0.01517333,0.0224343,0.0006430241],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9575115,0.001103557,0.003629731,0.004111908,0.0007975255,0.000506194,0.000005910349,0.00005912204,0.03227458],"genre_scores_gemma":[0.987996,0.00007220489,0.002730764,0.001836159,0.00003207158,0.00001474763,0.00000260532,0.000004992492,0.007310387],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.42295,"threshold_uncertainty_score":0.9999908,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1157862929031439,"score_gpt":0.4215548920575341,"score_spread":0.3057685991543903,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}