{"id":"W2799424953","doi":"10.1162/tacl_a_00018","title":"Questionable Answers in Question Answering Research: Reproducibility and Variability of Published Results","year":2018,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":73,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Reproducibility; Computer science; Popularity; Reliability (semiconductor); Sample (material); Field (mathematics); Range (aeronautics); Code (set theory); Artificial intelligence; Data science; Statistics; Psychology; Mathematics; Social psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01194193,0.00007815933,0.0001828203,0.0001864756,0.000262208,0.00005541673,0.0003454961,0.00009912577,9.353631e-7],"category_scores_gemma":[0.02501033,0.00007229423,0.00006476821,0.0007528743,0.0001683729,0.0001359262,0.00003179788,0.0001744432,5.293787e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002983564,"about_ca_system_score_gemma":0.0002173248,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004834671,"about_ca_topic_score_gemma":0.00009956647,"domain_scores_codex":[0.9976965,0.0004682724,0.0006501746,0.0004974864,0.000510078,0.0001775381],"domain_scores_gemma":[0.9936976,0.002086462,0.0003854324,0.0006454781,0.003148409,0.0000366158],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0002556212,0.0008296469,0.08088548,0.0003592137,0.0001288388,2.614372e-7,0.00666531,0.1441174,0.0006060645,0.7634407,0.001212427,0.001499018],"study_design_scores_gemma":[0.001540354,0.0003705999,0.1695891,0.0002595996,0.00002710918,0.000001795414,0.0001445967,0.3996072,0.002871558,0.4223044,0.00304046,0.0002432712],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04610717,0.00003354694,0.9450108,0.002623426,0.002328465,0.0008299167,0.000210907,0.00008359756,0.002772204],"genre_scores_gemma":[0.9599282,0.000002767869,0.03972273,0.000009646972,0.0001305943,0.00001607539,0.00001085968,0.000005334638,0.0001737846],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.913821,"threshold_uncertainty_score":0.9832024,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03935335336173296,"score_gpt":0.3372124674052018,"score_spread":0.2978591140434689,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}