{"id":"W3015713034","doi":"10.1007/978-3-030-45442-5_4","title":"Which BM25 Do You Mean? A Large-Scale Reproducibility Study of Scoring Variants","year":2020,"lang":"en","type":"book-chapter","venue":"Lecture notes in computer science","topic":"Topic Modeling","field":"Computer Science","cited_by":54,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Reproducibility; Scale (ratio); Information retrieval; Artificial intelligence; Statistics; Mathematics; Cartography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"simulation_or_modeling","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reproducibility","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.07356825,0.001326642,0.001739654,0.003176496,0.001702419,0.003960364,0.002743089,0.001847242,0.001993373],"category_scores_gemma":[0.2529285,0.0005604086,0.002257745,0.003456109,0.002260648,0.003076736,0.002014762,0.002239325,0.001681646],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001094458,"about_ca_system_score_gemma":0.000851398,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003010122,"about_ca_topic_score_gemma":0.00361878,"domain_scores_codex":[0.9495412,0.0303763,0.003388435,0.009881474,0.00616115,0.0006514071],"domain_scores_gemma":[0.6871519,0.2507562,0.009507655,0.03205353,0.01806521,0.002465583],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.009009751,0.0007260534,0.5411126,0.001457931,0.01540028,0.0003747955,0.004775363,0.00928698,0.006855055,0.00323477,0.03887367,0.3688927],"study_design_scores_gemma":[0.001369588,0.004207273,0.8038408,0.0006396945,0.01020276,0.003120408,0.0029255,0.1242059,0.01358202,0.01537635,0.01980715,0.0007225506],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9182201,0.01248321,0.05591474,0.001219135,0.001056509,0.0003033553,0.003883669,0.001794694,0.005124629],"genre_scores_gemma":[0.9758304,0.0005911762,0.0168667,0.0002055205,0.0002245143,0.0001487946,0.004126769,0.0007668785,0.001239191],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9264318,"threshold_uncertainty_score":0.3890707,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03285745192629517,"score_gpt":0.2676645239402604,"score_spread":0.2348070720139653,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}