{"id":"W4318239436","doi":"10.48550/arxiv.2301.10493","title":"From Baseline to Top Performer: A Reproducibility Study of Approaches at the TREC 2021 Conversational Assistance Track","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Norges Forskningsråd; University of Waterloo","keywords":"Computer science; Baseline (sea); Pipeline (software); Information retrieval; Set (abstract data type); Margin (machine learning); Measure (data warehouse); Track (disk drive); Data mining; Machine learning","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reproducibility","study_design":"design_other","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1160564,0.00217854,0.001887607,0.004526152,0.003642777,0.006167244,0.003726289,0.002590586,0.001951023],"category_scores_gemma":[0.2443363,0.0009346714,0.002257153,0.003886826,0.002549288,0.006389244,0.005344937,0.003497346,0.003843758],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002660676,"about_ca_system_score_gemma":0.002591354,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006933659,"about_ca_topic_score_gemma":0.008013124,"domain_scores_codex":[0.8487178,0.08447782,0.01171578,0.02469096,0.02729041,0.003107227],"domain_scores_gemma":[0.6571738,0.1568911,0.01828405,0.09622718,0.0651237,0.006300152],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.01040107,0.003128353,0.2547584,0.00628782,0.007729565,0.001208656,0.0192718,0.01783797,0.04710061,0.004999686,0.1190072,0.5082688],"study_design_scores_gemma":[0.001602024,0.01714965,0.5544614,0.001508436,0.004860919,0.00419324,0.01876961,0.1257832,0.1153575,0.01866782,0.1358226,0.001823483],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7938733,0.01751921,0.1177665,0.004992595,0.004972924,0.00195998,0.01699546,0.01662716,0.02529287],"genre_scores_gemma":[0.9354722,0.0007035139,0.03676039,0.0008337418,0.0008846003,0.0007545959,0.016744,0.003161782,0.004685131],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8839436,"threshold_uncertainty_score":0.6137722,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2084443694460633,"score_gpt":0.2211223094169392,"score_spread":0.01267793997087588,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}