{"id":"W4400528755","doi":"10.1145/3626772.3657861","title":"Systematic Evaluation of Neural Retrieval Models on the Touché 2020 Argument Retrieval Subset of BEIR","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; Deutsche Forschungsgemeinschaft; European Commission; HORIZON EUROPE Framework Programme; Compute Canada","keywords":"Argument (complex analysis); Computer science; Artificial intelligence; Information retrieval; Natural language processing; Biology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00323758,0.0001251491,0.0002568745,0.00008515924,0.00003747344,0.00008598249,0.0006878216,0.00005266371,0.00004809389],"category_scores_gemma":[0.0002471227,0.00007571108,0.0001161752,0.0005116625,0.00002789989,0.0002847227,0.0001528148,0.0001324134,0.00001431622],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00009766183,"about_ca_system_score_gemma":0.0001395976,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001442409,"about_ca_topic_score_gemma":0.000001783293,"domain_scores_codex":[0.9970674,0.0003502864,0.0006073396,0.0003331461,0.001479021,0.0001628257],"domain_scores_gemma":[0.9983214,0.0004311824,0.0001360964,0.0008130433,0.0002589487,0.00003938121],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00006845247,0.0001372791,0.00002061376,0.007394675,0.0002540308,0.000009985408,0.005183301,0.2213615,0.006910122,0.7551441,0.001244921,0.002271009],"study_design_scores_gemma":[0.0001216025,0.00008843088,0.00002013932,0.0008547716,0.00005641969,0.000005102418,0.00007604291,0.9743919,0.01039945,0.01391126,0.000001502273,0.0000733369],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6290485,0.001104916,0.3612565,0.002949752,0.0009812848,0.001393139,0.000006348269,0.0001314054,0.00312816],"genre_scores_gemma":[0.9981429,0.000008513184,0.001500904,0.00009014991,0.00004523851,0.000007991765,0.000001293706,0.000007304505,0.0001957087],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7530304,"threshold_uncertainty_score":0.3087408,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06782064987352145,"score_gpt":0.29225979395797,"score_spread":0.2244391440844485,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}