{"id":"W4404782900","doi":"10.18653/v1/2024.emnlp-main.840","title":"LitSearch: A Retrieval Benchmark for Scientific Literature Search","year":2024,"lang":"en","type":"article","venue":"","topic":"Semantic Web and Ontologies","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Institute for Catastrophic Loss Reduction; National Science Foundation","keywords":"Benchmark (surveying); Computer science; Information retrieval","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01165293,0.002095586,0.001513615,0.02195791,0.001564255,0.003526789,0.002312821,0.002419523,0.007773666],"category_scores_gemma":[0.06063072,0.000484618,0.001389204,0.01479765,0.0008887898,0.004966263,0.003315182,0.001148984,0.00800814],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002022699,"about_ca_system_score_gemma":0.003815812,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006264809,"about_ca_topic_score_gemma":0.01225337,"domain_scores_codex":[0.9898459,0.004102041,0.002142592,0.001112171,0.002450115,0.0003471815],"domain_scores_gemma":[0.9680653,0.0188008,0.002139215,0.003961583,0.005965643,0.001067507],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001640422,0.001028481,0.01186447,0.02627281,0.0009336974,0.0006337555,0.001612515,0.02288046,0.02363081,0.01120726,0.4447652,0.45353],"study_design_scores_gemma":[0.002089708,0.003517718,0.03487084,0.003064604,0.0009530216,0.003085288,0.002775447,0.2197358,0.05645772,0.03691848,0.6359046,0.0006267297],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"dataset","genre_gemma":"empirical","genre_scores_codex":[0.1913147,0.06985029,0.1938476,0.007232751,0.00217107,0.006031151,0.3553523,0.1201366,0.05406355],"genre_scores_gemma":[0.1725522,0.00706449,0.2955488,0.001358401,0.000532236,0.003391151,0.5102673,0.00292944,0.00635595],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02195791,"threshold_uncertainty_score":0.06162733,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05469280636575525,"score_gpt":0.3208167587591216,"score_spread":0.2661239523933664,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}