{"id":"W4404782900","doi":"10.18653/v1/2024.emnlp-main.840","title":"LitSearch: A Retrieval Benchmark for Scientific Literature Search","year":2024,"lang":"en","type":"article","venue":"","topic":"Semantic Web and Ontologies","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Institute for Catastrophic Loss Reduction; National Science Foundation","keywords":"Benchmark (surveying); Computer science; Information retrieval","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00105592,0.00009811596,0.0001025016,0.0002563689,0.0002064431,0.003335321,0.0008531869,0.00007150469,0.00004399042],"category_scores_gemma":[0.0001440159,0.00006922532,0.0001024844,0.001367633,0.0001033807,0.0005800037,0.0002569181,0.0001627838,0.00009446026],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002696718,"about_ca_system_score_gemma":0.0002105392,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000003815285,"about_ca_topic_score_gemma":0.000004111604,"domain_scores_codex":[0.9984958,0.00003773137,0.0001319965,0.0005757086,0.0003642759,0.0003945577],"domain_scores_gemma":[0.9988515,0.0003812616,0.00000697519,0.0005053591,0.0001706498,0.0000841886],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001420058,0.00002735787,0.00006097025,0.0002264353,0.00002244928,0.00007335551,0.002956792,0.000004141658,0.003373164,0.9011757,0.04857785,0.04348754],"study_design_scores_gemma":[0.0004409126,0.0003977379,0.001393531,0.0004104223,0.00001284557,0.0001254836,0.0002493053,0.4266391,0.03709698,0.1004298,0.4322599,0.0005440287],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04312974,0.01170736,0.9022489,0.01530185,0.005504328,0.0007751405,0.0000248627,0.00144092,0.01986686],"genre_scores_gemma":[0.8566146,0.00003846633,0.1031221,0.0002773513,0.0003127377,0.00001838028,0.00002045718,0.00001350953,0.03958245],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8134848,"threshold_uncertainty_score":0.9976993,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05469280636575525,"score_gpt":0.3208167587591216,"score_spread":0.2661239523933664,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}