{"id":"W7123335448","doi":"10.1109/esem64174.2025.00053","title":"SESR-Eval: Dataset for Evaluating LLMs in the Title-Abstract Screening of Systematic Reviews","year":2025,"lang":"","type":"article","venue":"","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Dalhousie University","funders":"","keywords":"Benchmark (surveying); Systematic review; Task (project management); Process (computing); Plan (archaeology); Risk assessment","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04383286,0.003612768,0.00360361,0.0186271,0.001905521,0.004628152,0.005006634,0.004696905,0.01315586],"category_scores_gemma":[0.216571,0.001294626,0.007996562,0.01072352,0.001081933,0.00241842,0.005036531,0.002078957,0.005618712],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003121644,"about_ca_system_score_gemma":0.008653234,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00752718,"about_ca_topic_score_gemma":0.0270423,"domain_scores_codex":[0.9558938,0.02270596,0.01235537,0.004145863,0.004366661,0.0005323873],"domain_scores_gemma":[0.7581722,0.1948807,0.0181825,0.01590015,0.01131373,0.001550631],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003302573,0.0005756072,0.03611569,0.1878286,0.01229283,0.000824909,0.0008614957,0.01433434,0.003285478,0.004594795,0.6333275,0.1026561],"study_design_scores_gemma":[0.01778272,0.002396505,0.08158429,0.04776967,0.01893204,0.002650303,0.0008903362,0.05018779,0.009153511,0.01734933,0.7503783,0.0009252573],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"dataset","genre_gemma":"empirical","genre_scores_codex":[0.01111194,0.01373041,0.0106344,0.001607888,0.0003375436,0.002857005,0.9523107,0.005036322,0.002373805],"genre_scores_gemma":[0.02081134,0.00205866,0.05047637,0.0009596334,0.0001187724,0.01005848,0.9144639,0.0004711181,0.0005817324],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9561672,"threshold_uncertainty_score":0.2318131,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9375302035275612,"score_gpt":0.6578025851641262,"score_spread":0.2797276183634351,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}