{"id":"W4402324443","doi":"10.21203/rs.3.rs-5038817/v1","title":"LLMs for Closed-Library Multi-Document Query, Test Generation, and Evaluation","year":2024,"lang":"en","type":"preprint","venue":"Research Square","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"ca_institutions":"Artificial Intelligence in Medicine (Canada)","funders":"","keywords":"Computer science; Leverage (statistics); Knowledge base; World Wide Web; Test (biology); Information retrieval; Knowledge management; Data science; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"simulation_or_modeling","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"simulation_or_modeling","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006455031,0.001851644,0.001761341,0.003281872,0.001040247,0.003358833,0.004057306,0.002346548,0.02349807],"category_scores_gemma":[0.0346248,0.000924364,0.001672796,0.002264274,0.001309407,0.004902925,0.003865416,0.001823746,0.009175009],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00291096,"about_ca_system_score_gemma":0.003405689,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009283123,"about_ca_topic_score_gemma":0.01394993,"domain_scores_codex":[0.9887221,0.005367466,0.0009855211,0.001287866,0.003056126,0.0005809257],"domain_scores_gemma":[0.9738683,0.01572269,0.001116613,0.005613847,0.00308009,0.0005985025],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002346099,0.0005406651,0.002941375,0.0008141412,0.0002071929,0.0003282643,0.0003094885,0.04354068,0.02114481,0.02700487,0.06869471,0.8321277],"study_design_scores_gemma":[0.000297829,0.0003207347,0.0008788981,0.00007350941,0.00007788125,0.0002100507,0.0001152359,0.9216456,0.02742967,0.03697973,0.01191504,0.00005585479],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01383284,0.000593941,0.8725924,0.0003845357,0.00008890052,0.0003370516,0.001891323,0.1069157,0.003363245],"genre_scores_gemma":[0.2641968,0.0001832376,0.7187841,0.0003715603,0.0001638901,0.000525351,0.005591858,0.005107696,0.005075464],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02349807,"threshold_uncertainty_score":0.07860881,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1093112554538163,"score_gpt":0.4531445710293902,"score_spread":0.3438333155755738,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}