{"id":"W4399625707","doi":"10.1007/s10922-024-09831-x","title":"Benchmarking Large Language Models for Log Analysis, Security, and Interpretation","year":2024,"lang":"en","type":"article","venue":"Journal of Network and Systems Management","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"ca_institutions":"Dalhousie University","funders":"","keywords":"Computer science; Benchmarking; Pipeline (software); Adaptation (eye); Language model; Domain (mathematical analysis); Interpretation (philosophy); Variety (cybernetics); Data mining; Data science; Artificial intelligence; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01108564,0.001585024,0.001291454,0.002315005,0.001032402,0.003283874,0.005123607,0.002317988,0.007113389],"category_scores_gemma":[0.06902754,0.001020211,0.001957976,0.002415854,0.001608669,0.007735878,0.003131323,0.003022281,0.001775242],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003618676,"about_ca_system_score_gemma":0.005991705,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01612484,"about_ca_topic_score_gemma":0.02228942,"domain_scores_codex":[0.9852858,0.008163824,0.001439222,0.001322808,0.003144798,0.0006436338],"domain_scores_gemma":[0.9186718,0.05862104,0.001482067,0.0135316,0.006736482,0.0009570509],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002361568,0.002002982,0.008761331,0.0006686432,0.0003671743,0.0002973686,0.0005469926,0.7535442,0.005705442,0.03920877,0.02251149,0.1640241],"study_design_scores_gemma":[0.00007711009,0.00009431512,0.0003135481,0.00001613729,0.00003035891,0.00001995059,0.00005887905,0.9861093,0.002079932,0.0100002,0.001182731,0.00001760989],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3086082,0.001624615,0.6277397,0.002364144,0.0006712358,0.0006846269,0.005334809,0.0446659,0.008306711],"genre_scores_gemma":[0.7826391,0.0004227367,0.204379,0.0003421816,0.0001275005,0.0004374812,0.007097801,0.002667027,0.001887142],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01612484,"threshold_uncertainty_score":0.05862719,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.008274013247924983,"score_gpt":0.2573795766463653,"score_spread":0.2491055633984403,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}