{"id":"W4399625707","doi":"10.1007/s10922-024-09831-x","title":"Benchmarking Large Language Models for Log Analysis, Security, and Interpretation","year":2024,"lang":"en","type":"article","venue":"Journal of Network and Systems Management","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"ca_institutions":"Dalhousie University","funders":"","keywords":"Computer science; Benchmarking; Pipeline (software); Adaptation (eye); Language model; Domain (mathematical analysis); Interpretation (philosophy); Variety (cybernetics); Data mining; Data science; Artificial intelligence; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001888692,0.00009052012,0.0002604542,0.0002053208,0.00008353164,0.0003719316,0.0001649615,0.00003789784,7.019486e-7],"category_scores_gemma":[0.000003288956,0.00006447937,0.0001097918,0.0003480006,0.00001029428,0.0004568774,0.0001201983,0.00007952742,4.039056e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003242328,"about_ca_system_score_gemma":0.000009577601,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000006297997,"about_ca_topic_score_gemma":0.000005749107,"domain_scores_codex":[0.9989774,0.00006487887,0.000413079,0.0001814479,0.0001953325,0.0001678616],"domain_scores_gemma":[0.9994798,0.0001002351,0.0001558341,0.0001486529,0.00006024223,0.00005525038],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001386746,0.0001601359,0.01581294,0.01100691,0.007990247,0.0003903724,0.03667725,0.4127492,0.000007888985,0.2471358,0.02125405,0.2466766],"study_design_scores_gemma":[0.0001760464,0.00008368438,0.0005896539,0.0004164717,0.0002102263,0.00003369054,0.000423719,0.9920997,2.928948e-7,0.00282744,0.00306356,0.00007551482],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.03686444,0.01767584,0.943458,0.0001089736,0.001273832,0.0002416354,0.000001713309,0.00002852298,0.0003471146],"genre_scores_gemma":[0.996254,0.0005249769,0.002708062,0.0000389565,0.0003894019,0.000009533459,0.000001608882,0.00000421409,0.00006921248],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9593896,"threshold_uncertainty_score":0.3586545,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.008274013247924983,"score_gpt":0.2573795766463653,"score_spread":0.2491055633984403,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}