{"id":"W4400391016","doi":"10.1016/j.jss.2024.112154","title":"AML: An accuracy metric model for effective evaluation of log parsing techniques","year":2024,"lang":"en","type":"article","venue":"Journal of Systems and Software","topic":"Software System Performance and Reliability","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Concordia University","funders":"Natural Sciences and Engineering Research Council of Canada; Ministère de la Défense Nationale; Innovation for Defence Excellence and Security","keywords":"Parsing; Computer science; Metric (unit); Debugging; Data mining; Software; Task (project management); Artificial intelligence; Programming language","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01900255,0.002470652,0.00196826,0.007989378,0.0009305297,0.003766983,0.003091687,0.002451636,0.001685611],"category_scores_gemma":[0.1002627,0.000536292,0.001516011,0.005292355,0.001287339,0.006005986,0.002489767,0.002198067,0.001109864],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003357842,"about_ca_system_score_gemma":0.002857401,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007769758,"about_ca_topic_score_gemma":0.006689733,"domain_scores_codex":[0.9770197,0.006857167,0.002868922,0.002269745,0.0101499,0.000834582],"domain_scores_gemma":[0.9050675,0.06043775,0.007106089,0.008716216,0.01765065,0.001021755],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00115104,0.0007640053,0.06287087,0.001359944,0.0006915634,0.0002530925,0.0007246717,0.3977676,0.01045295,0.02371768,0.02756752,0.472679],"study_design_scores_gemma":[0.00004473784,0.0004031654,0.008224437,0.0001116913,0.0001183898,0.0001780191,0.0001327774,0.9644658,0.006414879,0.01310515,0.006717028,0.00008404296],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.06176266,0.002142227,0.9132184,0.0009680391,0.0002446403,0.0007044065,0.004199662,0.01240808,0.00435193],"genre_scores_gemma":[0.5029173,0.0007641546,0.4833333,0.0003770941,0.0002078036,0.001260619,0.008050468,0.00121856,0.001870667],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.01900255,"threshold_uncertainty_score":0.1004964,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03965729576152576,"score_gpt":0.3410378737279783,"score_spread":0.3013805779664525,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}