From b8d08ef6d1d69e7694c65e13fb9d56c5dabe402c Mon Sep 17 00:00:00 2001 From: opsx Date: Thu, 29 Oct 2026 13:46:46 +0800 Subject: [PATCH] =?UTF-8?q?fix(alerts):=20=E5=86=85=E7=BD=AE=E8=A7=84?= =?UTF-8?q?=E5=88=99=E6=8E=92=E9=99=A4=20Loki=20=E8=87=AA=E8=BA=AB?= =?UTF-8?q?=E6=97=A5=E5=BF=97,=E6=B6=88=E9=99=A4=E8=87=AA=E6=88=91?= =?UTF-8?q?=E6=B1=A1=E6=9F=93=E8=AF=AF=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loki 会把执行过的查询语句记成自身 info 日志,查询文本包含规则关键字时 告警会'命中自己'(如 StackOverflowError 规则匹配到告警引擎自己的查询记录), 造成持续误报——全命名空间类内置规则一律加 app!="loki";存量规则启动时自动升级; 顺带清理旧版遗留的重复规则「应用错误过多(error > 1000)」(与现模板重复且阈值会误报); 前端新建/模板默认查询同步;红线测试锁死 --- backend/seed/alert_rules.go | 27 ++++++++++++++----- backend/seed/alert_rules_test.go | 17 ++++++++++++ .../alerts/components/RuleFormDialog.tsx | 6 ++--- 3 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 backend/seed/alert_rules_test.go diff --git a/backend/seed/alert_rules.go b/backend/seed/alert_rules.go index 02e8d3f..7fdb442 100644 --- a/backend/seed/alert_rules.go +++ b/backend/seed/alert_rules.go @@ -8,6 +8,7 @@ package seed // 通道未就绪时命中只会把失败原因写进发送历史,不影响规则评估。 import ( "slices" + "strings" "time" "lokistack/models" @@ -69,10 +70,12 @@ func channelIDsFor(id uint) []uint { var builtinRules = []builtinRule{ // —— 致命类：这类日志本该极少出现，但也不能一条就报 —— - {name: "致命日志 FATAL", query: `{namespace=~".+"} |= "FATAL"`, windowMin: 15, threshold: 10, cooldown: 120}, - {name: "Go panic", query: `{namespace=~".+"} |= "panic:"`, windowMin: 15, threshold: 10, cooldown: 120}, - {name: "JVM OutOfMemoryError", query: `{namespace=~".+"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 5, cooldown: 120}, - {name: "StackOverflowError", query: `{namespace=~".+"} |= "StackOverflowError"`, windowMin: 15, threshold: 5, cooldown: 120}, + // 全命名空间类查询一律排除 app=loki:Loki 会把执行过的查询语句记成自身 info 日志, + // 查询文本包含规则关键字时告警会"命中自己",造成持续误报。 + {name: "致命日志 FATAL", query: `{namespace=~".+", app!="loki"} |= "FATAL"`, windowMin: 15, threshold: 10, cooldown: 120}, + {name: "Go panic", query: `{namespace=~".+", app!="loki"} |= "panic:"`, windowMin: 15, threshold: 10, cooldown: 120}, + {name: "JVM OutOfMemoryError", query: `{namespace=~".+", app!="loki"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 5, cooldown: 120}, + {name: "StackOverflowError", query: `{namespace=~".+", app!="loki"} |= "StackOverflowError"`, windowMin: 15, threshold: 5, cooldown: 120}, // 注意：这里**只放日志类告警**。 // 「节点 NotReady」「容器 OOMKilled」属于基础设施指标监控（kube-state-metrics / // Prometheus）的职责，用日志去匹配这些字符串既不可靠（日志没开或轮转就漏）， @@ -81,19 +84,24 @@ var builtinRules = []builtinRule{ // 第二轮上调（500~5000 这一档实测仍然太敏感）： // 这几个量级在业务高峰期本来就是常态，阈值贴着基线就等于"每天准点报警"。 // 原则：宁可只报"明显异常"，也不要让人对告警脱敏。 - {name: "应用错误激增(error > 10万)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240}, + {name: "应用错误激增(error > 10万)", query: `{namespace=~".+", app!="loki"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240}, {name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) "`, windowMin: 15, threshold: 2000, cooldown: 240}, {name: "ingress-nginx 4xx 激增", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " 4[0-9][0-9] "`, windowMin: 15, threshold: 20000, cooldown: 240}, // 数据库连接类错误：覆盖 MySQL / PostgreSQL / Redis 的主流报错文案 // （详见函数上方注释，不要加裸 "EOF" / "timeout"，那会匹配到几乎所有网络日志） - {name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240}, + {name: "数据库连接失败", query: `{namespace=~".+", app!="loki"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240}, } // seedBuiltinAlertRules 为每个平台管理的 LogSource(managed=true)补齐内置规则: // 首次部署 Loki 后即有一套可直接启用的告警模板;通知通道留空,管理员配置后启用。 // legacyInfraRules 早期版本误种的 Kubernetes 基础设施类规则 —— 日志平台只关注应用日志, // 这类节点/容器状态告警属 Prometheus + Alertmanager 职责,升级时一次性清理存量。 -var legacyInfraRules = []string{"Kubernetes 节点 NotReady", "容器 OOMKilled"} +var legacyInfraRules = []string{ + "Kubernetes 节点 NotReady", + "容器 OOMKilled", + // 旧版 seed 遗留:与现模板「应用错误激增」重复,且 1000 阈值实测会误报平台自身查询日志 + "应用错误过多(error > 1000)", +} // SeedAlertRules 补齐内置告警规则(幂等;公开给启动与部署成功两处调用) func SeedAlertRules() { @@ -113,6 +121,11 @@ func SeedAlertRules() { changed = true continue } + if builtinNames[rule.Name] && strings.Contains(rule.Query, `{namespace=~".+"}`) && !strings.Contains(rule.Query, `app!="loki"`) { + rule.Query = strings.ReplaceAll(rule.Query, `{namespace=~".+"}`, `{namespace=~".+", app!="loki"}`) + rule.UpdatedAt = time.Now() + changed = true + } if builtinNames[rule.Name] && len(rule.ChannelIDs) == 0 { rule.ChannelIDs = channelIDsFor(defaultChannelID) if !rule.Enabled { diff --git a/backend/seed/alert_rules_test.go b/backend/seed/alert_rules_test.go new file mode 100644 index 0000000..d613b3e --- /dev/null +++ b/backend/seed/alert_rules_test.go @@ -0,0 +1,17 @@ +package seed + +import ( + "strings" + "testing" +) + +// TestBuiltinRulesExcludeLokiSelfLogs 全命名空间类内置规则必须排除 app=loki: +// Loki 会把执行过的查询语句记成自身 info 日志,查询文本包含规则关键字时 +// 告警会"命中自己"(自我污染),造成持续误报——用测试锁死这条红线。 +func TestBuiltinRulesExcludeLokiSelfLogs(t *testing.T) { + for _, r := range builtinRules { + if strings.Contains(r.query, `{namespace=~".+"}`) && !strings.Contains(r.query, `app!="loki"`) { + t.Fatalf("内置规则 %q 使用全命名空间选择器但未排除 app=loki(自我污染): %s", r.name, r.query) + } + } +} diff --git a/frontend/src/pages/alerts/components/RuleFormDialog.tsx b/frontend/src/pages/alerts/components/RuleFormDialog.tsx index 3891748..f7d6d2c 100644 --- a/frontend/src/pages/alerts/components/RuleFormDialog.tsx +++ b/frontend/src/pages/alerts/components/RuleFormDialog.tsx @@ -24,7 +24,7 @@ interface RuleForm { } const DEFAULTS: RuleForm = { - name: '', sourceId: '', channelIds: [], query: '{namespace=~".+"} |= "error"', + name: '', sourceId: '', channelIds: [], query: '{namespace=~".+", app!="loki"} |= "error"', windowMin: '15', threshold: '0', cooldownMin: '60', enabled: true, } @@ -45,7 +45,7 @@ const TEMPLATES: AlertTemplate[] = [ }, { key: 'allNsError', - query: '{namespace=~".+"} |~ "(?i)error"', + query: '{namespace=~".+", app!="loki"} |~ "(?i)error"', windowMin: 15, threshold: 1000, cooldownMin: 60, }, { @@ -60,7 +60,7 @@ const TEMPLATES: AlertTemplate[] = [ }, { key: 'fatal', - query: '{namespace=~".+"} |= "FATAL"', + query: '{namespace=~".+", app!="loki"} |= "FATAL"', windowMin: 5, threshold: 0, cooldownMin: 15, }, ]