fix(alerts): 内置规则排除 Loki 自身日志,消除自我污染误报

Loki 会把执行过的查询语句记成自身 info 日志,查询文本包含规则关键字时
告警会'命中自己'(如 StackOverflowError 规则匹配到告警引擎自己的查询记录),
造成持续误报——全命名空间类内置规则一律加 app!="loki";存量规则启动时自动升级;
顺带清理旧版遗留的重复规则「应用错误过多(error > 1000)」(与现模板重复且阈值会误报);
前端新建/模板默认查询同步;红线测试锁死

backend/seed/alert_rules.go

@@ -8,6 +8,7 @@ package seed
 // 通道未就绪时命中只会把失败原因写进发送历史,不影响规则评估。
 import (
 	"slices"
+	"strings"
 	"time"
 
 	"lokistack/models"
@@ -69,10 +70,12 @@ func channelIDsFor(id uint) []uint {
 
 var builtinRules = []builtinRule{
 	// —— 致命类：这类日志本该极少出现，但也不能一条就报 ——
-	{name: "致命日志 FATAL", query: `{namespace=~".+"} |= "FATAL"`, windowMin: 15, threshold: 10, cooldown: 120},
-	{name: "Go panic", query: `{namespace=~".+"} |= "panic:"`, windowMin: 15, threshold: 10, cooldown: 120},
-	{name: "JVM OutOfMemoryError", query: `{namespace=~".+"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 5, cooldown: 120},
-	{name: "StackOverflowError", query: `{namespace=~".+"} |= "StackOverflowError"`, windowMin: 15, threshold: 5, cooldown: 120},
+	// 全命名空间类查询一律排除 app=loki:Loki 会把执行过的查询语句记成自身 info 日志,
+	// 查询文本包含规则关键字时告警会"命中自己",造成持续误报。
+	{name: "致命日志 FATAL", query: `{namespace=~".+", app!="loki"} |= "FATAL"`, windowMin: 15, threshold: 10, cooldown: 120},
+	{name: "Go panic", query: `{namespace=~".+", app!="loki"} |= "panic:"`, windowMin: 15, threshold: 10, cooldown: 120},
+	{name: "JVM OutOfMemoryError", query: `{namespace=~".+", app!="loki"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 5, cooldown: 120},
+	{name: "StackOverflowError", query: `{namespace=~".+", app!="loki"} |= "StackOverflowError"`, windowMin: 15, threshold: 5, cooldown: 120},
 	// 注意：这里**只放日志类告警**。
 	// 「节点 NotReady」「容器 OOMKilled」属于基础设施指标监控（kube-state-metrics /
 	// Prometheus）的职责，用日志去匹配这些字符串既不可靠（日志没开或轮转就漏），
@@ -81,19 +84,24 @@ var builtinRules = []builtinRule{
 	// 第二轮上调（500~5000 这一档实测仍然太敏感）：
 	// 这几个量级在业务高峰期本来就是常态，阈值贴着基线就等于"每天准点报警"。
 	// 原则：宁可只报"明显异常"，也不要让人对告警脱敏。
-	{name: "应用错误激增(error > 10万)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240},
+	{name: "应用错误激增(error > 10万)", query: `{namespace=~".+", app!="loki"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240},
 	{name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) "`, windowMin: 15, threshold: 2000, cooldown: 240},
 	{name: "ingress-nginx 4xx 激增", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " 4[0-9][0-9] "`, windowMin: 15, threshold: 20000, cooldown: 240},
 	// 数据库连接类错误：覆盖 MySQL / PostgreSQL / Redis 的主流报错文案
 	// （详见函数上方注释，不要加裸 "EOF" / "timeout"，那会匹配到几乎所有网络日志）
-	{name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240},
+	{name: "数据库连接失败", query: `{namespace=~".+", app!="loki"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240},
 }
 
 // seedBuiltinAlertRules 为每个平台管理的 LogSource(managed=true)补齐内置规则:
 // 首次部署 Loki 后即有一套可直接启用的告警模板;通知通道留空,管理员配置后启用。
 // legacyInfraRules 早期版本误种的 Kubernetes 基础设施类规则 —— 日志平台只关注应用日志,
 // 这类节点/容器状态告警属 Prometheus + Alertmanager 职责,升级时一次性清理存量。
-var legacyInfraRules = []string{"Kubernetes 节点 NotReady", "容器 OOMKilled"}
+var legacyInfraRules = []string{
+	"Kubernetes 节点 NotReady",
+	"容器 OOMKilled",
+	// 旧版 seed 遗留:与现模板「应用错误激增」重复,且 1000 阈值实测会误报平台自身查询日志
+	"应用错误过多(error > 1000)",
+}
 
 // SeedAlertRules 补齐内置告警规则(幂等;公开给启动与部署成功两处调用)
 func SeedAlertRules() {
@@ -113,6 +121,11 @@ func SeedAlertRules() {
 			changed = true
 			continue
 		}
+		if builtinNames[rule.Name] && strings.Contains(rule.Query, `{namespace=~".+"}`) && !strings.Contains(rule.Query, `app!="loki"`) {
+			rule.Query = strings.ReplaceAll(rule.Query, `{namespace=~".+"}`, `{namespace=~".+", app!="loki"}`)
+			rule.UpdatedAt = time.Now()
+			changed = true
+		}
 		if builtinNames[rule.Name] && len(rule.ChannelIDs) == 0 {
 			rule.ChannelIDs = channelIDsFor(defaultChannelID)
 			if !rule.Enabled {

backend/seed/alert_rules_test.go

@@ -0,0 +1,17 @@
+package seed
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestBuiltinRulesExcludeLokiSelfLogs 全命名空间类内置规则必须排除 app=loki:
+// Loki 会把执行过的查询语句记成自身 info 日志,查询文本包含规则关键字时
+// 告警会"命中自己"(自我污染),造成持续误报——用测试锁死这条红线。
+func TestBuiltinRulesExcludeLokiSelfLogs(t *testing.T) {
+	for _, r := range builtinRules {
+		if strings.Contains(r.query, `{namespace=~".+"}`) && !strings.Contains(r.query, `app!="loki"`) {
+			t.Fatalf("内置规则 %q 使用全命名空间选择器但未排除 app=loki(自我污染): %s", r.name, r.query)
+		}
+	}
+}