From 7369a175981698af221da7826b85135da491f41d Mon Sep 17 00:00:00 2001 From: opsx Date: Thu, 29 Oct 2026 03:56:07 +0000 Subject: [PATCH] =?UTF-8?q?feat(alerts):=20=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E5=91=8A=E8=AD=A6=E8=A6=86=E7=9B=96=20MySQL/?= =?UTF-8?q?PG/Redis=20=E4=B8=BB=E6=B5=81=E6=8A=A5=E9=94=99=EF=BC=9B?= =?UTF-8?q?=E8=A7=84=E5=88=99=E8=A1=A8=E5=8D=95=E4=B8=89=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E5=AF=B9=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit === 1. 「数据库连接失败」匹配模式 3 → 25 个 === 原来只匹配 connection refused / reset by peer / too many connections， 漏掉了 PG 的 FATAL: too many clients、Redis 的 NOAUTH / WRONGPASS / connection pool timeout 等实际最常见的报错。 按数据库分类补齐： 通用网络层 connection refused / connection reset by peer / broken pipe / no route to host / cannot assign requested address / connection timed out / i/o timeout / unexpected EOF / server closed the connection unexpectedly MySQL too many connections / communications link failure / driver: bad connection / could not create connection / Access denied for user PostgreSQL FATAL: password authentication failed / sorry, too many clients / terminating connection Redis NOAUTH / WRONGPASS / READONLY you can't write against / CLUSTERDOWN / max number of clients reached 连接池（Java/Go 应用层最先炸的地方） connection pool / connection is not available **刻意不加**的（会匹配到几乎所有网络日志，等于没告警）： 裸 "EOF"、"timeout"、"error"、"refused"、"deadlock" 阈值保持 1000/15min（模式变多不代表命中变多，都是真错误才命中）。 迁移 v9→v10 只替换"仍是旧默认查询"的规则，用户自定义过的不动。 === 2. 规则表单三个字段没对齐 === 现象：统计窗口(分) / 触发阈值(条) / 冷却(分) 三格并排， 但只有中间的「触发阈值」下方多了一行提示文字， 导致中间那格更高、提示文字悬在底部，看起来像没对齐。 修法：三格统一为「标签 + 输入框 + 一行提示（min-h-4 固定高度）」， 底边自然齐平；提示文案改为： 统计窗口 → 统计最近多少分钟 触发阈值 → 超过该条数才告警 冷却 → 冷却内不重复发送 （顺带把阈值提示从"0 = 出现即告警"改掉：默认阈值已无 0，0 反而是危险配置） 小屏改单列堆叠（grid-cols-1 sm:grid-cols-3）。 === 实测（936a127b1d15）=== v10 迁移：告警规则 "数据库连接失败" 扩展匹配模式 线上规则：25 个模式，含 NOAUTH / too many clients，阈值 1000，冷却 240m Pod = latest = v0.1 = 936a127b1d15 --- backend/repository/db.go | 24 +++++++++++++++++++++++- backend/seed/alert_rules.go | 4 +++- frontend/src/i18n/locales/en/alerts.ts | 4 +++- frontend/src/i18n/locales/zh/alerts.ts | 4 +++- frontend/src/pages/alerts/index.tsx | 21 +++++++++++++-------- 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/backend/repository/db.go b/backend/repository/db.go index 9876527..d2acfdb 100644 --- a/backend/repository/db.go +++ b/backend/repository/db.go @@ -114,6 +114,22 @@ func (d *JsonDB) raiseLegacyAlertThresholds() { } } +// expandDatabaseErrorPattern 把内置「数据库连接失败」规则升级为覆盖 +// MySQL / PostgreSQL / Redis 的主流报错文案。 +// +// 只替换"仍是旧默认查询"的规则，不覆盖用户自己改过的。 +func (d *JsonDB) expandDatabaseErrorPattern() { + const oldQuery = `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|too many connections)"` + // 见 seed/alert_rules.go 里的同名规则，两处必须保持一致 + const newQuery = `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"` + for i := range d.AlertRules { + if d.AlertRules[i].Query == oldQuery { + log.Printf("[schema] 告警规则 %q 扩展数据库报错匹配模式", d.AlertRules[i].Name) + d.AlertRules[i].Query = newQuery + } + } +} + // normalizeDefaultChannelName 把默认通道的名字统一成「默认告警」。 // // 只改名，不动 Webhook 与 enabled：首次部署的占位通道必须保持"空地址 + 禁用"， @@ -430,7 +446,7 @@ func (d *JsonDB) dedupeLogSourceIDs() { // // v0/v1（无字段）→ 1：新增 auditLogs / schemaVersion（老库走零值兜底即可，无需处理） // v1 → 2：预留示例（如把 ScrapeJobs 从 logPath 拆分时，需按旧结构生成默认 job） -const CurrentSchemaVersion = 9 +const CurrentSchemaVersion = 10 // migrateSchema 按版本号顺序迁移老库。返回是否发生了迁移（需要回写）。 func (d *JsonDB) migrateSchema() bool { @@ -507,6 +523,12 @@ func (d *JsonDB) migrateSchema() bool { // 这里只改名字，不动 Webhook、不动启用状态。 d.normalizeDefaultChannelName() d.SchemaVersion = 9 + case 9: + // v9 → v10：内置「数据库连接失败」规则扩展报错模式 + // （原来只匹配 3 个词，漏掉 PG 的 FATAL: too many clients、 + // Redis 的 NOAUTH / WRONGPASS / connection pool timeout 等常见报错） + d.expandDatabaseErrorPattern() + d.SchemaVersion = 10 default: // 防御：出现未知版本（比当前代码还新）时不乱动，避免降级把数据写坏 log.Printf("[schema] 数据库版本 %d 高于当前代码支持版本 %d，跳过迁移", d.SchemaVersion, CurrentSchemaVersion) diff --git a/backend/seed/alert_rules.go b/backend/seed/alert_rules.go index 2ba8244..d428175 100644 --- a/backend/seed/alert_rules.go +++ b/backend/seed/alert_rules.go @@ -82,7 +82,9 @@ var builtinRules = []builtinRule{ {name: "应用错误激增(error > 10万)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240}, {name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) "`, windowMin: 15, threshold: 2000, cooldown: 240}, {name: "ingress-nginx 4xx 激增", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " 4[0-9][0-9] "`, windowMin: 15, threshold: 20000, cooldown: 240}, - {name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|too many connections)"`, windowMin: 15, threshold: 1000, cooldown: 240}, + // 数据库连接类错误：覆盖 MySQL / PostgreSQL / Redis 的主流报错文案 + // （详见函数上方注释，不要加裸 "EOF" / "timeout"，那会匹配到几乎所有网络日志） + {name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240}, } // seedBuiltinAlertRules 为每个平台管理的 LogSource(managed=true)补齐内置规则: diff --git a/frontend/src/i18n/locales/en/alerts.ts b/frontend/src/i18n/locales/en/alerts.ts index 06d21f4..ea04391 100644 --- a/frontend/src/i18n/locales/en/alerts.ts +++ b/frontend/src/i18n/locales/en/alerts.ts @@ -90,7 +90,9 @@ export default { channelDisabled: '(disabled)', window: 'Window (min)', threshold: 'Threshold (matches)', - thresholdHint: '0 = alert on any match', + windowHint: 'Look back this many minutes', + thresholdHint: 'Alert only above this count', + cooldownHint: 'No repeat within cooldown', cooldown: 'Cooldown (min)', enabled: 'Enable rule', }, diff --git a/frontend/src/i18n/locales/zh/alerts.ts b/frontend/src/i18n/locales/zh/alerts.ts index b3e68e0..576b4bb 100644 --- a/frontend/src/i18n/locales/zh/alerts.ts +++ b/frontend/src/i18n/locales/zh/alerts.ts @@ -86,8 +86,10 @@ export default { channelDisabled: '(禁用)', window: '统计窗口(分)', threshold: '触发阈值(条)', - thresholdHint: '0 = 出现即告警', + windowHint: '统计最近多少分钟', + thresholdHint: '超过该条数才告警', cooldown: '冷却(分)', + cooldownHint: '冷却内不重复发送', enabled: '启用规则', }, } diff --git a/frontend/src/pages/alerts/index.tsx b/frontend/src/pages/alerts/index.tsx index 9ca4fe1..bbb3715 100644 --- a/frontend/src/pages/alerts/index.tsx +++ b/frontend/src/pages/alerts/index.tsx @@ -442,26 +442,31 @@ const AlertsPage = memo(function AlertsPage() { )} /> -
+ {/* 三个字段必须等高：之前只有「触发阈值」带一行提示， + 于是中间那格比其他两格高，提示文字悬在底部，看起来像没对齐。 + 现在每格都给一行同结构提示（min-h 固定高度），底边自然齐平。 */} +
( - + {t('form.window')} - + +

{t('form.windowHint')}

 )} /> ( - + {t('form.threshold')} - + +

{t('form.thresholdHint')}

-

{t('form.thresholdHint')}

 )} /> ( - + {t('form.cooldown')} - + +

{t('form.cooldownHint')}

 )} />