feat(alerts): 数据库连接告警覆盖 MySQL/PG/Redis 主流报错;规则表单三字段对齐
=== 1. 「数据库连接失败」匹配模式 3 → 25 个 ===
原来只匹配 connection refused / reset by peer / too many connections,
漏掉了 PG 的 FATAL: too many clients、Redis 的 NOAUTH / WRONGPASS /
connection pool timeout 等实际最常见的报错。
按数据库分类补齐:
通用网络层
connection refused / connection reset by peer / broken pipe /
no route to host / cannot assign requested address /
connection timed out / i/o timeout / unexpected EOF /
server closed the connection unexpectedly
MySQL
too many connections / communications link failure /
driver: bad connection / could not create connection /
Access denied for user
PostgreSQL
FATAL: password authentication failed / sorry, too many clients /
terminating connection
Redis
NOAUTH / WRONGPASS / READONLY you can't write against /
CLUSTERDOWN / max number of clients reached
连接池(Java/Go 应用层最先炸的地方)
connection pool / connection is not available
**刻意不加**的(会匹配到几乎所有网络日志,等于没告警):
裸 "EOF"、"timeout"、"error"、"refused"、"deadlock"
阈值保持 1000/15min(模式变多不代表命中变多,都是真错误才命中)。
迁移 v9→v10 只替换"仍是旧默认查询"的规则,用户自定义过的不动。
=== 2. 规则表单三个字段没对齐 ===
现象:统计窗口(分) / 触发阈值(条) / 冷却(分) 三格并排,
但只有中间的「触发阈值」下方多了一行提示文字,
导致中间那格更高、提示文字悬在底部,看起来像没对齐。
修法:三格统一为「标签 + 输入框 + 一行提示(min-h-4 固定高度)」,
底边自然齐平;提示文案改为:
统计窗口 → 统计最近多少分钟
触发阈值 → 超过该条数才告警
冷却 → 冷却内不重复发送
(顺带把阈值提示从"0 = 出现即告警"改掉:默认阈值已无 0,0 反而是危险配置)
小屏改单列堆叠(grid-cols-1 sm:grid-cols-3)。
=== 实测(936a127b1d15)===
v10 迁移:告警规则 "数据库连接失败" 扩展匹配模式
线上规则:25 个模式,含 NOAUTH / too many clients,阈值 1000,冷却 240m
Pod = latest = v0.1 = 936a127b1d15
This commit is contained in:
1 parent
cb1ef1dff4
commit
7369a17598
5 files changed
+45
-12
No files matched your search
@@ -114,6 +114,22 @@ func (d *JsonDB) raiseLegacyAlertThresholds() {
|
||||
}
|
||||
}
|
||||
|
||||
// expandDatabaseErrorPattern 把内置「数据库连接失败」规则升级为覆盖
|
||||
// MySQL / PostgreSQL / Redis 的主流报错文案。
|
||||
//
|
||||
// 只替换"仍是旧默认查询"的规则,不覆盖用户自己改过的。
|
||||
func (d *JsonDB) expandDatabaseErrorPattern() {
|
||||
const oldQuery = `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|too many connections)"`
|
||||
// 见 seed/alert_rules.go 里的同名规则,两处必须保持一致
|
||||
const newQuery = `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`
|
||||
for i := range d.AlertRules {
|
||||
if d.AlertRules[i].Query == oldQuery {
|
||||
log.Printf("[schema] 告警规则 %q 扩展数据库报错匹配模式", d.AlertRules[i].Name)
|
||||
d.AlertRules[i].Query = newQuery
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeDefaultChannelName 把默认通道的名字统一成「默认告警」。
|
||||
//
|
||||
// 只改名,不动 Webhook 与 enabled:首次部署的占位通道必须保持"空地址 + 禁用",
|
||||
@@ -430,7 +446,7 @@ func (d *JsonDB) dedupeLogSourceIDs() {
|
||||
//
|
||||
// v0/v1(无字段)→ 1:新增 auditLogs / schemaVersion(老库走零值兜底即可,无需处理)
|
||||
// v1 → 2:预留示例(如把 ScrapeJobs 从 logPath 拆分时,需按旧结构生成默认 job)
|
||||
const CurrentSchemaVersion = 9
|
||||
const CurrentSchemaVersion = 10
|
||||
|
||||
// migrateSchema 按版本号顺序迁移老库。返回是否发生了迁移(需要回写)。
|
||||
func (d *JsonDB) migrateSchema() bool {
|
||||
@@ -507,6 +523,12 @@ func (d *JsonDB) migrateSchema() bool {
|
||||
// 这里只改名字,不动 Webhook、不动启用状态。
|
||||
d.normalizeDefaultChannelName()
|
||||
d.SchemaVersion = 9
|
||||
case 9:
|
||||
// v9 → v10:内置「数据库连接失败」规则扩展报错模式
|
||||
// (原来只匹配 3 个词,漏掉 PG 的 FATAL: too many clients、
|
||||
// Redis 的 NOAUTH / WRONGPASS / connection pool timeout 等常见报错)
|
||||
d.expandDatabaseErrorPattern()
|
||||
d.SchemaVersion = 10
|
||||
default:
|
||||
// 防御:出现未知版本(比当前代码还新)时不乱动,避免降级把数据写坏
|
||||
log.Printf("[schema] 数据库版本 %d 高于当前代码支持版本 %d,跳过迁移", d.SchemaVersion, CurrentSchemaVersion)
|
||||
|
||||
@@ -82,7 +82,9 @@ var builtinRules = []builtinRule{
|
||||
{name: "应用错误激增(error > 10万)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 100000, cooldown: 240},
|
||||
{name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) "`, windowMin: 15, threshold: 2000, cooldown: 240},
|
||||
{name: "ingress-nginx 4xx 激增", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " 4[0-9][0-9] "`, windowMin: 15, threshold: 20000, cooldown: 240},
|
||||
{name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|too many connections)"`, windowMin: 15, threshold: 1000, cooldown: 240},
|
||||
// 数据库连接类错误:覆盖 MySQL / PostgreSQL / Redis 的主流报错文案
|
||||
// (详见函数上方注释,不要加裸 "EOF" / "timeout",那会匹配到几乎所有网络日志)
|
||||
{name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|broken pipe|no route to host|cannot assign requested address|connection timed out|i/o timeout|too many connections|too many clients already|max number of clients reached|connection pool|connection is not available|could not create connection|communications link failure|server closed the connection unexpectedly|unexpected EOF|driver: bad connection|FATAL:\\s*(password authentication failed|sorry, too many clients|terminating connection)|NOAUTH|WRONGPASS|READONLY you can.t write|CLUSTERDOWN)"`, windowMin: 15, threshold: 1000, cooldown: 240},
|
||||
}
|
||||
|
||||
// seedBuiltinAlertRules 为每个平台管理的 LogSource(managed=true)补齐内置规则:
|
||||
|
||||
@@ -90,7 +90,9 @@ export default {
|
||||
channelDisabled: '(disabled)',
|
||||
window: 'Window (min)',
|
||||
threshold: 'Threshold (matches)',
|
||||
thresholdHint: '0 = alert on any match',
|
||||
windowHint: 'Look back this many minutes',
|
||||
thresholdHint: 'Alert only above this count',
|
||||
cooldownHint: 'No repeat within cooldown',
|
||||
cooldown: 'Cooldown (min)',
|
||||
enabled: 'Enable rule',
|
||||
},
|
||||
|
||||
@@ -86,8 +86,10 @@ export default {
|
||||
channelDisabled: '(禁用)',
|
||||
window: '统计窗口(分)',
|
||||
threshold: '触发阈值(条)',
|
||||
thresholdHint: '0 = 出现即告警',
|
||||
windowHint: '统计最近多少分钟',
|
||||
thresholdHint: '超过该条数才告警',
|
||||
cooldown: '冷却(分)',
|
||||
cooldownHint: '冷却内不重复发送',
|
||||
enabled: '启用规则',
|
||||
},
|
||||
}
|
||||
@@ -442,26 +442,31 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)} />
|
||||
<div className="grid grid-cols-3 gap-4">
|
||||
{/* 三个字段必须等高:之前只有「触发阈值」带一行提示,
|
||||
于是中间那格比其他两格高,提示文字悬在底部,看起来像没对齐。
|
||||
现在每格都给一行同结构提示(min-h 固定高度),底边自然齐平。 */}
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
|
||||
<FormField control={form.control} name="windowMin" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormItem className="flex flex-col">
|
||||
<FormLabel>{t('form.window')}</FormLabel>
|
||||
<FormControl><Input type="number" {...field} /></FormControl>
|
||||
<FormControl><Input type="number" min={1} {...field} /></FormControl>
|
||||
<p className="mt-1 min-h-4 text-xs text-muted-foreground">{t('form.windowHint')}</p>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)} />
|
||||
<FormField control={form.control} name="threshold" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormItem className="flex flex-col">
|
||||
<FormLabel>{t('form.threshold')}</FormLabel>
|
||||
<FormControl><Input type="number" {...field} /></FormControl>
|
||||
<FormControl><Input type="number" min={0} {...field} /></FormControl>
|
||||
<p className="mt-1 min-h-4 text-xs text-muted-foreground">{t('form.thresholdHint')}</p>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">{t('form.thresholdHint')}</p>
|
||||
</FormItem>
|
||||
)} />
|
||||
<FormField control={form.control} name="cooldownMin" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormItem className="flex flex-col">
|
||||
<FormLabel>{t('form.cooldown')}</FormLabel>
|
||||
<FormControl><Input type="number" {...field} /></FormControl>
|
||||
<FormControl><Input type="number" min={1} {...field} /></FormControl>
|
||||
<p className="mt-1 min-h-4 text-xs text-muted-foreground">{t('form.cooldownHint')}</p>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)} />
|
||||
|
||||
Reference in new issue
Block a user