logan
2d73661dd1
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application. Validated with go test ./..., npm run build, and npm run lint.
84 lines
3.4 KiB
Go
84 lines
3.4 KiB
Go
package seed
|
|
|
|
// 内置告警规则 — 新环境开箱即用的通用日志告警(参照运维告警脚本的两类模式):
|
|
// 致命类(出现即报):FATAL / panic / OutOfMemoryError / StackOverflowError
|
|
// 突增类(超阈值才报):应用 error 过多 / ingress-nginx 4xx / 5xx
|
|
// 绑定数据源:部署成功自动注册的 LogSource(managed)。
|
|
// 规则默认不启用(通知通道由管理员后期在设置页配置后再开启),ChannelIDs 留空。
|
|
import (
|
|
"time"
|
|
|
|
"opsxloki/models"
|
|
"opsxloki/repository"
|
|
)
|
|
|
|
type builtinRule struct {
|
|
name string
|
|
query string
|
|
windowMin int
|
|
threshold int
|
|
cooldown int
|
|
}
|
|
|
|
var builtinRules = []builtinRule{
|
|
// —— 致命类:15 分钟内出现即报 ——
|
|
{name: "致命日志 FATAL", query: `{namespace=~".+"} |= "FATAL"`, windowMin: 15, threshold: 0, cooldown: 60},
|
|
{name: "Go panic", query: `{namespace=~".+"} |= "panic:"`, windowMin: 15, threshold: 0, cooldown: 60},
|
|
{name: "JVM OutOfMemoryError", query: `{namespace=~".+"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 0, cooldown: 60},
|
|
{name: "StackOverflowError", query: `{namespace=~".+"} |= "StackOverflowError"`, windowMin: 15, threshold: 0, cooldown: 60},
|
|
{name: "K8s 节点 NotReady", query: `{app="kube-state-metrics"} |= "NotReady"`, windowMin: 15, threshold: 0, cooldown: 120},
|
|
{name: "容器 OOMKilled", query: `{app=~"kube-state-metrics|event-exporter"} |= "OOMKilled"`, windowMin: 15, threshold: 0, cooldown: 120},
|
|
// —— 突增类:超过基线阈值才报 ——
|
|
{name: "应用错误过多(error > 1000)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 1000, cooldown: 60},
|
|
{name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) `, windowMin: 15, threshold: 100, cooldown: 60},
|
|
{name: "ingress-nginx 4xx 激增", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " 4[0-9][0-9] "`, windowMin: 15, threshold: 1000, cooldown: 60},
|
|
{name: "数据库连接失败", query: `{namespace=~".+"} |~ "(?i)(connection refused|connection reset by peer|too many connections)"`, windowMin: 15, threshold: 50, cooldown: 60},
|
|
}
|
|
|
|
// seedBuiltinAlertRules 为每个平台管理的 LogSource(managed=true)补齐内置规则:
|
|
// 首次部署 Loki 后即有一套可直接启用的告警模板;通知通道留空,管理员配置后启用。
|
|
// SeedAlertRules 补齐内置告警规则(幂等;公开给启动与部署成功两处调用)
|
|
func SeedAlertRules() {
|
|
db := repository.GetDB()
|
|
db.RLock()
|
|
type srcRef struct {
|
|
id uint
|
|
name string
|
|
}
|
|
var sources []srcRef
|
|
for i := range db.LogSources {
|
|
if db.LogSources[i].Managed {
|
|
sources = append(sources, srcRef{id: db.LogSources[i].ID, name: db.LogSources[i].Name})
|
|
}
|
|
}
|
|
// 已有规则(按 name)跳过
|
|
existing := map[string]bool{}
|
|
for i := range db.AlertRules {
|
|
existing[db.AlertRules[i].Name] = true
|
|
}
|
|
db.RUnlock()
|
|
|
|
if len(sources) == 0 {
|
|
return
|
|
}
|
|
// 绑定第一个 managed 数据源(通常是唯一部署)
|
|
target := sources[0]
|
|
|
|
newID := db.NextID("alertRules")
|
|
db.Lock()
|
|
defer db.Unlock()
|
|
for _, r := range builtinRules {
|
|
if existing[r.name] {
|
|
continue
|
|
}
|
|
newID++
|
|
db.AlertRules = append(db.AlertRules, models.AlertRule{
|
|
ID: newID, Name: r.name, SourceID: target.id,
|
|
Query: r.query, WindowMin: r.windowMin, Threshold: r.threshold,
|
|
Enabled: false, CooldownMin: r.cooldown,
|
|
CreatedAt: time.Now(), UpdatedAt: time.Now(),
|
|
})
|
|
}
|
|
db.RequestSave()
|
|
}
|