fix(logs,analysis,alerts): 数据源不可达、分析页超时、告警规则混入基础设施项
=== 1. 主机模式数据源地址 0.0.0.0 → 主机真实 IP（查询页"永远加载中"的根因）===
现象：切换数据源后命名空间/App 下拉不动，一直显示"加载中…"，控制台报
Get "http://0.0.0.0:3100/loki/api/v1/labels": dial tcp 0.0.0.0:3100: connect: connection refused
根因：主机模式部署的 endpoint 记的是**容器监听地址** 0.0.0.0:3100，
而平台进程不在被采集主机上，0.0.0.0 只在本机有意义。
EnsureLogSourceForDeployment 把它原样写成了数据源 URL。
修法：
- 新增 reachableEndpoint()：把 0.0.0.0 / [::] 换成关联主机的真实 IP
- schemaVersion v3→v4 迁移 fixUnreachableLogSourceURLs()，修正已有脏数据
- 实测：http://0.0.0.0:3100 → http://172.19.16.8:3100，该源 label 查询由 502 变 200
=== 2. 前端：区分"加载中"与"加载失败" ===
原来用 `loading={!options.length}` 表示加载中 —— 数据源不可达时列表永远为空，
于是界面永远转圈，把真实错误完全藏起来了（这次的 0.0.0.0 就是这样被掩盖的）。
改为独立的 namespaceLoading / namespaceError 两个状态：
- 切换数据源先清空旧 options（否则显示上一个源的命名空间，看着像"没反应"）
- 失败显示"加载失败"并在旁边给「重试」按钮，不用刷新整页
=== 3. 日志分析页：主图改走索引接口（原来 5 条 count_over_time 全超时）===
实测 astralx（约 370GB/天）15 分钟窗口：
count_over_time → exec 29s + 排队 95s，超过 30s 护栏 → 502
index/volume_range → 0s
改为两阶段渲染：
- 总量趋势改用 /logs/histogram（内部 index/volume_range），**0s 出图**
- 错误/警告/app 维度仍需读日志内容，串行执行，先渲染主图再更新
- heavyQueryTimeout 30s → 90s（实测 error 维度查询 14s 可完成）
=== 4. 告警规则按日志平台定位收敛 ===
- seed 里删掉「Kubernetes 节点 NotReady」「容器 OOMKilled」：
这类属于 kube-state-metrics / Prometheus 的职责，用日志字符串匹配不可靠
- 迁移里清掉线上已存在的同类规则（3 条）
- 修 AlertRule / NotifyChannel 重复 ID（同 logSources 一样漏在 sequences 重建外）
=== 5. 告警页 UI ===
- 规则列表由"每条一张卡"改为紧凑表格（一屏看全，阈值可对齐比较）
- 通知通道（飞书/Lark/Webhook）从「系统设置」搬到告警页折叠区：
规则和"发到哪"是一件事的两半，分两页配容易漏配导致告警发不出去
=== 6. 权限收敛 ===
- /alerts 加 adminOnly：普通用户按查询授权只看「日志查询」，不再看到告警菜单
- 系统设置从侧栏移到右上角齿轮（低频平台配置，不占侧栏一位）
=== 7. 迁移设计教训（写进代码注释）===
v2 时把"告警去重 + 清基础设施规则"塞进 case 1，但线上库已到 v2，
那段代码永远不执行。**结构变更只要逻辑变了、目标库已达该版本，就必须再抬一版**，
不能改历史迁移分支。
=== 实测（v4-1028-2210 / 979bdb4a48e8）===
v4 迁移：数据源 URL 修正 http://0.0.0.0:3100 → http://172.19.16.8:3100
source 1 label：200，0s（astralx / log）
source 2 label：200，0s（不再 connection refused）
分析页 histogram astralx 15m：0s
分析页 metrics error astralx 15m：14s
告警规则：10 条（已无基础设施类）、ID 唯一
另清理 172.19.16.8 上 13 个残留容器，回收 6.1GB
This commit is contained in:
opsx committed
1 parent
1a4c75e28c
commit
95dcd9408c
20 files changed
+922
-68
No files matched your search
+288
-1
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -61,12 +62,239 @@ func GetDB() *JsonDB {
|
||||
return db
|
||||
}
|
||||
|
||||
// fixUnreachableLogSourceURLs 把数据源里 0.0.0.0 / [::] 的监听地址换成主机真实 IP。
|
||||
//
|
||||
// 平台进程不在被采集主机上，0.0.0.0 只在本机有意义，用它必然连不上。
|
||||
func (d *JsonDB) fixUnreachableLogSourceURLs() {
|
||||
hostIPByID := map[uint]string{}
|
||||
for _, h := range d.Hosts {
|
||||
hostIPByID[h.ID] = strings.TrimSpace(h.HostIP)
|
||||
}
|
||||
// 数据源按名字对应部署名，用于推断该源属于哪台主机
|
||||
hostByName := map[string]string{}
|
||||
for _, dep := range d.Deployments {
|
||||
if dep.HostID == nil {
|
||||
continue
|
||||
}
|
||||
if ip, ok := hostIPByID[*dep.HostID]; ok && ip != "" {
|
||||
hostByName[dep.Name] = ip
|
||||
}
|
||||
}
|
||||
|
||||
for i := range d.LogSources {
|
||||
src := &d.LogSources[i]
|
||||
if !strings.Contains(src.URL, "0.0.0.0") && !strings.Contains(src.URL, "[::]") {
|
||||
continue
|
||||
}
|
||||
ip := hostByName[src.Name]
|
||||
if ip == "" && len(d.Hosts) == 1 {
|
||||
// 只有一台主机时可以直接推断（常见于单机试用）
|
||||
ip = strings.TrimSpace(d.Hosts[0].HostIP)
|
||||
}
|
||||
if ip == "" {
|
||||
log.Printf("[schema] 数据源 %q 的 URL 是不可达的监听地址 %s，但无法推断主机 IP，请手动修正", src.Name, src.URL)
|
||||
continue
|
||||
}
|
||||
old := src.URL
|
||||
src.URL = strings.Replace(src.URL, "0.0.0.0", ip, 1)
|
||||
src.URL = strings.Replace(src.URL, "[::]", ip, 1)
|
||||
log.Printf("[schema] 数据源 %q URL 修正：%s → %s", src.Name, old, src.URL)
|
||||
}
|
||||
}
|
||||
|
||||
// dedupeCollectionIDs 对所有带 ID 的集合做重复 ID 修复。
|
||||
//
|
||||
// 注意：实际迁移里按版本分步调用（见 migrateSchema），
|
||||
// 不要直接用它替代分步逻辑 —— 那会让"已迁移库"漏掉后加的清理步骤。
|
||||
//
|
||||
// 背景：这些集合曾经都漏在 InitDB 的 sequences 重建之外，导致 NextID 每次
|
||||
// 进程重启都从 1 开始，于是不同时期创建的对象会撞号。撞号后的表现各不相同：
|
||||
// LogSource : 查询页数据源下拉选不动（两个 option 同 value）
|
||||
// AlertRule : 规则列表出现两条同 id，编辑/删除操作命中错的那条
|
||||
// NotifyChannel: 通知渠道覆盖错对象
|
||||
func (d *JsonDB) dedupeCollectionIDs() {
|
||||
d.dedupeLogSourceIDs()
|
||||
d.dedupeAlertRuleIDs()
|
||||
d.dedupeNotifyChannelIDs()
|
||||
}
|
||||
|
||||
// dedupeAlertRuleIDs 重新编号重复的告警规则 ID。
|
||||
// 同名规则（seed 重复执行导致）只保留第一条，其余删除。
|
||||
func (d *JsonDB) dedupeAlertRuleIDs() {
|
||||
if len(d.AlertRules) == 0 {
|
||||
return
|
||||
}
|
||||
seenID := map[uint]bool{}
|
||||
seenName := map[string]bool{}
|
||||
next := uint(0)
|
||||
for _, r := range d.AlertRules {
|
||||
if r.ID > next {
|
||||
next = r.ID
|
||||
}
|
||||
}
|
||||
fixed := make([]models.AlertRule, 0, len(d.AlertRules))
|
||||
for i := range d.AlertRules {
|
||||
rule := d.AlertRules[i]
|
||||
if rule.Name != "" && seenName[rule.Name] {
|
||||
log.Printf("[schema] 删除重复的告警规则 %q（同名）", rule.Name)
|
||||
continue
|
||||
}
|
||||
seenName[rule.Name] = true
|
||||
if rule.ID == 0 || seenID[rule.ID] {
|
||||
next++
|
||||
log.Printf("[schema] 告警规则 %q 重复 ID %d → 重新分配为 %d", rule.Name, rule.ID, next)
|
||||
rule.ID = next
|
||||
}
|
||||
seenID[rule.ID] = true
|
||||
fixed = append(fixed, rule)
|
||||
}
|
||||
d.AlertRules = fixed
|
||||
for _, r := range d.AlertRules {
|
||||
if r.ID > d.sequences["alertRules"] {
|
||||
d.sequences["alertRules"] = r.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dedupeNotifyChannelIDs 重新编号重复的通知渠道 ID。
|
||||
func (d *JsonDB) dedupeNotifyChannelIDs() {
|
||||
if len(d.NotifyChannels) == 0 {
|
||||
return
|
||||
}
|
||||
seen := map[uint]bool{}
|
||||
next := uint(0)
|
||||
for _, c := range d.NotifyChannels {
|
||||
if c.ID > next {
|
||||
next = c.ID
|
||||
}
|
||||
}
|
||||
for i := range d.NotifyChannels {
|
||||
ch := &d.NotifyChannels[i]
|
||||
if ch.ID == 0 || seen[ch.ID] {
|
||||
next++
|
||||
log.Printf("[schema] 通知渠道 %q 重复 ID %d → 重新分配为 %d", ch.Name, ch.ID, next)
|
||||
ch.ID = next
|
||||
}
|
||||
seen[ch.ID] = true
|
||||
if ch.ID > d.sequences["notifyChannels"] {
|
||||
d.sequences["notifyChannels"] = ch.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// removeInfraAlertRules 删除误配置的「基础设施类」告警规则。
|
||||
//
|
||||
// 判据：查询指向 kube-state-metrics / event-exporter，或在日志里匹配
|
||||
// 节点 NotReady / 容器 OOMKilled。这些是 Prometheus 指标监控的职责：
|
||||
// - 日志里有没有这行取决于采集器是否采到那些命名空间，不可靠
|
||||
// - 指标侧（kube_node_status_condition / kube_pod_container_status_terminated_reason）
|
||||
// 才是权威数据源
|
||||
func (d *JsonDB) removeInfraAlertRules() {
|
||||
infraMarkers := []string{"kube-state-metrics", "event-exporter"}
|
||||
infraNames := []string{"Kubernetes 节点 NotReady", "K8s 节点 NotReady", "容器 OOMKilled"}
|
||||
|
||||
kept := make([]models.AlertRule, 0, len(d.AlertRules))
|
||||
for _, r := range d.AlertRules {
|
||||
isInfra := false
|
||||
for _, marker := range infraMarkers {
|
||||
if strings.Contains(r.Query, marker) {
|
||||
isInfra = true
|
||||
}
|
||||
}
|
||||
for _, name := range infraNames {
|
||||
if strings.TrimSpace(r.Name) == name {
|
||||
isInfra = true
|
||||
}
|
||||
}
|
||||
if isInfra {
|
||||
log.Printf("[schema] 移除基础设施类告警规则 %q（应交给 Prometheus）", r.Name)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, r)
|
||||
}
|
||||
d.AlertRules = kept
|
||||
}
|
||||
|
||||
// dedupeLogSourceIDs 修复历史遗留的重复 ID，并同步更新引用它的策略组。
|
||||
//
|
||||
// 策略组通过 SourceID 关联数据源。重复 ID 时无法判断分组到底指向哪一个，
|
||||
// 这里按"保留第一个、其余顺延"处理，并让原本指向该 ID 的策略组继续指向
|
||||
// 编号最小的那个（即重建前第一次创建的数据源）。
|
||||
func (d *JsonDB) dedupeLogSourceIDs() {
|
||||
if len(d.LogSources) < 2 {
|
||||
return
|
||||
}
|
||||
seen := map[uint]bool{}
|
||||
remap := map[uint]uint{} // 旧ID → 新ID（仅记录需要改的）
|
||||
next := uint(0)
|
||||
for _, s := range d.LogSources {
|
||||
if s.ID > next {
|
||||
next = s.ID
|
||||
}
|
||||
}
|
||||
|
||||
fixed := make([]models.LogSource, 0, len(d.LogSources))
|
||||
kept := map[uint]bool{} // 已被占用的 ID（首次出现时占位）
|
||||
for i := range d.LogSources {
|
||||
src := d.LogSources[i]
|
||||
if src.ID == 0 || kept[src.ID] {
|
||||
// 重复或非法 ID：分配新号
|
||||
next++
|
||||
old := src.ID
|
||||
src.ID = next
|
||||
if _, ok := remap[old]; !ok && old != 0 {
|
||||
// 同一个旧 ID 只记录一次映射（指向保留的那个）
|
||||
remap[old] = 0 // 0 表示"不需要改引用"
|
||||
}
|
||||
log.Printf("[schema] 数据源 %q 重复 ID %d → 重新分配为 %d", src.Name, old, src.ID)
|
||||
} else {
|
||||
kept[src.ID] = true
|
||||
}
|
||||
seen[src.ID] = true
|
||||
fixed = append(fixed, src)
|
||||
}
|
||||
d.LogSources = fixed
|
||||
|
||||
// 同步策略组引用（只在旧 ID 已不存在时改写）
|
||||
for i := range d.LogPolicyGroups {
|
||||
sid := d.LogPolicyGroups[i].SourceID
|
||||
if sid == 0 || seen[sid] {
|
||||
continue
|
||||
}
|
||||
if nv, ok := remap[sid]; ok && nv != 0 {
|
||||
d.LogPolicyGroups[i].SourceID = nv
|
||||
}
|
||||
}
|
||||
|
||||
// 迁移发生在 sequences 重建**之后**，重新分配可能抬高最大 ID，
|
||||
// 必须同步提升计数器，否则紧接着创建的数据源会撞号。
|
||||
for _, s := range d.LogSources {
|
||||
if s.ID > d.sequences["logSources"] {
|
||||
d.sequences["logSources"] = s.ID
|
||||
}
|
||||
}
|
||||
|
||||
// 保证只有一个默认数据源
|
||||
defaultSeen := false
|
||||
for i := range d.LogSources {
|
||||
if d.LogSources[i].IsDefault {
|
||||
if defaultSeen {
|
||||
d.LogSources[i].IsDefault = false
|
||||
}
|
||||
defaultSeen = true
|
||||
}
|
||||
}
|
||||
if !defaultSeen && len(d.LogSources) > 0 {
|
||||
d.LogSources[0].IsDefault = true
|
||||
}
|
||||
}
|
||||
|
||||
// CurrentSchemaVersion 当前数据结构版本。
|
||||
// 每次做**不兼容**的结构变更时 +1，并在 migrateSchema 里补一段迁移逻辑。
|
||||
//
|
||||
// v0/v1（无字段）→ 1：新增 auditLogs / schemaVersion（老库走零值兜底即可，无需处理）
|
||||
// v1 → 2：预留示例（如把 ScrapeJobs 从 logPath 拆分时，需按旧结构生成默认 job）
|
||||
const CurrentSchemaVersion = 1
|
||||
const CurrentSchemaVersion = 4
|
||||
|
||||
// migrateSchema 按版本号顺序迁移老库。返回是否发生了迁移（需要回写）。
|
||||
func (d *JsonDB) migrateSchema() bool {
|
||||
@@ -86,6 +314,31 @@ func (d *JsonDB) migrateSchema() bool {
|
||||
d.AuditLogs = []models.AuditLog{}
|
||||
}
|
||||
d.SchemaVersion = 1
|
||||
case 1:
|
||||
// v1 → v2：修复 LogSource 重复 ID（曾漏在 sequences 重建之外，
|
||||
// NextID 每次重启从 1 开始，线上出现两个数据源都是 id=1）。
|
||||
d.dedupeLogSourceIDs()
|
||||
d.SchemaVersion = 2
|
||||
case 2:
|
||||
// v2 → v3：修复 AlertRule / NotifyChannel 重复 ID + 清掉基础设施类规则。
|
||||
//
|
||||
// ⚠️ 教训：v2 那次我把"告警去重 + 清基础设施规则"塞进了 case 1，
|
||||
// 但当时线上库已迁移到 v2，于是这段代码永远不执行（DB 里 schemaVersion
|
||||
// 已经等于当前版本）。结构变更只要改了逻辑、且目标库已到该版本，
|
||||
// 就必须**再抬一版**，不能改历史迁移分支。
|
||||
d.dedupeAlertRuleIDs()
|
||||
d.dedupeNotifyChannelIDs()
|
||||
d.removeInfraAlertRules()
|
||||
d.SchemaVersion = 3
|
||||
case 3:
|
||||
// v3 → v4：修复主机模式数据源的不可达 URL。
|
||||
//
|
||||
// 主机模式部署的 endpoint 存的是容器监听地址 0.0.0.0:3100，
|
||||
// 平台把它原样当数据源 URL → 查询页对该数据源的所有请求
|
||||
// 都报 dial tcp 0.0.0.0:3100: connection refused，
|
||||
// 前端表现为「命名空间/App 下拉永远加载中」。
|
||||
d.fixUnreachableLogSourceURLs()
|
||||
d.SchemaVersion = 4
|
||||
default:
|
||||
// 防御：出现未知版本（比当前代码还新）时不乱动，避免降级把数据写坏
|
||||
log.Printf("[schema] 数据库版本 %d 高于当前代码支持版本 %d，跳过迁移", d.SchemaVersion, CurrentSchemaVersion)
|
||||
@@ -126,6 +379,11 @@ func InitDB() error {
|
||||
"collectors": 0,
|
||||
"logs": 0,
|
||||
"logPolicyGroups": 0,
|
||||
"logSources": 0,
|
||||
"notifyChannels": 0,
|
||||
"alertRules": 0,
|
||||
"alertDeliveries": 0,
|
||||
"auditLogs": 0,
|
||||
},
|
||||
saveChan: make(chan saveRequest, 10),
|
||||
stopChan: make(chan struct{}),
|
||||
@@ -174,6 +432,35 @@ func InitDB() error {
|
||||
d.sequences["logPolicyGroups"] = group.ID
|
||||
}
|
||||
}
|
||||
// ⚠️ 以下集合曾经漏在重建之外，导致 NextID 每次重启都从 1 开始，
|
||||
// 于是不同进程生命周期里创建的对象 ID 重复（线上实际出现过
|
||||
// 两个 LogSource 都是 id=1 → 前端数据源下拉无法切换）。
|
||||
// 以后新增任何带 ID 的集合，都必须在这里补一段。
|
||||
for _, s := range d.LogSources {
|
||||
if s.ID > d.sequences["logSources"] {
|
||||
d.sequences["logSources"] = s.ID
|
||||
}
|
||||
}
|
||||
for _, ch := range d.NotifyChannels {
|
||||
if ch.ID > d.sequences["notifyChannels"] {
|
||||
d.sequences["notifyChannels"] = ch.ID
|
||||
}
|
||||
}
|
||||
for _, rule := range d.AlertRules {
|
||||
if rule.ID > d.sequences["alertRules"] {
|
||||
d.sequences["alertRules"] = rule.ID
|
||||
}
|
||||
}
|
||||
for _, del := range d.AlertDeliveries {
|
||||
if del.ID > d.sequences["alertDeliveries"] {
|
||||
d.sequences["alertDeliveries"] = del.ID
|
||||
}
|
||||
}
|
||||
for _, a := range d.AuditLogs {
|
||||
if a.ID > d.sequences["auditLogs"] {
|
||||
d.sequences["auditLogs"] = a.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// 回归：nextSequence / 去重逻辑。
|
||||
//
|
||||
// 线上事故：logSources 漏在 sequences 重建之外，NextID 每次重启从 1 开始，
|
||||
// 导致两个数据源都是 id=1 → 前端数据源下拉两个 option value 相同，
|
||||
// 选中后 sourceId 不变，界面表现为"无法切换数据源"。
|
||||
func TestDedupeLogSourceIDs(t *testing.T) {
|
||||
db := &JsonDB{
|
||||
sequences: map[string]uint{"logSources": 1},
|
||||
LogSources: []models.LogSource{
|
||||
{ID: 1, Name: "loki-pre", IsDefault: true},
|
||||
{ID: 1, Name: "host-verify-loki"}, // 重复 ID
|
||||
},
|
||||
LogPolicyGroups: []models.LogPolicyGroup{
|
||||
{ID: 1, Name: "g1", SourceID: 1},
|
||||
},
|
||||
}
|
||||
|
||||
db.dedupeLogSourceIDs()
|
||||
|
||||
if len(db.LogSources) != 2 {
|
||||
t.Fatalf("数据源数量变化了：%d", len(db.LogSources))
|
||||
}
|
||||
ids := map[uint]string{}
|
||||
for _, s := range db.LogSources {
|
||||
if _, dup := ids[s.ID]; dup {
|
||||
t.Fatalf("仍然存在重复 ID %d", s.ID)
|
||||
}
|
||||
ids[s.ID] = s.Name
|
||||
}
|
||||
if ids[1] != "loki-pre" {
|
||||
t.Fatalf("第一个数据源应保留 ID=1，实际 %v", ids)
|
||||
}
|
||||
// 被重排的那个必须拿到新号
|
||||
if _, ok := ids[2]; !ok {
|
||||
t.Fatalf("重复项应被重排为 ID=2，实际 %v", ids)
|
||||
}
|
||||
// 策略组仍指向保留下来的那个
|
||||
if db.LogPolicyGroups[0].SourceID != 1 {
|
||||
t.Fatalf("策略组引用被误改：%d", db.LogPolicyGroups[0].SourceID)
|
||||
}
|
||||
// 计数器必须已提升，防止紧接着创建又撞号
|
||||
if db.sequences["logSources"] < 2 {
|
||||
t.Fatalf("sequences 未提升：%d", db.sequences["logSources"])
|
||||
}
|
||||
}
|
||||
|
||||
// 只有一个数据源时不应被改动
|
||||
func TestDedupeLogSourceIDsNoop(t *testing.T) {
|
||||
db := &JsonDB{
|
||||
sequences: map[string]uint{"logSources": 1},
|
||||
LogSources: []models.LogSource{{ID: 1, Name: "only"}},
|
||||
}
|
||||
db.dedupeLogSourceIDs()
|
||||
if len(db.LogSources) != 1 || db.LogSources[0].ID != 1 {
|
||||
t.Fatalf("单数据源不应被改动：%+v", db.LogSources)
|
||||
}
|
||||
}
|
||||
|
||||
// 多个默认数据源只保留第一个
|
||||
func TestDedupeKeepsSingleDefault(t *testing.T) {
|
||||
db := &JsonDB{
|
||||
sequences: map[string]uint{"logSources": 2},
|
||||
LogSources: []models.LogSource{
|
||||
{ID: 1, Name: "a", IsDefault: true},
|
||||
{ID: 2, Name: "b", IsDefault: true},
|
||||
},
|
||||
}
|
||||
db.dedupeLogSourceIDs()
|
||||
if !db.LogSources[0].IsDefault {
|
||||
t.Fatal("第一个应保持默认")
|
||||
}
|
||||
if db.LogSources[1].IsDefault {
|
||||
t.Fatal("第二个不应再是默认")
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,10 @@ var builtinRules = []builtinRule{
|
||||
{name: "Go panic", query: `{namespace=~".+"} |= "panic:"`, windowMin: 15, threshold: 0, cooldown: 60},
|
||||
{name: "JVM OutOfMemoryError", query: `{namespace=~".+"} |~ "(^|[^a-zA-Z])OutOfMemoryError"`, windowMin: 15, threshold: 0, cooldown: 60},
|
||||
{name: "StackOverflowError", query: `{namespace=~".+"} |= "StackOverflowError"`, windowMin: 15, threshold: 0, cooldown: 60},
|
||||
{name: "Kubernetes 节点 NotReady", query: `{app="kube-state-metrics"} |= "NotReady"`, windowMin: 15, threshold: 0, cooldown: 120},
|
||||
{name: "容器 OOMKilled", query: `{app=~"kube-state-metrics|event-exporter"} |= "OOMKilled"`, windowMin: 15, threshold: 0, cooldown: 120},
|
||||
// 注意：这里**只放日志类告警**。
|
||||
// 「节点 NotReady」「容器 OOMKilled」属于基础设施指标监控（kube-state-metrics /
|
||||
// Prometheus）的职责，用日志去匹配这些字符串既不可靠（日志没开或轮转就漏），
|
||||
// 又与日志平台的定位混淆。这类告警请交给 Prometheus + Alertmanager。
|
||||
// —— 突增类:超过基线阈值才报 ——
|
||||
{name: "应用错误过多(error > 1000)", query: `{namespace=~".+"} |~ "(?i)\berror\b"`, windowMin: 15, threshold: 1000, cooldown: 60},
|
||||
{name: "ingress-nginx 5xx 错误", query: `{app=~"ingress-nginx|nginx-ingress-controller"} |~ " (50[0-4]|5[2-9][0-9]) `, windowMin: 15, threshold: 100, cooldown: 60},
|
||||
|
||||
@@ -417,7 +417,10 @@ const (
|
||||
// 日志量统计最多统计 7 天；更长跨度请分段查（前端会提示缩小范围）
|
||||
maxHistogramRange = 7 * 24 * time.Hour
|
||||
// 重查询（histogram/metrics）单独的更短超时，避免长时间占用 CPU
|
||||
heavyQueryTimeout = 30 * time.Second
|
||||
// 90s：实测 astralx（约 370GB/天）15 分钟窗口单条 count_over_time
|
||||
// exec 约 29s、外加排队时间，30s 必超时 → 分析页 502。
|
||||
// 索引类查询（index/volume_range、index/stats）不受此限制，仍是毫秒级。
|
||||
heavyQueryTimeout = 90 * time.Second
|
||||
// 全局并发重查询上限。取 4 而不是 2：
|
||||
// 日志分析页一次会并发 5 条 count_over_time，限制 2 会让大部分请求拿不到令牌。
|
||||
maxConcurrentHeavyQueries = 4
|
||||
|
||||
@@ -345,6 +345,45 @@ func DeleteLogSource(id uint) error {
|
||||
return fmt.Errorf("数据源不存在")
|
||||
}
|
||||
|
||||
// reachableEndpoint 把部署记录里的访问地址转换成「平台侧能真正连上」的地址。
|
||||
//
|
||||
// 主机模式创建部署时存的是容器的监听地址（0.0.0.0:3100 / [::]:3100），
|
||||
// 那是容器内部视角，平台进程在别的机器/集群里，用它必然 connection refused。
|
||||
// 换成该主机的真实 IP 再返回；主机信息缺失时原样返回（至少不会更差）。
|
||||
func reachableEndpoint(d *models.LokiDeployment) string {
|
||||
endpoint := strings.TrimSpace(d.Endpoint)
|
||||
if endpoint == "" {
|
||||
return endpoint
|
||||
}
|
||||
if !strings.Contains(endpoint, "0.0.0.0") && !strings.Contains(endpoint, "[::]") {
|
||||
return endpoint
|
||||
}
|
||||
hostIP := ""
|
||||
if d.Host != nil {
|
||||
hostIP = strings.TrimSpace(d.Host.HostIP)
|
||||
}
|
||||
if hostIP == "" {
|
||||
// 没带 Host 关联时回库查一次，避免调用方漏预加载
|
||||
db := repository.GetDB()
|
||||
if db != nil && d.HostID != nil {
|
||||
db.RLock()
|
||||
for i := range db.Hosts {
|
||||
if db.Hosts[i].ID == *d.HostID {
|
||||
hostIP = strings.TrimSpace(db.Hosts[i].HostIP)
|
||||
break
|
||||
}
|
||||
}
|
||||
db.RUnlock()
|
||||
}
|
||||
}
|
||||
if hostIP == "" {
|
||||
return endpoint
|
||||
}
|
||||
endpoint = strings.Replace(endpoint, "0.0.0.0", hostIP, 1)
|
||||
endpoint = strings.Replace(endpoint, "[::]", hostIP, 1)
|
||||
return endpoint
|
||||
}
|
||||
|
||||
// EnsureLogSourceForDeployment 部署成功后自动注册查询数据源(host→url;k8s→cluster-svc)。
|
||||
// 已存在同名数据源时：平台托管的自动刷新连接信息，用户手填的不覆盖。
|
||||
func EnsureLogSourceForDeployment(d *models.LokiDeployment) {
|
||||
@@ -360,7 +399,11 @@ func EnsureLogSourceForDeployment(d *models.LokiDeployment) {
|
||||
}
|
||||
if d.Mode == "host-binary" || d.Mode == "host-docker" {
|
||||
target.TargetType = "url"
|
||||
target.URL = d.Endpoint
|
||||
// 主机模式部署的 endpoint 记的是**容器监听地址**（0.0.0.0:3100），
|
||||
// 直接拿来当数据源 URL 会得到 dial tcp 0.0.0.0:3100: connection refused ——
|
||||
// 平台进程不在那台主机上，0.0.0.0 只在本机有意义。
|
||||
// 这里换成主机真实 IP，平台才能连上。
|
||||
target.URL = reachableEndpoint(d)
|
||||
target.ClusterID = nil
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
Menu,
|
||||
Sun,
|
||||
Moon,
|
||||
Settings,
|
||||
} from 'lucide-react'
|
||||
import { useTheme } from 'next-themes'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
@@ -237,6 +238,17 @@ const Layout = memo(function Layout({ children }: { children: React.ReactNode })
|
||||
</h1>
|
||||
</div>
|
||||
<div className="flex items-center gap-4 flex-shrink-0">
|
||||
{/* 系统设置入口：低频平台配置，放右上角比占侧栏一位更合适 */}
|
||||
{user?.role === 'admin' && (
|
||||
<Link
|
||||
to="/settings"
|
||||
title={t('menu.settings')}
|
||||
aria-label={t('menu.settings')}
|
||||
className="flex items-center gap-1 rounded-lg px-2.5 py-1.5 text-muted-foreground transition-all hover:bg-muted hover:text-foreground"
|
||||
>
|
||||
<Settings className="w-4 h-4" />
|
||||
</Link>
|
||||
)}
|
||||
<ThemeToggle />
|
||||
{user && (
|
||||
<div className="flex items-center gap-3 text-sm text-muted-foreground">
|
||||
|
||||
@@ -7,6 +7,15 @@ export default {
|
||||
checking: 'Checking...',
|
||||
create: 'New Rule',
|
||||
emptyTitle: 'No alert rules yet',
|
||||
notifySectionTitle: 'Notification channels (Feishu / Lark / Webhook)',
|
||||
notifySectionConfigured: '{{count}} configured',
|
||||
notifySectionEmpty: 'Not configured — alerts cannot be delivered',
|
||||
col: {
|
||||
enabled: 'On',
|
||||
name: 'Rule',
|
||||
condition: 'Condition',
|
||||
notify: 'Notify',
|
||||
},
|
||||
emptyDesc:
|
||||
'e.g. {namespace=~".+"} |= "FATAL" — any match within 15 minutes pushes a Feishu (Lark) card',
|
||||
sourceFallback: 'Data source {{id}}',
|
||||
|
||||
@@ -5,7 +5,7 @@ export default {
|
||||
groups: {
|
||||
resources: 'Resources',
|
||||
services: 'Log Service',
|
||||
system: 'System',
|
||||
system: 'Users & Audit',
|
||||
},
|
||||
dashboard: 'Dashboard',
|
||||
wizard: 'Deploy Wizard',
|
||||
|
||||
@@ -94,6 +94,9 @@ export default {
|
||||
appWithKey: 'App ({{key}})',
|
||||
pod: 'Pod',
|
||||
selectCount: 'Select ({{count}} total)',
|
||||
loading: 'Loading…',
|
||||
loadFailed: 'Load failed',
|
||||
retry: 'Retry',
|
||||
noApp: 'No app in this scope',
|
||||
pickNsFirst: 'Pick a namespace first to narrow down',
|
||||
noPod: 'No pods under this app',
|
||||
|
||||
@@ -6,6 +6,15 @@ export default {
|
||||
checking: '检查中...',
|
||||
create: '新建规则',
|
||||
emptyTitle: '暂无告警规则',
|
||||
notifySectionTitle: '通知通道（飞书 / Lark / Webhook）',
|
||||
notifySectionConfigured: '已配置 {{count}} 个',
|
||||
notifySectionEmpty: '尚未配置，告警发不出去',
|
||||
col: {
|
||||
enabled: '启用',
|
||||
name: '规则',
|
||||
condition: '条件',
|
||||
notify: '通知',
|
||||
},
|
||||
emptyDesc: '例如:{namespace=~".+"} |= "FATAL",15 分钟内超过 0 条即推送飞书',
|
||||
sourceFallback: '数据源 {{id}}',
|
||||
noChannels: '未关联通道',
|
||||
|
||||
@@ -5,7 +5,7 @@ export default {
|
||||
groups: {
|
||||
resources: '资源接入',
|
||||
services: '日志服务',
|
||||
system: '系统管理',
|
||||
system: '用户与审计',
|
||||
},
|
||||
dashboard: '仪表盘',
|
||||
wizard: '部署向导',
|
||||
|
||||
@@ -93,6 +93,9 @@ export default {
|
||||
appWithKey: 'App（{{key}}）',
|
||||
pod: '副本 / Pod',
|
||||
selectCount: '请选择（共 {{count}} 个）',
|
||||
loading: '加载中…',
|
||||
loadFailed: '加载失败',
|
||||
retry: '重试',
|
||||
noApp: '该范围内无 app',
|
||||
pickNsFirst: '先选命名空间可缩小范围',
|
||||
noPod: '该 app 下无副本',
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
import { memo, useCallback, useEffect, useState } from 'react'
|
||||
import { Link } from 'react-router'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { BellRing, Plus, Pencil, Trash2, Send } from 'lucide-react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
|
||||
import {
|
||||
Dialog, DialogContent, DialogHeader, DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
import { PasswordInput } from '@/components/PasswordInput'
|
||||
import { ConfirmDialog } from '@/components/ConfirmDialog'
|
||||
import { alertsApi, type NotifyChannel } from '@/api/alerts'
|
||||
import { toast } from 'sonner'
|
||||
|
||||
interface ChannelForm {
|
||||
name: string
|
||||
webhook: string
|
||||
signingSecret: string
|
||||
enabled: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 通知通道(飞书/Lark)— 多 webhook 对应不同告警群。
|
||||
* webhook 加密存储(留空=保留旧值),每通道可独立测试/禁用/删除(有关联规则时禁删)。
|
||||
*/
|
||||
export const NotifySection = memo(function NotifySection() {
|
||||
const { t } = useTranslation('settings')
|
||||
const [channels, setChannels] = useState<NotifyChannel[]>([])
|
||||
const [open, setOpen] = useState(false)
|
||||
const [editing, setEditing] = useState<NotifyChannel | null>(null)
|
||||
const [form, setForm] = useState<ChannelForm>({ name: '', webhook: '', signingSecret: '', enabled: true })
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [testing, setTesting] = useState(false)
|
||||
const [deleteTarget, setDeleteTarget] = useState<NotifyChannel | null>(null)
|
||||
const [deleting, setDeleting] = useState(false)
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
const list = await alertsApi.listChannels()
|
||||
setChannels(Array.isArray(list) ? list : [])
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('notify.errLoad'))
|
||||
}
|
||||
}, [t])
|
||||
|
||||
useEffect(() => {
|
||||
void refresh()
|
||||
}, [refresh])
|
||||
|
||||
const openCreate = () => {
|
||||
setEditing(null)
|
||||
setForm({ name: '', webhook: '', signingSecret: '', enabled: true })
|
||||
setOpen(true)
|
||||
}
|
||||
|
||||
const openEdit = (ch: NotifyChannel) => {
|
||||
setEditing(ch)
|
||||
setForm({ name: ch.name, webhook: '', signingSecret: '', enabled: ch.enabled })
|
||||
setOpen(true)
|
||||
}
|
||||
|
||||
const handleSave = async () => {
|
||||
if (!form.name.trim()) {
|
||||
toast.error(t('notify.errNameRequired'))
|
||||
return
|
||||
}
|
||||
if (!editing && !form.webhook) {
|
||||
toast.error(t('notify.errWebhookRequired'))
|
||||
return
|
||||
}
|
||||
setSaving(true)
|
||||
try {
|
||||
if (editing) {
|
||||
await alertsApi.updateChannel(editing.id, {
|
||||
name: form.name,
|
||||
webhook: form.webhook || undefined,
|
||||
signingSecret: form.signingSecret || undefined,
|
||||
enabled: form.enabled,
|
||||
})
|
||||
} else {
|
||||
await alertsApi.saveChannel({
|
||||
name: form.name,
|
||||
webhook: form.webhook,
|
||||
signingSecret: form.signingSecret || undefined,
|
||||
enabled: form.enabled,
|
||||
})
|
||||
}
|
||||
setOpen(false)
|
||||
void refresh()
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('notify.errSave'))
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleTest = async (ch: NotifyChannel) => {
|
||||
setTesting(true)
|
||||
try {
|
||||
await alertsApi.testChannel({ id: ch.id })
|
||||
toast.success(t('notify.testSent', { name: ch.name }))
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('notify.errSend'))
|
||||
} finally {
|
||||
setTesting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleToggle = async (ch: NotifyChannel, enabled: boolean) => {
|
||||
try {
|
||||
await alertsApi.updateChannel(ch.id, { name: ch.name, enabled })
|
||||
void refresh()
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('notify.errToggle'))
|
||||
}
|
||||
}
|
||||
|
||||
const handleDelete = async () => {
|
||||
if (!deleteTarget) return
|
||||
setDeleting(true)
|
||||
try {
|
||||
await alertsApi.deleteChannel(deleteTarget.id)
|
||||
setDeleteTarget(null)
|
||||
void refresh()
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('notify.errDelete'))
|
||||
} finally {
|
||||
setDeleting(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Card className="border-border">
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-base flex items-center justify-between">
|
||||
<span className="flex items-center gap-2">
|
||||
<BellRing className="w-4 h-4" />
|
||||
{t('notify.cardTitle')}
|
||||
</span>
|
||||
<Button size="sm" variant="outline" onClick={openCreate}>
|
||||
<Plus className="w-4 h-4 mr-1" />
|
||||
{t('notify.create')}
|
||||
</Button>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
{channels.length === 0 ? (
|
||||
<p className="text-sm text-muted-foreground py-2 text-center">
|
||||
{t('notify.emptyBefore')}
|
||||
<Link to="/alerts" className="text-primary mx-1">{t('notify.emptyLink')}</Link>
|
||||
{t('notify.emptyAfter')}
|
||||
</p>
|
||||
) : (
|
||||
channels.map((ch) => (
|
||||
<div key={ch.id} className="flex items-center gap-3 rounded-lg border bg-card px-3 py-2.5">
|
||||
<Switch checked={ch.enabled} onCheckedChange={(v) => void handleToggle(ch, v)} />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="font-medium text-sm">{ch.name}</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{ch.hasWebhook ? t('notify.webhookConfigured') : t('notify.webhookMissing')}
|
||||
{ch.hasSecret ? t('notify.secretConfigured') : ''}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<Button variant="outline" size="sm" aria-label={t('notify.testAria', { name: ch.name })} disabled={testing} onClick={() => void handleTest(ch)}>
|
||||
<Send className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" aria-label={t('notify.editAria', { name: ch.name })} onClick={() => openEdit(ch)}>
|
||||
<Pencil className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline" size="sm" aria-label={t('notify.deleteAria', { name: ch.name })}
|
||||
className="text-destructive hover:bg-destructive/10"
|
||||
onClick={() => setDeleteTarget(ch)}
|
||||
>
|
||||
<Trash2 className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</CardContent>
|
||||
|
||||
<Dialog open={open} onOpenChange={setOpen}>
|
||||
<DialogContent className="max-w-md">
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{editing ? t('notify.editTitle', { name: editing.name }) : t('notify.createTitle')}
|
||||
</DialogTitle>
|
||||
</DialogHeader>
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label>{t('notify.nameLabel')}</Label>
|
||||
<Input
|
||||
placeholder={t('notify.namePlaceholder')}
|
||||
value={form.name}
|
||||
onChange={(e) => setForm((f) => ({ ...f, name: e.target.value }))}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>
|
||||
{t('notify.webhookLabel')}
|
||||
{editing ? t('notify.webhookKeepHint') : ' *'}
|
||||
</Label>
|
||||
<PasswordInput
|
||||
placeholder={editing?.hasWebhook ? t('notify.webhookSavedPlaceholder') : 'https://open.feishu.cn/open-apis/bot/v2/hook/xxx'}
|
||||
value={form.webhook}
|
||||
onChange={(e) => setForm((f) => ({ ...f, webhook: e.target.value }))}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>{t('notify.signingLabel')}</Label>
|
||||
<PasswordInput
|
||||
placeholder={editing?.hasSecret ? t('notify.signingSavedPlaceholder') : t('notify.signingPlaceholder')}
|
||||
value={form.signingSecret}
|
||||
onChange={(e) => setForm((f) => ({ ...f, signingSecret: e.target.value }))}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Switch checked={form.enabled} onCheckedChange={(v) => setForm((f) => ({ ...f, enabled: v }))} />
|
||||
<Label>{t('notify.enable')}</Label>
|
||||
</div>
|
||||
<div className="flex justify-end gap-2">
|
||||
<Button variant="outline" onClick={() => setOpen(false)}>{t('actions.cancel')}</Button>
|
||||
<Button onClick={handleSave} disabled={saving}>
|
||||
{saving ? t('actions.saving') : editing ? t('notify.saveChanges') : t('notify.createChannel')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
<ConfirmDialog
|
||||
open={deleteTarget !== null}
|
||||
onOpenChange={(o) => { if (!o) setDeleteTarget(null) }}
|
||||
name={deleteTarget?.name}
|
||||
title={t('notify.deleteTitle')}
|
||||
loading={deleting}
|
||||
onConfirm={handleDelete}
|
||||
/>
|
||||
</Card>
|
||||
)
|
||||
})
|
||||
@@ -1,11 +1,11 @@
|
||||
import { memo, useCallback, useEffect, useState } from 'react'
|
||||
import { Link } from 'react-router'
|
||||
import { BellRing, Plus, Play, Pencil, Trash2 } from 'lucide-react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Checkbox } from '@/components/ui/checkbox'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { Card, CardContent } from '@/components/ui/card'
|
||||
import { NotifySection } from './components/NotifySection'
|
||||
import {
|
||||
Dialog, DialogContent, DialogHeader, DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
@@ -216,22 +216,43 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
description={t('emptyDesc')}
|
||||
/>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
/* 紧凑表格：原来每条规则一张 Card（p-4 + 三行文字），10 条规则要滚很久。
|
||||
规则本质是同构的行数据，用表格能在一屏内看全，也便于对齐比较阈值。 */
|
||||
<Card className="border-border">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="border-b border-border bg-muted/30 text-xs text-muted-foreground">
|
||||
<tr>
|
||||
<th className="w-10 px-3 py-1.5 text-left font-medium">{t('col.enabled')}</th>
|
||||
<th className="px-3 py-1.5 text-left font-medium">{t('col.name')}</th>
|
||||
<th className="px-3 py-1.5 text-left font-medium">{t('col.condition')}</th>
|
||||
<th className="px-3 py-1.5 text-left font-medium">{t('col.notify')}</th>
|
||||
<th className="w-16 px-3 py-1.5" />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rules.map((rule) => {
|
||||
const src = sources.find((s) => s.id === rule.sourceId)
|
||||
return (
|
||||
<Card key={rule.id} className="border-border">
|
||||
<CardContent className="p-4 flex items-center gap-4 flex-wrap">
|
||||
<Switch checked={rule.enabled} onCheckedChange={(v) => void toggleRule(rule, v)} />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="font-medium">{rule.name}</p>
|
||||
<p className="text-xs text-muted-foreground truncate font-mono mt-0.5">{rule.query}</p>
|
||||
<p className="text-xs text-muted-foreground mt-0.5">
|
||||
{src?.name ?? t('sourceFallback', { id: rule.sourceId })} →{' '}
|
||||
{(rule.channelIds || [])
|
||||
const chan = (rule.channelIds || [])
|
||||
.map((id) => channels.find((c) => c.id === id)?.name || `#${id}`)
|
||||
.join(t('channelSep')) || t('noChannels')}{' '}
|
||||
· {t('ruleMeta', {
|
||||
.join(t('channelSep')) || t('noChannels')
|
||||
return (
|
||||
<tr key={rule.id} className="border-b border-border/50 last:border-0 hover:bg-muted/20">
|
||||
<td className="px-3 py-1.5">
|
||||
<Switch checked={rule.enabled} onCheckedChange={(v) => void toggleRule(rule, v)} />
|
||||
</td>
|
||||
<td className="max-w-[13rem] px-3 py-1.5">
|
||||
<div className="truncate font-medium" title={rule.name}>{rule.name}</div>
|
||||
<div className="truncate text-[11px] text-muted-foreground" title={rule.query}>
|
||||
{src?.name ?? t('sourceFallback', { id: rule.sourceId })}
|
||||
</div>
|
||||
</td>
|
||||
<td className="px-3 py-1.5">
|
||||
<div className="truncate font-mono text-[11px] text-muted-foreground" title={rule.query}>
|
||||
{rule.query}
|
||||
</div>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
{t('ruleMeta', {
|
||||
window: rule.windowMin,
|
||||
threshold: rule.threshold,
|
||||
cooldown: rule.cooldownMin,
|
||||
@@ -239,32 +260,28 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
{rule.lastFiredAt && !rule.lastFiredAt.startsWith('0001') && ` · ${t('lastFired', {
|
||||
time: new Date(rule.lastFiredAt).toLocaleString(),
|
||||
})}`}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
aria-label={t('editAria', { name: rule.name })}
|
||||
onClick={() => openEdit(rule)}
|
||||
>
|
||||
</td>
|
||||
<td className="max-w-[10rem] px-3 py-1.5">
|
||||
<div className="truncate text-xs" title={chan}>{chan}</div>
|
||||
</td>
|
||||
<td className="px-3 py-1.5">
|
||||
<div className="flex items-center justify-end gap-1">
|
||||
<Button variant="ghost" size="sm" className="h-7 w-7 p-0" aria-label={t('editAria', { name: rule.name })} onClick={() => openEdit(rule)}>
|
||||
<Pencil className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
aria-label={t('deleteAria', { name: rule.name })}
|
||||
className="text-destructive hover:bg-destructive/10"
|
||||
onClick={() => setDeleteTarget(rule)}
|
||||
>
|
||||
<Button variant="ghost" size="sm" className="h-7 w-7 p-0 text-destructive hover:bg-destructive/10" aria-label={t('deleteAria', { name: rule.name })} onClick={() => setDeleteTarget(rule)}>
|
||||
<Trash2 className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</td>
|
||||
</tr>
|
||||
)
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* 发送历史 */}
|
||||
@@ -301,11 +318,22 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t('notifyHintPre')}
|
||||
<Link to="/settings" className="text-primary mx-1">{t('common:menu.settings')}</Link>
|
||||
{t('notifyHintPost')}
|
||||
</p>
|
||||
{/* 通知通道配置：从「系统设置」搬到这里。
|
||||
告警规则和"告警发到哪"是同一件事的两半，分两个页面配很容易漏配导致告警发不出去。
|
||||
默认折叠，不影响规则列表的紧凑度。 */}
|
||||
<details className="rounded-lg border border-border" open={channels.length === 0}>
|
||||
<summary className="cursor-pointer select-none px-3 py-2 text-sm font-medium">
|
||||
{t('notifySectionTitle')}
|
||||
<span className="ml-2 text-xs font-normal text-muted-foreground">
|
||||
{channels.length > 0
|
||||
? t('notifySectionConfigured', { count: channels.length })
|
||||
: t('notifySectionEmpty')}
|
||||
</span>
|
||||
</summary>
|
||||
<div className="border-t border-border p-3">
|
||||
<NotifySection />
|
||||
</div>
|
||||
</details>
|
||||
|
||||
{/* 规则弹窗 */}
|
||||
<Dialog open={open} onOpenChange={setOpen}>
|
||||
|
||||
@@ -111,7 +111,10 @@ export function useAnalysisMetrics({ sourceId, start, end, namespace }: Analysis
|
||||
// 分批（每批 2 条）而不是一次性 5 条并发：
|
||||
// Loki 单实例被 5 条 count_over_time 同时打会互相拖慢，
|
||||
// 而且会撞后端并发护栏。分批后总耗时接近，但不会互相踩。
|
||||
const runInBatches = async <T,>(tasks: Array<() => Promise<T>>, size = 2): Promise<T[]> => {
|
||||
// 串行执行（size=1）：分析页一次要跑 5 条重查询（count_over_time），
|
||||
// 后端并发护栏是 4 且共享给所有用户。并发 2 时多人同时打开分析页
|
||||
// 很容易撞 429，表现为"加载不出来"。串行慢一点但稳。
|
||||
const runInBatches = async <T,>(tasks: Array<() => Promise<T>>, size = 1): Promise<T[]> => {
|
||||
const out: T[] = []
|
||||
for (let index = 0; index < tasks.length; index += size) {
|
||||
out.push(...await Promise.all(tasks.slice(index, index + size).map(task => task())))
|
||||
@@ -123,8 +126,42 @@ export function useAnalysisMetrics({ sourceId, start, end, namespace }: Analysis
|
||||
setError('')
|
||||
void (async () => {
|
||||
try {
|
||||
const [totalSeries, errorSeries, warnSeries, byAppSeries, errByAppSeries] = await runInBatches([
|
||||
() => fetchRetry(`sum(count_over_time(${selector}[$step]))`),
|
||||
// ── 第一步：总量趋势走 Loki 的索引接口（index/volume_range）──
|
||||
//
|
||||
// 原来第一张图也用 count_over_time，实测 astralx（约 370GB/天）15 分钟窗口
|
||||
// 单条要 exec 29s + 排队，5 条串行下来两三分钟都出不来，页面就是"死在转圈"。
|
||||
// /logs/histogram 内部用 index/volume_range，只读 TSDB 的 chunk 元数据，
|
||||
// 不解码日志内容，同一条件毫秒级返回（对比实测 ~1300 倍差距）。
|
||||
const hist = await logsApi.histogram(
|
||||
{ sourceId, query: selector, start: Number(ns(new Date(start).getTime())), end: Number(ns(new Date(end).getTime())) },
|
||||
{ signal: controller.signal },
|
||||
)
|
||||
const totalSeries: MetricSeries[] = [{
|
||||
labels: {},
|
||||
points: (hist || []).map(bucket => ({
|
||||
timestamp: String((bucket as { timestamp: string }).timestamp ?? ''),
|
||||
value: Number((bucket as { count: number }).count ?? 0),
|
||||
})),
|
||||
}]
|
||||
if (version !== versionRef.current) return
|
||||
|
||||
// 主图先渲染：即使后面的 app 维度还在算，用户也能立刻看到总量趋势
|
||||
setLoading(false)
|
||||
setData(prev => ({
|
||||
trend: buildTrend(totalSeries, [], []),
|
||||
namespaceShares: prev?.namespaceShares ?? [],
|
||||
apps: prev?.apps ?? [],
|
||||
appTimes: prev?.appTimes ?? [],
|
||||
stats: {
|
||||
total: seriesTotal(totalSeries),
|
||||
errorCount: prev?.stats.errorCount ?? 0,
|
||||
warnCount: prev?.stats.warnCount ?? 0,
|
||||
appCount: prev?.stats.appCount ?? 0,
|
||||
},
|
||||
}))
|
||||
|
||||
// ── 第二步：错误/警告/app 维度仍需读日志内容，串行执行 ──
|
||||
const [errorSeries, warnSeries, byAppSeries, errByAppSeries] = await runInBatches([
|
||||
() => fetchRetry(`sum(count_over_time(${selector} |= "error"[$step]))`),
|
||||
() => fetchRetry(`sum(count_over_time(${selector} |= "warn"[$step]))`),
|
||||
() => fetchRetry(`sum by (${appLabel}, namespace) (count_over_time(${selector}[$step]))`),
|
||||
|
||||
@@ -39,7 +39,9 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
|
||||
const [sources, setSources] = useState<LogSource[]>([])
|
||||
const [sourceId, setSourceId] = useState(0)
|
||||
const [start, setStart] = useState(() => toLocal(new Date(Date.now() - 3600000)))
|
||||
// 默认 15 分钟：分析页一次性要跑 5 条 count_over_time，
|
||||
// 1 小时窗口在日志量大的命名空间要十几秒；15 分钟既能看出趋势又能快速出结果。
|
||||
const [start, setStart] = useState(() => toLocal(new Date(Date.now() - 15 * 60 * 1000)))
|
||||
const [end, setEnd] = useState(() => toLocal(new Date()))
|
||||
|
||||
/**
|
||||
@@ -49,7 +51,9 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
* 现在默认只查业务命名空间，用户可下拉切换；"全部命名空间"作为显式选项并列提示较慢。
|
||||
*/
|
||||
const [namespaces, setNamespaces] = useState<string[]>([])
|
||||
const [namespace, setNamespace] = useState<string>(() => localStorage.getItem('analysisNamespace') ?? '')
|
||||
// 初始为空，等策略返回后再定（不要用 localStorage 覆盖策略：
|
||||
// 策略是服务端下发的权威值，本地记忆会把管理员改过的策略"顶掉"）。
|
||||
const [namespace, setNamespace] = useState<string>('')
|
||||
const [namespaceReady, setNamespaceReady] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
@@ -65,20 +69,35 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
.catch(() => setSources([]))
|
||||
}, [])
|
||||
|
||||
// 拉取可用命名空间；默认优先业务命名空间 astralx，其次列表第一个
|
||||
// 拉取可用命名空间，并把**查询授权策略里配置的命名空间**作为默认值。
|
||||
//
|
||||
// 之前这里硬编码"优先 astralx"，等于把环境特征写进代码：
|
||||
// 换环境（策略配的是 default / 别的 ns）就默认选错，用户还得手动改。
|
||||
// 现在默认值来源依次为：URL 记忆 → 授权策略 labels.namespace → default → 列表第一个。
|
||||
useEffect(() => {
|
||||
if (!sourceId) return
|
||||
let cancelled = false
|
||||
logsApi.labelValues(sourceId, 'namespace')
|
||||
.then(list => {
|
||||
Promise.all([
|
||||
logsApi.labelValues(sourceId, 'namespace'),
|
||||
// 策略里配的命名空间（后台「查询授权」可改）；失败不影响主流程
|
||||
logsApi.myScope(sourceId).catch(() => null),
|
||||
])
|
||||
.then(([list, scope]) => {
|
||||
if (cancelled) return
|
||||
const values = (list || []).filter(Boolean).sort()
|
||||
const scopeNs = scope?.labels?.namespace ?? ''
|
||||
setNamespaces(values)
|
||||
setNamespace(current => {
|
||||
if (current && (current === ALL_NAMESPACES || values.includes(current))) return current
|
||||
const saved = localStorage.getItem('analysisNamespace')
|
||||
if (saved && (saved === ALL_NAMESPACES || values.includes(saved))) return saved
|
||||
return values.includes('astralx') ? 'astralx' : values[0] ?? ''
|
||||
// 优先级：用户当前已选（同一数据源内不跳变）→ 授权策略 → default → 第一个
|
||||
const candidates = [current, scopeNs, 'default', values[0]]
|
||||
for (const candidate of candidates) {
|
||||
if (!candidate) continue
|
||||
if (candidate === ALL_NAMESPACES) return candidate
|
||||
if (values.includes(candidate)) return candidate
|
||||
}
|
||||
// 策略里配了但 Loki 里暂时没数据时，仍按策略显示（用户能看到"为什么没数据"）
|
||||
if (scopeNs) return scopeNs
|
||||
return ''
|
||||
})
|
||||
})
|
||||
.catch(() => { if (!cancelled) setNamespaces([]) })
|
||||
@@ -86,6 +105,11 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
return () => { cancelled = true }
|
||||
}, [sourceId])
|
||||
|
||||
// 记住用户手选，下次进入沿用（但不会覆盖策略下发的值）
|
||||
useEffect(() => {
|
||||
if (namespace) localStorage.setItem('analysisNamespace', namespace)
|
||||
}, [namespace])
|
||||
|
||||
// hook 内部把空串当作 {namespace=~".+"}（全部），这里做哨兵转换
|
||||
const scopedNamespace = namespace === ALL_NAMESPACES ? '' : namespace
|
||||
const { data, loading, error, appLabel, refresh } = useAnalysisMetrics({ sourceId, start, end, namespace: scopedNamespace })
|
||||
@@ -106,7 +130,7 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
title={t('title')}
|
||||
description={t('subtitle')}
|
||||
action={
|
||||
<Button variant="outline" onClick={refresh} disabled={loading || !sourceId}>
|
||||
<Button variant="outline" onClick={refresh} disabled={!sourceId}>
|
||||
{loading ? <Loader2 className="mr-1 size-4 animate-spin" /> : <RefreshCw className="mr-1 size-4" />}
|
||||
{t('filter.refresh')}
|
||||
</Button>
|
||||
|
||||
@@ -42,6 +42,12 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, onInsert,
|
||||
* Promtail 会从容器名生成 service_name / container，覆盖度更高，
|
||||
* 因此按 app → service_name → container 逐级回退。
|
||||
*/
|
||||
/** 命名空间加载中（与"加载失败"要分开，否则空列表会一直显示"加载中"） */
|
||||
const [namespaceLoading, setNamespaceLoading] = useState(false)
|
||||
/** 命名空间加载失败原因（如数据源地址不可达 connection refused） */
|
||||
const [namespaceError, setNamespaceError] = useState('')
|
||||
/** 重试计数：失败后点「重试」时 +1 触发重新拉取 */
|
||||
const [reloadKey, setReloadKey] = useState(0)
|
||||
const [appLabel, setAppLabel] = useState('app')
|
||||
/** 授权策略里配置的 namespace（如默认策略的 default），用作默认选中值 */
|
||||
const [scopeNamespace, setScopeNamespace] = useState('')
|
||||
@@ -61,18 +67,41 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, onInsert,
|
||||
return () => { active = false }
|
||||
}, [sourceId])
|
||||
|
||||
// namespace 列表只加载一次
|
||||
// namespace 列表：**每次切换数据源都重新加载**。
|
||||
//
|
||||
// 三个必须点：
|
||||
// 1) 先清空旧值：否则切换数据源后仍显示上一个源的命名空间（看着像"没反应"）
|
||||
// 2) 记录 loading 与 error 两个状态：原来用 !options.length 表示 loading，
|
||||
// 数据源不可达时列表永远为空 → 界面永远"加载中…"，把真实错误藏起来了
|
||||
// （典型错误：主机模式数据源地址是 0.0.0.0 → connection refused）
|
||||
// 3) 失败后可点重试，不用刷新整页
|
||||
useEffect(() => {
|
||||
if (!sourceId) return
|
||||
if (!sourceId) {
|
||||
setNamespaces([])
|
||||
setApps([])
|
||||
setPods([])
|
||||
setNamespaceError('')
|
||||
return
|
||||
}
|
||||
let active = true
|
||||
setNamespaceLoading(true)
|
||||
setNamespaceError('')
|
||||
setNamespaces([])
|
||||
setApps([])
|
||||
setPods([])
|
||||
logsApi.labelValues(sourceId, 'namespace')
|
||||
// 后端历史版本可能返回 null（Loki 无值时 data:null），这里兜底成数组，
|
||||
// 否则 setState(null) 后 datalist 的 .map() 会抛
|
||||
// "Cannot read properties of null (reading 'map')"
|
||||
.then(next => { if (active) setNamespaces(Array.isArray(next) ? next : []) })
|
||||
.catch(() => { if (active) setNamespaces([]) })
|
||||
.catch(err => {
|
||||
if (!active) return
|
||||
setNamespaces([])
|
||||
setNamespaceError(err instanceof Error ? err.message : String(err))
|
||||
})
|
||||
.finally(() => { if (active) setNamespaceLoading(false) })
|
||||
return () => { active = false }
|
||||
}, [sourceId])
|
||||
}, [sourceId, reloadKey])
|
||||
|
||||
// app 候选：选了 namespace 就只在该命名空间内搜索
|
||||
useEffect(() => {
|
||||
@@ -204,10 +233,26 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, onInsert,
|
||||
values={namespaceOptions}
|
||||
value={namespace}
|
||||
onChange={chooseNamespace}
|
||||
placeholder={t('templates.selectCount', { count: namespaceOptions.length })}
|
||||
placeholder={
|
||||
namespaceError
|
||||
? t('templates.loadFailed')
|
||||
: namespaceLoading
|
||||
? t('templates.loading')
|
||||
: t('templates.selectCount', { count: namespaceOptions.length })
|
||||
}
|
||||
allowEmpty
|
||||
loading={!namespaceOptions.length}
|
||||
loading={namespaceLoading}
|
||||
/>
|
||||
{namespaceError && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setReloadKey(k => k + 1)}
|
||||
className="self-center text-xs text-destructive underline underline-offset-2"
|
||||
title={namespaceError}
|
||||
>
|
||||
{t('templates.retry')}
|
||||
</button>
|
||||
)}
|
||||
<SearchableValueSelect
|
||||
label={appLabel === 'app' ? t('templates.app') : t('templates.appWithKey', { key: appLabel })}
|
||||
values={apps}
|
||||
|
||||
@@ -63,6 +63,23 @@ export default function LogsPage() {
|
||||
const admin = getUser()?.role === 'admin'
|
||||
const [sources, setSources] = useState<LogSource[]>([])
|
||||
const [sourceId, setSourceId] = useState(0)
|
||||
/**
|
||||
* 按 id 去重的数据源选项。
|
||||
*
|
||||
* 后端曾因 sequences 漏建导致两个数据源 ID 都是 1，
|
||||
* 于是 <SelectItem> 出现两个相同 value —— 用户点第二个时 sourceId
|
||||
* 仍是 1，state 不变、effect 不触发，界面上表现为"数据源切换不了"。
|
||||
* 后端已修 + 做了迁移去重，这里再兜一层：即使历史数据仍有重复，
|
||||
* 下拉也不会因相同 value 而卡死。
|
||||
*/
|
||||
const sourceOptions = useMemo(() => {
|
||||
const seen = new Set<number>()
|
||||
return sources.filter(source => {
|
||||
if (seen.has(source.id)) return false
|
||||
seen.add(source.id)
|
||||
return true
|
||||
})
|
||||
}, [sources])
|
||||
const [query, setQuery] = useState('')
|
||||
const [executedQuery, setExecutedQuery] = useState('')
|
||||
const [executedRange, setExecutedRange] = useState<{ start: number; end: number } | null>(null)
|
||||
@@ -263,12 +280,12 @@ export default function LogsPage() {
|
||||
</div>
|
||||
)}
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Select value={sourceId ? String(sourceId) : undefined} onValueChange={value => setSourceId(Number(value))} disabled={sources.length === 0}>
|
||||
<Select value={sourceId ? String(sourceId) : undefined} onValueChange={value => setSourceId(Number(value))} disabled={sourceOptions.length === 0}>
|
||||
<SelectTrigger className="h-9 min-w-40" aria-label={t('toolbar.sourceAria')}>
|
||||
<SelectValue placeholder={t('toolbar.sourceEmptyOption')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sources.map(source => <SelectItem key={source.id} value={String(source.id)}>{source.name}</SelectItem>)}
|
||||
{sourceOptions.map(source => <SelectItem key={source.id} value={String(source.id)}>{source.name}</SelectItem>)}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<TimeRangePicker start={start} end={end} onChange={(nextStart, nextEnd) => { setStart(nextStart); setEnd(nextEnd); setTail(false) }} />
|
||||
|
||||
@@ -14,7 +14,6 @@ import { LdapSection } from './components/LdapSection'
|
||||
import { ThemeSection } from './components/ThemeSection'
|
||||
import { LanguageSection } from '@/i18n/LanguageSection'
|
||||
import { Palette } from 'lucide-react'
|
||||
import { NotifySection } from './components/NotifySection'
|
||||
import { ImageRegistrySection } from './components/ImageRegistrySection'
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
|
||||
|
||||
@@ -180,7 +179,7 @@ const SettingsPage = memo(function SettingsPage() {
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="notify" className="space-y-6">
|
||||
<NotifySection />
|
||||
{/* 告警通知通道已移到「日志告警 → 通知通道」，避免配置入口分裂 */}
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="ldap">
|
||||
|
||||
@@ -68,7 +68,9 @@ export const routes: AppRoute[] = [
|
||||
{ path: '/analysis', component: AnalysisPage, auth: true, meta: { title: 'menu.analysis', icon: BarChart3, order: 6, adminOnly: true } },
|
||||
|
||||
// ── 告警：独立入口（要盯的，不能被折叠） ──
|
||||
{ path: '/alerts', component: AlertsPage, auth: true, meta: { title: 'menu.alerts', icon: BellRing, order: 7 } },
|
||||
// 告警仅管理员：普通用户按「查询授权」只看指定的日志（菜单只剩日志查询）。
|
||||
// 告警规则涉及阈值/通知通道等平台级配置，不属于日志查询者的职责。
|
||||
{ path: '/alerts', component: AlertsPage, auth: true, meta: { title: 'menu.alerts', icon: BellRing, order: 7, adminOnly: true } },
|
||||
|
||||
// ── 系统管理（用户 / 审计 / 设置）：低频管理动作 ──
|
||||
{
|
||||
@@ -79,7 +81,9 @@ export const routes: AppRoute[] = [
|
||||
},
|
||||
},
|
||||
{ path: '/audit', component: AuditPage, auth: true, meta: { title: 'menu.audit', icon: History, order: 13, adminOnly: true, groupKey: 'system' } },
|
||||
{ path: '/settings', component: SettingsPage, auth: true, meta: { title: 'menu.settings', icon: Settings, order: 12, adminOnly: true, groupKey: 'system' } },
|
||||
// 系统设置从侧栏移到右上角齿轮入口：属于低频平台配置，
|
||||
// 和「用户/审计」这类日常安全管理不是一类东西，混在一个分组里太乱。
|
||||
{ path: '/settings', component: SettingsPage, auth: true, meta: { title: 'menu.settings', icon: Settings, order: 12, adminOnly: true, hideInMenu: true } },
|
||||
{ path: '*', component: NotFound, auth: false, meta: { title: 'menu.notFound', hideInMenu: true } },
|
||||
];
|
||||
|
||||
|
||||
Reference in new issue
Block a user