feat(settings,analysis): 日志保留时间可配(发送历史保留 7 天)+ 分析页手动刷新;修复我引入的 RWMutex 死锁
=== 0. 【严重】修复上一版引入的死锁(平台写路径全卡死)===
上一版把保留期读取写成 `GetRetentionSettings()`(内部 db.RLock()),
却在 `db.Lock()` 已持有的情况下调用(recordDelivery / RecordAudit 的裁剪路径)。
Go 的 sync.RWMutex **不可重入** → 第一次写投递历史/审计日志就永久死锁。
现象:登录 28 秒返回空(登录也要写审计)、所有写接口挂住。
go build / go vet / 之前的单测(不触发写路径)全都查不出来。
修法:
- 拆出 retentionFromDBLocked(db) / alertDeliveryRetentionLocked(db) /
auditLogRetentionLocked(db),调用方已持锁时用无锁版本
- upsertSettingLocked 里**不能**用 db.NextID()(内部 d.mu.Lock(),同样死锁),
改为锁内自算 max(id)+1
- 新增 retention_test.go:持写锁调用裁剪函数,2 秒超时未返回即判死锁
实测修复后:登录 103ms(原 28000ms)、panic/deadlock 命中 0。
=== 1. 平台数据保留期可配(系统设置 → 通用配置 → 日志保留时间)===
新增 GET/PUT /api/v1/settings/retention:
alertDeliveryDays 告警发送历史保留天数,默认 7(原写死"最近 50 条")
auditLogDays 审计日志保留天数,默认 90(原写死常量)
保存后立即按新策略裁剪一次,不必等下次触发。
校验:发送历史 1~365 天、审计 1~3650 天,超出返回明确中文原因(不是 500)。
注意与「Loki 日志保留期」区分:那个是部署级配置(组件部署 → 保留策略)。
=== 2. 发送历史:50 条硬上限 → 保留 7 天 ===
后端 Deliveries 接口原来写死 `len(out) < 50`,现在按保留期取(响应上限 500 可调),
标题改为「发送历史(保留 N 天)」,N 从设置里读。
=== 3. 菜单「日志服务」→「组件部署」===
=== 4. 分析页:去掉「全部命名空间」+ 改手动刷新 ===
- 移除「全部命名空间」选项:{namespace=~".+"} 要扫全集群(约 370GB/天),
一次分析几十秒,得不偿失;历史值 __all__ 也会被忽略并回退到策略/第一个 ns
- 筛选改为「选择值 / 已生效值」分离:切换命名空间或时间范围**不再自动查询**,
只把刷新按钮点亮并提示「筛选条件已改,点刷新生效」;
点一次才应用并取数。
原因:一次分析要跑多条 count_over_time,切一下下拉就重查一轮,
快速连点会把 Loki 打满(并发护栏 429),还拖慢别人。
=== 实测(9739c7c092e7)===
登录 103ms / token 273 字符
GET retention → {alertDeliveryDays:7, auditLogDays:90}
PUT 0 天 → 400「告警发送历史保留天数需在 1~365 之间」
平台日志 panic/deadlock 命中 0
go test ./... 全绿(含 2 个新死锁回归用例)
This commit is contained in:
1 parent
7369a17598
commit
5e59347c40
27 files changed
+491
-54
No files matched your search
@@ -182,10 +182,18 @@ func (ctrl *AlertsController) CheckNow(c *gin.Context) {
|
||||
|
||||
// Deliveries 发送历史(最近 50)
|
||||
func (ctrl *AlertsController) Deliveries(c *gin.Context) {
|
||||
// 保留期由系统设置控制(默认 7 天),库里已是裁剪后的数据;
|
||||
// 这里只是限制单次响应大小,避免历史多了以后一次吐出上万条。
|
||||
limit := 500
|
||||
if raw := c.Query("limit"); raw != "" {
|
||||
if v, err := strconv.Atoi(raw); err == nil && v > 0 && v <= 2000 {
|
||||
limit = v
|
||||
}
|
||||
}
|
||||
db := repository.GetDB()
|
||||
db.RLock()
|
||||
out := make([]map[string]interface{}, 0, len(db.AlertDeliveries))
|
||||
for i := len(db.AlertDeliveries) - 1; i >= 0 && len(out) < 50; i-- {
|
||||
out := make([]map[string]interface{}, 0, limit)
|
||||
for i := len(db.AlertDeliveries) - 1; i >= 0 && len(out) < limit; i-- {
|
||||
d := db.AlertDeliveries[i]
|
||||
out = append(out, map[string]interface{}{
|
||||
"id": d.ID, "ruleId": d.RuleID, "ruleName": d.RuleName,
|
||||
|
||||
@@ -84,6 +84,25 @@ func (ctrl *SystemController) Backups(c *gin.Context) {
|
||||
utils.Success(c, gin.H{"list": services.ListBackups()})
|
||||
}
|
||||
|
||||
// GetRetention 读取平台数据保留期(告警发送历史 / 审计日志)。
|
||||
func (ctrl *SystemController) GetRetention(c *gin.Context) {
|
||||
utils.Success(c, services.GetRetentionSettings())
|
||||
}
|
||||
|
||||
// SaveRetention 保存平台数据保留期。
|
||||
// 校验范围而不是直接接受:保留 0 天等于"什么都不留",通常是误操作。
|
||||
func (ctrl *SystemController) SaveRetention(c *gin.Context) {
|
||||
var req services.RetentionSettings
|
||||
if !utils.BindAndValidate(c, &req) {
|
||||
return
|
||||
}
|
||||
if err := services.SaveRetentionSettings(req.AlertDeliveryDays, req.AuditLogDays); err != nil {
|
||||
utils.BadRequest(c, err.Error())
|
||||
return
|
||||
}
|
||||
utils.Success(c, services.GetRetentionSettings())
|
||||
}
|
||||
|
||||
// Health 平台自检:一键诊断用。
|
||||
// 聚合「能不能用」的判断,避免排障时在多个页面之间跳。
|
||||
func (ctrl *SystemController) Health(c *gin.Context) {
|
||||
|
||||
@@ -105,6 +105,10 @@ func SetupRouter() *gin.Engine {
|
||||
logs.GET("/health", logsCtrl.Health)
|
||||
}
|
||||
|
||||
// 平台数据保留期(告警发送历史 / 审计日志)
|
||||
authorized.GET("/settings/retention", systemCtrl.GetRetention)
|
||||
authorized.PUT("/settings/retention", middleware.AdminAuth(), systemCtrl.SaveRetention)
|
||||
|
||||
// 镜像源(国内加速/海外/私有仓库)
|
||||
authorized.GET("/settings/image-registry", registryCtrl.Get)
|
||||
authorized.PUT("/settings/image-registry", middleware.AdminAuth(), registryCtrl.Save)
|
||||
|
||||
@@ -227,6 +227,37 @@ func recordCooldownThrottled(ruleID uint) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// pruneDeliveriesLocked 按保留期 + 条数上限裁剪投递历史(调用方必须已持锁)。
|
||||
func pruneDeliveriesLocked(db *repository.JsonDB) {
|
||||
cut := 0
|
||||
// 注意:调用方已持写锁,必须用无锁读取版本,否则 RWMutex 会死锁
|
||||
deadline := time.Now().Add(-alertDeliveryRetentionLocked(db))
|
||||
for cut < len(db.AlertDeliveries) && db.AlertDeliveries[cut].CreatedAt.Before(deadline) {
|
||||
cut++
|
||||
}
|
||||
if remain := len(db.AlertDeliveries) - cut; remain > maxAlertDeliveries {
|
||||
cut += remain - maxAlertDeliveries
|
||||
}
|
||||
if cut > 0 {
|
||||
db.AlertDeliveries = append([]models.AlertDelivery{}, db.AlertDeliveries[cut:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// PruneAlertDeliveries 按当前保留期裁剪一次(供保存设置后立即生效)。
|
||||
func PruneAlertDeliveries() {
|
||||
db := repository.GetDB()
|
||||
if db == nil {
|
||||
return
|
||||
}
|
||||
db.Lock()
|
||||
defer db.Unlock()
|
||||
before := len(db.AlertDeliveries)
|
||||
pruneDeliveriesLocked(db)
|
||||
if len(db.AlertDeliveries) != before {
|
||||
db.RequestSave()
|
||||
}
|
||||
}
|
||||
|
||||
func recordDelivery(ruleID uint, ruleName string, count int, status, reason string) {
|
||||
db := repository.GetDB()
|
||||
newID := db.NextID("alertDeliveries")
|
||||
@@ -235,10 +266,9 @@ func recordDelivery(ruleID uint, ruleName string, count int, status, reason stri
|
||||
ID: newID, RuleID: ruleID, RuleName: ruleName,
|
||||
Count: count, Status: status, Reason: reason,
|
||||
})
|
||||
// 只保留最近 50 条
|
||||
if len(db.AlertDeliveries) > 50 {
|
||||
db.AlertDeliveries = db.AlertDeliveries[len(db.AlertDeliveries)-50:]
|
||||
}
|
||||
// 保留策略:按 告警发送历史保留天数(默认 7 天,系统设置可改)+ 条数硬上限兜底。
|
||||
// 原来写死 只留最近 50 条,真实发送记录一小时内就被冷却记录挤光了。
|
||||
pruneDeliveriesLocked(db)
|
||||
db.RequestSave()
|
||||
db.Unlock()
|
||||
}
|
||||
|
||||
@@ -15,8 +15,7 @@ const (
|
||||
// maxAuditLogs 审计日志条数上限。JSON DB 是全量读进内存、全量写盘,
|
||||
// 不设上限的话审计表会把内存和落盘时间一起拖垮。
|
||||
maxAuditLogs = 20000
|
||||
// auditRetention 审计日志保留天数(合规常见要求 90 天)
|
||||
auditRetention = 90 * 24 * time.Hour
|
||||
// 保留期改由系统设置配置(见 services/retention.go)
|
||||
)
|
||||
|
||||
// RecordAudit 写入一条审计日志。
|
||||
@@ -60,7 +59,9 @@ func RecordAudit(entry models.AuditLog) {
|
||||
func pruneAuditLocked(db *repository.JsonDB) {
|
||||
logs := db.AuditLogs
|
||||
cut := 0
|
||||
deadline := time.Now().Add(-auditRetention)
|
||||
// 保留期来自 系统设置->通用配置->日志保留时间,默认 90 天
|
||||
// 注意:调用方已持写锁,必须用无锁读取版本,否则 RWMutex 会死锁
|
||||
deadline := time.Now().Add(-auditLogRetentionLocked(db))
|
||||
|
||||
// 1) 先按保留期裁
|
||||
for cut < len(logs) && logs[cut].CreatedAt.Before(deadline) {
|
||||
@@ -75,6 +76,21 @@ func pruneAuditLocked(db *repository.JsonDB) {
|
||||
}
|
||||
}
|
||||
|
||||
// PruneAuditLogs 按当前保留期裁剪一次审计日志(供保存设置后立即生效)。
|
||||
func PruneAuditLogs() {
|
||||
db := repository.GetDB()
|
||||
if db == nil {
|
||||
return
|
||||
}
|
||||
db.Lock()
|
||||
defer db.Unlock()
|
||||
before := len(db.AuditLogs)
|
||||
pruneAuditLocked(db)
|
||||
if len(db.AuditLogs) != before {
|
||||
db.RequestSave()
|
||||
}
|
||||
}
|
||||
|
||||
// ListAuditLogs 分页查询审计日志(按时间倒序)。
|
||||
func ListAuditLogs(q models.AuditQuery) ([]models.AuditLog, int64) {
|
||||
db := repository.GetDB()
|
||||
|
||||
@@ -3,9 +3,9 @@ package services
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"lokistack/models"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"lokistack/models"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
// indent 必须与该模板里 `external_labels:` 的层级匹配:
|
||||
// - 主机模板(4 空格)→ 子项 6 空格
|
||||
// - DaemonSet 模板(6 空格)→ 子项 8 空格
|
||||
//
|
||||
// 早先写死 8 空格,主机模式一旦配了标签就会生成非法 YAML,Promtail 直接装不上。
|
||||
func promtailExternalLabels(cluster string, labelsJSON string, indent string) string {
|
||||
lines := []string{fmt.Sprintf("%scluster: %s", indent, yamlScalar(cluster))}
|
||||
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
// 统一一个值要么被限流(COS 503 SlowDown),要么白等(S3 能扛更多)。
|
||||
func TestCompactorDeleteWorkersPerStorageType(t *testing.T) {
|
||||
cases := map[string]int{
|
||||
"cos": 16, // 实测 150 触发 503 SlowDown
|
||||
"obs": 32, // 华为 OBS 保守值
|
||||
"minio": 32, // 自建,取决于磁盘
|
||||
"azure": 32, // 存储账户限流较严
|
||||
"oss": 64, // 阿里云 OSS QPS 较高
|
||||
"cos": 16, // 实测 150 触发 503 SlowDown
|
||||
"obs": 32, // 华为 OBS 保守值
|
||||
"minio": 32, // 自建,取决于磁盘
|
||||
"azure": 32, // 存储账户限流较严
|
||||
"oss": 64, // 阿里云 OSS QPS 较高
|
||||
"gcs": 64,
|
||||
"s3": 96, // AWS S3 限流宽松
|
||||
"s3": 96, // AWS S3 限流宽松
|
||||
"": defaultCompactorDeleteWorkers,
|
||||
"unknown": defaultCompactorDeleteWorkers,
|
||||
}
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"lokistack/models"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// resolveKubeconfig 取集群最终的 kubeconfig 文本(优先内存字段,其次磁盘文件)
|
||||
|
||||
@@ -301,7 +301,6 @@ func patchDeployment(ctx context.Context, cluster *models.Cluster, namespace, na
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
// podDNSBroken 在指定 deployment 的容器里探测 hostname 是否可解析(wget /ready 连通性)。
|
||||
func podDNSBroken(ctx context.Context, cluster *models.Cluster, namespace, deployment, container, host string) bool {
|
||||
pods, err := listDeploymentPods(ctx, cluster, namespace, deployment)
|
||||
@@ -355,7 +354,6 @@ func listDeploymentPods(ctx context.Context, cluster *models.Cluster, namespace,
|
||||
return names, nil
|
||||
}
|
||||
|
||||
|
||||
// lokiNamespaceOf 从 endpoint 的 svc FQDN 提取 Loki 命名空间(log.ns.svc → ns)。
|
||||
func lokiNamespaceOf(loki *models.LokiDeployment, endpoint string) string {
|
||||
if loki != nil && loki.Namespace != "" {
|
||||
|
||||
@@ -123,12 +123,12 @@ type lokiConfigData struct {
|
||||
CompactorStoreType string
|
||||
// CompactorDeleteWorkers 过期 chunk 删除并发,按对象存储类型取默认值
|
||||
CompactorDeleteWorkers int
|
||||
RingInstanceAddr string // 单副本 127.0.0.1;多副本 0.0.0.0(须可被对端寻址)
|
||||
RingKVStore string // 单副本 inmemory;多副本 memberlist
|
||||
MemberlistBlock string // 多副本时非空,注入 memberlist 顶级配置
|
||||
RulerStorageBlock string // 对象存储时非空,注入 ruler.storage.s3/gcs/azure 嵌套配置
|
||||
MultiReplica bool
|
||||
FilesystemStorage bool
|
||||
RingInstanceAddr string // 单副本 127.0.0.1;多副本 0.0.0.0(须可被对端寻址)
|
||||
RingKVStore string // 单副本 inmemory;多副本 memberlist
|
||||
MemberlistBlock string // 多副本时非空,注入 memberlist 顶级配置
|
||||
RulerStorageBlock string // 对象存储时非空,注入 ruler.storage.s3/gcs/azure 嵌套配置
|
||||
MultiReplica bool
|
||||
FilesystemStorage bool
|
||||
}
|
||||
|
||||
func (d *LokiDeployer) generateLokiConfig() string {
|
||||
@@ -181,20 +181,20 @@ func (d *LokiDeployer) generateLokiConfig() string {
|
||||
}
|
||||
|
||||
data := lokiConfigData{
|
||||
AuthEnabled: d.deployment.AuthEnabled,
|
||||
ReplicationFactor: d.replicationFactor(),
|
||||
RetentionPeriod: retentionHours,
|
||||
StorageConfigBlock: storageBlock,
|
||||
ObjectStore: objectStore,
|
||||
RulerStoreType: rulerStore,
|
||||
AuthEnabled: d.deployment.AuthEnabled,
|
||||
ReplicationFactor: d.replicationFactor(),
|
||||
RetentionPeriod: retentionHours,
|
||||
StorageConfigBlock: storageBlock,
|
||||
ObjectStore: objectStore,
|
||||
RulerStoreType: rulerStore,
|
||||
CompactorStoreType: compactorStore,
|
||||
CompactorDeleteWorkers: compactorDeleteWorkers(d.deployment.StorageConfig),
|
||||
RingInstanceAddr: ringAddr,
|
||||
RingKVStore: ringStore,
|
||||
MemberlistBlock: memberlistBlock,
|
||||
RulerStorageBlock: d.buildRulerStorageBlock(),
|
||||
MultiReplica: d.deployment.Replicas >= 2,
|
||||
FilesystemStorage: objectStore == "filesystem",
|
||||
RingInstanceAddr: ringAddr,
|
||||
RingKVStore: ringStore,
|
||||
MemberlistBlock: memberlistBlock,
|
||||
RulerStorageBlock: d.buildRulerStorageBlock(),
|
||||
MultiReplica: d.deployment.Replicas >= 2,
|
||||
FilesystemStorage: objectStore == "filesystem",
|
||||
}
|
||||
|
||||
tmpl, err := template.New("loki-config").Parse(lokiConfigTemplate)
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"lokistack/models"
|
||||
"lokistack/repository"
|
||||
)
|
||||
|
||||
// 平台自身数据的保留期设置(不是 Loki 日志的保留期)。
|
||||
//
|
||||
// 为什么需要可配:不同团队对"告警发送历史留多久 / 审计日志留多久"要求不同 —
|
||||
// 合规场景要 90~180 天,小团队 7 天就够。写死在代码里只能改一次发一次版。
|
||||
//
|
||||
// 注意区分:Loki 的日志保留期是**部署级**配置(组件部署 → 保留策略),
|
||||
// 这里管的是平台自己产生的小数据(投递历史、审计日志)。
|
||||
const (
|
||||
settingKeyAlertDeliveryDays = "alertDeliveryRetentionDays"
|
||||
settingKeyAuditLogDays = "auditLogRetentionDays"
|
||||
|
||||
defaultAlertDeliveryDays = 7
|
||||
defaultAuditLogDays = 90
|
||||
|
||||
// 硬上限:即使配置得很大,也不能让 JSON 文件型 DB 无界增长
|
||||
maxAlertDeliveries = 20000
|
||||
)
|
||||
|
||||
// RetentionSettings 平台数据保留期(天)
|
||||
type RetentionSettings struct {
|
||||
AlertDeliveryDays int `json:"alertDeliveryDays"`
|
||||
AuditLogDays int `json:"auditLogDays"`
|
||||
}
|
||||
|
||||
// GetRetentionSettings 读取保留期配置,未配置时返回默认值。
|
||||
func GetRetentionSettings() RetentionSettings {
|
||||
db := repository.GetDB()
|
||||
if db == nil {
|
||||
return retentionDefaults()
|
||||
}
|
||||
db.RLock()
|
||||
defer db.RUnlock()
|
||||
return retentionFromDBLocked(db)
|
||||
}
|
||||
|
||||
func retentionDefaults() RetentionSettings {
|
||||
return RetentionSettings{
|
||||
AlertDeliveryDays: defaultAlertDeliveryDays,
|
||||
AuditLogDays: defaultAuditLogDays,
|
||||
}
|
||||
}
|
||||
|
||||
// retentionFromDBLocked 从已持有的锁内读取配置,**不会再加锁**。
|
||||
//
|
||||
// 为什么必须单独一个函数:Go 的 sync.RWMutex 不可重入。
|
||||
// 调用方已经 db.Lock() 时再调 GetRetentionSettings()(内部 RLock)会**永久死锁** ——
|
||||
// 实测把整个平台写路径卡死(登录都要写审计,于是登录 28 秒返回空)。
|
||||
func retentionFromDBLocked(db *repository.JsonDB) RetentionSettings {
|
||||
out := retentionDefaults()
|
||||
for _, item := range db.Settings {
|
||||
switch item.Key {
|
||||
case settingKeyAlertDeliveryDays:
|
||||
if v, err := strconv.Atoi(item.Value); err == nil && v > 0 {
|
||||
out.AlertDeliveryDays = v
|
||||
}
|
||||
case settingKeyAuditLogDays:
|
||||
if v, err := strconv.Atoi(item.Value); err == nil && v > 0 {
|
||||
out.AuditLogDays = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SaveRetentionSettings 保存保留期配置(天数)。
|
||||
func SaveRetentionSettings(alertDays, auditDays int) error {
|
||||
if alertDays < 1 || alertDays > 365 {
|
||||
return errors.New("告警发送历史保留天数需在 1~365 之间")
|
||||
}
|
||||
if auditDays < 1 || auditDays > 3650 {
|
||||
return errors.New("审计日志保留天数需在 1~3650 之间")
|
||||
}
|
||||
db := repository.GetDB()
|
||||
if db == nil {
|
||||
return errors.New("数据库未初始化")
|
||||
}
|
||||
db.Lock()
|
||||
upsertSettingLocked(db, settingKeyAlertDeliveryDays, strconv.Itoa(alertDays))
|
||||
upsertSettingLocked(db, settingKeyAuditLogDays, strconv.Itoa(auditDays))
|
||||
db.RequestSave()
|
||||
db.Unlock()
|
||||
|
||||
// 立刻按新策略裁剪一次,避免"改了配置但旧数据还在"
|
||||
PruneAlertDeliveries()
|
||||
PruneAuditLogs()
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertSettingLocked 写入/更新一条设置(调用方必须已持写锁)。
|
||||
func upsertSettingLocked(db *repository.JsonDB, key, value string) {
|
||||
for i := range db.Settings {
|
||||
if db.Settings[i].Key == key {
|
||||
db.Settings[i].Value = value
|
||||
return
|
||||
}
|
||||
}
|
||||
// 不能用 db.NextID():它内部会 d.mu.Lock(),而调用方已经持有写锁,
|
||||
// sync.Mutex 不可重入 → 永久死锁(这个坑在审计日志里踩过一次)。
|
||||
// 这里在锁内自己算最大 ID + 1。
|
||||
nextID := uint(1)
|
||||
for _, item := range db.Settings {
|
||||
if item.ID >= nextID {
|
||||
nextID = item.ID + 1
|
||||
}
|
||||
}
|
||||
db.Settings = append(db.Settings, models.SystemSetting{
|
||||
ID: nextID,
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
|
||||
// alertDeliveryRetentionLocked 返回投递历史保留时长(调用方必须已持锁)。
|
||||
func alertDeliveryRetentionLocked(db *repository.JsonDB) time.Duration {
|
||||
return time.Duration(retentionFromDBLocked(db).AlertDeliveryDays) * 24 * time.Hour
|
||||
}
|
||||
|
||||
// auditLogRetentionLocked 返回审计日志保留时长(调用方必须已持锁)。
|
||||
func auditLogRetentionLocked(db *repository.JsonDB) time.Duration {
|
||||
return time.Duration(retentionFromDBLocked(db).AuditLogDays) * 24 * time.Hour
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lokistack/repository"
|
||||
)
|
||||
|
||||
// 回归:持写锁时裁剪逻辑绝不能再去拿读锁。
|
||||
//
|
||||
// 线上事故:一个 day-retention 配置读取被写成在 db.Lock() 内部调用
|
||||
// GetRetentionSettings()(内部 db.RLock())。Go 的 sync.RWMutex 不可重入,
|
||||
// 于是**第一次写投递历史/审计日志就永久死锁** —— 平台表现是"登录 28 秒返回空",
|
||||
// 因为登录也要写审计。go build / go vet / 单测(不触发写路径)全都查不出来。
|
||||
func TestPruneWhileLockedDoesNotDeadlock(t *testing.T) {
|
||||
db := &repository.JsonDB{}
|
||||
db.Lock()
|
||||
defer db.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pruneDeliveriesLocked(db)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("pruneDeliveriesLocked 在持有写锁时死锁(内部又去拿了读锁)")
|
||||
}
|
||||
}
|
||||
|
||||
// 同理:审计日志裁剪。
|
||||
func TestPruneAuditWhileLockedDoesNotDeadlock(t *testing.T) {
|
||||
db := &repository.JsonDB{}
|
||||
db.Lock()
|
||||
defer db.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pruneAuditLocked(db)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("pruneAuditLocked 在持有写锁时死锁(内部又去拿了读锁)")
|
||||
}
|
||||
}
|
||||
@@ -177,8 +177,8 @@ func stripBucketPrefix(endpoint, bucket string) string {
|
||||
|
||||
// bucketPathFor 根据 endpoint 风格决定桶路径(用于建桶 PUT)。
|
||||
//
|
||||
// 1) 区域端点(推荐): cos.ap-hongkong.myqcloud.com → 路径式 /<bucket>
|
||||
// 2) 虚拟主机式: mybucket-1250000000.cos.<region>… → 路径为 /
|
||||
// 1. 区域端点(推荐): cos.ap-hongkong.myqcloud.com → 路径式 /<bucket>
|
||||
// 2. 虚拟主机式: mybucket-1250000000.cos.<region>… → 路径为 /
|
||||
func bucketPathFor(endpoint, bucket string) string {
|
||||
if endpointHasBucket(endpoint, bucket) {
|
||||
return "/"
|
||||
|
||||
@@ -51,6 +51,14 @@ export interface SelfCheckItem {
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
/** 平台自身数据的保留期(天) */
|
||||
export interface RetentionSettings {
|
||||
/** 告警发送历史保留天数,默认 7 */
|
||||
alertDeliveryDays: number
|
||||
/** 审计日志保留天数,默认 90 */
|
||||
auditLogDays: number
|
||||
}
|
||||
|
||||
export interface SelfCheckResult {
|
||||
checkedAt: string;
|
||||
overall: 'ok' | 'warn' | 'error';
|
||||
@@ -99,4 +107,8 @@ export const systemApi = {
|
||||
},
|
||||
restoreApply: () =>
|
||||
http.post<{ message: string; dbPath: string }>('/system/restore/apply', { confirm: true }),
|
||||
|
||||
// 平台数据保留期(告警发送历史 / 审计日志)
|
||||
retention: () => http.get<RetentionSettings>('/settings/retention'),
|
||||
saveRetention: (data: RetentionSettings) => http.put<RetentionSettings>('/settings/retention', data),
|
||||
};
|
||||
@@ -32,7 +32,7 @@ export default {
|
||||
lastFired: 'Last fired {{time}}',
|
||||
editAria: 'Edit {{name}}',
|
||||
deleteAria: 'Delete {{name}}',
|
||||
historyTitle: 'Delivery history (last 50)',
|
||||
historyTitle: 'Delivery history (kept {{days}} days)',
|
||||
notifyHintPre: 'Notifications not configured? Go to',
|
||||
notifyHintPost: 'to configure a Feishu (Lark) Webhook',
|
||||
editTitle: 'Edit rule · {{name}}',
|
||||
|
||||
@@ -18,6 +18,7 @@ export default {
|
||||
namespaceEmpty: 'Loading namespaces…',
|
||||
namespaceAll: 'All namespaces (slow)',
|
||||
refresh: 'Refresh',
|
||||
pendingApply: 'Filters changed — click Refresh to apply',
|
||||
},
|
||||
stats: {
|
||||
total: 'Total logs',
|
||||
|
||||
@@ -4,7 +4,7 @@ export default {
|
||||
menu: {
|
||||
groups: {
|
||||
resources: 'Resources',
|
||||
services: 'Log Service',
|
||||
services: 'Component Deploy',
|
||||
system: 'System Settings',
|
||||
},
|
||||
dashboard: 'Dashboard',
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
// settings namespace — Settings page (general/password/about, notifications, LDAP, image registry), English copy.
|
||||
// Note: theme & language card titles and ThemeSection copy are owned elsewhere and are not in this dictionary.
|
||||
export default {
|
||||
retention: {
|
||||
title: 'Log retention',
|
||||
desc: 'How long the platform keeps its own records. Loki log retention itself is configured per deployment (Component Deploy → Retention).',
|
||||
alertLabel: 'Alert delivery history (days)',
|
||||
alertHint: 'Expired records are purged automatically; default 7 days',
|
||||
auditLabel: 'Audit logs (days)',
|
||||
auditHint: '90–180 days recommended for compliance',
|
||||
save: 'Save',
|
||||
saving: 'Saving…',
|
||||
saved: 'Retention saved; expired records purged',
|
||||
saveFailed: 'Save failed',
|
||||
},
|
||||
title: 'General',
|
||||
tabs: {
|
||||
general: 'General',
|
||||
|
||||
@@ -29,7 +29,7 @@ export default {
|
||||
lastFired: '上次触发 {{time}}',
|
||||
editAria: '编辑 {{name}}',
|
||||
deleteAria: '删除 {{name}}',
|
||||
historyTitle: '发送历史(最近 50 条)',
|
||||
historyTitle: '发送历史(保留 {{days}} 天)',
|
||||
notifyHintPre: '通知未配置?去',
|
||||
notifyHintPost: '配置飞书 Webhook',
|
||||
editTitle: '编辑规则 · {{name}}',
|
||||
|
||||
@@ -18,6 +18,7 @@ export default {
|
||||
namespaceEmpty: '正在加载命名空间…',
|
||||
namespaceAll: '全部命名空间(较慢)',
|
||||
refresh: '刷新',
|
||||
pendingApply: '筛选条件已改,点「刷新」生效',
|
||||
},
|
||||
stats: {
|
||||
total: '日志总量',
|
||||
|
||||
@@ -4,7 +4,7 @@ export default {
|
||||
menu: {
|
||||
groups: {
|
||||
resources: '资源接入',
|
||||
services: '日志服务',
|
||||
services: '组件部署',
|
||||
system: '系统设置',
|
||||
},
|
||||
dashboard: '仪表盘',
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
// settings 命名空间 — 系统设置页(通用配置/密码/关于、告警通知、LDAP 接入、镜像源)中文文案
|
||||
// 注意:主题卡片与语言卡片的标题、ThemeSection 内文案由其他模块负责,不在本词典内。
|
||||
export default {
|
||||
retention: {
|
||||
title: '日志保留时间',
|
||||
desc: '平台自身数据的保留期。注意:Loki 日志本身的保留期在「组件部署 → 保留策略」里配置。',
|
||||
alertLabel: '告警发送历史(天)',
|
||||
alertHint: '过期记录自动清理,默认 7 天',
|
||||
auditLabel: '审计日志(天)',
|
||||
auditHint: '合规场景建议 90~180 天',
|
||||
save: '保存',
|
||||
saving: '保存中…',
|
||||
saved: '保留策略已保存,过期数据已清理',
|
||||
saveFailed: '保存失败',
|
||||
},
|
||||
title: '通用配置',
|
||||
tabs: {
|
||||
general: '通用配置',
|
||||
|
||||
@@ -21,6 +21,7 @@ import { EmptyState } from '@/components/EmptyState'
|
||||
import { ConfirmDialog } from '@/components/ConfirmDialog'
|
||||
import { alertsApi, type AlertRule, type AlertDelivery, type NotifyChannel } from '@/api/alerts'
|
||||
import { logsApi, type LogSource } from '@/api/logs'
|
||||
import { systemApi } from '@/api/system'
|
||||
import { toast } from 'sonner'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
@@ -85,6 +86,8 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
const [sources, setSources] = useState<LogSource[]>([])
|
||||
const [channels, setChannels] = useState<NotifyChannel[]>([])
|
||||
const [deliveries, setDeliveries] = useState<AlertDelivery[]>([])
|
||||
/** 发送历史保留天数(系统设置 → 通用配置 → 日志保留时间),标题里展示 */
|
||||
const [historyDays, setHistoryDays] = useState(7)
|
||||
const [open, setOpen] = useState(false)
|
||||
const [editing, setEditing] = useState<AlertRule | null>(null)
|
||||
const [deleteTarget, setDeleteTarget] = useState<AlertRule | null>(null)
|
||||
@@ -99,9 +102,11 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
const [r, s, d, ch] = await Promise.all([
|
||||
const [r, s, d, ch, ret] = await Promise.all([
|
||||
alertsApi.listRules(), logsApi.sources(), alertsApi.deliveries(), alertsApi.listChannels(),
|
||||
systemApi.retention().catch(() => null),
|
||||
])
|
||||
if (ret?.alertDeliveryDays) setHistoryDays(ret.alertDeliveryDays)
|
||||
setRules(Array.isArray(r) ? r : (r as unknown as { list: AlertRule[] }).list ?? [])
|
||||
setSources(s.list)
|
||||
setDeliveries(Array.isArray(d) ? d : [])
|
||||
@@ -308,7 +313,7 @@ const AlertsPage = memo(function AlertsPage() {
|
||||
) : (
|
||||
<Card className="border-border">
|
||||
<CardContent className="p-4">
|
||||
<h2 className="text-sm font-semibold mb-3">{t('historyTitle')}</h2>
|
||||
<h2 className="text-sm font-semibold mb-3">{t('historyTitle', { days: historyDays })}</h2>
|
||||
<div className="divide-y divide-border text-sm">
|
||||
{deliveries.map((d) => (
|
||||
<div key={d.id} className="py-2 flex items-center gap-3 flex-wrap">
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { memo, useEffect, useState } from 'react'
|
||||
import { memo, useCallback, useEffect, useState } from 'react'
|
||||
import { useNavigate } from 'react-router'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import '@/i18n/domains'
|
||||
@@ -43,8 +43,16 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
const [sourceId, setSourceId] = useState(0)
|
||||
// 默认 15 分钟:分析页一次性要跑 5 条 count_over_time,
|
||||
// 1 小时窗口在日志量大的命名空间要十几秒;15 分钟既能看出趋势又能快速出结果。
|
||||
// ── "选择值" 与 "已生效值" 分离 ──────────────────────────────────
|
||||
// start/end/namespace 是**已生效**值,查询只认它们;
|
||||
// sel* 是筛选栏里的**选择**值。
|
||||
// 这样切换命名空间/时间只改变选择,不触发查询;点「刷新」才生效。
|
||||
// 原因:一次分析要跑多条 count_over_time,切一下下拉就重查一轮很浪费,
|
||||
// 用户快速连点几下会把 Loki 打满(并发护栏 429)。
|
||||
const [start, setStart] = useState(() => toLocal(new Date(Date.now() - 15 * 60 * 1000)))
|
||||
const [end, setEnd] = useState(() => toLocal(new Date()))
|
||||
const [selStart, setSelStart] = useState(() => toLocal(new Date(Date.now() - 15 * 60 * 1000)))
|
||||
const [selEnd, setSelEnd] = useState(() => toLocal(new Date()))
|
||||
|
||||
/**
|
||||
* 命名空间筛选。
|
||||
@@ -56,6 +64,8 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
// 初始为空,等策略返回后再定(不要用 localStorage 覆盖策略:
|
||||
// 策略是服务端下发的权威值,本地记忆会把管理员改过的策略"顶掉")。
|
||||
const [namespace, setNamespace] = useState<string>('')
|
||||
/** 命名空间下拉里的选择值(未点刷新前不参与查询) */
|
||||
const [selNamespace, setSelNamespace] = useState<string>('')
|
||||
const [namespaceReady, setNamespaceReady] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
@@ -89,17 +99,24 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
const values = (list || []).filter(Boolean).sort()
|
||||
const scopeNs = scope?.labels?.namespace ?? ''
|
||||
setNamespaces(values)
|
||||
setNamespace(current => {
|
||||
const resolve = (current: string) => {
|
||||
// 优先级:用户当前已选(同一数据源内不跳变)→ 授权策略 → default → 第一个
|
||||
// 注意:已移除「全部命名空间」选项 —— {namespace=~".+"} 要扫全集群
|
||||
// (实测约 370GB/天),一次分析跑几十秒,得不偿失。
|
||||
const candidates = [current, scopeNs, 'default', values[0]]
|
||||
for (const candidate of candidates) {
|
||||
if (!candidate) continue
|
||||
if (candidate === ALL_NAMESPACES) return candidate
|
||||
if (!candidate || candidate === ALL_NAMESPACES) continue
|
||||
if (values.includes(candidate)) return candidate
|
||||
}
|
||||
// 策略里配了但 Loki 里暂时没数据时,仍按策略显示(用户能看到"为什么没数据")
|
||||
if (scopeNs) return scopeNs
|
||||
return ''
|
||||
}
|
||||
// 首次解析结果同时作为"选择值"和"已生效值",保证进页面就有一屏数据
|
||||
setNamespace(current => {
|
||||
const next = resolve(current)
|
||||
setSelNamespace(next)
|
||||
return next
|
||||
})
|
||||
})
|
||||
.catch(() => { if (!cancelled) setNamespaces([]) })
|
||||
@@ -133,6 +150,20 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
end,
|
||||
namespace: scopedNamespace,
|
||||
})
|
||||
|
||||
// 选择值与已生效值不一致 → 提示"点刷新生效"
|
||||
const pendingApply =
|
||||
namespaceReady && (selNamespace !== namespace || selStart !== start || selEnd !== end)
|
||||
|
||||
/** 把筛选栏里的选择值应用到查询:点一次「刷新」才生效 */
|
||||
const applySelection = useCallback(() => {
|
||||
setNamespace(selNamespace)
|
||||
setStart(selStart)
|
||||
setEnd(selEnd)
|
||||
// 参数没变(只是点刷新)时也要重新取数
|
||||
refresh()
|
||||
}, [selNamespace, selStart, selEnd, refresh])
|
||||
|
||||
// 每日统计固定近 7 天,不随上方时间范围变化;错误就地展示,不影响主区
|
||||
// 每日量可看范围:7 / 30 / 90 / 365 天(数据来自平台留存历史)
|
||||
const [dailyDays, setDailyDays] = useState(7)
|
||||
@@ -156,7 +187,7 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
title={t('title')}
|
||||
description={t('subtitle')}
|
||||
action={
|
||||
<Button variant="outline" onClick={refresh} disabled={!sourceId}>
|
||||
<Button variant={pendingApply ? 'default' : 'outline'} onClick={applySelection} disabled={!sourceId}>
|
||||
{loading ? <Loader2 className="mr-1 size-4 animate-spin" /> : <RefreshCw className="mr-1 size-4" />}
|
||||
{t('filter.refresh')}
|
||||
</Button>
|
||||
@@ -179,9 +210,10 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select
|
||||
value={namespace || undefined}
|
||||
value={selNamespace || undefined}
|
||||
onValueChange={value => {
|
||||
setNamespace(value)
|
||||
// 只改选择值,不触发查询 —— 统一由「刷新」按钮生效
|
||||
setSelNamespace(value)
|
||||
try { localStorage.setItem('analysisNamespace', value) } catch { /* 隐私模式忽略 */ }
|
||||
}}
|
||||
>
|
||||
@@ -190,10 +222,12 @@ const AnalysisPage = memo(function AnalysisPage() {
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{namespaces.map(item => <SelectItem key={item} value={item}>{item}</SelectItem>)}
|
||||
<SelectItem value={ALL_NAMESPACES}>{t('filter.namespaceAll')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<TimeRangePicker start={start} end={end} onChange={(nextStart, nextEnd) => { setStart(nextStart); setEnd(nextEnd) }} />
|
||||
<TimeRangePicker start={selStart} end={selEnd} onChange={(nextStart, nextEnd) => { setSelStart(nextStart); setSelEnd(nextEnd) }} />
|
||||
{pendingApply && (
|
||||
<span className="text-xs text-warning">{t('filter.pendingApply')}</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{!namespaceReady && sourceId ? (
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import { memo, useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
|
||||
import { systemApi, type RetentionSettings } from '@/api/system'
|
||||
import { toast } from 'sonner'
|
||||
|
||||
/**
|
||||
* 数据保留 — 平台自身产生的小数据保留多久。
|
||||
*
|
||||
* 注意与「Loki 日志保留期」区分:那个是部署级配置(组件部署 → 保留策略),
|
||||
* 管的是日志本身;这里管的是平台库里的投递历史与审计日志。
|
||||
*/
|
||||
export const RetentionSection = memo(function RetentionSection() {
|
||||
const { t } = useTranslation('settings')
|
||||
const [settings, setSettings] = useState<RetentionSettings>({ alertDeliveryDays: 7, auditLogDays: 90 })
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
systemApi
|
||||
.retention()
|
||||
.then(value => {
|
||||
if (!cancelled && value) setSettings(value)
|
||||
})
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoading(false)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [])
|
||||
|
||||
const save = useCallback(async () => {
|
||||
setSaving(true)
|
||||
try {
|
||||
const value = await systemApi.saveRetention(settings)
|
||||
if (value) setSettings(value)
|
||||
toast.success(t('retention.saved'))
|
||||
} catch (err: unknown) {
|
||||
toast.error(err instanceof Error ? err.message : t('retention.saveFailed'))
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}, [settings, t])
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-base">{t('retention.title')}</CardTitle>
|
||||
<p className="text-xs text-muted-foreground">{t('retention.desc')}</p>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
|
||||
<div className="flex flex-col">
|
||||
<label className="text-sm font-medium" htmlFor="retention-alert">
|
||||
{t('retention.alertLabel')}
|
||||
</label>
|
||||
<Input
|
||||
id="retention-alert"
|
||||
type="number"
|
||||
min={1}
|
||||
max={365}
|
||||
className="mt-1"
|
||||
disabled={loading}
|
||||
value={settings.alertDeliveryDays}
|
||||
onChange={e => setSettings(s => ({ ...s, alertDeliveryDays: Number(e.target.value) || 0 }))}
|
||||
/>
|
||||
<p className="mt-1 min-h-4 text-xs text-muted-foreground">{t('retention.alertHint')}</p>
|
||||
</div>
|
||||
<div className="flex flex-col">
|
||||
<label className="text-sm font-medium" htmlFor="retention-audit">
|
||||
{t('retention.auditLabel')}
|
||||
</label>
|
||||
<Input
|
||||
id="retention-audit"
|
||||
type="number"
|
||||
min={1}
|
||||
max={3650}
|
||||
className="mt-1"
|
||||
disabled={loading}
|
||||
value={settings.auditLogDays}
|
||||
onChange={e => setSettings(s => ({ ...s, auditLogDays: Number(e.target.value) || 0 }))}
|
||||
/>
|
||||
<p className="mt-1 min-h-4 text-xs text-muted-foreground">{t('retention.auditHint')}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex justify-end">
|
||||
<Button onClick={() => void save()} disabled={saving || loading}>
|
||||
{saving ? t('retention.saving') : t('retention.save')}
|
||||
</Button>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)
|
||||
})
|
||||
@@ -11,6 +11,7 @@ import { clearAuth } from '@/lib/auth-storage'
|
||||
import { toast } from 'sonner'
|
||||
import { PageContainer } from '@/components/PageContainer'
|
||||
import { ThemeSection } from './components/ThemeSection'
|
||||
import { RetentionSection } from './components/RetentionSection'
|
||||
import { LanguageSection } from '@/i18n/LanguageSection'
|
||||
import { Palette } from 'lucide-react'
|
||||
|
||||
@@ -80,6 +81,7 @@ const SettingsPage = memo(function SettingsPage() {
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<RetentionSection />
|
||||
<ThemeSection />
|
||||
<LanguageSection />
|
||||
</CardContent>
|
||||
|
||||
Reference in new issue
Block a user