feat(log): Gin 改结构化 JSON 访问日志 + 保留期自动清理（含 VACUUM 回收）
=== 一、审计结论（用户问：gin 日志是否标准、会不会打爆磁盘）===
现状是标准的：gin.Default() 写 stdout，代码里没有任何
gin.DefaultWriter / log.SetOutput / lumberjack / os.OpenFile 写日志文件；
容器内 /var/log 是空的，find / -name "*.log" 无结果；
kubelet containerLogMaxSize=10Mi / containerLogMaxFiles=5 自动轮转，
实测 /var/log 只占 120G 磁盘的 0.8%。
但发现两个真问题：
1. 访问日志是 gin 默认文本格式，Loki 里没法按字段过滤
2. 审计日志/查询历史等只配了保留天数，**没有任何自动清理** ——
只有手动按钮，表会一直涨，SQLite 文件删了也不缩
=== 二、结构化 JSON 访问日志（middleware/accesslog.go）===
gin.Default() → gin.New() + AccessLog() + gin.Recovery() + CORSMiddleware
- 一行一个 JSON：ts/level/type/method/path/status/latency_ms/bytes/ip/ua
- 自动分级：5xx=error / 4xx=warn / 其余=info
- 鉴权后带 user_id / username，便于审计
- 健康检查与 /assets 静音（K8s 探针每 10s 一次，会淹没真实请求）
- Recovery 显式加回（gin.Default 里包含它，漏了 panic 会打挂进程）
⚠️ 关键细节：Go 的 map 序列化按 key 字母序，ts 会排到最后，
一行以 {"bytes":... 开头根本看不出时间。LogJSON 手工拼装，
把 ts/level/type/msg 固定在最前，其余排序输出。
=== 三、保留期自动清理（handler/cleanup.go）===
启动 2 分钟后跑一次，之后每 24h 一次；cleanup_auto_enabled=false 可关。
覆盖 8 张会无限增长的表（原来只管 4 张）：
audit_logs / query_histories / slow_logs / tickets
+ operation_logs / workflow_execution_logs / scheduled_task_logs（新增）
+ refresh_tokens（按 expires_at < now，过期不清理会每次登录加一行）
设计取舍：
- 每批删 5000 行、批间 sleep 50ms：SQLite 单写者，一条 DELETE 全删会把库锁死
- 单表单轮最多 20 万行，剩下的留给下一轮
- deleted 后看 freelist/page_count，>25% 才 VACUUM（DELETE 不还磁盘，必须 VACUUM）
- 保留天数 <=0 视为不清理，避免误配把数据全删
手动接口同步改造：/settings/cleanup/{preview,run} 复用同一套逻辑，
返回结构改为 {deleted, total, vacuumed, freed_bytes, cost_ms}。
新增系统设置（seed 自动补，已实测 8 个 key 全部落库）：
operation_log_retention_days=365 workflow_log_retention_days=180
scheduled_log_retention_days=180 cleanup_auto_enabled=true
=== 四、实测 ===
访问日志：{"ts":"2026-10-27T14:46:39.58Z","level":"info","type":"access","bytes":668,"ip":"...","latency_ms":78.6,"method":"POST","path":"/api/v1/auth/login","status":200,...}
health 日志行数: 0 ✅ [GIN] 文本行数: 0 ✅ 非 JSON 行: 4（仅启动）
cleanup preview: 8 张表的 count/days ✅
cleanup run: {"cost_ms":1,"deleted":{},"freed_bytes":0,"vacuumed":false,"total":0} ✅
镜像：lghost/opsxsql:latest + v0.1 = sha256:30593de3fc09
This commit is contained in:
opsx committed
1 parent
4c2c5e182a
commit
0c57e18f19
4 files changed
+392
-61
No files matched your search
@@ -0,0 +1,219 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"sql-platform/internal/middleware"
|
||||
"sql-platform/internal/model"
|
||||
)
|
||||
|
||||
// 保留期清理任务的调参。设计取舍：
|
||||
// - 分批删（每批 5000）而不是一条 DELETE 全删：SQLite 是单写者，
|
||||
// 大事务会把整个库锁住，前端所有请求都会排队等锁。
|
||||
// - 单表单轮最多删 20 万行：极端情况下（比如首次清理积压半年的审计日志），
|
||||
// 剩下的留给下一轮或手动跑，避免一次任务跑太久。
|
||||
// - 删完看 freelist 比例决定要不要 VACUUM：SQLite 的 DELETE 只把页标记为空闲，
|
||||
// 文件不会自动变小，必须 VACUUM 才真正归还磁盘。
|
||||
const (
|
||||
cleanupBatchSize = 5000
|
||||
cleanupMaxPerTable = 200000
|
||||
cleanupInterval = 24 * time.Hour
|
||||
cleanupFirstDelay = 2 * time.Minute
|
||||
vacuumFreeRatio = 0.25
|
||||
)
|
||||
|
||||
// retentionRule 一条「按时间列保留 N 天」的清理规则
|
||||
type retentionRule struct {
|
||||
Table string
|
||||
TimeCol string
|
||||
DaysKey string // system_settings 里对应的保留天数 key
|
||||
Default int
|
||||
}
|
||||
|
||||
// 会无限增长的表清单。tickets 不在这里（要按状态过滤，单独处理）。
|
||||
var retentionRules = []retentionRule{
|
||||
{"audit_logs", "created_at", "audit_log_retention_days", 365},
|
||||
{"query_histories", "created_at", "query_history_retention_days", 90},
|
||||
{"slow_logs", "created_at", "slow_log_retention_days", 180},
|
||||
{"operation_logs", "created_at", "operation_log_retention_days", 365},
|
||||
{"workflow_execution_logs", "created_at", "workflow_log_retention_days", 180},
|
||||
{"scheduled_task_logs", "started_at", "scheduled_log_retention_days", 180},
|
||||
}
|
||||
|
||||
// deleteInBatches 分批删除，返回实际删除行数。
|
||||
//
|
||||
// 用 `DELETE ... WHERE id IN (SELECT id ... LIMIT n)` 而不是 GORM 的 Delete：
|
||||
// GORM 不支持 Limit，一次性删会生成超大事务。
|
||||
func deleteInBatches(table, where string, args []any, maxRows int64) int64 {
|
||||
var total int64
|
||||
for total < maxRows {
|
||||
params := make([]any, 0, len(args)+1)
|
||||
params = append(params, args...)
|
||||
params = append(params, cleanupBatchSize)
|
||||
|
||||
res := model.DB.Exec(
|
||||
fmt.Sprintf("DELETE FROM %s WHERE id IN (SELECT id FROM %s WHERE %s LIMIT ?)",
|
||||
table, table, where),
|
||||
params...,
|
||||
)
|
||||
if res.Error != nil {
|
||||
middleware.LogJSON(map[string]any{
|
||||
"ts": time.Now().Format(time.RFC3339Nano), "level": "error", "type": "cleanup",
|
||||
"msg": "删除失败", "table": table, "error": res.Error.Error(),
|
||||
})
|
||||
break
|
||||
}
|
||||
if res.RowsAffected == 0 {
|
||||
break
|
||||
}
|
||||
total += res.RowsAffected
|
||||
time.Sleep(50 * time.Millisecond) // 让出写锁，避免饿死业务请求
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// RunRetentionCleanup 执行一轮清理，返回 {表名: 删除行数}
|
||||
func RunRetentionCleanup() map[string]int64 {
|
||||
now := time.Now()
|
||||
deleted := map[string]int64{}
|
||||
|
||||
for _, r := range retentionRules {
|
||||
days := getSettingInt(r.DaysKey, r.Default)
|
||||
if days <= 0 {
|
||||
continue // <=0 视为不清理，避免误配把数据全删
|
||||
}
|
||||
cutoff := now.AddDate(0, 0, -days)
|
||||
n := deleteInBatches(r.Table, r.TimeCol+" < ?", []any{cutoff}, cleanupMaxPerTable)
|
||||
if n > 0 {
|
||||
deleted[r.Table] = n
|
||||
}
|
||||
}
|
||||
|
||||
// 工单：只删终态（success/failed/closed/rejected），进行中的不能动
|
||||
ticketDays := getSettingInt("ticket_retention_days", 180)
|
||||
if ticketDays > 0 {
|
||||
n := deleteInBatches("tickets",
|
||||
"status IN ('success','failed','closed','rejected') AND create_time < ?",
|
||||
[]any{now.AddDate(0, 0, -ticketDays)}, cleanupMaxPerTable)
|
||||
if n > 0 {
|
||||
deleted["tickets"] = n
|
||||
}
|
||||
}
|
||||
|
||||
// 刷新令牌：过期/已吊销的直接清掉（按时间点，不是保留天数）
|
||||
// 不清的话每次登录都会新增一行，永远不减少。
|
||||
n := deleteInBatches("refresh_tokens", "expires_at < ?", []any{now}, 100000)
|
||||
if n > 0 {
|
||||
deleted["refresh_tokens"] = n
|
||||
}
|
||||
|
||||
return deleted
|
||||
}
|
||||
|
||||
// CleanupPreview 返回各表「按当前保留策略会被删掉多少行」
|
||||
func CleanupPreview() map[string]any {
|
||||
now := time.Now()
|
||||
out := map[string]any{}
|
||||
count := func(table, where string, args ...any) int64 {
|
||||
var n int64
|
||||
model.DB.Raw(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", table, where), args...).Scan(&n)
|
||||
return n
|
||||
}
|
||||
|
||||
for _, r := range retentionRules {
|
||||
days := getSettingInt(r.DaysKey, r.Default)
|
||||
out[r.Table] = map[string]any{
|
||||
"days": days,
|
||||
"count": count(r.Table, r.TimeCol+" < ?", now.AddDate(0, 0, -days)),
|
||||
}
|
||||
}
|
||||
ticketDays := getSettingInt("ticket_retention_days", 180)
|
||||
out["tickets"] = map[string]any{
|
||||
"days": ticketDays,
|
||||
"count": count("tickets", "status IN ('success','failed','closed','rejected') AND create_time < ?", now.AddDate(0, 0, -ticketDays)),
|
||||
}
|
||||
out["refresh_tokens"] = map[string]any{
|
||||
"days": 0,
|
||||
"count": count("refresh_tokens", "expires_at < ?", now),
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// VacuumIfNeeded 空闲页占比超过阈值时执行 VACUUM 回收磁盘，返回 (是否执行, 释放字节数)
|
||||
func VacuumIfNeeded() (bool, int64) {
|
||||
var pageCount, freeList, pageSize int64
|
||||
model.DB.Raw("PRAGMA page_count").Scan(&pageCount)
|
||||
model.DB.Raw("PRAGMA freelist_count").Scan(&freeList)
|
||||
model.DB.Raw("PRAGMA page_size").Scan(&pageSize)
|
||||
if pageCount <= 0 || pageSize <= 0 {
|
||||
return false, 0
|
||||
}
|
||||
if float64(freeList)/float64(pageCount) < vacuumFreeRatio {
|
||||
return false, 0
|
||||
}
|
||||
before := pageCount * pageSize
|
||||
if err := model.DB.Exec("VACUUM").Error; err != nil {
|
||||
middleware.LogJSON(map[string]any{
|
||||
"ts": time.Now().Format(time.RFC3339Nano), "level": "error", "type": "cleanup",
|
||||
"msg": "VACUUM 失败", "error": err.Error(),
|
||||
})
|
||||
return false, 0
|
||||
}
|
||||
model.DB.Raw("PRAGMA page_count").Scan(&pageCount)
|
||||
after := pageCount * pageSize
|
||||
return true, before - after
|
||||
}
|
||||
|
||||
// StartCleanupScheduler 启动后台清理：启动 2 分钟后跑一次，之后每 24h 一次。
|
||||
// 可通过系统设置 cleanup_auto_enabled=false 关掉（关掉后仍可手动点「立即清理」）。
|
||||
func StartCleanupScheduler(ctx context.Context) {
|
||||
go func() {
|
||||
select {
|
||||
case <-time.After(cleanupFirstDelay):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
runScheduledCleanup()
|
||||
|
||||
ticker := time.NewTicker(cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
runScheduledCleanup()
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func runScheduledCleanup() {
|
||||
if getSetting("cleanup_auto_enabled", "true") != "true" {
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
deleted := RunRetentionCleanup()
|
||||
vacuumed, freed := VacuumIfNeeded()
|
||||
|
||||
var total int64
|
||||
for _, n := range deleted {
|
||||
total += n
|
||||
}
|
||||
if total == 0 && !vacuumed {
|
||||
return // 没什么可清的，不刷日志
|
||||
}
|
||||
middleware.LogJSON(map[string]any{
|
||||
"ts": time.Now().Format(time.RFC3339Nano),
|
||||
"level": "info",
|
||||
"type": "cleanup",
|
||||
"msg": "保留期清理完成",
|
||||
"deleted": deleted,
|
||||
"total": total,
|
||||
"vacuumed": vacuumed,
|
||||
"freed_bytes": freed,
|
||||
"cost_ms": time.Since(start).Milliseconds(),
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user