feat(log): Gin 改结构化 JSON 访问日志 + 保留期自动清理(含 VACUUM 回收)

=== 一、审计结论(用户问:gin 日志是否标准、会不会打爆磁盘)===
现状是标准的:gin.Default() 写 stdout,代码里没有任何
gin.DefaultWriter / log.SetOutput / lumberjack / os.OpenFile 写日志文件;
容器内 /var/log 是空的,find / -name "*.log" 无结果;
kubelet containerLogMaxSize=10Mi / containerLogMaxFiles=5 自动轮转,
实测 /var/log 只占 120G 磁盘的 0.8%。

但发现两个真问题:
  1. 访问日志是 gin 默认文本格式,Loki 里没法按字段过滤
  2. 审计日志/查询历史等只配了保留天数,**没有任何自动清理** ——
     只有手动按钮,表会一直涨,SQLite 文件删了也不缩

=== 二、结构化 JSON 访问日志(middleware/accesslog.go)===
gin.Default() → gin.New() + AccessLog() + gin.Recovery() + CORSMiddleware
  - 一行一个 JSON:ts/level/type/method/path/status/latency_ms/bytes/ip/ua
  - 自动分级:5xx=error / 4xx=warn / 其余=info
  - 鉴权后带 user_id / username,便于审计
  - 健康检查与 /assets 静音(K8s 探针每 10s 一次,会淹没真实请求)
  - Recovery 显式加回(gin.Default 里包含它,漏了 panic 会打挂进程)

⚠️ 关键细节:Go 的 map 序列化按 key 字母序,ts 会排到最后,
   一行以 {"bytes":... 开头根本看不出时间。LogJSON 手工拼装,
   把 ts/level/type/msg 固定在最前,其余排序输出。

=== 三、保留期自动清理(handler/cleanup.go)===
启动 2 分钟后跑一次,之后每 24h 一次;cleanup_auto_enabled=false 可关。

覆盖 8 张会无限增长的表(原来只管 4 张):
  audit_logs / query_histories / slow_logs / tickets
  + operation_logs / workflow_execution_logs / scheduled_task_logs(新增)
  + refresh_tokens(按 expires_at < now,过期不清理会每次登录加一行)

设计取舍:
  - 每批删 5000 行、批间 sleep 50ms:SQLite 单写者,一条 DELETE 全删会把库锁死
  - 单表单轮最多 20 万行,剩下的留给下一轮
  - deleted 后看 freelist/page_count,>25% 才 VACUUM(DELETE 不还磁盘,必须 VACUUM)
  - 保留天数 <=0 视为不清理,避免误配把数据全删

手动接口同步改造:/settings/cleanup/{preview,run} 复用同一套逻辑,
返回结构改为 {deleted, total, vacuumed, freed_bytes, cost_ms}。

新增系统设置(seed 自动补,已实测 8 个 key 全部落库):
  operation_log_retention_days=365  workflow_log_retention_days=180
  scheduled_log_retention_days=180  cleanup_auto_enabled=true

=== 四、实测 ===
访问日志:{"ts":"2026-10-27T14:46:39.58Z","level":"info","type":"access","bytes":668,"ip":"...","latency_ms":78.6,"method":"POST","path":"/api/v1/auth/login","status":200,...}
health 日志行数: 0 ✅   [GIN] 文本行数: 0 ✅   非 JSON 行: 4(仅启动)
cleanup preview: 8 张表的 count/days ✅
cleanup run: {"cost_ms":1,"deleted":{},"freed_bytes":0,"vacuumed":false,"total":0} ✅

镜像:lghost/opsxsql:latest + v0.1 = sha256:30593de3fc09
This commit is contained in:
opsx committed 2026-10-27 14:49:45 +00:00
1 parent 4c2c5e182a
commit 0c57e18f19
4 files changed
+392 -61

No files matched your search

+21 -59
View File
@@ -508,71 +508,33 @@ func UpdateSettings(c *gin.Context) {
}
// GET /api/v1/settings/cleanup/preview
//
// 统计口径统一收敛到 CleanupPreview()(handler/cleanup.go),
// 与后台定时清理共用同一份规则表,避免「预览说有、真跑不删」这类漂移。
func SettingsCleanupPreview(c *gin.Context) {
now := time.Now()
stats := gin.H{}
// Tickets
ticketDays := getSettingInt("ticket_retention_days", 180)
var ticketCount int64
model.DB.Model(&model.Ticket{}).
Where("status IN ? AND create_time < ?", []string{"success", "failed", "closed", "rejected"}, now.AddDate(0, 0, -ticketDays)).
Count(&ticketCount)
stats["tickets"] = gin.H{"days": ticketDays, "count": ticketCount}
// Query history
queryDays := getSettingInt("query_history_retention_days", 90)
var queryCount int64
model.DB.Model(&model.QueryHistory{}).
Where("created_at < ?", now.AddDate(0, 0, -queryDays)).
Count(&queryCount)
stats["query_history"] = gin.H{"days": queryDays, "count": queryCount}
// Slow logs
slowDays := getSettingInt("slow_log_retention_days", 180)
var slowCount int64
model.DB.Model(&model.SlowLog{}).
Where("created_at < ?", now.AddDate(0, 0, -slowDays)).
Count(&slowCount)
stats["slow_logs"] = gin.H{"days": slowDays, "count": slowCount}
// Audit logs — skip if no AuditLog model; use raw SQL
auditDays := getSettingInt("audit_log_retention_days", 365)
var auditCount int64
model.DB.Raw("SELECT COUNT(*) FROM audit_logs WHERE created_at < ?", now.AddDate(0, 0, -auditDays)).Scan(&auditCount)
stats["audit_logs"] = gin.H{"days": auditDays, "count": auditCount}
c.JSON(http.StatusOK, stats)
c.JSON(http.StatusOK, CleanupPreview())
}
// POST /api/v1/settings/cleanup/run
//
// 手动触发:与后台定时任务走同一套分批删除 + VACUUM 逻辑。
func SettingsCleanupRun(c *gin.Context) {
now := time.Now()
stats := gin.H{}
start := time.Now()
deleted := RunRetentionCleanup()
vacuumed, freed := VacuumIfNeeded()
// Tickets
ticketDays := getSettingInt("ticket_retention_days", 180)
ticketResult := model.DB.Where("status IN ? AND create_time < ?",
[]string{"success", "failed", "closed", "rejected"}, now.AddDate(0, 0, -ticketDays)).
Delete(&model.Ticket{})
stats["tickets_deleted"] = ticketResult.RowsAffected
// Query history
queryDays := getSettingInt("query_history_retention_days", 90)
queryResult := model.DB.Where("created_at < ?", now.AddDate(0, 0, -queryDays)).Delete(&model.QueryHistory{})
stats["query_history_deleted"] = queryResult.RowsAffected
// Slow logs
slowDays := getSettingInt("slow_log_retention_days", 180)
slowResult := model.DB.Where("created_at < ?", now.AddDate(0, 0, -slowDays)).Delete(&model.SlowLog{})
stats["slow_logs_deleted"] = slowResult.RowsAffected
// Audit logs
auditDays := getSettingInt("audit_log_retention_days", 365)
auditResult := model.DB.Exec("DELETE FROM audit_logs WHERE created_at < ?", now.AddDate(0, 0, -auditDays))
stats["audit_logs_deleted"] = auditResult.RowsAffected
c.JSON(http.StatusOK, gin.H{"message": "Cleanup completed", "stats": stats})
var total int64
for _, n := range deleted {
total += n
}
c.JSON(http.StatusOK, gin.H{
"message": "Cleanup completed",
"deleted": deleted,
"total": total,
"vacuumed": vacuumed,
"freed_bytes": freed,
"cost_ms": time.Since(start).Milliseconds(),
})
}
// GET /api/v1/notifications/channels