diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 06d0cfb..f77bbad 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "log" "net/http" @@ -9,6 +10,7 @@ import ( "strings" "sql-platform/config" + "sql-platform/internal/handler" "sql-platform/internal/middleware" "sql-platform/internal/model" "sql-platform/router" @@ -75,8 +77,13 @@ func main() { config.EnsureJWTSecret() // Setup Gin - r := gin.Default() - r.Use(middleware.CORSMiddleware()) + // + // 用 gin.New() 而不是 gin.Default():Default 自带的是文本格式的 gin.Logger(), + // 这里换成结构化 JSON 访问日志(middleware.AccessLog),便于 Promtail→Loki 按字段过滤, + // 同时跳过健康检查/静态资源,避免探针轮询把日志量刷满。 + // Recovery 必须显式加回来(Default 里包含它),否则 panic 会打挂整个进程。 + r := gin.New() + r.Use(middleware.AccessLog(), gin.Recovery(), middleware.CORSMiddleware()) // Health check r.GET("/health", func(c *gin.Context) { @@ -86,6 +93,11 @@ func main() { // Routes router.Setup(r) + // 后台保留期清理:启动 2 分钟后跑第一次,之后每 24h 一次。 + // 解决「只配了保留天数但没人执行」的问题 —— 之前审计日志/查询历史 + // 只能靠管理员手动点「立即清理」,表会一直涨、SQLite 文件也不会缩。 + handler.StartCleanupScheduler(context.Background()) + // 单镜像部署:直接托管前端产物(DIST_DIR 为空则跳过,开发模式由 vite 独立提供) if distDir := os.Getenv("DIST_DIR"); distDir != "" { indexPath := filepath.Join(distDir, "index.html") @@ -145,6 +157,10 @@ func seedDefaultSettings(db *gorm.DB) { {Key: "audit_log_retention_days", Value: "365", ValueType: "number", Group: "cleanup", Label: "审计日志保留天数", Desc: strPtr("超过此天数的审计日志将被清理"), DefaultValue: strPtr("365"), SortOrder: 2}, {Key: "query_history_retention_days", Value: "90", ValueType: "number", Group: "cleanup", Label: "查询历史保留天数", Desc: strPtr("超过此天数的查询历史将被清理"), DefaultValue: strPtr("90"), SortOrder: 3}, {Key: "slow_log_retention_days", Value: "180", ValueType: "number", Group: "cleanup", Label: "慢日志保留天数", Desc: strPtr("超过此天数的慢查询日志将被清理"), DefaultValue: strPtr("180"), SortOrder: 4}, + {Key: "operation_log_retention_days", Value: "365", ValueType: "number", Group: "cleanup", Label: "操作日志保留天数", Desc: strPtr("超过此天数的操作日志将被清理"), DefaultValue: strPtr("365"), SortOrder: 5}, + {Key: "workflow_log_retention_days", Value: "180", ValueType: "number", Group: "cleanup", Label: "工单流程日志保留天数", Desc: strPtr("超过此天数的工单流程执行日志将被清理"), DefaultValue: strPtr("180"), SortOrder: 6}, + {Key: "scheduled_log_retention_days", Value: "180", ValueType: "number", Group: "cleanup", Label: "定时任务日志保留天数", Desc: strPtr("超过此天数的定时任务执行日志将被清理"), DefaultValue: strPtr("180"), SortOrder: 7}, + {Key: "cleanup_auto_enabled", Value: "true", ValueType: "boolean", Group: "cleanup", Label: "自动清理", Desc: strPtr("每天自动按上面的保留天数清理一次;关闭后仍可在本页点「立即清理」"), DefaultValue: strPtr("true"), SortOrder: 99}, // ── 执行器 ── {Key: "executor_timeout", Value: "300", ValueType: "number", Group: "executor", Label: "执行超时(秒)", Desc: strPtr("工单中 SQL 语句执行的最大等待时间"), DefaultValue: strPtr("300"), SortOrder: 1}, diff --git a/backend/internal/handler/cleanup.go b/backend/internal/handler/cleanup.go new file mode 100644 index 0000000..bfea43d --- /dev/null +++ b/backend/internal/handler/cleanup.go @@ -0,0 +1,219 @@ +package handler + +import ( + "context" + "fmt" + "time" + + "sql-platform/internal/middleware" + "sql-platform/internal/model" +) + +// 保留期清理任务的调参。设计取舍: +// - 分批删(每批 5000)而不是一条 DELETE 全删:SQLite 是单写者, +// 大事务会把整个库锁住,前端所有请求都会排队等锁。 +// - 单表单轮最多删 20 万行:极端情况下(比如首次清理积压半年的审计日志), +// 剩下的留给下一轮或手动跑,避免一次任务跑太久。 +// - 删完看 freelist 比例决定要不要 VACUUM:SQLite 的 DELETE 只把页标记为空闲, +// 文件不会自动变小,必须 VACUUM 才真正归还磁盘。 +const ( + cleanupBatchSize = 5000 + cleanupMaxPerTable = 200000 + cleanupInterval = 24 * time.Hour + cleanupFirstDelay = 2 * time.Minute + vacuumFreeRatio = 0.25 +) + +// retentionRule 一条「按时间列保留 N 天」的清理规则 +type retentionRule struct { + Table string + TimeCol string + DaysKey string // system_settings 里对应的保留天数 key + Default int +} + +// 会无限增长的表清单。tickets 不在这里(要按状态过滤,单独处理)。 +var retentionRules = []retentionRule{ + {"audit_logs", "created_at", "audit_log_retention_days", 365}, + {"query_histories", "created_at", "query_history_retention_days", 90}, + {"slow_logs", "created_at", "slow_log_retention_days", 180}, + {"operation_logs", "created_at", "operation_log_retention_days", 365}, + {"workflow_execution_logs", "created_at", "workflow_log_retention_days", 180}, + {"scheduled_task_logs", "started_at", "scheduled_log_retention_days", 180}, +} + +// deleteInBatches 分批删除,返回实际删除行数。 +// +// 用 `DELETE ... WHERE id IN (SELECT id ... LIMIT n)` 而不是 GORM 的 Delete: +// GORM 不支持 Limit,一次性删会生成超大事务。 +func deleteInBatches(table, where string, args []any, maxRows int64) int64 { + var total int64 + for total < maxRows { + params := make([]any, 0, len(args)+1) + params = append(params, args...) + params = append(params, cleanupBatchSize) + + res := model.DB.Exec( + fmt.Sprintf("DELETE FROM %s WHERE id IN (SELECT id FROM %s WHERE %s LIMIT ?)", + table, table, where), + params..., + ) + if res.Error != nil { + middleware.LogJSON(map[string]any{ + "ts": time.Now().Format(time.RFC3339Nano), "level": "error", "type": "cleanup", + "msg": "删除失败", "table": table, "error": res.Error.Error(), + }) + break + } + if res.RowsAffected == 0 { + break + } + total += res.RowsAffected + time.Sleep(50 * time.Millisecond) // 让出写锁,避免饿死业务请求 + } + return total +} + +// RunRetentionCleanup 执行一轮清理,返回 {表名: 删除行数} +func RunRetentionCleanup() map[string]int64 { + now := time.Now() + deleted := map[string]int64{} + + for _, r := range retentionRules { + days := getSettingInt(r.DaysKey, r.Default) + if days <= 0 { + continue // <=0 视为不清理,避免误配把数据全删 + } + cutoff := now.AddDate(0, 0, -days) + n := deleteInBatches(r.Table, r.TimeCol+" < ?", []any{cutoff}, cleanupMaxPerTable) + if n > 0 { + deleted[r.Table] = n + } + } + + // 工单:只删终态(success/failed/closed/rejected),进行中的不能动 + ticketDays := getSettingInt("ticket_retention_days", 180) + if ticketDays > 0 { + n := deleteInBatches("tickets", + "status IN ('success','failed','closed','rejected') AND create_time < ?", + []any{now.AddDate(0, 0, -ticketDays)}, cleanupMaxPerTable) + if n > 0 { + deleted["tickets"] = n + } + } + + // 刷新令牌:过期/已吊销的直接清掉(按时间点,不是保留天数) + // 不清的话每次登录都会新增一行,永远不减少。 + n := deleteInBatches("refresh_tokens", "expires_at < ?", []any{now}, 100000) + if n > 0 { + deleted["refresh_tokens"] = n + } + + return deleted +} + +// CleanupPreview 返回各表「按当前保留策略会被删掉多少行」 +func CleanupPreview() map[string]any { + now := time.Now() + out := map[string]any{} + count := func(table, where string, args ...any) int64 { + var n int64 + model.DB.Raw(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", table, where), args...).Scan(&n) + return n + } + + for _, r := range retentionRules { + days := getSettingInt(r.DaysKey, r.Default) + out[r.Table] = map[string]any{ + "days": days, + "count": count(r.Table, r.TimeCol+" < ?", now.AddDate(0, 0, -days)), + } + } + ticketDays := getSettingInt("ticket_retention_days", 180) + out["tickets"] = map[string]any{ + "days": ticketDays, + "count": count("tickets", "status IN ('success','failed','closed','rejected') AND create_time < ?", now.AddDate(0, 0, -ticketDays)), + } + out["refresh_tokens"] = map[string]any{ + "days": 0, + "count": count("refresh_tokens", "expires_at < ?", now), + } + return out +} + +// VacuumIfNeeded 空闲页占比超过阈值时执行 VACUUM 回收磁盘,返回 (是否执行, 释放字节数) +func VacuumIfNeeded() (bool, int64) { + var pageCount, freeList, pageSize int64 + model.DB.Raw("PRAGMA page_count").Scan(&pageCount) + model.DB.Raw("PRAGMA freelist_count").Scan(&freeList) + model.DB.Raw("PRAGMA page_size").Scan(&pageSize) + if pageCount <= 0 || pageSize <= 0 { + return false, 0 + } + if float64(freeList)/float64(pageCount) < vacuumFreeRatio { + return false, 0 + } + before := pageCount * pageSize + if err := model.DB.Exec("VACUUM").Error; err != nil { + middleware.LogJSON(map[string]any{ + "ts": time.Now().Format(time.RFC3339Nano), "level": "error", "type": "cleanup", + "msg": "VACUUM 失败", "error": err.Error(), + }) + return false, 0 + } + model.DB.Raw("PRAGMA page_count").Scan(&pageCount) + after := pageCount * pageSize + return true, before - after +} + +// StartCleanupScheduler 启动后台清理:启动 2 分钟后跑一次,之后每 24h 一次。 +// 可通过系统设置 cleanup_auto_enabled=false 关掉(关掉后仍可手动点「立即清理」)。 +func StartCleanupScheduler(ctx context.Context) { + go func() { + select { + case <-time.After(cleanupFirstDelay): + case <-ctx.Done(): + return + } + runScheduledCleanup() + + ticker := time.NewTicker(cleanupInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + runScheduledCleanup() + case <-ctx.Done(): + return + } + } + }() +} + +func runScheduledCleanup() { + if getSetting("cleanup_auto_enabled", "true") != "true" { + return + } + start := time.Now() + deleted := RunRetentionCleanup() + vacuumed, freed := VacuumIfNeeded() + + var total int64 + for _, n := range deleted { + total += n + } + if total == 0 && !vacuumed { + return // 没什么可清的,不刷日志 + } + middleware.LogJSON(map[string]any{ + "ts": time.Now().Format(time.RFC3339Nano), + "level": "info", + "type": "cleanup", + "msg": "保留期清理完成", + "deleted": deleted, + "total": total, + "vacuumed": vacuumed, + "freed_bytes": freed, + "cost_ms": time.Since(start).Milliseconds(), + }) +} diff --git a/backend/internal/handler/ghost.go b/backend/internal/handler/ghost.go index 4595cbd..c291124 100644 --- a/backend/internal/handler/ghost.go +++ b/backend/internal/handler/ghost.go @@ -508,71 +508,33 @@ func UpdateSettings(c *gin.Context) { } // GET /api/v1/settings/cleanup/preview +// +// 统计口径统一收敛到 CleanupPreview()(handler/cleanup.go), +// 与后台定时清理共用同一份规则表,避免「预览说有、真跑不删」这类漂移。 func SettingsCleanupPreview(c *gin.Context) { - now := time.Now() - stats := gin.H{} - - // Tickets - ticketDays := getSettingInt("ticket_retention_days", 180) - var ticketCount int64 - model.DB.Model(&model.Ticket{}). - Where("status IN ? AND create_time < ?", []string{"success", "failed", "closed", "rejected"}, now.AddDate(0, 0, -ticketDays)). - Count(&ticketCount) - stats["tickets"] = gin.H{"days": ticketDays, "count": ticketCount} - - // Query history - queryDays := getSettingInt("query_history_retention_days", 90) - var queryCount int64 - model.DB.Model(&model.QueryHistory{}). - Where("created_at < ?", now.AddDate(0, 0, -queryDays)). - Count(&queryCount) - stats["query_history"] = gin.H{"days": queryDays, "count": queryCount} - - // Slow logs - slowDays := getSettingInt("slow_log_retention_days", 180) - var slowCount int64 - model.DB.Model(&model.SlowLog{}). - Where("created_at < ?", now.AddDate(0, 0, -slowDays)). - Count(&slowCount) - stats["slow_logs"] = gin.H{"days": slowDays, "count": slowCount} - - // Audit logs — skip if no AuditLog model; use raw SQL - auditDays := getSettingInt("audit_log_retention_days", 365) - var auditCount int64 - model.DB.Raw("SELECT COUNT(*) FROM audit_logs WHERE created_at < ?", now.AddDate(0, 0, -auditDays)).Scan(&auditCount) - stats["audit_logs"] = gin.H{"days": auditDays, "count": auditCount} - - c.JSON(http.StatusOK, stats) + c.JSON(http.StatusOK, CleanupPreview()) } // POST /api/v1/settings/cleanup/run +// +// 手动触发:与后台定时任务走同一套分批删除 + VACUUM 逻辑。 func SettingsCleanupRun(c *gin.Context) { - now := time.Now() - stats := gin.H{} + start := time.Now() + deleted := RunRetentionCleanup() + vacuumed, freed := VacuumIfNeeded() - // Tickets - ticketDays := getSettingInt("ticket_retention_days", 180) - ticketResult := model.DB.Where("status IN ? AND create_time < ?", - []string{"success", "failed", "closed", "rejected"}, now.AddDate(0, 0, -ticketDays)). - Delete(&model.Ticket{}) - stats["tickets_deleted"] = ticketResult.RowsAffected - - // Query history - queryDays := getSettingInt("query_history_retention_days", 90) - queryResult := model.DB.Where("created_at < ?", now.AddDate(0, 0, -queryDays)).Delete(&model.QueryHistory{}) - stats["query_history_deleted"] = queryResult.RowsAffected - - // Slow logs - slowDays := getSettingInt("slow_log_retention_days", 180) - slowResult := model.DB.Where("created_at < ?", now.AddDate(0, 0, -slowDays)).Delete(&model.SlowLog{}) - stats["slow_logs_deleted"] = slowResult.RowsAffected - - // Audit logs - auditDays := getSettingInt("audit_log_retention_days", 365) - auditResult := model.DB.Exec("DELETE FROM audit_logs WHERE created_at < ?", now.AddDate(0, 0, -auditDays)) - stats["audit_logs_deleted"] = auditResult.RowsAffected - - c.JSON(http.StatusOK, gin.H{"message": "Cleanup completed", "stats": stats}) + var total int64 + for _, n := range deleted { + total += n + } + c.JSON(http.StatusOK, gin.H{ + "message": "Cleanup completed", + "deleted": deleted, + "total": total, + "vacuumed": vacuumed, + "freed_bytes": freed, + "cost_ms": time.Since(start).Milliseconds(), + }) } // GET /api/v1/notifications/channels diff --git a/backend/internal/middleware/accesslog.go b/backend/internal/middleware/accesslog.go new file mode 100644 index 0000000..4f221f4 --- /dev/null +++ b/backend/internal/middleware/accesslog.go @@ -0,0 +1,134 @@ +package middleware + +import ( + "bytes" + "encoding/json" + "os" + "sort" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +// LogJSON 输出一行 JSON 到标准输出(stdout)。 +// +// 统一走 stdout 的原因:容器里日志由容器运行时接管(kubelet 侧 +// containerLogMaxSize=10Mi / containerLogMaxFiles=5 自动轮转), +// 应用自己写文件会绕过这套轮转、把磁盘写爆。 +// +// 关键细节:Go 的 map 序列化是**按 key 字母序**输出,ts 会排到最后, +// kubectl logs 里一行以 {"bytes":... 开头完全看不出时间。 +// 这里手工拼装,把 ts/level/type/msg 固定在最前面,其余字段排序输出。 +func LogJSON(fields map[string]any) { + head := []string{"ts", "level", "type", "msg"} + + var buf bytes.Buffer + buf.WriteByte('{') + first := true + write := func(k string) { + v, ok := fields[k] + if !ok { + return + } + kb, err := json.Marshal(k) + if err != nil { + return + } + vb, err := json.Marshal(v) + if err != nil { + return + } + if !first { + buf.WriteByte(',') + } + buf.Write(kb) + buf.WriteByte(':') + buf.Write(vb) + first = false + } + for _, k := range head { + write(k) + } + rest := make([]string, 0, len(fields)) + for k := range fields { + switch k { + case "ts", "level", "type", "msg": + continue + } + rest = append(rest, k) + } + sort.Strings(rest) + for _, k := range rest { + write(k) + } + buf.WriteString("}\n") + + _, _ = os.Stdout.Write(buf.Bytes()) +} + +// AccessLog 结构化访问日志,替代 gin.Default() 自带的文本 Logger。 +// +// 与 gin.Logger() 的差异: +// - 一行一个 JSON 对象,字段可直接在 Loki 里过滤(status>=500、latency_ms>1000) +// - 按状态码自动分级:5xx=error / 4xx=warn / 其余=info +// - 健康检查与静态资源不打日志(K8s 探针每 10s 一次,纯噪音) +func AccessLog() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + path := c.Request.URL.Path + + c.Next() + + if isQuietPath(path) { + return + } + + status := c.Writer.Status() + level := "info" + switch { + case status >= 500: + level = "error" + case status >= 400: + level = "warn" + } + + entry := map[string]any{ + "ts": time.Now().Format(time.RFC3339Nano), + "level": level, + "type": "access", + "method": c.Request.Method, + "path": path, + "status": status, + "latency_ms": float64(time.Since(start).Microseconds()) / 1000.0, + "bytes": c.Writer.Size(), + "ip": c.ClientIP(), + } + if q := c.Request.URL.RawQuery; q != "" { + entry["query"] = q + } + if ua := c.Request.UserAgent(); ua != "" { + entry["ua"] = ua + } + // 鉴权中间件把用户信息放在 context 里,日志带上便于审计 + if uid, ok := c.Get("userId"); ok { + entry["user_id"] = uid + } + if uname, ok := c.Get("username"); ok { + entry["username"] = uname + } + if len(c.Errors) > 0 { + entry["error"] = c.Errors.String() + } + LogJSON(entry) + } +} + +// isQuietPath 探针/静态资源不记访问日志,避免日志量被轮询请求淹没 +func isQuietPath(p string) bool { + switch p { + case "/health", "/api/v1/health", "/favicon.ico", "/favicon.svg", "/robots.txt": + return true + } + return strings.HasPrefix(p, "/assets/") +}