128 lines
4.2 KiB
Go
128 lines
4.2 KiB
Go
// 高频数据保留与批量清理，避免 SQLite/MySQL 大表无限增长。 // Logan.Li
|
|
package retention
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"ezops/config"
|
|
"ezops/model"
|
|
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
const defaultBatchSize = 10000
|
|
|
|
// legacyMetricsAggregate 兼容旧聚合表；新写入统一使用 monitor_metric_aggregates。 // Logan.Li
|
|
type legacyMetricsAggregate struct {
|
|
ID uint `gorm:"primaryKey"`
|
|
BucketTime time.Time `gorm:"column:bucket_time"`
|
|
}
|
|
|
|
// TableName 固定旧聚合表名称，避免历史数据在迁移完成前无限增长。 // Logan.Li
|
|
func (legacyMetricsAggregate) TableName() string {
|
|
return "monitor_metrics_aggregate"
|
|
}
|
|
|
|
// ItemResult 描述单张表的清理结果，供系统任务审计与告警使用。 // Logan.Li
|
|
type ItemResult struct {
|
|
Table string `json:"table"`
|
|
Retention int `json:"retention_days"`
|
|
RowsDeleted int64 `json:"rows_deleted"`
|
|
Skipped bool `json:"skipped,omitempty"`
|
|
}
|
|
|
|
// Result 汇总一次全量高频数据清理的可审计结果。 // Logan.Li
|
|
type Result struct {
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinishedAt time.Time `json:"finished_at"`
|
|
Items []ItemResult `json:"items"`
|
|
Total int64 `json:"total_rows_deleted"`
|
|
}
|
|
|
|
// Run 仅清理已存在的高频表；每批先读取主键再删除，避免一次 DELETE 长事务。 // Logan.Li
|
|
func Run(db *gorm.DB, cfg config.RetentionConfig) (Result, error) {
|
|
result := Result{StartedAt: time.Now()}
|
|
batchSize := normalizeBatchSize(cfg.CleanupBatchSize)
|
|
|
|
items := []struct {
|
|
model interface{}
|
|
table string
|
|
column string
|
|
retentionDays int
|
|
}{
|
|
{&model.AgentReport{}, "monitor_agent_reports", "report_time", cfg.AgentReportsDays},
|
|
{&model.MetricData{}, "monitor_metric_data", "timestamp", cfg.MetricHistoryDays},
|
|
{&model.MetricHistory{}, "monitor_metric_histories", "timestamp", cfg.MetricHistoryDays},
|
|
{&model.MetricsAggregate{}, "monitor_metric_aggregates", "bucket_time", cfg.MetricAggregateDays},
|
|
{&legacyMetricsAggregate{}, "monitor_metrics_aggregate", "bucket_time", cfg.MetricAggregateDays},
|
|
{&model.SNMPMetric{}, "monitor_snmp_samples", "timestamp", cfg.SNMPSamplesDays},
|
|
{&model.SiteResult{}, "monitor_site_results", "checked_at", cfg.SiteResultsDays},
|
|
{&model.LogEntry{}, "sys_log_entries", "timestamp", cfg.LogEntriesDays},
|
|
}
|
|
|
|
for _, item := range items {
|
|
retentionDays := normalizeRetentionDays(item.retentionDays)
|
|
entry := ItemResult{Table: item.table, Retention: retentionDays}
|
|
if !db.Migrator().HasTable(item.model) {
|
|
entry.Skipped = true
|
|
result.Items = append(result.Items, entry)
|
|
continue
|
|
}
|
|
|
|
cutoff := time.Now().AddDate(0, 0, -retentionDays)
|
|
var err error
|
|
if item.table == "monitor_metric_histories" || item.table == "monitor_snmp_samples" {
|
|
err = deleteInBatches(db, item.model, item.column+" < ?", cutoff.Unix(), batchSize, &entry.RowsDeleted)
|
|
} else {
|
|
err = deleteInBatches(db, item.model, item.column+" < ?", cutoff, batchSize, &entry.RowsDeleted)
|
|
}
|
|
if err != nil {
|
|
return result, fmt.Errorf("清理 %s 失败: %w", item.table, err)
|
|
}
|
|
result.Total += entry.RowsDeleted
|
|
result.Items = append(result.Items, entry)
|
|
}
|
|
|
|
result.FinishedAt = time.Now()
|
|
return result, nil
|
|
}
|
|
|
|
// deleteInBatches 通过主键分页删除，兼容 SQLite 与 MySQL，失败时不会隐藏受影响行数。 // Logan.Li
|
|
func deleteInBatches(db *gorm.DB, target interface{}, condition string, value interface{}, batchSize int, total *int64) error {
|
|
for {
|
|
var ids []uint
|
|
if err := db.Model(target).Where(condition, value).Order("id ASC").Limit(batchSize).Pluck("id", &ids).Error; err != nil {
|
|
return err
|
|
}
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
deleted := db.Where("id IN ?", ids).Delete(target)
|
|
if deleted.Error != nil {
|
|
return deleted.Error
|
|
}
|
|
if deleted.RowsAffected != int64(len(ids)) {
|
|
return fmt.Errorf("批量删除行数不一致: 预期 %d，实际 %d", len(ids), deleted.RowsAffected)
|
|
}
|
|
*total += deleted.RowsAffected
|
|
}
|
|
}
|
|
|
|
func normalizeRetentionDays(days int) int {
|
|
if days <= 0 {
|
|
return 30
|
|
}
|
|
return days
|
|
}
|
|
|
|
func normalizeBatchSize(size int) int {
|
|
if size <= 0 {
|
|
return defaultBatchSize
|
|
}
|
|
if size > defaultBatchSize {
|
|
return defaultBatchSize
|
|
}
|
|
return size
|
|
}
|