chore: 统一本地服务器与远程仓库源码
- 合并 main 与 master 历史并以服务器结构为基线 - 保留迁移、数据保留、AWS 账单与部署安全能力 - 建立 SDD 规格和 CI 验证闭环
This commit is contained in:
Logan.Li committed
commit
fd60361d93
460 files changed
+37492
-23871
No files matched your search
@@ -0,0 +1,127 @@
|
||||
// 高频数据保留与批量清理，避免 SQLite/MySQL 大表无限增长。 // Logan.Li
|
||||
package retention
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"ezops/config"
|
||||
"ezops/model"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const defaultBatchSize = 10000
|
||||
|
||||
// legacyMetricsAggregate 兼容旧聚合表；新写入统一使用 monitor_metric_aggregates。 // Logan.Li
|
||||
type legacyMetricsAggregate struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
BucketTime time.Time `gorm:"column:bucket_time"`
|
||||
}
|
||||
|
||||
// TableName 固定旧聚合表名称，避免历史数据在迁移完成前无限增长。 // Logan.Li
|
||||
func (legacyMetricsAggregate) TableName() string {
|
||||
return "monitor_metrics_aggregate"
|
||||
}
|
||||
|
||||
// ItemResult 描述单张表的清理结果，供系统任务审计与告警使用。 // Logan.Li
|
||||
type ItemResult struct {
|
||||
Table string `json:"table"`
|
||||
Retention int `json:"retention_days"`
|
||||
RowsDeleted int64 `json:"rows_deleted"`
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// Result 汇总一次全量高频数据清理的可审计结果。 // Logan.Li
|
||||
type Result struct {
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
Items []ItemResult `json:"items"`
|
||||
Total int64 `json:"total_rows_deleted"`
|
||||
}
|
||||
|
||||
// Run 仅清理已存在的高频表；每批先读取主键再删除，避免一次 DELETE 长事务。 // Logan.Li
|
||||
func Run(db *gorm.DB, cfg config.RetentionConfig) (Result, error) {
|
||||
result := Result{StartedAt: time.Now()}
|
||||
batchSize := normalizeBatchSize(cfg.CleanupBatchSize)
|
||||
|
||||
items := []struct {
|
||||
model interface{}
|
||||
table string
|
||||
column string
|
||||
retentionDays int
|
||||
}{
|
||||
{&model.AgentReport{}, "monitor_agent_reports", "report_time", cfg.AgentReportsDays},
|
||||
{&model.MetricData{}, "monitor_metric_data", "timestamp", cfg.MetricHistoryDays},
|
||||
{&model.MetricHistory{}, "monitor_metric_histories", "timestamp", cfg.MetricHistoryDays},
|
||||
{&model.MetricsAggregate{}, "monitor_metric_aggregates", "bucket_time", cfg.MetricAggregateDays},
|
||||
{&legacyMetricsAggregate{}, "monitor_metrics_aggregate", "bucket_time", cfg.MetricAggregateDays},
|
||||
{&model.SNMPMetric{}, "monitor_snmp_samples", "timestamp", cfg.SNMPSamplesDays},
|
||||
{&model.SiteResult{}, "monitor_site_results", "checked_at", cfg.SiteResultsDays},
|
||||
{&model.LogEntry{}, "sys_log_entries", "timestamp", cfg.LogEntriesDays},
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
retentionDays := normalizeRetentionDays(item.retentionDays)
|
||||
entry := ItemResult{Table: item.table, Retention: retentionDays}
|
||||
if !db.Migrator().HasTable(item.model) {
|
||||
entry.Skipped = true
|
||||
result.Items = append(result.Items, entry)
|
||||
continue
|
||||
}
|
||||
|
||||
cutoff := time.Now().AddDate(0, 0, -retentionDays)
|
||||
var err error
|
||||
if item.table == "monitor_metric_histories" || item.table == "monitor_snmp_samples" {
|
||||
err = deleteInBatches(db, item.model, item.column+" < ?", cutoff.Unix(), batchSize, &entry.RowsDeleted)
|
||||
} else {
|
||||
err = deleteInBatches(db, item.model, item.column+" < ?", cutoff, batchSize, &entry.RowsDeleted)
|
||||
}
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("清理 %s 失败: %w", item.table, err)
|
||||
}
|
||||
result.Total += entry.RowsDeleted
|
||||
result.Items = append(result.Items, entry)
|
||||
}
|
||||
|
||||
result.FinishedAt = time.Now()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// deleteInBatches 通过主键分页删除，兼容 SQLite 与 MySQL，失败时不会隐藏受影响行数。 // Logan.Li
|
||||
func deleteInBatches(db *gorm.DB, target interface{}, condition string, value interface{}, batchSize int, total *int64) error {
|
||||
for {
|
||||
var ids []uint
|
||||
if err := db.Model(target).Where(condition, value).Order("id ASC").Limit(batchSize).Pluck("id", &ids).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
deleted := db.Where("id IN ?", ids).Delete(target)
|
||||
if deleted.Error != nil {
|
||||
return deleted.Error
|
||||
}
|
||||
if deleted.RowsAffected != int64(len(ids)) {
|
||||
return fmt.Errorf("批量删除行数不一致: 预期 %d，实际 %d", len(ids), deleted.RowsAffected)
|
||||
}
|
||||
*total += deleted.RowsAffected
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRetentionDays(days int) int {
|
||||
if days <= 0 {
|
||||
return 30
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
func normalizeBatchSize(size int) int {
|
||||
if size <= 0 {
|
||||
return defaultBatchSize
|
||||
}
|
||||
if size > defaultBatchSize {
|
||||
return defaultBatchSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"ezops/config"
|
||||
"ezops/model"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// TestRun 验证高频数据按保留期分批清理且不影响新数据。 // Logan.Li
|
||||
func TestRun(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:retention?mode=memory&cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.AgentReport{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := model.AgentReport{Hostname: "old", ReportTime: time.Now().AddDate(0, 0, -8)}
|
||||
current := model.AgentReport{Hostname: "new", ReportTime: time.Now()}
|
||||
if err := db.Create(&old).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(¤t).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := Run(db, config.RetentionConfig{AgentReportsDays: 7, CleanupBatchSize: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Total != 1 {
|
||||
t.Fatalf("应清理 1 条记录，实际 %d", result.Total)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&model.AgentReport{}).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("应保留 1 条新记录，实际 %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunCleansLegacyMetricTables 验证保留期覆盖尚未下线的旧时序表。 // Logan.Li
|
||||
func TestRunCleansLegacyMetricTables(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:retention_legacy?mode=memory&cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.MetricData{}, &legacyMetricsAggregate{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.MetricData{Name: "old", Timestamp: time.Now().AddDate(0, 0, -31)}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.MetricData{Name: "current", Timestamp: time.Now()}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO monitor_metrics_aggregate (bucket_time) VALUES (?), (?)", time.Now().AddDate(0, 0, -366), time.Now()).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := Run(db, config.RetentionConfig{MetricHistoryDays: 30, MetricAggregateDays: 365, CleanupBatchSize: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Total != 2 {
|
||||
t.Fatalf("应清理两条旧指标，实际 %d", result.Total)
|
||||
}
|
||||
assertTableCount(t, db, "monitor_metric_data", 1)
|
||||
assertTableCount(t, db, "monitor_metrics_aggregate", 1)
|
||||
}
|
||||
|
||||
// assertTableCount 断言时序表保留数据量。 // Logan.Li
|
||||
func assertTableCount(t *testing.T, db *gorm.DB, table string, want int64) {
|
||||
t.Helper()
|
||||
var count int64
|
||||
if err := db.Table(table).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != want {
|
||||
t.Fatalf("表 %s 记录数错误: got %d want %d", table, count, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user