perf: 修复 Agent 校验 CPU 热点并实现指标分层聚合与频率下发
CPU 根因：每个 Agent 请求都做一次 bcrypt 校验（实测 72ms）， 30 台主机 18 req/s × 72ms ≈ 1.35 核，实测容器 CPU 135%。 - agentauth: 校验结果按 Token 哈希缓存（TTL 5min，轮换即失效）， last_used_at 写入按 60s 节流。实测 CPU 135% → 1.6%，请求延迟 103ms → 1.6ms - metricagg: 新增后台逐级降采样聚合 raw→1m/5m→1h→6h/1d，取代前端手动触发 - metrics_aggregate: 补 uniq_metric_agg 唯一索引（原 idx_agg 非唯一导致 upsert 插重复行） - agent 合并配置接口 GET /agent/config：一次返回任务+探针+端口策略+频率， 配置轮询 5s→30s；服务端可下发 report_interval，Agent 热更新 ticker 无需重装 - retention: agent_reports 保留期 7d→1d（趋势由聚合表承担） - redis: 增加 maxmemory 512mb + volatile-lru - fix(upgrade): 升级脚本改为 fail-fast，修复下载失败仍报 UPGRADE_OK 的静默谎报 // Logan.Li
This commit is contained in:
Logan.Li committed
1 parent
93abf81e40
commit
59320aa410
23 files changed
+902
-29
No files matched your search
@@ -2,9 +2,11 @@ package agentauth
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
@@ -13,6 +15,120 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// Token 校验缓存
|
||||
//
|
||||
// bcrypt 被设计成故意慢（本项目实测单次 Cost=10 约 72ms CPU）。Agent 每 10 秒
|
||||
// 轮询 3 个接口，每个接口都要校验一次 Token，30 台主机即 9 req/s×72ms ≈ 0.65 核，
|
||||
// 纯粹烧在重复校验同一条不可变凭证上。
|
||||
//
|
||||
// 由于 Token 在轮换前不可变（Issue 才会改写 token_hash），校验结果可以安全缓存：
|
||||
// - 缓存键：token 的 SHA-256（不落明文，避免内存里出现可直接使用的凭证）
|
||||
// - 有效期：默认 5 分钟，过期后回落一次 bcrypt 并续期
|
||||
// - 轮换：Issue 时全量清空，保证「旧 Token 立即失效」语义不变
|
||||
//
|
||||
// last_used_at 的写入同样按前缀节流（默认 60s 一次），避免每请求一次 UPDATE。
|
||||
// // Logan.Li
|
||||
type cachedCredential struct {
|
||||
credential *model.AgentCredential
|
||||
expiresAt time.Time
|
||||
// lastWrite 记录最近一次把 last_used_at 落库的时间，用于写入节流。 // Logan.Li
|
||||
lastWrite time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
defaultCacheTTL = 5 * time.Minute
|
||||
defaultWriteThrottle = 60 * time.Second
|
||||
// cacheMaxEntries 上限，防止异常 Token 洪水把内存撑爆。 // Logan.Li
|
||||
cacheMaxEntries = 4096
|
||||
)
|
||||
|
||||
var (
|
||||
credCacheMu sync.Mutex
|
||||
credCache = make(map[string]*cachedCredential)
|
||||
// cacheTTL / writeThrottle 允许测试覆盖。 // Logan.Li
|
||||
cacheTTL = defaultCacheTTL
|
||||
writeThrottle = defaultWriteThrottle
|
||||
)
|
||||
|
||||
// cacheKey 只缓存 Token 的哈希，内存中不保留明文 Token。 // Logan.Li
|
||||
func cacheKey(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// lookupCache 返回命中的凭证副本；命中时同时按节流更新 last_used_at。 // Logan.Li
|
||||
func lookupCache(db *gorm.DB, token string) (*model.AgentCredential, bool) {
|
||||
key := cacheKey(token)
|
||||
credCacheMu.Lock()
|
||||
defer credCacheMu.Unlock()
|
||||
|
||||
entry, ok := credCache[key]
|
||||
if !ok || time.Now().After(entry.expiresAt) {
|
||||
if ok {
|
||||
delete(credCache, key)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
if entry.credential.ExpiresAt != nil && entry.credential.ExpiresAt.Before(time.Now()) {
|
||||
delete(credCache, key)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// 节流写入 last_used_at：仅当距上次落库超过阈值才真正 UPDATE。 // Logan.Li
|
||||
now := time.Now()
|
||||
if now.Sub(entry.lastWrite) >= writeThrottle {
|
||||
entry.lastWrite = now
|
||||
prefix := entry.credential.TokenPrefix
|
||||
go func() {
|
||||
_ = db.Model(&model.AgentCredential{}).
|
||||
Where("token_prefix = ?", prefix).
|
||||
Update("last_used_at", now).Error
|
||||
}()
|
||||
}
|
||||
|
||||
cp := *entry.credential
|
||||
cp.LastUsedAt = &now
|
||||
return &cp, true
|
||||
}
|
||||
|
||||
// storeCache 写入缓存，必要时按容量上限淘汰过期项。 // Logan.Li
|
||||
func storeCache(token string, credential *model.AgentCredential) {
|
||||
key := cacheKey(token)
|
||||
credCacheMu.Lock()
|
||||
defer credCacheMu.Unlock()
|
||||
|
||||
if len(credCache) >= cacheMaxEntries {
|
||||
now := time.Now()
|
||||
for k, v := range credCache {
|
||||
if now.After(v.expiresAt) {
|
||||
delete(credCache, k)
|
||||
}
|
||||
}
|
||||
// 仍超上限则整体清空，宁可多算几次 bcrypt 也不无限占内存。 // Logan.Li
|
||||
if len(credCache) >= cacheMaxEntries {
|
||||
credCache = make(map[string]*cachedCredential)
|
||||
}
|
||||
}
|
||||
|
||||
cp := *credential
|
||||
credCache[key] = &cachedCredential{
|
||||
credential: &cp,
|
||||
expiresAt: time.Now().Add(cacheTTL),
|
||||
lastWrite: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// invalidateCache 清空全部缓存。Token 轮换/撤销后调用，保证旧 Token 立即失效。 // Logan.Li
|
||||
func invalidateCache() {
|
||||
credCacheMu.Lock()
|
||||
credCache = make(map[string]*cachedCredential)
|
||||
credCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// Invalidate 提供给凭证变更入口（轮换/撤销/删除）显式失效缓存。 // Logan.Li
|
||||
func Invalidate() { invalidateCache() }
|
||||
|
||||
// Issue 为指定主机轮换 Agent Token，旧 Token 会立即失效。 // Logan.Li
|
||||
func Issue(db *gorm.DB, hostname, issuedBy string) (string, error) {
|
||||
hostname = strings.TrimSpace(hostname)
|
||||
@@ -47,11 +163,20 @@ func Issue(db *gorm.DB, hostname, issuedBy string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// 轮换后必须让缓存失效，否则被缓存的旧 Token 还能继续通过校验。 // Logan.Li
|
||||
invalidateCache()
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Authenticate 校验 Token 并返回绑定的主机名。 // Logan.Li
|
||||
//
|
||||
// 命中缓存时跳过 bcrypt 与凭证查询，只做一次可选节流的 last_used_at 更新；
|
||||
// 未命中时才回落完整校验路径。 // Logan.Li
|
||||
func Authenticate(db *gorm.DB, token string) (*model.AgentCredential, error) {
|
||||
if credential, ok := lookupCache(db, token); ok {
|
||||
return credential, nil
|
||||
}
|
||||
|
||||
parts := strings.SplitN(token, ".", 2)
|
||||
if len(parts) != 2 || !strings.HasPrefix(parts[0], "agt_") || parts[1] == "" {
|
||||
return nil, fmt.Errorf("invalid agent token")
|
||||
@@ -69,6 +194,9 @@ func Authenticate(db *gorm.DB, token string) (*model.AgentCredential, error) {
|
||||
now := time.Now()
|
||||
_ = db.Model(&credential).Update("last_used_at", now).Error
|
||||
credential.LastUsedAt = &now
|
||||
|
||||
// 校验通过后写入缓存，后续请求直接命中。 // Logan.Li
|
||||
storeCache(token, &credential)
|
||||
return &credential, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
// Package metricagg 指标分层滚动聚合。
|
||||
//
|
||||
// 目标：把高频原始采样（Redis ZSET）逐级降采样为多分辨率聚合行，落库供趋势查询。
|
||||
//
|
||||
// 层级（逐级降采样，每级只读上一级，避免每次从原始数据重算）：
|
||||
//
|
||||
// raw(10s, Redis) --1min--> 1m --1h--> 1h --6h/24h--> 6h / 1d
|
||||
//
|
||||
// 调度（全部后台，替代原先依赖前端手动触发 POST /agent/aggregate）：
|
||||
// - fine 每 1 分钟：raw → 1m / 5m
|
||||
// - mid 每 15 分钟：1m(DB) → 1h
|
||||
// - coarse 每 1 小时：1h(DB) → 6h / 1d
|
||||
//
|
||||
// 幂等：依赖 monitor_metric_aggregates 的 uniq_metric_agg 唯一索引 + ON CONFLICT。 // Logan.Li
|
||||
package metricagg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
ezredis "ezops/pkg/redis"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// metrics 需要聚合的指标清单（与 Agent 写入 Redis 的 key 保持一致）。 // Logan.Li
|
||||
var metrics = []string{"cpu", "mem", "load1"}
|
||||
|
||||
const (
|
||||
// rawKeyPrefix 与 controller.writeHotMetric 写入的 key 前缀保持一致。 // Logan.Li
|
||||
rawKeyPrefix = "metrics:"
|
||||
|
||||
fineEvery = time.Minute
|
||||
midEvery = 15 * time.Minute
|
||||
coarseEvery = time.Hour
|
||||
|
||||
// 各级回看窗口：只需覆盖该级「刚闭合的桶 + 边界抖动」。 // Logan.Li
|
||||
fineLookback = 3 * time.Hour
|
||||
midLookback = 6 * time.Hour
|
||||
coarseLookback = 8 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// fineIntervals / midIntervals / coarseIntervals 各级负责的聚合周期。 // Logan.Li
|
||||
var (
|
||||
fineIntervals = []interval{{"1m", time.Minute}, {"5m", 5 * time.Minute}}
|
||||
midIntervals = []interval{{"1h", time.Hour}}
|
||||
coarseIntervals = []interval{{"6h", 6 * time.Hour}, {"1d", 24 * time.Hour}}
|
||||
)
|
||||
|
||||
type interval struct {
|
||||
name string
|
||||
d time.Duration
|
||||
}
|
||||
|
||||
// Start 启动后台聚合调度，ctx 取消后退出。 // Logan.Li
|
||||
func Start(ctx context.Context, db *gorm.DB, cache ezredis.Cache, log *zap.Logger) {
|
||||
// 无 ZSet 能力（redka/local 模式）时不做 Redis 侧聚合。 // Logan.Li
|
||||
if cache == nil || !cache.Capabilities().SortedSet {
|
||||
log.Info("指标聚合未启用：当前缓存后端不支持有序集合")
|
||||
return
|
||||
}
|
||||
|
||||
go run(ctx, db, cache, log, fineEvery, "fine", func(c context.Context) {
|
||||
rollupFromRaw(c, db, cache, log, fineIntervals, fineLookback)
|
||||
})
|
||||
go run(ctx, db, cache, log, midEvery, "mid", func(c context.Context) {
|
||||
rollupFromDB(db, log, "1m", midIntervals, midLookback)
|
||||
})
|
||||
go run(ctx, db, cache, log, coarseEvery, "coarse", func(c context.Context) {
|
||||
rollupFromDB(db, log, "1h", coarseIntervals, coarseLookback)
|
||||
})
|
||||
}
|
||||
|
||||
// run 周期性执行任务，起跑前先等一个周期，避免与启动期初始化抢资源。 // Logan.Li
|
||||
func run(ctx context.Context, _ *gorm.DB, _ ezredis.Cache, log *zap.Logger, every time.Duration, name string, fn func(context.Context)) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
start := time.Now()
|
||||
fn(ctx)
|
||||
log.Debug("指标聚合完成", zap.String("level", name), zap.Duration("cost", time.Since(start)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rollupFromRaw 从 Redis 原始采样聚合出细粒度桶。 // Logan.Li
|
||||
func rollupFromRaw(ctx context.Context, db *gorm.DB, cache ezredis.Cache, log *zap.Logger, ivs []interval, lookback time.Duration) {
|
||||
hosts := activeHosts(db)
|
||||
if len(hosts) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
since := now.Add(-lookback)
|
||||
|
||||
for _, host := range hosts {
|
||||
for _, metric := range metrics {
|
||||
key := rawKeyPrefix + metric + ":" + host
|
||||
members, err := cache.ZRangeByScore(ctx, key,
|
||||
formatUnix(since), formatUnix(now.Add(time.Minute)))
|
||||
if err != nil || len(members) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, iv := range ivs {
|
||||
buckets := bucketizeMembers(members, iv.d)
|
||||
if err := upsertBuckets(db, host, metric, iv.name, buckets); err != nil {
|
||||
log.Warn("写入聚合桶失败", zap.String("host", host), zap.String("metric", metric), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rollupFromDB 从上一级聚合行再聚合出粗粒度桶（逐级降采样）。 // Logan.Li
|
||||
func rollupFromDB(db *gorm.DB, log *zap.Logger, sourceInterval string, ivs []interval, lookback time.Duration) {
|
||||
hosts := activeHosts(db)
|
||||
if len(hosts) == 0 {
|
||||
return
|
||||
}
|
||||
since := time.Now().Add(-lookback)
|
||||
|
||||
for _, host := range hosts {
|
||||
for _, metric := range metrics {
|
||||
var rows []model.MetricsAggregate
|
||||
if err := db.Where("hostname = ? AND metric = ? AND `interval` = ? AND bucket_time >= ?",
|
||||
host, metric, sourceInterval, since).
|
||||
Order("bucket_time ASC").Find(&rows).Error; err != nil || len(rows) == 0 {
|
||||
continue
|
||||
}
|
||||
samples := make([]weightedSample, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
samples = append(samples, weightedSample{ts: r.BucketTime, avg: r.Avg, count: r.Count})
|
||||
}
|
||||
for _, iv := range ivs {
|
||||
buckets := bucketizeSamples(samples, iv.d)
|
||||
if err := upsertBuckets(db, host, metric, iv.name, buckets); err != nil {
|
||||
log.Warn("写入降采样桶失败", zap.String("host", host), zap.String("metric", metric), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// activeHosts 只聚合近期在线的 Agent 主机，避免对已下线主机做无用扫描。 // Logan.Li
|
||||
func activeHosts(db *gorm.DB) []string {
|
||||
var hosts []string
|
||||
cutoff := time.Now().Add(-15 * time.Minute)
|
||||
db.Model(&model.Server{}).
|
||||
Where("agent_status = ? AND last_check_at >= ?", "online", cutoff).
|
||||
Distinct().Pluck("hostname", &hosts)
|
||||
return hosts
|
||||
}
|
||||
|
||||
type bucketStats struct {
|
||||
avg, maxV, minV, p95 float64
|
||||
count int
|
||||
}
|
||||
|
||||
type weightedSample struct {
|
||||
ts time.Time
|
||||
avg float64
|
||||
count int
|
||||
}
|
||||
|
||||
// bucketizeMembers 把原始采样按周期分桶。 // Logan.Li
|
||||
func bucketizeMembers(members []ezredis.SortedSetMember, d time.Duration) map[time.Time][]float64 {
|
||||
sec := int64(d.Seconds())
|
||||
out := make(map[time.Time][]float64)
|
||||
for _, m := range members {
|
||||
ts := int64(m.Score)
|
||||
start := (ts / sec) * sec
|
||||
out[time.Unix(start, 0).UTC()] = append(out[time.Unix(start, 0).UTC()], parseValue(m.Member))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bucketizeSamples 把上一级聚合行按周期再聚合（用 count 加权，避免小样本被放大）。 // Logan.Li
|
||||
func bucketizeSamples(samples []weightedSample, d time.Duration) map[time.Time][]float64 {
|
||||
sec := int64(d.Seconds())
|
||||
out := make(map[time.Time][]float64)
|
||||
for _, s := range samples {
|
||||
ts := s.ts.Unix()
|
||||
start := (ts / sec) * sec
|
||||
key := time.Unix(start, 0).UTC()
|
||||
// 用 avg 重复 count 次近似还原样本分布，保证 max/min/p95 有意义。 // Logan.Li
|
||||
n := s.count
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
if n > 1000 {
|
||||
n = 1000
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
out[key] = append(out[key], s.avg)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// upsertBuckets 幂等写入聚合行（依赖 uniq_metric_agg 唯一索引）。 // Logan.Li
|
||||
func upsertBuckets(db *gorm.DB, host, metric, intervalName string, buckets map[time.Time][]float64) error {
|
||||
for bucketTime, values := range buckets {
|
||||
if len(values) == 0 {
|
||||
continue
|
||||
}
|
||||
st := calcStats(values)
|
||||
row := model.MetricsAggregate{
|
||||
Hostname: host,
|
||||
Metric: metric,
|
||||
Interval: intervalName,
|
||||
Avg: st.avg,
|
||||
MaxV: st.maxV,
|
||||
MinV: st.minV,
|
||||
P95: st.p95,
|
||||
Count: st.count,
|
||||
BucketTime: bucketTime,
|
||||
}
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "hostname"}, {Name: "metric"}, {Name: "interval"}, {Name: "bucket_time"},
|
||||
},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"avg", "max_v", "min_v", "p95", "count"}),
|
||||
}).Create(&row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// calcStats 计算 avg / max / min / p95，结果保留两位小数。 // Logan.Li
|
||||
func calcStats(values []float64) bucketStats {
|
||||
sorted := make([]float64, len(values))
|
||||
copy(sorted, values)
|
||||
sort.Float64s(sorted)
|
||||
|
||||
var sum float64
|
||||
for _, v := range sorted {
|
||||
sum += v
|
||||
}
|
||||
avg := sum / float64(len(sorted))
|
||||
p95 := sorted[int(math.Ceil(float64(len(sorted))*0.95))-1]
|
||||
|
||||
return bucketStats{
|
||||
avg: round2(avg),
|
||||
maxV: round2(sorted[len(sorted)-1]),
|
||||
minV: round2(sorted[0]),
|
||||
p95: round2(p95),
|
||||
count: len(sorted),
|
||||
}
|
||||
}
|
||||
|
||||
func round2(v float64) float64 { return math.Round(v*100) / 100 }
|
||||
|
||||
// parseValue 从 "45.2300:1234567890123456789" 中取出数值部分。 // Logan.Li
|
||||
func parseValue(member string) float64 {
|
||||
for i := 0; i < len(member); i++ {
|
||||
if member[i] == ':' {
|
||||
return parseFloat(member[:i])
|
||||
}
|
||||
}
|
||||
return parseFloat(member)
|
||||
}
|
||||
|
||||
func parseFloat(s string) float64 {
|
||||
var v float64
|
||||
var frac, div float64 = 0, 1
|
||||
neg := false
|
||||
i := 0
|
||||
if i < len(s) && (s[i] == '-' || s[i] == '+') {
|
||||
neg = s[i] == '-'
|
||||
i++
|
||||
}
|
||||
for ; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c == '.' {
|
||||
i++
|
||||
break
|
||||
}
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
v = v*10 + float64(c-'0')
|
||||
}
|
||||
for ; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
frac = frac*10 + float64(c-'0')
|
||||
div *= 10
|
||||
}
|
||||
v += frac / div
|
||||
if neg {
|
||||
v = -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func formatUnix(t time.Time) string {
|
||||
// Redis ZSET score 为 Unix 秒；用整数文本作为 ZRANGEBYSCORE 的边界。 // Logan.Li
|
||||
return strconv.FormatInt(t.Unix(), 10)
|
||||
}
|
||||
Reference in new issue
Block a user