perf: 修复 Agent 校验 CPU 热点并实现指标分层聚合与频率下发
CPU 根因：每个 Agent 请求都做一次 bcrypt 校验（实测 72ms）， 30 台主机 18 req/s × 72ms ≈ 1.35 核，实测容器 CPU 135%。 - agentauth: 校验结果按 Token 哈希缓存（TTL 5min，轮换即失效）， last_used_at 写入按 60s 节流。实测 CPU 135% → 1.6%，请求延迟 103ms → 1.6ms - metricagg: 新增后台逐级降采样聚合 raw→1m/5m→1h→6h/1d，取代前端手动触发 - metrics_aggregate: 补 uniq_metric_agg 唯一索引（原 idx_agg 非唯一导致 upsert 插重复行） - agent 合并配置接口 GET /agent/config：一次返回任务+探针+端口策略+频率， 配置轮询 5s→30s；服务端可下发 report_interval，Agent 热更新 ticker 无需重装 - retention: agent_reports 保留期 7d→1d（趋势由聚合表承担） - redis: 增加 maxmemory 512mb + volatile-lru - fix(upgrade): 升级脚本改为 fail-fast，修复下载失败仍报 UPGRADE_OK 的静默谎报 // Logan.Li
This commit is contained in:
Logan.Li committed
1 parent
93abf81e40
commit
59320aa410
23 files changed
+902
-29
No files matched your search
@@ -0,0 +1,198 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"ezops/middleware"
|
||||
"ezops/model"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Agent 采集策略的兜底默认值（表内无记录时使用）。 // Logan.Li
|
||||
const (
|
||||
defaultReportInterval = 10
|
||||
defaultConfigPollInterval = 30
|
||||
// intervalFloor 与 Agent 侧校验保持一致：过密会打爆平台。 // Logan.Li
|
||||
intervalFloor = 5
|
||||
)
|
||||
|
||||
// GetAgentConfig 合并下发 Agent 配置。
|
||||
//
|
||||
// 原先 Agent 每 5 秒要分别请求 tasks/poll、health-probes/config、port-policy 三个接口，
|
||||
// 配置类请求量是上报本身的 6 倍；合并为单次响应后请求量降到原来的 1/3。 // Logan.Li
|
||||
//
|
||||
// GET /api/v1/agent/config
|
||||
func (h *AgentHandler) GetAgentConfig(c *gin.Context) {
|
||||
hostname := c.Query("hostname")
|
||||
if hostname == "" {
|
||||
c.JSON(http.StatusBadRequest, model.Response{Code: 400, Message: "hostname 不能为空"})
|
||||
return
|
||||
}
|
||||
if !middleware.RequireAgentHostname(c, hostname) {
|
||||
return
|
||||
}
|
||||
|
||||
setting := h.resolveAgentSetting(hostname)
|
||||
|
||||
// 任务：拉取 pending 并标记 running（与 /tasks/poll 行为一致）。 // Logan.Li
|
||||
tasks := h.pickPendingTasks(hostname)
|
||||
|
||||
// 探针配置。 // Logan.Li
|
||||
probes := []model.HealthProbe{}
|
||||
h.db.Where("hostname = ? AND enabled = ?", hostname, true).Order("id ASC").Find(&probes)
|
||||
|
||||
// 端口策略。 // Logan.Li
|
||||
policy := h.resolvePortPolicy(hostname)
|
||||
|
||||
dto := model.AgentConfigDTO{
|
||||
ReportInterval: setting.ReportInterval,
|
||||
ConfigPollInterval: setting.ConfigPollInterval,
|
||||
Tasks: tasks,
|
||||
HealthProbes: probes,
|
||||
PortPolicy: policy,
|
||||
}
|
||||
dto.ConfigVersion = agentConfigVersion(dto)
|
||||
|
||||
c.JSON(http.StatusOK, model.Response{Code: 200, Message: "success", Data: dto})
|
||||
}
|
||||
|
||||
// SaveAgentSetting 平台侧设置采集策略：hostname 为空串表示全局默认。 // Logan.Li
|
||||
// PUT /api/v1/agent/settings
|
||||
func (h *AgentHandler) SaveAgentSetting(c *gin.Context) {
|
||||
var req struct {
|
||||
Hostname string `json:"hostname"`
|
||||
ReportInterval int `json:"report_interval"`
|
||||
ConfigPollInterval int `json:"config_poll_interval"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, model.Response{Code: 400, Message: "请求格式错误: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.ReportInterval < intervalFloor {
|
||||
c.JSON(http.StatusBadRequest, model.Response{
|
||||
Code: 400, Message: fmt.Sprintf("上报间隔不能小于 %d 秒", intervalFloor)})
|
||||
return
|
||||
}
|
||||
if req.ConfigPollInterval < intervalFloor {
|
||||
c.JSON(http.StatusBadRequest, model.Response{
|
||||
Code: 400, Message: fmt.Sprintf("配置轮询间隔不能小于 %d 秒", intervalFloor)})
|
||||
return
|
||||
}
|
||||
|
||||
setting := model.AgentSetting{Hostname: req.Hostname}
|
||||
h.db.Where("hostname = ?", req.Hostname).Assign(map[string]interface{}{
|
||||
"report_interval": req.ReportInterval,
|
||||
"config_poll_interval": req.ConfigPollInterval,
|
||||
"updated_by": middleware.GetUsername(c),
|
||||
}).FirstOrCreate(&setting)
|
||||
|
||||
c.JSON(http.StatusOK, model.Response{Code: 200, Message: "已保存", Data: setting})
|
||||
}
|
||||
|
||||
// ListAgentSettings 列出全局默认与所有按主机覆盖项。 // Logan.Li
|
||||
// GET /api/v1/agent/settings
|
||||
func (h *AgentHandler) ListAgentSettings(c *gin.Context) {
|
||||
var items []model.AgentSetting
|
||||
h.db.Order("hostname ASC").Find(&items)
|
||||
if items == nil {
|
||||
items = []model.AgentSetting{}
|
||||
}
|
||||
c.JSON(http.StatusOK, model.Response{Code: 200, Message: "success", Data: gin.H{
|
||||
"defaults": gin.H{
|
||||
"report_interval": defaultReportInterval,
|
||||
"config_poll_interval": defaultConfigPollInterval,
|
||||
},
|
||||
"items": items,
|
||||
}})
|
||||
}
|
||||
|
||||
// resolveAgentSetting 解析生效的采集策略：按主机覆盖 > 全局默认 > 内置默认。 // Logan.Li
|
||||
func (h *AgentHandler) resolveAgentSetting(hostname string) model.AgentSetting {
|
||||
effective := model.AgentSetting{
|
||||
Hostname: hostname,
|
||||
ReportInterval: defaultReportInterval,
|
||||
ConfigPollInterval: defaultConfigPollInterval,
|
||||
}
|
||||
|
||||
// 全局默认。 // Logan.Li
|
||||
var global model.AgentSetting
|
||||
if err := h.db.Where("hostname = ?", "").Take(&global).Error; err == nil {
|
||||
applyAgentSetting(&effective, global)
|
||||
}
|
||||
// 按主机覆盖。 // Logan.Li
|
||||
var perHost model.AgentSetting
|
||||
if err := h.db.Where("hostname = ?", hostname).Take(&perHost).Error; err == nil {
|
||||
applyAgentSetting(&effective, perHost)
|
||||
}
|
||||
return effective
|
||||
}
|
||||
|
||||
// applyAgentSetting 仅在来源值合法时覆盖，避免脏数据把间隔写成 0。 // Logan.Li
|
||||
func applyAgentSetting(dst *model.AgentSetting, src model.AgentSetting) {
|
||||
if src.ReportInterval >= intervalFloor {
|
||||
dst.ReportInterval = src.ReportInterval
|
||||
}
|
||||
if src.ConfigPollInterval >= intervalFloor {
|
||||
dst.ConfigPollInterval = src.ConfigPollInterval
|
||||
}
|
||||
}
|
||||
|
||||
// pickPendingTasks 取出该主机待执行任务并标记 running，最多 5 个。 // Logan.Li
|
||||
func (h *AgentHandler) pickPendingTasks(hostname string) []model.AgentTask {
|
||||
tasks := []model.AgentTask{}
|
||||
h.db.Where("hostname = ? AND status = ?", hostname, "pending").
|
||||
Order("created_at ASC").Limit(5).Find(&tasks)
|
||||
if len(tasks) == 0 {
|
||||
return tasks
|
||||
}
|
||||
now := time.Now()
|
||||
for _, t := range tasks {
|
||||
// 条件更新防止多 Agent 重复认领同一任务。 // Logan.Li
|
||||
h.db.Model(&model.AgentTask{}).
|
||||
Where("id = ? AND status = ?", t.ID, "pending").
|
||||
Updates(map[string]interface{}{"status": "running", "picked_at": now})
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
// resolvePortPolicy 读取主机端口策略，无记录时返回空白名单。 // Logan.Li
|
||||
func (h *AgentHandler) resolvePortPolicy(hostname string) *model.AgentPortPolicyDTO {
|
||||
empty := &model.AgentPortPolicyDTO{Hostname: hostname, PortFilterMode: "whitelist", PortWhitelist: []int{}}
|
||||
var policy model.AgentPortPolicy
|
||||
if err := h.db.Where("hostname = ?", hostname).First(&policy).Error; err != nil {
|
||||
return empty
|
||||
}
|
||||
whitelist := []int{}
|
||||
_ = json.Unmarshal([]byte(policy.PortWhitelist), &whitelist)
|
||||
return &model.AgentPortPolicyDTO{
|
||||
Hostname: policy.Hostname,
|
||||
PortFilterMode: policy.PortFilterMode,
|
||||
PortWhitelist: whitelist,
|
||||
}
|
||||
}
|
||||
|
||||
// agentConfigVersion 对配置内容取指纹，Agent 可据此跳过未变更配置的处理。 // Logan.Li
|
||||
func agentConfigVersion(dto model.AgentConfigDTO) string {
|
||||
payload := struct {
|
||||
ReportInterval int
|
||||
ConfigPollInterval int
|
||||
Tasks []model.AgentTask
|
||||
HealthProbes []model.HealthProbe
|
||||
PortPolicy *model.AgentPortPolicyDTO
|
||||
}{
|
||||
dto.ReportInterval, dto.ConfigPollInterval, dto.Tasks, dto.HealthProbes, dto.PortPolicy,
|
||||
}
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return strconv.FormatInt(time.Now().Unix(), 10)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -695,9 +695,23 @@ func (h *AgentInstallHandler) BatchUpgradeAgent(c *gin.Context) {
|
||||
binaryFile = "aiops-agent-linux-arm64"
|
||||
}
|
||||
|
||||
// 下载新二进制 + 获取版本号 + 重启服务
|
||||
// 下载新二进制 + 校验 + 重启服务。
|
||||
// 必须 fail-fast：旧版脚本用 `;` 串联并把 UPGRADE_OK 无条件 echo，
|
||||
// 下载失败时同样报"成功"，导致升级结果完全不可信。 // Logan.Li
|
||||
upgradeScript := fmt.Sprintf(
|
||||
`curl -fsSL -o /tmp/aiops-agent %s && chmod +x /tmp/aiops-agent && systemctl stop aiops-agent 2>/dev/null; mv /tmp/aiops-agent /usr/local/bin/aiops-agent && systemctl start aiops-agent; ver=$(/usr/local/bin/aiops-agent version 2>/dev/null | awk '/AIOps Agent/{print $3}'); echo "UPGRADE_OK ver=$ver"`,
|
||||
`set -e; `+
|
||||
`curl -fsSL --connect-timeout 10 -o /tmp/aiops-agent.new %s || { echo "UPGRADE_FAIL download"; exit 1; }; `+
|
||||
`size=$(wc -c < /tmp/aiops-agent.new); `+
|
||||
`if [ "$size" -lt 1000000 ]; then echo "UPGRADE_FAIL size=$size"; exit 1; fi; `+
|
||||
`chmod +x /tmp/aiops-agent.new; `+
|
||||
`/tmp/aiops-agent.new version >/dev/null 2>&1 || { echo "UPGRADE_FAIL not-executable"; exit 1; }; `+
|
||||
`systemctl stop aiops-agent 2>/dev/null || true; `+
|
||||
`mv /tmp/aiops-agent.new /usr/local/bin/aiops-agent; `+
|
||||
`systemctl start aiops-agent; `+
|
||||
`sleep 1; `+
|
||||
`ver=$(/usr/local/bin/aiops-agent version 2>/dev/null | awk '/AIOps Agent/{print $3}'); `+
|
||||
`if [ -z "$ver" ]; then echo "UPGRADE_FAIL version-empty"; exit 1; fi; `+
|
||||
`echo "UPGRADE_OK ver=$ver"`,
|
||||
shellQuote(serverURL+"/api/agent/bin/"+binaryFile),
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user