feat: v0.34 健康探针(Architecture B) + Agent v2.1 + 端口监控 + 告警联系人重构

- v0.31: AlertContact 重构(channel_type下拉+webhook_url替代三分立字段) - v0.32: Agent ProcessTop 增强(cmdline/user/cpu_pct/mem_pct) - v0.33: 端口监控 Tab(监听端口自动发现+down联动告警) - v0.34: 健康探针(Agent本地执行HTTP/TCP/Exec→结果随心跳上报) - Agent新增: ContextSwitch/FD/HealthProbe采集器 - 后端新增: health_probe handler + Agent model扩展 - HostMonitor新增Tab: 健康探针/端口监控
2026-06-24 19:18:10 +08:00
1 parent e1f75f0bc4
commit d656520428
@@ -1,6 +1,7 @@
 // Package main 是 AIOps Agent 的入口。
 // Agent 定时采集 CPU、内存、磁盘、端口、进程 TOP、Swap、FD、上下文切换等指标，
 // 并通过 HTTP 上报至 AIOps 后端。
+// v2.1: 共享 HTTP client + 独立探针拉取 + 配置结构体
 //
 // 优化要点（1000 台服务器规模）：
 //   - 启动随机 Jitter 避免惊群效应
@@ -15,7 +16,6 @@ package main
 import (
 	"context"
 	"flag"
-	"fmt"
 	"log"
 	"math/rand"
 	"net"
@@ -25,6 +25,7 @@ import (
 	"syscall"
 	"time"

+	agentclient "ezops/agent/client"
 	"ezops/agent/collector"
 	"ezops/agent/model"
 	"ezops/agent/reporter"
@@ -32,38 +33,51 @@ import (
 )

 var (
-	version   = "v2.0.0"
+	version   = "v2.1.0"
 	buildTime = "unknown"
 )

+// Config Agent 运行配置（v2.1 从 flag 集中解析）
+type Config struct {
+	Server   string
+	Token    string
+	Hostname string
+	Interval int
+	TopN     int
+}
+
 func main() {
-	server := flag.String("server", "http://localhost:8080", "AIOps 后端上报地址")
-	interval := flag.Int("interval", 10, "采集与上报间隔，单位：秒")
-	token := flag.String("token", "", "Bearer 鉴权令牌")
-	hostnameFlag := flag.String("hostname", "", "上报主机名，默认使用系统主机名")
-	topN := flag.Int("top-n", 3, "进程 TOP N 数量")
+	cfg := Config{
+		Server:   *flag.String("server", "http://localhost:8080", "AIOps 后端上报地址"),
+		Token:    *flag.String("token", "", "Bearer 鉴权令牌"),
+		Hostname: *flag.String("hostname", "", "上报主机名，默认使用系统主机名"),
+		Interval: *flag.Int("interval", 10, "采集与上报间隔，单位：秒"),
+		TopN:     *flag.Int("top-n", 3, "进程 TOP N 数量"),
+	}
 	flag.Parse()

 	logger := log.New(os.Stdout, "[aiops-agent] ", log.LstdFlags|log.Lmicroseconds)

-	hostname := *hostnameFlag
-	if hostname == "" {
+	if cfg.Hostname == "" {
 		if h, err := os.Hostname(); err == nil {
-			hostname = h
+			cfg.Hostname = h
 		} else {
-			hostname = "unknown"
+			cfg.Hostname = "unknown"
 		}
 	}

-	if *server == "" {
+	if cfg.Server == "" {
 		logger.Fatal("参数 -server 不能为空")
 	}
-	if *interval < 5 {
-		logger.Fatalf("参数 -interval 必须 >= 5，当前为 %d（1000 台规模推荐 30~60s）", *interval)
+	if cfg.Interval < 5 {
+		logger.Fatalf("参数 -interval 必须 >= 5，当前为 %d（1000 台规模推荐 30~60s）", cfg.Interval)
 	}

 	logger.Printf("AIOps Agent v%s (built %s) go=%s", version, buildTime, runtime.Version())
-	logger.Printf("配置: server=%s interval=%ds top-n=%d hostname=%q", *server, *interval, *topN, hostname)
+	logger.Printf("配置: server=%s interval=%ds top-n=%d hostname=%q", cfg.Server, cfg.Interval, cfg.TopN, cfg.Hostname)
+
+	// ═══ 共享 HTTP 客户端（Reporter + TaskClient 共用）═══
+	httpCli := agentclient.New(cfg.Server, cfg.Token, logger)

 	// ═══ 初始化采集器 ═══
 	cpuCollector := collector.NewCPUCollector()
@@ -74,7 +88,10 @@ func main() {
 	diskIOCollector := collector.NewDiskIOCollector()
 	procTopCollector := collector.NewProcessTopCollector()
 	swapCollector := collector.NewSwapCollector()
-	rep := reporter.New(*server, *token, logger)
+	healthProbeCollector := collector.NewHealthProbeCollector()
+	fdCollector := collector.NewFDCollector()
+	csCollector := collector.NewContextSwitchCollector()
+	rep := reporter.New(httpCli, logger)

 	// 系统静态信息（仅启动时一次）
 	sysInfo := collector.CollectSystem()
@@ -84,111 +101,97 @@ func main() {
 	localIP := detectLocalIP()
 	logger.Printf("探测本机 IP: %s", localIP)

-	sysReportCount := 0 // 前 3 次上报携带系统信息
+	sysReportCount := 0

-	// 任务执行器
+	// 任务执行器 + 任务客户端
 	taskExecutor := task.New(logger)
-	taskClient := task.NewClient(*server, *token, hostname, logger)
+	taskClient := task.NewClient(httpCli, cfg.Hostname, logger)

-	// ═══ 启动随机 Jitter：避免 1000 台同时上报 ═══
-	startupJitter := time.Duration(rand.Intn(*interval)) * time.Second
+	// ═══ 启动随机 Jitter ═══
+	startupJitter := time.Duration(rand.Intn(cfg.Interval)) * time.Second
 	logger.Printf("启动 Jitter: %v（防止惊群效应）", startupJitter)

 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()

-	// Agent 自检序列号
 	var reportSeq uint64
+	var probeConfigs []model.ProbeConfig

-	// collectAndReport 执行单次采集并上报
+	// ═══ collectAndReport 采集 + 上报 ═══
 	collectAndReport := func() {
 		report := &model.Report{
-			Hostname:     hostname,
+			Hostname:     cfg.Hostname,
 			IP:           localIP,
 			Timestamp:    time.Now().UTC().Format(time.RFC3339),
 			AgentVersion: version,
 		}

-		// 系统信息（前 3 次）
 		if sysReportCount < 3 && (sysInfo.OSVersion != "" || sysInfo.Kernel != "") {
 			report.System = &model.SystemStat{
-				OSVersion:    sysInfo.OSVersion,
-				Kernel:       sysInfo.Kernel,
-				Uptime:       sysInfo.UptimeSeconds,
+				OSVersion: sysInfo.OSVersion,
+				Kernel:    sysInfo.Kernel,
+				Uptime:    sysInfo.UptimeSeconds,
 			}
 			sysReportCount++
 		}

-		// CPU
+		// 系统指标采集
 		if stat, err := cpuCollector.Collect(); err != nil {
 			logger.Printf("CPU 采集异常: %v", err)
 		} else {
 			report.CPU = stat
 		}
-
-		// 内存
 		if stat, err := memCollector.Collect(); err != nil {
 			logger.Printf("内存采集异常: %v", err)
 		} else {
 			report.Memory = stat
 		}
-
-		// Swap（仅当系统配置了 swap 才上报）
 		if swap, err := swapCollector.Collect(); err != nil {
 			logger.Printf("Swap 采集异常: %v", err)
 		} else if swap != nil {
 			report.Swap = swap
 		}
-
-		// 磁盘
 		if disks, err := diskCollector.Collect(); err != nil {
 			logger.Printf("磁盘采集异常: %v", err)
 		} else {
 			report.Disks = disks
 		}
-
-		// 监听端口
 		if ports, err := portCollector.Collect(); err != nil {
 			logger.Printf("端口采集异常: %v", err)
 		} else {
 			report.Ports = ports
 		}
-
-		// 网络
 		if netStat, err := netCollector.Collect(); err != nil {
 			logger.Printf("网络采集异常: %v", err)
 		} else {
 			report.Network = netStat
 		}
-
-		// 磁盘 IO
 		if dioStat, err := diskIOCollector.Collect(); err != nil {
 			logger.Printf("磁盘IO采集异常: %v", err)
 		} else {
 			report.DiskIO = dioStat
 		}
-
-		// 进程 TOP N
-		if procs, err := procTopCollector.Collect(*topN); err != nil {
+		if procs, err := procTopCollector.Collect(cfg.TopN); err != nil {
 			logger.Printf("进程TOP采集异常: %v", err)
 		} else {
 			report.ProcessTop = procs
 		}
-
-		// 系统 FD 句柄
-		if fdStat, err := collectFD(); err != nil {
+		if fdStat, err := fdCollector.Collect(); err != nil {
 			logger.Printf("FD采集异常: %v", err)
 		} else {
 			report.FD = fdStat
 		}
-
-		// 上下文切换
-		if csStat, err := collectContextSwitches(); err != nil {
-			// 静默跳过（某些内核/容器环境不可用）
+		if csStat, err := csCollector.Collect(); err != nil {
+			// 静默跳过
 		} else {
 			report.ContextSwitches = csStat
 		}

+		// v2.1 健康探针（使用缓存配置）
+		if len(probeConfigs) > 0 {
+			report.ProbeResults = healthProbeCollector.Execute(ctx, probeConfigs)
+		}
+
 		// Agent 自检心跳
 		reportSeq++
 		var memStats runtime.MemStats
@@ -196,17 +199,19 @@ func main() {
 		report.Heartbeat = model.AgentHeartbeat{
 			Version:   version,
 			UptimeSec: sysInfo.UptimeSeconds,
-			SelfCPU:   0, // TODO: self-CPU via /proc/self/stat
+			SelfCPU:   0,
 			SelfRSSMB: float64(memStats.Alloc) / 1024.0 / 1024.0,
 			ReportSeq: reportSeq,
 		}

-		// 上报
+		// 上报指标（纯上报，不解析响应）
 		if err := rep.Send(ctx, report); err != nil {
 			logger.Printf("上报失败: %v", err)
 			return
 		}
-		logger.Printf("上报成功: cpu=%.1f%% mem=%.1f%% swap=%.1f%% disks=%d ports=%d procs=%d fd=%d/%d cs=%.0f/s",
+
+		probeCount := len(report.ProbeResults)
+		logger.Printf("上报成功: cpu=%.1f%% mem=%.1f%% swap=%.1f%% disks=%d ports=%d procs=%d fd=%d/%d cs=%.0f/s probes=%d",
 			report.CPU.UsagePercent,
 			report.Memory.UsagePercent,
 			func() float64 {
@@ -215,9 +220,7 @@ func main() {
 				}
 				return 0
 			}(),
-			len(report.Disks),
-			len(report.Ports),
-			len(report.ProcessTop),
+			len(report.Disks), len(report.Ports), len(report.ProcessTop),
 			func() int {
 				if report.FD != nil {
 					return report.FD.Allocated
@@ -236,10 +239,11 @@ func main() {
 				}
 				return 0
 			}(),
+			probeCount,
 		)
 	}

-	// pollAndExecute 任务轮询
+	// ═══ pollAndExecute 任务轮询 ═══
 	pollAndExecute := func() {
 		tasks, err := taskClient.Poll(ctx)
 		if err != nil {
@@ -249,46 +253,59 @@ func main() {
 			logger.Printf("任务轮询失败: %v", err)
 			return
 		}
-		if len(tasks) == 0 {
-			return
-		}
-		logger.Printf("拉取到 %d 个待执行任务", len(tasks))
-		for _, t := range tasks {
-			if ctx.Err() != nil {
-				return
-			}
-			result := taskExecutor.ExecuteTask(ctx, t)
-			if err := taskClient.Submit(ctx, result); err != nil {
+		if len(tasks) > 0 {
+			logger.Printf("拉取到 %d 个待执行任务", len(tasks))
+			for _, t := range tasks {
 				if ctx.Err() != nil {
 					return
 				}
-				logger.Printf("任务 %s 结果回传失败: %v", t.TaskID, err)
-			} else {
-				logger.Printf("任务 %s 结果回传成功 status=%s", t.TaskID, result.Status)
+				result := taskExecutor.ExecuteTask(ctx, t)
+				if err := taskClient.Submit(ctx, result); err != nil {
+					logger.Printf("任务 %s 结果回传失败: %v", t.TaskID, err)
+				} else {
+					logger.Printf("任务 %s 结果回传成功 status=%s", t.TaskID, result.Status)
+				}
 			}
 		}
 	}

-	// ═══ 任务轮询协程（5s 间隔，不受 -interval 影响）═══
-	const taskPollInterval = 5 * time.Second
+	// ═══ pullProbes 探针配置拉取（v2.1 独立于上报）═══
+	pullProbes := func() {
+		configs, err := taskClient.PollProbes(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			logger.Printf("探针配置拉取失败: %v", err)
+			return
+		}
+		if len(configs) > 0 {
+			probeConfigs = configs
+		}
+	}
+
+	// ═══ 任务轮询 + 探针拉取协程（5s 间隔）═══
+	const pollInterval = 5 * time.Second
 	go func() {
 		pollAndExecute()
-		ticker := time.NewTicker(taskPollInterval)
+		pullProbes()
+		ticker := time.NewTicker(pollInterval)
 		defer ticker.Stop()
 		for {
 			select {
 			case <-ticker.C:
 				pollAndExecute()
+				pullProbes()
 			case <-ctx.Done():
 				return
 			}
 		}
 	}()

-	// ═══ 主循环：启动先 sleep jitter，再按 interval 采集 ═══
+	// ═══ 主循环：采集 + 上报 ═══
 	time.Sleep(startupJitter)
 	collectAndReport()
-	ticker := time.NewTicker(time.Duration(*interval) * time.Second)
+	ticker := time.NewTicker(time.Duration(cfg.Interval) * time.Second)
 	defer ticker.Stop()
 	for {
 		select {
@@ -301,17 +318,6 @@ func main() {
 	}
 }

-// maskToken 对令牌做脱敏处理
-func maskToken(token string) string {
-	if token == "" {
-		return "(empty)"
-	}
-	if len(token) <= 4 {
-		return "****"
-	}
-	return token[:4] + "****"
-}
-
 // detectLocalIP 探测本机主 IP
 func detectLocalIP() string {
 	conn, err := net.Dial("udp", "8.8.8.8:53")
@@ -337,73 +343,3 @@ func detectLocalIP() string {
 	}
 	return "unknown"
 }
-
-// ══════════════════════════════════════════════
-// 补充采集器（内联简单实现，不单独建文件）
-// ══════════════════════════════════════════════
-
-// collectFD 读取 /proc/sys/fs/file-nr 和 /proc/sys/fs/file-max
-func collectFD() (*model.SocketFDStat, error) {
-	alloc, err := os.ReadFile("/proc/sys/fs/file-nr")
-	if err != nil {
-		return nil, err
-	}
-	var allocated, _unused1, _unused2 int
-	fmt.Sscanf(string(alloc), "%d %d %d", &allocated, &_unused1, &_unused2)
-
-	max, err := os.ReadFile("/proc/sys/fs/file-max")
-	if err != nil {
-		return nil, err
-	}
-	var maxFD int64
-	fmt.Sscanf(string(max), "%d", &maxFD)
-
-	stat := &model.SocketFDStat{
-		Allocated: allocated,
-		Max:       maxFD,
-	}
-	if maxFD > 0 {
-		stat.UsagePct = float64(allocated) / float64(maxFD) * 100.0
-	}
-	return stat, nil
-}
-
-// collectContextSwitches 读取 /proc/stat 中的 ctxt 和 intr 行
-// 需要两次采样才能计算速率；这里使用简单的累计值（首次返回 0）
-func collectContextSwitches() (*model.ContextSwitchStat, error) {
-	data, err := os.ReadFile("/proc/stat")
-	if err != nil {
-		return nil, err
-	}
-	stat := &model.ContextSwitchStat{}
-	lines := string(data)
-	for _, line := range splitLines(lines) {
-		if hasPrefix(line, "ctxt ") {
-			fmt.Sscanf(line, "ctxt %f", &stat.CTXPerSec)
-		}
-		if hasPrefix(line, "intr ") {
-			fmt.Sscanf(line, "intr %f", &stat.IRQPerSec)
-		}
-	}
-	// NOTE: 这里返回的是累计值而非速率；后端可用两次上报的差值计算速率
-	return stat, nil
-}
-
-func splitLines(s string) []string {
-	var result []string
-	start := 0
-	for i := 0; i < len(s); i++ {
-		if s[i] == '\n' {
-			result = append(result, s[start:i])
-			start = i + 1
-		}
-	}
-	if start < len(s) {
-		result = append(result, s[start:])
-	}
-	return result
-}
-
-func hasPrefix(s, prefix string) bool {
-	return len(s) >= len(prefix) && s[:len(prefix)] == prefix
-}