feat: v0.34 健康探针(Architecture B) + Agent v2.1 + 端口监控 + 告警联系人重构

- v0.31: AlertContact 重构(channel_type下拉+webhook_url替代三分立字段)
- v0.32: Agent ProcessTop 增强(cmdline/user/cpu_pct/mem_pct)
- v0.33: 端口监控 Tab(监听端口自动发现+down联动告警)
- v0.34: 健康探针(Agent本地执行HTTP/TCP/Exec→结果随心跳上报)
- Agent新增: ContextSwitch/FD/HealthProbe采集器
- 后端新增: health_probe handler + Agent model扩展
- HostMonitor新增Tab: 健康探针/端口监控
This commit is contained in:
attacker committed 2026-06-24 19:18:10 +08:00
1 parent e1f75f0bc4
commit d656520428
25 files changed
+1700 -619

No files matched your search

+39
View File
@@ -0,0 +1,39 @@
// Package collector 的上下文切换采集器。
// 读取 /proc/stat 中的 ctxt(上下文切换)和 intr(中断)累计值。
// 注意:此处返回累计值而非速率,后端可用两次上报差值计算。
//
// 署名 Logan.Li
package collector
import (
"fmt"
"os"
"ezops/agent/model"
)
// ContextSwitchCollector 上下文切换采集器
type ContextSwitchCollector struct{}
// NewContextSwitchCollector 创建采集器实例
func NewContextSwitchCollector() *ContextSwitchCollector {
return &ContextSwitchCollector{}
}
// Collect 读取 /proc/stat 中的上下文切换和中断统计
func (c *ContextSwitchCollector) Collect() (*model.ContextSwitchStat, error) {
data, err := os.ReadFile("/proc/stat")
if err != nil {
return nil, fmt.Errorf("读取 /proc/stat 失败: %w", err)
}
stat := &model.ContextSwitchStat{}
for _, line := range SplitLines(string(data)) {
if HasPrefix(line, "ctxt ") {
fmt.Sscanf(line, "ctxt %f", &stat.CTXPerSec)
}
if HasPrefix(line, "intr ") {
fmt.Sscanf(line, "intr %f", &stat.IRQPerSec)
}
}
return stat, nil
}
+49
View File
@@ -0,0 +1,49 @@
// Package collector 的 FD 句柄采集器。
// 读取 /proc/sys/fs/file-nr 和 /proc/sys/fs/file-max 获取系统级文件句柄使用情况。
//
// 署名 Logan.Li
package collector
import (
"fmt"
"os"
"ezops/agent/model"
)
// FDCollector 系统文件句柄采集器
type FDCollector struct{}
// NewFDCollector 创建采集器实例
func NewFDCollector() *FDCollector {
return &FDCollector{}
}
// Collect 读取 /proc/sys/fs/file-nr 和 file-max
func (c *FDCollector) Collect() (*model.SocketFDStat, error) {
alloc, err := os.ReadFile("/proc/sys/fs/file-nr")
if err != nil {
return nil, fmt.Errorf("读取 file-nr 失败: %w", err)
}
var allocated, _unused1, _unused2 int
fmt.Sscanf(string(alloc), "%d %d %d", &allocated, &_unused1, &_unused2)
max, err := os.ReadFile("/proc/sys/fs/file-max")
if err != nil {
return nil, fmt.Errorf("读取 file-max 失败: %w", err)
}
var maxFD int64
fmt.Sscanf(string(max), "%d", &maxFD)
stat := &model.SocketFDStat{
Allocated: allocated,
Max: maxFD,
}
if maxFD > 100_000_000 {
stat.Max = -1
stat.UsagePct = 0
} else if maxFD > 0 {
stat.UsagePct = float64(allocated) / float64(maxFD) * 100.0
}
return stat, nil
}
+216
View File
@@ -0,0 +1,216 @@
// Package collector 的健康探针采集器。
// 在后端下发的探针配置驱动下,本地执行 HTTP GET / TCP 连接 / 命令执行,
// 并将结果返回供上报。
//
// 署名 Logan.Li
package collector
import (
"context"
"fmt"
"io"
"net"
"net/http"
"os/exec"
"strings"
"time"
"ezops/agent/model"
)
// HealthProbeCollector 健康探针执行器
type HealthProbeCollector struct {
httpClient *http.Client
}
// NewHealthProbeCollector 创建采集器实例
func NewHealthProbeCollector() *HealthProbeCollector {
return &HealthProbeCollector{
httpClient: &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
},
},
}
}
// Execute 执行所有探针配置,返回结果列表。
// parentCtx 用于传播取消信号——探针执行尽量尊重主循环的生命周期。
func (h *HealthProbeCollector) Execute(parentCtx context.Context, configs []model.ProbeConfig) []model.ProbeResult {
results := make([]model.ProbeResult, 0, len(configs))
for _, cfg := range configs {
// 检查父上下文是否已取消
if err := parentCtx.Err(); err != nil {
break
}
result := model.ProbeResult{
ProbeID: cfg.ID,
Name: cfg.Name,
ProbeType: cfg.ProbeType,
}
start := time.Now()
switch cfg.ProbeType {
case "http_get":
result = h.execHTTP(parentCtx, cfg, start)
case "tcp_socket":
result = h.execTCP(cfg, start)
case "exec_command":
result = h.execCmd(parentCtx, cfg, start)
default:
result.Status = "down"
result.Message = fmt.Sprintf("unknown probe type: %s", cfg.ProbeType)
}
results = append(results, result)
}
return results
}
// execHTTP 执行 HTTP 探测,接收父 context
func (h *HealthProbeCollector) execHTTP(parentCtx context.Context, cfg model.ProbeConfig, start time.Time) model.ProbeResult {
result := model.ProbeResult{
ProbeID: cfg.ID,
Name: cfg.Name,
ProbeType: cfg.ProbeType,
}
timeout := time.Duration(cfg.Timeout) * time.Second
if timeout <= 0 {
timeout = 10 * time.Second
}
ctx, cancel := context.WithTimeout(parentCtx, timeout)
defer cancel()
method := cfg.HTTPMethod
if method == "" {
method = "GET"
}
req, err := http.NewRequestWithContext(ctx, method, cfg.HTTPURL, nil)
if err != nil {
result.Status = "down"
result.Message = fmt.Sprintf("构造请求失败: %v", err)
result.LatencyMs = int(time.Since(start).Milliseconds())
return result
}
req.Header.Set("User-Agent", "aiops-agent-health-probe/2.1")
resp, err := h.httpClient.Do(req)
latencyMs := int(time.Since(start).Milliseconds())
result.LatencyMs = latencyMs
if err != nil {
result.Status = "down"
result.Message = fmt.Sprintf("HTTP请求失败: %v", err)
return result
}
defer resp.Body.Close()
expectCode := cfg.HTTPExpectCode
if expectCode == 0 {
expectCode = 200
}
result.StatusCode = resp.StatusCode
if resp.StatusCode != expectCode {
result.Status = "down"
result.Message = fmt.Sprintf("状态码不匹配: got %d, want %d", resp.StatusCode, expectCode)
return result
}
// 可选:检查响应体
if cfg.HTTPExpectBody != "" {
bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
if err != nil {
result.Status = "down"
result.Message = fmt.Sprintf("读取响应体失败: %v", err)
return result
}
if !strings.Contains(string(bodyBytes), cfg.HTTPExpectBody) {
result.Status = "down"
result.Message = fmt.Sprintf("响应体不包含期望字符串: %q", cfg.HTTPExpectBody)
return result
}
}
result.Status = "up"
result.Message = fmt.Sprintf("HTTP %d OK (%dms)", resp.StatusCode, latencyMs)
return result
}
// execTCP 执行 TCP 连接探测
func (h *HealthProbeCollector) execTCP(cfg model.ProbeConfig, start time.Time) model.ProbeResult {
result := model.ProbeResult{
ProbeID: cfg.ID,
Name: cfg.Name,
ProbeType: cfg.ProbeType,
}
timeout := time.Duration(cfg.Timeout) * time.Second
if timeout <= 0 {
timeout = 10 * time.Second
}
addr := fmt.Sprintf("%s:%d", cfg.TCPHost, cfg.TCPPort)
conn, err := net.DialTimeout("tcp", addr, timeout)
latencyMs := int(time.Since(start).Milliseconds())
result.LatencyMs = latencyMs
if err != nil {
result.Status = "down"
result.Message = fmt.Sprintf("TCP连接失败 %s: %v", addr, err)
return result
}
conn.Close()
result.Status = "up"
result.Message = fmt.Sprintf("TCP %s connected (%dms)", addr, latencyMs)
return result
}
// execCmd 执行命令探测(exit code 0 = up),接收父 context
func (h *HealthProbeCollector) execCmd(parentCtx context.Context, cfg model.ProbeConfig, start time.Time) model.ProbeResult {
result := model.ProbeResult{
ProbeID: cfg.ID,
Name: cfg.Name,
ProbeType: cfg.ProbeType,
}
timeout := time.Duration(cfg.Timeout) * time.Second
if timeout <= 0 {
timeout = 10 * time.Second
}
ctx, cancel := context.WithTimeout(parentCtx, timeout)
defer cancel()
cmd := exec.CommandContext(ctx, "sh", "-c", cfg.ExecCommand)
output, err := cmd.CombinedOutput()
latencyMs := int(time.Since(start).Milliseconds())
result.LatencyMs = latencyMs
result.StatusCode = cmd.ProcessState.ExitCode()
if err != nil {
result.Status = "down"
// 截断输出避免过大
outMsg := strings.TrimSpace(string(output))
if len(outMsg) > 200 {
outMsg = outMsg[:200] + "..."
}
result.Message = fmt.Sprintf("命令执行失败(exit=%d): %s", cmd.ProcessState.ExitCode(), outMsg)
return result
}
result.Status = "up"
result.Message = fmt.Sprintf("命令执行成功 (%dms)", latencyMs)
return result
}
+230 -67
View File
@@ -1,6 +1,6 @@
// Package collector — 进程 TOP3 采集器
// 扫描 /proc/*/stat + /proc/*/status 获取 CPU/内存/句柄 TOP3 进程
// 1000 服务器规模下的优化:只取 TOP3,不遍历全部进程的 cmdline
// Package collector — 进程 TOP 采集器
// 扫描 /proc 获取 TOP N 进程完整信息:PID/User/CMD/CPU%/MEM%/FDs
// 阿里云 ECS 监控风格 — 直观 6 列布局
//
// 署名 Logan.Li
package collector
@@ -8,6 +8,7 @@ package collector
import (
"bufio"
"os"
"os/user"
"path/filepath"
"sort"
"strconv"
@@ -16,33 +17,166 @@ import (
"ezops/agent/model"
)
const clktck = 100.0 // Linux CLK_TCK = 100 Hz
// ProcessTopCollector 进程 TOP 采集器
type ProcessTopCollector struct{}
type ProcessTopCollector struct {
memTotalKB uint64 // 系统总内存(KB),初始化时读取
numCPU int // CPU 核心数
}
// NewProcessTopCollector 创建进程 TOP 采集器
func NewProcessTopCollector() *ProcessTopCollector {
return &ProcessTopCollector{}
pt := &ProcessTopCollector{}
pt.memTotalKB = readMemTotalKB()
pt.numCPU = readNumCPU()
if pt.numCPU < 1 {
pt.numCPU = 1
}
return pt
}
// procSample holds key metrics for a single process
type procSample struct {
PID int
Name string
CPU float64 // CPU 使用率(需两次采样 + 系统总 jiffies 才能算,这里用 utime+stime ticks 近似)
RSSKB uint64 // 物理内存 (KB)
FDCount int // 打开的文件句柄数
Threads int // 线程数
// readMemTotalKB reads MemTotal from /proc/meminfo
func readMemTotalKB() uint64 {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 1
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
if strings.HasPrefix(sc.Text(), "MemTotal:") {
f := strings.Fields(sc.Text())
if len(f) >= 2 {
v, _ := strconv.ParseUint(f[1], 10, 64)
return v
}
}
}
return 1
}
// Collect gathers process metrics and returns top N sorted by RSS.
// Sorting by RSS is the most stable and reliable metric without multi-sampling CPU.
// readNumCPU reads CPU count from /proc/stat (cpuN lines)
func readNumCPU() int {
f, err := os.Open("/proc/stat")
if err != nil {
return 1
}
defer f.Close()
n := 0
sc := bufio.NewScanner(f)
for sc.Scan() {
if strings.HasPrefix(sc.Text(), "cpu") && len(sc.Text()) > 3 && sc.Text()[3] >= '0' && sc.Text()[3] <= '9' {
n++
}
}
return n
}
// readProcCPU reads utime+stime from /proc/<pid>/stat
func readProcCPUJiffies(pid string) uint64 {
data, err := os.ReadFile(filepath.Join("/proc", pid, "stat"))
if err != nil {
return 0
}
// stat: pid (comm) state ppid ... utime stime ...
// We need fields 14 (utime) and 15 (stime) in 1-based indexing
// Parse: skip past the closing ')' of comm
s := string(data)
idx := strings.LastIndex(s, ")")
if idx < 0 {
return 0
}
fields := strings.Fields(s[idx+2:])
if len(fields) < 13 {
return 0
}
// After ')', fields are: state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt cmajflt utime stime
// So utime = fields[11], stime = fields[12]
utime, _ := strconv.ParseUint(fields[11], 10, 64)
stime, _ := strconv.ParseUint(fields[12], 10, 64)
return utime + stime
}
// readProcUser reads the owner username from /proc/<pid>/status (Uid line)
func readProcUser(pid string) string {
statusFile := filepath.Join("/proc", pid, "status")
f, err := os.Open(statusFile)
if err != nil {
return "?"
}
defer f.Close()
sc := bufio.NewScanner(f)
var uidStr string
for sc.Scan() {
line := sc.Text()
if strings.HasPrefix(line, "Uid:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
uidStr = fields[1]
}
break
}
}
if uidStr == "" || uidStr == "0" {
return "root"
}
// Try to look up username
u, err := user.LookupId(uidStr)
if err == nil {
return u.Username
}
return uidStr
}
// readProcCmdline reads /proc/<pid>/cmdline, replaces null bytes with spaces
func readProcCmdline(pid string) string {
data, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline"))
if err != nil || len(data) == 0 {
// Fallback to comm
data2, err2 := os.ReadFile(filepath.Join("/proc", pid, "comm"))
if err2 == nil {
return strings.TrimSpace(string(data2))
}
return "?"
}
// Replace null bytes with spaces
out := strings.Builder{}
for _, b := range data {
if b == 0 {
out.WriteByte(' ')
} else {
out.WriteByte(b)
}
}
return strings.TrimSpace(out.String())
}
// Collect gathers process metrics and returns top N sorted by RSS
func (p *ProcessTopCollector) Collect(topN int) ([]model.ProcessStat, error) {
uptimeJiffies := readUptimeJiffies()
if uptimeJiffies < 1 {
uptimeJiffies = 1
}
entries, err := os.ReadDir("/proc")
if err != nil {
return nil, err
}
var samples []procSample
type sample struct {
pid int
name string
cmdline string
user string
cpuPct float64
memPct float64
rssKB uint64
fdCount int
threads int
}
var samples []sample
for _, entry := range entries {
if !entry.IsDir() {
continue
@@ -51,32 +185,56 @@ func (p *ProcessTopCollector) Collect(topN int) ([]model.ProcessStat, error) {
if err != nil {
continue
}
pidStr := entry.Name()
name := readProcName(entry.Name())
// Skip kernel threads (empty cmdline + running in kernel)
if name == "" || strings.HasPrefix(name, "[") {
// Read comm to filter kernel threads
commData, err := os.ReadFile(filepath.Join("/proc", pidStr, "comm"))
if err != nil {
continue
}
name := strings.TrimSpace(string(commData))
if name == "" || strings.HasPrefix(name, "[") {
continue // skip kernel threads
}
rssKB := readProcRSS(entry.Name())
fdCount := countFD(entry.Name())
threads := readProcThreads(entry.Name())
// Gather all metrics
jiffies := readProcCPUJiffies(pidStr)
rssKB := readProcRSS(pidStr)
fdCount := countFD(pidStr)
threads := readProcThreads(pidStr)
userName := readProcUser(pidStr)
cmdline := readProcCmdline(pidStr)
samples = append(samples, procSample{
PID: pid,
Name: name,
RSSKB: rssKB,
FDCount: fdCount,
Threads: threads,
// CPU% = (utime+stime) / uptime_jiffies * 100 / num_cpus
cpuPct := float64(jiffies) / float64(uptimeJiffies) * 100.0 / float64(p.numCPU)
if cpuPct > 100*float64(p.numCPU) {
cpuPct = 0 // invalid
}
// Memory%
var memPct float64
if p.memTotalKB > 0 {
memPct = float64(rssKB) / float64(p.memTotalKB) * 100.0
}
samples = append(samples, sample{
pid: pid,
name: name,
cmdline: cmdline,
user: userName,
cpuPct: cpuPct,
memPct: memPct,
rssKB: rssKB,
fdCount: fdCount,
threads: threads,
})
}
// Sort by RSS descending
sort.Slice(samples, func(i, j int) bool {
return samples[i].RSSKB > samples[j].RSSKB
return samples[i].rssKB > samples[j].rssKB
})
// Take top N
if topN > len(samples) {
topN = len(samples)
}
@@ -84,23 +242,41 @@ func (p *ProcessTopCollector) Collect(topN int) ([]model.ProcessStat, error) {
for i := 0; i < topN; i++ {
s := samples[i]
result[i] = model.ProcessStat{
PID: s.PID,
Name: s.Name,
MemoryMB: float64(s.RSSKB) / 1024.0,
FDCount: s.FDCount,
Threads: s.Threads,
PID: s.pid,
Name: s.name,
Cmdline: s.cmdline,
User: s.user,
CPUPct: s.cpuPct,
MemPct: s.memPct,
MemoryMB: float64(s.rssKB) / 1024.0,
FDCount: s.fdCount,
Threads: s.threads,
}
}
return result, nil
}
// readProcName reads /proc/<pid>/comm
func readProcName(pid string) string {
data, err := os.ReadFile(filepath.Join("/proc", pid, "comm"))
// readUptimeJiffies reads /proc/uptime and converts to jiffies
func readUptimeJiffies() uint64 {
data, err := os.ReadFile("/proc/uptime")
if err != nil {
return ""
return 1
}
return strings.TrimSpace(string(data))
f := strings.Fields(string(data))
if len(f) < 1 {
return 1
}
sec, _ := strconv.ParseFloat(f[0], 64)
return uint64(sec * clktck)
}
// countFD counts open file descriptors for a process
func countFD(pid string) int {
entries, err := os.ReadDir(filepath.Join("/proc", pid, "fd"))
if err != nil {
return 0
}
return len(entries)
}
// readProcRSS reads VmRSS from /proc/<pid>/status
@@ -110,15 +286,13 @@ func readProcRSS(pid string) uint64 {
return 0
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "VmRSS:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
val, _ := strconv.ParseUint(fields[1], 10, 64)
return val
sc := bufio.NewScanner(f)
for sc.Scan() {
if strings.HasPrefix(sc.Text(), "VmRSS:") {
f := strings.Fields(sc.Text())
if len(f) >= 2 {
v, _ := strconv.ParseUint(f[1], 10, 64)
return v
}
}
}
@@ -132,26 +306,15 @@ func readProcThreads(pid string) int {
return 0
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Threads:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
val, _ := strconv.Atoi(fields[1])
return val
sc := bufio.NewScanner(f)
for sc.Scan() {
if strings.HasPrefix(sc.Text(), "Threads:") {
f := strings.Fields(sc.Text())
if len(f) >= 2 {
v, _ := strconv.Atoi(f[1])
return v
}
}
}
return 0
}
// countFD counts open file descriptors for a process
func countFD(pid string) int {
entries, err := os.ReadDir(filepath.Join("/proc", pid, "fd"))
if err != nil {
return 0
}
return len(entries)
}
+25
View File
@@ -0,0 +1,25 @@
// Package collector 的通用工具函数。
//
// 署名 Logan.Li
package collector
// SplitLines 按换行符切割字符串
func SplitLines(s string) []string {
var result []string
start := 0
for i := 0; i < len(s); i++ {
if s[i] == '\n' {
result = append(result, s[start:i])
start = i + 1
}
}
if start < len(s) {
result = append(result, s[start:])
}
return result
}
// HasPrefix 检查字符串前缀
func HasPrefix(s, prefix string) bool {
return len(s) >= len(prefix) && s[:len(prefix)] == prefix
}