attacker
5333f6f04a
- 后端: NotifyChannel 模型新增 feishu 类型, FeishuWebhookConfig - 后端: TestNotifyChannel 真实发送飞书测试消息(文本格式) - 后端: 告警引擎触发/恢复时自动发送飞书通知(sendFeishuAlert/sendFeishuRecovery) - 后端: AlertEngine 新增 tcp_time_wait/tcp_close_wait/service_process 指标检测 - 后端: NetworkStatModel 新增 TCPCloseWait 字段 - 后端: QuickSetupRules 一键创建6条告警规则(CPU/内存/磁盘/TIME_WAIT/CLOSE_WAIT/进程) - 后端: 注册 /alerts/quick-setup 路由 - 前端: AlertChannelDialog 新增飞书类型选项 - 前端: helpers 新增 getChannelIcon/getChannelTypeLabel 支持 feishu/dingtalk/wechat
417 lines
12 KiB
Go
417 lines
12 KiB
Go
package handlers
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"time"
|
||
|
||
"ezops/internal/models"
|
||
|
||
"gorm.io/gorm"
|
||
)
|
||
|
||
// AlertEngine 告警引擎 — 定时扫描 Agent 上报数据并触发告警 // Logan.Li
|
||
type AlertEngine struct {
|
||
db *gorm.DB
|
||
}
|
||
|
||
func NewAlertEngine(db *gorm.DB) *AlertEngine {
|
||
return &AlertEngine{db: db}
|
||
}
|
||
|
||
// Check 每 30 秒调用一次
|
||
func (e *AlertEngine) Check() {
|
||
e.checkResourceAlerts()
|
||
e.checkOfflineAlerts()
|
||
e.checkPortAlerts()
|
||
e.autoResolveAlerts()
|
||
}
|
||
|
||
// 检查 CPU/内存/磁盘/进程/TCP连接/端口 阈值
|
||
func (e *AlertEngine) checkResourceAlerts() {
|
||
var rules []models.AgentRule
|
||
e.db.Where("enabled = ? AND metric IN ?", true, []string{"cpu", "mem", "disk", "service_process", "tcp_time_wait", "tcp_close_wait"}).Find(&rules)
|
||
|
||
for _, rule := range rules {
|
||
// 获取目标主机最新上报
|
||
var report models.AgentReport
|
||
query := e.db.Model(&models.AgentReport{})
|
||
if rule.Hostname != "" {
|
||
query = query.Where("hostname = ?", rule.Hostname)
|
||
}
|
||
query.Order("id DESC").First(&report)
|
||
if report.ID == 0 {
|
||
continue
|
||
}
|
||
|
||
var value float64
|
||
switch rule.Metric {
|
||
case "cpu":
|
||
value = report.CPUUsage
|
||
case "mem":
|
||
value = report.MemUsage
|
||
case "disk":
|
||
var disks []models.DiskMetric
|
||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||
for _, d := range disks {
|
||
if d.UsagePercent > value {
|
||
value = d.UsagePercent
|
||
}
|
||
}
|
||
case "tcp_time_wait":
|
||
var netStat models.NetworkStatModel
|
||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||
value = float64(netStat.TCPTimeWait)
|
||
case "tcp_close_wait":
|
||
var netStat models.NetworkStatModel
|
||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||
value = float64(netStat.TCPCloseWait)
|
||
case "service_process":
|
||
// 进程不存在告警 — 检查 process_top_info 中是否包含指定进程名
|
||
var processes []models.ProcessMetric
|
||
json.Unmarshal([]byte(report.ProcessTopInfo), &processes)
|
||
found := false
|
||
for _, p := range processes {
|
||
if p.Name == rule.Hostname || rule.Hostname == "" {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if found {
|
||
continue // 进程存在，不触发
|
||
}
|
||
value = 0 // 进程不存在时触发
|
||
}
|
||
|
||
// 检查是否超阈值
|
||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||
continue
|
||
}
|
||
|
||
// 避免重复告警：如果最近 10 分钟已有同规则未恢复告警，不重复
|
||
var existing models.AgentAlert
|
||
tenMinAgo := time.Now().Add(-10 * time.Minute)
|
||
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ? AND created_at > ?",
|
||
rule.Name, report.Hostname, false, tenMinAgo).First(&existing)
|
||
if existing.ID > 0 {
|
||
continue
|
||
}
|
||
|
||
// 创建告警
|
||
alert := models.AgentAlert{
|
||
RuleName: rule.Name,
|
||
Hostname: report.Hostname,
|
||
Metric: rule.Metric,
|
||
Severity: "warning",
|
||
Message: fmt.Sprintf("%s: %s %.1f%% %s %.1f%%", report.Hostname, rule.Metric, value, rule.Operator, rule.Threshold),
|
||
Value: value,
|
||
Threshold: rule.Threshold,
|
||
}
|
||
if value > rule.Threshold*1.5 {
|
||
alert.Severity = "critical"
|
||
}
|
||
e.db.Create(&alert)
|
||
|
||
// 发送告警通知
|
||
e.notifyAlert(&rule, report.Hostname, alert)
|
||
|
||
// 更新规则最后触发时间
|
||
now := time.Now()
|
||
e.db.Model(&rule).Update("last_triggered", now)
|
||
}
|
||
}
|
||
|
||
// 检查主机离线（超过 3 分钟无上报）— 内置规则，无需手动创建。
|
||
// 以 servers 表为中心：遍历所有服务器，检查其 IP 上是否有 Agent 在最近 3 分钟内上报。
|
||
// 如果同一 IP 下有多个 Agent hostname，只要有一个在活跃上报，该服务器就算在线。
|
||
func (e *AlertEngine) checkOfflineAlerts() {
|
||
offlineThreshold := 3 * time.Minute
|
||
|
||
// 获取所有已登记的主机（去重 IP）
|
||
var servers []models.Server
|
||
e.db.Select("ip, hostname").Group("ip").Find(&servers)
|
||
|
||
// 收集所有 agent_reports 中活跃上报的主机名（最近 3 分钟内有上报）
|
||
threeMinAgo := time.Now().Add(-offlineThreshold)
|
||
|
||
var activeHostnames []string
|
||
e.db.Model(&models.AgentReport{}).
|
||
Where("report_time >= ?", threeMinAgo).
|
||
Distinct("hostname").
|
||
Pluck("hostname", &activeHostnames)
|
||
|
||
// 对于每个已登记的服务器，检查是否有活跃 Agent
|
||
for _, s := range servers {
|
||
if s.IP == "" {
|
||
continue
|
||
}
|
||
|
||
// 检查该 IP 对应的服务器是否有活跃 Agent 上报
|
||
hasActiveAgent := false
|
||
for _, h := range activeHostnames {
|
||
// 通过 hostname 精确匹配或 servers 表 IP 匹配
|
||
if h == s.Hostname {
|
||
hasActiveAgent = true
|
||
break
|
||
}
|
||
// 检查该 hostname 的最新上报——其源 IP 是否匹配此 server IP
|
||
// （通过 servers 表中查找 hostname 对应的 IP）
|
||
var matchedServer models.Server
|
||
e.db.Where("hostname = ?", h).First(&matchedServer)
|
||
if matchedServer.ID > 0 && matchedServer.IP == s.IP {
|
||
hasActiveAgent = true
|
||
break
|
||
}
|
||
}
|
||
|
||
if hasActiveAgent {
|
||
continue // 有活跃 Agent，在线
|
||
}
|
||
|
||
// 该服务器离线：检查是否已有未恢复的离线告警
|
||
var existing models.AgentAlert
|
||
e.db.Where("hostname = ? AND metric = ? AND resolved = ?",
|
||
s.Hostname, "offline", false).First(&existing)
|
||
|
||
if existing.ID == 0 {
|
||
// 获取最后一次上报时间计算离线时长
|
||
var lastReport models.AgentReport
|
||
e.db.Where("hostname = ?", s.Hostname).Order("id DESC").First(&lastReport)
|
||
offlineMinutes := 3.0
|
||
var lastReportTime time.Time
|
||
if lastReport.ID > 0 {
|
||
offlineMinutes = time.Since(lastReport.ReportTime).Minutes()
|
||
lastReportTime = lastReport.ReportTime
|
||
}
|
||
|
||
alert := models.AgentAlert{
|
||
RuleName: "主机离线",
|
||
Hostname: s.Hostname,
|
||
Metric: "offline",
|
||
Severity: "critical",
|
||
Message: fmt.Sprintf("主机 %s (%s) 已超过 3 分钟无数据上报，可能已离线", s.Hostname, s.IP),
|
||
Value: offlineMinutes,
|
||
Threshold: 3,
|
||
}
|
||
e.db.Create(&alert)
|
||
|
||
// 发送离线告警通知
|
||
e.notifyAlertRule("主机离线", s.Hostname, "offline", "critical",
|
||
alert.Message, offlineMinutes, 3)
|
||
|
||
// 记录离线上报时间用于恢复通知
|
||
if !lastReportTime.IsZero() {
|
||
e.db.Model(&alert).Update("created_at", time.Now())
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 检查端口消失（之前有的监听端口现在没了）
|
||
func (e *AlertEngine) checkPortAlerts() {
|
||
var rules []models.AgentRule
|
||
e.db.Where("enabled = ? AND metric = ?", true, "port_down").Find(&rules)
|
||
|
||
for _, rule := range rules {
|
||
if rule.Port == 0 {
|
||
continue
|
||
}
|
||
|
||
// 获取目标主机最新上报
|
||
var report models.AgentReport
|
||
query := e.db.Model(&models.AgentReport{})
|
||
if rule.Hostname != "" {
|
||
query = query.Where("hostname = ?", rule.Hostname)
|
||
}
|
||
query.Order("id DESC").First(&report)
|
||
if report.ID == 0 {
|
||
continue
|
||
}
|
||
|
||
// 检查端口是否在最新上报中
|
||
var ports []models.PortMetric
|
||
json.Unmarshal([]byte(report.PortInfo), &ports)
|
||
|
||
portUp := false
|
||
for _, p := range ports {
|
||
if p.Port == rule.Port {
|
||
portUp = true
|
||
break
|
||
}
|
||
}
|
||
|
||
if portUp {
|
||
continue // 端口正常
|
||
}
|
||
|
||
// 端口消失了，检查是否已有未恢复告警
|
||
var existing models.AgentAlert
|
||
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ?",
|
||
rule.Name, report.Hostname, false).First(&existing)
|
||
if existing.ID > 0 {
|
||
continue
|
||
}
|
||
|
||
alert := models.AgentAlert{
|
||
RuleName: rule.Name,
|
||
Hostname: report.Hostname,
|
||
Metric: "port_down",
|
||
Severity: "critical",
|
||
Message: fmt.Sprintf("主机 %s 端口 %d 服务疑似下线", report.Hostname, rule.Port),
|
||
Value: float64(rule.Port),
|
||
Threshold: float64(rule.Port),
|
||
}
|
||
e.db.Create(&alert)
|
||
|
||
// 发送端口Down告警通知
|
||
e.notifyAlertRule(rule.Name, report.Hostname, "port_down", "critical",
|
||
alert.Message, float64(rule.Port), float64(rule.Port))
|
||
|
||
now := time.Now()
|
||
e.db.Model(&rule).Update("last_triggered", now)
|
||
}
|
||
}
|
||
|
||
// 自动恢复告警（指标恢复正常）
|
||
func (e *AlertEngine) autoResolveAlerts() {
|
||
var alerts []models.AgentAlert
|
||
e.db.Where("resolved = ?", false).Find(&alerts)
|
||
|
||
for _, alert := range alerts {
|
||
resolved := false
|
||
|
||
switch alert.Metric {
|
||
case "offline":
|
||
// 主机重新上报了
|
||
var report models.AgentReport
|
||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||
if report.ID > 0 && time.Since(report.ReportTime) < 3*time.Minute {
|
||
resolved = true
|
||
}
|
||
case "cpu", "mem", "disk", "tcp_time_wait", "tcp_close_wait":
|
||
// 获取对应规则，检查是否恢复正常
|
||
var rule models.AgentRule
|
||
e.db.Where("name = ?", alert.RuleName).First(&rule)
|
||
if rule.ID == 0 {
|
||
continue
|
||
}
|
||
var report models.AgentReport
|
||
if rule.Hostname != "" {
|
||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||
} else {
|
||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||
}
|
||
if report.ID == 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
switch alert.Metric {
|
||
case "cpu":
|
||
value = report.CPUUsage
|
||
case "mem":
|
||
value = report.MemUsage
|
||
case "disk":
|
||
var disks []models.DiskMetric
|
||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||
for _, d := range disks {
|
||
if d.UsagePercent > value {
|
||
value = d.UsagePercent
|
||
}
|
||
}
|
||
case "tcp_time_wait":
|
||
var netStat models.NetworkStatModel
|
||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||
value = float64(netStat.TCPTimeWait)
|
||
case "tcp_close_wait":
|
||
var netStat models.NetworkStatModel
|
||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||
value = float64(netStat.TCPCloseWait)
|
||
}
|
||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||
resolved = true
|
||
}
|
||
case "port_down":
|
||
var rule models.AgentRule
|
||
e.db.Where("name = ?", alert.RuleName).First(&rule)
|
||
if rule.ID > 0 {
|
||
var report models.AgentReport
|
||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||
var ports []models.PortMetric
|
||
json.Unmarshal([]byte(report.PortInfo), &ports)
|
||
for _, p := range ports {
|
||
if p.Port == rule.Port {
|
||
resolved = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if resolved {
|
||
now := time.Now()
|
||
e.db.Model(&alert).Updates(map[string]interface{}{
|
||
"resolved": true,
|
||
"resolved_at": now,
|
||
})
|
||
|
||
// 发送恢复通知
|
||
e.notifyRecovery(alert.RuleName, alert.Hostname, alert.Metric, alert.Message)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (e *AlertEngine) compareOperator(op string, val, threshold float64) bool {
|
||
switch op {
|
||
case ">":
|
||
return val > threshold
|
||
case ">=":
|
||
return val >= threshold
|
||
case "<":
|
||
return val < threshold
|
||
case "<=":
|
||
return val <= threshold
|
||
case "==":
|
||
return val == threshold
|
||
default:
|
||
return val > threshold
|
||
}
|
||
}
|
||
|
||
// notifyAlert 发送告警通知（通过 AgentRule 对应的 notify_channels）
|
||
func (e *AlertEngine) notifyAlert(rule *models.AgentRule, hostname string, alert models.AgentAlert) {
|
||
e.notifyAlertRule(rule.Name, hostname, alert.Metric, alert.Severity,
|
||
alert.Message, alert.Value, alert.Threshold)
|
||
}
|
||
|
||
// notifyAlertRule 按规则发送告警通知到所有启用的飞书渠道
|
||
func (e *AlertEngine) notifyAlertRule(ruleName, hostname, metric, severity, message string, value, threshold float64) {
|
||
var channels []models.NotifyChannel
|
||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||
|
||
for _, ch := range channels {
|
||
title := fmt.Sprintf("[%s] %s — %s (%s)", severity, ruleName, hostname, metric)
|
||
content := fmt.Sprintf(
|
||
"**规则名称：** %s\\n**目标主机：** %s\\n**指标类型：** %s\\n**告警级别：** %s\\n**当前值：** %.2f\\n**阈值：** %.2f\\n**详细信息：** %s\\n**触发时间：** %s",
|
||
ruleName, hostname, metric, severity, value, threshold, message, time.Now().Format("2006-01-02 15:04:05"),
|
||
)
|
||
if err := sendFeishuAlert(ch.Config, title, content, severity); err != nil {
|
||
fmt.Printf("[AlertEngine] 飞书通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// notifyRecovery 发送恢复通知到所有启用的飞书渠道
|
||
func (e *AlertEngine) notifyRecovery(ruleName, hostname, metric, message string) {
|
||
var channels []models.NotifyChannel
|
||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||
|
||
for _, ch := range channels {
|
||
title := fmt.Sprintf("%s — 已恢复正常", ruleName)
|
||
content := fmt.Sprintf("**告警已恢复**\\n\\n主机：%s\\n指标：%s\\n详情：%s\\n恢复时间：%s",
|
||
hostname, metric, message, time.Now().Format("2006-01-02 15:04:05"))
|
||
if err := sendFeishuRecovery(ch.Config, title, content); err != nil {
|
||
fmt.Printf("[AlertEngine] 飞书恢复通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||
}
|
||
}
|
||
}
|