feat(alert-center): 添加飞书Webhook通知渠道 + 告警引擎实时通知
- 后端: NotifyChannel 模型新增 feishu 类型, FeishuWebhookConfig - 后端: TestNotifyChannel 真实发送飞书测试消息(文本格式) - 后端: 告警引擎触发/恢复时自动发送飞书通知(sendFeishuAlert/sendFeishuRecovery) - 后端: AlertEngine 新增 tcp_time_wait/tcp_close_wait/service_process 指标检测 - 后端: NetworkStatModel 新增 TCPCloseWait 字段 - 后端: QuickSetupRules 一键创建6条告警规则(CPU/内存/磁盘/TIME_WAIT/CLOSE_WAIT/进程) - 后端: 注册 /alerts/quick-setup 路由 - 前端: AlertChannelDialog 新增飞书类型选项 - 前端: helpers 新增 getChannelIcon/getChannelTypeLabel 支持 feishu/dingtalk/wechat
This commit is contained in:
attacker committed
1 parent
7bc5169e8f
commit
5333f6f04a
30 files changed
+2406
-1001
No files matched your search
@@ -27,10 +27,10 @@ func (e *AlertEngine) Check() {
|
||||
e.autoResolveAlerts()
|
||||
}
|
||||
|
||||
// 检查 CPU/内存/磁盘 阈值
|
||||
// 检查 CPU/内存/磁盘/进程/TCP连接/端口 阈值
|
||||
func (e *AlertEngine) checkResourceAlerts() {
|
||||
var rules []models.AgentRule
|
||||
e.db.Where("enabled = ? AND metric IN ?", true, []string{"cpu", "mem", "disk"}).Find(&rules)
|
||||
e.db.Where("enabled = ? AND metric IN ?", true, []string{"cpu", "mem", "disk", "service_process", "tcp_time_wait", "tcp_close_wait"}).Find(&rules)
|
||||
|
||||
for _, rule := range rules {
|
||||
// 获取目标主机最新上报
|
||||
@@ -58,6 +58,29 @@ func (e *AlertEngine) checkResourceAlerts() {
|
||||
value = d.UsagePercent
|
||||
}
|
||||
}
|
||||
case "tcp_time_wait":
|
||||
var netStat models.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPTimeWait)
|
||||
case "tcp_close_wait":
|
||||
var netStat models.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPCloseWait)
|
||||
case "service_process":
|
||||
// 进程不存在告警 — 检查 process_top_info 中是否包含指定进程名
|
||||
var processes []models.ProcessMetric
|
||||
json.Unmarshal([]byte(report.ProcessTopInfo), &processes)
|
||||
found := false
|
||||
for _, p := range processes {
|
||||
if p.Name == rule.Hostname || rule.Hostname == "" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue // 进程存在，不触发
|
||||
}
|
||||
value = 0 // 进程不存在时触发
|
||||
}
|
||||
|
||||
// 检查是否超阈值
|
||||
@@ -89,6 +112,9 @@ func (e *AlertEngine) checkResourceAlerts() {
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 发送告警通知
|
||||
e.notifyAlert(&rule, report.Hostname, alert)
|
||||
|
||||
// 更新规则最后触发时间
|
||||
now := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
@@ -150,10 +176,12 @@ func (e *AlertEngine) checkOfflineAlerts() {
|
||||
if existing.ID == 0 {
|
||||
// 获取最后一次上报时间计算离线时长
|
||||
var lastReport models.AgentReport
|
||||
e.db.Order("id DESC").First(&lastReport)
|
||||
e.db.Where("hostname = ?", s.Hostname).Order("id DESC").First(&lastReport)
|
||||
offlineMinutes := 3.0
|
||||
var lastReportTime time.Time
|
||||
if lastReport.ID > 0 {
|
||||
offlineMinutes = time.Since(lastReport.ReportTime).Minutes()
|
||||
lastReportTime = lastReport.ReportTime
|
||||
}
|
||||
|
||||
alert := models.AgentAlert{
|
||||
@@ -166,6 +194,15 @@ func (e *AlertEngine) checkOfflineAlerts() {
|
||||
Threshold: 3,
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 发送离线告警通知
|
||||
e.notifyAlertRule("主机离线", s.Hostname, "offline", "critical",
|
||||
alert.Message, offlineMinutes, 3)
|
||||
|
||||
// 记录离线上报时间用于恢复通知
|
||||
if !lastReportTime.IsZero() {
|
||||
e.db.Model(&alert).Update("created_at", time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -226,6 +263,10 @@ func (e *AlertEngine) checkPortAlerts() {
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 发送端口Down告警通知
|
||||
e.notifyAlertRule(rule.Name, report.Hostname, "port_down", "critical",
|
||||
alert.Message, float64(rule.Port), float64(rule.Port))
|
||||
|
||||
now := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
}
|
||||
@@ -247,7 +288,7 @@ func (e *AlertEngine) autoResolveAlerts() {
|
||||
if report.ID > 0 && time.Since(report.ReportTime) < 3*time.Minute {
|
||||
resolved = true
|
||||
}
|
||||
case "cpu", "mem", "disk":
|
||||
case "cpu", "mem", "disk", "tcp_time_wait", "tcp_close_wait":
|
||||
// 获取对应规则，检查是否恢复正常
|
||||
var rule models.AgentRule
|
||||
e.db.Where("name = ?", alert.RuleName).First(&rule)
|
||||
@@ -277,6 +318,14 @@ func (e *AlertEngine) autoResolveAlerts() {
|
||||
value = d.UsagePercent
|
||||
}
|
||||
}
|
||||
case "tcp_time_wait":
|
||||
var netStat models.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPTimeWait)
|
||||
case "tcp_close_wait":
|
||||
var netStat models.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPCloseWait)
|
||||
}
|
||||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||||
resolved = true
|
||||
@@ -304,6 +353,9 @@ func (e *AlertEngine) autoResolveAlerts() {
|
||||
"resolved": true,
|
||||
"resolved_at": now,
|
||||
})
|
||||
|
||||
// 发送恢复通知
|
||||
e.notifyRecovery(alert.RuleName, alert.Hostname, alert.Metric, alert.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -324,3 +376,41 @@ func (e *AlertEngine) compareOperator(op string, val, threshold float64) bool {
|
||||
return val > threshold
|
||||
}
|
||||
}
|
||||
|
||||
// notifyAlert 发送告警通知（通过 AgentRule 对应的 notify_channels）
|
||||
func (e *AlertEngine) notifyAlert(rule *models.AgentRule, hostname string, alert models.AgentAlert) {
|
||||
e.notifyAlertRule(rule.Name, hostname, alert.Metric, alert.Severity,
|
||||
alert.Message, alert.Value, alert.Threshold)
|
||||
}
|
||||
|
||||
// notifyAlertRule 按规则发送告警通知到所有启用的飞书渠道
|
||||
func (e *AlertEngine) notifyAlertRule(ruleName, hostname, metric, severity, message string, value, threshold float64) {
|
||||
var channels []models.NotifyChannel
|
||||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||||
|
||||
for _, ch := range channels {
|
||||
title := fmt.Sprintf("[%s] %s — %s (%s)", severity, ruleName, hostname, metric)
|
||||
content := fmt.Sprintf(
|
||||
"**规则名称：** %s\\n**目标主机：** %s\\n**指标类型：** %s\\n**告警级别：** %s\\n**当前值：** %.2f\\n**阈值：** %.2f\\n**详细信息：** %s\\n**触发时间：** %s",
|
||||
ruleName, hostname, metric, severity, value, threshold, message, time.Now().Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
if err := sendFeishuAlert(ch.Config, title, content, severity); err != nil {
|
||||
fmt.Printf("[AlertEngine] 飞书通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// notifyRecovery 发送恢复通知到所有启用的飞书渠道
|
||||
func (e *AlertEngine) notifyRecovery(ruleName, hostname, metric, message string) {
|
||||
var channels []models.NotifyChannel
|
||||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||||
|
||||
for _, ch := range channels {
|
||||
title := fmt.Sprintf("%s — 已恢复正常", ruleName)
|
||||
content := fmt.Sprintf("**告警已恢复**\\n\\n主机：%s\\n指标：%s\\n详情：%s\\n恢复时间：%s",
|
||||
hostname, metric, message, time.Now().Format("2006-01-02 15:04:05"))
|
||||
if err := sendFeishuRecovery(ch.Config, title, content); err != nil {
|
||||
fmt.Printf("[AlertEngine] 飞书恢复通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user