feat: 监控告警体系统一重构 + 自动化任务调度引擎 + 监控大屏拆分
告警中心 - 新增统一告警(alert_unified)、监控告警规则(monitor_alert_rule)、指标目录(alert_metric_catalog) - 通知策略与消息分发(alert_notify_policy/dispatch/message)、静默规则 - 告警引擎按阈值/SNMP 拆分(alert_engine_monitor/snmp) 监控采集 - SNMP 常量/模板与历史指标(snmp templates/constants、snmp_metrics_history、snmp_collect) - 中间件监控 Redis 采集与种子(middleware_monitor_redis/seed) - 站点探测 pkg(siteprobe) + 站点视图(site_view) 自动化任务 - cron 调度器(cronscheduler)、脚本运行器(scriptrunner)、系统任务(systask)及运行记录(cron_job_run、system_task) 审计与凭证 - 审计设置(audit_setting) + 历史清理(auditcleaner) 前端 - 监控页面(host/middleware/site/snmp)拆分为子模块并大屏化 - 告警中心多 Tab(规则/指标目录/通知策略/静默)、任务管理组件重构、Agent CLI Markdown 渲染 - 移除旧脚本卡片/详情/日志组件，统一为 ScriptTable/ScriptCodeEditor/TerminalOutput Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
LoganandClaude committed
1 parent
092d0e4b9e
commit
471353e3d7
190 files changed
+16640
-3484
No files matched your search
@@ -64,6 +64,10 @@ func (e *AlertEngine) Check() {
|
||||
e.checkResourceAlerts()
|
||||
e.checkOfflineAlerts()
|
||||
e.checkPortAlerts()
|
||||
e.checkMiddlewareAlerts()
|
||||
e.checkSNMPAlerts()
|
||||
e.checkSiteAlerts()
|
||||
e.autoResolveSiteSslAlerts()
|
||||
e.checkLogAlertRules()
|
||||
e.autoResolveAlerts()
|
||||
e.checkLogAlertRecovery()
|
||||
@@ -124,8 +128,15 @@ func (e *AlertEngine) GetEngineStatus() map[string]interface{} {
|
||||
|
||||
// 检查 CPU/内存/磁盘/进程/TCP连接/端口 阈值
|
||||
func (e *AlertEngine) checkResourceAlerts() {
|
||||
hostMetrics := []string{
|
||||
"cpu", "cpu_iowait", "mem", "mem_available", "swap", "load1",
|
||||
"disk", "disk_inode", "disk_io_util",
|
||||
"tcp_time_wait", "tcp_close_wait", "tcp_established",
|
||||
"net_rx_bps", "net_tx_bps", "net_rx_errors", "fd",
|
||||
"service_process",
|
||||
}
|
||||
var rules []model.AgentRule
|
||||
e.db.Where("enabled = ? AND metric IN ?", true, []string{"cpu", "mem", "disk", "service_process", "tcp_time_wait", "tcp_close_wait", "fd"}).Find(&rules)
|
||||
e.db.Where("enabled = ? AND metric IN ?", true, hostMetrics).Find(&rules)
|
||||
|
||||
for _, rule := range rules {
|
||||
// 获取目标主机最新上报
|
||||
@@ -140,29 +151,7 @@ func (e *AlertEngine) checkResourceAlerts() {
|
||||
}
|
||||
|
||||
var value float64
|
||||
switch rule.Metric {
|
||||
case "cpu":
|
||||
value = report.CPUUsage
|
||||
case "mem":
|
||||
value = report.MemUsage
|
||||
case "disk":
|
||||
var disks []model.DiskMetric
|
||||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||||
for _, d := range disks {
|
||||
if d.UsagePercent > value {
|
||||
value = d.UsagePercent
|
||||
}
|
||||
}
|
||||
case "tcp_time_wait":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPTimeWait)
|
||||
case "tcp_close_wait":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
value = float64(netStat.TCPCloseWait)
|
||||
case "service_process":
|
||||
// 进程不存在告警 — 检查 process_top_info 中是否包含指定进程名
|
||||
if rule.Metric == "service_process" {
|
||||
var processes []model.ProcessMetric
|
||||
json.Unmarshal([]byte(report.ProcessTopInfo), &processes)
|
||||
found := false
|
||||
@@ -173,13 +162,15 @@ func (e *AlertEngine) checkResourceAlerts() {
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue // 进程存在，不触发
|
||||
continue
|
||||
}
|
||||
value = 0
|
||||
} else {
|
||||
var ok bool
|
||||
value, ok = extractHostMetricValue(report, rule.Metric)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
value = 0 // 进程不存在时触发
|
||||
case "fd":
|
||||
var fdStat model.FDMetric
|
||||
json.Unmarshal([]byte(report.FDInfo), &fdStat)
|
||||
value = fdStat.UsagePct
|
||||
}
|
||||
|
||||
// 检查是否超阈值
|
||||
@@ -305,7 +296,7 @@ func (e *AlertEngine) checkOfflineAlerts() {
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 发送离线告警通知
|
||||
e.notifyAlertRule("主机离线", s.Hostname, "offline", "critical",
|
||||
e.notifyAlertRule("host", "主机离线", s.Hostname, "offline", "critical",
|
||||
alert.Message, offlineMinutes, 3)
|
||||
|
||||
// 记录离线上报时间用于恢复通知
|
||||
@@ -378,7 +369,7 @@ func (e *AlertEngine) checkPortAlerts() {
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 发送端口Down告警通知
|
||||
e.notifyAlertRule(rule.Name, report.Hostname, "port_down", "critical",
|
||||
e.notifyAlertRule("host", rule.Name, report.Hostname, "port_down", "critical",
|
||||
alert.Message, float64(rule.Port), float64(rule.Port))
|
||||
|
||||
now := time.Now()
|
||||
@@ -440,7 +431,7 @@ func (e *AlertEngine) checkLogAlertRules() {
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
e.notifyAlertRule(rule.Name, alert.Target, "log", rule.Severity, alert.Message, float64(count), 0)
|
||||
e.notifyAlertRule("host", rule.Name, alert.Target, "log", rule.Severity, alert.Message, float64(count), 0)
|
||||
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
}
|
||||
@@ -610,7 +601,7 @@ func (e *AlertEngine) autoResolveAlerts() {
|
||||
})
|
||||
|
||||
// 发送恢复通知
|
||||
e.notifyRecovery(alert.RuleName, alert.Hostname, alert.Metric, alert.Message)
|
||||
e.notifyRecovery("host", alert.RuleName, alert.Hostname, alert.Metric, alert.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -648,7 +639,7 @@ func (e *AlertEngine) checkLogAlertRecovery() {
|
||||
"status": model.AlertEventResolved,
|
||||
"resolved_at": now,
|
||||
})
|
||||
e.notifyRecovery(rule.Name, evt.Target, "log", evt.Message)
|
||||
e.notifyRecovery("host", rule.Name, evt.Target, "log", evt.Message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -669,42 +660,20 @@ func (e *AlertEngine) compareOperator(op string, val, threshold float64) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// notifyAlert 发送告警通知（通过 AgentRule 对应的 notify_channels）
|
||||
// notifyAlert 发送主机监控告警通知 // Logan.Li
|
||||
func (e *AlertEngine) notifyAlert(rule *model.AgentRule, hostname string, alert model.AgentAlert) {
|
||||
e.notifyAlertRule(rule.Name, hostname, alert.Metric, alert.Severity,
|
||||
e.notifyAlertRule("host", rule.Name, hostname, alert.Metric, alert.Severity,
|
||||
alert.Message, alert.Value, alert.Threshold)
|
||||
}
|
||||
|
||||
// notifyAlertRule 按规则发送告警通知到所有启用的飞书渠道
|
||||
func (e *AlertEngine) notifyAlertRule(ruleName, hostname, metric, severity, message string, value, threshold float64) {
|
||||
var channels []model.NotifyChannel
|
||||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||||
|
||||
for _, ch := range channels {
|
||||
title := fmt.Sprintf("[%s] %s — %s (%s)", severity, ruleName, hostname, metric)
|
||||
content := fmt.Sprintf(
|
||||
"**规则名称：** %s\\n**目标主机：** %s\\n**指标类型：** %s\\n**告警级别：** %s\\n**当前值：** %.2f\\n**阈值：** %.2f\\n**详细信息：** %s\\n**触发时间：** %s",
|
||||
ruleName, hostname, metric, severity, value, threshold, message, time.Now().Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
if err := sendFeishuAlert(ch.Config, title, content, severity); err != nil {
|
||||
fmt.Printf("[AlertEngine] 飞书通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||||
}
|
||||
}
|
||||
// notifyAlertRule 按告警集合策略发送通知（无策略时回退全局渠道）// Logan.Li
|
||||
func (e *AlertEngine) notifyAlertRule(category, ruleName, hostname, metric, severity, message string, value, threshold float64) {
|
||||
dispatchCategoryAlert(e.db, category, ruleName, hostname, metric, severity, message, value, threshold)
|
||||
}
|
||||
|
||||
// notifyRecovery 发送恢复通知到所有启用的飞书渠道
|
||||
func (e *AlertEngine) notifyRecovery(ruleName, hostname, metric, message string) {
|
||||
var channels []model.NotifyChannel
|
||||
e.db.Where("enabled = ? AND type = ?", true, "feishu").Find(&channels)
|
||||
|
||||
for _, ch := range channels {
|
||||
title := fmt.Sprintf("%s — 已恢复正常", ruleName)
|
||||
content := fmt.Sprintf("**告警已恢复**\\\\n\\\\n主机：%s\\\\n指标：%s\\\\n详情：%s\\\\n恢复时间：%s",
|
||||
hostname, metric, message, time.Now().Format("2006-01-02 15:04:05"))
|
||||
if err := sendFeishuRecovery(ch.Config, title, content); err != nil {
|
||||
fmt.Printf("[AlertEngine] 飞书恢复通知失败 (渠道:%s): %v\n", ch.Name, err)
|
||||
}
|
||||
}
|
||||
// notifyRecovery 按告警集合策略发送恢复通知 // Logan.Li
|
||||
func (e *AlertEngine) notifyRecovery(category, ruleName, hostname, metric, message string) {
|
||||
dispatchCategoryRecovery(e.db, category, ruleName, hostname, metric, message)
|
||||
}
|
||||
|
||||
// isSilenced 检查给定告警是否被静默规则匹配
|
||||
|
||||
Reference in new issue
Block a user