feat: 监控告警体系统一重构 + 自动化任务调度引擎 + 监控大屏拆分
告警中心 - 新增统一告警(alert_unified)、监控告警规则(monitor_alert_rule)、指标目录(alert_metric_catalog) - 通知策略与消息分发(alert_notify_policy/dispatch/message)、静默规则 - 告警引擎按阈值/SNMP 拆分(alert_engine_monitor/snmp) 监控采集 - SNMP 常量/模板与历史指标(snmp templates/constants、snmp_metrics_history、snmp_collect) - 中间件监控 Redis 采集与种子(middleware_monitor_redis/seed) - 站点探测 pkg(siteprobe) + 站点视图(site_view) 自动化任务 - cron 调度器(cronscheduler)、脚本运行器(scriptrunner)、系统任务(systask)及运行记录(cron_job_run、system_task) 审计与凭证 - 审计设置(audit_setting) + 历史清理(auditcleaner) 前端 - 监控页面(host/middleware/site/snmp)拆分为子模块并大屏化 - 告警中心多 Tab(规则/指标目录/通知策略/静默)、任务管理组件重构、Agent CLI Markdown 渲染 - 移除旧脚本卡片/详情/日志组件，统一为 ScriptTable/ScriptCodeEditor/TerminalOutput Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
LoganandClaude committed
1 parent
092d0e4b9e
commit
471353e3d7
190 files changed
+16640
-3484
No files matched your search
@@ -0,0 +1,329 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
"ezops/pkg/siteprobe"
|
||||
)
|
||||
|
||||
// checkMiddlewareAlerts 中间件指标告警 // Logan.Li
|
||||
func (e *AlertEngine) checkMiddlewareAlerts() {
|
||||
var rules []model.MonitorAlertRule
|
||||
e.db.Where("enabled = ? AND category = ?", true, model.MonitorAlertMiddleware).Find(&rules)
|
||||
if len(rules) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var targets []model.MonitorTarget
|
||||
e.db.Where("type = ?", model.TargetTypeMiddleware).Find(&targets)
|
||||
targetMap := make(map[uint]model.MonitorTarget)
|
||||
for _, t := range targets {
|
||||
targetMap[t.ID] = t
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
checkTargets := targets
|
||||
if rule.TargetID > 0 {
|
||||
if t, ok := targetMap[rule.TargetID]; ok {
|
||||
checkTargets = []model.MonitorTarget{t}
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
for _, target := range checkTargets {
|
||||
e.evalMiddlewareRule(rule, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *AlertEngine) evalMiddlewareRule(rule model.MonitorAlertRule, target model.MonitorTarget) {
|
||||
if rule.Metric == "middleware_offline" {
|
||||
if target.Status == model.TargetStatusOffline || target.Status == model.TargetStatusUnknown {
|
||||
e.fireMiddlewareAlert(rule, target, 1, 1, "中间件不可达或采集失败")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
value, ok := e.getLatestMetricValue(target.ID, rule.Metric)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||||
return
|
||||
}
|
||||
msg := fmt.Sprintf("%s %s %.2f %s %.2f", target.Name, rule.Metric, value, rule.Operator, rule.Threshold)
|
||||
e.fireMiddlewareAlert(rule, target, value, rule.Threshold, msg)
|
||||
}
|
||||
|
||||
func (e *AlertEngine) getLatestMetricValue(targetID uint, metric string) (float64, bool) {
|
||||
var hist model.MetricHistory
|
||||
e.db.Where("target_id = ? AND metric_name = ?", targetID, metric).
|
||||
Order("timestamp DESC").First(&hist)
|
||||
if hist.ID == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return hist.Value, true
|
||||
}
|
||||
|
||||
func (e *AlertEngine) fireMiddlewareAlert(rule model.MonitorAlertRule, target model.MonitorTarget, value, threshold float64, msg string) {
|
||||
tenMinAgo := time.Now().Add(-10 * time.Minute)
|
||||
var existing model.AgentAlert
|
||||
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ? AND created_at > ?",
|
||||
rule.Name, target.Name, false, tenMinAgo).First(&existing)
|
||||
if existing.ID > 0 {
|
||||
return
|
||||
}
|
||||
if e.isSilenced(rule.Name, target.Name, rule.ID) {
|
||||
return
|
||||
}
|
||||
sev := rule.Severity
|
||||
if sev == "" {
|
||||
sev = "warning"
|
||||
}
|
||||
alert := model.AgentAlert{
|
||||
RuleName: rule.Name, Hostname: target.Name, Metric: rule.Metric,
|
||||
Severity: sev, Message: msg, Value: value, Threshold: threshold,
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
e.notifyAlertRule("middleware", rule.Name, target.Name, rule.Metric, sev, msg, value, threshold)
|
||||
now := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
}
|
||||
|
||||
// checkSiteAlerts 站点拨测告警 // Logan.Li
|
||||
func (e *AlertEngine) checkSiteAlerts() {
|
||||
var rules []model.MonitorAlertRule
|
||||
e.db.Where("enabled = ? AND category = ?", true, model.MonitorAlertSite).Find(&rules)
|
||||
if len(rules) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var tasks []model.SiteTask
|
||||
e.db.Find(&tasks)
|
||||
taskMap := make(map[uint]model.SiteTask)
|
||||
for _, t := range tasks {
|
||||
taskMap[t.ID] = t
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
checkTasks := tasks
|
||||
if rule.TargetID > 0 {
|
||||
if t, ok := taskMap[rule.TargetID]; ok {
|
||||
checkTasks = []model.SiteTask{t}
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
for _, task := range checkTasks {
|
||||
if task.Status != model.SiteTaskStatusRunning {
|
||||
continue
|
||||
}
|
||||
e.evalSiteRule(rule, task)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *AlertEngine) evalSiteRule(rule model.MonitorAlertRule, task model.SiteTask) {
|
||||
var value float64
|
||||
var triggered bool
|
||||
var msg string
|
||||
|
||||
switch rule.Metric {
|
||||
case "probe_fail":
|
||||
triggered = task.LastResult == "fail" || task.LastResult == "timeout"
|
||||
value = 1
|
||||
if triggered {
|
||||
msg = fmt.Sprintf("拨测任务 %s 失败: %s", task.Name, task.LastResult)
|
||||
}
|
||||
case "probe_timeout":
|
||||
triggered = task.LastResult == "timeout"
|
||||
value = 1
|
||||
if triggered {
|
||||
msg = fmt.Sprintf("拨测任务 %s 超时", task.Name)
|
||||
}
|
||||
case "response_time":
|
||||
value = task.AvgResponse
|
||||
triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
|
||||
if triggered {
|
||||
msg = fmt.Sprintf("拨测 %s 延迟 %.0fms 超过阈值 %.0fms", task.Name, value, rule.Threshold)
|
||||
}
|
||||
case "availability":
|
||||
value = task.SuccessRate
|
||||
triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
|
||||
if triggered {
|
||||
msg = fmt.Sprintf("拨测 %s 可用率 %.1f%% 低于阈值 %.1f%%", task.Name, value, rule.Threshold)
|
||||
}
|
||||
case "http_status_error":
|
||||
triggered = task.LastResult == "fail"
|
||||
value = 1
|
||||
if triggered {
|
||||
msg = fmt.Sprintf("拨测 %s HTTP 状态码异常", task.Name)
|
||||
}
|
||||
case "ssl_expiry_days":
|
||||
if !siteprobe.IsSSLProbeTask(task) {
|
||||
return
|
||||
}
|
||||
var result model.SiteResult
|
||||
e.db.Where("task_id = ? AND ssl_cert_expiry IS NOT NULL", task.ID).
|
||||
Order("checked_at DESC").First(&result)
|
||||
if result.ID == 0 {
|
||||
return
|
||||
}
|
||||
value = float64(result.SSLDaysRemaining)
|
||||
triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
|
||||
if triggered {
|
||||
expiryStr := ""
|
||||
if result.SSLCertExpiry != nil {
|
||||
expiryStr = result.SSLCertExpiry.Format("2006-01-02")
|
||||
}
|
||||
if value < 0 {
|
||||
msg = fmt.Sprintf("SSL 拨测 %s 证书已过期", task.Name)
|
||||
} else if expiryStr != "" {
|
||||
msg = fmt.Sprintf("SSL 拨测 %s 证书剩余 %.0f 天（到期 %s）", task.Name, value, expiryStr)
|
||||
} else {
|
||||
msg = fmt.Sprintf("SSL 拨测 %s 证书剩余 %.0f 天", task.Name, value)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
if !triggered {
|
||||
return
|
||||
}
|
||||
|
||||
tenMinAgo := time.Now().Add(-10 * time.Minute)
|
||||
var existing model.SiteAlert
|
||||
e.db.Where("task_id = ? AND title = ? AND status = ? AND started_at > ?",
|
||||
task.ID, rule.Name, model.SiteAlertStatusActive, tenMinAgo).First(&existing)
|
||||
if existing.ID > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
sev := model.SiteAlertWarning
|
||||
if rule.Severity == "critical" || (rule.Metric == "ssl_expiry_days" && value <= 7) {
|
||||
sev = model.SiteAlertCritical
|
||||
}
|
||||
now := time.Now()
|
||||
siteAlert := model.SiteAlert{
|
||||
TaskID: task.ID, Severity: sev, Status: model.SiteAlertStatusActive,
|
||||
Title: rule.Name, Message: msg, TriggerType: rule.Metric,
|
||||
Threshold: rule.Threshold, ResponseTime: task.AvgResponse,
|
||||
StartedAt: now,
|
||||
}
|
||||
e.db.Create(&siteAlert)
|
||||
e.notifyAlertRule("site", rule.Name, task.Name, rule.Metric, string(sev), msg, value, rule.Threshold)
|
||||
nt := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", nt)
|
||||
}
|
||||
|
||||
// extractHostMetricValue 从 Agent 上报提取扩展指标值 // Logan.Li
|
||||
func extractHostMetricValue(report model.AgentReport, metric string) (float64, bool) {
|
||||
switch metric {
|
||||
case "cpu":
|
||||
return report.CPUUsage, true
|
||||
case "cpu_iowait":
|
||||
return report.CPUIOwait, true
|
||||
case "mem":
|
||||
return report.MemUsage, true
|
||||
case "mem_available":
|
||||
return report.MemAvailable, true
|
||||
case "load1":
|
||||
return report.Load1, true
|
||||
case "disk":
|
||||
var disks []model.DiskMetric
|
||||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||||
max := 0.0
|
||||
for _, d := range disks {
|
||||
if d.UsagePercent > max {
|
||||
max = d.UsagePercent
|
||||
}
|
||||
}
|
||||
return max, true
|
||||
case "disk_inode":
|
||||
var disks []model.DiskMetric
|
||||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||||
max := 0.0
|
||||
for _, d := range disks {
|
||||
if d.InodeUsagePct > max {
|
||||
max = d.InodeUsagePct
|
||||
}
|
||||
}
|
||||
return max, true
|
||||
case "disk_io_util":
|
||||
var dio model.DiskIOStatModel
|
||||
json.Unmarshal([]byte(report.DiskIOInfo), &dio)
|
||||
return dio.Util, true
|
||||
case "swap":
|
||||
var swap model.SwapMetric
|
||||
json.Unmarshal([]byte(report.SwapInfo), &swap)
|
||||
return swap.UsagePercent, true
|
||||
case "tcp_time_wait":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return float64(netStat.TCPTimeWait), true
|
||||
case "tcp_close_wait":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return float64(netStat.TCPCloseWait), true
|
||||
case "tcp_established":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return float64(netStat.TCPEstablished), true
|
||||
case "net_rx_bps":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return netStat.NetRxBPS, true
|
||||
case "net_tx_bps":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return netStat.NetTxBPS, true
|
||||
case "net_rx_errors":
|
||||
var netStat model.NetworkStatModel
|
||||
json.Unmarshal([]byte(report.NetInfo), &netStat)
|
||||
return float64(netStat.NetRxErrors), true
|
||||
case "fd":
|
||||
var fdStat model.FDMetric
|
||||
json.Unmarshal([]byte(report.FDInfo), &fdStat)
|
||||
return fdStat.UsagePct, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
// autoResolveSiteSslAlerts 证书续期后自动恢复 SSL 过期告警 // Logan.Li
|
||||
func (e *AlertEngine) autoResolveSiteSslAlerts() {
|
||||
var alerts []model.SiteAlert
|
||||
e.db.Where("status = ? AND trigger_type = ?", model.SiteAlertStatusActive, "ssl_expiry_days").Find(&alerts)
|
||||
if len(alerts) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for _, alert := range alerts {
|
||||
var result model.SiteResult
|
||||
e.db.Where("task_id = ? AND ssl_cert_expiry IS NOT NULL", alert.TaskID).
|
||||
Order("checked_at DESC").First(&result)
|
||||
if result.ID == 0 {
|
||||
continue
|
||||
}
|
||||
if float64(result.SSLDaysRemaining) >= 30 {
|
||||
e.db.Model(&alert).Updates(map[string]interface{}{
|
||||
"status": model.SiteAlertStatusResolved, "resolved_at": now,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isMiddlewareMetric 判断 AgentAlert 是否属于中间件 // Logan.Li
|
||||
func isMiddlewareMetric(metric string) bool {
|
||||
prefixes := []string{"redis_", "mysql_", "middleware_"}
|
||||
for _, p := range prefixes {
|
||||
if len(metric) >= len(p) && metric[:len(p)] == p {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user