告警中心 - 新增统一告警(alert_unified)、监控告警规则(monitor_alert_rule)、指标目录(alert_metric_catalog) - 通知策略与消息分发(alert_notify_policy/dispatch/message)、静默规则 - 告警引擎按阈值/SNMP 拆分(alert_engine_monitor/snmp) 监控采集 - SNMP 常量/模板与历史指标(snmp templates/constants、snmp_metrics_history、snmp_collect) - 中间件监控 Redis 采集与种子(middleware_monitor_redis/seed) - 站点探测 pkg(siteprobe) + 站点视图(site_view) 自动化任务 - cron 调度器(cronscheduler)、脚本运行器(scriptrunner)、系统任务(systask)及运行记录(cron_job_run、system_task) 审计与凭证 - 审计设置(audit_setting) + 历史清理(auditcleaner) 前端 - 监控页面(host/middleware/site/snmp)拆分为子模块并大屏化 - 告警中心多 Tab(规则/指标目录/通知策略/静默)、任务管理组件重构、Agent CLI Markdown 渲染 - 移除旧脚本卡片/详情/日志组件,统一为 ScriptTable/ScriptCodeEditor/TerminalOutput Co-Authored-By: Claude <noreply@anthropic.com>
184 lines
5.5 KiB
Go
184 lines
5.5 KiB
Go
package controller
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"ezops/model"
|
|
)
|
|
|
|
// SNMP 指标名映射(与采集模板一致)// Logan.Li
|
|
var snmpCPUMetricNames = []string{
|
|
"h3c_cpu_usage", "hw_cpu_usage", "cisco_cpu_5min", "ruijie_cpu_usage", "inspur_cpu_usage",
|
|
}
|
|
var snmpMemMetricNames = []string{
|
|
"h3c_mem_usage", "hw_mem_usage", "ruijie_mem_usage", "inspur_mem_usage",
|
|
}
|
|
var snmpTempMetricNames = []string{
|
|
"h3c_temperature", "hw_temperature", "ruijie_temperature", "dell_temperature",
|
|
"inspur_temperature", "cisco_sensor_value",
|
|
}
|
|
|
|
// checkSNMPAlerts SNMP 设备指标告警 // Logan.Li
|
|
func (e *AlertEngine) checkSNMPAlerts() {
|
|
var rules []model.MonitorAlertRule
|
|
e.db.Where("enabled = ? AND category = ?", true, model.MonitorAlertSNMP).Find(&rules)
|
|
if len(rules) == 0 {
|
|
return
|
|
}
|
|
|
|
var devices []model.SNMPDevice
|
|
e.db.Where("template_id > 0").Find(&devices)
|
|
deviceMap := make(map[uint]model.SNMPDevice, len(devices))
|
|
for _, d := range devices {
|
|
deviceMap[d.ID] = d
|
|
}
|
|
|
|
for _, rule := range rules {
|
|
checkDevices := devices
|
|
if rule.TargetID > 0 {
|
|
if d, ok := deviceMap[rule.TargetID]; ok {
|
|
checkDevices = []model.SNMPDevice{d}
|
|
} else {
|
|
continue
|
|
}
|
|
}
|
|
for _, device := range checkDevices {
|
|
e.evalSNMPRule(rule, device)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *AlertEngine) evalSNMPRule(rule model.MonitorAlertRule, device model.SNMPDevice) {
|
|
var value float64
|
|
var triggered bool
|
|
var msg string
|
|
|
|
switch rule.Metric {
|
|
case "snmp_offline":
|
|
triggered = device.Status != model.SNMPDeviceOnline
|
|
value = 1
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP 设备 %s (%s) 离线或采集异常", device.Name, device.IP)
|
|
}
|
|
case "snmp_cpu_usage":
|
|
value, triggered = e.evalSNMPGaugeRule(device.ID, snmpCPUMetricNames, rule)
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP %s CPU %.1f%% %s %.1f%%", device.Name, value, rule.Operator, rule.Threshold)
|
|
}
|
|
case "snmp_mem_usage":
|
|
value, triggered = e.evalSNMPGaugeRule(device.ID, snmpMemMetricNames, rule)
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP %s 内存 %.1f%% %s %.1f%%", device.Name, value, rule.Operator, rule.Threshold)
|
|
}
|
|
case "snmp_temperature":
|
|
value, triggered = e.evalSNMPGaugeRule(device.ID, snmpTempMetricNames, rule)
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP %s 温度 %.1f°C %s %.1f°C", device.Name, value, rule.Operator, rule.Threshold)
|
|
}
|
|
case "snmp_if_in_bps":
|
|
value, triggered = e.evalSNMPRateRule(device.ID, "if_in_octets", rule)
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP %s 入流量 %.0f bps %s %.0f bps", device.Name, value, rule.Operator, rule.Threshold)
|
|
}
|
|
case "snmp_if_out_bps":
|
|
value, triggered = e.evalSNMPRateRule(device.ID, "if_out_octets", rule)
|
|
if triggered {
|
|
msg = fmt.Sprintf("SNMP %s 出流量 %.0f bps %s %.0f bps", device.Name, value, rule.Operator, rule.Threshold)
|
|
}
|
|
default:
|
|
return
|
|
}
|
|
|
|
if !triggered {
|
|
return
|
|
}
|
|
e.fireSNMPAlert(rule, device, value, rule.Threshold, msg)
|
|
}
|
|
|
|
func (e *AlertEngine) evalSNMPGaugeRule(deviceID uint, names []string, rule model.MonitorAlertRule) (float64, bool) {
|
|
value, ok := e.getMaxSNMPMetric(deviceID, names)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
return value, e.compareOperator(rule.Operator, value, rule.Threshold)
|
|
}
|
|
|
|
func (e *AlertEngine) evalSNMPRateRule(deviceID uint, counterName string, rule model.MonitorAlertRule) (float64, bool) {
|
|
value, ok := e.getSNMPCounterRate(deviceID, counterName)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
return value, e.compareOperator(rule.Operator, value, rule.Threshold)
|
|
}
|
|
|
|
func (e *AlertEngine) getMaxSNMPMetric(deviceID uint, names []string) (float64, bool) {
|
|
var latestTS int64
|
|
e.db.Model(&model.SNMPMetric{}).Where("device_id = ?", deviceID).
|
|
Select("COALESCE(MAX(timestamp), 0)").Scan(&latestTS)
|
|
if latestTS == 0 {
|
|
return 0, false
|
|
}
|
|
var metrics []model.SNMPMetric
|
|
e.db.Where("device_id = ? AND timestamp = ? AND name IN ?", deviceID, latestTS, names).Find(&metrics)
|
|
if len(metrics) == 0 {
|
|
return 0, false
|
|
}
|
|
max := 0.0
|
|
for _, m := range metrics {
|
|
if m.Value > max {
|
|
max = m.Value
|
|
}
|
|
}
|
|
return max, true
|
|
}
|
|
|
|
// getSNMPCounterRate 取 counter 最近两次差分比特速率 // Logan.Li
|
|
func (e *AlertEngine) getSNMPCounterRate(deviceID uint, name string) (float64, bool) {
|
|
var metrics []model.SNMPMetric
|
|
e.db.Where("device_id = ? AND name = ?", deviceID, name).
|
|
Order("timestamp DESC").Limit(2).Find(&metrics)
|
|
if len(metrics) < 2 {
|
|
return 0, false
|
|
}
|
|
cur, prev := metrics[0], metrics[1]
|
|
dt := float64(cur.Timestamp - prev.Timestamp)
|
|
if dt <= 0 {
|
|
return 0, false
|
|
}
|
|
delta := cur.Value - prev.Value
|
|
if delta < 0 {
|
|
return 0, false
|
|
}
|
|
return (delta * 8) / dt, true
|
|
}
|
|
|
|
func (e *AlertEngine) fireSNMPAlert(rule model.MonitorAlertRule, device model.SNMPDevice, value, threshold float64, msg string) {
|
|
tenMinAgo := time.Now().Add(-10 * time.Minute)
|
|
var existing model.AgentAlert
|
|
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ? AND created_at > ?",
|
|
rule.Name, device.Name, false, tenMinAgo).First(&existing)
|
|
if existing.ID > 0 {
|
|
return
|
|
}
|
|
if e.isSilenced(rule.Name, device.Name, rule.ID) {
|
|
return
|
|
}
|
|
sev := rule.Severity
|
|
if sev == "" {
|
|
sev = "warning"
|
|
}
|
|
alert := model.AgentAlert{
|
|
RuleName: rule.Name, Hostname: device.Name, Metric: rule.Metric,
|
|
Severity: sev, Message: msg, Value: value, Threshold: threshold,
|
|
}
|
|
e.db.Create(&alert)
|
|
e.notifyAlertRule("snmp", rule.Name, device.Name, rule.Metric, sev, msg, value, threshold)
|
|
now := time.Now()
|
|
e.db.Model(&rule).Update("last_triggered", now)
|
|
}
|
|
|
|
// isSNMPMetric 判断 AgentAlert 是否属于 SNMP // Logan.Li
|
|
func isSNMPMetric(metric string) bool {
|
|
return len(metric) >= 5 && metric[:5] == "snmp_"
|
|
} |