aiops/backend/controller/alert_engine_monitor.go

package controller

import (
	"encoding/json"
	"fmt"
	"time"

	"ezops/model"
	"ezops/pkg/siteprobe"
)

// checkMiddlewareAlerts 中间件指标告警 // Logan.Li
func (e *AlertEngine) checkMiddlewareAlerts() {
	var rules []model.MonitorAlertRule
	e.db.Where("enabled = ? AND category = ?", true, model.MonitorAlertMiddleware).Find(&rules)
	if len(rules) == 0 {
		return
	}

	var targets []model.MonitorTarget
	e.db.Where("type = ?", model.TargetTypeMiddleware).Find(&targets)
	targetMap := make(map[uint]model.MonitorTarget)
	for _, t := range targets {
		targetMap[t.ID] = t
	}

	for _, rule := range rules {
		checkTargets := targets
		if rule.TargetID > 0 {
			if t, ok := targetMap[rule.TargetID]; ok {
				checkTargets = []model.MonitorTarget{t}
			} else {
				continue
			}
		}
		for _, target := range checkTargets {
			e.evalMiddlewareRule(rule, target)
		}
	}
}

func (e *AlertEngine) evalMiddlewareRule(rule model.MonitorAlertRule, target model.MonitorTarget) {
	if rule.Metric == "middleware_offline" {
		if target.Status == model.TargetStatusOffline || target.Status == model.TargetStatusUnknown {
			e.fireMiddlewareAlert(rule, target, 1, 1, "中间件不可达或采集失败")
		}
		return
	}

	value, ok := e.getLatestMetricValue(target.ID, rule.Metric)
	if !ok {
		return
	}
	if !e.compareOperator(rule.Operator, value, rule.Threshold) {
		return
	}
	msg := fmt.Sprintf("%s %s %.2f %s %.2f", target.Name, rule.Metric, value, rule.Operator, rule.Threshold)
	e.fireMiddlewareAlert(rule, target, value, rule.Threshold, msg)
}

func (e *AlertEngine) getLatestMetricValue(targetID uint, metric string) (float64, bool) {
	var hist model.MetricHistory
	e.db.Where("target_id = ? AND metric_name = ?", targetID, metric).
		Order("timestamp DESC").First(&hist)
	if hist.ID == 0 {
		return 0, false
	}
	return hist.Value, true
}

func (e *AlertEngine) fireMiddlewareAlert(rule model.MonitorAlertRule, target model.MonitorTarget, value, threshold float64, msg string) {
	tenMinAgo := time.Now().Add(-10 * time.Minute)
	var existing model.AgentAlert
	e.db.Where("rule_name = ? AND hostname = ? AND resolved = ? AND created_at > ?",
		rule.Name, target.Name, false, tenMinAgo).First(&existing)
	if existing.ID > 0 {
		return
	}
	if e.isSilenced(rule.Name, target.Name, rule.ID) {
		return
	}
	sev := rule.Severity
	if sev == "" {
		sev = "warning"
	}
	alert := model.AgentAlert{
		RuleName: rule.Name, Hostname: target.Name, Metric: rule.Metric,
		Severity: sev, Message: msg, Value: value, Threshold: threshold,
	}
	e.db.Create(&alert)
	e.notifyAlertRule("middleware", rule.Name, target.Name, rule.Metric, sev, msg, value, threshold)
	now := time.Now()
	e.db.Model(&rule).Update("last_triggered", now)
}

// checkSiteAlerts 站点拨测告警 // Logan.Li
func (e *AlertEngine) checkSiteAlerts() {
	var rules []model.MonitorAlertRule
	e.db.Where("enabled = ? AND category = ?", true, model.MonitorAlertSite).Find(&rules)
	if len(rules) == 0 {
		return
	}

	var tasks []model.SiteTask
	e.db.Find(&tasks)
	taskMap := make(map[uint]model.SiteTask)
	for _, t := range tasks {
		taskMap[t.ID] = t
	}

	for _, rule := range rules {
		checkTasks := tasks
		if rule.TargetID > 0 {
			if t, ok := taskMap[rule.TargetID]; ok {
				checkTasks = []model.SiteTask{t}
			} else {
				continue
			}
		}
		for _, task := range checkTasks {
			if task.Status != model.SiteTaskStatusRunning {
				continue
			}
			e.evalSiteRule(rule, task)
		}
	}
}

func (e *AlertEngine) evalSiteRule(rule model.MonitorAlertRule, task model.SiteTask) {
	var value float64
	var triggered bool
	var msg string

	switch rule.Metric {
	case "probe_fail":
		triggered = task.LastResult == "fail" || task.LastResult == "timeout"
		value = 1
		if triggered {
			msg = fmt.Sprintf("拨测任务 %s 失败: %s", task.Name, task.LastResult)
		}
	case "probe_timeout":
		triggered = task.LastResult == "timeout"
		value = 1
		if triggered {
			msg = fmt.Sprintf("拨测任务 %s 超时", task.Name)
		}
	case "response_time":
		value = task.AvgResponse
		triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
		if triggered {
			msg = fmt.Sprintf("拨测 %s 延迟 %.0fms 超过阈值 %.0fms", task.Name, value, rule.Threshold)
		}
	case "availability":
		value = task.SuccessRate
		triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
		if triggered {
			msg = fmt.Sprintf("拨测 %s 可用率 %.1f%% 低于阈值 %.1f%%", task.Name, value, rule.Threshold)
		}
	case "http_status_error":
		triggered = task.LastResult == "fail"
		value = 1
		if triggered {
			msg = fmt.Sprintf("拨测 %s HTTP 状态码异常", task.Name)
		}
	case "ssl_expiry_days":
		if !siteprobe.IsSSLProbeTask(task) {
			return
		}
		var result model.SiteResult
		e.db.Where("task_id = ? AND ssl_cert_expiry IS NOT NULL", task.ID).
			Order("checked_at DESC").First(&result)
		if result.ID == 0 {
			return
		}
		value = float64(result.SSLDaysRemaining)
		triggered = e.compareOperator(rule.Operator, value, rule.Threshold)
		if triggered {
			expiryStr := ""
			if result.SSLCertExpiry != nil {
				expiryStr = result.SSLCertExpiry.Format("2006-01-02")
			}
			if value < 0 {
				msg = fmt.Sprintf("SSL 拨测 %s 证书已过期", task.Name)
			} else if expiryStr != "" {
				msg = fmt.Sprintf("SSL 拨测 %s 证书剩余 %.0f 天（到期 %s）", task.Name, value, expiryStr)
			} else {
				msg = fmt.Sprintf("SSL 拨测 %s 证书剩余 %.0f 天", task.Name, value)
			}
		}
	default:
		return
	}

	if !triggered {
		return
	}

	tenMinAgo := time.Now().Add(-10 * time.Minute)
	var existing model.SiteAlert
	e.db.Where("task_id = ? AND title = ? AND status = ? AND started_at > ?",
		task.ID, rule.Name, model.SiteAlertStatusActive, tenMinAgo).First(&existing)
	if existing.ID > 0 {
		return
	}

	sev := model.SiteAlertWarning
	if rule.Severity == "critical" || (rule.Metric == "ssl_expiry_days" && value <= 7) {
		sev = model.SiteAlertCritical
	}
	now := time.Now()
	siteAlert := model.SiteAlert{
		TaskID: task.ID, Severity: sev, Status: model.SiteAlertStatusActive,
		Title: rule.Name, Message: msg, TriggerType: rule.Metric,
		Threshold: rule.Threshold, ResponseTime: task.AvgResponse,
		StartedAt: now,
	}
	e.db.Create(&siteAlert)
	e.notifyAlertRule("site", rule.Name, task.Name, rule.Metric, string(sev), msg, value, rule.Threshold)
	nt := time.Now()
	e.db.Model(&rule).Update("last_triggered", nt)
}

// extractHostMetricValue 从 Agent 上报提取扩展指标值 // Logan.Li
func extractHostMetricValue(report model.AgentReport, metric string) (float64, bool) {
	switch metric {
	case "cpu":
		return report.CPUUsage, true
	case "cpu_iowait":
		return report.CPUIOwait, true
	case "mem":
		return report.MemUsage, true
	case "mem_available":
		return report.MemAvailable, true
	case "load1":
		return report.Load1, true
	case "disk":
		var disks []model.DiskMetric
		json.Unmarshal([]byte(report.DiskInfo), &disks)
		max := 0.0
		for _, d := range disks {
			if d.UsagePercent > max {
				max = d.UsagePercent
			}
		}
		return max, true
	case "disk_inode":
		var disks []model.DiskMetric
		json.Unmarshal([]byte(report.DiskInfo), &disks)
		max := 0.0
		for _, d := range disks {
			if d.InodeUsagePct > max {
				max = d.InodeUsagePct
			}
		}
		return max, true
	case "disk_io_util":
		var dio model.DiskIOStatModel
		json.Unmarshal([]byte(report.DiskIOInfo), &dio)
		return dio.Util, true
	case "swap":
		var swap model.SwapMetric
		json.Unmarshal([]byte(report.SwapInfo), &swap)
		return swap.UsagePercent, true
	case "tcp_time_wait":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return float64(netStat.TCPTimeWait), true
	case "tcp_close_wait":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return float64(netStat.TCPCloseWait), true
	case "tcp_established":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return float64(netStat.TCPEstablished), true
	case "net_rx_bps":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return netStat.NetRxBPS, true
	case "net_tx_bps":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return netStat.NetTxBPS, true
	case "net_rx_errors":
		var netStat model.NetworkStatModel
		json.Unmarshal([]byte(report.NetInfo), &netStat)
		return float64(netStat.NetRxErrors), true
	case "fd":
		var fdStat model.FDMetric
		json.Unmarshal([]byte(report.FDInfo), &fdStat)
		return fdStat.UsagePct, true
	default:
		return 0, false
	}
}

// autoResolveSiteSslAlerts 证书续期后自动恢复 SSL 过期告警 // Logan.Li
func (e *AlertEngine) autoResolveSiteSslAlerts() {
	var alerts []model.SiteAlert
	e.db.Where("status = ? AND trigger_type = ?", model.SiteAlertStatusActive, "ssl_expiry_days").Find(&alerts)
	if len(alerts) == 0 {
		return
	}
	now := time.Now()
	for _, alert := range alerts {
		var result model.SiteResult
		e.db.Where("task_id = ? AND ssl_cert_expiry IS NOT NULL", alert.TaskID).
			Order("checked_at DESC").First(&result)
		if result.ID == 0 {
			continue
		}
		if float64(result.SSLDaysRemaining) >= 30 {
			e.db.Model(&alert).Updates(map[string]interface{}{
				"status": model.SiteAlertStatusResolved, "resolved_at": now,
			})
		}
	}
}

// isMiddlewareMetric 判断 AgentAlert 是否属于中间件 // Logan.Li
func isMiddlewareMetric(metric string) bool {
	prefixes := []string{"redis_", "mysql_", "middleware_"}
	for _, p := range prefixes {
		if len(metric) >= len(p) && metric[:len(p)] == p {
			return true
		}
	}
	return false
}