feat(monitor): F6 日志告警引擎 — AlertEngine 支持 type=log 规则查询Loki/ES+触发/恢复 + 数据源认证升级(none/basic/bearer/custom) + 从查询创建告警真实API对接

2026-06-28 04:00:38 +08:00
1 parent 894cb16a05
commit 6bba6144f3
@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 	"time"

@@ -67,6 +66,7 @@ func (e *AlertEngine) Check() {
 	e.checkPortAlerts()
 	e.checkLogAlertRules()
 	e.autoResolveAlerts()
+	e.checkLogAlertRecovery()
 }

 // tryAcquireLock 尝试获取分布式锁，返回是否获得 Leader 角色
@@ -386,6 +386,143 @@ func (e *AlertEngine) checkPortAlerts() {
 	}
 }

+// checkLogAlertRules 执行 type == "log" 的告警规则
+// 通过数据源查询 Loki/ES，结果条数 > 0 即触发。// Logan.Li
+func (e *AlertEngine) checkLogAlertRules() {
+	if e.ds == nil {
+		return
+	}
+	var rules []model.AlertRule
+	e.db.Where("enabled = ? AND type = ?", true, "log").Find(&rules)
+
+	for _, rule := range rules {
+		ds, err := loadLogDS(e.ds, rule.DatasourceID)
+		if err != nil || ds.Type != rule.Source {
+			log.Printf("[AlertEngine] 日志告警规则 %s 数据源无效: id=%d source=%s", rule.Name, rule.DatasourceID, rule.Source)
+			continue
+		}
+
+		count, err := e.queryLogDataSource(ds, rule.Expression)
+		if err != nil {
+			log.Printf("[AlertEngine] 日志告警规则 %s 查询失败: %v", rule.Name, err)
+			continue
+		}
+
+		// 简单策略：查询结果数量 > 0 即触发（后续可扩展为 count_over_time 阈值）// Logan.Li
+		if count == 0 {
+			continue
+		}
+
+		// 避免重复告警：最近 10 分钟已有同规则未恢复事件，不重复
+		var existing model.AlertEvent
+		tenMinAgo := time.Now().Add(-10 * time.Minute)
+		e.db.Where("rule_id = ? AND status = ? AND started_at > ?", rule.ID, model.AlertEventFiring, tenMinAgo).First(&existing)
+		if existing.ID > 0 {
+			continue
+		}
+
+		if e.isSilenced(rule.Name, "", rule.ID) {
+			continue
+		}
+
+		now := time.Now()
+		alert := model.AlertEvent{
+			RuleID:     rule.ID,
+			RuleName:   rule.Name,
+			Severity:   rule.Severity,
+			Status:     string(model.AlertEventFiring),
+			Target:     fmt.Sprintf("%s/%s", ds.Type, ds.Name),
+			Message:    fmt.Sprintf("日志告警触发: %s 在 %s 中匹配到 %d 条日志", rule.Name, ds.Name, count),
+			Value:      float64(count),
+			Threshold:  0,
+			StartedAt:  now,
+			LastEvalAt: now,
+		}
+		e.db.Create(&alert)
+
+		e.notifyAlertRule(rule.Name, alert.Target, "log", rule.Severity, alert.Message, float64(count), 0)
+
+		e.db.Model(&rule).Update("last_triggered", now)
+	}
+}
+
+// queryLogDataSource 根据数据源类型查询日志条数 // Logan.Li
+func (e *AlertEngine) queryLogDataSource(ds *model.LogDataSource, expression string) (int, error) {
+	end := time.Now()
+	start := end.Add(-5 * time.Minute)
+	client := newLogHTTPClient(ds.SkipTLSVerify, 30*time.Second)
+
+	switch strings.ToLower(ds.Type) {
+	case "loki":
+		params := url.Values{}
+		params.Set("query", expression)
+		params.Set("limit", "1")
+		params.Set("direction", "backward")
+		params.Set("start", fmt.Sprintf("%d", start.UnixNano()))
+		params.Set("end", fmt.Sprintf("%d", end.UnixNano()))
+		target := ds.URL + "/loki/api/v1/query_range?" + params.Encode()
+		req, _ := http.NewRequest(http.MethodGet, target, nil)
+		applyLogDSAuthV2(req, ds)
+		resp, err := client.Do(req)
+		if err != nil {
+			return 0, err
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		if resp.StatusCode >= 400 {
+			return 0, fmt.Errorf("HTTP %s: %s", resp.Status, string(body))
+		}
+		var raw map[string]interface{}
+		_ = json.Unmarshal(body, &raw)
+		entries := flattenLokiStreams(raw)
+		return len(entries), nil
+	case "elasticsearch":
+		searchBody := map[string]interface{}{
+			"size": 0,
+			"query": map[string]interface{}{
+				"bool": map[string]interface{}{
+					"must": []interface{}{
+						map[string]interface{}{"query_string": map[string]interface{}{"query": defaultESQuery(expression)}},
+					},
+				},
+			},
+		}
+		rangeCond := map[string]interface{}{
+			"bool": map[string]interface{}{
+				"should": []interface{}{
+					map[string]interface{}{"range": map[string]interface{}{"@timestamp": map[string]string{"gte": start.Format(time.RFC3339Nano), "lte": end.Format(time.RFC3339Nano)}}},
+					map[string]interface{}{"range": map[string]interface{}{"timestamp": map[string]string{"gte": start.Format(time.RFC3339Nano), "lte": end.Format(time.RFC3339Nano)}}},
+				},
+				"minimum_should_match": 1,
+			},
+		}
+		searchBody["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(
+			searchBody["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]interface{}),
+			rangeCond,
+		)
+		payload, _ := json.Marshal(searchBody)
+		target := fmt.Sprintf("%s/%s/_search", ds.URL, "*")
+		req, _ := http.NewRequest(http.MethodPost, target, bytes.NewReader(payload))
+		req.Header.Set("Content-Type", "application/json")
+		applyLogDSAuthV2(req, ds)
+		resp, err := client.Do(req)
+		if err != nil {
+			return 0, err
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		if resp.StatusCode >= 400 {
+			return 0, fmt.Errorf("HTTP %s: %s", resp.Status, string(body))
+		}
+		var raw map[string]interface{}
+		_ = json.Unmarshal(body, &raw)
+		total := esHitsTotal(raw)
+		return int(total), nil
+	default:
+		return 0, fmt.Errorf("未知日志数据源类型: %s", ds.Type)
+	}
+}
+
 // 自动恢复告警（指标恢复正常）
 func (e *AlertEngine) autoResolveAlerts() {
 	var alerts []model.AgentAlert
@@ -478,6 +615,43 @@ func (e *AlertEngine) autoResolveAlerts() {
 	}
 }

+// checkLogAlertRecovery 恢复已解决的日志告警事件 // Logan.Li
+func (e *AlertEngine) checkLogAlertRecovery() {
+	if e.ds == nil {
+		return
+	}
+	var events []model.AlertEvent
+	e.db.Where("status = ?", model.AlertEventFiring).Find(&events)
+
+	for _, evt := range events {
+		if evt.RuleID == 0 {
+			continue
+		}
+		var rule model.AlertRule
+		if err := e.db.First(&rule, evt.RuleID).Error; err != nil || rule.Type != "log" {
+			continue
+		}
+		ds, err := loadLogDS(e.ds, rule.DatasourceID)
+		if err != nil {
+			continue
+		}
+		count, err := e.queryLogDataSource(ds, rule.Expression)
+		if err != nil {
+			continue
+		}
+		// 连续两个周期无日志匹配则恢复（简化策略）// Logan.Li
+		if count > 0 {
+			continue
+		}
+		now := time.Now()
+		e.db.Model(&evt).Updates(map[string]interface{}{
+			"status":      model.AlertEventResolved,
+			"resolved_at": now,
+		})
+		e.notifyRecovery(rule.Name, evt.Target, "log", evt.Message)
+	}
+}
+
 func (e *AlertEngine) compareOperator(op string, val, threshold float64) bool {
 	switch op {
 	case ">":