attacker
ca036a85ca
1. EventsTab.tsx: DEMO_EVENTS硬编码→kubernetesApi.getEvents真实K8s事件 - 复用 normalizeK8sEvent/formatEventRelative 工具（与 EventsPage 一致） - 新增 loading/empty 状态 + 刷新按钮 - Warning事件优先排序 + 限100条防溢出 - KubernetesClusterDetail 传入 clusterCode prop - 修复 Q28 组件拆分时遗留的 demo 数据未对接缺口 2. aiops_alert_analysis.go: log.Printf %s→%d 修复 go vet 失败 - AlertID 类型为 uint, %s 格式符不匹配致 go test 构建失败 - go vet ./... 全量零错误 验证: tsc -b=0, npm build✓ (18.00s), go build=0, go vet=0, 17/17测试通过 Events API 返回5条真实事件 (type=Normal, reason=Pulled, count=5487)
453 lines
15 KiB
Go
453 lines
15 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"ezops/middleware"
|
|
"ezops/model"
|
|
"ezops/pkg/agentcli/llm"
|
|
"ezops/pkg/aiopscontext"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/google/uuid"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// AIOpsAlertAnalysisHandler 基于上下文包执行告警 RCA。 // Logan.Li
|
|
type AIOpsAlertAnalysisHandler struct {
|
|
db *gorm.DB
|
|
context *aiopscontext.Service
|
|
}
|
|
|
|
// NewAIOpsAlertAnalysisHandler 创建告警分析处理器。 // Logan.Li
|
|
func NewAIOpsAlertAnalysisHandler(db *gorm.DB) *AIOpsAlertAnalysisHandler {
|
|
return &AIOpsAlertAnalysisHandler{db: db, context: aiopscontext.NewService(db)}
|
|
}
|
|
|
|
// AlertAnalysisRequest 告警分析请求，前端传告警 ID 和少量兜底字段。 // Logan.Li
|
|
type AlertAnalysisRequest struct {
|
|
AlertID uint `json:"alert_id"`
|
|
Name string `json:"name"`
|
|
Severity string `json:"severity"`
|
|
Target string `json:"target"`
|
|
Metric string `json:"metric"`
|
|
CurrentValue string `json:"current_value"`
|
|
Threshold string `json:"threshold"`
|
|
Duration string `json:"duration"`
|
|
Description string `json:"description"`
|
|
TimeRange string `json:"time_range"`
|
|
}
|
|
|
|
// AlertAnalysisResponse 告警 RCA 结构化响应。 // Logan.Li
|
|
type AlertAnalysisResponse struct {
|
|
TraceID string `json:"trace_id"`
|
|
SpanID string `json:"span_id"`
|
|
RootCause string `json:"root_cause"`
|
|
Impact string `json:"impact"`
|
|
Recommendations []string `json:"recommendations"`
|
|
AutoFixScript string `json:"auto_fix_script,omitempty"`
|
|
SimilarCases int `json:"similar_cases"`
|
|
Confidence int `json:"confidence"`
|
|
Evidence []aiopscontext.EvidenceItem `json:"evidence"`
|
|
ContextPack *aiopscontext.ContextPack `json:"context_pack,omitempty"`
|
|
Model string `json:"model"`
|
|
Source string `json:"source"`
|
|
GeneratedAt time.Time `json:"generated_at"`
|
|
}
|
|
|
|
// Analyze POST /api/v1/aiops/alert-analysis。 // Logan.Li
|
|
func (h *AIOpsAlertAnalysisHandler) Analyze(c *gin.Context) {
|
|
start := time.Now()
|
|
traceID := uuid.NewString()
|
|
spanID := uuid.NewString()
|
|
userID := middleware.GetUserID(c)
|
|
|
|
var req AlertAnalysisRequest
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
c.JSON(http.StatusBadRequest, model.Response{Code: 400, Message: "请求参数错误: " + err.Error()})
|
|
return
|
|
}
|
|
timeRange := req.TimeRange
|
|
if strings.TrimSpace(timeRange) == "" {
|
|
timeRange = "24h"
|
|
}
|
|
metrics := metricsForAlert(req.Metric)
|
|
pack, err := h.context.BuildPack(c.Request.Context(), aiopscontext.ContextPackRequest{
|
|
AlertID: req.AlertID, Target: req.Target, Metrics: metrics, TimeRange: timeRange, Limit: 12,
|
|
})
|
|
if err != nil {
|
|
h.saveAnalysisTrace(c.Request.Context(), analysisTraceInput{
|
|
TraceID: traceID, SpanID: spanID, UserID: userID, Feature: "alert_analysis",
|
|
Target: req.Target, AlertID: req.AlertID, Status: "error", Source: "context-pack",
|
|
Duration: time.Since(start).Milliseconds(), Error: err.Error(),
|
|
Result: gin.H{"error": "构建上下文失败: " + err.Error()},
|
|
})
|
|
c.JSON(http.StatusOK, model.Response{Code: 500, Message: "构建上下文失败: " + err.Error()})
|
|
return
|
|
}
|
|
|
|
fallback := fallbackAlertAnalysis(req, pack)
|
|
attachTrace(fallback, traceID, spanID)
|
|
var cfg model.LLMConfig
|
|
if err := h.db.First(&cfg).Error; err != nil || !cfg.Enabled {
|
|
h.saveAnalysisTrace(c.Request.Context(), analysisTraceInput{
|
|
TraceID: traceID, SpanID: spanID, UserID: userID, Feature: "alert_analysis",
|
|
Target: firstText(req.Target, pack.Scope), AlertID: req.AlertID, Provider: cfg.Provider,
|
|
Model: fallback.Model, Source: fallback.Source, Status: "fallback", Pack: pack, Result: fallback,
|
|
Duration: time.Since(start).Milliseconds(),
|
|
})
|
|
c.JSON(http.StatusOK, model.Response{Code: 200, Data: fallback})
|
|
return
|
|
}
|
|
|
|
result, usage, err := h.callAlertRCA(c.Request.Context(), cfg, req, pack, fallback)
|
|
if err != nil {
|
|
fallback.Source = "rule-fallback: " + err.Error()
|
|
attachTrace(fallback, traceID, spanID)
|
|
h.saveAnalysisTrace(c.Request.Context(), analysisTraceInput{
|
|
TraceID: traceID, SpanID: spanID, UserID: userID, Feature: "alert_analysis",
|
|
Target: firstText(req.Target, pack.Scope), AlertID: req.AlertID, Provider: cfg.Provider,
|
|
Model: cfg.Model, Source: "rule-fallback", Status: "fallback", Pack: pack, Result: fallback,
|
|
Duration: time.Since(start).Milliseconds(), Error: err.Error(),
|
|
})
|
|
c.JSON(http.StatusOK, model.Response{Code: 200, Data: fallback})
|
|
return
|
|
}
|
|
attachTrace(result, traceID, spanID)
|
|
h.saveAnalysisTrace(c.Request.Context(), analysisTraceInput{
|
|
TraceID: traceID, SpanID: spanID, UserID: userID, Feature: "alert_analysis",
|
|
Target: firstText(req.Target, pack.Scope), AlertID: req.AlertID, Provider: cfg.Provider,
|
|
Model: result.Model, Source: result.Source, Status: "success", Pack: pack, Result: result,
|
|
PromptTokens: usage.PromptTokens, CompletionTokens: usage.CompletionTokens, TotalTokens: usage.TotalTokens,
|
|
Duration: time.Since(start).Milliseconds(),
|
|
})
|
|
c.JSON(http.StatusOK, model.Response{Code: 200, Data: result})
|
|
}
|
|
|
|
func (h *AIOpsAlertAnalysisHandler) callAlertRCA(ctx context.Context, cfg model.LLMConfig, req AlertAnalysisRequest, pack *aiopscontext.ContextPack, fallback *AlertAnalysisResponse) (*AlertAnalysisResponse, llm.Usage, error) {
|
|
packJSON, _ := json.MarshalIndent(pack, "", " ")
|
|
prompt := fmt.Sprintf(`你是 AIOps 平台的企业级告警 RCA 分析器。请只基于【告警基本信息】和【AIOps Context Pack】做判断，不要编造不存在的数据。
|
|
|
|
【告警基本信息】
|
|
- ID: %d
|
|
- 名称: %s
|
|
- 级别: %s
|
|
- 目标: %s
|
|
- 指标: %s
|
|
- 当前值: %s
|
|
- 阈值: %s
|
|
- 持续: %s
|
|
- 描述: %s
|
|
|
|
【AIOps Context Pack】
|
|
%s
|
|
|
|
请返回严格 JSON，不要 markdown，不要代码块，字段如下：
|
|
{
|
|
"root_cause": "根因分析，2-4句，必须引用证据",
|
|
"impact": "影响范围，1-2句",
|
|
"recommendations": ["建议1", "建议2", "建议3"],
|
|
"auto_fix_script": "仅限只读诊断脚本；没有则为空字符串",
|
|
"similar_cases": 0,
|
|
"confidence": 0-100
|
|
}`, req.AlertID, req.Name, req.Severity, req.Target, req.Metric, req.CurrentValue, req.Threshold, req.Duration, req.Description, string(packJSON))
|
|
|
|
resp, err := buildLLMClientFromConfig(cfg).Complete(ctx, &llm.CompletionRequest{
|
|
Messages: []llm.Message{
|
|
{Role: "system", Content: "你是 AIOps 平台的告警根因分析专家，输出必须是可解析 JSON。"},
|
|
{Role: "user", Content: prompt},
|
|
},
|
|
Temperature: 0.2,
|
|
MaxTokens: cfg.MaxTokens,
|
|
})
|
|
if err != nil {
|
|
return nil, llm.Usage{}, err
|
|
}
|
|
parsed := *fallback
|
|
if err := parseAlertAnalysisJSON(resp.Message.Content, &parsed); err != nil {
|
|
parsed.RootCause = strings.TrimSpace(resp.Message.Content)
|
|
}
|
|
parsed.ContextPack = pack
|
|
parsed.Evidence = pack.Evidence
|
|
parsed.Model = cfg.Model
|
|
parsed.Source = "llm"
|
|
parsed.GeneratedAt = time.Now()
|
|
return &parsed, resp.Usage, nil
|
|
}
|
|
|
|
// ListAnalysisTraces 查询 AI 分析审计记录，普通用户只看自己的记录。 // Logan.Li
|
|
func (h *AIOpsAlertAnalysisHandler) ListAnalysisTraces(c *gin.Context) {
|
|
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
|
if page < 1 {
|
|
page = 1
|
|
}
|
|
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
|
|
if pageSize <= 0 || pageSize > 100 {
|
|
pageSize = 20
|
|
}
|
|
|
|
q := h.db.Model(&model.AIAnalysisTrace{})
|
|
if !middleware.GetIsAdmin(c) {
|
|
q = q.Where("user_id = ?", middleware.GetUserID(c))
|
|
}
|
|
if feature := strings.TrimSpace(c.Query("feature")); feature != "" {
|
|
q = q.Where("feature = ?", feature)
|
|
}
|
|
if target := strings.TrimSpace(c.Query("target")); target != "" {
|
|
q = q.Where("target LIKE ?", "%"+target+"%")
|
|
}
|
|
if status := strings.TrimSpace(c.Query("status")); status != "" {
|
|
q = q.Where("status = ?", status)
|
|
}
|
|
|
|
var total int64
|
|
q.Count(&total)
|
|
var rows []model.AIAnalysisTrace
|
|
if err := q.Select("id, trace_id, span_id, parent_span_id, user_id, feature, target, alert_id, provider, model, source, status, evidence_count, prompt_tokens, completion_tokens, total_tokens, duration, error, created_at, updated_at").
|
|
Order("created_at DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
|
|
c.JSON(http.StatusOK, model.Response{Code: 500, Message: "查询失败: " + err.Error()})
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, model.Response{Code: 200, Data: gin.H{"list": rows, "total": total}})
|
|
}
|
|
|
|
// GetAnalysisTrace 按 trace_id 回放某次 AI 分析看到的上下文包和输出结论。 // Logan.Li
|
|
func (h *AIOpsAlertAnalysisHandler) GetAnalysisTrace(c *gin.Context) {
|
|
traceID := strings.TrimSpace(c.Param("trace_id"))
|
|
q := h.db.Where("trace_id = ?", traceID)
|
|
if !middleware.GetIsAdmin(c) {
|
|
q = q.Where("user_id = ?", middleware.GetUserID(c))
|
|
}
|
|
var row model.AIAnalysisTrace
|
|
if err := q.Order("id ASC").First(&row).Error; err != nil {
|
|
c.JSON(http.StatusOK, model.Response{Code: 404, Message: "分析记录不存在"})
|
|
return
|
|
}
|
|
|
|
c.JSON(http.StatusOK, model.Response{Code: 200, Data: gin.H{
|
|
"trace": row,
|
|
"context_pack": decodeJSONField(row.ContextPack),
|
|
"result": decodeJSONField(row.Result),
|
|
}})
|
|
}
|
|
|
|
func parseAlertAnalysisJSON(raw string, out *AlertAnalysisResponse) error {
|
|
clean := strings.TrimSpace(raw)
|
|
clean = strings.TrimPrefix(clean, "```json")
|
|
clean = strings.TrimPrefix(clean, "```")
|
|
clean = strings.TrimSuffix(clean, "```")
|
|
clean = strings.TrimSpace(clean)
|
|
var tmp struct {
|
|
RootCause string `json:"root_cause"`
|
|
Impact string `json:"impact"`
|
|
Recommendations []string `json:"recommendations"`
|
|
AutoFixScript string `json:"auto_fix_script"`
|
|
SimilarCases int `json:"similar_cases"`
|
|
Confidence int `json:"confidence"`
|
|
}
|
|
if err := json.Unmarshal([]byte(clean), &tmp); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(tmp.RootCause) != "" {
|
|
out.RootCause = tmp.RootCause
|
|
}
|
|
if strings.TrimSpace(tmp.Impact) != "" {
|
|
out.Impact = tmp.Impact
|
|
}
|
|
if len(tmp.Recommendations) > 0 {
|
|
out.Recommendations = tmp.Recommendations
|
|
}
|
|
out.AutoFixScript = tmp.AutoFixScript
|
|
out.SimilarCases = tmp.SimilarCases
|
|
if tmp.Confidence > 0 {
|
|
if tmp.Confidence > 100 {
|
|
tmp.Confidence = 100
|
|
}
|
|
out.Confidence = tmp.Confidence
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fallbackAlertAnalysis(req AlertAnalysisRequest, pack *aiopscontext.ContextPack) *AlertAnalysisResponse {
|
|
evidenceText := "暂无足够平台证据"
|
|
if len(pack.Evidence) > 0 {
|
|
evidenceText = pack.Evidence[0].Summary
|
|
}
|
|
target := firstText(req.Target, pack.Scope, "未知目标")
|
|
metric := firstText(req.Metric, "unknown")
|
|
recs := []string{
|
|
fmt.Sprintf("先查看 %s 的近期指标趋势和告警时间线，确认是否持续异常", target),
|
|
"结合变更、发布、定时任务和日志错误进一步缩小根因范围",
|
|
"修复动作前先执行只读诊断，涉及重启、清理、扩容等动作需走确认或审批",
|
|
}
|
|
return &AlertAnalysisResponse{
|
|
RootCause: fmt.Sprintf("告警 %s 在目标 %s 上触发，指标 %s 当前值 %s，阈值 %s。基于当前上下文，最直接证据是：%s。",
|
|
firstText(req.Name, "未命名告警"), target, metric, req.CurrentValue, req.Threshold, evidenceText),
|
|
Impact: fmt.Sprintf("该告警级别为 %s，可能影响目标 %s 的稳定性，需要结合业务等级判断优先级。", firstText(req.Severity, "unknown"), target),
|
|
Recommendations: recs,
|
|
SimilarCases: 0,
|
|
Confidence: fallbackConfidence(pack),
|
|
Evidence: pack.Evidence,
|
|
ContextPack: pack,
|
|
Model: "rule-fallback",
|
|
Source: "rule-fallback",
|
|
GeneratedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
func firstText(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func metricsForAlert(metric string) []string {
|
|
m := strings.ToLower(metric)
|
|
switch {
|
|
case strings.Contains(m, "mem"):
|
|
return []string{"mem_usage", "load1"}
|
|
case strings.Contains(m, "load"):
|
|
return []string{"load1", "load5", "cpu_usage"}
|
|
case strings.Contains(m, "io"):
|
|
return []string{"cpu_iowait", "load1"}
|
|
default:
|
|
return []string{"cpu_usage", "mem_usage", "load1"}
|
|
}
|
|
}
|
|
|
|
func fallbackConfidence(pack *aiopscontext.ContextPack) int {
|
|
score := 45
|
|
if len(pack.Alerts) > 0 {
|
|
score += 10
|
|
}
|
|
if len(pack.Metrics) > 0 {
|
|
score += 15
|
|
}
|
|
if len(pack.Assets) > 0 {
|
|
score += 5
|
|
}
|
|
if score > 80 {
|
|
score = 80
|
|
}
|
|
return score
|
|
}
|
|
|
|
type analysisTraceInput struct {
|
|
TraceID string
|
|
SpanID string
|
|
ParentSpanID string
|
|
UserID uint
|
|
Feature string
|
|
Target string
|
|
AlertID uint
|
|
Provider string
|
|
Model string
|
|
Source string
|
|
Status string
|
|
Pack *aiopscontext.ContextPack
|
|
Result interface{}
|
|
PromptTokens int
|
|
CompletionTokens int
|
|
TotalTokens int
|
|
Duration int64
|
|
Error string
|
|
}
|
|
|
|
func attachTrace(resp *AlertAnalysisResponse, traceID, spanID string) {
|
|
if resp == nil {
|
|
return
|
|
}
|
|
resp.TraceID = traceID
|
|
resp.SpanID = spanID
|
|
}
|
|
|
|
func (h *AIOpsAlertAnalysisHandler) saveAnalysisTrace(ctx context.Context, in analysisTraceInput) {
|
|
packJSON := marshalTraceJSON(in.Pack)
|
|
resultJSON := marshalTraceJSON(traceResultForStorage(in.Result))
|
|
evidenceCount := 0
|
|
if in.Pack != nil {
|
|
evidenceCount = len(in.Pack.Evidence)
|
|
}
|
|
row := model.AIAnalysisTrace{
|
|
TraceID: in.TraceID,
|
|
SpanID: in.SpanID,
|
|
ParentSpanID: in.ParentSpanID,
|
|
UserID: in.UserID,
|
|
Feature: firstText(in.Feature, "unknown"),
|
|
Target: in.Target,
|
|
AlertID: in.AlertID,
|
|
Provider: in.Provider,
|
|
Model: in.Model,
|
|
Source: trimTraceText(in.Source, 64),
|
|
Status: firstText(in.Status, "unknown"),
|
|
ContextPack: packJSON,
|
|
Result: resultJSON,
|
|
EvidenceCount: evidenceCount,
|
|
PromptTokens: in.PromptTokens,
|
|
CompletionTokens: in.CompletionTokens,
|
|
TotalTokens: in.TotalTokens,
|
|
Duration: in.Duration,
|
|
Error: in.Error,
|
|
}
|
|
_ = h.db.WithContext(ctx).Create(&row).Error
|
|
}
|
|
|
|
func marshalTraceJSON(v interface{}) string {
|
|
if v == nil {
|
|
return ""
|
|
}
|
|
raw, err := json.Marshal(v)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(raw)
|
|
}
|
|
|
|
func traceResultForStorage(v interface{}) interface{} {
|
|
switch r := v.(type) {
|
|
case *AlertAnalysisResponse:
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
cp := *r
|
|
cp.ContextPack = nil
|
|
return cp
|
|
case AlertAnalysisResponse:
|
|
r.ContextPack = nil
|
|
return r
|
|
default:
|
|
return v
|
|
}
|
|
}
|
|
|
|
func decodeJSONField(raw string) interface{} {
|
|
if strings.TrimSpace(raw) == "" {
|
|
return nil
|
|
}
|
|
var out interface{}
|
|
if err := json.Unmarshal([]byte(raw), &out); err != nil {
|
|
return raw
|
|
}
|
|
return out
|
|
}
|
|
|
|
func trimTraceText(s string, max int) string {
|
|
if max <= 0 {
|
|
return s
|
|
}
|
|
runes := []rune(s)
|
|
if len(runes) <= max {
|
|
return s
|
|
}
|
|
return string(runes[:max])
|
|
}
|