This commit is contained in:
Logan committed 2026-07-09 14:16:56 +08:00
1 parent 0690285147
commit 5618cd2738
154 files changed
+5021 -3161

No files matched your search

+1
View File
@@ -96,6 +96,7 @@ func NewEngine(cfg EngineConfig) (*Engine, error) {
return nil, fmt.Errorf("Kubernetes 模式缺少集群客户端")
}
kt := cfg.Kubernetes
registry.Register(tools.NewKubernetesDiagnoseTool(kt.Client, kt.Namespace))
registry.Register(tools.NewKubernetesGetTool(kt.Client, kt.Namespace))
registry.Register(tools.NewPodLogsTool(kt.Client, kt.Namespace, kt.PodName, kt.Container))
registry.Register(tools.NewPodExecuteTool(kt.Client, kt.Namespace, kt.PodName, kt.Container, coreCfg.ToolTimeout))
+132 -24
View File
@@ -116,6 +116,7 @@ type Agent struct {
evolver *evolver.EvolverEngine
callback EventCallback
totalTokens int
cache map[string]string
mu sync.Mutex
}
@@ -130,7 +131,7 @@ func NewAgent(client llm.Client, registry *tools.Registry, cfg *Config, sc *safe
if cfg.MaxContextTokens <= 0 {
cfg.MaxContextTokens = 8000
}
return &Agent{llm: client, registry: registry, config: cfg, safetyCtl: sc, evolver: ev}
return &Agent{llm: client, registry: registry, config: cfg, safetyCtl: sc, evolver: ev, cache: make(map[string]string)}
}
// SetCallback 注入事件回调 // Logan.Li
@@ -293,6 +294,14 @@ func (a *Agent) processToolCalls(ctx context.Context, calls []llm.ToolCall, hist
continue
}
if msg := a.suppressBroadKubernetesGet(tc.Function.Name, args, history); msg != "" {
a.emit(Event{Type: EventToolCall, Phase: PhaseToolRunning, Round: round, ToolName: tc.Function.Name, ToolArgs: args, Risk: tools.RiskSafe.Display(), Summary: "工具执行中"})
a.emit(Event{Type: EventToolResult, Phase: PhaseObserving, Round: round, ToolName: tc.Function.Name,
Output: msg, Success: true, Duration: "0s", OutputSize: len(msg)})
obs = append(obs, llm.Message{Role: "tool", ToolCallID: tc.ID, Name: tc.Function.Name, Content: msg})
continue
}
risk := a.toolRisk(tool, args)
a.emit(Event{Type: EventStatus, Phase: PhaseSafety, Round: round, ToolName: tc.Function.Name, ToolArgs: args, Risk: risk.Display(), Summary: "安全检查"})
@@ -315,6 +324,16 @@ func (a *Agent) processToolCalls(ctx context.Context, calls []llm.ToolCall, hist
// 执行
a.emit(Event{Type: EventToolCall, Phase: PhaseToolRunning, Round: round, ToolName: tc.Function.Name, ToolArgs: args, Risk: risk.Display(), Summary: "工具执行中"})
if cached, ok := a.cachedToolResult(tc.Function.Name, args); ok {
a.emit(Event{Type: EventToolResult, Phase: PhaseObserving, Round: round, ToolName: tc.Function.Name,
Output: cached, Success: true, Duration: "0s", OutputSize: len(cached)})
records = append(records, evolver.ToolCallRecord{
ToolName: tc.Function.Name, Args: args, Output: cached,
Duration: 0, Success: true, RiskLevel: int(risk),
})
obs = append(obs, llm.Message{Role: "tool", ToolCallID: tc.ID, Name: tc.Function.Name, Content: cached})
continue
}
toolStart := time.Now()
toolCtx, cancel := context.WithTimeout(ctx, a.config.ToolTimeout)
result, execErr := tool.Execute(toolCtx, args)
@@ -346,6 +365,7 @@ func (a *Agent) processToolCalls(ctx context.Context, calls []llm.ToolCall, hist
a.emit(Event{Type: EventToolResult, Phase: PhaseObserving, Round: round, ToolName: tc.Function.Name,
Output: output, Success: success, Duration: dur.Truncate(time.Millisecond).String(), OutputSize: len(output)})
a.storeToolResult(tc.Function.Name, args, output, success)
records = append(records, evolver.ToolCallRecord{
ToolName: tc.Function.Name, Args: args, Output: output,
@@ -369,6 +389,82 @@ func (a *Agent) processToolCalls(ctx context.Context, calls []llm.ToolCall, hist
return obs, records, failInc
}
// cachedToolResult 返回同一任务内只读工具的缓存结果,避免 LLM 重复消耗 token。 // Logan.Li
func (a *Agent) cachedToolResult(toolName string, args map[string]interface{}) (string, bool) {
if !cacheableTool(toolName) {
return "", false
}
key := toolCacheKey(toolName, args)
if key == "" {
return "", false
}
if out, ok := a.cache[key]; ok {
return "已复用本轮任务缓存结果;不要重复查询同一对象,请基于首次结果总结,或改用更具体的 name/status_filter/pod 参数补查。\n\n" + out, true
}
return "", false
}
// storeToolResult 缓存成功的只读工具输出。 // Logan.Li
func (a *Agent) storeToolResult(toolName string, args map[string]interface{}, output string, success bool) {
if !success || !cacheableTool(toolName) {
return
}
key := toolCacheKey(toolName, args)
if key == "" {
return
}
a.cache[key] = output
}
func cacheableTool(toolName string) bool {
switch toolName {
case "kubernetes_diagnose", "kubernetes_get", "pod_logs", "aiops_asset_search", "aiops_alert_list", "aiops_metric_summary":
return true
default:
return false
}
}
func toolCacheKey(toolName string, args map[string]interface{}) string {
b, err := json.Marshal(args)
if err != nil {
return ""
}
return toolName + ":" + string(b)
}
// suppressBroadKubernetesGet 在聚合诊断后拦截全量 K8s 查询,迫使模型按异常对象下钻。 // Logan.Li
func (a *Agent) suppressBroadKubernetesGet(toolName string, args map[string]interface{}, history map[string]int) string {
if a.config.Mode != "kubernetes" && a.config.Mode != "k8s" {
return ""
}
if toolName != "kubernetes_get" || !hasDiagnoseHistory(history) {
return ""
}
resource := strings.ToLower(strings.TrimSpace(fmt.Sprint(args["resource"])))
name := strings.TrimSpace(fmt.Sprint(args["name"]))
labelSelector := strings.TrimSpace(fmt.Sprint(args["label_selector"]))
statusFilter := strings.TrimSpace(fmt.Sprint(args["status_filter"]))
if name != "" && name != "<nil>" || labelSelector != "" && labelSelector != "<nil>" || statusFilter != "" && statusFilter != "<nil>" {
return ""
}
switch resource {
case "pods", "pod", "deployments", "deployment", "deploy", "services", "service", "svc", "nodes", "node", "events", "event":
return "已拦截全量 Kubernetes 查询:本轮已经执行过 kubernetes_diagnose,里面包含异常 Workload、异常 Pod、Top Events 和节点摘要。请不要再次拉全量资源;如需补证据,请按诊断结果中的具体对象调用 kubernetes_get(name/status_filter) 或 pod_logs(pod)。"
default:
return ""
}
}
func hasDiagnoseHistory(history map[string]int) bool {
for key, count := range history {
if count > 0 && strings.HasPrefix(key, "kubernetes_diagnose:") {
return true
}
}
return false
}
// triggerEvolve 异步触发进化(带超时 + panic 恢复,绝不阻塞主循环) // Logan.Li
func (a *Agent) triggerEvolve(ctx context.Context, query, serverHost, answer string, records []evolver.ToolCallRecord, success bool, start time.Time) {
if a.evolver == nil {
@@ -522,34 +618,46 @@ func (a *Agent) trimMessages(msgs []llm.Message) []llm.Message {
return ensureToolMessageIntegrity(result)
}
// ensureToolMessageIntegrity 确保 tool 消息前面有对应 assistant(tool_calls) // Logan.Li
// ensureToolMessageIntegrity 确保 assistant(tool_calls) 后紧跟完整 tool 结果组 // Logan.Li
func ensureToolMessageIntegrity(msgs []llm.Message) []llm.Message {
validIDs := make(map[string]bool)
for _, m := range msgs {
if m.Role == "assistant" {
for _, tc := range m.ToolCalls {
validIDs[tc.ID] = true
}
}
}
out := make([]llm.Message, 0, len(msgs))
for i, m := range msgs {
for i := 0; i < len(msgs); i++ {
m := msgs[i]
if m.Role == "tool" {
if !validIDs[m.ToolCallID] || i == 0 || msgs[i-1].Role != "assistant" {
continue
}
matched := false
for _, tc := range msgs[i-1].ToolCalls {
if tc.ID == m.ToolCallID {
matched = true
break
}
}
if !matched {
continue
continue
}
if m.Role != "assistant" || len(m.ToolCalls) == 0 {
out = append(out, m)
continue
}
required := make(map[string]bool, len(m.ToolCalls))
for _, tc := range m.ToolCalls {
if tc.ID != "" {
required[tc.ID] = true
}
}
out = append(out, m)
if len(required) == 0 {
continue
}
seen := make(map[string]bool, len(required))
tools := make([]llm.Message, 0, len(required))
j := i + 1
for ; j < len(msgs) && msgs[j].Role == "tool"; j++ {
tm := msgs[j]
if !required[tm.ToolCallID] || seen[tm.ToolCallID] {
continue
}
seen[tm.ToolCallID] = true
tools = append(tools, tm)
}
if len(seen) == len(required) {
out = append(out, m)
out = append(out, tools...)
}
i = j - 1
}
return out
}
+98
View File
@@ -0,0 +1,98 @@
package core
import (
"testing"
"ezops/pkg/agentcli/llm"
)
func TestEnsureToolMessageIntegrityKeepsCompleteMultiToolGroup(t *testing.T) {
msgs := []llm.Message{
{Role: "system", Content: "sys"},
{Role: "user", Content: "查看异常副本"},
{Role: "assistant", Content: "并行检查", ToolCalls: []llm.ToolCall{
{ID: "call-1", Type: "function", Function: llm.FunctionCall{Name: "kubernetes_get", Arguments: "{}"}},
{ID: "call-2", Type: "function", Function: llm.FunctionCall{Name: "kubernetes_get", Arguments: "{}"}},
{ID: "call-3", Type: "function", Function: llm.FunctionCall{Name: "kubernetes_get", Arguments: "{}"}},
}},
{Role: "tool", ToolCallID: "call-1", Name: "kubernetes_get", Content: "deployments"},
{Role: "tool", ToolCallID: "call-2", Name: "kubernetes_get", Content: "statefulsets"},
{Role: "tool", ToolCallID: "call-3", Name: "kubernetes_get", Content: "pods"},
}
got := ensureToolMessageIntegrity(msgs)
if len(got) != len(msgs) {
t.Fatalf("完整多工具结果组应全部保留,got=%d want=%d", len(got), len(msgs))
}
for i := 3; i < len(got); i++ {
if got[i].Role != "tool" || got[i].ToolCallID == "" {
t.Fatalf("第 %d 条工具结果异常: %+v", i, got[i])
}
}
}
func TestEnsureToolMessageIntegrityDropsIncompleteToolGroup(t *testing.T) {
msgs := []llm.Message{
{Role: "system", Content: "sys"},
{Role: "assistant", Content: "并行检查", ToolCalls: []llm.ToolCall{
{ID: "call-1", Type: "function", Function: llm.FunctionCall{Name: "kubernetes_get", Arguments: "{}"}},
{ID: "call-2", Type: "function", Function: llm.FunctionCall{Name: "kubernetes_get", Arguments: "{}"}},
}},
{Role: "tool", ToolCallID: "call-1", Name: "kubernetes_get", Content: "deployments"},
{Role: "user", Content: "继续"},
}
got := ensureToolMessageIntegrity(msgs)
if len(got) != 2 {
t.Fatalf("不完整工具结果组应整体丢弃,got=%d want=2: %+v", len(got), got)
}
if got[0].Role != "system" || got[1].Role != "user" {
t.Fatalf("剩余消息异常: %+v", got)
}
}
func TestSuppressBroadKubernetesGetAfterDiagnose(t *testing.T) {
agent := NewAgent(nil, nil, &Config{Mode: "kubernetes"}, nil, nil)
history := map[string]int{`kubernetes_diagnose:{"namespace":"astralx"}`: 1}
msg := agent.suppressBroadKubernetesGet("kubernetes_get", map[string]interface{}{"resource": "pods"}, history)
if msg == "" {
t.Fatal("expected broad pods query to be suppressed after diagnose")
}
}
func TestSuppressBroadKubernetesGetBlocksAllPodAliasesAfterDiagnose(t *testing.T) {
agent := NewAgent(nil, nil, &Config{Mode: "kubernetes"}, nil, nil)
history := map[string]int{`kubernetes_diagnose:{"namespace":"astralx","focus":"all"}`: 1}
for _, resource := range []string{"pods", "pod"} {
msg := agent.suppressBroadKubernetesGet("kubernetes_get", map[string]interface{}{"resource": resource}, history)
if msg == "" {
t.Fatalf("expected broad %s query to be suppressed after diagnose", resource)
}
}
}
func TestSuppressBroadKubernetesGetAllowsFilteredQuery(t *testing.T) {
agent := NewAgent(nil, nil, &Config{Mode: "kubernetes"}, nil, nil)
history := map[string]int{`kubernetes_diagnose:{"namespace":"astralx"}`: 1}
msg := agent.suppressBroadKubernetesGet("kubernetes_get", map[string]interface{}{
"resource": "pods", "status_filter": "NotRunning",
}, history)
if msg != "" {
t.Fatalf("expected filtered pods query to pass, got %q", msg)
}
}
func TestSuppressBroadKubernetesGetAllowsStatusFilteredPodsAfterDiagnose(t *testing.T) {
agent := NewAgent(nil, nil, &Config{Mode: "kubernetes"}, nil, nil)
history := map[string]int{`kubernetes_diagnose:{"namespace":"astralx","focus":"pods"}`: 1}
cases := []map[string]interface{}{
{"resource": "pods", "status_filter": "NotRunning"},
{"resource": "pod", "status_filter": "CrashLoopBackOff"},
}
for _, args := range cases {
msg := agent.suppressBroadKubernetesGet("kubernetes_get", args, history)
if msg != "" {
t.Fatalf("expected status_filter query to pass for args=%v, got %q", args, msg)
}
}
}
+108
View File
@@ -0,0 +1,108 @@
// Package guardrail 提供 Agent CLI 输入输出治理能力。 // Logan.Li
package guardrail
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"regexp"
"strings"
)
var sensitivePatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)(bearer\s+)[a-z0-9._~+/=-]{16,}`),
regexp.MustCompile(`(?i)((?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|secret[_-]?key)\s*[:=]\s*)[^\s'"&]+`),
regexp.MustCompile(`(?i)((?:mysql|postgres|postgresql|redis|mongodb)://[^:\s/@]+:)[^@\s]+(@)`),
regexp.MustCompile(`(?is)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----`),
regexp.MustCompile(`(?i)((?:Cookie|Set-Cookie):\s*)[^\r\n]+`),
regexp.MustCompile(`(?i)(AKIA[0-9A-Z]{16})`),
}
var sensitiveKeys = map[string]bool{
"password": true, "passwd": true, "pwd": true, "secret": true,
"token": true, "api_key": true, "apikey": true, "access_key": true,
"secret_key": true, "private_key": true, "cookie": true,
}
// SanitizeText 脱敏工具输出、LLM 文本和错误信息,避免敏感信息进入前端/DB/记忆。 // Logan.Li
func SanitizeText(input string) string {
if input == "" {
return ""
}
out := input
for _, re := range sensitivePatterns {
out = re.ReplaceAllStringFunc(out, func(match string) string {
if strings.Contains(match, "PRIVATE KEY") {
return "[REDACTED_PRIVATE_KEY]"
}
sub := re.FindStringSubmatch(match)
if len(sub) >= 3 {
return sub[1] + "[REDACTED]" + sub[2]
}
if len(sub) >= 2 {
return sub[1] + "[REDACTED]"
}
return "[REDACTED]"
})
}
return out
}
// SanitizeJSON 脱敏 JSON 文本;非法 JSON 时退回文本脱敏。 // Logan.Li
func SanitizeJSON(raw string) string {
if strings.TrimSpace(raw) == "" {
return raw
}
var v interface{}
if err := json.Unmarshal([]byte(raw), &v); err != nil {
return SanitizeText(raw)
}
v = sanitizeValue(v)
b, err := json.Marshal(v)
if err != nil {
return SanitizeText(raw)
}
return string(b)
}
func sanitizeValue(v interface{}) interface{} {
switch val := v.(type) {
case map[string]interface{}:
out := make(map[string]interface{}, len(val))
for k, item := range val {
if sensitiveKeys[strings.ToLower(strings.ReplaceAll(k, "-", "_"))] {
out[k] = "[REDACTED]"
continue
}
out[k] = sanitizeValue(item)
}
return out
case []interface{}:
for i := range val {
val[i] = sanitizeValue(val[i])
}
return val
case string:
return SanitizeText(val)
default:
return val
}
}
// NewTraceID 生成短 trace id,便于串联一次 Agent 任务。 // Logan.Li
func NewTraceID() string {
return "tr-" + randomHex(12)
}
// NewSpanID 生成短 span id,便于串联工具和消息事件。 // Logan.Li
func NewSpanID() string {
return "sp-" + randomHex(8)
}
func randomHex(n int) string {
buf := make([]byte, n)
if _, err := rand.Read(buf); err != nil {
return "0000000000000000"
}
return hex.EncodeToString(buf)
}
@@ -0,0 +1,31 @@
package guardrail
import (
"strings"
"testing"
)
func TestSanitizeTextMasksCommonSecrets(t *testing.T) {
input := "Authorization: Bearer abcdefghijklmnopqrstuvwxyz password=secret123 mysql://root:pass@db/prod"
got := SanitizeText(input)
for _, leak := range []string{"abcdefghijklmnopqrstuvwxyz", "secret123", "root:pass@db"} {
if strings.Contains(got, leak) {
t.Fatalf("敏感信息未脱敏: %s in %s", leak, got)
}
}
if !strings.Contains(got, "[REDACTED]") {
t.Fatalf("脱敏标记缺失: %s", got)
}
}
func TestSanitizeJSONMasksSensitiveKeys(t *testing.T) {
got := SanitizeJSON(`{"command":"echo ok","password":"p1","nested":{"api_key":"k1"}}`)
for _, leak := range []string{"p1", "k1"} {
if strings.Contains(got, leak) {
t.Fatalf("JSON 敏感字段未脱敏: %s in %s", leak, got)
}
}
if !strings.Contains(got, `"command":"echo ok"`) {
t.Fatalf("非敏感字段不应被改写: %s", got)
}
}
+6 -5
View File
@@ -51,9 +51,10 @@ func BuildSystemPrompt(mode, targetInfo, toolDescs, evolveContext string) string
sb.WriteString("1. 破坏性命令(rm -rf /、mkfs、dd 覆写磁盘等)被严格禁止,绝不执行。\n")
sb.WriteString("2. 高危操作(删除 Pod、重启 Deployment、扩缩容、换镜像等)需要用户确认后才能执行。\n")
if mode == runModeKubernetes {
sb.WriteString("3. 排障时优先使用只读工具:kubernetes_get 查资源状态、pod_logs 看日志;确认根因后再 pod_execute 或 kubernetes_action。\n")
sb.WriteString("4. 容器内可能没有 systemctl、宿主机路径;不要假设节点级命令可用。\n")
sb.WriteString("5. 未选定 Pod 时,用 kubernetes_get 和 pod_logs(带 pod 参数)排障;选定 Pod 后可直接 pod_execute。\n")
sb.WriteString("3. 排障时优先使用只读工具;涉及异常副本、集群健康、Pod 异常、ImagePullBackOff、CrashLoopBackOff、探针失败时,第一步必须调用 kubernetes_diagnose。\n")
sb.WriteString("4. kubernetes_diagnose 已聚合 Workload/Pod/Event/Node,不要重复拉全量 pods/deployments/events;只针对异常对象补查 pod_logs 或 kubernetes_get name。\n")
sb.WriteString("5. 容器内可能没有 systemctl、宿主机路径;不要假设节点级命令可用。\n")
sb.WriteString("6. 未选定 Pod 时,用 kubernetes_diagnose 和必要的 pod_logs(带 pod 参数)排障;选定 Pod 后可直接 pod_execute。\n")
} else {
sb.WriteString("3. 排障时优先使用只读命令(df/ps/cat/grep/tail/systemctl status/journalctl)收集信息,确认根因后再动手修复。\n")
sb.WriteString("4. 不要执行任何你不确定后果的命令;宁可多问用户,也不盲目操作。\n")
@@ -70,7 +71,7 @@ func BuildSystemPrompt(mode, targetInfo, toolDescs, evolveContext string) string
sb.WriteString("你采用「思考-行动-观察」循环解决任务:\n")
sb.WriteString("1. 思考:分析当前状态,决定下一步。在回复中先用简短文字说明你的判断和计划。\n")
if mode == runModeKubernetes {
sb.WriteString("2. 行动:优先 kubernetes_get / pod_logs;需要容器内信息时用 pod_execute;变更用 kubernetes_action(一次一步)。\n")
sb.WriteString("2. 行动:健康巡检/异常副本先用 kubernetes_diagnose;需要细节时对异常对象调用 kubernetes_get / pod_logs;需要容器内信息时用 pod_execute;变更用 kubernetes_action(一次一步)。\n")
sb.WriteString("3. 观察:仔细阅读工具输出;pod_execute 输出含 [exit_code=0] 表示成功。\n")
} else {
sb.WriteString("2. 行动:调用 ssh_execute 执行命令(一次只做关键的、可验证的一步)。\n")
@@ -98,4 +99,4 @@ func FormatToolDescriptions(items []struct{ Name, Desc string }) string {
sb.WriteString(fmt.Sprintf("- %s: %s\n", it.Name, it.Desc))
}
return sb.String()
}
}
@@ -0,0 +1,601 @@
// kubernetes_diagnose.go - Kubernetes 异常副本与健康诊断聚合工具 // Logan.Li
package tools
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
k8sclient "ezops/pkg/kubernetesclient"
)
const defaultDiagnoseLimit = 8
// KubernetesDiagnoseTool 聚合查询异常 Workload、Pod、Event 和节点健康,减少 LLM 反复拉全量资源。 // Logan.Li
type KubernetesDiagnoseTool struct {
client *k8sclient.Client
namespace string
}
// NewKubernetesDiagnoseTool 创建 Kubernetes 聚合诊断工具。 // Logan.Li
func NewKubernetesDiagnoseTool(client *k8sclient.Client, namespace string) *KubernetesDiagnoseTool {
return &KubernetesDiagnoseTool{client: client, namespace: namespace}
}
func (t *KubernetesDiagnoseTool) Name() string { return "kubernetes_diagnose" }
func (t *KubernetesDiagnoseTool) Description() string {
return "一键诊断 Kubernetes 命名空间健康和异常副本(只读)。优先用于“异常副本、集群健康、Pod 异常、Deployment 不可用、ImagePullBackOff、CrashLoopBackOff”等任务。" +
"工具会聚合 Deployment/StatefulSet/DaemonSet/Pod/Event/Node,只返回异常摘要、可能根因和建议补充检查,避免反复查询全量资源。"
}
func (t *KubernetesDiagnoseTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"namespace": map[string]interface{}{
"type": "string",
"description": "命名空间;空则使用会话绑定命名空间",
},
"focus": map[string]interface{}{
"type": "string",
"description": "关注点:replicas|pods|events|image|crash|probe|all,默认 all",
},
"limit": map[string]interface{}{
"type": "integer",
"description": "每类最多返回数量,默认 8,最大 20",
},
},
}
}
func (t *KubernetesDiagnoseTool) RiskLevel() RiskLevel { return RiskSafe }
func (t *KubernetesDiagnoseTool) Execute(ctx context.Context, args map[string]interface{}) (*Result, error) {
ns := strings.TrimSpace(StringParam(args, "namespace"))
if ns == "" {
ns = t.namespace
}
if ns == "" {
return &Result{Success: false, Error: "namespace 不能为空"}, nil
}
limit := IntParam(args, "limit")
if limit <= 0 {
limit = defaultDiagnoseLimit
}
if limit > 20 {
limit = 20
}
focus := strings.ToLower(strings.TrimSpace(StringParam(args, "focus")))
if focus == "" {
focus = "all"
}
report, err := t.buildReport(ns, focus, limit)
if err != nil {
return &Result{Success: false, Error: err.Error(), Output: err.Error()}, nil
}
b, err := json.MarshalIndent(report, "", " ")
if err != nil {
return nil, fmt.Errorf("诊断结果 JSON 序列化失败: %w", err)
}
return &Result{Success: true, Output: string(b)}, nil
}
type k8sDiagnoseReport struct {
Namespace string `json:"namespace"`
Focus string `json:"focus"`
Summary k8sDiagnoseSummary `json:"summary"`
AbnormalWorkloads []k8sWorkloadFinding `json:"abnormal_workloads"`
AbnormalPods []k8sPodFinding `json:"abnormal_pods"`
TopEvents []k8sEventFinding `json:"top_events"`
NodeHealth []k8sNodeFinding `json:"node_health,omitempty"`
LikelyRootCauses []string `json:"likely_root_causes"`
RecommendedActions []string `json:"recommended_actions"`
NextTools []map[string]interface{} `json:"next_tools"`
}
type k8sDiagnoseSummary struct {
Deployments int `json:"deployments"`
StatefulSets int `json:"statefulsets"`
DaemonSets int `json:"daemonsets"`
Pods int `json:"pods"`
AbnormalWorkloads int `json:"abnormal_workloads"`
AbnormalPods int `json:"abnormal_pods"`
WarningEvents int `json:"warning_events"`
NotReadyNodes int `json:"not_ready_nodes"`
}
type k8sWorkloadFinding struct {
Kind string `json:"kind"`
Namespace string `json:"namespace"`
Name string `json:"name"`
Desired int `json:"desired"`
Ready int `json:"ready"`
Available int `json:"available,omitempty"`
Reason string `json:"reason"`
Image string `json:"image,omitempty"`
}
type k8sPodFinding struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Status string `json:"status"`
Reason string `json:"reason"`
Restarts int `json:"restarts"`
Node string `json:"node"`
IP string `json:"ip,omitempty"`
Image string `json:"image,omitempty"`
Owner string `json:"owner,omitempty"`
Messages []string `json:"messages,omitempty"`
}
type k8sEventFinding struct {
Type string `json:"type"`
Reason string `json:"reason"`
Object string `json:"object"`
Count int32 `json:"count"`
LastSeen string `json:"last_seen,omitempty"`
Message string `json:"message"`
}
type k8sNodeFinding struct {
Name string `json:"name"`
Ready string `json:"ready"`
Reason string `json:"reason,omitempty"`
Message string `json:"message,omitempty"`
}
func (t *KubernetesDiagnoseTool) buildReport(ns, focus string, limit int) (*k8sDiagnoseReport, error) {
pods, err := t.client.ListPods()
if err != nil {
return nil, fmt.Errorf("获取 Pod 失败: %w", err)
}
deployments, err := t.client.ListDeployments()
if err != nil {
return nil, fmt.Errorf("获取 Deployment 失败: %w", err)
}
statefulsets, err := t.client.ListStatefulSets()
if err != nil {
return nil, fmt.Errorf("获取 StatefulSet 失败: %w", err)
}
daemonsets, err := t.client.ListDaemonSets()
if err != nil {
return nil, fmt.Errorf("获取 DaemonSet 失败: %w", err)
}
events, err := t.client.ListEvents(ns)
if err != nil {
return nil, fmt.Errorf("获取 Event 失败: %w", err)
}
nodes, _ := t.client.ListNodes()
report := &k8sDiagnoseReport{Namespace: ns, Focus: focus}
report.AbnormalWorkloads = diagnoseWorkloads(ns, deployments.Items, statefulsets.Items, daemonsets.Items)
report.AbnormalPods = diagnosePods(ns, pods.Items, deployments.Items)
report.TopEvents = diagnoseEvents(events.Items, limit)
report.NodeHealth = diagnoseNodes(nodes)
report.Summary = k8sDiagnoseSummary{
Deployments: countDeployments(ns, deployments.Items), StatefulSets: countStatefulSets(ns, statefulsets.Items),
DaemonSets: countDaemonSets(ns, daemonsets.Items), Pods: countPods(ns, pods.Items),
AbnormalWorkloads: len(report.AbnormalWorkloads), AbnormalPods: len(report.AbnormalPods),
WarningEvents: countWarningEvents(events.Items), NotReadyNodes: len(report.NodeHealth),
}
report.LikelyRootCauses = inferK8sRootCauses(report.AbnormalPods, report.TopEvents)
report.RecommendedActions = recommendK8sActions(report.AbnormalPods, report.AbnormalWorkloads)
report.NextTools = recommendK8sNextTools(report.AbnormalPods, report.TopEvents, limit)
report.AbnormalWorkloads = limitWorkloadFindings(report.AbnormalWorkloads, limit)
report.AbnormalPods = limitPodFindings(report.AbnormalPods, limit)
return report, nil
}
func diagnoseWorkloads(ns string, deployments []k8sclient.DeploymentItem, statefulsets []k8sclient.StatefulSetItem, daemonsets []k8sclient.DaemonSetItem) []k8sWorkloadFinding {
var out []k8sWorkloadFinding
for _, d := range deployments {
if d.Metadata.Namespace != ns || d.Spec.Replicas == 0 {
continue
}
if d.Status.ReadyReplicas < d.Spec.Replicas || d.Status.AvailableReplicas < d.Spec.Replicas {
out = append(out, k8sWorkloadFinding{
Kind: "Deployment", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name,
Desired: d.Spec.Replicas, Ready: d.Status.ReadyReplicas, Available: d.Status.AvailableReplicas,
Reason: "ready/available replicas less than desired", Image: firstDeploymentImage(d),
})
}
}
for _, s := range statefulsets {
if s.Metadata.Namespace != ns || s.Spec.Replicas == 0 {
continue
}
if s.Status.ReadyReplicas < s.Spec.Replicas {
out = append(out, k8sWorkloadFinding{
Kind: "StatefulSet", Namespace: s.Metadata.Namespace, Name: s.Metadata.Name,
Desired: s.Spec.Replicas, Ready: s.Status.ReadyReplicas,
Reason: "ready replicas less than desired", Image: firstStatefulSetImage(s),
})
}
}
for _, d := range daemonsets {
if d.Metadata.Namespace != ns || d.Status.DesiredNumberScheduled == 0 {
continue
}
if d.Status.NumberReady < d.Status.DesiredNumberScheduled {
out = append(out, k8sWorkloadFinding{
Kind: "DaemonSet", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name,
Desired: d.Status.DesiredNumberScheduled, Ready: d.Status.NumberReady,
Reason: "ready pods less than desired scheduled pods", Image: firstDaemonSetImage(d),
})
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Kind == out[j].Kind {
return out[i].Name < out[j].Name
}
return out[i].Kind < out[j].Kind
})
return out
}
func diagnosePods(ns string, pods []k8sclient.PodItem, deployments []k8sclient.DeploymentItem) []k8sPodFinding {
var out []k8sPodFinding
for _, p := range pods {
if p.Metadata.Namespace != ns {
continue
}
status, reason, restarts, messages := podDiagnosisStatus(p)
if !isAbnormalPod(status, reason, restarts) {
continue
}
out = append(out, k8sPodFinding{
Namespace: p.Metadata.Namespace, Name: p.Metadata.Name, Status: status, Reason: reason,
Restarts: restarts, Node: p.Spec.NodeName, IP: p.Status.PodIP, Image: firstPodImage(p),
Owner: inferDeploymentOwner(p, deployments), Messages: messages,
})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Reason == out[j].Reason {
return out[i].Name < out[j].Name
}
return podReasonRank(out[i].Reason) < podReasonRank(out[j].Reason)
})
return out
}
func podDiagnosisStatus(p k8sclient.PodItem) (string, string, int, []string) {
status := p.Status.Phase
reason := ""
restarts := 0
var messages []string
for _, cs := range p.Status.ContainerStatuses {
if cs.RestartCount > restarts {
restarts = cs.RestartCount
}
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" {
status = cs.State.Waiting.Reason
reason = cs.State.Waiting.Reason
if cs.State.Waiting.Message != "" {
messages = append(messages, truncateK8sText(cs.State.Waiting.Message, 220))
}
}
if cs.LastState.Terminated != nil && cs.LastState.Terminated.Reason != "" {
if reason == "" {
reason = cs.LastState.Terminated.Reason
}
msg := fmt.Sprintf("last terminated: %s exit=%d", cs.LastState.Terminated.Reason, cs.LastState.Terminated.ExitCode)
messages = append(messages, msg)
}
}
for _, c := range p.Status.Conditions {
if c.Type == "Ready" && c.Status != "True" {
if reason == "" && c.Reason != "" {
reason = c.Reason
}
if c.Message != "" {
messages = append(messages, truncateK8sText(c.Message, 220))
}
}
}
if reason == "" {
reason = status
}
return status, reason, restarts, dedupeStrings(messages)
}
func diagnoseEvents(events []k8sclient.EventItem, limit int) []k8sEventFinding {
sort.Slice(events, func(i, j int) bool {
return eventTime(events[i]).After(eventTime(events[j]))
})
var out []k8sEventFinding
seen := make(map[string]bool)
for _, e := range events {
if e.Type != "Warning" && !isImportantEventReason(e.Reason) {
continue
}
key := fmt.Sprintf("%s/%s/%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message)
if seen[key] {
continue
}
seen[key] = true
out = append(out, k8sEventFinding{
Type: e.Type, Reason: e.Reason,
Object: fmt.Sprintf("%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name),
Count: e.Count, LastSeen: eventTime(e).Format(time.RFC3339),
Message: truncateK8sText(e.Message, 260),
})
if len(out) >= limit {
break
}
}
return out
}
func diagnoseNodes(nodes *k8sclient.NodeList) []k8sNodeFinding {
if nodes == nil {
return nil
}
var out []k8sNodeFinding
for _, n := range nodes.Items {
for _, c := range n.Status.Conditions {
if c.Type == "Ready" && c.Status != "True" {
out = append(out, k8sNodeFinding{Name: n.Metadata.Name, Ready: c.Status, Reason: c.Reason, Message: truncateK8sText(c.Message, 220)})
break
}
}
}
return out
}
func inferK8sRootCauses(pods []k8sPodFinding, events []k8sEventFinding) []string {
flags := make(map[string]bool)
for _, p := range pods {
r := strings.ToLower(p.Reason + " " + p.Status)
switch {
case strings.Contains(r, "imagepull") || strings.Contains(r, "errimagepull"):
flags["存在镜像拉取失败,优先检查镜像 tag、Harbor 仓库、imagePullSecrets 和节点到仓库网络。"] = true
case strings.Contains(r, "crashloop"):
flags["存在容器反复崩溃,优先查看 previous logs、启动参数、依赖连接和最近变更。"] = true
case strings.Contains(r, "oom"):
flags["存在 OOMKilled 风险,优先检查内存 limit、最近内存趋势和应用内存泄漏。"] = true
case strings.Contains(r, "containersnotready") || strings.Contains(r, "unready"):
flags["存在容器未就绪,需结合 readiness/liveness probe、事件和应用日志确认。"] = true
case strings.Contains(r, "pending") || strings.Contains(r, "failedscheduling"):
flags["存在调度失败或 Pending,优先检查节点资源、污点、亲和性和 PVC。"] = true
}
}
for _, e := range events {
msg := strings.ToLower(e.Reason + " " + e.Message)
if strings.Contains(msg, "liveness probe") || strings.Contains(msg, "readiness probe") {
flags["事件显示健康探针失败,需检查探针路径、超时时间、应用响应耗时和依赖服务。"] = true
}
}
return sortedMapKeys(flags)
}
func recommendK8sActions(pods []k8sPodFinding, workloads []k8sWorkloadFinding) []string {
var out []string
if len(workloads) > 0 {
out = append(out, "先处理 ready/available 副本不足的 Workload,再回查副本是否恢复。")
}
if hasPodReason(pods, "ImagePullBackOff") || hasPodReason(pods, "ErrImagePull") {
out = append(out, "对镜像拉取失败的 Pod,确认镜像 tag 是否存在、仓库权限是否有效、节点能否访问镜像仓库。")
}
if hasPodReason(pods, "CrashLoopBackOff") {
out = append(out, "对 CrashLoopBackOff 的 Pod,补查 previous logs 和启动命令,避免只看当前空日志。")
}
if hasPodReason(pods, "ContainersNotReady") {
out = append(out, "对未就绪 Pod,查看事件中的 readiness/liveness probe 失败原因和应用健康端点耗时。")
}
if len(out) == 0 {
out = append(out, "当前聚合诊断未发现明显异常;如仍有业务问题,继续按服务名查看日志、Ingress、Service selector 和指标。")
}
return out
}
func recommendK8sNextTools(pods []k8sPodFinding, events []k8sEventFinding, limit int) []map[string]interface{} {
var out []map[string]interface{}
for _, p := range pods {
if len(out) >= limit {
break
}
if strings.Contains(strings.ToLower(p.Reason), "imagepull") {
out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events", "name": p.Name}})
continue
}
if p.Status == "CrashLoopBackOff" || p.Restarts >= 5 || strings.Contains(strings.ToLower(p.Reason), "notready") {
out = append(out, map[string]interface{}{"tool": "pod_logs", "args": map[string]interface{}{"namespace": p.Namespace, "pod": p.Name, "tail": 120}})
}
}
if len(out) == 0 && len(events) > 0 {
out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events"}})
}
return out
}
func isAbnormalPod(status, reason string, restarts int) bool {
s := strings.ToLower(status + " " + reason)
if status != "Running" && status != "Succeeded" && status != "PodCompleted" && status != "Completed" {
return true
}
if strings.Contains(s, "notready") || strings.Contains(s, "crashloop") || strings.Contains(s, "imagepull") || strings.Contains(s, "oom") {
return true
}
return restarts >= 10
}
func podReasonRank(reason string) int {
r := strings.ToLower(reason)
switch {
case strings.Contains(r, "imagepull"):
return 1
case strings.Contains(r, "crashloop"):
return 2
case strings.Contains(r, "oom"):
return 3
case strings.Contains(r, "notready"):
return 4
default:
return 9
}
}
func inferDeploymentOwner(p k8sclient.PodItem, deployments []k8sclient.DeploymentItem) string {
for _, d := range deployments {
if d.Metadata.Namespace != p.Metadata.Namespace || d.Spec.Selector == nil {
continue
}
if labelsMatch(p.Metadata.Labels, d.Spec.Selector.MatchLabels) {
return "Deployment/" + d.Metadata.Name
}
}
return ""
}
func firstDeploymentImage(d k8sclient.DeploymentItem) string {
if len(d.Spec.Template.Spec.Containers) == 0 {
return ""
}
return d.Spec.Template.Spec.Containers[0].Image
}
func firstStatefulSetImage(s k8sclient.StatefulSetItem) string {
if len(s.Spec.Template.Spec.Containers) == 0 {
return ""
}
return s.Spec.Template.Spec.Containers[0].Image
}
func firstDaemonSetImage(d k8sclient.DaemonSetItem) string {
if len(d.Spec.Template.Spec.Containers) == 0 {
return ""
}
return d.Spec.Template.Spec.Containers[0].Image
}
func firstPodImage(p k8sclient.PodItem) string {
if len(p.Spec.Containers) == 0 {
return ""
}
return p.Spec.Containers[0].Image
}
func eventTime(e k8sclient.EventItem) time.Time {
if !e.LastTimestamp.IsZero() {
return e.LastTimestamp
}
if !e.EventTime.IsZero() {
return e.EventTime
}
return e.FirstTimestamp
}
func isImportantEventReason(reason string) bool {
r := strings.ToLower(reason)
return strings.Contains(r, "failed") || strings.Contains(r, "backoff") || strings.Contains(r, "unhealthy")
}
func countDeployments(ns string, items []k8sclient.DeploymentItem) int {
n := 0
for _, it := range items {
if it.Metadata.Namespace == ns {
n++
}
}
return n
}
func countStatefulSets(ns string, items []k8sclient.StatefulSetItem) int {
n := 0
for _, it := range items {
if it.Metadata.Namespace == ns {
n++
}
}
return n
}
func countDaemonSets(ns string, items []k8sclient.DaemonSetItem) int {
n := 0
for _, it := range items {
if it.Metadata.Namespace == ns {
n++
}
}
return n
}
func countPods(ns string, items []k8sclient.PodItem) int {
n := 0
for _, it := range items {
if it.Metadata.Namespace == ns {
n++
}
}
return n
}
func countWarningEvents(items []k8sclient.EventItem) int {
n := 0
for _, it := range items {
if it.Type == "Warning" {
n++
}
}
return n
}
func hasPodReason(pods []k8sPodFinding, reason string) bool {
needle := strings.ToLower(reason)
for _, p := range pods {
if strings.Contains(strings.ToLower(p.Reason+" "+p.Status), needle) {
return true
}
}
return false
}
func limitWorkloadFindings(items []k8sWorkloadFinding, limit int) []k8sWorkloadFinding {
if len(items) <= limit {
return items
}
return items[:limit]
}
func limitPodFindings(items []k8sPodFinding, limit int) []k8sPodFinding {
if len(items) <= limit {
return items
}
return items[:limit]
}
func truncateK8sText(s string, max int) string {
s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
if len([]rune(s)) <= max {
return s
}
r := []rune(s)
return string(r[:max]) + "..."
}
func dedupeStrings(items []string) []string {
seen := make(map[string]bool)
var out []string
for _, item := range items {
if item == "" || seen[item] {
continue
}
seen[item] = true
out = append(out, item)
}
return out
}
func sortedMapKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -0,0 +1,64 @@
package tools
import (
"testing"
k8sclient "ezops/pkg/kubernetesclient"
)
func TestPodDiagnosisStatusDetectsWaitingReason(t *testing.T) {
pod := k8sclient.PodItem{
Status: k8sclient.PodStatus{
Phase: "Pending",
ContainerStatuses: []k8sclient.ContainerStatus{{
Name: "app",
State: k8sclient.ContainerStatusState{Waiting: &k8sclient.ContainerStateWaiting{
Reason: "ImagePullBackOff", Message: "Back-off pulling image",
}},
}},
},
}
status, reason, restarts, messages := podDiagnosisStatus(pod)
if status != "ImagePullBackOff" || reason != "ImagePullBackOff" {
t.Fatalf("unexpected status=%s reason=%s", status, reason)
}
if restarts != 0 {
t.Fatalf("unexpected restarts=%d", restarts)
}
if len(messages) != 1 {
t.Fatalf("expected waiting message, got %v", messages)
}
if !isAbnormalPod(status, reason, restarts) {
t.Fatal("ImagePullBackOff should be abnormal")
}
}
func TestPodDiagnosisStatusDetectsOOMKilled(t *testing.T) {
pod := k8sclient.PodItem{
Status: k8sclient.PodStatus{
Phase: "Running",
ContainerStatuses: []k8sclient.ContainerStatus{{
Name: "app", Ready: true, RestartCount: 3,
LastState: k8sclient.ContainerStatusState{Terminated: &k8sclient.ContainerStateTerminated{
Reason: "OOMKilled", ExitCode: 137,
}},
}},
},
}
status, reason, restarts, _ := podDiagnosisStatus(pod)
if status != "Running" || reason != "OOMKilled" || restarts != 3 {
t.Fatalf("unexpected status=%s reason=%s restarts=%d", status, reason, restarts)
}
if !isAbnormalPod(status, reason, restarts) {
t.Fatal("OOMKilled should be abnormal")
}
}
func TestIsAbnormalPodByRestartThreshold(t *testing.T) {
if !isAbnormalPod("Running", "Running", 10) {
t.Fatal("high restart pod should be abnormal")
}
if isAbnormalPod("Running", "Running", 2) {
t.Fatal("healthy running pod should not be abnormal")
}
}
+38 -11
View File
@@ -96,7 +96,6 @@ func AutoMigrate(db *gorm.DB) error {
&model.AlertNotifyPolicy{}, // 告警集合通知策略 // Logan.Li
&model.AlertNotifySettings{}, // 告警消息模版 // Logan.Li
&model.NotifyChannel{},
&model.MetricData{},
&model.Pipeline{},
&model.PipelineRun{},
&model.BuildNode{},
@@ -121,11 +120,10 @@ func AutoMigrate(db *gorm.DB) error {
&model.AgentTask{},
&model.AgentRule{},
&model.AgentAlert{},
&model.AgentPortPolicy{}, // Agent 端口策略 // Logan.Li
&model.AnsiblePlaybook{},
&model.AnsibleJob{},
&model.AnsibleInventory{},
&model.LogSource{},
&model.LogEntry{},
&model.TopologyLayout{}, // 拓扑布局保存 // Logan.Li
&model.TerminalSession{}, // 终端会话录制(终端审计) // Logan.Li
&model.RDPSession{}, // RDP 图形会话审计 // Logan.Li
@@ -136,7 +134,6 @@ func AutoMigrate(db *gorm.DB) error {
&model.MetricsAggregate{}, // 指标时序聚合(Redis → MySQL)// Logan.Li
&model.MonitorTarget{}, // 监控目标(主机/中间件/设备/站点) // Logan.Li
&model.MonitorMetric{}, // 监控指标定义 // Logan.Li
&model.MetricValue{}, // 指标当前值 // Logan.Li
&model.MetricHistory{}, // 指标历史时序 // Logan.Li
&model.HostGroup{}, // 主机组 // Logan.Li
&model.SNMPDevice{}, // SNMP设备 // Logan.Li
@@ -152,8 +149,6 @@ func AutoMigrate(db *gorm.DB) error {
&model.ZabbixDataSource{}, // Zabbix 数据源配置 // Logan.Li
&model.LogDataSource{}, // Loki / ES 日志数据源 // Logan.Li
&model.Credential{}, // 凭据管理(SSH密钥/API Token) // Logan.Li
&model.PipelineRepo{}, // Git 仓库认证 // Logan.Li
&model.PipelineRegistry{}, // 镜像仓库认证 // Logan.Li
&model.DeployEnvironment{}, // 发布环境分组 // Logan.Li
&model.DeployPipelineMember{}, // 流水线成员 // Logan.Li
&model.DeployUserGroup{}, // 用户组 // Logan.Li
@@ -172,9 +167,13 @@ func AutoMigrate(db *gorm.DB) error {
&model.HealthProbe{}, // 健康探针配置 // Logan.Li
&model.ApprovalWorkflow{}, // 审批流程定义 // Logan.Li
&model.ApprovalWorkflowStep{}, // 审批流程步骤 // Logan.Li
&model.AgentCLISession{}, // Agent CLI 任务会话 // Logan.Li
&model.AgentCLIMessage{}, // Agent CLI 推理消息 // Logan.Li
&model.CustomDashboard{}, // 自定义监控仪表盘 // Logan.Li
&model.AIRun{}, // AI 任务统一运行记录 // Logan.Li
&model.AIRunMessage{}, // AI 任务回放消息 // Logan.Li
&model.AIRunSpan{}, // AI 工具/LLM/Guardrail Trace Span // Logan.Li
&model.AICapability{}, // AI 内置/MCP/技能能力注册 // Logan.Li
&model.AICapabilityPolicy{}, // AI 工具能力策略 // Logan.Li
&model.OpsSession{}, // 人机统一执行会话 // Logan.Li
&model.OpsCommand{}, // 人机统一命令审计 // Logan.Li
&model.TraceDataSource{}, // 链路追踪数据源 Jaeger/Tempo // Logan.Li
}
@@ -231,8 +230,10 @@ func applySchemaPatches(db *gorm.DB) {
}
}
}
ensureTable(&model.LogScopeRole{}, "monitor_log_scope_roles")
ensureTable(&model.UserLogScopeRole{}, "monitor_user_log_scope_roles")
ensureTable(&model.LogScopeRole{}, "iam_log_scope_roles")
if !m.HasTable("iam_user_log_scope_roles") {
createUserLogScopeRolesTable(db)
}
// LDAP 目录类型列(openldap | ad) // Logan.Li
if m.HasTable(&model.LDAPConfig{}) && !m.HasColumn(&model.LDAPConfig{}, "server_type") {
@@ -252,6 +253,9 @@ func applySchemaPatches(db *gorm.DB) {
patchServerColumn("OSType", "os_type")
patchServerColumn("Protocol", "protocol")
patchServerColumn("RDPPort", "rdp_port")
patchServerColumn("Alias", "alias")
patchServerColumn("CloudInstanceName", "cloud_instance_name")
patchServerColumn("CloudInstanceID", "cloud_instance_id")
// 将历史仅填 os=windows 的记录补齐协议字段 // Logan.Li
if m.HasColumn(&model.Server{}, "os_type") && m.HasColumn(&model.Server{}, "protocol") {
@@ -263,3 +267,26 @@ func applySchemaPatches(db *gorm.DB) {
`).Error
}
}
func createUserLogScopeRolesTable(db *gorm.DB) {
sql := `CREATE TABLE IF NOT EXISTS iam_user_log_scope_roles (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
user_id BIGINT UNSIGNED NOT NULL,
log_scope_role_id BIGINT UNSIGNED NOT NULL,
created_at DATETIME(3) NULL,
UNIQUE KEY idx_iam_user_log_scope_role (user_id, log_scope_role_id),
KEY idx_iam_user_log_scope_role_role (log_scope_role_id)
)`
if db.Dialector.Name() == "sqlite" {
sql = `CREATE TABLE IF NOT EXISTS iam_user_log_scope_roles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
log_scope_role_id INTEGER NOT NULL,
created_at DATETIME,
UNIQUE(user_id, log_scope_role_id)
)`
}
if err := db.Exec(sql).Error; err != nil {
log.Printf("[DB] 创建 iam_user_log_scope_roles 失败: %v", err)
}
}
+3 -3
View File
@@ -51,13 +51,13 @@ func CleanupSNMPMetricsSQL(driver string, retentionDays int) string {
switch driver {
case "sqlite":
return fmt.Sprintf(
"DELETE FROM ops_snmp_metrics WHERE timestamp < CAST(strftime('%%s', 'now', '-%d days') AS INTEGER)",
"DELETE FROM monitor_snmp_samples WHERE timestamp < CAST(strftime('%%s', 'now', '-%d days') AS INTEGER)",
retentionDays,
)
default:
return fmt.Sprintf(
"DELETE FROM ops_snmp_metrics WHERE timestamp < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL %d DAY))",
"DELETE FROM monitor_snmp_samples WHERE timestamp < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL %d DAY))",
retentionDays,
)
}
}
}
+149 -113
View File
@@ -203,16 +203,22 @@ func (c *Client) get(path string, result interface{}) error {
// getRaw performs a GET request returning raw bytes with custom Accept header
func (c *Client) getRaw(path string, accept string) ([]byte, error) {
req, err := http.NewRequest("GET", c.BaseURL+path, nil)
if err != nil { return nil, err }
if err != nil {
return nil, err
}
req.Header.Set("Accept", accept)
if c.BearerToken != "" {
req.Header.Set("Authorization", "Bearer "+c.BearerToken)
}
resp, err := c.HTTPClient.Do(req)
if err != nil { return nil, fmt.Errorf("GET %s: %w", path, err) }
if err != nil {
return nil, fmt.Errorf("GET %s: %w", path, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil { return nil, fmt.Errorf("read body: %w", err) }
if err != nil {
return nil, fmt.Errorf("read body: %w", err)
}
if resp.StatusCode >= 400 {
return nil, fmt.Errorf("K8s API error %d: %s", resp.StatusCode, string(body[:min(len(body), 200)]))
}
@@ -222,13 +228,17 @@ func (c *Client) getRaw(path string, accept string) ([]byte, error) {
// putRaw performs a PUT request with raw body and custom Content-Type
func (c *Client) putRaw(path string, contentType string, data []byte) error {
req, err := http.NewRequest("PUT", c.BaseURL+path, strings.NewReader(string(data)))
if err != nil { return err }
if err != nil {
return err
}
req.Header.Set("Content-Type", contentType)
if c.BearerToken != "" {
req.Header.Set("Authorization", "Bearer "+c.BearerToken)
}
resp, err := c.HTTPClient.Do(req)
if err != nil { return fmt.Errorf("PUT %s: %w", path, err) }
if err != nil {
return fmt.Errorf("PUT %s: %w", path, err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(resp.Body)
@@ -245,9 +255,9 @@ type NodeList struct {
}
type NodeItem struct {
Metadata ObjectMeta `json:"metadata"`
Status NodeStatus `json:"status"`
Spec NodeSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Status NodeStatus `json:"status"`
Spec NodeSpec `json:"spec"`
}
type NodeSpec struct {
@@ -263,11 +273,11 @@ type NodeTaint struct {
}
type NodeStatus struct {
Capacity ResourceList `json:"capacity"`
Allocatable ResourceList `json:"allocatable"`
Capacity ResourceList `json:"capacity"`
Allocatable ResourceList `json:"allocatable"`
Conditions []NodeCondition `json:"conditions"`
NodeInfo NodeInfo `json:"nodeInfo"`
Addresses []NodeAddress `json:"addresses"`
NodeInfo NodeInfo `json:"nodeInfo"`
Addresses []NodeAddress `json:"addresses"`
}
type ResourceList map[string]string
@@ -305,14 +315,14 @@ type PodItem struct {
}
type PodSpec struct {
NodeName string `json:"nodeName"`
NodeName string `json:"nodeName"`
Containers []Container `json:"containers"`
}
type Container struct {
Name string `json:"name"`
Image string `json:"image"`
Resources ContainerRes `json:"resources"`
Name string `json:"name"`
Image string `json:"image"`
Resources ContainerRes `json:"resources"`
}
type ContainerRes struct {
@@ -321,10 +331,10 @@ type ContainerRes struct {
}
type PodStatus struct {
Phase string `json:"phase"`
PodIP string `json:"podIP"`
ContainerStatuses []ContainerStatus `json:"containerStatuses"`
Conditions []PodCondition `json:"conditions,omitempty"`
Phase string `json:"phase"`
PodIP string `json:"podIP"`
ContainerStatuses []ContainerStatus `json:"containerStatuses"`
Conditions []PodCondition `json:"conditions,omitempty"`
}
// PodCondition for pod status conditions (Scheduled, Initialized, Ready etc)
@@ -337,8 +347,30 @@ type PodCondition struct {
}
type ContainerStatus struct {
RestartCount int `json:"restartCount"`
Ready bool `json:"ready"`
Name string `json:"name"`
RestartCount int `json:"restartCount"`
Ready bool `json:"ready"`
State ContainerStatusState `json:"state,omitempty"`
LastState ContainerStatusState `json:"lastState,omitempty"`
}
// ContainerStatusState 解析容器等待/终止状态,用于 AI 诊断 ImagePullBackOff、OOMKilled 等原因。 // Logan.Li
type ContainerStatusState struct {
Waiting *ContainerStateWaiting `json:"waiting,omitempty"`
Terminated *ContainerStateTerminated `json:"terminated,omitempty"`
}
// ContainerStateWaiting 容器等待状态。 // Logan.Li
type ContainerStateWaiting struct {
Reason string `json:"reason,omitempty"`
Message string `json:"message,omitempty"`
}
// ContainerStateTerminated 容器终止状态。 // Logan.Li
type ContainerStateTerminated struct {
Reason string `json:"reason,omitempty"`
Message string `json:"message,omitempty"`
ExitCode int `json:"exitCode,omitempty"`
}
// DeploymentList response
@@ -380,8 +412,8 @@ type ServiceList struct {
}
type ServiceItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec ServiceSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Spec ServiceSpec `json:"spec"`
}
type ServiceSpec struct {
@@ -392,11 +424,11 @@ type ServiceSpec struct {
}
type ServicePort struct {
Name string `json:"name"`
Port int `json:"port"`
Name string `json:"name"`
Port int `json:"port"`
TargetPort interface{} `json:"targetPort"`
NodePort int `json:"nodePort"`
Protocol string `json:"protocol"`
NodePort int `json:"nodePort"`
Protocol string `json:"protocol"`
}
// IngressList response
@@ -405,13 +437,13 @@ type IngressList struct {
}
type IngressItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec IngressSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Spec IngressSpec `json:"spec"`
}
type IngressSpec struct {
TLS []IngressTLS `json:"tls"`
Rules []IngressRule `json:"rules"`
TLS []IngressTLS `json:"tls"`
Rules []IngressRule `json:"rules"`
}
type IngressTLS struct {
@@ -419,7 +451,7 @@ type IngressTLS struct {
}
type IngressRule struct {
Host string `json:"host"`
Host string `json:"host"`
HTTP *IngressHTTP `json:"http,omitempty"`
}
@@ -428,8 +460,8 @@ type IngressHTTP struct {
}
type IngressPath struct {
Path string `json:"path"`
PathType string `json:"pathType"`
Path string `json:"path"`
PathType string `json:"pathType"`
Backend IngressBackend `json:"backend"`
}
@@ -438,7 +470,7 @@ type IngressBackend struct {
}
type IngressBackendService struct {
Name string `json:"name"`
Name string `json:"name"`
Port IngressServicePort `json:"port"`
}
@@ -461,20 +493,20 @@ type IngressClassItem struct {
// ObjectMeta
type ObjectMeta struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
CreationTimestamp string `json:"creationTimestamp"`
Name string `json:"name"`
Namespace string `json:"namespace"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
CreationTimestamp string `json:"creationTimestamp"`
}
// SyncResult holds sync counts
type SyncResult struct {
Nodes int `json:"nodes"`
Pods int `json:"pods"`
Deployments int `json:"deployments"`
Services int `json:"services"`
Ingresses int `json:"ingresses"`
Nodes int `json:"nodes"`
Pods int `json:"pods"`
Deployments int `json:"deployments"`
Services int `json:"services"`
Ingresses int `json:"ingresses"`
Errors []string `json:"errors,omitempty"` // 各资源同步错误,便于排查 // Logan.Li
}
@@ -812,8 +844,8 @@ func (c *Client) ExecInPod(namespace, podName, container string, command []strin
url.PathEscape(namespace), url.PathEscape(podName), params.Encode())
dialer := websocket.Dialer{
TLSClientConfig: c.TLSConfig,
Subprotocols: []string{"v4.channel.k8s.io", "v3.channel.k8s.io", "channel.k8s.io"},
TLSClientConfig: c.TLSConfig,
Subprotocols: []string{"v4.channel.k8s.io", "v3.channel.k8s.io", "channel.k8s.io"},
HandshakeTimeout: 10 * time.Second,
}
header := http.Header{}
@@ -863,9 +895,9 @@ type HPAList struct {
// HPAItem represents a HorizontalPodAutoscaler
type HPAItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec HPASpec `json:"spec"`
Status HPAStatus `json:"status"`
Metadata ObjectMeta `json:"metadata"`
Spec HPASpec `json:"spec"`
Status HPAStatus `json:"status"`
}
// HPASpec defines the HPA spec
@@ -885,7 +917,7 @@ type ScaleTargetRef struct {
// HPAMetric defines a metric for HPA
type HPAMetric struct {
Type string `json:"type"`
Type string `json:"type"`
Resource *struct {
Name string `json:"name"`
Target struct {
@@ -914,12 +946,12 @@ func (c *Client) ListHPAs() (*HPAList, error) {
type CreateHPARequest struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
TargetKind string `json:"targetKind"` // Deployment, StatefulSet
TargetKind string `json:"targetKind"` // Deployment, StatefulSet
TargetName string `json:"targetName"`
MinReplicas int `json:"minReplicas"`
MaxReplicas int `json:"maxReplicas"`
CPUPercent int `json:"cpuPercent"` // target average CPU utilization
MemPercent int `json:"memPercent"` // target average memory utilization (0 = disabled)
CPUPercent int `json:"cpuPercent"` // target average CPU utilization
MemPercent int `json:"memPercent"` // target average memory utilization (0 = disabled)
}
// CreateHPA creates a new HPA
@@ -1004,9 +1036,9 @@ type CronJobList struct {
// CronJobItem represents a CronJob
type CronJobItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec CronJobSpec `json:"spec"`
Status CronJobStatus `json:"status"`
Metadata ObjectMeta `json:"metadata"`
Spec CronJobSpec `json:"spec"`
Status CronJobStatus `json:"status"`
}
// CronJobSpec defines the CronJob spec
@@ -1042,10 +1074,10 @@ func (c *Client) ListCronJobs() (*CronJobList, error) {
// CreateCronJobRequest is the payload for creating a CronJob
type CreateCronJobRequest struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Schedule string `json:"schedule"` // cron expression
Image string `json:"image"`
Name string `json:"name"`
Namespace string `json:"namespace"`
Schedule string `json:"schedule"` // cron expression
Image string `json:"image"`
Command []string `json:"command"`
}
@@ -1145,9 +1177,9 @@ func (c *Client) ListNamespaces() ([]string, error) {
// NamespaceItem holds full namespace metadata
type NamespaceItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec NamespaceSpec `json:"spec"`
Status NamespaceStatus `json:"status"`
Metadata ObjectMeta `json:"metadata"`
Spec NamespaceSpec `json:"spec"`
Status NamespaceStatus `json:"status"`
}
// NamespaceSpec holds namespace spec
@@ -1209,15 +1241,15 @@ func (c *Client) DeleteNamespace(name string) error {
// ResourceQuotaItem represents a K8s ResourceQuota
type ResourceQuotaItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec ResourceQuotaSpec `json:"spec"`
Status ResourceQuotaStatus `json:"status"`
Metadata ObjectMeta `json:"metadata"`
Spec ResourceQuotaSpec `json:"spec"`
Status ResourceQuotaStatus `json:"status"`
}
// ResourceQuotaSpec holds hard limits
type ResourceQuotaSpec struct {
Hard map[string]string `json:"hard"`
Scopes []string `json:"scopes,omitempty"`
Hard map[string]string `json:"hard"`
Scopes []string `json:"scopes,omitempty"`
}
// ResourceQuotaStatus holds used/hard counts
@@ -1245,8 +1277,8 @@ func (c *Client) ListResourceQuotas(namespace string) ([]ResourceQuotaItem, erro
// LimitRangeItem represents a K8s LimitRange
type LimitRangeItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec LimitRangeSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Spec LimitRangeSpec `json:"spec"`
}
// LimitRangeSpec holds limit items
@@ -1303,17 +1335,17 @@ type SecretList struct {
// SecretItem represents a Secret (return keys only, not values)
type SecretItem struct {
Metadata ObjectMeta `json:"metadata"`
Type string `json:"type"`
DataKeys []string `json:"-"`
Metadata ObjectMeta `json:"metadata"`
Type string `json:"type"`
DataKeys []string `json:"-"`
}
// ListSecrets fetches Secrets in a namespace (names+types only, no data)
func (c *Client) ListSecrets(namespace string) (*SecretList, error) {
var result struct {
Items []struct {
Metadata ObjectMeta `json:"metadata"`
Type string `json:"type"`
Metadata ObjectMeta `json:"metadata"`
Type string `json:"type"`
Data map[string]string `json:"data"`
} `json:"items"`
}
@@ -1348,14 +1380,14 @@ type PVCItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec PVCSpec `json:"spec"`
Status struct {
Phase string `json:"phase"`
Capacity map[string]string `json:"capacity"`
Phase string `json:"phase"`
Capacity map[string]string `json:"capacity"`
} `json:"status"`
}
type PVCSpec struct {
AccessModes []string `json:"accessModes"`
Resources struct {
AccessModes []string `json:"accessModes"`
Resources struct {
Requests map[string]string `json:"requests"`
} `json:"resources"`
StorageClassName string `json:"storageClassName"`
@@ -1386,16 +1418,16 @@ type StatefulSetList struct {
}
type StatefulSetItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec StatefulSetSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Spec StatefulSetSpec `json:"spec"`
Status struct {
Replicas int `json:"replicas"`
ReadyReplicas int `json:"readyReplicas"`
Replicas int `json:"replicas"`
ReadyReplicas int `json:"readyReplicas"`
} `json:"status"`
}
type StatefulSetSpec struct {
Replicas int `json:"replicas"`
Replicas int `json:"replicas"`
Template struct {
Spec struct {
Containers []struct {
@@ -1424,8 +1456,8 @@ type DaemonSetList struct {
}
type DaemonSetItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec DaemonSetSpec `json:"spec"`
Metadata ObjectMeta `json:"metadata"`
Spec DaemonSetSpec `json:"spec"`
Status struct {
DesiredNumberScheduled int `json:"desiredNumberScheduled"`
CurrentNumberScheduled int `json:"currentNumberScheduled"`
@@ -1589,18 +1621,18 @@ type EventSource struct {
}
type EventItem struct {
Metadata ObjectMeta `json:"metadata"`
InvolvedObject InvolvedObject `json:"involvedObject"`
Type string `json:"type"`
Reason string `json:"reason"`
Message string `json:"message"`
Source EventSource `json:"source"`
ReportingComponent string `json:"reportingComponent"`
ReportingInstance string `json:"reportingInstance"`
Count int32 `json:"count"`
FirstTimestamp time.Time `json:"firstTimestamp"`
LastTimestamp time.Time `json:"lastTimestamp"`
EventTime time.Time `json:"eventTime"`
Metadata ObjectMeta `json:"metadata"`
InvolvedObject InvolvedObject `json:"involvedObject"`
Type string `json:"type"`
Reason string `json:"reason"`
Message string `json:"message"`
Source EventSource `json:"source"`
ReportingComponent string `json:"reportingComponent"`
ReportingInstance string `json:"reportingInstance"`
Count int32 `json:"count"`
FirstTimestamp time.Time `json:"firstTimestamp"`
LastTimestamp time.Time `json:"lastTimestamp"`
EventTime time.Time `json:"eventTime"`
}
type InvolvedObject struct {
@@ -1673,7 +1705,7 @@ type RoleList struct {
}
type RoleItem struct {
Metadata ObjectMeta `json:"metadata"`
Metadata ObjectMeta `json:"metadata"`
Rules []PolicyRule `json:"rules"`
}
@@ -1700,9 +1732,9 @@ type RoleBindingList struct {
}
type RoleBindingItem struct {
Metadata ObjectMeta `json:"metadata"`
RoleRef RoleRef `json:"roleRef"`
Subjects []Subject `json:"subjects"`
Metadata ObjectMeta `json:"metadata"`
RoleRef RoleRef `json:"roleRef"`
Subjects []Subject `json:"subjects"`
}
type RoleRef struct {
@@ -1775,7 +1807,7 @@ func (c *Client) applyRaw(yamlContent string) (map[string]interface{}, error) {
}
// Try GET first — if 200, it exists, do PUT. If 404, do POST.
parentPath := apiPath[:strings.LastIndex(apiPath, "/")] // without /name suffix
parentPath := apiPath[:strings.LastIndex(apiPath, "/")] // without /name suffix
_, getErr := c.getRaw(apiPath, "application/yaml")
if getErr == nil {
// Resource exists → PUT update
@@ -1801,8 +1833,10 @@ func (c *Client) applyRaw(yamlContent string) (map[string]interface{}, error) {
func (c *Client) resourceAPIPath(apiVersion, kind, namespace, name string) (string, bool) {
resource := strings.ToLower(kind) + "s"
switch kind {
case "Endpoints": resource = "endpoints"
case "PodMetrics": resource = "pods"
case "Endpoints":
resource = "endpoints"
case "PodMetrics":
resource = "pods"
}
var groupPrefix string
@@ -1829,7 +1863,9 @@ func (c *Client) resourceAPIPath(apiVersion, kind, namespace, name string) (stri
}
ns := namespace
if ns == "" { ns = "default" }
if ns == "" {
ns = "default"
}
if groupPrefix != "" {
switch kind {
@@ -1866,15 +1902,15 @@ type ReplicaSetList struct {
// ReplicaSetItem 表示一个 ReplicaSet(含完整 spec/status/metadata,用于回滚取 template)
type ReplicaSetItem struct {
Metadata ObjectMeta `json:"metadata"`
Spec ReplicaSetSpec `json:"spec"`
Status ReplicaSetStatus `json:"status"`
Metadata ObjectMeta `json:"metadata"`
Spec ReplicaSetSpec `json:"spec"`
Status ReplicaSetStatus `json:"status"`
}
// ReplicaSetSpec 与 DeploymentSpec 结构一致,关键是 template
type ReplicaSetSpec struct {
Replicas int `json:"replicas"`
Template ReplicaSetTemplate `json:"template"`
Replicas int `json:"replicas"`
Template ReplicaSetTemplate `json:"template"`
}
// ReplicaSetTemplate pod 模板
+3 -3
View File
@@ -27,9 +27,9 @@ var (
// GetUserLogScopeRoles 获取用户已启用的日志权限策略 // Logan.Li
func GetUserLogScopeRoles(db *gorm.DB, userID uint) ([]model.LogScopeRole, error) {
var roles []model.LogScopeRole
err := db.Table("monitor_log_scope_roles").
Joins("JOIN monitor_user_log_scope_roles ON monitor_user_log_scope_roles.log_scope_role_id = monitor_log_scope_roles.id").
Where("monitor_user_log_scope_roles.user_id = ? AND monitor_log_scope_roles.status = 1", userID).
err := db.Table("iam_log_scope_roles").
Joins("JOIN iam_user_log_scope_roles ON iam_user_log_scope_roles.log_scope_role_id = iam_log_scope_roles.id").
Where("iam_user_log_scope_roles.user_id = ? AND iam_log_scope_roles.status = 1", userID).
Find(&roles).Error
return roles, err
}