// kubernetes_diagnose.go - Kubernetes 异常副本与健康诊断聚合工具 // Logan.Li package tools import ( "context" "encoding/json" "fmt" "sort" "strings" "time" k8sclient "ezops/pkg/kubernetesclient" ) const defaultDiagnoseLimit = 8 // KubernetesDiagnoseTool 聚合查询异常 Workload、Pod、Event 和节点健康,减少 LLM 反复拉全量资源。 // Logan.Li type KubernetesDiagnoseTool struct { client *k8sclient.Client namespace string } // NewKubernetesDiagnoseTool 创建 Kubernetes 聚合诊断工具。 // Logan.Li func NewKubernetesDiagnoseTool(client *k8sclient.Client, namespace string) *KubernetesDiagnoseTool { return &KubernetesDiagnoseTool{client: client, namespace: namespace} } func (t *KubernetesDiagnoseTool) Name() string { return "kubernetes_diagnose" } func (t *KubernetesDiagnoseTool) Description() string { return "一键诊断 Kubernetes 命名空间健康和异常副本(只读)。优先用于“异常副本、集群健康、Pod 异常、Deployment 不可用、ImagePullBackOff、CrashLoopBackOff”等任务。" + "工具会聚合 Deployment/StatefulSet/DaemonSet/Pod/Event/Node,只返回异常摘要、可能根因和建议补充检查,避免反复查询全量资源。" } func (t *KubernetesDiagnoseTool) Parameters() map[string]interface{} { return map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "namespace": map[string]interface{}{ "type": "string", "description": "命名空间;空则使用会话绑定命名空间", }, "focus": map[string]interface{}{ "type": "string", "description": "关注点:replicas|pods|events|image|crash|probe|all,默认 all", }, "limit": map[string]interface{}{ "type": "integer", "description": "每类最多返回数量,默认 8,最大 20", }, }, } } func (t *KubernetesDiagnoseTool) RiskLevel() RiskLevel { return RiskSafe } func (t *KubernetesDiagnoseTool) Execute(ctx context.Context, args map[string]interface{}) (*Result, error) { ns := strings.TrimSpace(StringParam(args, "namespace")) if ns == "" { ns = t.namespace } if ns == "" { return &Result{Success: false, Error: "namespace 不能为空"}, nil } limit := IntParam(args, "limit") if limit <= 0 { limit = defaultDiagnoseLimit } if limit > 20 { limit = 20 } focus := strings.ToLower(strings.TrimSpace(StringParam(args, "focus"))) if focus == "" { focus = "all" } report, err := t.buildReport(ns, focus, limit) if err != nil { return &Result{Success: false, Error: err.Error(), Output: err.Error()}, nil } b, err := json.MarshalIndent(report, "", " ") if err != nil { return nil, fmt.Errorf("诊断结果 JSON 序列化失败: %w", err) } return &Result{Success: true, Output: string(b)}, nil } type k8sDiagnoseReport struct { Namespace string `json:"namespace"` Focus string `json:"focus"` Summary k8sDiagnoseSummary `json:"summary"` AbnormalWorkloads []k8sWorkloadFinding `json:"abnormal_workloads"` AbnormalPods []k8sPodFinding `json:"abnormal_pods"` TopEvents []k8sEventFinding `json:"top_events"` NodeHealth []k8sNodeFinding `json:"node_health,omitempty"` LikelyRootCauses []string `json:"likely_root_causes"` RecommendedActions []string `json:"recommended_actions"` NextTools []map[string]interface{} `json:"next_tools"` } type k8sDiagnoseSummary struct { Deployments int `json:"deployments"` StatefulSets int `json:"statefulsets"` DaemonSets int `json:"daemonsets"` Pods int `json:"pods"` AbnormalWorkloads int `json:"abnormal_workloads"` AbnormalPods int `json:"abnormal_pods"` WarningEvents int `json:"warning_events"` NotReadyNodes int `json:"not_ready_nodes"` } type k8sWorkloadFinding struct { Kind string `json:"kind"` Namespace string `json:"namespace"` Name string `json:"name"` Desired int `json:"desired"` Ready int `json:"ready"` Available int `json:"available,omitempty"` Reason string `json:"reason"` Image string `json:"image,omitempty"` } type k8sPodFinding struct { Namespace string `json:"namespace"` Name string `json:"name"` Status string `json:"status"` Reason string `json:"reason"` Restarts int `json:"restarts"` Node string `json:"node"` IP string `json:"ip,omitempty"` Image string `json:"image,omitempty"` Owner string `json:"owner,omitempty"` Messages []string `json:"messages,omitempty"` } type k8sEventFinding struct { Type string `json:"type"` Reason string `json:"reason"` Object string `json:"object"` Count int32 `json:"count"` LastSeen string `json:"last_seen,omitempty"` Message string `json:"message"` } type k8sNodeFinding struct { Name string `json:"name"` Ready string `json:"ready"` Reason string `json:"reason,omitempty"` Message string `json:"message,omitempty"` } func (t *KubernetesDiagnoseTool) buildReport(ns, focus string, limit int) (*k8sDiagnoseReport, error) { pods, err := t.client.ListPods() if err != nil { return nil, fmt.Errorf("获取 Pod 失败: %w", err) } deployments, err := t.client.ListDeployments() if err != nil { return nil, fmt.Errorf("获取 Deployment 失败: %w", err) } statefulsets, err := t.client.ListStatefulSets() if err != nil { return nil, fmt.Errorf("获取 StatefulSet 失败: %w", err) } daemonsets, err := t.client.ListDaemonSets() if err != nil { return nil, fmt.Errorf("获取 DaemonSet 失败: %w", err) } events, err := t.client.ListEvents(ns) if err != nil { return nil, fmt.Errorf("获取 Event 失败: %w", err) } nodes, _ := t.client.ListNodes() report := &k8sDiagnoseReport{Namespace: ns, Focus: focus} report.AbnormalWorkloads = diagnoseWorkloads(ns, deployments.Items, statefulsets.Items, daemonsets.Items) report.AbnormalPods = diagnosePods(ns, pods.Items, deployments.Items) report.TopEvents = diagnoseEvents(events.Items, limit) report.NodeHealth = diagnoseNodes(nodes) report.Summary = k8sDiagnoseSummary{ Deployments: countDeployments(ns, deployments.Items), StatefulSets: countStatefulSets(ns, statefulsets.Items), DaemonSets: countDaemonSets(ns, daemonsets.Items), Pods: countPods(ns, pods.Items), AbnormalWorkloads: len(report.AbnormalWorkloads), AbnormalPods: len(report.AbnormalPods), WarningEvents: countWarningEvents(events.Items), NotReadyNodes: len(report.NodeHealth), } report.LikelyRootCauses = inferK8sRootCauses(report.AbnormalPods, report.TopEvents) report.RecommendedActions = recommendK8sActions(report.AbnormalPods, report.AbnormalWorkloads) report.NextTools = recommendK8sNextTools(report.AbnormalPods, report.TopEvents, limit) report.AbnormalWorkloads = limitWorkloadFindings(report.AbnormalWorkloads, limit) report.AbnormalPods = limitPodFindings(report.AbnormalPods, limit) return report, nil } func diagnoseWorkloads(ns string, deployments []k8sclient.DeploymentItem, statefulsets []k8sclient.StatefulSetItem, daemonsets []k8sclient.DaemonSetItem) []k8sWorkloadFinding { var out []k8sWorkloadFinding for _, d := range deployments { if d.Metadata.Namespace != ns || d.Spec.Replicas == 0 { continue } if d.Status.ReadyReplicas < d.Spec.Replicas || d.Status.AvailableReplicas < d.Spec.Replicas { out = append(out, k8sWorkloadFinding{ Kind: "Deployment", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name, Desired: d.Spec.Replicas, Ready: d.Status.ReadyReplicas, Available: d.Status.AvailableReplicas, Reason: "ready/available replicas less than desired", Image: firstDeploymentImage(d), }) } } for _, s := range statefulsets { if s.Metadata.Namespace != ns || s.Spec.Replicas == 0 { continue } if s.Status.ReadyReplicas < s.Spec.Replicas { out = append(out, k8sWorkloadFinding{ Kind: "StatefulSet", Namespace: s.Metadata.Namespace, Name: s.Metadata.Name, Desired: s.Spec.Replicas, Ready: s.Status.ReadyReplicas, Reason: "ready replicas less than desired", Image: firstStatefulSetImage(s), }) } } for _, d := range daemonsets { if d.Metadata.Namespace != ns || d.Status.DesiredNumberScheduled == 0 { continue } if d.Status.NumberReady < d.Status.DesiredNumberScheduled { out = append(out, k8sWorkloadFinding{ Kind: "DaemonSet", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name, Desired: d.Status.DesiredNumberScheduled, Ready: d.Status.NumberReady, Reason: "ready pods less than desired scheduled pods", Image: firstDaemonSetImage(d), }) } } sort.Slice(out, func(i, j int) bool { if out[i].Kind == out[j].Kind { return out[i].Name < out[j].Name } return out[i].Kind < out[j].Kind }) return out } func diagnosePods(ns string, pods []k8sclient.PodItem, deployments []k8sclient.DeploymentItem) []k8sPodFinding { var out []k8sPodFinding for _, p := range pods { if p.Metadata.Namespace != ns { continue } status, reason, restarts, messages := podDiagnosisStatus(p) if !isAbnormalPod(status, reason, restarts) { continue } out = append(out, k8sPodFinding{ Namespace: p.Metadata.Namespace, Name: p.Metadata.Name, Status: status, Reason: reason, Restarts: restarts, Node: p.Spec.NodeName, IP: p.Status.PodIP, Image: firstPodImage(p), Owner: inferDeploymentOwner(p, deployments), Messages: messages, }) } sort.Slice(out, func(i, j int) bool { if out[i].Reason == out[j].Reason { return out[i].Name < out[j].Name } return podReasonRank(out[i].Reason) < podReasonRank(out[j].Reason) }) return out } func podDiagnosisStatus(p k8sclient.PodItem) (string, string, int, []string) { status := p.Status.Phase reason := "" restarts := 0 var messages []string for _, cs := range p.Status.ContainerStatuses { if cs.RestartCount > restarts { restarts = cs.RestartCount } if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" { status = cs.State.Waiting.Reason reason = cs.State.Waiting.Reason if cs.State.Waiting.Message != "" { messages = append(messages, truncateK8sText(cs.State.Waiting.Message, 220)) } } if cs.LastState.Terminated != nil && cs.LastState.Terminated.Reason != "" { if reason == "" { reason = cs.LastState.Terminated.Reason } msg := fmt.Sprintf("last terminated: %s exit=%d", cs.LastState.Terminated.Reason, cs.LastState.Terminated.ExitCode) messages = append(messages, msg) } } for _, c := range p.Status.Conditions { if c.Type == "Ready" && c.Status != "True" { if reason == "" && c.Reason != "" { reason = c.Reason } if c.Message != "" { messages = append(messages, truncateK8sText(c.Message, 220)) } } } if reason == "" { reason = status } return status, reason, restarts, dedupeStrings(messages) } func diagnoseEvents(events []k8sclient.EventItem, limit int) []k8sEventFinding { sort.Slice(events, func(i, j int) bool { return eventTime(events[i]).After(eventTime(events[j])) }) var out []k8sEventFinding seen := make(map[string]bool) for _, e := range events { if e.Type != "Warning" && !isImportantEventReason(e.Reason) { continue } key := fmt.Sprintf("%s/%s/%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message) if seen[key] { continue } seen[key] = true out = append(out, k8sEventFinding{ Type: e.Type, Reason: e.Reason, Object: fmt.Sprintf("%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name), Count: e.Count, LastSeen: eventTime(e).Format(time.RFC3339), Message: truncateK8sText(e.Message, 260), }) if len(out) >= limit { break } } return out } func diagnoseNodes(nodes *k8sclient.NodeList) []k8sNodeFinding { if nodes == nil { return nil } var out []k8sNodeFinding for _, n := range nodes.Items { for _, c := range n.Status.Conditions { if c.Type == "Ready" && c.Status != "True" { out = append(out, k8sNodeFinding{Name: n.Metadata.Name, Ready: c.Status, Reason: c.Reason, Message: truncateK8sText(c.Message, 220)}) break } } } return out } func inferK8sRootCauses(pods []k8sPodFinding, events []k8sEventFinding) []string { flags := make(map[string]bool) for _, p := range pods { r := strings.ToLower(p.Reason + " " + p.Status) switch { case strings.Contains(r, "imagepull") || strings.Contains(r, "errimagepull"): flags["存在镜像拉取失败,优先检查镜像 tag、Harbor 仓库、imagePullSecrets 和节点到仓库网络。"] = true case strings.Contains(r, "crashloop"): flags["存在容器反复崩溃,优先查看 previous logs、启动参数、依赖连接和最近变更。"] = true case strings.Contains(r, "oom"): flags["存在 OOMKilled 风险,优先检查内存 limit、最近内存趋势和应用内存泄漏。"] = true case strings.Contains(r, "containersnotready") || strings.Contains(r, "unready"): flags["存在容器未就绪,需结合 readiness/liveness probe、事件和应用日志确认。"] = true case strings.Contains(r, "pending") || strings.Contains(r, "failedscheduling"): flags["存在调度失败或 Pending,优先检查节点资源、污点、亲和性和 PVC。"] = true } } for _, e := range events { msg := strings.ToLower(e.Reason + " " + e.Message) if strings.Contains(msg, "liveness probe") || strings.Contains(msg, "readiness probe") { flags["事件显示健康探针失败,需检查探针路径、超时时间、应用响应耗时和依赖服务。"] = true } } return sortedMapKeys(flags) } func recommendK8sActions(pods []k8sPodFinding, workloads []k8sWorkloadFinding) []string { var out []string if len(workloads) > 0 { out = append(out, "先处理 ready/available 副本不足的 Workload,再回查副本是否恢复。") } if hasPodReason(pods, "ImagePullBackOff") || hasPodReason(pods, "ErrImagePull") { out = append(out, "对镜像拉取失败的 Pod,确认镜像 tag 是否存在、仓库权限是否有效、节点能否访问镜像仓库。") } if hasPodReason(pods, "CrashLoopBackOff") { out = append(out, "对 CrashLoopBackOff 的 Pod,补查 previous logs 和启动命令,避免只看当前空日志。") } if hasPodReason(pods, "ContainersNotReady") { out = append(out, "对未就绪 Pod,查看事件中的 readiness/liveness probe 失败原因和应用健康端点耗时。") } if len(out) == 0 { out = append(out, "当前聚合诊断未发现明显异常;如仍有业务问题,继续按服务名查看日志、Ingress、Service selector 和指标。") } return out } func recommendK8sNextTools(pods []k8sPodFinding, events []k8sEventFinding, limit int) []map[string]interface{} { var out []map[string]interface{} for _, p := range pods { if len(out) >= limit { break } if strings.Contains(strings.ToLower(p.Reason), "imagepull") { out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events", "name": p.Name}}) continue } if p.Status == "CrashLoopBackOff" || p.Restarts >= 5 || strings.Contains(strings.ToLower(p.Reason), "notready") { out = append(out, map[string]interface{}{"tool": "pod_logs", "args": map[string]interface{}{"namespace": p.Namespace, "pod": p.Name, "tail": 120}}) } } if len(out) == 0 && len(events) > 0 { out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events"}}) } return out } func isAbnormalPod(status, reason string, restarts int) bool { s := strings.ToLower(status + " " + reason) if status != "Running" && status != "Succeeded" && status != "PodCompleted" && status != "Completed" { return true } if strings.Contains(s, "notready") || strings.Contains(s, "crashloop") || strings.Contains(s, "imagepull") || strings.Contains(s, "oom") { return true } return restarts >= 10 } func podReasonRank(reason string) int { r := strings.ToLower(reason) switch { case strings.Contains(r, "imagepull"): return 1 case strings.Contains(r, "crashloop"): return 2 case strings.Contains(r, "oom"): return 3 case strings.Contains(r, "notready"): return 4 default: return 9 } } func inferDeploymentOwner(p k8sclient.PodItem, deployments []k8sclient.DeploymentItem) string { for _, d := range deployments { if d.Metadata.Namespace != p.Metadata.Namespace || d.Spec.Selector == nil { continue } if labelsMatch(p.Metadata.Labels, d.Spec.Selector.MatchLabels) { return "Deployment/" + d.Metadata.Name } } return "" } func firstDeploymentImage(d k8sclient.DeploymentItem) string { if len(d.Spec.Template.Spec.Containers) == 0 { return "" } return d.Spec.Template.Spec.Containers[0].Image } func firstStatefulSetImage(s k8sclient.StatefulSetItem) string { if len(s.Spec.Template.Spec.Containers) == 0 { return "" } return s.Spec.Template.Spec.Containers[0].Image } func firstDaemonSetImage(d k8sclient.DaemonSetItem) string { if len(d.Spec.Template.Spec.Containers) == 0 { return "" } return d.Spec.Template.Spec.Containers[0].Image } func firstPodImage(p k8sclient.PodItem) string { if len(p.Spec.Containers) == 0 { return "" } return p.Spec.Containers[0].Image } func eventTime(e k8sclient.EventItem) time.Time { if !e.LastTimestamp.IsZero() { return e.LastTimestamp } if !e.EventTime.IsZero() { return e.EventTime } return e.FirstTimestamp } func isImportantEventReason(reason string) bool { r := strings.ToLower(reason) return strings.Contains(r, "failed") || strings.Contains(r, "backoff") || strings.Contains(r, "unhealthy") } func countDeployments(ns string, items []k8sclient.DeploymentItem) int { n := 0 for _, it := range items { if it.Metadata.Namespace == ns { n++ } } return n } func countStatefulSets(ns string, items []k8sclient.StatefulSetItem) int { n := 0 for _, it := range items { if it.Metadata.Namespace == ns { n++ } } return n } func countDaemonSets(ns string, items []k8sclient.DaemonSetItem) int { n := 0 for _, it := range items { if it.Metadata.Namespace == ns { n++ } } return n } func countPods(ns string, items []k8sclient.PodItem) int { n := 0 for _, it := range items { if it.Metadata.Namespace == ns { n++ } } return n } func countWarningEvents(items []k8sclient.EventItem) int { n := 0 for _, it := range items { if it.Type == "Warning" { n++ } } return n } func hasPodReason(pods []k8sPodFinding, reason string) bool { needle := strings.ToLower(reason) for _, p := range pods { if strings.Contains(strings.ToLower(p.Reason+" "+p.Status), needle) { return true } } return false } func limitWorkloadFindings(items []k8sWorkloadFinding, limit int) []k8sWorkloadFinding { if len(items) <= limit { return items } return items[:limit] } func limitPodFindings(items []k8sPodFinding, limit int) []k8sPodFinding { if len(items) <= limit { return items } return items[:limit] } func truncateK8sText(s string, max int) string { s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " ")) if len([]rune(s)) <= max { return s } r := []rune(s) return string(r[:max]) + "..." } func dedupeStrings(items []string) []string { seen := make(map[string]bool) var out []string for _, item := range items { if item == "" || seen[item] { continue } seen[item] = true out = append(out, item) } return out } func sortedMapKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out }