update
This commit is contained in:
Logan committed
1 parent
0690285147
commit
5618cd2738
154 files changed
+5021
-3161
No files matched your search
@@ -0,0 +1,601 @@
|
||||
// kubernetes_diagnose.go - Kubernetes 异常副本与健康诊断聚合工具 // Logan.Li
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
k8sclient "ezops/pkg/kubernetesclient"
|
||||
)
|
||||
|
||||
const defaultDiagnoseLimit = 8
|
||||
|
||||
// KubernetesDiagnoseTool 聚合查询异常 Workload、Pod、Event 和节点健康，减少 LLM 反复拉全量资源。 // Logan.Li
|
||||
type KubernetesDiagnoseTool struct {
|
||||
client *k8sclient.Client
|
||||
namespace string
|
||||
}
|
||||
|
||||
// NewKubernetesDiagnoseTool 创建 Kubernetes 聚合诊断工具。 // Logan.Li
|
||||
func NewKubernetesDiagnoseTool(client *k8sclient.Client, namespace string) *KubernetesDiagnoseTool {
|
||||
return &KubernetesDiagnoseTool{client: client, namespace: namespace}
|
||||
}
|
||||
|
||||
func (t *KubernetesDiagnoseTool) Name() string { return "kubernetes_diagnose" }
|
||||
|
||||
func (t *KubernetesDiagnoseTool) Description() string {
|
||||
return "一键诊断 Kubernetes 命名空间健康和异常副本（只读）。优先用于“异常副本、集群健康、Pod 异常、Deployment 不可用、ImagePullBackOff、CrashLoopBackOff”等任务。" +
|
||||
"工具会聚合 Deployment/StatefulSet/DaemonSet/Pod/Event/Node，只返回异常摘要、可能根因和建议补充检查，避免反复查询全量资源。"
|
||||
}
|
||||
|
||||
func (t *KubernetesDiagnoseTool) Parameters() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"namespace": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "命名空间；空则使用会话绑定命名空间",
|
||||
},
|
||||
"focus": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "关注点：replicas|pods|events|image|crash|probe|all，默认 all",
|
||||
},
|
||||
"limit": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "每类最多返回数量，默认 8，最大 20",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *KubernetesDiagnoseTool) RiskLevel() RiskLevel { return RiskSafe }
|
||||
|
||||
func (t *KubernetesDiagnoseTool) Execute(ctx context.Context, args map[string]interface{}) (*Result, error) {
|
||||
ns := strings.TrimSpace(StringParam(args, "namespace"))
|
||||
if ns == "" {
|
||||
ns = t.namespace
|
||||
}
|
||||
if ns == "" {
|
||||
return &Result{Success: false, Error: "namespace 不能为空"}, nil
|
||||
}
|
||||
limit := IntParam(args, "limit")
|
||||
if limit <= 0 {
|
||||
limit = defaultDiagnoseLimit
|
||||
}
|
||||
if limit > 20 {
|
||||
limit = 20
|
||||
}
|
||||
focus := strings.ToLower(strings.TrimSpace(StringParam(args, "focus")))
|
||||
if focus == "" {
|
||||
focus = "all"
|
||||
}
|
||||
|
||||
report, err := t.buildReport(ns, focus, limit)
|
||||
if err != nil {
|
||||
return &Result{Success: false, Error: err.Error(), Output: err.Error()}, nil
|
||||
}
|
||||
b, err := json.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("诊断结果 JSON 序列化失败: %w", err)
|
||||
}
|
||||
return &Result{Success: true, Output: string(b)}, nil
|
||||
}
|
||||
|
||||
type k8sDiagnoseReport struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Focus string `json:"focus"`
|
||||
Summary k8sDiagnoseSummary `json:"summary"`
|
||||
AbnormalWorkloads []k8sWorkloadFinding `json:"abnormal_workloads"`
|
||||
AbnormalPods []k8sPodFinding `json:"abnormal_pods"`
|
||||
TopEvents []k8sEventFinding `json:"top_events"`
|
||||
NodeHealth []k8sNodeFinding `json:"node_health,omitempty"`
|
||||
LikelyRootCauses []string `json:"likely_root_causes"`
|
||||
RecommendedActions []string `json:"recommended_actions"`
|
||||
NextTools []map[string]interface{} `json:"next_tools"`
|
||||
}
|
||||
|
||||
type k8sDiagnoseSummary struct {
|
||||
Deployments int `json:"deployments"`
|
||||
StatefulSets int `json:"statefulsets"`
|
||||
DaemonSets int `json:"daemonsets"`
|
||||
Pods int `json:"pods"`
|
||||
AbnormalWorkloads int `json:"abnormal_workloads"`
|
||||
AbnormalPods int `json:"abnormal_pods"`
|
||||
WarningEvents int `json:"warning_events"`
|
||||
NotReadyNodes int `json:"not_ready_nodes"`
|
||||
}
|
||||
|
||||
type k8sWorkloadFinding struct {
|
||||
Kind string `json:"kind"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Desired int `json:"desired"`
|
||||
Ready int `json:"ready"`
|
||||
Available int `json:"available,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
Image string `json:"image,omitempty"`
|
||||
}
|
||||
|
||||
type k8sPodFinding struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
Restarts int `json:"restarts"`
|
||||
Node string `json:"node"`
|
||||
IP string `json:"ip,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Messages []string `json:"messages,omitempty"`
|
||||
}
|
||||
|
||||
type k8sEventFinding struct {
|
||||
Type string `json:"type"`
|
||||
Reason string `json:"reason"`
|
||||
Object string `json:"object"`
|
||||
Count int32 `json:"count"`
|
||||
LastSeen string `json:"last_seen,omitempty"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type k8sNodeFinding struct {
|
||||
Name string `json:"name"`
|
||||
Ready string `json:"ready"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
func (t *KubernetesDiagnoseTool) buildReport(ns, focus string, limit int) (*k8sDiagnoseReport, error) {
|
||||
pods, err := t.client.ListPods()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取 Pod 失败: %w", err)
|
||||
}
|
||||
deployments, err := t.client.ListDeployments()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取 Deployment 失败: %w", err)
|
||||
}
|
||||
statefulsets, err := t.client.ListStatefulSets()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取 StatefulSet 失败: %w", err)
|
||||
}
|
||||
daemonsets, err := t.client.ListDaemonSets()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取 DaemonSet 失败: %w", err)
|
||||
}
|
||||
events, err := t.client.ListEvents(ns)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取 Event 失败: %w", err)
|
||||
}
|
||||
nodes, _ := t.client.ListNodes()
|
||||
|
||||
report := &k8sDiagnoseReport{Namespace: ns, Focus: focus}
|
||||
report.AbnormalWorkloads = diagnoseWorkloads(ns, deployments.Items, statefulsets.Items, daemonsets.Items)
|
||||
report.AbnormalPods = diagnosePods(ns, pods.Items, deployments.Items)
|
||||
report.TopEvents = diagnoseEvents(events.Items, limit)
|
||||
report.NodeHealth = diagnoseNodes(nodes)
|
||||
report.Summary = k8sDiagnoseSummary{
|
||||
Deployments: countDeployments(ns, deployments.Items), StatefulSets: countStatefulSets(ns, statefulsets.Items),
|
||||
DaemonSets: countDaemonSets(ns, daemonsets.Items), Pods: countPods(ns, pods.Items),
|
||||
AbnormalWorkloads: len(report.AbnormalWorkloads), AbnormalPods: len(report.AbnormalPods),
|
||||
WarningEvents: countWarningEvents(events.Items), NotReadyNodes: len(report.NodeHealth),
|
||||
}
|
||||
report.LikelyRootCauses = inferK8sRootCauses(report.AbnormalPods, report.TopEvents)
|
||||
report.RecommendedActions = recommendK8sActions(report.AbnormalPods, report.AbnormalWorkloads)
|
||||
report.NextTools = recommendK8sNextTools(report.AbnormalPods, report.TopEvents, limit)
|
||||
report.AbnormalWorkloads = limitWorkloadFindings(report.AbnormalWorkloads, limit)
|
||||
report.AbnormalPods = limitPodFindings(report.AbnormalPods, limit)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func diagnoseWorkloads(ns string, deployments []k8sclient.DeploymentItem, statefulsets []k8sclient.StatefulSetItem, daemonsets []k8sclient.DaemonSetItem) []k8sWorkloadFinding {
|
||||
var out []k8sWorkloadFinding
|
||||
for _, d := range deployments {
|
||||
if d.Metadata.Namespace != ns || d.Spec.Replicas == 0 {
|
||||
continue
|
||||
}
|
||||
if d.Status.ReadyReplicas < d.Spec.Replicas || d.Status.AvailableReplicas < d.Spec.Replicas {
|
||||
out = append(out, k8sWorkloadFinding{
|
||||
Kind: "Deployment", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name,
|
||||
Desired: d.Spec.Replicas, Ready: d.Status.ReadyReplicas, Available: d.Status.AvailableReplicas,
|
||||
Reason: "ready/available replicas less than desired", Image: firstDeploymentImage(d),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, s := range statefulsets {
|
||||
if s.Metadata.Namespace != ns || s.Spec.Replicas == 0 {
|
||||
continue
|
||||
}
|
||||
if s.Status.ReadyReplicas < s.Spec.Replicas {
|
||||
out = append(out, k8sWorkloadFinding{
|
||||
Kind: "StatefulSet", Namespace: s.Metadata.Namespace, Name: s.Metadata.Name,
|
||||
Desired: s.Spec.Replicas, Ready: s.Status.ReadyReplicas,
|
||||
Reason: "ready replicas less than desired", Image: firstStatefulSetImage(s),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, d := range daemonsets {
|
||||
if d.Metadata.Namespace != ns || d.Status.DesiredNumberScheduled == 0 {
|
||||
continue
|
||||
}
|
||||
if d.Status.NumberReady < d.Status.DesiredNumberScheduled {
|
||||
out = append(out, k8sWorkloadFinding{
|
||||
Kind: "DaemonSet", Namespace: d.Metadata.Namespace, Name: d.Metadata.Name,
|
||||
Desired: d.Status.DesiredNumberScheduled, Ready: d.Status.NumberReady,
|
||||
Reason: "ready pods less than desired scheduled pods", Image: firstDaemonSetImage(d),
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Kind == out[j].Kind {
|
||||
return out[i].Name < out[j].Name
|
||||
}
|
||||
return out[i].Kind < out[j].Kind
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func diagnosePods(ns string, pods []k8sclient.PodItem, deployments []k8sclient.DeploymentItem) []k8sPodFinding {
|
||||
var out []k8sPodFinding
|
||||
for _, p := range pods {
|
||||
if p.Metadata.Namespace != ns {
|
||||
continue
|
||||
}
|
||||
status, reason, restarts, messages := podDiagnosisStatus(p)
|
||||
if !isAbnormalPod(status, reason, restarts) {
|
||||
continue
|
||||
}
|
||||
out = append(out, k8sPodFinding{
|
||||
Namespace: p.Metadata.Namespace, Name: p.Metadata.Name, Status: status, Reason: reason,
|
||||
Restarts: restarts, Node: p.Spec.NodeName, IP: p.Status.PodIP, Image: firstPodImage(p),
|
||||
Owner: inferDeploymentOwner(p, deployments), Messages: messages,
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Reason == out[j].Reason {
|
||||
return out[i].Name < out[j].Name
|
||||
}
|
||||
return podReasonRank(out[i].Reason) < podReasonRank(out[j].Reason)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func podDiagnosisStatus(p k8sclient.PodItem) (string, string, int, []string) {
|
||||
status := p.Status.Phase
|
||||
reason := ""
|
||||
restarts := 0
|
||||
var messages []string
|
||||
for _, cs := range p.Status.ContainerStatuses {
|
||||
if cs.RestartCount > restarts {
|
||||
restarts = cs.RestartCount
|
||||
}
|
||||
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" {
|
||||
status = cs.State.Waiting.Reason
|
||||
reason = cs.State.Waiting.Reason
|
||||
if cs.State.Waiting.Message != "" {
|
||||
messages = append(messages, truncateK8sText(cs.State.Waiting.Message, 220))
|
||||
}
|
||||
}
|
||||
if cs.LastState.Terminated != nil && cs.LastState.Terminated.Reason != "" {
|
||||
if reason == "" {
|
||||
reason = cs.LastState.Terminated.Reason
|
||||
}
|
||||
msg := fmt.Sprintf("last terminated: %s exit=%d", cs.LastState.Terminated.Reason, cs.LastState.Terminated.ExitCode)
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
}
|
||||
for _, c := range p.Status.Conditions {
|
||||
if c.Type == "Ready" && c.Status != "True" {
|
||||
if reason == "" && c.Reason != "" {
|
||||
reason = c.Reason
|
||||
}
|
||||
if c.Message != "" {
|
||||
messages = append(messages, truncateK8sText(c.Message, 220))
|
||||
}
|
||||
}
|
||||
}
|
||||
if reason == "" {
|
||||
reason = status
|
||||
}
|
||||
return status, reason, restarts, dedupeStrings(messages)
|
||||
}
|
||||
|
||||
func diagnoseEvents(events []k8sclient.EventItem, limit int) []k8sEventFinding {
|
||||
sort.Slice(events, func(i, j int) bool {
|
||||
return eventTime(events[i]).After(eventTime(events[j]))
|
||||
})
|
||||
var out []k8sEventFinding
|
||||
seen := make(map[string]bool)
|
||||
for _, e := range events {
|
||||
if e.Type != "Warning" && !isImportantEventReason(e.Reason) {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s/%s/%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, k8sEventFinding{
|
||||
Type: e.Type, Reason: e.Reason,
|
||||
Object: fmt.Sprintf("%s/%s", e.InvolvedObject.Kind, e.InvolvedObject.Name),
|
||||
Count: e.Count, LastSeen: eventTime(e).Format(time.RFC3339),
|
||||
Message: truncateK8sText(e.Message, 260),
|
||||
})
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func diagnoseNodes(nodes *k8sclient.NodeList) []k8sNodeFinding {
|
||||
if nodes == nil {
|
||||
return nil
|
||||
}
|
||||
var out []k8sNodeFinding
|
||||
for _, n := range nodes.Items {
|
||||
for _, c := range n.Status.Conditions {
|
||||
if c.Type == "Ready" && c.Status != "True" {
|
||||
out = append(out, k8sNodeFinding{Name: n.Metadata.Name, Ready: c.Status, Reason: c.Reason, Message: truncateK8sText(c.Message, 220)})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func inferK8sRootCauses(pods []k8sPodFinding, events []k8sEventFinding) []string {
|
||||
flags := make(map[string]bool)
|
||||
for _, p := range pods {
|
||||
r := strings.ToLower(p.Reason + " " + p.Status)
|
||||
switch {
|
||||
case strings.Contains(r, "imagepull") || strings.Contains(r, "errimagepull"):
|
||||
flags["存在镜像拉取失败，优先检查镜像 tag、Harbor 仓库、imagePullSecrets 和节点到仓库网络。"] = true
|
||||
case strings.Contains(r, "crashloop"):
|
||||
flags["存在容器反复崩溃，优先查看 previous logs、启动参数、依赖连接和最近变更。"] = true
|
||||
case strings.Contains(r, "oom"):
|
||||
flags["存在 OOMKilled 风险，优先检查内存 limit、最近内存趋势和应用内存泄漏。"] = true
|
||||
case strings.Contains(r, "containersnotready") || strings.Contains(r, "unready"):
|
||||
flags["存在容器未就绪，需结合 readiness/liveness probe、事件和应用日志确认。"] = true
|
||||
case strings.Contains(r, "pending") || strings.Contains(r, "failedscheduling"):
|
||||
flags["存在调度失败或 Pending，优先检查节点资源、污点、亲和性和 PVC。"] = true
|
||||
}
|
||||
}
|
||||
for _, e := range events {
|
||||
msg := strings.ToLower(e.Reason + " " + e.Message)
|
||||
if strings.Contains(msg, "liveness probe") || strings.Contains(msg, "readiness probe") {
|
||||
flags["事件显示健康探针失败，需检查探针路径、超时时间、应用响应耗时和依赖服务。"] = true
|
||||
}
|
||||
}
|
||||
return sortedMapKeys(flags)
|
||||
}
|
||||
|
||||
func recommendK8sActions(pods []k8sPodFinding, workloads []k8sWorkloadFinding) []string {
|
||||
var out []string
|
||||
if len(workloads) > 0 {
|
||||
out = append(out, "先处理 ready/available 副本不足的 Workload，再回查副本是否恢复。")
|
||||
}
|
||||
if hasPodReason(pods, "ImagePullBackOff") || hasPodReason(pods, "ErrImagePull") {
|
||||
out = append(out, "对镜像拉取失败的 Pod，确认镜像 tag 是否存在、仓库权限是否有效、节点能否访问镜像仓库。")
|
||||
}
|
||||
if hasPodReason(pods, "CrashLoopBackOff") {
|
||||
out = append(out, "对 CrashLoopBackOff 的 Pod，补查 previous logs 和启动命令，避免只看当前空日志。")
|
||||
}
|
||||
if hasPodReason(pods, "ContainersNotReady") {
|
||||
out = append(out, "对未就绪 Pod，查看事件中的 readiness/liveness probe 失败原因和应用健康端点耗时。")
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = append(out, "当前聚合诊断未发现明显异常；如仍有业务问题，继续按服务名查看日志、Ingress、Service selector 和指标。")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func recommendK8sNextTools(pods []k8sPodFinding, events []k8sEventFinding, limit int) []map[string]interface{} {
|
||||
var out []map[string]interface{}
|
||||
for _, p := range pods {
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
if strings.Contains(strings.ToLower(p.Reason), "imagepull") {
|
||||
out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events", "name": p.Name}})
|
||||
continue
|
||||
}
|
||||
if p.Status == "CrashLoopBackOff" || p.Restarts >= 5 || strings.Contains(strings.ToLower(p.Reason), "notready") {
|
||||
out = append(out, map[string]interface{}{"tool": "pod_logs", "args": map[string]interface{}{"namespace": p.Namespace, "pod": p.Name, "tail": 120}})
|
||||
}
|
||||
}
|
||||
if len(out) == 0 && len(events) > 0 {
|
||||
out = append(out, map[string]interface{}{"tool": "kubernetes_get", "args": map[string]interface{}{"resource": "events"}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isAbnormalPod(status, reason string, restarts int) bool {
|
||||
s := strings.ToLower(status + " " + reason)
|
||||
if status != "Running" && status != "Succeeded" && status != "PodCompleted" && status != "Completed" {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(s, "notready") || strings.Contains(s, "crashloop") || strings.Contains(s, "imagepull") || strings.Contains(s, "oom") {
|
||||
return true
|
||||
}
|
||||
return restarts >= 10
|
||||
}
|
||||
|
||||
func podReasonRank(reason string) int {
|
||||
r := strings.ToLower(reason)
|
||||
switch {
|
||||
case strings.Contains(r, "imagepull"):
|
||||
return 1
|
||||
case strings.Contains(r, "crashloop"):
|
||||
return 2
|
||||
case strings.Contains(r, "oom"):
|
||||
return 3
|
||||
case strings.Contains(r, "notready"):
|
||||
return 4
|
||||
default:
|
||||
return 9
|
||||
}
|
||||
}
|
||||
|
||||
func inferDeploymentOwner(p k8sclient.PodItem, deployments []k8sclient.DeploymentItem) string {
|
||||
for _, d := range deployments {
|
||||
if d.Metadata.Namespace != p.Metadata.Namespace || d.Spec.Selector == nil {
|
||||
continue
|
||||
}
|
||||
if labelsMatch(p.Metadata.Labels, d.Spec.Selector.MatchLabels) {
|
||||
return "Deployment/" + d.Metadata.Name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstDeploymentImage(d k8sclient.DeploymentItem) string {
|
||||
if len(d.Spec.Template.Spec.Containers) == 0 {
|
||||
return ""
|
||||
}
|
||||
return d.Spec.Template.Spec.Containers[0].Image
|
||||
}
|
||||
|
||||
func firstStatefulSetImage(s k8sclient.StatefulSetItem) string {
|
||||
if len(s.Spec.Template.Spec.Containers) == 0 {
|
||||
return ""
|
||||
}
|
||||
return s.Spec.Template.Spec.Containers[0].Image
|
||||
}
|
||||
|
||||
func firstDaemonSetImage(d k8sclient.DaemonSetItem) string {
|
||||
if len(d.Spec.Template.Spec.Containers) == 0 {
|
||||
return ""
|
||||
}
|
||||
return d.Spec.Template.Spec.Containers[0].Image
|
||||
}
|
||||
|
||||
func firstPodImage(p k8sclient.PodItem) string {
|
||||
if len(p.Spec.Containers) == 0 {
|
||||
return ""
|
||||
}
|
||||
return p.Spec.Containers[0].Image
|
||||
}
|
||||
|
||||
func eventTime(e k8sclient.EventItem) time.Time {
|
||||
if !e.LastTimestamp.IsZero() {
|
||||
return e.LastTimestamp
|
||||
}
|
||||
if !e.EventTime.IsZero() {
|
||||
return e.EventTime
|
||||
}
|
||||
return e.FirstTimestamp
|
||||
}
|
||||
|
||||
func isImportantEventReason(reason string) bool {
|
||||
r := strings.ToLower(reason)
|
||||
return strings.Contains(r, "failed") || strings.Contains(r, "backoff") || strings.Contains(r, "unhealthy")
|
||||
}
|
||||
|
||||
func countDeployments(ns string, items []k8sclient.DeploymentItem) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Metadata.Namespace == ns {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func countStatefulSets(ns string, items []k8sclient.StatefulSetItem) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Metadata.Namespace == ns {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func countDaemonSets(ns string, items []k8sclient.DaemonSetItem) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Metadata.Namespace == ns {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func countPods(ns string, items []k8sclient.PodItem) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Metadata.Namespace == ns {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func countWarningEvents(items []k8sclient.EventItem) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Type == "Warning" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func hasPodReason(pods []k8sPodFinding, reason string) bool {
|
||||
needle := strings.ToLower(reason)
|
||||
for _, p := range pods {
|
||||
if strings.Contains(strings.ToLower(p.Reason+" "+p.Status), needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func limitWorkloadFindings(items []k8sWorkloadFinding, limit int) []k8sWorkloadFinding {
|
||||
if len(items) <= limit {
|
||||
return items
|
||||
}
|
||||
return items[:limit]
|
||||
}
|
||||
|
||||
func limitPodFindings(items []k8sPodFinding, limit int) []k8sPodFinding {
|
||||
if len(items) <= limit {
|
||||
return items
|
||||
}
|
||||
return items[:limit]
|
||||
}
|
||||
|
||||
func truncateK8sText(s string, max int) string {
|
||||
s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
|
||||
if len([]rune(s)) <= max {
|
||||
return s
|
||||
}
|
||||
r := []rune(s)
|
||||
return string(r[:max]) + "..."
|
||||
}
|
||||
|
||||
func dedupeStrings(items []string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, item := range items {
|
||||
if item == "" || seen[item] {
|
||||
continue
|
||||
}
|
||||
seen[item] = true
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedMapKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
k8sclient "ezops/pkg/kubernetesclient"
|
||||
)
|
||||
|
||||
func TestPodDiagnosisStatusDetectsWaitingReason(t *testing.T) {
|
||||
pod := k8sclient.PodItem{
|
||||
Status: k8sclient.PodStatus{
|
||||
Phase: "Pending",
|
||||
ContainerStatuses: []k8sclient.ContainerStatus{{
|
||||
Name: "app",
|
||||
State: k8sclient.ContainerStatusState{Waiting: &k8sclient.ContainerStateWaiting{
|
||||
Reason: "ImagePullBackOff", Message: "Back-off pulling image",
|
||||
}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
status, reason, restarts, messages := podDiagnosisStatus(pod)
|
||||
if status != "ImagePullBackOff" || reason != "ImagePullBackOff" {
|
||||
t.Fatalf("unexpected status=%s reason=%s", status, reason)
|
||||
}
|
||||
if restarts != 0 {
|
||||
t.Fatalf("unexpected restarts=%d", restarts)
|
||||
}
|
||||
if len(messages) != 1 {
|
||||
t.Fatalf("expected waiting message, got %v", messages)
|
||||
}
|
||||
if !isAbnormalPod(status, reason, restarts) {
|
||||
t.Fatal("ImagePullBackOff should be abnormal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodDiagnosisStatusDetectsOOMKilled(t *testing.T) {
|
||||
pod := k8sclient.PodItem{
|
||||
Status: k8sclient.PodStatus{
|
||||
Phase: "Running",
|
||||
ContainerStatuses: []k8sclient.ContainerStatus{{
|
||||
Name: "app", Ready: true, RestartCount: 3,
|
||||
LastState: k8sclient.ContainerStatusState{Terminated: &k8sclient.ContainerStateTerminated{
|
||||
Reason: "OOMKilled", ExitCode: 137,
|
||||
}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
status, reason, restarts, _ := podDiagnosisStatus(pod)
|
||||
if status != "Running" || reason != "OOMKilled" || restarts != 3 {
|
||||
t.Fatalf("unexpected status=%s reason=%s restarts=%d", status, reason, restarts)
|
||||
}
|
||||
if !isAbnormalPod(status, reason, restarts) {
|
||||
t.Fatal("OOMKilled should be abnormal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAbnormalPodByRestartThreshold(t *testing.T) {
|
||||
if !isAbnormalPod("Running", "Running", 10) {
|
||||
t.Fatal("high restart pod should be abnormal")
|
||||
}
|
||||
if isAbnormalPod("Running", "Running", 2) {
|
||||
t.Fatal("healthy running pod should not be abnormal")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user