package services import ( "bytes" "context" "fmt" "io" "strings" "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" yamlutil "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" kcScheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/remotecommand" "opsxloki/models" ) func ensureNamespace(ctx context.Context, cluster *models.Cluster, namespace string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } _, err = client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if apierrors.IsNotFound(err) { _, err = client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) } return err } // applyManifest 使用 API Server 的 server-side apply,保留已有 YAML 模板与字段所有权。 func applyManifest(ctx context.Context, cluster *models.Cluster, manifest string) error { cfg, err := clusterRESTConfig(cluster, 30*time.Second) if err != nil { return err } dc, err := discovery.NewDiscoveryClientForConfig(cfg) if err != nil { return err } groups, err := restmapper.GetAPIGroupResources(dc) if err != nil { return fmt.Errorf("读取 Kubernetes API 资源列表: %w", err) } mapper := restmapper.NewDiscoveryRESTMapper(groups) client, err := dynamic.NewForConfig(cfg) if err != nil { return err } decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(manifest), 4096) for { var raw map[string]interface{} if err := decoder.Decode(&raw); err == io.EOF { break } else if err != nil { return fmt.Errorf("解析 Kubernetes YAML: %w", err) } if len(raw) == 0 { continue } obj := &unstructured.Unstructured{Object: raw} mapping, err := mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version) if err != nil { return err } name := obj.GetName() if name == "" { return fmt.Errorf("Kubernetes %s 缺少 metadata.name", obj.GetKind()) } data, err := obj.MarshalJSON() if err != nil { return err } var resource dynamic.ResourceInterface = client.Resource(mapping.Resource) if mapping.Scope.Name() == "namespace" { resource = client.Resource(mapping.Resource).Namespace(obj.GetNamespace()) } force := true if _, err := resource.Patch(ctx, name, types.ApplyPatchType, data, metav1.PatchOptions{FieldManager: "lokideploy", Force: &force}); err != nil { return fmt.Errorf("应用 %s %s/%s: %w", obj.GetKind(), obj.GetNamespace(), name, err) } } return nil } func waitStatefulSet(ctx context.Context, cluster *models.Cluster, namespace, name string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } limit, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) { item, err := client.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, err } desired := int32(1) if item.Spec.Replicas != nil { desired = *item.Spec.Replicas } return item.Status.ObservedGeneration >= item.Generation && item.Status.ReadyReplicas >= desired && item.Status.UpdatedReplicas >= desired, nil }) } func waitDeployment(ctx context.Context, cluster *models.Cluster, namespace, name string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } limit, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) { item, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, err } desired := int32(1) if item.Spec.Replicas != nil { desired = *item.Spec.Replicas } return item.Status.ObservedGeneration >= item.Generation && item.Status.ReadyReplicas >= desired && item.Status.UpdatedReplicas >= desired, nil }) } func waitDaemonSet(ctx context.Context, cluster *models.Cluster, namespace, name string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } limit, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) { item, err := client.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, err } return item.Status.ObservedGeneration >= item.Generation && item.Status.NumberReady >= item.Status.DesiredNumberScheduled && item.Status.UpdatedNumberScheduled >= item.Status.DesiredNumberScheduled, nil }) } func restartDeployment(ctx context.Context, cluster *models.Cluster, namespace, name string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } item, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } item.Spec.Template.Annotations = copyAnnotations(item.Spec.Template.Annotations) item.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().UTC().Format(time.RFC3339) _, err = client.AppsV1().Deployments(namespace).Update(ctx, item, metav1.UpdateOptions{}) return err } func copyAnnotations(source map[string]string) map[string]string { result := map[string]string{} for k, v := range source { result[k] = v } return result } func deleteK8sResource(ctx context.Context, cluster *models.Cluster, kind, namespace, name string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } var result error switch kind { case "statefulset": result = client.AppsV1().StatefulSets(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "deployment": result = client.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "daemonset": result = client.AppsV1().DaemonSets(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "service": result = client.CoreV1().Services(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "configmap": result = client.CoreV1().ConfigMaps(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "serviceaccount": result = client.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "persistentvolumeclaim": result = client.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, name, metav1.DeleteOptions{}) case "clusterrole": result = client.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{}) case "clusterrolebinding": result = client.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{}) default: return fmt.Errorf("未知 Kubernetes 资源: %s", kind) } if apierrors.IsNotFound(result) { return nil } return result } func serviceClusterIP(ctx context.Context, cluster *models.Cluster, namespace, name string) (string, error) { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return "", err } svc, err := client.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return "", err } return svc.Spec.ClusterIP, nil } func serviceNodePort(ctx context.Context, cluster *models.Cluster, namespace, name string) (int32, error) { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return 0, err } svc, err := client.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return 0, err } if len(svc.Spec.Ports) == 0 { return 0, fmt.Errorf("Service %s/%s 没有端口", namespace, name) } return svc.Spec.Ports[0].NodePort, nil } func firstNodeIP(ctx context.Context, cluster *models.Cluster) (string, error) { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return "", err } nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1}) if err != nil { return "", err } for _, node := range nodes.Items { for _, address := range node.Status.Addresses { if address.Type == corev1.NodeInternalIP { return address.Address, nil } } } return "", fmt.Errorf("集群无可用节点内网地址") } func patchDeployment(ctx context.Context, cluster *models.Cluster, namespace, name, yamlPatch string) error { client, err := clusterClient(cluster, 30*time.Second) if err != nil { return err } patch, err := yamlutil.ToJSON([]byte(yamlPatch)) if err != nil { return err } _, err = client.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) return err } // podDNSBroken 在指定 deployment 的容器里探测 hostname 是否可解析(wget /ready 连通性)。 func podDNSBroken(ctx context.Context, cluster *models.Cluster, namespace, deployment, container, host string) bool { pods, err := listDeploymentPods(ctx, cluster, namespace, deployment) if err != nil || len(pods) == 0 { return false // 拿不到 pod 时不做自愈,交给后续日志排查 } client, err := clusterClient(cluster, 15*time.Second) if err != nil { return false } restCfg, err := clusterRESTConfig(cluster, 15*time.Second) if err != nil { return false } execReq := client.CoreV1().RESTClient().Post(). Namespace(namespace).Resource("pods").Name(pods[0]). SubResource("exec"). VersionedParams(&corev1.PodExecOptions{ Command: []string{"sh", "-c", "wget -q -O /dev/null -T 3 http://" + host + ":3100/ready"}, Container: container, Stdout: true, Stderr: true, }, kcScheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", execReq.URL()) if err != nil { return false } var out bytes.Buffer err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &out, Stderr: &out}) return err != nil // 执行失败(连接/解析不通)视为 DNS 故障 } // listDeploymentPods 按 deployment selector 列出 pod 名。 func listDeploymentPods(ctx context.Context, cluster *models.Cluster, namespace, deployment string) ([]string, error) { client, err := clusterClient(cluster, 15*time.Second) if err != nil { return nil, err } dep, err := client.AppsV1().Deployments(namespace).Get(ctx, deployment, metav1.GetOptions{}) if err != nil { return nil, err } selector := metav1.FormatLabelSelector(dep.Spec.Selector) podList, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) if err != nil { return nil, err } names := make([]string, 0, len(podList.Items)) for i := range podList.Items { names = append(names, podList.Items[i].Name) } return names, nil } // lokiNamespaceOf 从 endpoint 的 svc FQDN 提取 Loki 命名空间(log.ns.svc → ns)。 func lokiNamespaceOf(loki *models.LokiDeployment, endpoint string) string { if loki != nil && loki.Namespace != "" { return loki.Namespace } parts := strings.SplitN(endpoint, "//", 2) if len(parts) == 2 { host := strings.SplitN(parts[1], ".", 3) if len(host) >= 2 { return host[1] } } return "log" }