package services import ( "context" "fmt" "os" "os/exec" "strings" "opsxloki/models" ) // CollectorDeployer 采集器部署器,按 type 分发到三种安装/卸载流程 type CollectorDeployer struct { collector *models.Collector loki *models.LokiDeployment logFunc func(level, message, detail string) } func NewCollectorDeployer(c *models.Collector, l *models.LokiDeployment, logFunc func(level, message, detail string)) *CollectorDeployer { return &CollectorDeployer{collector: c, loki: l, logFunc: logFunc} } func (d *CollectorDeployer) log(level, message, detail string) { if d.logFunc != nil { d.logFunc(level, message, detail) } } // kubeconfigEnv 构造 kubectl/helm 所需的 KUBECONFIG 环境变量,返回 cleanup 删除临时文件 func kubeconfigEnv(cluster *models.Cluster) (env []string, cleanup func(), err error) { kubeconfig, err := resolveKubeconfig(cluster) if err != nil { return nil, nil, err } kubeFile, err := os.CreateTemp("", "kube-*.yaml") if err != nil { return nil, nil, fmt.Errorf("create kubeconfig temp file failed: %w", err) } if _, err := kubeFile.WriteString(kubeconfig); err != nil { kubeFile.Close() os.Remove(kubeFile.Name()) return nil, nil, fmt.Errorf("write kubeconfig temp file failed: %w", err) } kubeFile.Close() os.Chmod(kubeFile.Name(), 0600) env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile.Name())) cleanup = func() { os.Remove(kubeFile.Name()) } return env, cleanup, nil } // Install 按采集器类型分发安装 func (d *CollectorDeployer) Install(ctx context.Context) error { endpoint := d.loki.Endpoint if endpoint == "" { return fmt.Errorf("关联的 Loki 尚未部署,缺少推送地址(endpoint),请先成功部署该 Loki") } d.log("info", fmt.Sprintf("开始部署采集器 %s(类型 %s)", d.collector.Name, d.collector.Type), "") var err error switch d.collector.Type { case "host-promtail": err = d.installHost(ctx, endpoint) case "k8s-daemonset": err = d.installDaemonset(ctx, endpoint) case "k8s-fluentbit-sidecar": err = d.installSidecar(ctx, endpoint) default: return fmt.Errorf("不支持的采集器类型: %s", d.collector.Type) } if err != nil { d.log("error", fmt.Sprintf("采集器部署失败: %v", err), "") return err } d.log("success", "采集器部署完成", "") return nil } // Uninstall 按采集器类型分发卸载 func (d *CollectorDeployer) Uninstall(ctx context.Context) error { d.log("info", fmt.Sprintf("开始卸载采集器 %s(类型 %s)", d.collector.Name, d.collector.Type), "") switch d.collector.Type { case "host-promtail": return d.uninstallHost(ctx) case "k8s-daemonset": return d.uninstallDaemonset(ctx) case "k8s-fluentbit-sidecar": return d.uninstallSidecar(ctx) } return nil } // ===== host-promtail:SSH 安装 Promtail 二进制 + systemd ===== func (d *CollectorDeployer) installHost(ctx context.Context, endpoint string) error { host := d.collector.Host if host == nil { return fmt.Errorf("主机采集器缺少目标主机") } client, err := NewSSHClientWithAuth(HostSSHAuth(host)) if err != nil { return err } defer client.Close() // 检测架构 archOut, _, _ := client.Execute(ctx, "uname -m") arch := strings.TrimSpace(archOut) switch arch { case "x86_64": arch = "amd64" case "aarch64", "arm64": arch = "arm64" default: return fmt.Errorf("不支持的主机架构: %s", arch) } // 确定 Promtail 版本(优先使用关联的 Loki 版本) version := "3.0.0" if d.loki != nil && d.loki.Version != "" { version = d.loki.Version } // 创建必要目录 d.log("info", "准备目录 /opt/promtail /etc/promtail /var/lib/promtail", "") if _, _, err := client.Execute(ctx, "mkdir -p /opt/promtail /etc/promtail /var/lib/promtail"); err != nil { return fmt.Errorf("创建 Promtail 目录失败: %w", err) } // 下载 Promtail d.log("info", fmt.Sprintf("下载 Promtail %s(%s)", version, arch), "") url := fmt.Sprintf("https://github.com/grafana/loki/releases/download/v%s/promtail-linux-%s.zip", version, arch) dl := fmt.Sprintf("cd /tmp && curl -fsSL -o p.zip '%s' && unzip -o p.zip -d /opt/promtail/ && chmod +x /opt/promtail/promtail-linux-%s", url, arch) if _, _, err := client.Execute(ctx, dl); err != nil { return fmt.Errorf("下载 Promtail 失败: %w", err) } client.Execute(ctx, "ln -sf /opt/promtail/promtail-linux-* /usr/local/bin/promtail 2>/dev/null || true") // 生成并上传配置 logPath := d.collector.LogPath if logPath == "" { logPath = "/var/log/*.log" } conf := PromtailConfig(endpoint, logPath, "host", d.collector.Labels) d.log("info", "上传 Promtail 配置", "") if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/promtail/promtail.yaml << 'EOF'\n%s\nEOF", conf)); err != nil { return fmt.Errorf("上传配置失败: %w", err) } // systemd 服务(生产级) service := `[Unit] Description=Promtail Log Collector Wants=network-online.target After=network-online.target [Service] Type=simple User=root Group=root ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml -positions.enabled=true ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=10 LimitNOFILE=65536 StandardOutput=append:/var/log/promtail.log StandardError=append:/var/log/promtail.log [Install] WantedBy=multi-user.target ` d.log("info", "安装 systemd 服务", "") if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/systemd/system/promtail.service << 'EOF'\n%s\nEOF", service)); err != nil { return fmt.Errorf("创建 systemd 服务失败: %w", err) } if _, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable promtail && systemctl restart promtail"); err != nil { return fmt.Errorf("启动 Promtail 失败: %w", err) } // 保存配置快照 d.collector.Config = conf d.log("success", "Promtail 部署完成", "") return nil } func (d *CollectorDeployer) uninstallHost(ctx context.Context) error { host := d.collector.Host if host == nil { return nil } client, err := NewSSHClientWithAuth(HostSSHAuth(host)) if err != nil { return err } defer client.Close() client.Execute(ctx, "systemctl stop promtail 2>/dev/null || true") client.Execute(ctx, "systemctl disable promtail 2>/dev/null || true") client.Execute(ctx, "rm -rf /etc/systemd/system/promtail.service /usr/local/bin/promtail /etc/promtail /opt/promtail /var/lib/promtail") client.Execute(ctx, "systemctl daemon-reload") return nil } // ===== k8s-daemonset:helm 安装 Promtail DaemonSet ===== func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint string) error { cluster := d.collector.Cluster if cluster == nil { return fmt.Errorf("集群采集器缺少目标集群") } env, cleanup, err := kubeconfigEnv(cluster) if err != nil { return err } defer cleanup() // 默认部署到关联 Loki 的命名空间(统一 log 命名空间策略) ns := d.collector.Namespace if ns == "" && d.loki != nil { ns = d.loki.Namespace } if ns == "" { ns = "log" } clusterName := "unknown" if cluster.Name != "" { clusterName = cluster.Name } // 幂等:先清掉同名残留(如历史失败安装留下的无注解资源) del := exec.CommandContext(ctx, "kubectl", "delete", "daemonset/promtail", "configmap/promtail", "serviceaccount/promtail", "-n", ns, "--ignore-not-found=true", "--wait=false") del.Env = env del.CombinedOutput() // 纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),不依赖后端安装 helm manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels) if err := d.applyYAML(ctx, env, manifest, "promtail"); err != nil { return err } d.log("info", fmt.Sprintf("等待 DaemonSet promtail 就绪(命名空间 %s)...", ns), "") waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "daemonset/promtail", "-n", ns, "--timeout=5m") waitCmd.Env = env waitOut, _ := waitCmd.CombinedOutput() d.log("info", string(waitOut), "") // DNS 自愈:CoreDNS 异常的集群上服务名不可解析,回退 Loki ClusterIP 重建配置 svcURL := endpoint if strings.Contains(endpoint, ".svc") { probe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, "daemonset/promtail", "--", "wget", "-q", "-O", "/dev/null", "-T", "5", endpoint+"/ready") probe.Env = env if pout, perr := probe.CombinedOutput(); perr != nil { d.log("warn", fmt.Sprintf("promtail 无法解析 Loki 服务名(%s),回退 ClusterIP...", strings.TrimSpace(string(pout))), "") ip := d.lokiClusterIP(ctx, env, endpoint) if ip != "" { // 只替换 host 部分,保留 scheme/port/path: http://loki.ns.svc:3100/... → http://IP:3100/... afterScheme := strings.SplitN(endpoint, "//", 2) if len(afterScheme) == 2 { rest := afterScheme[1] host := rest if i := strings.IndexAny(rest, ":/"); i >= 0 { host = rest[:i] } fallback := afterScheme[0] + "//" + strings.Replace(rest, host, ip, 1) if err := d.applyYAML(ctx, env, PromtailDaemonsetYAML(ns, fallback, clusterName, d.collector.Labels), "promtail"); err == nil { rs := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "daemonset/promtail", "-n", ns) rs.Env = env rs.CombinedOutput() d.log("info", fmt.Sprintf("promtail 推送地址已回退为 ClusterIP: %s", fallback), "") svcURL = fallback } } } } } d.collector.Config = svcURL d.log("success", fmt.Sprintf("Promtail DaemonSet 已部署到 %s(推送至 %s)", ns, svcURL), "") return nil } // applyYAML kubectl apply 一个 manifest(临时文件) func (d *CollectorDeployer) applyYAML(ctx context.Context, env []string, manifest, label string) error { f, cleanup, err := writeTempFile("collector-*.yaml", manifest, 0600) if err != nil { return err } defer cleanup() cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", f) cmd.Env = env out, err := cmd.CombinedOutput() if err != nil { d.log("error", fmt.Sprintf("kubectl apply %s failed: %v", label, err), string(out)) return fmt.Errorf("kubectl apply %s failed: %w\n%s", label, err, string(out)) } return nil } // lokiClusterIP 从 endpoint 的命名空间取 loki Service 的 ClusterIP(不经 DNS) func (d *CollectorDeployer) lokiClusterIP(ctx context.Context, env []string, endpoint string) string { ns := "log" if d.loki != nil && d.loki.Namespace != "" { ns = d.loki.Namespace } cmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns, "-o", "jsonpath={.spec.clusterIP}") cmd.Env = env out, _ := cmd.CombinedOutput() return strings.TrimSpace(string(out)) } func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error { cluster := d.collector.Cluster if cluster == nil { return nil } env, cleanup, err := kubeconfigEnv(cluster) if err != nil { return err } defer cleanup() ns := d.collector.Namespace if ns == "" && d.loki != nil { ns = d.loki.Namespace } if ns == "" { ns = "log" } cmd := exec.CommandContext(ctx, "kubectl", "delete", "daemonset/promtail", "configmap/promtail", "serviceaccount/promtail", "clusterrole/opsxloki-promtail", "clusterrolebinding/opsxloki-promtail", "-n", ns, "--ignore-not-found=true") cmd.Env = env cmd.CombinedOutput() return nil } // ===== k8s-fluentbit-sidecar:kubectl patch 注入 Fluent Bit sidecar ===== func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string) error { cluster := d.collector.Cluster if cluster == nil { return fmt.Errorf("Sidecar 采集器缺少目标集群") } target := d.collector.TargetDeployment if target == "" { return fmt.Errorf("Sidecar 采集器缺少目标 Deployment") } env, cleanup, err := kubeconfigEnv(cluster) if err != nil { return err } defer cleanup() ns := d.collector.Namespace if ns == "" { ns = "default" } logPath := d.collector.LogPath if logPath == "" { logPath = "/var/log/app/*.log" } app := target // 1. 检查目标 deployment 是否存在 d.log("info", fmt.Sprintf("检查目标 Deployment %s/%s", ns, target), "") check := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns) check.Env = env if out, err := check.CombinedOutput(); err != nil { return fmt.Errorf("目标 Deployment 不存在: %s/%s\n%s", ns, target, string(out)) } // 2. 幂等检查:是否已有 fluent-bit 容器 getNames := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns, "-o", "jsonpath={.spec.template.spec.containers[*].name}") getNames.Env = env namesOut, _ := getNames.CombinedOutput() if strings.Contains(string(namesOut), "fluent-bit") { d.log("warn", "目标 Deployment 已含 fluent-bit 容器,跳过注入", "") return nil } // 3. 部署 ConfigMap lokiHost, lokiPort := parseLokiURL(endpoint) conf := FluentbitConfig(lokiHost, lokiPort, logPath, app, ns) cm := FluentbitConfigMapYAML(conf, ns) cmFile, cmCleanup, err := writeTempFile("fluent-bit-cm-*.yaml", cm, 0600) if err != nil { return err } defer cmCleanup() d.log("info", "部署 Fluent Bit ConfigMap", "") applyCm := exec.CommandContext(ctx, "kubectl", "apply", "-f", cmFile) applyCm.Env = env if out, err := applyCm.CombinedOutput(); err != nil { return fmt.Errorf("ConfigMap 部署失败: %w\n%s", err, string(out)) } // 4. patch 注入 sidecar d.log("info", fmt.Sprintf("为 %s 注入 Fluent Bit Sidecar", target), "") patchFile, patchCleanup, err := writeTempFile("sidecar-patch-*.yaml", SidecarPatchYAML, 0600) if err != nil { return err } defer patchCleanup() patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", target, "-n", ns, "--type=strategic", "--patch-file", patchFile) patch.Env = env if out, err := patch.CombinedOutput(); err != nil { return fmt.Errorf("Sidecar 注入失败: %w\n%s", err, string(out)) } // 5. 等待 rollout d.log("info", "等待 Pod 重建", "") rollout := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/"+target, "-n", ns, "--timeout=5m") rollout.Env = env rollout.CombinedOutput() return nil } func (d *CollectorDeployer) uninstallSidecar(ctx context.Context) error { cluster := d.collector.Cluster if cluster == nil || d.collector.TargetDeployment == "" { return nil } env, cleanup, err := kubeconfigEnv(cluster) if err != nil { return err } defer cleanup() ns := d.collector.Namespace if ns == "" { ns = "default" } patchFile, patchCleanup, err := writeTempFile("sidecar-rm-*.yaml", SidecarRemovePatchYAML, 0600) if err != nil { return err } defer patchCleanup() patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", d.collector.TargetDeployment, "-n", ns, "--type=strategic", "--patch-file", patchFile) patch.Env = env patch.CombinedOutput() return nil }