package services import ( "context" "encoding/base64" "fmt" "io" "net/http" "os/exec" "strings" "time" "opsxloki/models" ) // GrafanaDeployer 与 Loki 同环境部署 Grafana 并自动配置 Loki 数据源。 // k8s 模式:manifest(Deployment + NodePort Service + 数据源 ConfigMap),与 Loki 同 namespace; // host-binary 模式:tarball + systemd,与 Loki 同机。 type GrafanaDeployer struct { deployment *models.LokiDeployment log func(level, message, detail string) } const defaultGrafanaVersion = "11.3.0" func (d *GrafanaDeployer) version() string { if d.deployment.GrafanaVersion != "" { return d.deployment.GrafanaVersion } return defaultGrafanaVersion } // lokiPushURL 数据源指向的 Loki 地址(k8s 走集群内服务名,主机走本机) func (d *GrafanaDeployer) lokiPushURL() string { if d.deployment.Mode == "host-binary" { return "http://127.0.0.1:3100" } ns := d.deployment.Namespace if ns == "" { ns = "log" } return fmt.Sprintf("http://loki.%s.svc.cluster.local:3100", ns) } // Deploy 部署入口:按模式分发 func (d *GrafanaDeployer) Deploy(ctx context.Context) error { d.log("info", fmt.Sprintf("Deploying Grafana %s (mode: %s)...", d.version(), d.deployment.Mode), "") switch d.deployment.Mode { case "k8s-helm", "k8s-yaml": return d.deployK8s(ctx) case "host-binary": return d.deployHost(ctx) default: return fmt.Errorf("unsupported mode for grafana: %s", d.deployment.Mode) } } // ==================== K8s manifest 部署 ==================== // grafanaManifest 生成 Grafana 资源;lokiURL 显式传入(默认服务名,DNS 异常的集群可回退 ClusterIP)。 // 数据卷用 emptyDir:数据源/配置由 ConfigMap 供给,重启即恢复;避免无默认 // StorageClass 的集群上 PVC Pending 阻塞(UI 内自建的面板需重新导入,见已知限制)。 func (d *GrafanaDeployer) grafanaManifest(ns, lokiURL string) string { version := d.version() datasources := fmt.Sprintf(`apiVersion: 1 datasources: - name: Loki type: loki uid: loki-ds access: proxy url: %s isDefault: true editable: false`, lokiURL) return fmt.Sprintf(`apiVersion: v1 kind: ConfigMap metadata: name: grafana-datasources namespace: %s data: datasources.yaml: | %s --- apiVersion: apps/v1 kind: Deployment metadata: name: grafana namespace: %s labels: app: grafana spec: replicas: 1 selector: matchLabels: app: grafana template: metadata: labels: app: grafana spec: containers: - name: grafana image: grafana/grafana:%s imagePullPolicy: IfNotPresent ports: - name: http containerPort: 3000 env: - name: GF_SECURITY_ADMIN_USER value: admin - name: GF_SECURITY_ADMIN_PASSWORD value: admin readinessProbe: httpGet: path: /api/health port: http initialDelaySeconds: 10 periodSeconds: 10 resources: requests: cpu: 100m memory: 128Mi limits: cpu: 500m memory: 512Mi volumeMounts: - name: datasources mountPath: /etc/grafana/provisioning/datasources volumes: - name: datasources configMap: name: grafana-datasources --- apiVersion: v1 kind: Service metadata: name: grafana namespace: %s labels: app: grafana spec: # NodePort 便于集群外直接访问(端口由集群自动分配,实际端口见部署日志) type: NodePort ports: - name: http port: 3000 targetPort: 3000 selector: app: grafana `, ns, indentBlock(datasources, " "), ns, version, ns) } func (d *GrafanaDeployer) deployK8s(ctx context.Context) error { if d.deployment.Cluster == nil { return fmt.Errorf("please select target cluster") } env, cleanup, err := kubeconfigEnv(d.deployment.Cluster) if err != nil { return err } defer cleanup() ns := d.deployment.Namespace if ns == "" { ns = "log" } lokiURL := d.lokiPushURL() manifestFile, manifestCleanup, err := writeTempFile("grafana-*.yaml", d.grafanaManifest(ns, lokiURL), 0600) if err != nil { return err } defer manifestCleanup() d.log("info", "Applying Grafana manifest (Deployment + NodePort Service + datasource ConfigMap)...", "") cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", manifestFile) cmd.Env = env out, err := cmd.CombinedOutput() if err != nil { d.log("error", fmt.Sprintf("kubectl apply grafana failed: %v", err), string(out)) return fmt.Errorf("kubectl apply grafana failed: %w\n%s", err, string(out)) } waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana", "-n", ns, "--timeout=5m") waitCmd.Env = env waitOut, _ := waitCmd.CombinedOutput() d.log("info", string(waitOut), "") // DNS 自愈:部分集群 CoreDNS 对 Service 记录异常(如 watch 失效), // 经 grafana 容器探测 Loki 服务名,不通则回退 ClusterIP 并热更新数据源。 dnsProbe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, "deploy/grafana", "--", "wget", "-q", "-O", "/dev/null", "-T", "5", lokiURL+"/ready") dnsProbe.Env = env if probeOut, probeErr := dnsProbe.CombinedOutput(); probeErr != nil { d.log("warn", fmt.Sprintf("Grafana 无法解析 Loki 服务名(%s),回退 ClusterIP…", strings.TrimSpace(string(probeOut))), "") ipCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns, "-o", "jsonpath={.spec.clusterIP}") ipCmd.Env = env ipOut, ipErr := ipCmd.CombinedOutput() clusterIP := strings.TrimSpace(string(ipOut)) if ipErr != nil || clusterIP == "" { d.log("error", "获取 Loki ClusterIP 失败,数据源保持服务名形式", "") } else { fallbackURL := fmt.Sprintf("http://%s:3100", clusterIP) fbFile, fbCleanup, ferr := writeTempFile("grafana-fb-*.yaml", d.grafanaManifest(ns, fallbackURL), 0600) if ferr == nil { fbApply := exec.CommandContext(ctx, "kubectl", "apply", "-f", fbFile) fbApply.Env = env fbApply.CombinedOutput() fbCleanup() restart := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "deployment/grafana", "-n", ns) restart.Env = env restart.CombinedOutput() wait2 := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana", "-n", ns, "--timeout=5m") wait2.Env = env w2, _ := wait2.CombinedOutput() d.log("info", string(w2), "") d.log("info", fmt.Sprintf("数据源已回退为 ClusterIP: %s", fallbackURL), "") } } } // 回读实际分配的 NodePort(固定端口易与集群已有服务冲突) portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns, "-o", "jsonpath={.spec.ports[0].nodePort}") portCmd.Env = env portOut, _ := portCmd.CombinedOutput() nodePort := strings.TrimSpace(string(portOut)) access := fmt.Sprintf("http://<节点IP>:%s", nodePort) if nodePort == "" { access = "kubectl get svc grafana -n " + ns + " 查看访问端口" } d.log("success", fmt.Sprintf("Grafana deployed. 访问: %s (NodePort, 账号 admin/admin), 数据源已指向 %s", access, d.lokiPushURL()), "") // 自动导入 Loki 日志面板(打开即可看图;密码被用户修改过则跳过并提示) if err := d.importDashboard(ctx, env, ns); err != nil { d.log("warn", fmt.Sprintf("面板自动导入跳过: %v", err), "") } else { d.log("info", "已导入「Loki 日志」面板(Explore 可用,默认账号 admin/admin)", "") } return nil } // importDashboard 通过 Grafana API 导入预置的 Loki 日志面板 func (d *GrafanaDeployer) importDashboard(ctx context.Context, env []string, ns string) error { payload := grafanaDashboardPayload() postDashboard := func(baseURL string) error { req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimSuffix(baseURL, "/")+"/api/dashboards/db", strings.NewReader(payload)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("admin:admin"))) resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode >= 400 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) } return nil } switch d.deployment.Mode { case "k8s-helm", "k8s-yaml": // 经 NodePort 直连(kube service proxy 的 POST 需额外 RBAC,直连最稳) ipCmd := exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", "jsonpath={.items[0].status.addresses[0].address}") ipCmd.Env = env ipOut, _ := ipCmd.CombinedOutput() portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns, "-o", "jsonpath={.spec.ports[0].nodePort}") portCmd.Env = env portOut, _ := portCmd.CombinedOutput() ip, port := strings.TrimSpace(string(ipOut)), strings.TrimSpace(string(portOut)) if ip == "" || port == "" { return fmt.Errorf("获取 NodePort 失败") } return postDashboard(fmt.Sprintf("http://%s:%s", ip, port)) case "host-binary": if d.deployment.Host == nil { return fmt.Errorf("no host") } return postDashboard(fmt.Sprintf("http://%s:3000", d.deployment.Host.HostIP)) } return fmt.Errorf("unsupported mode") } // ==================== 主机 tarball 部署 ==================== func (d *GrafanaDeployer) deployHost(ctx context.Context) error { host := d.deployment.Host if host == nil { return fmt.Errorf("please select target host") } client, err := NewSSHClientWithAuth(HostSSHAuth(host)) if err != nil { return err } defer client.Close() arch, _, _ := client.Execute(ctx, "uname -m") grafanaArch := "amd64" if strings.Contains(arch, "aarch64") || strings.Contains(arch, "arm64") { grafanaArch = "arm64" } version := d.version() d.log("info", fmt.Sprintf("Downloading Grafana %s (%s)...", version, grafanaArch), "") url := fmt.Sprintf("https://dl.grafana.com/oss/release/grafana-%s.linux-%s.tar.gz", version, grafanaArch) dl := fmt.Sprintf("curl -fsSL -o /tmp/grafana.tar.gz %s && tar -xzf /tmp/grafana.tar.gz -C /opt/ && rm -rf /opt/grafana && ln -sfn /opt/grafana-v%s /opt/grafana && rm -f /tmp/grafana.tar.gz", url, version) if out, _, err := client.Execute(ctx, dl); err != nil { return fmt.Errorf("download grafana failed: %w\n%s", err, out) } // 数据源 provisioning(指向同机 Loki) dsYAML := fmt.Sprintf(`apiVersion: 1 datasources: - name: Loki type: loki access: proxy url: %s isDefault: true `, d.lokiPushURL()) client.Execute(ctx, "mkdir -p /etc/grafana/provisioning/datasources /var/lib/grafana /var/log/grafana") writeCmd := fmt.Sprintf("cat > /etc/grafana/provisioning/datasources/loki.yaml <<'EOF'\n%s\nEOF", dsYAML) if out, _, err := client.Execute(ctx, writeCmd); err != nil { return fmt.Errorf("write datasource config failed: %w\n%s", err, out) } unit := `[Unit] Description=Grafana After=network.target [Service] Type=simple User=root WorkingDirectory=/opt/grafana ExecStart=/opt/grafana/bin/grafana server --homepath /opt/grafana --config /opt/grafana/conf/defaults.ini Restart=always LimitNOFILE=65536 [Install] WantedBy=multi-user.target` writeUnit := fmt.Sprintf("cat > /etc/systemd/system/grafana.service <<'EOF'\n%s\nEOF", unit) if out, _, err := client.Execute(ctx, writeUnit); err != nil { return fmt.Errorf("write systemd unit failed: %w\n%s", err, out) } if out, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable grafana && systemctl restart grafana"); err != nil { return fmt.Errorf("start grafana failed: %w\n%s", err, out) } // 健康检查(最多 10 次) for i := 0; i < 10; i++ { out, _, _ := client.Execute(ctx, "curl -fsSL http://localhost:3000/api/health") if strings.Contains(out, "\"version\"") { d.log("success", fmt.Sprintf("Grafana running on %s:3000 (admin/admin), 数据源已指向本机 Loki", host.HostIP), "") return nil } select { case <-ctx.Done(): return ctx.Err() case <-time.After(2 * time.Second): } } d.log("warn", "Grafana 服务已启动但健康检查未通过,请稍后确认 http://<主机IP>:3000", "") return nil } // CleanupGrafana 卸载 Grafana(停止部署时同步调用) func CleanupGrafana(ctx context.Context, deployment *models.LokiDeployment) { if deployment == nil { return } switch deployment.Mode { case "k8s-helm", "k8s-yaml": if deployment.Cluster == nil { return } env, cleanup, err := kubeconfigEnv(deployment.Cluster) if err != nil { return } defer cleanup() ns := deployment.Namespace if ns == "" { ns = "log" } cmd := exec.CommandContext(ctx, "kubectl", "delete", "deployment/grafana", "service/grafana", "configmap/grafana-datasources", "-n", ns, "--ignore-not-found=true") cmd.Env = env cmd.CombinedOutput() case "host-binary": if deployment.Host == nil { return } client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host)) if err != nil { return } defer client.Close() client.Execute(ctx, "systemctl stop grafana 2>/dev/null; systemctl disable grafana 2>/dev/null; rm -f /etc/systemd/system/grafana.service; systemctl daemon-reload") } } // grafanaDashboardPayload 预置「Loki 日志」面板:日志量(按命名空间)/错误日志/最近日志 func grafanaDashboardPayload() string { return `{"dashboard":{ "uid":"loki-logs","title":"Loki 日志","tags":["lokideploy"], "timezone":"browser","schemaVersion":39,"refresh":"30s","version":1, "time":{"from":"now-1h","to":"now"}, "panels":[ {"id":1,"type":"timeseries","title":"日志量(按命名空间)", "datasource":{"type":"loki","uid":"loki-ds"}, "gridPos":{"h":8,"w":12,"x":0,"y":0}, "fieldConfig":{"defaults":{"unit":"short"},"overrides":[]}, "targets":[{"refId":"A","expr":"sum by (namespace) (count_over_time({namespace=\".+\"}[5m]))","queryType":"range"}]}, {"id":2,"type":"timeseries","title":"错误日志", "datasource":{"type":"loki","uid":"loki-ds"}, "gridPos":{"h":8,"w":12,"x":12,"y":0}, "fieldConfig":{"defaults":{"unit":"short","color":{"mode":"palette-classic"}},"overrides":[]}, "targets":[{"refId":"A","expr":"sum(count_over_time({namespace=\".+\"} |~ \"(?i)error\" [5m]))","queryType":"range"}]}, {"id":3,"type":"logs","title":"最近日志", "datasource":{"type":"loki","uid":"loki-ds"}, "gridPos":{"h":10,"w":24,"x":0,"y":8}, "targets":[{"refId":"A","expr":"{namespace=\".+\"}","queryType":"range"}]} ] },"overwrite":true}` }