LokiStack/backend/services/grafana_deployer.go

package services

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"os/exec"
	"strings"
	"time"

	"opsxloki/models"
)

// GrafanaDeployer 与 Loki 同环境部署 Grafana 并自动配置 Loki 数据源。
// k8s 模式:manifest(Deployment + NodePort Service + 数据源 ConfigMap),与 Loki 同 namespace;
// host-binary 模式:tarball + systemd,与 Loki 同机。
type GrafanaDeployer struct {
	deployment *models.LokiDeployment
	log        func(level, message, detail string)
}

const defaultGrafanaVersion = "11.3.0"

func (d *GrafanaDeployer) version() string {
	if d.deployment.GrafanaVersion != "" {
		return d.deployment.GrafanaVersion
	}
	return defaultGrafanaVersion
}

// lokiPushURL 数据源指向的 Loki 地址(k8s 走集群内服务名,主机走本机)
func (d *GrafanaDeployer) lokiPushURL() string {
	if d.deployment.Mode == "host-binary" {
		return "http://127.0.0.1:3100"
	}
	ns := d.deployment.Namespace
	if ns == "" {
		ns = "log"
	}
	return fmt.Sprintf("http://loki.%s.svc.cluster.local:3100", ns)
}

// Deploy 部署入口:按模式分发
func (d *GrafanaDeployer) Deploy(ctx context.Context) error {
	d.log("info", fmt.Sprintf("Deploying Grafana %s (mode: %s)...", d.version(), d.deployment.Mode), "")
	switch d.deployment.Mode {
	case "k8s-helm", "k8s-yaml":
		return d.deployK8s(ctx)
	case "host-binary":
		return d.deployHost(ctx)
	default:
		return fmt.Errorf("unsupported mode for grafana: %s", d.deployment.Mode)
	}
}

// ==================== K8s manifest 部署 ====================

// grafanaManifest 生成 Grafana 资源;lokiURL 显式传入(默认服务名,DNS 异常的集群可回退 ClusterIP)。
// 数据卷用 emptyDir:数据源/配置由 ConfigMap 供给,重启即恢复;避免无默认
// StorageClass 的集群上 PVC Pending 阻塞(UI 内自建的面板需重新导入,见已知限制)。
func (d *GrafanaDeployer) grafanaManifest(ns, lokiURL string) string {
	version := d.version()

	datasources := fmt.Sprintf(`apiVersion: 1
datasources:
- name: Loki
  type: loki
  uid: loki-ds
  access: proxy
  url: %s
  isDefault: true
  editable: false`, lokiURL)

	return fmt.Sprintf(`apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources
  namespace: %s
data:
  datasources.yaml: |
%s
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: %s
  labels:
    app: grafana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
      - name: grafana
        image: grafana/grafana:%s
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 3000
        env:
        - name: GF_SECURITY_ADMIN_USER
          value: admin
        - name: GF_SECURITY_ADMIN_PASSWORD
          value: admin
        readinessProbe:
          httpGet:
            path: /api/health
            port: http
          initialDelaySeconds: 10
          periodSeconds: 10
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 512Mi
        volumeMounts:
        - name: datasources
          mountPath: /etc/grafana/provisioning/datasources
      volumes:
      - name: datasources
        configMap:
          name: grafana-datasources
---
apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: %s
  labels:
    app: grafana
spec:
  # NodePort 便于集群外直接访问(端口由集群自动分配,实际端口见部署日志)
  type: NodePort
  ports:
  - name: http
    port: 3000
    targetPort: 3000
  selector:
    app: grafana
`, ns, indentBlock(datasources, "    "), ns, version, ns)
}

func (d *GrafanaDeployer) deployK8s(ctx context.Context) error {
	if d.deployment.Cluster == nil {
		return fmt.Errorf("please select target cluster")
	}
	env, cleanup, err := kubeconfigEnv(d.deployment.Cluster)
	if err != nil {
		return err
	}
	defer cleanup()

	ns := d.deployment.Namespace
	if ns == "" {
		ns = "log"
	}

	lokiURL := d.lokiPushURL()
	manifestFile, manifestCleanup, err := writeTempFile("grafana-*.yaml", d.grafanaManifest(ns, lokiURL), 0600)
	if err != nil {
		return err
	}
	defer manifestCleanup()

	d.log("info", "Applying Grafana manifest (Deployment + NodePort Service + datasource ConfigMap)...", "")
	cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", manifestFile)
	cmd.Env = env
	out, err := cmd.CombinedOutput()
	if err != nil {
		d.log("error", fmt.Sprintf("kubectl apply grafana failed: %v", err), string(out))
		return fmt.Errorf("kubectl apply grafana failed: %w\n%s", err, string(out))
	}

	waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
		"-n", ns, "--timeout=5m")
	waitCmd.Env = env
	waitOut, _ := waitCmd.CombinedOutput()
	d.log("info", string(waitOut), "")

	// DNS 自愈:部分集群 CoreDNS 对 Service 记录异常(如 watch 失效),
	// 经 grafana 容器探测 Loki 服务名,不通则回退 ClusterIP 并热更新数据源。
	dnsProbe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, "deploy/grafana", "--",
		"wget", "-q", "-O", "/dev/null", "-T", "5", lokiURL+"/ready")
	dnsProbe.Env = env
	if probeOut, probeErr := dnsProbe.CombinedOutput(); probeErr != nil {
		d.log("warn", fmt.Sprintf("Grafana 无法解析 Loki 服务名(%s),回退 ClusterIP…", strings.TrimSpace(string(probeOut))), "")
		ipCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
			"-o", "jsonpath={.spec.clusterIP}")
		ipCmd.Env = env
		ipOut, ipErr := ipCmd.CombinedOutput()
		clusterIP := strings.TrimSpace(string(ipOut))
		if ipErr != nil || clusterIP == "" {
			d.log("error", "获取 Loki ClusterIP 失败,数据源保持服务名形式", "")
		} else {
			fallbackURL := fmt.Sprintf("http://%s:3100", clusterIP)
			fbFile, fbCleanup, ferr := writeTempFile("grafana-fb-*.yaml", d.grafanaManifest(ns, fallbackURL), 0600)
			if ferr == nil {
				fbApply := exec.CommandContext(ctx, "kubectl", "apply", "-f", fbFile)
				fbApply.Env = env
				fbApply.CombinedOutput()
				fbCleanup()
				restart := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "deployment/grafana", "-n", ns)
				restart.Env = env
				restart.CombinedOutput()
				wait2 := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
					"-n", ns, "--timeout=5m")
				wait2.Env = env
				w2, _ := wait2.CombinedOutput()
				d.log("info", string(w2), "")
				d.log("info", fmt.Sprintf("数据源已回退为 ClusterIP: %s", fallbackURL), "")
			}
		}
	}

	// 回读实际分配的 NodePort(固定端口易与集群已有服务冲突)
	portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
		"-o", "jsonpath={.spec.ports[0].nodePort}")
	portCmd.Env = env
	portOut, _ := portCmd.CombinedOutput()
	nodePort := strings.TrimSpace(string(portOut))
	access := fmt.Sprintf("http://<节点IP>:%s", nodePort)
	if nodePort == "" {
		access = "kubectl get svc grafana -n " + ns + " 查看访问端口"
	}

	d.log("success", fmt.Sprintf("Grafana deployed. 访问: %s (NodePort, 账号 admin/admin), 数据源已指向 %s", access, d.lokiPushURL()), "")

	// 自动导入 Loki 日志面板(打开即可看图;密码被用户修改过则跳过并提示)
	if err := d.importDashboard(ctx, env, ns); err != nil {
		d.log("warn", fmt.Sprintf("面板自动导入跳过: %v", err), "")
	} else {
		d.log("info", "已导入「Loki 日志」面板(Explore 可用,默认账号 admin/admin)", "")
	}
	return nil
}

// importDashboard 通过 Grafana API 导入预置的 Loki 日志面板
func (d *GrafanaDeployer) importDashboard(ctx context.Context, env []string, ns string) error {
	payload := grafanaDashboardPayload()

	postDashboard := func(baseURL string) error {
		req, err := http.NewRequestWithContext(ctx, http.MethodPost,
			strings.TrimSuffix(baseURL, "/")+"/api/dashboards/db", strings.NewReader(payload))
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("admin:admin")))
		resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		if resp.StatusCode >= 400 {
			body, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
			return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
		}
		return nil
	}

	switch d.deployment.Mode {
	case "k8s-helm", "k8s-yaml":
		// 经 NodePort 直连(kube service proxy 的 POST 需额外 RBAC,直连最稳)
		ipCmd := exec.CommandContext(ctx, "kubectl", "get", "nodes",
			"-o", "jsonpath={.items[0].status.addresses[0].address}")
		ipCmd.Env = env
		ipOut, _ := ipCmd.CombinedOutput()
		portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
			"-o", "jsonpath={.spec.ports[0].nodePort}")
		portCmd.Env = env
		portOut, _ := portCmd.CombinedOutput()
		ip, port := strings.TrimSpace(string(ipOut)), strings.TrimSpace(string(portOut))
		if ip == "" || port == "" {
			return fmt.Errorf("获取 NodePort 失败")
		}
		return postDashboard(fmt.Sprintf("http://%s:%s", ip, port))
	case "host-binary":
		if d.deployment.Host == nil {
			return fmt.Errorf("no host")
		}
		return postDashboard(fmt.Sprintf("http://%s:3000", d.deployment.Host.HostIP))
	}
	return fmt.Errorf("unsupported mode")
}

// ==================== 主机 tarball 部署 ====================

func (d *GrafanaDeployer) deployHost(ctx context.Context) error {
	host := d.deployment.Host
	if host == nil {
		return fmt.Errorf("please select target host")
	}
	client, err := NewSSHClientWithAuth(HostSSHAuth(host))
	if err != nil {
		return err
	}
	defer client.Close()

	arch, _, _ := client.Execute(ctx, "uname -m")
	grafanaArch := "amd64"
	if strings.Contains(arch, "aarch64") || strings.Contains(arch, "arm64") {
		grafanaArch = "arm64"
	}
	version := d.version()

	d.log("info", fmt.Sprintf("Downloading Grafana %s (%s)...", version, grafanaArch), "")
	url := fmt.Sprintf("https://dl.grafana.com/oss/release/grafana-%s.linux-%s.tar.gz", version, grafanaArch)
	dl := fmt.Sprintf("curl -fsSL -o /tmp/grafana.tar.gz %s && tar -xzf /tmp/grafana.tar.gz -C /opt/ && rm -rf /opt/grafana && ln -sfn /opt/grafana-v%s /opt/grafana && rm -f /tmp/grafana.tar.gz", url, version)
	if out, _, err := client.Execute(ctx, dl); err != nil {
		return fmt.Errorf("download grafana failed: %w\n%s", err, out)
	}

	// 数据源 provisioning(指向同机 Loki)
	dsYAML := fmt.Sprintf(`apiVersion: 1
datasources:
- name: Loki
  type: loki
  access: proxy
  url: %s
  isDefault: true
`, d.lokiPushURL())
	client.Execute(ctx, "mkdir -p /etc/grafana/provisioning/datasources /var/lib/grafana /var/log/grafana")
	writeCmd := fmt.Sprintf("cat > /etc/grafana/provisioning/datasources/loki.yaml <<'EOF'\n%s\nEOF", dsYAML)
	if out, _, err := client.Execute(ctx, writeCmd); err != nil {
		return fmt.Errorf("write datasource config failed: %w\n%s", err, out)
	}

	unit := `[Unit]
Description=Grafana
After=network.target

[Service]
Type=simple
User=root
WorkingDirectory=/opt/grafana
ExecStart=/opt/grafana/bin/grafana server --homepath /opt/grafana --config /opt/grafana/conf/defaults.ini
Restart=always
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target`
	writeUnit := fmt.Sprintf("cat > /etc/systemd/system/grafana.service <<'EOF'\n%s\nEOF", unit)
	if out, _, err := client.Execute(ctx, writeUnit); err != nil {
		return fmt.Errorf("write systemd unit failed: %w\n%s", err, out)
	}
	if out, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable grafana && systemctl restart grafana"); err != nil {
		return fmt.Errorf("start grafana failed: %w\n%s", err, out)
	}

	// 健康检查(最多 10 次)
	for i := 0; i < 10; i++ {
		out, _, _ := client.Execute(ctx, "curl -fsSL http://localhost:3000/api/health")
		if strings.Contains(out, "\"version\"") {
			d.log("success", fmt.Sprintf("Grafana running on %s:3000 (admin/admin), 数据源已指向本机 Loki", host.HostIP), "")
			return nil
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(2 * time.Second):
		}
	}
	d.log("warn", "Grafana 服务已启动但健康检查未通过,请稍后确认 http://<主机IP>:3000", "")
	return nil
}

// CleanupGrafana 卸载 Grafana(停止部署时同步调用)
func CleanupGrafana(ctx context.Context, deployment *models.LokiDeployment) {
	if deployment == nil {
		return
	}
	switch deployment.Mode {
	case "k8s-helm", "k8s-yaml":
		if deployment.Cluster == nil {
			return
		}
		env, cleanup, err := kubeconfigEnv(deployment.Cluster)
		if err != nil {
			return
		}
		defer cleanup()
		ns := deployment.Namespace
		if ns == "" {
			ns = "log"
		}
		cmd := exec.CommandContext(ctx, "kubectl", "delete",
			"deployment/grafana", "service/grafana", "configmap/grafana-datasources",
			"-n", ns, "--ignore-not-found=true")
		cmd.Env = env
		cmd.CombinedOutput()
	case "host-binary":
		if deployment.Host == nil {
			return
		}
		client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
		if err != nil {
			return
		}
		defer client.Close()
		client.Execute(ctx, "systemctl stop grafana 2>/dev/null; systemctl disable grafana 2>/dev/null; rm -f /etc/systemd/system/grafana.service; systemctl daemon-reload")
	}
}

// grafanaDashboardPayload 预置「Loki 日志」面板:日志量(按命名空间)/错误日志/最近日志
func grafanaDashboardPayload() string {
	return `{"dashboard":{
		"uid":"loki-logs","title":"Loki 日志","tags":["lokideploy"],
		"timezone":"browser","schemaVersion":39,"refresh":"30s","version":1,
		"time":{"from":"now-1h","to":"now"},
		"panels":[
			{"id":1,"type":"timeseries","title":"日志量(按命名空间)",
			 "datasource":{"type":"loki","uid":"loki-ds"},
			 "gridPos":{"h":8,"w":12,"x":0,"y":0},
			 "fieldConfig":{"defaults":{"unit":"short"},"overrides":[]},
			 "targets":[{"refId":"A","expr":"sum by (namespace) (count_over_time({namespace=\".+\"}[5m]))","queryType":"range"}]},
			{"id":2,"type":"timeseries","title":"错误日志",
			 "datasource":{"type":"loki","uid":"loki-ds"},
			 "gridPos":{"h":8,"w":12,"x":12,"y":0},
			 "fieldConfig":{"defaults":{"unit":"short","color":{"mode":"palette-classic"}},"overrides":[]},
			 "targets":[{"refId":"A","expr":"sum(count_over_time({namespace=\".+\"} |~ \"(?i)error\" [5m]))","queryType":"range"}]},
			{"id":3,"type":"logs","title":"最近日志",
			 "datasource":{"type":"loki","uid":"loki-ds"},
			 "gridPos":{"h":10,"w":24,"x":0,"y":8},
			 "targets":[{"refId":"A","expr":"{namespace=\".+\"}","queryType":"range"}]}
		]
	},"overwrite":true}`
}