LokiStack/backend/services/loki_deployer.go

package services

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"text/template"

	"opsxloki/models"
	"opsxloki/pkg/crypto"
)

// LokiDeployer 负责 Loki 的三种部署模式：k8s-helm / k8s-yaml / host-binary
type LokiDeployer struct {
	deployment *models.LokiDeployment
	logFunc    func(level, message, detail string)
}

func NewLokiDeployer(deployment *models.LokiDeployment, logFunc func(level, message, detail string)) *LokiDeployer {
	return &LokiDeployer{deployment: deployment, logFunc: logFunc}
}

// Deploy 根据部署模式分发到对应实现
func (d *LokiDeployer) Deploy(ctx context.Context) error {
	d.log("info", fmt.Sprintf("Starting Loki deployment: %s (mode: %s, version: %s)",
		d.deployment.Name, d.deployment.Mode, d.effectiveVersion()), "")

	// 版本兼容性检查
	if err := d.checkVersionCompatibility(); err != nil {
		d.log("warn", err.Error(), "")
	}

	var err error
	switch d.deployment.Mode {
	case "k8s-helm":
		err = d.deployHelm(ctx)
	case "k8s-yaml":
		err = d.deployYaml(ctx)
	case "host-binary":
		err = d.deployBinary(ctx)
	default:
		return fmt.Errorf("unsupported deployment mode: %s", d.deployment.Mode)
	}
	if err != nil {
		return err
	}

	// Loki 就绪后,按需同环境部署 Grafana(数据源自动指向本 Loki)
	if d.deployment.GrafanaEnabled {
		g := &GrafanaDeployer{deployment: d.deployment, log: d.log}
		if gerr := g.Deploy(ctx); gerr != nil {
			d.log("error", fmt.Sprintf("Grafana deployment failed: %v", gerr), "")
			return fmt.Errorf("grafana deployment failed: %w", gerr)
		}
	}
	return nil
}

// ==================== Helm 部署 (grafana/loki v6+) ====================

func (d *LokiDeployer) deployHelm(ctx context.Context) error {
	if d.deployment.Cluster == nil {
		return fmt.Errorf("please select target cluster")
	}

	kubeconfig, err := resolveKubeconfig(d.deployment.Cluster)
	if err != nil {
		return err
	}

	kubeFile, cleanup, err := writeTempFile("kube-*.yaml", kubeconfig, 0600)
	if err != nil {
		return err
	}
	defer cleanup()

	env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile))
	ns := d.effectiveNamespace()

	// 创建 namespace（幂等）
	d.log("info", fmt.Sprintf("Ensuring namespace %s...", ns), "")
	nsCmd := exec.CommandContext(ctx, "kubectl", "create", "namespace", ns, "--dry-run=client", "-o", "yaml")
	nsCmd.Env = env
	nsOut, _ := nsCmd.CombinedOutput()
	applyNs := exec.CommandContext(ctx, "kubectl", "apply", "-f", "-")
	applyNs.Env = env
	applyNs.Stdin = bytes.NewReader(nsOut)
	applyNs.CombinedOutput()

	// 添加 Helm repo
	d.log("info", "Adding/updating Grafana Helm repo...", "")
	repoCmd := exec.CommandContext(ctx, "helm", "repo", "add", "grafana", "https://grafana.github.io/helm-charts")
	repoCmd.Env = env
	repoCmd.CombinedOutput()
	updCmd := exec.CommandContext(ctx, "helm", "repo", "update")
	updCmd.Env = env
	updCmd.CombinedOutput()

	// 生成 values.yaml
	valuesYAML := d.generateHelmValues()
	valuesFile, valuesCleanup, err := writeTempFile("loki-values-*.yaml", valuesYAML, 0600)
	if err != nil {
		return err
	}
	defer valuesCleanup()

	// Helm install/upgrade - 统一使用 grafana/loki (v6+)
	d.log("info", fmt.Sprintf("Installing Loki via Helm (grafana/loki, mode: %s)...", d.helmDeploymentMode()), "")
	releaseName := d.effectiveReleaseName()

	cmd := exec.CommandContext(ctx, "helm", "upgrade", "--install", releaseName, "grafana/loki",
		"--namespace", ns, "--create-namespace", "--wait", "--timeout", "10m",
		"-f", valuesFile,
	)
	cmd.Env = env
	out, err := cmd.CombinedOutput()
	if err != nil {
		d.log("error", fmt.Sprintf("Helm install failed: %v", err), string(out))
		return fmt.Errorf("helm install failed: %w\n%s", err, string(out))
	}
	d.log("success", "Loki installed via Helm", string(out))
	return nil
}

// ==================== K8s YAML 部署 (StatefulSet) ====================

func (d *LokiDeployer) deployYaml(ctx context.Context) error {
	if d.deployment.Cluster == nil {
		return fmt.Errorf("please select target cluster")
	}

	kubeconfig, err := resolveKubeconfig(d.deployment.Cluster)
	if err != nil {
		return err
	}

	kubeFile, cleanup, err := writeTempFile("kube-*.yaml", kubeconfig, 0600)
	if err != nil {
		return err
	}
	defer cleanup()

	env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile))
	ns := d.effectiveNamespace()

	// 生成完整 manifest
	manifest := d.generateManifest(ns)
	manifestFile, manifestCleanup, err := writeTempFile("loki-*.yaml", manifest, 0600)
	if err != nil {
		return err
	}
	defer manifestCleanup()

	d.log("info", "Applying Loki manifest (StatefulSet + PVC + ConfigMap + Service)...", "")
	cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", manifestFile)
	cmd.Env = env
	out, err := cmd.CombinedOutput()
	if err != nil {
		d.log("error", fmt.Sprintf("kubectl apply failed: %v", err), string(out))
		return fmt.Errorf("kubectl apply failed: %w\n%s", err, string(out))
	}

	// 等待 StatefulSet ready
	d.log("info", "Waiting for StatefulSet to be ready...", "")
	waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset/loki",
		"-n", ns, "--timeout=5m")
	waitCmd.Env = env
	waitOut, _ := waitCmd.CombinedOutput()
	d.log("info", string(waitOut), "")

	d.log("success", "Loki deployed via YAML manifest", string(out))
	return nil
}

// ==================== 主机二进制部署 ====================

func (d *LokiDeployer) deployBinary(ctx context.Context) error {
	if d.deployment.Host == nil {
		return fmt.Errorf("please select target host")
	}
	host := d.deployment.Host
	client, err := NewSSHClientWithAuth(HostSSHAuth(host))
	if err != nil {
		return err
	}
	defer client.Close()

	// 检测架构
	arch, _, _ := client.Execute(ctx, "uname -m")
	lokiArch := strings.TrimSpace(arch)
	switch lokiArch {
	case "x86_64":
		lokiArch = "amd64"
	case "aarch64":
		lokiArch = "arm64"
	}

	version := d.effectiveVersion()
	d.log("info", fmt.Sprintf("SSH connected, downloading Loki %s (%s)...", version, lokiArch), "")

	// 创建目录
	client.Execute(ctx, "mkdir -p /opt/loki /var/lib/loki /var/lib/loki/rules /var/lib/loki/rules-temp /etc/loki")

	// 下载 Loki
	downloadURL := fmt.Sprintf("https://github.com/grafana/loki/releases/download/v%s/loki-linux-%s.zip", version, lokiArch)
	cmd := fmt.Sprintf("cd /tmp && curl -fsSL -o loki.zip '%s' && unzip -o loki.zip -d /opt/loki/ && chmod +x /opt/loki/loki-linux-%s",
		downloadURL, lokiArch)
	if _, _, err := client.Execute(ctx, cmd); err != nil {
		return fmt.Errorf("download Loki failed: %w", err)
	}
	client.Execute(ctx, "ln -sf /opt/loki/loki-linux-* /usr/local/bin/loki 2>/dev/null || true")

	// 上传配置
	config := d.generateLokiConfig()
	client.Execute(ctx, fmt.Sprintf("cat > /etc/loki/loki.yaml << 'EOF'\n%s\nEOF", config))

	// systemd 服务（生产级配置）
	service := `[Unit]
Description=Grafana Loki
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/bin/loki -config.file=/etc/loki/loki.yaml
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
RestartSec=10
LimitNOFILE=65536
StandardOutput=append:/var/log/loki.log
StandardError=append:/var/log/loki.log

[Install]
WantedBy=multi-user.target
`
	client.Execute(ctx, fmt.Sprintf("cat > /etc/systemd/system/loki.service << 'EOF'\n%s\nEOF", service))

	// 启动服务
	if _, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable loki && systemctl restart loki"); err != nil {
		return fmt.Errorf("start Loki failed: %w", err)
	}

	// 健康检查
	d.log("info", "Waiting for Loki to be ready...", "")
	for i := 0; i < 10; i++ {
		_, _, err := client.Execute(ctx, "curl -fsSL http://localhost:3100/ready")
		if err == nil {
			break
		}
		client.Execute(ctx, "sleep 2")
	}

	d.log("success", fmt.Sprintf("Loki %s deployed on %s", version, host.HostIP), "")
	return nil
}

// ==================== Helm Values 生成 ====================

func (d *LokiDeployer) generateHelmValues() string {
	mode := d.helmDeploymentMode()
	version := d.effectiveVersion()

	// 构建 storage 配置
	storageSection := d.buildHelmStorageSection()
	schemaSection := d.buildHelmSchemaSection()

	// 资源限制 + 持久化（合并到同一顶层键下，避免重复键导致 Helm 丢弃 resources）
	deploymentSection := d.buildHelmDeploymentSection()

	// 认证配置
	authSection := ""
	if d.deployment.AuthEnabled {
		authSection = `
auth:
  type: basic
  basic:
    username: admin`
	}

	values := fmt.Sprintf(`# Auto-generated Loki Helm Values
# Chart: grafana/loki (v6+)
# Deployment Mode: %s
# Version: %s

deploymentMode: %s

loki:
  image:
    tag: %s
  auth_enabled: %v
  commonConfig:
    replication_factor: %d
    path_prefix: /var/loki
  %s
  limits_config:
    retention_period: %dh
    ingestion_rate_mb: 16
    ingestion_burst_size_mb: 32
    per_stream_rate_limit: 3MB
    per_stream_rate_limit_burst: 15MB
    max_chunks_per_query: 2000000
    max_query_series: 1000
    max_query_parallelism: 32
  %s
  %s
  analytics:
    reporting_enabled: false

%s

monitoring:
  serviceMonitor:
    enabled: false
  selfMonitoring:
    enabled: false
    grafanaAgent:
      installOperator: false
  lokiCanary:
    enabled: false

test:
  enabled: false
`, mode, version, mode, version, d.deployment.AuthEnabled,
		d.replicationFactor(), storageSection, d.deployment.RetentionDays, schemaSection,
		authSection, deploymentSection)

	return values
}

func (d *LokiDeployer) helmDeploymentMode() string {
	if d.deployment.Replicas > 1 {
		return "Distributed"
	}
	return "SingleBinary"
}

func (d *LokiDeployer) replicationFactor() int {
	if d.deployment.Replicas >= 3 {
		return 3
	}
	if d.deployment.Replicas >= 2 {
		return 2
	}
	return 1
}

func (d *LokiDeployer) buildHelmStorageSection() string {
	storageType := d.effectiveStorageType()

	switch storageType {
	case "s3":
		sc := d.deployment.StorageConfig
		endpoint := sc.Endpoint
		if endpoint == "" {
			endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
		}
		return fmt.Sprintf(`storage:
    type: s3
    bucketNames:
      chunks: %s
      ruler: %s-ruler
      admin: %s-admin
    s3:
      endpoint: %s
      region: %s
      accessKeyId: %s
      secretAccessKey: %s
      insecure: %v
      s3ForcePathStyle: true`, sc.Bucket, sc.Bucket, sc.Bucket, endpoint, sc.Region, sc.AccessKey, sc.SecretKey, sc.Insecure)
	case "oss":
		sc := d.deployment.StorageConfig
		return fmt.Sprintf(`storage:
    type: s3
    bucketNames:
      chunks: %s
      ruler: %s-ruler
      admin: %s-admin
    s3:
      endpoint: %s
      region: %s
      accessKeyId: %s
      secretAccessKey: %s`, sc.Bucket, sc.Bucket, sc.Bucket, sc.Endpoint, sc.Region, sc.AccessKey, sc.SecretKey)
	case "minio":
		sc := d.deployment.StorageConfig
		return fmt.Sprintf(`storage:
    type: s3
    bucketNames:
      chunks: %s
      ruler: %s-ruler
      admin: %s-admin
    s3:
      endpoint: %s
      accessKeyId: %s
      secretAccessKey: %s
      insecure: %v
      s3ForcePathStyle: true`, sc.Bucket, sc.Bucket, sc.Bucket, sc.Endpoint, sc.AccessKey, sc.SecretKey, sc.Insecure)
	default:
		// filesystem
		return `storage:
    type: filesystem`
	}
}

func (d *LokiDeployer) buildHelmSchemaSection() string {
	storageType := d.effectiveStorageType()
	objectStore := "filesystem"
	switch storageType {
	case "s3", "oss", "minio":
		objectStore = "s3"
	case "gcs":
		objectStore = "gcs"
	case "azure":
		objectStore = "azure"
	}

	return fmt.Sprintf(`schemaConfig:
    configs:
      - from: "2024-01-01"
        store: tsdb
        object_store: %s
        schema: v13
        index:
          prefix: tsdb_index_
          period: 24h`, objectStore)
}

// buildHelmDeploymentSection 生成目标组件的 resources + persistence。
// 关键：必须把两者合并到同一顶层键(singleBinary / write / read / backend)之下，
// 否则 resources 与 persistence 各自 emit 一次同名顶层键，YAML/Helm 只取最后一个，
// 导致 resources 限制被静默丢弃。
func (d *LokiDeployer) buildHelmDeploymentSection() string {
	cpu := d.deployment.ResourcesCPU
	mem := d.deployment.ResourcesMem
	if cpu == "" {
		cpu = "1000m"
	}
	if mem == "" {
		mem = "1Gi"
	}
	storageSize := d.deployment.StorageSize
	if storageSize == "" {
		storageSize = "10Gi"
	}

	resources := fmt.Sprintf(`  resources:
    limits:
      cpu: %s
      memory: %s
    requests:
      cpu: 200m
      memory: 256Mi`, cpu, mem)

	persistence := fmt.Sprintf(`  persistence:
    enabled: true
    size: %s
    storageClassName: ""`, storageSize)

	if d.helmDeploymentMode() == "SingleBinary" {
		return fmt.Sprintf(`singleBinary:
  replicas: %d
%s
%s`, d.deployment.Replicas, resources, persistence)
	}

	// Distributed: write/backend 持久化，read 不持久化（无状态）
	return fmt.Sprintf(`write:
  replicas: %d
%s
%s
read:
  replicas: %d
%s
  persistence:
    enabled: false
backend:
  replicas: 1
%s
%s`,
		d.deployment.Replicas, resources, persistence,
		d.deployment.Replicas, resources,
		resources, persistence)
}

// ==================== K8s YAML Manifest 生成 (StatefulSet) ====================

func (d *LokiDeployer) generateManifest(ns string) string {
	version := d.effectiveVersion()
	replicas := d.deployment.Replicas
	if replicas < 1 {
		replicas = 1
	}
	storageSize := d.deployment.StorageSize
	if storageSize == "" {
		storageSize = "10Gi"
	}

	cpu := d.deployment.ResourcesCPU
	mem := d.deployment.ResourcesMem
	if cpu == "" {
		cpu = "1000m"
	}
	if mem == "" {
		mem = "1Gi"
	}

	configData := d.generateLokiConfig()

	manifest := fmt.Sprintf(`apiVersion: v1
kind: Namespace
metadata:
  name: %s
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-config
  namespace: %s
data:
  loki.yaml: |
%s
---
apiVersion: v1
kind: Service
metadata:
  name: loki
  namespace: %s
  labels:
    app: loki
spec:
  type: ClusterIP
  ports:
  - name: http
    port: 3100
    targetPort: 3100
    protocol: TCP
  - name: grpc
    port: 9095
    targetPort: 9095
    protocol: TCP
  selector:
    app: loki
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: loki-data
  namespace: %s
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: %s
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: loki
  namespace: %s
  labels:
    app: loki
spec:
  serviceName: loki
  replicas: %d
  selector:
    matchLabels:
      app: loki
  template:
    metadata:
      labels:
        app: loki
    spec:
      securityContext:
        runAsUser: 0
        runAsGroup: 0
        fsGroup: 0
      containers:
      - name: loki
        image: grafana/loki:%s
        imagePullPolicy: IfNotPresent
        args:
          - -config.file=/etc/loki/loki.yaml
        ports:
        - name: http
          containerPort: 3100
          protocol: TCP
        - name: grpc
          containerPort: 9095
          protocol: TCP
        resources:
          limits:
            cpu: "%s"
            memory: "%s"
          requests:
            cpu: "200m"
            memory: "256Mi"
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: false
          capabilities:
            drop:
            - ALL
        livenessProbe:
          httpGet:
            path: /ready
            port: http
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 6
        readinessProbe:
          httpGet:
            path: /ready
            port: http
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 3
        startupProbe:
          httpGet:
            path: /ready
            port: http
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 30
        volumeMounts:
        - name: config
          mountPath: /etc/loki
          readOnly: true
        - name: data
          mountPath: /var/lib/loki
        - name: wal
          mountPath: /var/lib/loki/wal
      volumes:
      - name: config
        configMap:
          name: loki-config
      - name: data
        persistentVolumeClaim:
          claimName: loki-data
      - name: wal
        emptyDir: {}
`, ns, ns, indentYAML(configData, 4), ns, ns, storageSize, ns, replicas, version, cpu, mem)

	// 多副本：追加 Headless Service（clusterIP: None + selector app:loki），
	// 使 memberlist 的 join_members (loki-headless.<ns>.svc.cluster.local) 能解析到所有 Pod IP。
	if replicas >= 2 {
		manifest += fmt.Sprintf(`
---
apiVersion: v1
kind: Service
metadata:
  name: loki-headless
  namespace: %s
  labels:
    app: loki
spec:
  clusterIP: None
  ports:
  - name: http
    port: 3100
    targetPort: 3100
    protocol: TCP
  - name: grpc
    port: 9095
    targetPort: 9095
    protocol: TCP
  - name: memberlist
    port: 7946
    targetPort: 7946
    protocol: TCP
  selector:
    app: loki
`, ns)
	}
	return manifest
}

// ==================== Loki 本地配置生成 (text/template) ====================

const lokiConfigTemplate = `auth_enabled: {{ .AuthEnabled }}

server:
  http_listen_port: 3100
  grpc_listen_port: 9095
  http_server_read_timeout: 3m
  http_server_write_timeout: 3m
  http_server_idle_timeout: 2m
  grpc_server_max_recv_msg_size: 104857600
  grpc_server_max_send_msg_size: 104857600
  grpc_server_max_concurrent_streams: 1000
  log_level: info
  log_format: logfmt

common:
  path_prefix: /var/lib/loki
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
  replication_factor: {{ .ReplicationFactor }}
  ring:
    instance_addr: {{ .RingInstanceAddr }}
    kvstore:
      store: {{ .RingKVStore }}

{{- if .StorageConfigBlock }}

storage_config:
{{ .StorageConfigBlock }}
{{- end }}
{{- if .MemberlistBlock }}

{{ .MemberlistBlock }}
{{- end }}

ingester:
  wal:
    enabled: true
    dir: /var/lib/loki/wal
    flush_on_shutdown: true
    replay_memory_ceiling: 1GB
  lifecycler:
    ring:
      kvstore:
        store: {{ .RingKVStore }}
      replication_factor: {{ .ReplicationFactor }}
    final_sleep: 0s
  chunk_idle_period: 30m
  chunk_retain_period: 30s
  chunk_target_size: 1048576
  chunk_encoding: snappy
  max_chunk_age: 2h
  autoforget_unhealthy: true

querier:
  max_concurrent: 10
  query_ingesters_within: 2h

query_scheduler:
  max_outstanding_requests_per_tenant: 4096

compactor:
  working_directory: /var/lib/loki/compactor
  compaction_interval: 10m
  retention_enabled: true
  retention_delete_delay: 2h
  retention_delete_worker_count: 150
  delete_request_store: {{ .CompactorStoreType }}

limits_config:
  retention_period: {{ .RetentionPeriod }}h
  ingestion_rate_mb: 16
  ingestion_burst_size_mb: 32
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  max_chunks_per_query: 2000000
  max_query_series: 1000
  max_query_parallelism: 32
  max_entries_limit_per_query: 5000
  max_label_name_length: 1024
  max_label_value_length: 2048
  max_label_names_per_series: 30
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  allow_structured_metadata: true

schema_config:
  configs:
    - from: "2024-01-01"
      store: tsdb
      object_store: {{ .ObjectStore }}
      schema: v13
      index:
        prefix: tsdb_index_
        period: 24h

ruler:
  storage:
    type: {{ .RulerStoreType }}
{{- if .RulerStorageBlock }}
{{ .RulerStorageBlock }}
{{- end }}
{{- if eq .RulerStoreType "local" }}
    local:
      directory: /var/lib/loki/rules
{{- end }}
  rule_path: /var/lib/loki/rules-temp
  alertmanager_url: http://localhost:9093
  ring:
    kvstore:
      store: {{ .RingKVStore }}
  enable_api: true
  enable_alertmanager_v2: true

frontend:
  max_outstanding_per_tenant: 4096
  compress_responses: true

frontend_worker: {}

chunk_store_config:
  chunk_cache_config:
    embedded_cache:
      enabled: true
      max_size_mb: 256
      ttl: 1h

analytics:
  reporting_enabled: false
`

type lokiConfigData struct {
	AuthEnabled        bool
	ReplicationFactor  int
	RetentionPeriod    int
	StorageConfigBlock string
	ObjectStore        string
	RulerStoreType     string
	CompactorStoreType string
	RingInstanceAddr   string // 单副本 127.0.0.1；多副本 0.0.0.0（须可被对端寻址）
	RingKVStore        string // 单副本 inmemory；多副本 memberlist
	MemberlistBlock    string // 多副本时非空，注入 memberlist 顶级配置
	RulerStorageBlock  string // 对象存储时非空，注入 ruler.storage.s3/gcs/azure 嵌套配置
}

func (d *LokiDeployer) generateLokiConfig() string {
	retentionHours := d.deployment.RetentionDays * 24
	if retentionHours <= 0 {
		retentionHours = 720 // 30 days default
	}

	storageType := d.effectiveStorageType()
	storageBlock := d.buildStorageConfigBlock()
	objectStore := "filesystem"
	rulerStore := "local"
	compactorStore := "filesystem"

	switch storageType {
	case "s3", "oss", "minio":
		objectStore = "s3"
		rulerStore = "s3"
		compactorStore = "s3"
	case "gcs":
		objectStore = "gcs"
		rulerStore = "gcs"
		compactorStore = "gcs"
	case "azure":
		objectStore = "azure"
		rulerStore = "azure"
		compactorStore = "azure"
	}

	// Ring 拓扑：单副本用 inmemory + 127.0.0.1（无需跨实例通信）；
	// 多副本必须切 memberlist + 0.0.0.0，否则 ring 无法收敛、写入失败。
	ringAddr := "127.0.0.1"
	ringStore := "inmemory"
	memberlistBlock := ""
	if d.deployment.Replicas >= 2 {
		ringAddr = "0.0.0.0"
		ringStore = "memberlist"
		ns := d.deployment.Namespace
		if ns == "" {
			ns = "log"
		}
		memberlistBlock = fmt.Sprintf(`memberlist:
  join_members:
    - loki-headless.%s.svc.cluster.local
  bind_port: 7946
  dead_node_reclaim_time: 30s
  gossip_to_dead_nodes_time: 2m
  left_ingesters_timeout: 5m`, ns)
	}

	data := lokiConfigData{
		AuthEnabled:        d.deployment.AuthEnabled,
		ReplicationFactor:  d.replicationFactor(),
		RetentionPeriod:    retentionHours,
		StorageConfigBlock: storageBlock,
		ObjectStore:        objectStore,
		RulerStoreType:     rulerStore,
		CompactorStoreType: compactorStore,
		RingInstanceAddr:   ringAddr,
		RingKVStore:        ringStore,
		MemberlistBlock:    memberlistBlock,
		RulerStorageBlock:  d.buildRulerStorageBlock(),
	}

	tmpl, err := template.New("loki-config").Parse(lokiConfigTemplate)
	if err != nil {
		return d.generateLokiConfigFallback(retentionHours)
	}

	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, data); err != nil {
		return d.generateLokiConfigFallback(retentionHours)
	}
	return buf.String()
}

func (d *LokiDeployer) buildStorageConfigBlock() string {
	if d.deployment.StorageConfig == nil {
		return ""
	}

	sc := d.deployment.StorageConfig
	switch sc.StorageType {
	case "s3":
		endpoint := sc.Endpoint
		if endpoint == "" {
			endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
		}
		insecure := "false"
		if sc.Insecure {
			insecure = "true"
		}
		return fmt.Sprintf(`  aws:
    s3: s3://%s/%s
    endpoint: %s
    region: %s
    access_key_id: %s
    secret_access_key: %s
    insecure: %s
    s3forcepathstyle: true`, sc.Bucket, sc.PathPrefix, endpoint, sc.Region, sc.AccessKey, sc.SecretKey, insecure)
	case "oss":
		return fmt.Sprintf(`  alibabacloud:
    bucket: %s
    endpoint: %s
    region: %s
    access_key_id: %s
    secret_access_key: %s`, sc.Bucket, sc.Endpoint, sc.Region, sc.AccessKey, sc.SecretKey)
	case "minio":
		insecure := "false"
		if sc.Insecure {
			insecure = "true"
		}
		return fmt.Sprintf(`  aws:
    s3: s3://%s/%s
    endpoint: %s
    access_key_id: %s
    secret_access_key: %s
    insecure: %s
    s3forcepathstyle: true`, sc.Bucket, sc.PathPrefix, sc.Endpoint, sc.AccessKey, sc.SecretKey, insecure)
	case "gcs":
		return fmt.Sprintf(`  gcs:
    bucket_name: %s`, sc.Bucket)
	case "azure":
		return fmt.Sprintf(`  azure:
    container_name: %s
    account_name: %s
    account_key: %s`, sc.Bucket, sc.AccessKey, sc.SecretKey)
	default:
		return ""
	}
}

// buildRulerStorageBlock 生成 ruler.storage 下与存储类型同名的嵌套配置
// (Loki 3.x 的 ruler 存储客户端不回退读取 storage_config,缺少此块会以空 bucket 启动失败)
func (d *LokiDeployer) buildRulerStorageBlock() string {
	if d.deployment.StorageConfig == nil {
		return ""
	}
	sc := d.deployment.StorageConfig
	switch sc.StorageType {
	case "s3":
		endpoint := sc.Endpoint
		if endpoint == "" {
			endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
		}
		insecure := "false"
		if sc.Insecure {
			insecure = "true"
		}
		return fmt.Sprintf(`    s3:
      s3: s3://%s/%s
      endpoint: %s
      region: %s
      access_key_id: %s
      secret_access_key: %s
      insecure: %s
      s3forcepathstyle: true`, sc.Bucket, sc.PathPrefix, endpoint, sc.Region, sc.AccessKey, sc.SecretKey, insecure)
	case "oss":
		return fmt.Sprintf(`    s3:
      s3: s3://%s/%s
      endpoint: %s
      access_key_id: %s
      secret_access_key: %s
      s3forcepathstyle: true`, sc.Bucket, sc.PathPrefix, sc.Endpoint, sc.AccessKey, sc.SecretKey)
	case "minio":
		insecure := "false"
		if sc.Insecure {
			insecure = "true"
		}
		return fmt.Sprintf(`    s3:
      s3: s3://%s/%s
      endpoint: %s
      access_key_id: %s
      secret_access_key: %s
      insecure: %s
      s3forcepathstyle: true`, sc.Bucket, sc.PathPrefix, sc.Endpoint, sc.AccessKey, sc.SecretKey, insecure)
	case "gcs":
		return fmt.Sprintf(`    gcs:
      bucket_name: %s`, sc.Bucket)
	case "azure":
		return fmt.Sprintf(`    azure:
      container_name: %s
      account_name: %s
      account_key: %s`, sc.Bucket, sc.AccessKey, sc.SecretKey)
	default:
		return ""
	}
}

// generateLokiConfigFallback 当模板解析失败时的降级配置
func (d *LokiDeployer) generateLokiConfigFallback(retentionHours int) string {
	return fmt.Sprintf(`auth_enabled: %v
server:
  http_listen_port: 3100
  grpc_listen_port: 9095
  http_server_read_timeout: 3m
  http_server_write_timeout: 3m
  log_level: info
common:
  path_prefix: /var/lib/loki
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
  replication_factor: %d
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory
ingester:
  wal:
    enabled: true
    dir: /var/lib/loki/wal
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: %d
    final_sleep: 0s
  chunk_idle_period: 30m
  max_chunk_age: 2h
querier:
  max_concurrent: 10
compactor:
  working_directory: /var/lib/loki/compactor
  retention_enabled: true
  retention_delete_delay: 2h
limits_config:
  retention_period: %dh
  ingestion_rate_mb: 16
  ingestion_burst_size_mb: 32
schema_config:
  configs:
    - from: "2024-01-01"
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: tsdb_index_
        period: 24h
analytics:
  reporting_enabled: false
`, d.deployment.AuthEnabled, d.replicationFactor(), d.replicationFactor(), retentionHours)
}

// ==================== 辅助函数 ====================

func (d *LokiDeployer) effectiveVersion() string {
	if d.deployment.Version != "" {
		return d.deployment.Version
	}
	return "3.0.0"
}

func (d *LokiDeployer) effectiveNamespace() string {
	if d.deployment.Namespace != "" {
		return d.deployment.Namespace
	}
	return "log"
}

func (d *LokiDeployer) effectiveReleaseName() string {
	if d.deployment.Name != "" {
		return d.deployment.Name
	}
	return "loki"
}

func (d *LokiDeployer) effectiveStorageType() string {
	if d.deployment.StorageConfig != nil && d.deployment.StorageConfig.StorageType != "" {
		return d.deployment.StorageConfig.StorageType
	}
	return "filesystem"
}

func (d *LokiDeployer) checkVersionCompatibility() error {
	version := d.effectiveVersion()
	parts := strings.Split(version, ".")
	if len(parts) < 1 {
		return fmt.Errorf("cannot parse version: %s", version)
	}
	major, err := strconv.Atoi(parts[0])
	if err != nil {
		return fmt.Errorf("cannot parse major version: %s", version)
	}
	if major < 2 {
		return fmt.Errorf("Loki version %s is too old, recommend 2.8+ or 3.x", version)
	}
	return nil
}

func (d *LokiDeployer) log(level, message, detail string) {
	if d.logFunc != nil {
		d.logFunc(level, message, detail)
	}
}

// ==================== 清理 / 卸载 ====================

func CleanupLoki(ctx context.Context, deployment *models.LokiDeployment) {
	// 同环境部署的 Grafana 一并清理
	if deployment.GrafanaEnabled {
		CleanupGrafana(ctx, deployment)
	}

	if deployment.Mode == "k8s-helm" && deployment.Cluster != nil {
		kubeconfig, err := resolveKubeconfig(deployment.Cluster)
		if err != nil {
			return
		}
		kubeFile, cleanup, err := writeTempFile("kube-*.yaml", kubeconfig, 0600)
		if err != nil {
			return
		}
		defer cleanup()

		env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile))
		ns := deployment.Namespace
		if ns == "" {
			ns = "log"
		}
		name := deployment.Name
		if name == "" {
			name = "loki"
		}

		cmd := exec.CommandContext(ctx, "helm", "uninstall", name, "--namespace", ns)
		cmd.Env = env
		cmd.CombinedOutput()

		// 清理 PVC（Helm 不会自动删除 PVC）
		cleanupPVC := exec.CommandContext(ctx, "kubectl", "delete", "pvc",
			"-l", "app.kubernetes.io/name=loki", "-n", ns, "--ignore-not-found=true")
		cleanupPVC.Env = env
		cleanupPVC.CombinedOutput()
	}

	if deployment.Mode == "k8s-yaml" && deployment.Cluster != nil {
		kubeconfig, err := resolveKubeconfig(deployment.Cluster)
		if err != nil {
			return
		}
		kubeFile, cleanup, err := writeTempFile("kube-*.yaml", kubeconfig, 0600)
		if err != nil {
			return
		}
		defer cleanup()

		env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile))
		ns := deployment.Namespace
		if ns == "" {
			ns = "log"
		}

		// 删除所有相关资源
		resources := []string{
			"statefulset/loki",
			"service/loki",
			"configmap/loki-config",
			"persistentvolumeclaim/loki-data",
		}
		for _, res := range resources {
			cmd := exec.CommandContext(ctx, "kubectl", "delete", res, "-n", ns, "--ignore-not-found=true")
			cmd.Env = env
			cmd.CombinedOutput()
		}
	}

	if deployment.Mode == "host-binary" && deployment.Host != nil {
		host := deployment.Host
		client, _ := NewSSHClientWithAuth(HostSSHAuth(host))
		if client != nil {
			defer client.Close()
			client.Execute(ctx, "systemctl stop loki 2>/dev/null || true")
			client.Execute(ctx, "systemctl disable loki 2>/dev/null || true")
			client.Execute(ctx, "rm -f /etc/systemd/system/loki.service /usr/local/bin/loki")
			client.Execute(ctx, "systemctl daemon-reload")
		}
	}
}

// DecryptCred 解密凭证
func DecryptCred(encrypted string) (string, error) {
	if strings.HasPrefix(encrypted, "ENC:") {
		return crypto.Decrypt(encrypted[4:])
	}
	return encrypted, nil
}

// ==================== 通用工具函数 ====================

// writeTempFile 安全地创建临时文件，返回文件路径和清理函数
func writeTempFile(pattern, content string, perm os.FileMode) (string, func(), error) {
	f, err := os.CreateTemp("", pattern)
	if err != nil {
		return "", nil, fmt.Errorf("create temp file failed: %w", err)
	}
	if _, err := f.WriteString(content); err != nil {
		f.Close()
		os.Remove(f.Name())
		return "", nil, fmt.Errorf("write temp file failed: %w", err)
	}
	f.Close()
	if perm > 0 {
		os.Chmod(f.Name(), perm)
	}
	cleanup := func() { os.Remove(f.Name()) }
	return f.Name(), cleanup, nil
}

// indentYAML 对 YAML 内容进行缩进（用于嵌入 ConfigMap）
func indentYAML(s string, spaces int) string {
	pad := strings.Repeat(" ", spaces)
	lines := strings.Split(s, "\n")
	for i, l := range lines {
		if l != "" {
			lines[i] = pad + l
		}
	}
	return strings.Join(lines, "\n")
}