Files
LokiStack/backend/services/loki_manifest.go
T
opsx 46e73d3d17 fix(loki): COS 存储全链路修复 —— Secret 生成/env 注入/凭据解密/endpoint 规范化
API 端到端跑通中暴露 4 个串联问题(腾讯云 COS + K8s 场景):

1. loki_manifest.go: switch 缺少 cos / obs
   → 不生成 loki-storage-credentials Secret、不注入 LOKI_STORAGE_* 环境变量、
     也不给 Loki 加 -config.expand-env=true
   → 配置里 ${LOKI_STORAGE_ACCESS_KEY} 无法展开 → AK 为空
   → COS 返回 403 InvalidAccessKeyId(ruler 每分钟报错)
   修复:case 补上 cos / obs

2. loki_manifest.go: 生成 Secret 时用了 ENC: 密文原始值
   → Secret data 里是 ENC:xxx 而非真实 AK/SK
   修复:crypto.DecryptField 解密后再 base64

3. loki_config.go: host-docker 模式同样把密文写进配置
   修复:解密后返回

4. loki_config.go: cos/oss/obs 的 endpoint 未剥离桶名前缀
   → 用户填虚拟主机式 endpoint(mybucket-1250000000.cos.<region>...)时,
     Loki 的 S3 客户端又拼一次 bucketnames,变成 <bucket>.<bucket>.cos...
   → TLS 证书校验失败(x509: certificate is valid for ... not <bucket>.<bucket>.cos...)
   修复:新增 stripBucketPrefix,s3/cos/oss/obs/minio 分支统一剥离桶名前缀

验证(log namespace,API 端到端):
  - Loki statefulset 1/1 Running,ruler 无 InvalidAccessKeyId/x509
  - 写入 + 查询日志成功(持久化到 COS)
  - Promtail DaemonSet 12/12 全节点就绪
  - collector status=active

运维侧附加动作(非代码,见部署文档):
  - 存储配置改用有效 AK/SK + 区域端点 cos.ap-hongkong.myqcloud.com
  - 平台 ServiceAccount log:lokistack 绑定 cluster-admin
    (否则无法创建 Promtail 所需的 ClusterRole/ClusterRoleBinding)
2026-10-26 19:32:38 +00:00

250 lines
5.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// LokiDeployer — k8s-yaml 模式 manifest 生成(StatefulSet/PVC/ConfigMap/Service)。
package services
import (
"crypto/sha256"
"encoding/base64"
"fmt"
"opsxloki/pkg/crypto"
)
func (d *LokiDeployer) generateManifest(ns string) string {
version := d.effectiveVersion()
image := ResolveImageFor("loki", "grafana/loki", version)
replicas := d.deployment.Replicas
if replicas < 1 {
replicas = 1
}
storageSize := d.deployment.StorageSize
if storageSize == "" {
storageSize = "20Gi"
}
cpu := d.deployment.ResourcesCPU
mem := d.deployment.ResourcesMem
if cpu == "" {
cpu = "1000m"
}
if mem == "" {
mem = "1Gi"
}
configData := d.generateLokiConfig()
secretManifest := ""
configExpandArg := ""
storageEnvBlock := ""
if sc := d.deployment.StorageConfig; sc != nil {
switch sc.StorageType {
// 注意:必须包含 cos / obs —— 腾讯云 COS、华为 OBS 同样走 S3 兼容协议,
// 也需要生成 loki-storage-credentials Secret、注入 LOKI_STORAGE_* 环境变量、
// 并给 Loki 加 -config.expand-env=true,否则配置里的 ${LOKI_STORAGE_ACCESS_KEY}
// 无法展开 → AK 为空 → 对象存储 403 InvalidAccessKeyId。
case "s3", "oss", "cos", "obs", "minio", "azure":
secretManifest = fmt.Sprintf(`---
apiVersion: v1
kind: Secret
metadata:
name: loki-storage-credentials
namespace: %s
type: Opaque
data:
accessKey: %s
secretKey: %s`, ns,
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.AccessKey))),
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.SecretKey))))
configExpandArg = "\n - -config.expand-env=true"
storageEnvBlock = `
env:
- name: LOKI_STORAGE_ACCESS_KEY
valueFrom:
secretKeyRef:
name: loki-storage-credentials
key: accessKey
- name: LOKI_STORAGE_SECRET_KEY
valueFrom:
secretKeyRef:
name: loki-storage-credentials
key: secretKey`
}
}
// ConfigMap/Secret 更新本身不会重建 Pod。把内容摘要写入 Pod 模板以触发滚动更新。
configHash := sha256.Sum256([]byte(configData + "\n" + secretManifest))
manifest := fmt.Sprintf(`apiVersion: v1
kind: Namespace
metadata:
name: %s
---
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-config
namespace: %s
data:
loki.yaml: |
%s
%s
---
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: %s
labels:
app: loki
spec:
type: ClusterIP
ports:
- name: http
port: 3100
targetPort: 3100
protocol: TCP
- name: grpc
port: 9095
targetPort: 9095
protocol: TCP
selector:
app: loki
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: loki-data
namespace: %s
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: %s
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: loki
namespace: %s
labels:
app: loki
spec:
serviceName: loki
replicas: %d
selector:
matchLabels:
app: loki
template:
metadata:
labels:
app: loki
annotations:
checksum/loki-config: "%x"
spec:
securityContext:
runAsUser: 0
runAsGroup: 0
fsGroup: 0
containers:
- name: loki
image: %s
imagePullPolicy: IfNotPresent
args:
- -config.file=/etc/loki/loki.yaml%s%s
ports:
- name: http
containerPort: 3100
protocol: TCP
- name: grpc
containerPort: 9095
protocol: TCP
resources:
limits:
cpu: "%s"
memory: "%s"
requests:
cpu: "500m"
memory: "512Mi"
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop:
- ALL
livenessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
startupProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 30
volumeMounts:
- name: config
mountPath: /etc/loki
readOnly: true
- name: data
mountPath: /var/lib/loki
- name: wal
mountPath: /var/lib/loki/wal
volumes:
- name: config
configMap:
name: loki-config
- name: data
persistentVolumeClaim:
claimName: loki-data
- name: wal
emptyDir: {}
`, ns, ns, indentYAML(configData, 4), secretManifest, ns, ns, storageSize, ns, replicas, configHash, image, configExpandArg, storageEnvBlock, cpu, mem)
// 多副本:追加 Headless Service(clusterIP: None + selector app:loki),
// 使 memberlist 的 join_members (loki-headless.<ns>.svc.cluster.local) 能解析到所有 Pod IP。
if replicas >= 2 {
manifest += fmt.Sprintf(`
---
apiVersion: v1
kind: Service
metadata:
name: loki-headless
namespace: %s
labels:
app: loki
spec:
clusterIP: None
ports:
- name: http
port: 3100
targetPort: 3100
protocol: TCP
- name: grpc
port: 9095
targetPort: 9095
protocol: TCP
- name: memberlist
port: 7946
targetPort: 7946
protocol: TCP
selector:
app: loki
`, ns)
}
return manifest
}
// ==================== Loki 本地配置生成 (text/template) ====================