API 端到端跑通中暴露 4 个串联问题(腾讯云 COS + K8s 场景):
1. loki_manifest.go: switch 缺少 cos / obs
→ 不生成 loki-storage-credentials Secret、不注入 LOKI_STORAGE_* 环境变量、
也不给 Loki 加 -config.expand-env=true
→ 配置里 ${LOKI_STORAGE_ACCESS_KEY} 无法展开 → AK 为空
→ COS 返回 403 InvalidAccessKeyId(ruler 每分钟报错)
修复:case 补上 cos / obs
2. loki_manifest.go: 生成 Secret 时用了 ENC: 密文原始值
→ Secret data 里是 ENC:xxx 而非真实 AK/SK
修复:crypto.DecryptField 解密后再 base64
3. loki_config.go: host-docker 模式同样把密文写进配置
修复:解密后返回
4. loki_config.go: cos/oss/obs 的 endpoint 未剥离桶名前缀
→ 用户填虚拟主机式 endpoint(mybucket-1250000000.cos.<region>...)时,
Loki 的 S3 客户端又拼一次 bucketnames,变成 <bucket>.<bucket>.cos...
→ TLS 证书校验失败(x509: certificate is valid for ... not <bucket>.<bucket>.cos...)
修复:新增 stripBucketPrefix,s3/cos/oss/obs/minio 分支统一剥离桶名前缀
验证(log namespace,API 端到端):
- Loki statefulset 1/1 Running,ruler 无 InvalidAccessKeyId/x509
- 写入 + 查询日志成功(持久化到 COS)
- Promtail DaemonSet 12/12 全节点就绪
- collector status=active
运维侧附加动作(非代码,见部署文档):
- 存储配置改用有效 AK/SK + 区域端点 cos.ap-hongkong.myqcloud.com
- 平台 ServiceAccount log:lokistack 绑定 cluster-admin
(否则无法创建 Promtail 所需的 ClusterRole/ClusterRoleBinding)
250 lines
5.7 KiB
Go
250 lines
5.7 KiB
Go
// LokiDeployer — k8s-yaml 模式 manifest 生成(StatefulSet/PVC/ConfigMap/Service)。
|
||
package services
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/base64"
|
||
"fmt"
|
||
|
||
"opsxloki/pkg/crypto"
|
||
)
|
||
|
||
func (d *LokiDeployer) generateManifest(ns string) string {
|
||
version := d.effectiveVersion()
|
||
image := ResolveImageFor("loki", "grafana/loki", version)
|
||
replicas := d.deployment.Replicas
|
||
if replicas < 1 {
|
||
replicas = 1
|
||
}
|
||
storageSize := d.deployment.StorageSize
|
||
if storageSize == "" {
|
||
storageSize = "20Gi"
|
||
}
|
||
|
||
cpu := d.deployment.ResourcesCPU
|
||
mem := d.deployment.ResourcesMem
|
||
if cpu == "" {
|
||
cpu = "1000m"
|
||
}
|
||
if mem == "" {
|
||
mem = "1Gi"
|
||
}
|
||
|
||
configData := d.generateLokiConfig()
|
||
secretManifest := ""
|
||
configExpandArg := ""
|
||
storageEnvBlock := ""
|
||
if sc := d.deployment.StorageConfig; sc != nil {
|
||
switch sc.StorageType {
|
||
// 注意:必须包含 cos / obs —— 腾讯云 COS、华为 OBS 同样走 S3 兼容协议,
|
||
// 也需要生成 loki-storage-credentials Secret、注入 LOKI_STORAGE_* 环境变量、
|
||
// 并给 Loki 加 -config.expand-env=true,否则配置里的 ${LOKI_STORAGE_ACCESS_KEY}
|
||
// 无法展开 → AK 为空 → 对象存储 403 InvalidAccessKeyId。
|
||
case "s3", "oss", "cos", "obs", "minio", "azure":
|
||
secretManifest = fmt.Sprintf(`---
|
||
apiVersion: v1
|
||
kind: Secret
|
||
metadata:
|
||
name: loki-storage-credentials
|
||
namespace: %s
|
||
type: Opaque
|
||
data:
|
||
accessKey: %s
|
||
secretKey: %s`, ns,
|
||
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.AccessKey))),
|
||
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.SecretKey))))
|
||
configExpandArg = "\n - -config.expand-env=true"
|
||
storageEnvBlock = `
|
||
env:
|
||
- name: LOKI_STORAGE_ACCESS_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: loki-storage-credentials
|
||
key: accessKey
|
||
- name: LOKI_STORAGE_SECRET_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: loki-storage-credentials
|
||
key: secretKey`
|
||
}
|
||
}
|
||
// ConfigMap/Secret 更新本身不会重建 Pod。把内容摘要写入 Pod 模板以触发滚动更新。
|
||
configHash := sha256.Sum256([]byte(configData + "\n" + secretManifest))
|
||
|
||
manifest := fmt.Sprintf(`apiVersion: v1
|
||
kind: Namespace
|
||
metadata:
|
||
name: %s
|
||
---
|
||
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: loki-config
|
||
namespace: %s
|
||
data:
|
||
loki.yaml: |
|
||
%s
|
||
%s
|
||
---
|
||
apiVersion: v1
|
||
kind: Service
|
||
metadata:
|
||
name: loki
|
||
namespace: %s
|
||
labels:
|
||
app: loki
|
||
spec:
|
||
type: ClusterIP
|
||
ports:
|
||
- name: http
|
||
port: 3100
|
||
targetPort: 3100
|
||
protocol: TCP
|
||
- name: grpc
|
||
port: 9095
|
||
targetPort: 9095
|
||
protocol: TCP
|
||
selector:
|
||
app: loki
|
||
---
|
||
apiVersion: v1
|
||
kind: PersistentVolumeClaim
|
||
metadata:
|
||
name: loki-data
|
||
namespace: %s
|
||
spec:
|
||
accessModes:
|
||
- ReadWriteOnce
|
||
resources:
|
||
requests:
|
||
storage: %s
|
||
---
|
||
apiVersion: apps/v1
|
||
kind: StatefulSet
|
||
metadata:
|
||
name: loki
|
||
namespace: %s
|
||
labels:
|
||
app: loki
|
||
spec:
|
||
serviceName: loki
|
||
replicas: %d
|
||
selector:
|
||
matchLabels:
|
||
app: loki
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: loki
|
||
annotations:
|
||
checksum/loki-config: "%x"
|
||
spec:
|
||
securityContext:
|
||
runAsUser: 0
|
||
runAsGroup: 0
|
||
fsGroup: 0
|
||
containers:
|
||
- name: loki
|
||
image: %s
|
||
imagePullPolicy: IfNotPresent
|
||
args:
|
||
- -config.file=/etc/loki/loki.yaml%s%s
|
||
ports:
|
||
- name: http
|
||
containerPort: 3100
|
||
protocol: TCP
|
||
- name: grpc
|
||
containerPort: 9095
|
||
protocol: TCP
|
||
resources:
|
||
limits:
|
||
cpu: "%s"
|
||
memory: "%s"
|
||
requests:
|
||
cpu: "500m"
|
||
memory: "512Mi"
|
||
securityContext:
|
||
allowPrivilegeEscalation: false
|
||
readOnlyRootFilesystem: false
|
||
capabilities:
|
||
drop:
|
||
- ALL
|
||
livenessProbe:
|
||
httpGet:
|
||
path: /ready
|
||
port: http
|
||
initialDelaySeconds: 30
|
||
periodSeconds: 10
|
||
timeoutSeconds: 5
|
||
failureThreshold: 6
|
||
readinessProbe:
|
||
httpGet:
|
||
path: /ready
|
||
port: http
|
||
initialDelaySeconds: 10
|
||
periodSeconds: 5
|
||
timeoutSeconds: 3
|
||
failureThreshold: 3
|
||
startupProbe:
|
||
httpGet:
|
||
path: /ready
|
||
port: http
|
||
initialDelaySeconds: 10
|
||
periodSeconds: 10
|
||
timeoutSeconds: 5
|
||
failureThreshold: 30
|
||
volumeMounts:
|
||
- name: config
|
||
mountPath: /etc/loki
|
||
readOnly: true
|
||
- name: data
|
||
mountPath: /var/lib/loki
|
||
- name: wal
|
||
mountPath: /var/lib/loki/wal
|
||
volumes:
|
||
- name: config
|
||
configMap:
|
||
name: loki-config
|
||
- name: data
|
||
persistentVolumeClaim:
|
||
claimName: loki-data
|
||
- name: wal
|
||
emptyDir: {}
|
||
`, ns, ns, indentYAML(configData, 4), secretManifest, ns, ns, storageSize, ns, replicas, configHash, image, configExpandArg, storageEnvBlock, cpu, mem)
|
||
|
||
// 多副本:追加 Headless Service(clusterIP: None + selector app:loki),
|
||
// 使 memberlist 的 join_members (loki-headless.<ns>.svc.cluster.local) 能解析到所有 Pod IP。
|
||
if replicas >= 2 {
|
||
manifest += fmt.Sprintf(`
|
||
---
|
||
apiVersion: v1
|
||
kind: Service
|
||
metadata:
|
||
name: loki-headless
|
||
namespace: %s
|
||
labels:
|
||
app: loki
|
||
spec:
|
||
clusterIP: None
|
||
ports:
|
||
- name: http
|
||
port: 3100
|
||
targetPort: 3100
|
||
protocol: TCP
|
||
- name: grpc
|
||
port: 9095
|
||
targetPort: 9095
|
||
protocol: TCP
|
||
- name: memberlist
|
||
port: 7946
|
||
targetPort: 7946
|
||
protocol: TCP
|
||
selector:
|
||
app: loki
|
||
`, ns)
|
||
}
|
||
return manifest
|
||
}
|
||
|
||
// ==================== Loki 本地配置生成 (text/template) ====================
|