fix(loki): COS 存储全链路修复 —— Secret 生成/env 注入/凭据解密/endpoint 规范化
API 端到端跑通中暴露 4 个串联问题(腾讯云 COS + K8s 场景):
1. loki_manifest.go: switch 缺少 cos / obs
→ 不生成 loki-storage-credentials Secret、不注入 LOKI_STORAGE_* 环境变量、
也不给 Loki 加 -config.expand-env=true
→ 配置里 ${LOKI_STORAGE_ACCESS_KEY} 无法展开 → AK 为空
→ COS 返回 403 InvalidAccessKeyId(ruler 每分钟报错)
修复:case 补上 cos / obs
2. loki_manifest.go: 生成 Secret 时用了 ENC: 密文原始值
→ Secret data 里是 ENC:xxx 而非真实 AK/SK
修复:crypto.DecryptField 解密后再 base64
3. loki_config.go: host-docker 模式同样把密文写进配置
修复:解密后返回
4. loki_config.go: cos/oss/obs 的 endpoint 未剥离桶名前缀
→ 用户填虚拟主机式 endpoint(mybucket-1250000000.cos.<region>...)时,
Loki 的 S3 客户端又拼一次 bucketnames,变成 <bucket>.<bucket>.cos...
→ TLS 证书校验失败(x509: certificate is valid for ... not <bucket>.<bucket>.cos...)
修复:新增 stripBucketPrefix,s3/cos/oss/obs/minio 分支统一剥离桶名前缀
验证(log namespace,API 端到端):
- Loki statefulset 1/1 Running,ruler 无 InvalidAccessKeyId/x509
- 写入 + 查询日志成功(持久化到 COS)
- Promtail DaemonSet 12/12 全节点就绪
- collector status=active
运维侧附加动作(非代码,见部署文档):
- 存储配置改用有效 AK/SK + 区域端点 cos.ap-hongkong.myqcloud.com
- 平台 ServiceAccount log:lokistack 绑定 cluster-admin
(否则无法创建 Promtail 所需的 ClusterRole/ClusterRoleBinding)
This commit is contained in:
1 parent
004375e0ce
commit
46e73d3d17
3 files changed
+47
-6
No files matched your search
@@ -5,6 +5,8 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
|
||||||
|
"opsxloki/pkg/crypto"
|
||||||
)
|
)
|
||||||
|
|
||||||
const lokiConfigTemplate = `auth_enabled: {{ .AuthEnabled }}
|
const lokiConfigTemplate = `auth_enabled: {{ .AuthEnabled }}
|
||||||
@@ -207,6 +209,14 @@ func (d *LokiDeployer) buildStorageConfigBlock() string {
|
|||||||
if sc.Insecure {
|
if sc.Insecure {
|
||||||
insecure = "true"
|
insecure = "true"
|
||||||
}
|
}
|
||||||
|
// 云厂商 S3 兼容端点有两种写法:
|
||||||
|
// 区域端点 cos.ap-hongkong.myqcloud.com ← 正确
|
||||||
|
// 虚拟主机式 mybucket-1250000000.cos.<region>... ← 桶名已在 host 里
|
||||||
|
// Loki 的 S3 客户端会自己把 bucketnames 拼到 endpoint 前面(s3forcepathstyle=false
|
||||||
|
// 即虚拟主机风格),所以 endpoint 必须是区域端点。若用户在平台里填了带桶名的写法,
|
||||||
|
// 这里剥掉桶名前缀,否则会拼出 <bucket>.<bucket>.cos... 导致 TLS 证书校验失败、
|
||||||
|
// 日志写入失败(ruler/ingester 持续报 x509 证书不匹配)。
|
||||||
|
endpoint := stripBucketPrefix(sc.Endpoint, sc.Bucket)
|
||||||
return fmt.Sprintf(` aws:
|
return fmt.Sprintf(` aws:
|
||||||
bucketnames: %s
|
bucketnames: %s
|
||||||
endpoint: %s
|
endpoint: %s
|
||||||
@@ -214,7 +224,7 @@ func (d *LokiDeployer) buildStorageConfigBlock() string {
|
|||||||
access_key_id: %s
|
access_key_id: %s
|
||||||
secret_access_key: %s
|
secret_access_key: %s
|
||||||
insecure: %s
|
insecure: %s
|
||||||
s3forcepathstyle: false`, sc.Bucket, sc.Endpoint, sc.Region, accessKey, secretKey, insecure)
|
s3forcepathstyle: false`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
|
||||||
case "minio":
|
case "minio":
|
||||||
insecure := "false"
|
insecure := "false"
|
||||||
if sc.Insecure {
|
if sc.Insecure {
|
||||||
@@ -226,7 +236,7 @@ func (d *LokiDeployer) buildStorageConfigBlock() string {
|
|||||||
access_key_id: %s
|
access_key_id: %s
|
||||||
secret_access_key: %s
|
secret_access_key: %s
|
||||||
insecure: %s
|
insecure: %s
|
||||||
s3forcepathstyle: true`, sc.Bucket, sc.Endpoint, accessKey, secretKey, insecure)
|
s3forcepathstyle: true`, sc.Bucket, stripBucketPrefix(sc.Endpoint, sc.Bucket), accessKey, secretKey, insecure)
|
||||||
case "gcs":
|
case "gcs":
|
||||||
return fmt.Sprintf(` gcs:
|
return fmt.Sprintf(` gcs:
|
||||||
bucket_name: %s`, sc.Bucket)
|
bucket_name: %s`, sc.Bucket)
|
||||||
@@ -311,7 +321,9 @@ func (d *LokiDeployer) storageCredentialValues() (string, string) {
|
|||||||
if d.deployment.Mode == "k8s-yaml" || d.deployment.Mode == "host-docker" {
|
if d.deployment.Mode == "k8s-yaml" || d.deployment.Mode == "host-docker" {
|
||||||
return "${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}"
|
return "${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}"
|
||||||
}
|
}
|
||||||
return d.deployment.StorageConfig.AccessKey, d.deployment.StorageConfig.SecretKey
|
// host-docker 模式没有 Secret 注入,必须把库里 ENC: 密文解密后再写进配置
|
||||||
|
return crypto.DecryptField(d.deployment.StorageConfig.AccessKey),
|
||||||
|
crypto.DecryptField(d.deployment.StorageConfig.SecretKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// generateLokiConfigFallback 当模板解析失败时的降级配置
|
// generateLokiConfigFallback 当模板解析失败时的降级配置
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"opsxloki/pkg/crypto"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (d *LokiDeployer) generateManifest(ns string) string {
|
func (d *LokiDeployer) generateManifest(ns string) string {
|
||||||
@@ -34,7 +36,11 @@ func (d *LokiDeployer) generateManifest(ns string) string {
|
|||||||
storageEnvBlock := ""
|
storageEnvBlock := ""
|
||||||
if sc := d.deployment.StorageConfig; sc != nil {
|
if sc := d.deployment.StorageConfig; sc != nil {
|
||||||
switch sc.StorageType {
|
switch sc.StorageType {
|
||||||
case "s3", "oss", "minio", "azure":
|
// 注意:必须包含 cos / obs —— 腾讯云 COS、华为 OBS 同样走 S3 兼容协议,
|
||||||
|
// 也需要生成 loki-storage-credentials Secret、注入 LOKI_STORAGE_* 环境变量、
|
||||||
|
// 并给 Loki 加 -config.expand-env=true,否则配置里的 ${LOKI_STORAGE_ACCESS_KEY}
|
||||||
|
// 无法展开 → AK 为空 → 对象存储 403 InvalidAccessKeyId。
|
||||||
|
case "s3", "oss", "cos", "obs", "minio", "azure":
|
||||||
secretManifest = fmt.Sprintf(`---
|
secretManifest = fmt.Sprintf(`---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Secret
|
kind: Secret
|
||||||
@@ -45,8 +51,8 @@ type: Opaque
|
|||||||
data:
|
data:
|
||||||
accessKey: %s
|
accessKey: %s
|
||||||
secretKey: %s`, ns,
|
secretKey: %s`, ns,
|
||||||
base64.StdEncoding.EncodeToString([]byte(sc.AccessKey)),
|
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.AccessKey))),
|
||||||
base64.StdEncoding.EncodeToString([]byte(sc.SecretKey)))
|
base64.StdEncoding.EncodeToString([]byte(crypto.DecryptField(sc.SecretKey))))
|
||||||
configExpandArg = "\n - -config.expand-env=true"
|
configExpandArg = "\n - -config.expand-env=true"
|
||||||
storageEnvBlock = `
|
storageEnvBlock = `
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -152,6 +152,29 @@ func endpointHasBucket(endpoint, bucket string) bool {
|
|||||||
return bucket != "" && strings.HasPrefix(strings.ToLower(host), strings.ToLower(bucket)+".")
|
return bucket != "" && strings.HasPrefix(strings.ToLower(host), strings.ToLower(bucket)+".")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stripBucketPrefix 从 endpoint 里剥掉桶名前缀,得到区域端点。
|
||||||
|
//
|
||||||
|
// mybucket-1250000000.cos.ap-hongkong.myqcloud.com → cos.ap-hongkong.myqcloud.com
|
||||||
|
// cos.ap-hongkong.myqcloud.com → 原样返回
|
||||||
|
//
|
||||||
|
// Loki 的 S3 客户端(s3forcepathstyle=false)会自己把 bucketnames 拼到 endpoint 前,
|
||||||
|
// 所以配置里必须是区域端点,否则会拼出 <bucket>.<bucket>.cos... 触发 TLS 证书错误。
|
||||||
|
func stripBucketPrefix(endpoint, bucket string) string {
|
||||||
|
if bucket == "" || endpoint == "" {
|
||||||
|
return endpoint
|
||||||
|
}
|
||||||
|
scheme := ""
|
||||||
|
host := strings.TrimSpace(endpoint)
|
||||||
|
if i := strings.Index(host, "://"); i >= 0 {
|
||||||
|
scheme, host = host[:i+3], host[i+3:]
|
||||||
|
}
|
||||||
|
prefix := strings.ToLower(bucket) + "."
|
||||||
|
if strings.HasPrefix(strings.ToLower(host), prefix) {
|
||||||
|
return scheme + host[len(prefix):]
|
||||||
|
}
|
||||||
|
return endpoint
|
||||||
|
}
|
||||||
|
|
||||||
// bucketPathFor 根据 endpoint 风格决定桶路径(用于建桶 PUT)。
|
// bucketPathFor 根据 endpoint 风格决定桶路径(用于建桶 PUT)。
|
||||||
//
|
//
|
||||||
// 1) 区域端点(推荐): cos.ap-hongkong.myqcloud.com → 路径式 /<bucket>
|
// 1) 区域端点(推荐): cos.ap-hongkong.myqcloud.com → 路径式 /<bucket>
|
||||||
|
|||||||
Reference in new issue
Block a user