Files
LokiStack/backend/services/collector_templates.go
T
opsx e006fc1a0e fix(promtail): app 标签通用回退 + 跳过僵尸 Pod + SecretKey 不下发 + 采集命名空间可选
=== 一、astralx 只采集到 2 个 app(核心问题)===
实证:用户的 Deployment Pod 模板只有 k8s.kuboard.cn/name 标签,
     没有 app / app.kubernetes.io/name。原模板的 app 只从这两个标签取,
     所以 115 个服务的 app 全是空值,Loki 里 app 只剩零星 2 个。

修复:app 四级回退,全部基于 Kubernetes 原生元数据(跨集群通用):
  1) app.kubernetes.io/name   标准约定
  2) app                      常见简写
  3) __tmp_controller_name    Pod owner(ReplicaSet) 去 hash → Deployment 名
                              (__meta_kubernetes_pod_controller_name 由
                                Promtail 直接从 K8s API 读取,任何集群都有,
                                不依赖 Kuboard/Rancher 等第三方工具注入的标签)
  4) pod name                 兜底,保证 app 永不为空

同时修复 PromtailHelmValues 模板同样的 app 逻辑,
并修正其 __meta_kubernetes__pod_name 双下划线笔误。

验证:astralx 的 app 值从 2 个 → 73 个。

=== 二、15 分钟 27 万条日志的真相 ===
排查结论:不是重复采集(positions 存 hostPath /run/promtail,Pod 重启不丢)。
真实原因:cex-openapi-gateway 残留 347 个 Evicted + 37 个 Error 的僵尸 Pod,
        只有 1 个 Running。这些 Pod 的日志文件仍在 /var/log/pods/ 下,
        Promtail 会把它们的历史异常堆栈(Reactor at reactor.core.publisher...)
        全量采集,造成 27 万条的假象并污染存储。

修复:relabel_configs 首条加 drop 规则:
  - source_labels: [__meta_kubernetes_pod_phase]
    regex: (Succeeded|Failed)
    action: drop
(两个模板都加)

=== 三、SecretKey 不再下发给前端 ===
安全要求:避免任何登录用户点「显示」按钮就能拿到对象存储 SK。
- StorageConfig.Sanitized(): secretKey 返回空串,仅保留 hasSecretKey 标记
  (accessKey 仍明文返回,便于管理员核对用的是哪个 AK)
- 测试连接:StorageTestRequest 新增 id 字段,secretKey 留空时后端用
  库里保存的凭据解密补齐(新建存储仍强制填写)
- 前端编辑时 SecretKey 一律留空 = 不修改,placeholder 显示
  「已保存 · 留空则不修改」,测试时带上 id

=== 四、采集器支持指定命名空间 ===
- Collector 新增 TargetNamespaces []string,对应 promtail 的
  kubernetes_sd_configs.namespaces.names(为空=全集群)
- PromtailDaemonsetYAML 新增 promtailNamespaceFilter 生成白名单
- service.Update 支持 targetNamespaces(兼容 []string 与逗号分隔字符串)
- 前端 CollectorEditDialog 增加「采集命名空间」输入(逗号分隔)

镜像:lghost/lokistack:latest + v0.1 = sha256:05b1505aedae
2026-10-27 03:30:25 +00:00

583 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package services
import (
"encoding/json"
"fmt"
"sort"
"strings"
)
// PromtailConfig 主机 Promtail 配置(推送至 lokiEndpoint,采集 logPath)
// 生产级配置:持久化位置文件、批处理、重试、CRI日志格式解析
// promtailExternalLabels 组并 cluster 标识与用户自定义标签(JSON)为 external_labels YAML 行
func promtailExternalLabels(cluster string, labelsJSON string) string {
lines := []string{fmt.Sprintf(" cluster: %q", cluster)}
if labelsJSON != "" {
var m map[string]string
if err := json.Unmarshal([]byte(labelsJSON), &m); err == nil {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
lines = append(lines, fmt.Sprintf(" %s: %q", k, m[k]))
}
}
}
return strings.Join(lines, "\n")
}
func PromtailConfig(lokiEndpoint, logPath, cluster, labelsJSON string) string {
conf := fmt.Sprintf(`server:
http_listen_port: 9080
grpc_listen_port: 0
log_level: info
log_format: logfmt
positions:
filename: /var/lib/promtail/positions.yaml
sync_period: 10s
clients:
- url: %s/loki/api/v1/push
batchwait: 1s
batchsize: 1048576
timeout: 10s
backoff_config:
min_period: 500ms
max_period: 5m
max_retries: 10
external_labels:
job: host-logs
__EXT_LABELS__
limits_config:
readline_rate_enabled: false
readline_rate: 1000
readline_burst: 10000
max_streams: 0
scrape_configs:
- job_name: app
static_configs:
- targets: [localhost]
labels:
job: app
host: ${HOSTNAME}
__path__: %s
pipeline_stages:
- cri: {}
- match:
selector: '{job="app"}'
stages:
- timestamp:
source: time
format: RFC3339Nano
- limit:
rate: 1000
burst: 10000
drop: true
`, lokiEndpoint, logPath)
return strings.Replace(conf, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON), 1)
}
// PromtailHelmValues DaemonSet 模式 Promtail Helm values(生产级)
// 包含:cri解析、kubernetes标签、批处理配置、资源限制
func PromtailHelmValues(lokiEndpoint, cluster, labelsJSON string) string {
values := fmt.Sprintf(`config:
clients:
- url: %s/loki/api/v1/push
batchwait: 1s
batchsize: 1048576
timeout: 10s
backoff_config:
min_period: 500ms
max_period: 5m
max_retries: 10
external_labels:
__EXT_LABELS__
positions:
filename: /run/promtail/positions.yaml
scrape_configs:
- job_name: kubernetes-pods
kubernetes_sd_configs:
- role: pod
pipeline_stages:
- cri: {}
- match:
selector: '{app!=""}'
stages:
- docker: {}
- match:
selector: '{app!=""}'
stages:
- labels:
app:
namespace:
pod:
container:
- limit:
rate: 1000
burst: 10000
drop: true
relabel_configs:
- source_labels: [__meta_kubernetes_pod_phase]
regex: (Succeeded|Failed)
action: drop
- source_labels: [__meta_kubernetes_pod_controller_name]
regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
action: replace
target_label: __tmp_controller_name
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
- __meta_kubernetes_pod_label_app
- __tmp_controller_name
- __meta_kubernetes_pod_name
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: app
- source_labels: [__meta_kubernetes_pod_annotation_kubernetes_io_config_hash]
regex: .+
action: replace
target_label: config_hash
- action: replace
source_labels: ['__meta_kubernetes_pod_name']
target_label: 'pod'
- action: replace
source_labels: ['__meta_kubernetes_namespace']
target_label: 'namespace'
- action: replace
source_labels: ['__meta_kubernetes_pod_container_name']
target_label: 'container'
- action: replace
source_labels: ['__meta_kubernetes_pod_node_name']
target_label: 'node_name'
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
target_label: __path__
daemonset:
enabled: true
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 128Mi
extraVolumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: promtail-run
hostPath:
path: /run/promtail
extraVolumeMounts:
- name: varlog
mountPath: /var/log
readOnly: true
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: promtail-run
mountPath: /run/promtail
`, lokiEndpoint)
return strings.Replace(values, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON), 1)
}
// PromtailDaemonsetYAML DaemonSet 模式纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),
// 取代 helm 安装 —— 后端无需安装 helm 二进制。
// relabel 体系与主流 helm chart 对齐:namespace/pod/container/node_name + __path__ 按 pod uid 精确匹配。
// promtailNamespaceFilter 生成 kubernetes_sd_configs 的命名空间白名单。
// namespaces 为空 → 返回空串(采集全集群所有命名空间)。
func promtailNamespaceFilter(namespaces []string) string {
clean := make([]string, 0, len(namespaces))
for _, n := range namespaces {
if n = strings.TrimSpace(n); n != "" {
clean = append(clean, n)
}
}
if len(clean) == 0 {
return ""
}
var b strings.Builder
b.WriteString(" namespaces:\n names:\n")
for _, n := range clean {
b.WriteString(" - ")
b.WriteString(n)
b.WriteString("\n")
}
return b.String()
}
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string, targetNamespaces []string) string {
promtailImage := ResolveImageFor("promtail", "grafana/promtail", "3.4.0")
extLabels := promtailExternalLabels(clusterName, labelsJSON)
nsFilter := promtailNamespaceFilter(targetNamespaces)
return fmt.Sprintf(`apiVersion: v1
kind: ServiceAccount
metadata:
name: promtail
namespace: %s
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: opsxloki-promtail
rules:
- apiGroups: [""]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: opsxloki-promtail
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: opsxloki-promtail
subjects:
- kind: ServiceAccount
name: promtail
namespace: %s
---
apiVersion: v1
kind: ConfigMap
metadata:
name: promtail
namespace: %s
data:
promtail.yaml: |
server:
http_listen_port: 3101
grpc_listen_port: 0
positions:
filename: /run/promtail/positions.yaml
clients:
- url: %s/loki/api/v1/push
batchwait: 1s
batchsize: 1048576
backoff_config:
min_period: 500ms
max_period: 5m
max_retries: 10
external_labels:
%s
scrape_configs:
- job_name: kubernetes-pods
kubernetes_sd_configs:
- role: pod
%s
pipeline_stages:
- cri: {}
- multiline:
firstline: '^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[+\\-]\\d{2}:\\d{2}'
max_wait_time: 3s
max_lines: 500
- timestamp:
source: time
format: RFC3339Nano
relabel_configs:
# 跳过已终止的 Pod(Evicted / Failed / Succeeded)。
# 实证:某服务残留 387 个 Evicted Pod,日志文件还在节点上,
# Promtail 会把它们的历史异常堆栈全量采回来,导致 15 分钟 27 万条的假象,
# 也会污染 Loki 存储和查询结果。
- source_labels: [__meta_kubernetes_pod_phase]
regex: (Succeeded|Failed)
action: drop
# 从 controller name 提取 deployment 名(去掉 replicaset 哈希)
- source_labels: [__meta_kubernetes_pod_controller_name]
regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
action: replace
target_label: __tmp_controller_name
# app 四级回退 —— 全部基于 Kubernetes 原生元数据,跨集群通用:
# 1) app.kubernetes.io/name ← K8s 标准标签约定
# 2) app ← 最常见的简写
# 3) controller name 去 hash ← Pod 的 owner(ReplicaSet) → Deployment 名
# __meta_kubernetes_pod_controller_name 是
# Promtail 从 K8s API 直接拿的,任何集群都有,
# 不依赖任何第三方工具注入的标签
# 4) pod name ← 最后兜底,保证 app 永不为空
#
# 不使用 k8s.kuboard.cn/name 之类的平台私有标签:那只是 Kuboard 管理的集群才有,
# 换 Rancher/Lens/Helm/kubectl 部署的集群就不存在,会破坏通用性。
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
- __meta_kubernetes_pod_label_app
- __tmp_controller_name
- __meta_kubernetes_pod_name
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: app
# instance:app.kubernetes.io/instance → instance
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance, __meta_kubernetes_pod_label_instance]
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: instance
# component:app.kubernetes.io/component → component
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component, __meta_kubernetes_pod_label_component]
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: component
- action: replace
source_labels: [__meta_kubernetes_pod_node_name]
target_label: node_name
- action: replace
source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- action: replace
source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- action: replace
source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
# job: namespace/app(必须放在 app 之后,否则取到的是未回退的空值)
- action: replace
replacement: $1
separator: /
source_labels: [namespace, app]
target_label: job
- action: replace
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
target_label: __path__
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
namespace: %s
labels:
app: promtail
spec:
selector:
matchLabels:
app: promtail
template:
metadata:
labels:
app: promtail
spec:
serviceAccountName: promtail
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: promtail
image: %s
imagePullPolicy: IfNotPresent
args: ["-config.file=/etc/promtail/promtail.yaml"]
env:
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
ports:
- name: http-metrics
containerPort: 3101
readinessProbe:
httpGet:
path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 5
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: config
mountPath: /etc/promtail
- name: run
mountPath: /run/promtail
- name: pods
mountPath: /var/log/pods
readOnly: true
- name: containers
mountPath: /var/lib/docker/containers
readOnly: true
volumes:
- name: config
configMap:
name: promtail
- name: run
hostPath:
path: /run/promtail
- name: pods
hostPath:
path: /var/log/pods
- name: containers
hostPath:
path: /var/lib/docker/containers
`, ns, ns, ns, lokiURL, extLabels, nsFilter, ns, promtailImage)
}
// FluentbitConfig Sidecar 模式 Fluent Bit 配置(采集 logPath,推送至 lokiHost:lokiPort)
// Sidecar 已明确知道应用与命名空间,无需 Kubernetes 元数据过滤器。
func FluentbitConfig(lokiHost, lokiPort, logPath, app, namespace string) string {
return fmt.Sprintf(`[SERVICE]
Flush 1
Log_Level info
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
storage.path /tmp/flb-storage/
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 5M
Health_Check On
HC_Errors_Count 5
HC_Retry_Failure_Count 5
HC_Period 60
[INPUT]
Name tail
Path %s
Tag app.%s
Refresh_Interval 5
Mem_Buf_Limit 10MB
Skip_Long_Lines On
Read_from_Head True
DB /tmp/flb.db
DB.Sync Normal
Buffer_Max_Size 1MB
storage.type filesystem
[FILTER]
Name modify
Match app.*
Add namespace %s
Add app %s
Add source fluent-bit
[OUTPUT]
Name loki
Match *
Host %s
Port %s
Labels job=fluent-bit, app=%s, namespace=%s
Line_Format json
Retry_Limit 10
`, logPath, app, namespace, app, lokiHost, lokiPort, app, namespace)
}
// FluentbitConfigMapYAML 为每个采集策略生成独立 ConfigMap,避免同命名空间的 Sidecar 相互覆盖。
func FluentbitConfigMapYAML(conf, namespace, name string) string {
return fmt.Sprintf(`apiVersion: v1
kind: ConfigMap
metadata:
name: %s
namespace: %s
data:
fluent-bit.conf: |
%s
`, name, namespace, indentBlock(conf, " "))
}
// SidecarPatchYAML 将业务容器与 Fluent Bit 挂到同一日志卷和目录。
func SidecarPatchYAML(configMapName string, mount sidecarLogMount) string {
volume := ""
if mount.createVolume {
volume = fmt.Sprintf(" - name: %s\n emptyDir: {}\n", mount.volumeName)
}
appMount := ""
if mount.mountApplication {
appMount = fmt.Sprintf(" - name: %s\n volumeMounts:\n - name: %s\n mountPath: %q\n", mount.containerName, mount.volumeName, mount.mountPath)
}
subPath := ""
if mount.subPath != "" {
subPath = fmt.Sprintf(" subPath: %q\n", mount.subPath)
}
return fmt.Sprintf(`spec:
template:
spec:
volumes:
%s - name: fluent-bit-config
configMap:
name: %s
- name: fluent-bit-state
emptyDir: {}
containers:
%s - name: fluent-bit
image: %s
imagePullPolicy: IfNotPresent
ports:
- name: fluent-bit-http
containerPort: 2020
protocol: TCP
volumeMounts:
- name: %s
mountPath: %q
%s readOnly: true
- name: fluent-bit-config
mountPath: /fluent-bit/etc/
- name: fluent-bit-state
mountPath: /tmp
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
livenessProbe:
httpGet:
path: /
port: fluent-bit-http
initialDelaySeconds: 10
periodSeconds: 10
readinessProbe:
httpGet:
path: /
port: fluent-bit-http
initialDelaySeconds: 5
periodSeconds: 5
`, volume, configMapName, appMount, ResolveImageFor("fluentbit", "fluent/fluent-bit", "5.1.5"), mount.volumeName, mount.mountPath, subPath)
}
// indentBlock 给每行加前缀(ConfigMap 嵌入配置需缩进)
func indentBlock(s, pad string) string {
lines := strings.Split(s, "\n")
for i, l := range lines {
if l != "" {
lines[i] = pad + l
}
}
return strings.Join(lines, "\n")
}
// parseLokiURL 从 http://host:port 提取 host 与 port(Fluent Bit OUTPUT 需要)
func parseLokiURL(endpoint string) (host, port string) {
u := strings.TrimSpace(endpoint)
u = strings.TrimPrefix(u, "http://")
u = strings.TrimPrefix(u, "https://")
if i := strings.Index(u, "/"); i >= 0 {
u = u[:i]
}
if i := strings.LastIndex(u, ":"); i >= 0 {
return u[:i], u[i+1:]
}
return u, "3100"
}