logan
2d73661dd1
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application. Validated with go test ./..., npm run build, and npm run lint.
500 lines
13 KiB
Go
500 lines
13 KiB
Go
package services
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
)
|
||
|
||
// PromtailConfig 主机 Promtail 配置（推送至 lokiEndpoint，采集 logPath）
|
||
// 生产级配置：持久化位置文件、批处理、重试、CRI日志格式解析
|
||
// promtailExternalLabels 组并 cluster 标识与用户自定义标签(JSON)为 external_labels YAML 行
|
||
func promtailExternalLabels(cluster string, labelsJSON string) string {
|
||
lines := []string{fmt.Sprintf(" cluster: %q", cluster)}
|
||
if labelsJSON != "" {
|
||
var m map[string]string
|
||
if err := json.Unmarshal([]byte(labelsJSON), &m); err == nil {
|
||
keys := make([]string, 0, len(m))
|
||
for k := range m {
|
||
keys = append(keys, k)
|
||
}
|
||
sort.Strings(keys)
|
||
for _, k := range keys {
|
||
lines = append(lines, fmt.Sprintf(" %s: %q", k, m[k]))
|
||
}
|
||
}
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
func PromtailConfig(lokiEndpoint, logPath, cluster, labelsJSON string) string {
|
||
conf := fmt.Sprintf(`server:
|
||
http_listen_port: 9080
|
||
grpc_listen_port: 0
|
||
log_level: info
|
||
log_format: logfmt
|
||
|
||
positions:
|
||
filename: /var/lib/promtail/positions.yaml
|
||
sync_period: 10s
|
||
|
||
clients:
|
||
- url: %s/loki/api/v1/push
|
||
batchwait: 1s
|
||
batchsize: 1048576
|
||
timeout: 10s
|
||
backoff_config:
|
||
min_period: 500ms
|
||
max_period: 5m
|
||
max_retries: 10
|
||
external_labels:
|
||
job: host-logs
|
||
__EXT_LABELS__
|
||
|
||
limits_config:
|
||
readline_rate_enabled: false
|
||
readline_rate: 1000
|
||
readline_burst: 10000
|
||
max_streams: 0
|
||
|
||
scrape_configs:
|
||
- job_name: app
|
||
static_configs:
|
||
- targets: [localhost]
|
||
labels:
|
||
job: app
|
||
host: ${HOSTNAME}
|
||
__path__: %s
|
||
pipeline_stages:
|
||
- cri: {}
|
||
- match:
|
||
selector: '{job="app"}'
|
||
stages:
|
||
- timestamp:
|
||
source: time
|
||
format: RFC3339Nano
|
||
- limit:
|
||
rate: 1000
|
||
burst: 10000
|
||
drop: true
|
||
`, lokiEndpoint, logPath)
|
||
return strings.Replace(conf, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON), 1)
|
||
}
|
||
|
||
// PromtailHelmValues DaemonSet 模式 Promtail Helm values（生产级）
|
||
// 包含：cri解析、kubernetes标签、批处理配置、资源限制
|
||
func PromtailHelmValues(lokiEndpoint, cluster, labelsJSON string) string {
|
||
values := fmt.Sprintf(`config:
|
||
clients:
|
||
- url: %s/loki/api/v1/push
|
||
batchwait: 1s
|
||
batchsize: 1048576
|
||
timeout: 10s
|
||
backoff_config:
|
||
min_period: 500ms
|
||
max_period: 5m
|
||
max_retries: 10
|
||
external_labels:
|
||
__EXT_LABELS__
|
||
positions:
|
||
filename: /run/promtail/positions.yaml
|
||
scrape_configs:
|
||
- job_name: kubernetes-pods
|
||
kubernetes_sd_configs:
|
||
- role: pod
|
||
pipeline_stages:
|
||
- cri: {}
|
||
- match:
|
||
selector: '{app!=""}'
|
||
stages:
|
||
- docker: {}
|
||
- match:
|
||
selector: '{app!=""}'
|
||
stages:
|
||
- labels:
|
||
app:
|
||
namespace:
|
||
pod:
|
||
container:
|
||
- limit:
|
||
rate: 1000
|
||
burst: 10000
|
||
drop: true
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_pod_controller_name]
|
||
regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
|
||
action: replace
|
||
target_label: __tmp_controller_name
|
||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||
regex: .+
|
||
action: replace
|
||
target_label: app
|
||
- source_labels: [__meta_kubernetes_pod_label_app]
|
||
regex: .+
|
||
action: replace
|
||
target_label: app
|
||
- source_labels: [__meta_kubernetes_pod_annotation_kubernetes_io_config_hash]
|
||
regex: .+
|
||
action: replace
|
||
target_label: config_hash
|
||
- action: replace
|
||
source_labels: ['__meta_kubernetes__pod_name']
|
||
target_label: 'pod'
|
||
- action: replace
|
||
source_labels: ['__meta_kubernetes_namespace']
|
||
target_label: 'namespace'
|
||
- action: replace
|
||
source_labels: ['__meta_kubernetes_pod_container_name']
|
||
target_label: 'container'
|
||
- action: replace
|
||
source_labels: ['__meta_kubernetes_pod_node_name']
|
||
target_label: 'node_name'
|
||
- replacement: /var/log/pods/*$1/*.log
|
||
separator: /
|
||
source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
|
||
target_label: __path__
|
||
|
||
daemonset:
|
||
enabled: true
|
||
|
||
resources:
|
||
limits:
|
||
cpu: 200m
|
||
memory: 256Mi
|
||
requests:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
|
||
extraVolumes:
|
||
- name: varlog
|
||
hostPath:
|
||
path: /var/log
|
||
- name: varlibdockercontainers
|
||
hostPath:
|
||
path: /var/lib/docker/containers
|
||
- name: promtail-run
|
||
hostPath:
|
||
path: /run/promtail
|
||
|
||
extraVolumeMounts:
|
||
- name: varlog
|
||
mountPath: /var/log
|
||
readOnly: true
|
||
- name: varlibdockercontainers
|
||
mountPath: /var/lib/docker/containers
|
||
readOnly: true
|
||
- name: promtail-run
|
||
mountPath: /run/promtail
|
||
`, lokiEndpoint)
|
||
return strings.Replace(values, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON), 1)
|
||
}
|
||
|
||
// PromtailDaemonsetYAML DaemonSet 模式纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),
|
||
// 取代 helm 安装 —— 后端无需安装 helm 二进制。
|
||
// relabel 体系与主流 helm chart 对齐:namespace/pod/container/node_name + __path__ 按 pod uid 精确匹配。
|
||
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string) string {
|
||
promtailImage := ResolveImage("docker.io/grafana/promtail:3.4.0")
|
||
extLabels := promtailExternalLabels(clusterName, labelsJSON)
|
||
return fmt.Sprintf(`apiVersion: v1
|
||
kind: ServiceAccount
|
||
metadata:
|
||
name: promtail
|
||
namespace: %s
|
||
---
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
kind: ClusterRole
|
||
metadata:
|
||
name: opsxloki-promtail
|
||
rules:
|
||
- apiGroups: [""]
|
||
resources: ["pods", "nodes"]
|
||
verbs: ["get", "list", "watch"]
|
||
---
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
kind: ClusterRoleBinding
|
||
metadata:
|
||
name: opsxloki-promtail
|
||
roleRef:
|
||
apiGroup: rbac.authorization.k8s.io
|
||
kind: ClusterRole
|
||
name: opsxloki-promtail
|
||
subjects:
|
||
- kind: ServiceAccount
|
||
name: promtail
|
||
namespace: %s
|
||
---
|
||
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: promtail
|
||
namespace: %s
|
||
data:
|
||
promtail.yaml: |
|
||
server:
|
||
http_listen_port: 3101
|
||
grpc_listen_port: 0
|
||
positions:
|
||
filename: /run/promtail/positions.yaml
|
||
clients:
|
||
- url: %s/loki/api/v1/push
|
||
batchwait: 1s
|
||
batchsize: 1048576
|
||
backoff_config:
|
||
min_period: 500ms
|
||
max_period: 5m
|
||
max_retries: 10
|
||
external_labels:
|
||
job: promtail
|
||
%s
|
||
scrape_configs:
|
||
- job_name: kubernetes-pods
|
||
kubernetes_sd_configs:
|
||
- role: pod
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_namespace]
|
||
target_label: namespace
|
||
- source_labels: [__meta_kubernetes_pod_name]
|
||
target_label: pod
|
||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||
target_label: container
|
||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||
target_label: node_name
|
||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app]
|
||
regex: ^;*([^;]+)(;.*)?$
|
||
target_label: app
|
||
- separator: /
|
||
source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||
target_label: job
|
||
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
|
||
separator: /
|
||
target_label: __path__
|
||
replacement: /var/log/pods/*$1/*.log
|
||
---
|
||
apiVersion: apps/v1
|
||
kind: DaemonSet
|
||
metadata:
|
||
name: promtail
|
||
namespace: %s
|
||
labels:
|
||
app: promtail
|
||
spec:
|
||
selector:
|
||
matchLabels:
|
||
app: promtail
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: promtail
|
||
spec:
|
||
serviceAccountName: promtail
|
||
tolerations:
|
||
- effect: NoSchedule
|
||
operator: Exists
|
||
containers:
|
||
- name: promtail
|
||
image: %s
|
||
imagePullPolicy: IfNotPresent
|
||
args: ["-config.file=/etc/promtail/promtail.yaml"]
|
||
env:
|
||
- name: HOSTNAME
|
||
valueFrom:
|
||
fieldRef:
|
||
fieldPath: spec.nodeName
|
||
ports:
|
||
- name: http-metrics
|
||
containerPort: 3101
|
||
readinessProbe:
|
||
httpGet:
|
||
path: /ready
|
||
port: http-metrics
|
||
initialDelaySeconds: 10
|
||
periodSeconds: 10
|
||
resources:
|
||
requests:
|
||
cpu: 50m
|
||
memory: 64Mi
|
||
limits:
|
||
cpu: 200m
|
||
memory: 256Mi
|
||
volumeMounts:
|
||
- name: config
|
||
mountPath: /etc/promtail
|
||
- name: run
|
||
mountPath: /run/promtail
|
||
- name: pods
|
||
mountPath: /var/log/pods
|
||
readOnly: true
|
||
- name: containers
|
||
mountPath: /var/lib/docker/containers
|
||
readOnly: true
|
||
volumes:
|
||
- name: config
|
||
configMap:
|
||
name: promtail
|
||
- name: run
|
||
hostPath:
|
||
path: /run/promtail
|
||
- name: pods
|
||
hostPath:
|
||
path: /var/log/pods
|
||
- name: containers
|
||
hostPath:
|
||
path: /var/lib/docker/containers
|
||
`, ns, ns, ns, lokiURL, extLabels, ns, promtailImage)
|
||
}
|
||
|
||
// FluentbitConfig Sidecar 模式 Fluent Bit 配置（采集 logPath，推送至 lokiHost:lokiPort）
|
||
// Sidecar 已明确知道应用与命名空间，无需 Kubernetes 元数据过滤器。
|
||
func FluentbitConfig(lokiHost, lokiPort, logPath, app, namespace string) string {
|
||
return fmt.Sprintf(`[SERVICE]
|
||
Flush 1
|
||
Log_Level info
|
||
HTTP_Server On
|
||
HTTP_Listen 0.0.0.0
|
||
HTTP_Port 2020
|
||
storage.path /tmp/flb-storage/
|
||
storage.sync normal
|
||
storage.checksum off
|
||
storage.backlog.mem_limit 5M
|
||
Health_Check On
|
||
HC_Errors_Count 5
|
||
HC_Retry_Failure_Count 5
|
||
HC_Period 60
|
||
|
||
[INPUT]
|
||
Name tail
|
||
Path %s
|
||
Tag app.%s
|
||
Refresh_Interval 5
|
||
Mem_Buf_Limit 10MB
|
||
Skip_Long_Lines On
|
||
Read_from_Head True
|
||
DB /tmp/flb.db
|
||
DB.Sync Normal
|
||
Buffer_Max_Size 1MB
|
||
storage.type filesystem
|
||
|
||
[FILTER]
|
||
Name modify
|
||
Match app.*
|
||
Add namespace %s
|
||
Add app %s
|
||
Add source fluent-bit
|
||
|
||
[OUTPUT]
|
||
Name loki
|
||
Match *
|
||
Host %s
|
||
Port %s
|
||
Labels job=fluent-bit, app=%s, namespace=%s
|
||
Line_Format json
|
||
Retry_Limit 10
|
||
`, logPath, app, namespace, app, lokiHost, lokiPort, app, namespace)
|
||
}
|
||
|
||
// FluentbitConfigMapYAML 为每个采集策略生成独立 ConfigMap，避免同命名空间的 Sidecar 相互覆盖。
|
||
func FluentbitConfigMapYAML(conf, namespace, name string) string {
|
||
return fmt.Sprintf(`apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: %s
|
||
namespace: %s
|
||
data:
|
||
fluent-bit.conf: |
|
||
%s
|
||
`, name, namespace, indentBlock(conf, " "))
|
||
}
|
||
|
||
// SidecarPatchYAML 将业务容器与 Fluent Bit 挂到同一日志卷和目录。
|
||
func SidecarPatchYAML(configMapName string, mount sidecarLogMount) string {
|
||
volume := ""
|
||
if mount.createVolume {
|
||
volume = fmt.Sprintf(" - name: %s\n emptyDir: {}\n", mount.volumeName)
|
||
}
|
||
appMount := ""
|
||
if mount.mountApplication {
|
||
appMount = fmt.Sprintf(" - name: %s\n volumeMounts:\n - name: %s\n mountPath: %q\n", mount.containerName, mount.volumeName, mount.mountPath)
|
||
}
|
||
subPath := ""
|
||
if mount.subPath != "" {
|
||
subPath = fmt.Sprintf(" subPath: %q\n", mount.subPath)
|
||
}
|
||
return fmt.Sprintf(`spec:
|
||
template:
|
||
spec:
|
||
volumes:
|
||
%s - name: fluent-bit-config
|
||
configMap:
|
||
name: %s
|
||
- name: fluent-bit-state
|
||
emptyDir: {}
|
||
containers:
|
||
%s - name: fluent-bit
|
||
image: %s
|
||
imagePullPolicy: IfNotPresent
|
||
ports:
|
||
- name: fluent-bit-http
|
||
containerPort: 2020
|
||
protocol: TCP
|
||
volumeMounts:
|
||
- name: %s
|
||
mountPath: %q
|
||
%s readOnly: true
|
||
- name: fluent-bit-config
|
||
mountPath: /fluent-bit/etc/
|
||
- name: fluent-bit-state
|
||
mountPath: /tmp
|
||
resources:
|
||
limits:
|
||
cpu: 200m
|
||
memory: 256Mi
|
||
requests:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
securityContext:
|
||
allowPrivilegeEscalation: false
|
||
readOnlyRootFilesystem: true
|
||
capabilities:
|
||
drop:
|
||
- ALL
|
||
livenessProbe:
|
||
httpGet:
|
||
path: /
|
||
port: fluent-bit-http
|
||
initialDelaySeconds: 10
|
||
periodSeconds: 10
|
||
readinessProbe:
|
||
httpGet:
|
||
path: /
|
||
port: fluent-bit-http
|
||
initialDelaySeconds: 5
|
||
periodSeconds: 5
|
||
`, volume, configMapName, appMount, ResolveImage("docker.io/fluent/fluent-bit:5.1.5"), mount.volumeName, mount.mountPath, subPath)
|
||
}
|
||
|
||
// indentBlock 给每行加前缀（ConfigMap 嵌入配置需缩进）
|
||
func indentBlock(s, pad string) string {
|
||
lines := strings.Split(s, "\n")
|
||
for i, l := range lines {
|
||
if l != "" {
|
||
lines[i] = pad + l
|
||
}
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
// parseLokiURL 从 http://host:port 提取 host 与 port（Fluent Bit OUTPUT 需要）
|
||
func parseLokiURL(endpoint string) (host, port string) {
|
||
u := strings.TrimSpace(endpoint)
|
||
u = strings.TrimPrefix(u, "http://")
|
||
u = strings.TrimPrefix(u, "https://")
|
||
if i := strings.Index(u, "/"); i >= 0 {
|
||
u = u[:i]
|
||
}
|
||
if i := strings.LastIndex(u, ":"); i >= 0 {
|
||
return u[:i], u[i+1:]
|
||
}
|
||
return u, "3100"
|
||
}
|