opsx
bad2a1e624
=== 一、主机「测试连接」永远显示旧错误（真实 bug）===
HostService.Validate 的失败分支也写 status=active，且从不写 statusMessage：
if info, err := client.GetHostInfo(ctx); err == nil { 更新 os/arch }
else { s.Update(id, {status:"active"}) } // ← 失败也标记成功
结果：连不上也显示正常，页面永远停在几分钟前的旧错误文案，
用户改了 IP 再点测试，还以为"改了还是不行"。
改为区分成功/失败并写清原因：
成功 → status=active + os/arch + statusMessage="连接成功"
失败 → status=error + statusMessage=FormatSSHError(err)
=== 二、回环地址在平台里必然连不上 ===
平台跑在容器/Pod 内，127.0.0.1 指向平台自身。
- 新建主机时拦截：hostIP 为 127.0.0.1/localhost/::1 直接报错并解释原因
- 前端表单同步校验 + 文案
- SSH 错误映射补充：连接被拒绝时提示可能是地址填了回环
=== 三、主机模式采集系统日志（两处会导致采集失败）===
1. pipeline 一律套 cri 阶段：cri 只认容器运行时格式，
套在 /var/log/messages 这类 syslog 上会取不到时间戳、整行被丢。
→ appPipelineStages() 按路径判断：容器日志用 cri，普通文本日志只做限流。
2. promtailExternalLabels 写死 8 空格缩进：主机模板 external_labels 只有 4 空格，
一旦给采集器配置了标签就生成非法 YAML，Promtail 装不上。
→ 增加 indent 参数，主机 6 空格 / DaemonSet 8 空格。
3. 新增 system 采集任务：
/var/log/{messages,secure,cron,maillog,syslog,dmesg}
正则解析 syslog 时间戳（Jan _2 15:04:05，Asia/Shanghai）+ 限流
测试：TestHostPromtailConfigValid / CRIPathKeepsCRIStage / PlainFileHasNoCRIStage
TestPromtailManifestParses 全部通过
=== 四、资源用量 CPU 解析（nanocores）===
metrics-server 返回的 CPU 常是纳核（如 "194533645n"=194.5m）。
原解析只处理 "m"，导致 194 亿核、页面出现"CPU 已达 limit 的 4863341125%"。
→ parseQuantityMilli 支持 n/u/m/整核
实测：CPU 当前=178m，峰值=3300m，建议"峰值占 limit 82%，可考虑提前扩容"
=== 五、其他 ===
- 添加集群默认改用 Token 认证（最小权限、可随时吊销）
- RBAC 片段补 metrics.k8s.io（资源面板需要，独立 API 组常被漏授）
- 文案去掉 Grafana 需求描述（平台自身不部署 Grafana）
镜像：lghost/lokistack:latest + v0.1 = sha256:7bddd0f4d1f1
468 lines
13 KiB
Go
468 lines
13 KiB
Go
// LokiDeployer — loki.yaml 配置生成:模板 + 对象存储/规则存储/ring 拓扑。
|
||
package services
|
||
|
||
import (
|
||
"bytes"
|
||
"fmt"
|
||
"os"
|
||
"strconv"
|
||
"strings"
|
||
"text/template"
|
||
|
||
"lokistack/models"
|
||
"lokistack/pkg/crypto"
|
||
)
|
||
|
||
const lokiConfigTemplate = `auth_enabled: {{ .AuthEnabled }}
|
||
|
||
common:
|
||
path_prefix: /var/lib/loki
|
||
{{- if .FilesystemStorage }}
|
||
storage:
|
||
filesystem:
|
||
chunks_directory: /var/lib/loki/chunks
|
||
rules_directory: /var/lib/loki/rules
|
||
{{- end }}
|
||
replication_factor: {{ .ReplicationFactor }}
|
||
ring:
|
||
instance_addr: {{ .RingInstanceAddr }}
|
||
kvstore:
|
||
store: {{ .RingKVStore }}
|
||
|
||
{{- if .StorageConfigBlock }}
|
||
|
||
storage_config:
|
||
{{ .StorageConfigBlock }}
|
||
{{- end }}
|
||
{{- if .MemberlistBlock }}
|
||
|
||
{{ .MemberlistBlock }}
|
||
{{- end }}
|
||
|
||
ingester:
|
||
wal:
|
||
dir: /var/lib/loki/wal
|
||
flush_on_shutdown: true
|
||
replay_memory_ceiling: 1GB
|
||
lifecycler:
|
||
ring:
|
||
kvstore:
|
||
store: {{ .RingKVStore }}
|
||
replication_factor: {{ .ReplicationFactor }}
|
||
{{- if .MultiReplica }}
|
||
autoforget_unhealthy: true
|
||
{{- end }}
|
||
|
||
compactor:
|
||
working_directory: /var/lib/loki/compactor
|
||
retention_enabled: true
|
||
delete_request_store: {{ .CompactorStoreType }}
|
||
# 过期 chunk 的删除并发。Loki 默认 150，但各云对象存储的 QPS 限制差别很大，
|
||
# 打太猛会被限流（腾讯云 COS 实测返回 503 SlowDown，整轮 apply_retention 失败，
|
||
# 残留 21,354 个对象 / 24.8 GB 删不掉）。该值按 provider 取值，可用
|
||
# 环境变量 LOKI_COMPACTOR_DELETE_WORKERS 覆盖。
|
||
retention_delete_worker_count: {{ .CompactorDeleteWorkers }}
|
||
# 过期 chunk 的「标记 → 真正删除」间隔。默认 2h 会让对象存储里
|
||
# 多留 2 小时的数据（实测桶内超过保留期的对象一度占到 24.6 GB）。
|
||
# 预发环境取 10m：既保留短暂的安全窗口，又不会长期占用存储成本。
|
||
retention_delete_delay: 10m
|
||
|
||
limits_config:
|
||
retention_period: {{ .RetentionPeriod }}h
|
||
# 单实例 Loki 的并发护栏：默认 32 高得离谱，一条重查询就能把单进程打满
|
||
max_query_parallelism: 8
|
||
|
||
# ── 查询结果缓存（官方推荐；此前完全没配，重复查询要重算一遍）───────
|
||
# align_queries_with_step 让不同时间范围的同类查询对齐到同一 step，
|
||
# 显著提高缓存命中率；命中后同样的"日志量统计"直接从缓存返回。
|
||
query_range:
|
||
align_queries_with_step: true
|
||
cache_results: true
|
||
results_cache:
|
||
cache:
|
||
embedded_cache:
|
||
enabled: true
|
||
max_size_mb: 100
|
||
|
||
schema_config:
|
||
configs:
|
||
- from: "2024-01-01"
|
||
store: tsdb
|
||
object_store: {{ .ObjectStore }}
|
||
schema: v13
|
||
index:
|
||
prefix: tsdb_index_
|
||
period: 24h
|
||
|
||
ruler:
|
||
storage:
|
||
type: {{ .RulerStoreType }}
|
||
{{- if .RulerStorageBlock }}
|
||
{{ .RulerStorageBlock }}
|
||
{{- end }}
|
||
{{- if eq .RulerStoreType "local" }}
|
||
local:
|
||
directory: /var/lib/loki/rules
|
||
{{- end }}
|
||
rule_path: /var/lib/loki/rules-temp
|
||
ring:
|
||
kvstore:
|
||
store: {{ .RingKVStore }}
|
||
|
||
analytics:
|
||
reporting_enabled: false
|
||
`
|
||
|
||
type lokiConfigData struct {
|
||
AuthEnabled bool
|
||
ReplicationFactor int
|
||
RetentionPeriod int
|
||
StorageConfigBlock string
|
||
ObjectStore string
|
||
RulerStoreType string
|
||
CompactorStoreType string
|
||
// CompactorDeleteWorkers 过期 chunk 删除并发，按对象存储类型取默认值
|
||
CompactorDeleteWorkers int
|
||
RingInstanceAddr string // 单副本 127.0.0.1；多副本 0.0.0.0（须可被对端寻址）
|
||
RingKVStore string // 单副本 inmemory；多副本 memberlist
|
||
MemberlistBlock string // 多副本时非空，注入 memberlist 顶级配置
|
||
RulerStorageBlock string // 对象存储时非空，注入 ruler.storage.s3/gcs/azure 嵌套配置
|
||
MultiReplica bool
|
||
FilesystemStorage bool
|
||
}
|
||
|
||
func (d *LokiDeployer) generateLokiConfig() string {
|
||
retentionHours := d.deployment.RetentionDays * 24
|
||
if retentionHours <= 0 {
|
||
retentionHours = DefaultRetentionDays * 24
|
||
}
|
||
|
||
storageType := d.effectiveStorageType()
|
||
storageBlock := d.buildStorageConfigBlock()
|
||
objectStore := "filesystem"
|
||
rulerStore := "local"
|
||
compactorStore := "filesystem"
|
||
|
||
switch storageType {
|
||
case "s3", "oss", "minio", "cos", "obs":
|
||
objectStore = "s3"
|
||
rulerStore = "s3"
|
||
// 删除请求存本地盘，避免各云 S3 兼容实现差异导致 compactor 启动失败
|
||
compactorStore = "filesystem"
|
||
case "gcs":
|
||
objectStore = "gcs"
|
||
rulerStore = "gcs"
|
||
compactorStore = "gcs"
|
||
case "azure":
|
||
objectStore = "azure"
|
||
rulerStore = "azure"
|
||
compactorStore = "azure"
|
||
}
|
||
|
||
// Ring 拓扑：单副本用 inmemory + 127.0.0.1（无需跨实例通信）；
|
||
// 多副本必须切 memberlist + 0.0.0.0，否则 ring 无法收敛、写入失败。
|
||
ringAddr := "127.0.0.1"
|
||
ringStore := "inmemory"
|
||
memberlistBlock := ""
|
||
if d.deployment.Replicas >= 2 {
|
||
ringAddr = "0.0.0.0"
|
||
ringStore = "memberlist"
|
||
ns := d.deployment.Namespace
|
||
if ns == "" {
|
||
ns = "log"
|
||
}
|
||
memberlistBlock = fmt.Sprintf(`memberlist:
|
||
join_members:
|
||
- loki-headless.%s.svc.cluster.local
|
||
bind_port: 7946
|
||
dead_node_reclaim_time: 30s
|
||
gossip_to_dead_nodes_time: 2m
|
||
left_ingesters_timeout: 5m`, ns)
|
||
}
|
||
|
||
data := lokiConfigData{
|
||
AuthEnabled: d.deployment.AuthEnabled,
|
||
ReplicationFactor: d.replicationFactor(),
|
||
RetentionPeriod: retentionHours,
|
||
StorageConfigBlock: storageBlock,
|
||
ObjectStore: objectStore,
|
||
RulerStoreType: rulerStore,
|
||
CompactorStoreType: compactorStore,
|
||
CompactorDeleteWorkers: compactorDeleteWorkers(d.deployment.StorageConfig),
|
||
RingInstanceAddr: ringAddr,
|
||
RingKVStore: ringStore,
|
||
MemberlistBlock: memberlistBlock,
|
||
RulerStorageBlock: d.buildRulerStorageBlock(),
|
||
MultiReplica: d.deployment.Replicas >= 2,
|
||
FilesystemStorage: objectStore == "filesystem",
|
||
}
|
||
|
||
tmpl, err := template.New("loki-config").Parse(lokiConfigTemplate)
|
||
if err != nil {
|
||
return d.generateLokiConfigFallback(retentionHours)
|
||
}
|
||
|
||
var buf bytes.Buffer
|
||
if err := tmpl.Execute(&buf, data); err != nil {
|
||
return d.generateLokiConfigFallback(retentionHours)
|
||
}
|
||
return buf.String()
|
||
}
|
||
|
||
// defaultCompactorDeleteWorkers 是 Loki 自身的默认值。
|
||
const defaultCompactorDeleteWorkers = 150
|
||
|
||
// compactorDeleteWorkers 按对象存储类型给出「过期 chunk 删除并发」。
|
||
//
|
||
// 为什么不能写死一个值：这个并发直接决定 compactor 对对象存储发出的 DELETE 速率，
|
||
// 而各云的限流差别很大。打满会被限流并让整轮 apply_retention 失败 ——
|
||
// 腾讯云 COS 实测默认 150 就返回 503 SlowDown，过期 chunk 残留 21,354 个 / 24.8 GB
|
||
// 删不掉；降到 16 后删除稳定推进。
|
||
//
|
||
// cos 16 实测 150 触发 503 SlowDown
|
||
// obs 32 华为 OBS 未给明确单桶 QPS，取保守值
|
||
// minio 32 自建，取决于后端磁盘 IO，别打满
|
||
// azure 32 存储账户限流较严
|
||
// oss 64 阿里云 OSS 单桶 QPS 万级，留余量
|
||
// gcs 64 GCS 高 QPS，但有 per-bucket 操作速率
|
||
// s3 96 AWS S3 单前缀约 3,500 DELETE/s，150 也能跑，留余量
|
||
//
|
||
// 任意 provider 都可用环境变量 LOKI_COMPACTOR_DELETE_WORKERS 覆盖（运维手调无需改代码）。
|
||
func compactorDeleteWorkers(sc *models.StorageConfig) int {
|
||
if raw := strings.TrimSpace(os.Getenv("LOKI_COMPACTOR_DELETE_WORKERS")); raw != "" {
|
||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||
return n
|
||
}
|
||
}
|
||
if sc == nil {
|
||
return defaultCompactorDeleteWorkers
|
||
}
|
||
switch strings.ToLower(strings.TrimSpace(sc.StorageType)) {
|
||
case "cos":
|
||
return 16
|
||
case "obs", "minio", "azure":
|
||
return 32
|
||
case "oss", "gcs":
|
||
return 64
|
||
case "s3":
|
||
return 96
|
||
default:
|
||
return defaultCompactorDeleteWorkers
|
||
}
|
||
}
|
||
|
||
func (d *LokiDeployer) buildStorageConfigBlock() string {
|
||
if d.deployment.StorageConfig == nil {
|
||
return ""
|
||
}
|
||
|
||
sc := d.deployment.StorageConfig
|
||
accessKey, secretKey := d.storageCredentialValues()
|
||
switch sc.StorageType {
|
||
case "s3":
|
||
endpoint := sc.Endpoint
|
||
if endpoint == "" {
|
||
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
|
||
}
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
return fmt.Sprintf(` aws:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
region: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: true`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
|
||
case "oss", "cos", "obs":
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
// 云厂商 S3 兼容端点有两种写法：
|
||
// 区域端点 cos.ap-hongkong.myqcloud.com ← 正确
|
||
// 虚拟主机式 mybucket-1250000000.cos.<region>... ← 桶名已在 host 里
|
||
// Loki 的 S3 客户端会自己把 bucketnames 拼到 endpoint 前面（s3forcepathstyle=false
|
||
// 即虚拟主机风格），所以 endpoint 必须是区域端点。若用户在平台里填了带桶名的写法，
|
||
// 这里剥掉桶名前缀，否则会拼出 <bucket>.<bucket>.cos... 导致 TLS 证书校验失败、
|
||
// 日志写入失败（ruler/ingester 持续报 x509 证书不匹配）。
|
||
endpoint := stripBucketPrefix(sc.Endpoint, sc.Bucket)
|
||
return fmt.Sprintf(` aws:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
region: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: false`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
|
||
case "minio":
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
return fmt.Sprintf(` aws:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: true`, sc.Bucket, stripBucketPrefix(sc.Endpoint, sc.Bucket), accessKey, secretKey, insecure)
|
||
case "gcs":
|
||
return fmt.Sprintf(` gcs:
|
||
bucket_name: %s`, sc.Bucket)
|
||
case "azure":
|
||
return fmt.Sprintf(` azure:
|
||
container_name: %s
|
||
account_name: %s
|
||
account_key: %s`, sc.Bucket, accessKey, secretKey)
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
// buildRulerStorageBlock 生成 ruler.storage 下与存储类型同名的嵌套配置
|
||
// (Loki 3.x 的 ruler 存储客户端不回退读取 storage_config,缺少此块会以空 bucket 启动失败)
|
||
func (d *LokiDeployer) buildRulerStorageBlock() string {
|
||
if d.deployment.StorageConfig == nil {
|
||
return ""
|
||
}
|
||
sc := d.deployment.StorageConfig
|
||
accessKey, secretKey := d.storageCredentialValues()
|
||
switch sc.StorageType {
|
||
case "s3":
|
||
endpoint := sc.Endpoint
|
||
if endpoint == "" {
|
||
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
|
||
}
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
return fmt.Sprintf(` s3:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
region: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: true`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
|
||
case "oss", "cos", "obs":
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
return fmt.Sprintf(` s3:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
region: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: false`, sc.Bucket, sc.Endpoint, sc.Region, accessKey, secretKey, insecure)
|
||
case "minio":
|
||
insecure := "false"
|
||
if sc.Insecure {
|
||
insecure = "true"
|
||
}
|
||
return fmt.Sprintf(` s3:
|
||
bucketnames: %s
|
||
endpoint: %s
|
||
access_key_id: %s
|
||
secret_access_key: %s
|
||
insecure: %s
|
||
s3forcepathstyle: true`, sc.Bucket, sc.Endpoint, accessKey, secretKey, insecure)
|
||
case "gcs":
|
||
return fmt.Sprintf(` gcs:
|
||
bucket_name: %s`, sc.Bucket)
|
||
case "azure":
|
||
return fmt.Sprintf(` azure:
|
||
container_name: %s
|
||
account_name: %s
|
||
account_key: %s`, sc.Bucket, accessKey, secretKey)
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
func (d *LokiDeployer) storageCredentialValues() (string, string) {
|
||
if d.deployment.StorageConfig == nil {
|
||
return "", ""
|
||
}
|
||
if d.deployment.Mode == "k8s-yaml" || d.deployment.Mode == "host-docker" {
|
||
return "${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}"
|
||
}
|
||
// host-docker 模式没有 Secret 注入，必须把库里 ENC: 密文解密后再写进配置
|
||
return crypto.DecryptField(d.deployment.StorageConfig.AccessKey),
|
||
crypto.DecryptField(d.deployment.StorageConfig.SecretKey)
|
||
}
|
||
|
||
// generateLokiConfigFallback 当模板解析失败时的降级配置
|
||
func (d *LokiDeployer) generateLokiConfigFallback(retentionHours int) string {
|
||
return fmt.Sprintf(`auth_enabled: %v
|
||
server:
|
||
http_listen_port: 3100
|
||
grpc_listen_port: 9095
|
||
http_server_read_timeout: 3m
|
||
http_server_write_timeout: 3m
|
||
log_level: info
|
||
common:
|
||
path_prefix: /var/lib/loki
|
||
storage:
|
||
filesystem:
|
||
chunks_directory: /var/lib/loki/chunks
|
||
rules_directory: /var/lib/loki/rules
|
||
replication_factor: %d
|
||
ring:
|
||
instance_addr: 127.0.0.1
|
||
kvstore:
|
||
store: inmemory
|
||
ingester:
|
||
wal:
|
||
enabled: true
|
||
dir: /var/lib/loki/wal
|
||
lifecycler:
|
||
ring:
|
||
kvstore:
|
||
store: inmemory
|
||
replication_factor: %d
|
||
final_sleep: 0s
|
||
chunk_idle_period: 30m
|
||
max_chunk_age: 2h
|
||
querier:
|
||
max_concurrent: 10
|
||
compactor:
|
||
working_directory: /var/lib/loki/compactor
|
||
retention_enabled: true
|
||
retention_delete_delay: 2h
|
||
limits_config:
|
||
retention_period: %dh
|
||
ingestion_rate_mb: 64
|
||
ingestion_burst_size_mb: 128
|
||
schema_config:
|
||
configs:
|
||
- from: "2024-01-01"
|
||
store: tsdb
|
||
object_store: filesystem
|
||
schema: v13
|
||
index:
|
||
prefix: tsdb_index_
|
||
period: 24h
|
||
analytics:
|
||
reporting_enabled: false
|
||
`, d.deployment.AuthEnabled, d.replicationFactor(), d.replicationFactor(), retentionHours)
|
||
}
|
||
|
||
// ==================== 辅助函数 ====================
|
||
|
||
func (d *LokiDeployer) replicationFactor() int {
|
||
if d.deployment.Replicas >= 3 {
|
||
return 3
|
||
}
|
||
if d.deployment.Replicas >= 2 {
|
||
return 2
|
||
}
|
||
return 1
|
||
}
|