feat(collectors): 可编辑采集配置(B 档结构化编辑器)+ 修 host 标签 / syslog 时间戳
=== 一、两个线上 bug(172.19.16.8 真机取证)===
1) host 标签是字面量 ${HOSTNAME}
Promtail 不做环境变量展开(除非 -config.expand-env + systemd Environment),
实测 21,825 条日志的 host 全是 "${HOSTNAME}",无法按主机过滤。
→ 安装时 `hostname -s` 探测真实主机名并渲染进配置,结果存 Collector.ResolvedHostname,
让「预览配置」与线上一致。
2) 系统日志时间戳解析失效
TencentOS 4 的 /var/log/messages 是 ISO8601(2026-07-17T15:38:24.923784+08:00),
旧正则按传统 syslog(Oct 28 18:11:06)写死 → timestamp 阶段匹配不上,用的是采集时间,
回填历史日志时间全错。
→ 安装时探测目标机格式:iso8601 → RFC3339Nano;rfc3164 → Jan _2 15:04:05 + Local;
系统日志路径同时探测(Ubuntu 没有 messages,写死会一条都采不到)。
实测:写一条 10 分钟前的 ISO8601 探针行,Loki 里 loki_ts 与该行自身时间偏差 **0 秒**;
另写一条 2 小时前的被 Loki 以 entry too far behind 拒绝 —— 恰好反证时间戳是按事件时间
解析后推送的(旧实现会用采集时间,反而被接收)。
=== 二、B 档:结构化多 Job 编辑器 ===
后端
- models/scrape_job.go:ScrapeJob + ValidateScrapeJobs
校验 job 名格式、绝对路径、**敏感路径拦截**(/etc/shadow、/etc/sudoers、/root/.ssh、
id_rsa、*.pem…)、正则可编译、job/host/__path__ 保留标签名、数量上限。
- services/promtail_host_config.go:多 job 渲染;多路径展开成多个 static_configs;
pipeline 顺序硬约束 cri → multiline → json → timestamp → drop → limit;
encoding 写在 scrape_config 级(放进 static_configs 里 promtail 直接起不来)。
- services/host_profile.go:探测 distro / init / hostname / 时区 / 架构 / journald
与真实存在的系统日志路径。
- CollectorService:renderConfig 实时渲染、PreviewConfig(只渲染不落库)、
Update 支持 scrapeJobs 并校验、安装结果(hostname/syslog 格式/配置快照)落库。
- 新增 POST /api/v1/collectors/:id/preview → {config, warnings, hostname, syslogFormat}。
- installHost:配置先写 .new + `promtail -check-syntax` 通过才替换,重启后轮询
systemctl is-active,失败把 journalctl 尾部带回前端。
前端
- 采集项结构化编辑器:名称/路径/标签常驻,多行合并 / JSON / 自定义时间戳 / 命中丢弃 /
限流折叠在「高级」里;「预览 YAML」弹窗在保存前看到将要下发的配置。
- 本地即时校验 + 旧数据(只有 logPath)自动迁移成 app + system 两个采集项。
=== 三、验证(host-verify-promtail @172.19.16.8)===
- 预览:host=pre-cex-ops-jenkins(无 ${HOSTNAME}),2 job / 3 条系统日志路径 /
ISO8601 正则 / 多行 + drop + 限流 全部按填写渲染。
- 部署:promtail active、NRestarts=0、-check-syntax 通过,主机侧配置与预览逐行一致。
- Loki:新数据 host 全为 pre-cex-ops-jenkins;job=system 415 条/5min、job=app 40 条/5min。
- 负向 7/7 返回指名到字段的 400:/etc/shadow、/root/.ssh/id_rsa、相对路径、坏正则、
job 名带空格、保留标签名、空路径。
镜像:lghost/lokistack:latest + v0.1 = sha256:26172f51187d
This commit is contained in:
1 parent
55d6a74eb3
commit
c983d0e3c1
18 files changed
+1678
-254
No files matched your search
@@ -26,6 +26,7 @@ func (ctrl *CollectorController) Create(c *gin.Context) {
|
||||
Name: req.Name, Type: req.Type, LokiDeploymentID: req.LokiDeploymentID,
|
||||
HostID: req.HostID, ClusterID: req.ClusterID, TargetDeployment: req.TargetDeployment,
|
||||
Namespace: req.Namespace, LogPath: req.LogPath, Labels: req.Labels,
|
||||
ScrapeJobs: req.ScrapeJobs,
|
||||
})
|
||||
if err != nil {
|
||||
utils.Error(c, 400, err.Error())
|
||||
@@ -100,6 +101,27 @@ func (ctrl *CollectorController) Install(c *gin.Context) {
|
||||
utils.Success(c, gin.H{"message": "已开始部署采集器"})
|
||||
}
|
||||
|
||||
// Preview 渲染「当前配置 + 本次改动」的 promtail.yaml,只预览不落库。
|
||||
// 供结构化编辑器里的「预览 YAML」按钮使用:用户能先看到将要写到目标机的配置,
|
||||
// 保存前就发现路径/正则写错,而不是点了部署才发现采集起不来。
|
||||
func (ctrl *CollectorController) Preview(c *gin.Context) {
|
||||
id, err := utils.IntParam(c, "id")
|
||||
if err != nil {
|
||||
utils.BadRequest(c, "ID 无效")
|
||||
return
|
||||
}
|
||||
var updates map[string]interface{}
|
||||
if !utils.BindAndValidate(c, &updates) {
|
||||
return
|
||||
}
|
||||
out, err := ctrl.service.PreviewConfig(uint(id), updates)
|
||||
if err != nil {
|
||||
utils.Error(c, 400, err.Error())
|
||||
return
|
||||
}
|
||||
utils.Success(c, out)
|
||||
}
|
||||
|
||||
func (ctrl *CollectorController) Uninstall(c *gin.Context) {
|
||||
id, err := utils.IntParam(c, "id")
|
||||
if err != nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package dto
|
||||
|
||||
// 采集器域
|
||||
import "lokistack/models"
|
||||
|
||||
type CreateCollectorRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Type string `json:"type" binding:"required"`
|
||||
@@ -11,4 +12,6 @@ type CreateCollectorRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
LogPath string `json:"logPath"`
|
||||
Labels string `json:"labels"`
|
||||
// ScrapeJobs 主机模式的结构化采集项(可选;留空按 LogPath 生成 app + system 两个默认项)
|
||||
ScrapeJobs []models.ScrapeJob `json:"scrapeJobs"`
|
||||
}
|
||||
@@ -25,14 +25,26 @@ type Collector struct {
|
||||
AppInclude []string `json:"appInclude"`
|
||||
// TargetNamespaces 只采集这些命名空间的 Pod 日志;为空表示采集全集群。
|
||||
// 对应 promtail 里 kubernetes_sd_configs.namespaces.names
|
||||
TargetNamespaces []string `json:"targetNamespaces"`
|
||||
LogPath string `json:"logPath"` // 采集日志路径
|
||||
Labels string `json:"labels"` // 额外标签(JSON)
|
||||
Status string `json:"status"` // pending | active | error
|
||||
StatusMessage string `json:"statusMessage"`
|
||||
Config string `json:"config"` // 生成的配置快照
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
TargetNamespaces []string `json:"targetNamespaces"`
|
||||
LogPath string `json:"logPath"` // 采集日志路径
|
||||
Labels string `json:"labels"` // 额外标签(JSON)
|
||||
// ScrapeJobs 主机模式的结构化采集项(多 job / 多路径 / 多行合并 / 丢弃 / 限流)。
|
||||
// 为空时按 LogPath 生成默认的两个 job(app + system),保持旧行为。
|
||||
ScrapeJobs []ScrapeJob `json:"scrapeJobs,omitempty"`
|
||||
// ResolvedHostname / SyslogFormat 是上一次安装到目标机时探测到的真实值。
|
||||
// 用途:① host 标签写真实主机名(不再写 ${HOSTNAME});② 系统日志按目标机
|
||||
// 实际格式(ISO8601 vs 传统 syslog)选正则 —— 这两条是安装期才能知道的信息,
|
||||
// 存下来才能让「配置预览」和线上真实配置一致。
|
||||
// SystemLogPaths 安装时探测到的系统日志绝对路径(Ubuntu/Debian 与 RHEL 系不同),
|
||||
// 存下来让「预览配置」和实际下发保持一致。
|
||||
SystemLogPaths []string `json:"systemLogPaths,omitempty"`
|
||||
ResolvedHostname string `json:"resolvedHostname,omitempty"`
|
||||
SyslogFormat string `json:"syslogFormat,omitempty"` // iso8601 | syslog
|
||||
Status string `json:"status"` // pending | active | error
|
||||
StatusMessage string `json:"statusMessage"`
|
||||
Config string `json:"config"` // 生成的配置快照
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Collector) Sanitized() map[string]interface{} {
|
||||
@@ -73,6 +85,10 @@ func (c *Collector) Sanitized() map[string]interface{} {
|
||||
"appInclude": c.AppInclude,
|
||||
"logPath": c.LogPath,
|
||||
"labels": c.Labels,
|
||||
"scrapeJobs": c.ScrapeJobs,
|
||||
"systemLogPaths": c.SystemLogPaths,
|
||||
"resolvedHostname": c.ResolvedHostname,
|
||||
"syslogFormat": c.SyslogFormat,
|
||||
"status": c.Status,
|
||||
"statusMessage": c.StatusMessage,
|
||||
"createdAt": c.CreatedAt,
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ScrapeJob 主机 Promtail 的一个采集项(对应配置里的 scrape_configs 一项)。
|
||||
//
|
||||
// 这套结构是「结构化编辑器(B 档)」的落点:用户按字段填,平台负责渲染成
|
||||
// 合法的 promtail YAML(见 services.PromtailConfigForHost)。
|
||||
//
|
||||
// 设计取舍:
|
||||
// - 不暴露任意 pipeline 语句,只暴露能覆盖 90% 场景的几个开关
|
||||
// (多行合并 / JSON 解析 / 自定义时间戳 / 命中丢弃 / 限流);
|
||||
// - System=true 的采集项由平台内置:安装时探测目标机 syslog 格式
|
||||
// (ISO8601 vs 传统 syslog),自动选择正则与 timestamp 格式 —— 这是
|
||||
// Bug4 的根治点,用户不需要懂正则。
|
||||
type ScrapeJob struct {
|
||||
Name string `json:"name"`
|
||||
Paths []string `json:"paths"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
// Encoding 仅支持 utf-8 / ascii;留空 = utf-8
|
||||
Encoding string `json:"encoding,omitempty"`
|
||||
// System 内置系统日志采集项:自动加 syslog 正则 + 时间戳解析,Paths 留空时用默认系统日志路径
|
||||
System bool `json:"system,omitempty"`
|
||||
// Multiline 多行合并(Java 堆栈等)
|
||||
Multiline *JobMultiline `json:"multiline,omitempty"`
|
||||
// JSON 是否用 json 阶段解析(解析后字段可用于 drop/labels)
|
||||
JSON bool `json:"json,omitempty"`
|
||||
// Timestamp 自定义时间戳解析;留空且非 System 时使用采集时间
|
||||
Timestamp *JobTimestamp `json:"timestamp,omitempty"`
|
||||
// DropIfMatch 命中这些正则的日志行直接丢弃(如健康检查日志)
|
||||
DropIfMatch []string `json:"dropIfMatch,omitempty"`
|
||||
// RateLimit 限流;留空用平台默认(rate=2000 burst=4000 drop)
|
||||
RateLimit *JobRateLimit `json:"rateLimit,omitempty"`
|
||||
}
|
||||
|
||||
type JobMultiline struct {
|
||||
// FirstLine 行首正则,匹配到即认为是新一条日志的开始
|
||||
FirstLine string `json:"firstline"`
|
||||
// MaxLines 单条合并的上限行数,默认 128,上限 1000
|
||||
MaxLines int `json:"maxLines,omitempty"`
|
||||
}
|
||||
|
||||
type JobTimestamp struct {
|
||||
Source string `json:"source"`
|
||||
// Format 是 Go 时间布局(RFC3339 / RFC3339Nano / 2006-01-02 15:04:05 …)
|
||||
Format string `json:"format"`
|
||||
Location string `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
type JobRateLimit struct {
|
||||
Rate int `json:"rate"`
|
||||
Burst int `json:"burst"`
|
||||
}
|
||||
|
||||
// DefaultSystemPaths 系统日志默认采集路径(不存在时 promtail 会跳过,不报错)
|
||||
var DefaultSystemPaths = []string{"/var/log/messages", "/var/log/secure", "/var/log/cron", "/var/log/maillog", "/var/log/syslog", "/var/log/dmesg"}
|
||||
|
||||
const (
|
||||
MaxScrapeJobs = 20
|
||||
MaxJobLabels = 20
|
||||
MaxJobPaths = 20
|
||||
maxJobNameLen = 64
|
||||
DefaultMultilineMax = 128
|
||||
MaxMultilineLines = 1000
|
||||
)
|
||||
|
||||
var jobNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
var labelNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// forbiddenPathPrefixes 绝对禁止采集的路径。
|
||||
//
|
||||
// 采集就是把文件内容推到 Loki —— 把 /etc/shadow、私钥目录写进采集项,
|
||||
// 等于把主机凭据搬到日志平台里(权限要求更低的系统),必须拦死。
|
||||
var forbiddenPathPrefixes = []string{
|
||||
"/etc/shadow", "/etc/gshadow", "/etc/sudoers", "/etc/ssh/",
|
||||
"/root/.ssh/", "/proc/", "/sys/", "/dev/",
|
||||
}
|
||||
|
||||
var forbiddenPathSubstrings = []string{
|
||||
"id_rsa", "id_ed25519", "id_ecdsa", ".pem", "private_key", "keystore",
|
||||
}
|
||||
|
||||
// ValidateScrapeJobs 校验采集项集合;返回的错误信息直接展示给用户,要指名到具体字段。
|
||||
func ValidateScrapeJobs(jobs []ScrapeJob) error {
|
||||
if len(jobs) > MaxScrapeJobs {
|
||||
return fmt.Errorf("采集项过多(%d 个),最多 %d 个", len(jobs), MaxScrapeJobs)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i := range jobs {
|
||||
j := &jobs[i]
|
||||
j.Name = strings.TrimSpace(j.Name)
|
||||
if j.Name == "" {
|
||||
return fmt.Errorf("第 %d 个采集项缺少名称(job_name)", i+1)
|
||||
}
|
||||
if len(j.Name) > maxJobNameLen || !jobNameRe.MatchString(j.Name) {
|
||||
return fmt.Errorf("采集项名称「%s」不合法:只能包含字母、数字、点、下划线、中划线,且不超过 %d 字符", j.Name, maxJobNameLen)
|
||||
}
|
||||
if seen[j.Name] {
|
||||
return fmt.Errorf("采集项名称「%s」重复", j.Name)
|
||||
}
|
||||
seen[j.Name] = true
|
||||
|
||||
paths := j.Paths
|
||||
if j.System && len(paths) == 0 {
|
||||
paths = DefaultSystemPaths
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return fmt.Errorf("采集项「%s」至少需要填写一个日志路径", j.Name)
|
||||
}
|
||||
if len(paths) > MaxJobPaths {
|
||||
return fmt.Errorf("采集项「%s」路径过多(%d 个),最多 %d 个", j.Name, len(paths), MaxJobPaths)
|
||||
}
|
||||
for _, p := range paths {
|
||||
if err := validateLogPath(p); err != nil {
|
||||
return fmt.Errorf("采集项「%s」的路径不合法:%w", j.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(j.Labels) > MaxJobLabels {
|
||||
return fmt.Errorf("采集项「%s」标签过多(%d 个),最多 %d 个(标签基数过高会拖垮 Loki)", j.Name, len(j.Labels), MaxJobLabels)
|
||||
}
|
||||
for k, v := range j.Labels {
|
||||
if !labelNameRe.MatchString(k) {
|
||||
return fmt.Errorf("采集项「%s」的标签名「%s」不合法:只能用字母、数字、下划线,且不能以数字开头", j.Name, k)
|
||||
}
|
||||
switch k {
|
||||
case "job", "host", "__path__":
|
||||
return fmt.Errorf("采集项「%s」的标签名「%s」是平台保留字段,不能自定义", j.Name, k)
|
||||
}
|
||||
if len(v) > 200 || strings.ContainsAny(v, "\n\r") {
|
||||
return fmt.Errorf("采集项「%s」的标签「%s」取值过长或包含换行", j.Name, k)
|
||||
}
|
||||
}
|
||||
|
||||
if j.Multiline != nil {
|
||||
if strings.TrimSpace(j.Multiline.FirstLine) == "" {
|
||||
return fmt.Errorf("采集项「%s」启用了多行合并,但没填「首行正则」", j.Name)
|
||||
}
|
||||
if _, err := regexp.Compile(j.Multiline.FirstLine); err != nil {
|
||||
return fmt.Errorf("采集项「%s」的首行正则无法编译:%v", j.Name, err)
|
||||
}
|
||||
if j.Multiline.MaxLines < 0 || j.Multiline.MaxLines > MaxMultilineLines {
|
||||
return fmt.Errorf("采集项「%s」的多行最大行数需在 0~%d 之间", j.Name, MaxMultilineLines)
|
||||
}
|
||||
}
|
||||
if j.Timestamp != nil && j.Timestamp.Format != "" {
|
||||
if strings.TrimSpace(j.Timestamp.Source) == "" {
|
||||
return fmt.Errorf("采集项「%s」配置了时间戳格式,但没填字段名(source)", j.Name)
|
||||
}
|
||||
}
|
||||
for _, exp := range j.DropIfMatch {
|
||||
if strings.TrimSpace(exp) == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := regexp.Compile(exp); err != nil {
|
||||
return fmt.Errorf("采集项「%s」的丢弃正则「%s」无法编译:%v", j.Name, exp, err)
|
||||
}
|
||||
}
|
||||
if j.RateLimit != nil {
|
||||
if j.RateLimit.Rate < 0 || j.RateLimit.Burst < 0 {
|
||||
return fmt.Errorf("采集项「%s」的限流参数不能为负", j.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLogPath(p string) error {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
return fmt.Errorf("路径为空")
|
||||
}
|
||||
if strings.ContainsAny(p, "\n\r") {
|
||||
return fmt.Errorf("路径包含换行")
|
||||
}
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
return fmt.Errorf("「%s」必须是绝对路径(以 / 开头)", p)
|
||||
}
|
||||
low := strings.ToLower(p)
|
||||
for _, prefix := range forbiddenPathPrefixes {
|
||||
if strings.HasPrefix(low, prefix) {
|
||||
return fmt.Errorf("「%s」属于禁止采集的敏感路径(凭据/内核接口)", p)
|
||||
}
|
||||
}
|
||||
for _, sub := range forbiddenPathSubstrings {
|
||||
if strings.Contains(low, sub) {
|
||||
return fmt.Errorf("「%s」看起来是密钥/凭据文件,禁止采集", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultScrapeJobs 未配置采集项时的默认值(保持旧行为:应用日志 + 系统日志)
|
||||
func DefaultScrapeJobs(logPath string) []ScrapeJob {
|
||||
if strings.TrimSpace(logPath) == "" {
|
||||
logPath = "/var/log/*.log"
|
||||
}
|
||||
return []ScrapeJob{
|
||||
{Name: "app", Paths: []string{logPath}},
|
||||
{Name: "system", System: true},
|
||||
}
|
||||
}
|
||||
@@ -196,6 +196,7 @@ func SetupRouter() *gin.Engine {
|
||||
authorized.DELETE("/collectors/:id", collectorCtrl.Delete)
|
||||
authorized.POST("/collectors/:id/install", collectorCtrl.Install)
|
||||
authorized.POST("/collectors/:id/uninstall", collectorCtrl.Uninstall)
|
||||
authorized.POST("/collectors/:id/preview", collectorCtrl.Preview)
|
||||
|
||||
// Static files - use executable directory as base
|
||||
distPath := findDistPath()
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -27,6 +28,7 @@ type CreateCollectorRequest struct {
|
||||
Namespace string
|
||||
LogPath string
|
||||
Labels string
|
||||
ScrapeJobs []models.ScrapeJob
|
||||
}
|
||||
|
||||
// fillAssociations 填充关联的 Loki / Host / Cluster(用于配置生成与展示)
|
||||
@@ -59,10 +61,33 @@ func (s *CollectorService) fillAssociations(c *models.Collector) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// 配置快照按当前字段实时渲染:DB 里存的 Config 只是上次安装时的副本,
|
||||
// 用户改完采集项还没点「部署」时,看配置应该看到「将要生效的 YAML」。
|
||||
// renderConfig 只读 c 的字段、不取锁,所以在 RLock 内调用是安全的。
|
||||
if cfg, err := s.renderConfig(c); err == nil {
|
||||
c.Config = cfg
|
||||
}
|
||||
}
|
||||
|
||||
// generateConfigSnapshot 按类型生成配置快照
|
||||
func (s *CollectorService) generateConfigSnapshot(c *models.Collector) string {
|
||||
// parseScrapeJobs 把接口层传来的 scrapeJobs([]interface{} / JSON 字符串)解析成模型。
|
||||
// gin 的 map 绑定拿不到强类型,统一过一遍 JSON 转换,避免手写一堆类型断言。
|
||||
func parseScrapeJobs(v interface{}) ([]models.ScrapeJob, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("采集项格式不合法:%w", err)
|
||||
}
|
||||
var jobs []models.ScrapeJob
|
||||
if err := json.Unmarshal(raw, &jobs); err != nil {
|
||||
return nil, fmt.Errorf("采集项格式不合法:%w", err)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// renderConfig 按类型渲染当前配置(不落库)。
|
||||
func (s *CollectorService) renderConfig(c *models.Collector) (string, error) {
|
||||
endpoint := ""
|
||||
if c.LokiDeployment != nil {
|
||||
endpoint = c.LokiDeployment.Endpoint
|
||||
@@ -73,7 +98,25 @@ func (s *CollectorService) generateConfigSnapshot(c *models.Collector) string {
|
||||
}
|
||||
switch c.Type {
|
||||
case "host-promtail":
|
||||
return PromtailConfig(endpoint, c.LogPath, "host", c.Labels)
|
||||
// 主机名与 syslog 格式:优先用上次安装时探测到的真实值,
|
||||
// 从未安装过时退回平台登记的主机名/IP(预览会提示这一点)。
|
||||
hostname := c.ResolvedHostname
|
||||
if hostname == "" && c.Host != nil {
|
||||
hostname = c.Host.Name
|
||||
}
|
||||
if hostname == "" && c.Host != nil {
|
||||
hostname = c.Host.HostIP
|
||||
}
|
||||
return PromtailConfigForHost(HostPromtailOptions{
|
||||
Endpoint: endpoint,
|
||||
Cluster: "host",
|
||||
LabelsJSON: c.Labels,
|
||||
Hostname: hostname,
|
||||
SyslogFormat: c.SyslogFormat,
|
||||
SystemLogPaths: c.SystemLogPaths,
|
||||
LogPath: c.LogPath,
|
||||
Jobs: c.ScrapeJobs,
|
||||
})
|
||||
case "k8s-daemonset":
|
||||
ns := c.Namespace
|
||||
if ns == "" && c.LokiDeployment != nil {
|
||||
@@ -82,7 +125,7 @@ func (s *CollectorService) generateConfigSnapshot(c *models.Collector) string {
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
return PromtailDaemonsetYAML(ns, endpoint, clusterName, c.Labels, c.TargetNamespaces, c.AppLabelSource, c.AppInclude)
|
||||
return PromtailDaemonsetYAML(ns, endpoint, clusterName, c.Labels, c.TargetNamespaces, c.AppLabelSource, c.AppInclude), nil
|
||||
case "k8s-fluentbit-sidecar":
|
||||
host, port := parseLokiURL(endpoint)
|
||||
app := c.TargetDeployment
|
||||
@@ -90,9 +133,59 @@ func (s *CollectorService) generateConfigSnapshot(c *models.Collector) string {
|
||||
if ns == "" {
|
||||
ns = "default"
|
||||
}
|
||||
return FluentbitConfig(host, port, c.LogPath, app, ns)
|
||||
return FluentbitConfig(host, port, c.LogPath, app, ns), nil
|
||||
}
|
||||
return ""
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// generateConfigSnapshot 按类型生成配置快照(渲染失败返回空串,不阻断保存)
|
||||
func (s *CollectorService) generateConfigSnapshot(c *models.Collector) string {
|
||||
cfg, err := s.renderConfig(c)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// PreviewConfig 按「当前已保存配置 + 本次提交的改动」渲染 YAML 预览。
|
||||
// 只渲染不落库:用户在编辑器里点「预览」时用,避免为了看一眼 YAML 就改线上采集。
|
||||
func (s *CollectorService) PreviewConfig(id uint, updates map[string]interface{}) (map[string]interface{}, error) {
|
||||
c, err := s.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v, ok := updates["logPath"].(string); ok {
|
||||
c.LogPath = v
|
||||
}
|
||||
if v, ok := updates["labels"].(string); ok {
|
||||
c.Labels = v
|
||||
}
|
||||
if v, ok := updates["scrapeJobs"]; ok {
|
||||
jobs, perr := parseScrapeJobs(v)
|
||||
if perr != nil {
|
||||
return nil, perr
|
||||
}
|
||||
c.ScrapeJobs = jobs
|
||||
}
|
||||
if c.Type == "host-promtail" {
|
||||
if err := models.ValidateScrapeJobs(c.ScrapeJobs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
cfg, err := s.renderConfig(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warnings := make([]string, 0, 2)
|
||||
if c.Type == "host-promtail" {
|
||||
if c.SyslogFormat == "" {
|
||||
warnings = append(warnings, "尚未成功安装过:系统日志时间戳格式会在安装时按目标机实际格式探测(ISO8601 / 传统 syslog)")
|
||||
}
|
||||
if c.ResolvedHostname == "" {
|
||||
warnings = append(warnings, "尚未成功安装过:host 标签将使用平台登记的主机名,实际安装时会替换为目标机真实 hostname")
|
||||
}
|
||||
}
|
||||
return map[string]interface{}{"config": cfg, "warnings": warnings, "hostname": c.ResolvedHostname, "syslogFormat": c.SyslogFormat}, nil
|
||||
}
|
||||
|
||||
func (s *CollectorService) Create(req *CreateCollectorRequest) (*models.Collector, error) {
|
||||
@@ -131,6 +224,7 @@ func (s *CollectorService) Create(req *CreateCollectorRequest) (*models.Collecto
|
||||
Namespace: req.Namespace,
|
||||
LogPath: req.LogPath,
|
||||
Labels: req.Labels,
|
||||
ScrapeJobs: req.ScrapeJobs,
|
||||
Status: "pending",
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
@@ -292,6 +386,35 @@ func (s *CollectorService) Update(id uint, updates map[string]interface{}) error
|
||||
}
|
||||
db.Collectors[i].AppInclude = apps
|
||||
}
|
||||
// scrapeJobs:主机模式的结构化采集项(B 档)
|
||||
if v, ok := updates["scrapeJobs"]; ok {
|
||||
jobs, jerr := parseScrapeJobs(v)
|
||||
if jerr != nil {
|
||||
return jerr
|
||||
}
|
||||
if verr := models.ValidateScrapeJobs(jobs); verr != nil {
|
||||
return verr
|
||||
}
|
||||
if len(jobs) != len(db.Collectors[i].ScrapeJobs) {
|
||||
db.Collectors[i].StatusMessage = "采集项已保存,点击部署以应用新配置"
|
||||
}
|
||||
db.Collectors[i].ScrapeJobs = jobs
|
||||
}
|
||||
// 以下三个字段由安装流程写回(探测结果 / 渲染快照),不在前端表单里
|
||||
if v, ok := updates["resolvedHostname"].(string); ok && v != "" {
|
||||
db.Collectors[i].ResolvedHostname = v
|
||||
}
|
||||
if v, ok := updates["syslogFormat"].(string); ok && v != "" {
|
||||
db.Collectors[i].SyslogFormat = v
|
||||
}
|
||||
// 系统日志路径也是探测结果(Ubuntu 与 RHEL 系不同),
|
||||
// 不落库的话「配置预览」会和线上实际采集路径不一致。
|
||||
if v, ok := updates["systemLogPaths"].([]string); ok && len(v) > 0 {
|
||||
db.Collectors[i].SystemLogPaths = v
|
||||
}
|
||||
if v, ok := updates["config"].(string); ok && v != "" {
|
||||
db.Collectors[i].Config = v
|
||||
}
|
||||
db.Collectors[i].UpdatedAt = time.Now()
|
||||
db.RequestSave()
|
||||
return nil
|
||||
@@ -340,7 +463,22 @@ func (s *CollectorService) Install(id uint) {
|
||||
s.Update(id, map[string]interface{}{"status": "error", "statusMessage": err.Error()})
|
||||
return
|
||||
}
|
||||
s.Update(id, map[string]interface{}{"status": "active", "statusMessage": "采集器已部署"})
|
||||
// 安装时探测到的真实 hostname / syslog 格式 + 实际生效的配置都要落库,
|
||||
// 否则「配置预览」与线上真实配置不一致(用户没法核对)。
|
||||
updates := map[string]interface{}{"status": "active", "statusMessage": "采集器已部署"}
|
||||
if collector.Config != "" {
|
||||
updates["config"] = collector.Config
|
||||
}
|
||||
if collector.ResolvedHostname != "" {
|
||||
updates["resolvedHostname"] = collector.ResolvedHostname
|
||||
}
|
||||
if collector.SyslogFormat != "" {
|
||||
updates["syslogFormat"] = collector.SyslogFormat
|
||||
}
|
||||
if len(collector.SystemLogPaths) > 0 {
|
||||
updates["systemLogPaths"] = collector.SystemLogPaths
|
||||
}
|
||||
s.Update(id, updates)
|
||||
}
|
||||
|
||||
// Uninstall 异步卸载采集器
|
||||
|
||||
@@ -120,11 +120,52 @@ func (d *CollectorDeployer) installHost(ctx context.Context, endpoint string) er
|
||||
if logPath == "" {
|
||||
logPath = "/var/log/*.log"
|
||||
}
|
||||
conf := PromtailConfig(endpoint, logPath, "host", d.collector.Labels)
|
||||
d.log("info", "上传 Promtail 配置", "")
|
||||
if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/promtail/promtail.yaml << 'EOF'\n%s\nEOF", conf)); err != nil {
|
||||
// 目标机真实身份与 syslog 格式只能在安装时探测(见 Bug3 / Bug4):
|
||||
// - host 标签写真实主机名,promtail 不做 ${HOSTNAME} 展开;
|
||||
// - 系统日志按实际格式(ISO8601 / 传统 syslog)选正则。
|
||||
hostname := detectHostname(ctx, client, host)
|
||||
syslogFormat := detectSyslogFormat(ctx, client)
|
||||
d.log("info", fmt.Sprintf("目标机 %s,syslog 格式 %s", hostname, syslogFormat), "")
|
||||
|
||||
// 系统日志路径同样必须探测:Ubuntu 只有 syslog/auth.log,没有 messages
|
||||
sysLogPaths := detectSystemLogPaths(ctx, client)
|
||||
d.log("info", fmt.Sprintf("系统日志路径:%s", strings.Join(sysLogPaths, ",")), "")
|
||||
|
||||
conf, err := PromtailConfigForHost(HostPromtailOptions{
|
||||
Endpoint: endpoint,
|
||||
Cluster: "host",
|
||||
LabelsJSON: d.collector.Labels,
|
||||
Hostname: hostname,
|
||||
SyslogFormat: syslogFormat,
|
||||
SystemLogPaths: sysLogPaths,
|
||||
LogPath: logPath,
|
||||
Jobs: d.collector.ScrapeJobs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("生成 Promtail 配置失败(现有配置未改动): %w", err)
|
||||
}
|
||||
d.collector.ResolvedHostname = hostname
|
||||
d.collector.SyslogFormat = syslogFormat
|
||||
d.collector.SystemLogPaths = sysLogPaths
|
||||
|
||||
// 先写临时文件并做语法校验,通过后才替换正式配置 ——
|
||||
// 配置坏了会把正在跑的采集干掉,不能直接覆盖。
|
||||
d.log("info", "上传 Promtail 配置并校验", "")
|
||||
if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/promtail/promtail.yaml.new << 'PROMTAIL_EOF'\n%s\nPROMTAIL_EOF", conf)); err != nil {
|
||||
return fmt.Errorf("上传配置失败: %w", err)
|
||||
}
|
||||
if out, _, err := client.Execute(ctx, "/usr/local/bin/promtail -check-syntax -config.file=/etc/promtail/promtail.yaml.new 2>&1"); err != nil {
|
||||
// 老版本 promtail 可能没有 -check-syntax:只有明确报配置错误才阻断
|
||||
msg := strings.TrimSpace(out)
|
||||
if strings.Contains(msg, "flag provided but not defined") {
|
||||
d.log("info", "当前 promtail 不支持 -check-syntax,跳过语法校验", "")
|
||||
} else {
|
||||
return fmt.Errorf("Promtail 配置校验未通过(现有配置未改动):%s", truncateForMessage(msg, 600))
|
||||
}
|
||||
}
|
||||
if _, _, err := client.Execute(ctx, "mv -f /etc/promtail/promtail.yaml.new /etc/promtail/promtail.yaml"); err != nil {
|
||||
return fmt.Errorf("替换 Promtail 配置失败: %w", err)
|
||||
}
|
||||
|
||||
// systemd 服务(生产级)
|
||||
service := `[Unit]
|
||||
@@ -201,6 +242,45 @@ func truncateForMessage(s string, max int) string {
|
||||
return s[:max] + "…"
|
||||
}
|
||||
|
||||
// detectHostname 取目标机真实主机名(Bug3 修复点)。
|
||||
// promtail 不做环境变量展开,把 ${HOSTNAME} 写进配置只会在 Loki 里留下一堆
|
||||
// 字面量标签,无法按主机过滤。优先 `hostname -s`,失败退回平台里登记的主机名/IP。
|
||||
func detectHostname(ctx context.Context, client *SSHClient, host *models.Host) string {
|
||||
if out, _, err := client.Execute(ctx, "hostname -s 2>/dev/null || hostname 2>/dev/null"); err == nil {
|
||||
if v := strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]); v != "" && !strings.Contains(v, "${") {
|
||||
return v
|
||||
}
|
||||
}
|
||||
if host != nil {
|
||||
if v := strings.TrimSpace(host.Name); v != "" {
|
||||
return v
|
||||
}
|
||||
if v := strings.TrimSpace(host.HostIP); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return "unknown-host"
|
||||
}
|
||||
|
||||
// detectSyslogFormat 探测目标机 /var/log 的时间戳格式(Bug4 修复点)。
|
||||
//
|
||||
// TencentOS / RHEL 系(rsyslog 新版): 2026-07-17T15:38:24.923784+08:00 → iso8601
|
||||
// Debian / 老 rsyslog: Oct 28 18:11:06 → syslog
|
||||
//
|
||||
// 探测不到(文件不存在/为空)时按传统 syslog 处理(保守默认)。
|
||||
func detectSyslogFormat(ctx context.Context, client *SSHClient) string {
|
||||
const probe = `f=""; for p in /var/log/messages /var/log/syslog /var/log/secure; do [ -s "$p" ] && f="$p" && break; done; [ -n "$f" ] && head -n 1 "$f"`
|
||||
out, _, err := client.Execute(ctx, probe)
|
||||
if err != nil || strings.TrimSpace(out) == "" {
|
||||
return SyslogFormatClassic
|
||||
}
|
||||
line := strings.TrimSpace(strings.SplitN(out, "\n", 2)[0])
|
||||
if len(line) >= 19 && line[4] == '-' && line[7] == '-' && line[10] == 'T' {
|
||||
return SyslogFormatISO8601
|
||||
}
|
||||
return SyslogFormatClassic
|
||||
}
|
||||
|
||||
func (d *CollectorDeployer) uninstallHost(ctx context.Context) error {
|
||||
host := d.collector.Host
|
||||
if host == nil {
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// - DaemonSet 模板(6 空格)→ 子项 8 空格
|
||||
// 早先写死 8 空格,主机模式一旦配了标签就会生成非法 YAML,Promtail 直接装不上。
|
||||
func promtailExternalLabels(cluster string, labelsJSON string, indent string) string {
|
||||
lines := []string{fmt.Sprintf("%scluster: %q", indent, cluster)}
|
||||
lines := []string{fmt.Sprintf("%scluster: %s", indent, yamlScalar(cluster))}
|
||||
if labelsJSON != "" {
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(labelsJSON), &m); err == nil {
|
||||
@@ -27,7 +27,7 @@ func promtailExternalLabels(cluster string, labelsJSON string, indent string) st
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
lines = append(lines, fmt.Sprintf("%s%s: %q", indent, k, m[k]))
|
||||
lines = append(lines, fmt.Sprintf("%s%s: %s", indent, k, yamlScalar(m[k])))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,97 +43,6 @@ func isCRILogPath(logPath string) bool {
|
||||
strings.Contains(p, "/docker/")
|
||||
}
|
||||
|
||||
// appPipelineStages 按日志来源生成 pipeline:
|
||||
// - 容器日志(CRI 格式)→ cri 解析 + 提取时间戳
|
||||
// - 其它文本日志(应用文件、系统日志)→ 不套 cri,只做限流,避免解析失败丢行
|
||||
//
|
||||
// 原实现不区分来源、一律塞 cri 阶段,导致主机模式下采集 /var/log/messages
|
||||
// 或普通应用文件时时间戳解析失败、日志被丢弃。
|
||||
func appPipelineStages(logPath string) string {
|
||||
if isCRILogPath(logPath) {
|
||||
return ` - cri: {}
|
||||
- match:
|
||||
selector: '{job="app"}'
|
||||
stages:
|
||||
- timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano
|
||||
- limit:
|
||||
rate: 1000
|
||||
burst: 10000
|
||||
drop: true`
|
||||
}
|
||||
return ` - limit:
|
||||
rate: 2000
|
||||
burst: 4000
|
||||
drop: true`
|
||||
}
|
||||
|
||||
func PromtailConfig(lokiEndpoint, logPath, cluster, labelsJSON string) string {
|
||||
conf := fmt.Sprintf(`server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
log_level: info
|
||||
log_format: logfmt
|
||||
|
||||
positions:
|
||||
filename: /var/lib/promtail/positions.yaml
|
||||
sync_period: 10s
|
||||
|
||||
clients:
|
||||
- url: %s/loki/api/v1/push
|
||||
batchwait: 1s
|
||||
batchsize: 1048576
|
||||
timeout: 10s
|
||||
backoff_config:
|
||||
min_period: 500ms
|
||||
max_period: 5m
|
||||
max_retries: 10
|
||||
external_labels:
|
||||
job: host-logs
|
||||
__EXT_LABELS__
|
||||
|
||||
limits_config:
|
||||
readline_rate_enabled: false
|
||||
readline_rate: 1000
|
||||
readline_burst: 10000
|
||||
max_streams: 0
|
||||
|
||||
scrape_configs:
|
||||
- job_name: app
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: app
|
||||
host: ${HOSTNAME}
|
||||
__path__: %s
|
||||
pipeline_stages:
|
||||
%s
|
||||
# 系统日志(/var/log/messages、secure、cron 等)是 syslog 文本格式,
|
||||
# 绝不能用 cri 阶段解析 —— cri 只认 K8s/容器运行时那种 "时间 流 内容" 的行,
|
||||
# 套在 syslog 上会取不到时间戳、甚至整行被丢。这里单独一个 job 走正则解析。
|
||||
- job_name: system
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: system
|
||||
host: ${HOSTNAME}
|
||||
__path__: /var/log/{messages,secure,cron,maillog,syslog,dmesg}
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '^(?P<ts>[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<hostname>\S+)\s+(?P<unit>[^:\[]+)(?:\[(?P<pid>\d+)\])?:\s*(?P<msg>.*)$'
|
||||
- timestamp:
|
||||
source: ts
|
||||
format: Jan _2 15:04:05
|
||||
location: Asia/Shanghai
|
||||
- limit:
|
||||
rate: 2000
|
||||
burst: 4000
|
||||
drop: true
|
||||
`, lokiEndpoint, logPath, appPipelineStages(logPath))
|
||||
return strings.Replace(conf, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON, " "), 1)
|
||||
}
|
||||
|
||||
// PromtailDaemonsetYAML DaemonSet 模式纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),
|
||||
// 经 Kubernetes API 直接提交,后端无需任何额外二进制。
|
||||
// relabel 体系与主流 Promtail 部署惯例对齐:namespace/pod/container/node_name + __path__ 按 pod uid 精确匹配。
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// systemLogCandidates 是主流 Linux 发行版上系统日志的候选路径。
|
||||
//
|
||||
// 为什么要探测而不是写死:
|
||||
//
|
||||
// RHEL / CentOS / TencentOS / AWS Linux / Rocky : messages, secure, cron, dmesg
|
||||
// Ubuntu / Debian : syslog, auth.log, kern.log(**没有 messages**)
|
||||
// Ubuntu 最小安装(无 rsyslog) : 以上全没有,只能读 systemd journal
|
||||
//
|
||||
// 早期实现固定采 messages/secure/cron,在 Ubuntu 上采集器"装成功了但一条系统日志都没有"。
|
||||
var systemLogCandidates = []string{
|
||||
"/var/log/messages",
|
||||
"/var/log/syslog",
|
||||
"/var/log/secure",
|
||||
"/var/log/auth.log",
|
||||
"/var/log/cron",
|
||||
"/var/log/kern.log",
|
||||
"/var/log/dmesg",
|
||||
}
|
||||
|
||||
// detectSystemLogPaths 在目标机上挑出真实存在且非空的系统日志文件。
|
||||
// 探测失败时返回 RHEL 系默认值(保守兜底)。
|
||||
func detectSystemLogPaths(ctx context.Context, client *SSHClient) []string {
|
||||
var quoted []string
|
||||
for _, p := range systemLogCandidates {
|
||||
quoted = append(quoted, "'"+p+"'")
|
||||
}
|
||||
// 只输出存在且大小 > 0 的文件,一行一个
|
||||
script := "for p in " + strings.Join(quoted, " ") + "; do [ -s \"$p\" ] && echo \"$p\"; done"
|
||||
out, _, err := client.Execute(ctx, script)
|
||||
if err != nil {
|
||||
return defaultSystemLogPaths
|
||||
}
|
||||
|
||||
var paths []string
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "/var/log/") {
|
||||
paths = append(paths, line)
|
||||
}
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return nil // 交给上层走 journald 兜底
|
||||
}
|
||||
return paths
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// 系统日志时间戳格式(安装期探测结果)。
|
||||
// 不同发行版/rsyslog 版本写法不同,正则必须跟着变,否则 timestamp 阶段不生效。
|
||||
const (
|
||||
// SyslogFormatISO8601 2026-07-17T15:38:24.923784+08:00(TencentOS 4 / 新版 rsyslog)
|
||||
SyslogFormatISO8601 = "iso8601"
|
||||
// SyslogFormatClassic Oct 28 18:11:06(Debian / 老 rsyslog)
|
||||
SyslogFormatClassic = "rfc3164"
|
||||
)
|
||||
|
||||
// HostPromtailOptions 生成主机模式 promtail.yaml 的入参。
|
||||
type HostPromtailOptions struct {
|
||||
Endpoint string
|
||||
Cluster string
|
||||
LabelsJSON string
|
||||
// Hostname 真机主机名(Promtail 不展开 ${HOSTNAME},必须写具体值)
|
||||
Hostname string
|
||||
// SyslogFormat 安装时探测:iso8601 / rfc3164
|
||||
SyslogFormat string
|
||||
// SystemLogPaths 安装时探测到的系统日志绝对路径。
|
||||
// Ubuntu/Debian 是 syslog+auth.log,RHEL 系是 messages+secure;
|
||||
// 留空时退回 RHEL 系默认值(兼容旧调用)。
|
||||
SystemLogPaths []string
|
||||
LogPath string
|
||||
Jobs []models.ScrapeJob
|
||||
}
|
||||
|
||||
// defaultSystemLogPaths 是探测失败时的兜底(RHEL/CentOS/TencentOS/AWS Linux)。
|
||||
var defaultSystemLogPaths = []string{
|
||||
"/var/log/messages", "/var/log/secure", "/var/log/cron",
|
||||
}
|
||||
|
||||
// PromtailConfigForHost 渲染主机模式配置。
|
||||
//
|
||||
// 跨云/跨发行版的三个关键点:
|
||||
// 1. 系统日志路径来自安装期探测(Ubuntu 没有 /var/log/messages,写死就采不到)。
|
||||
// 2. 时间戳格式按探测结果选:ISO8601 → RFC3339Nano;传统 syslog → Jan _2 15:04:05,
|
||||
// location 用 Local(跟随进程所在主机时区,不写死某个地区)。
|
||||
// 3. host 标签写真实主机名。
|
||||
func PromtailConfigForHost(o HostPromtailOptions) (string, error) {
|
||||
hostname := strings.TrimSpace(o.Hostname)
|
||||
if hostname == "" {
|
||||
hostname = "unknown-host"
|
||||
}
|
||||
cluster := o.Cluster
|
||||
if cluster == "" {
|
||||
cluster = "host"
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
log_level: info
|
||||
log_format: logfmt
|
||||
|
||||
positions:
|
||||
filename: /var/lib/promtail/positions.yaml
|
||||
sync_period: 10s
|
||||
|
||||
clients:
|
||||
- url: %s/loki/api/v1/push
|
||||
batchwait: 1s
|
||||
batchsize: 1048576
|
||||
timeout: 10s
|
||||
backoff_config:
|
||||
min_period: 500ms
|
||||
max_period: 5m
|
||||
max_retries: 10
|
||||
external_labels:
|
||||
job: host-logs
|
||||
%s
|
||||
|
||||
limits_config:
|
||||
readline_rate_enabled: false
|
||||
readline_rate: 1000
|
||||
readline_burst: 10000
|
||||
max_streams: 0
|
||||
|
||||
scrape_configs:
|
||||
`, strings.TrimSuffix(o.Endpoint, "/"), promtailExternalLabels(cluster, o.LabelsJSON, " "))
|
||||
|
||||
// 自定义采集项优先;一个都没有时用「应用日志 + 系统日志」的默认组合
|
||||
if len(o.Jobs) > 0 {
|
||||
jobs := make([]models.ScrapeJob, len(o.Jobs))
|
||||
copy(jobs, o.Jobs)
|
||||
for i := range jobs {
|
||||
if jobs[i].System && len(jobs[i].Paths) == 0 {
|
||||
jobs[i].Paths = pickSystemPaths(o.SystemLogPaths)
|
||||
}
|
||||
}
|
||||
// 用户填的正则/路径/标签必须校验:非法名字会拼出坏 YAML,
|
||||
// 敏感路径(/etc/shadow、私钥)会把主机凭据推到日志平台里。
|
||||
if err := models.ValidateScrapeJobs(jobs); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if job.System && len(job.Paths) == 0 {
|
||||
job.Paths = pickSystemPaths(o.SystemLogPaths)
|
||||
}
|
||||
rendered, err := renderScrapeJob(job, hostname, o.SyslogFormat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString(rendered)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// 默认 1:应用日志
|
||||
if path := strings.TrimSpace(o.LogPath); path != "" {
|
||||
rendered, err := renderScrapeJob(models.ScrapeJob{
|
||||
Name: "app",
|
||||
Paths: splitPaths(path),
|
||||
}, hostname, o.SyslogFormat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString(rendered)
|
||||
}
|
||||
|
||||
// 默认 2:系统日志
|
||||
rendered, err := renderScrapeJob(models.ScrapeJob{
|
||||
Name: "system",
|
||||
System: true,
|
||||
Paths: pickSystemPaths(o.SystemLogPaths),
|
||||
}, hostname, o.SyslogFormat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString(rendered)
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// yamlScalar 输出 YAML 标量:简单值不加引号,含特殊字符时用双引号包裹并转义。
|
||||
// external_labels / labels 里的值(env=pre、app=cex-api)加引号会变成
|
||||
// `env: "pre"`,与用户预期和既有配置风格不一致。
|
||||
func yamlScalar(v string) string {
|
||||
safe := true
|
||||
for _, r := range v {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
case r == '_' || r == '-' || r == '.' || r == '/' || r == ':' || r == '@':
|
||||
default:
|
||||
safe = false
|
||||
}
|
||||
}
|
||||
if safe && v != "" {
|
||||
return v
|
||||
}
|
||||
return strconv.Quote(v)
|
||||
}
|
||||
|
||||
// pickSystemPaths 选系统日志路径:优先用安装期探测结果,否则退回默认。
|
||||
func pickSystemPaths(detected []string) []string {
|
||||
if len(detected) > 0 {
|
||||
return detected
|
||||
}
|
||||
return defaultSystemLogPaths
|
||||
}
|
||||
|
||||
// splitPaths 兼容调用方用逗号或空白分隔多个路径。
|
||||
func splitPaths(raw string) []string {
|
||||
fields := strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == '\n' })
|
||||
out := make([]string, 0, len(fields))
|
||||
for _, f := range fields {
|
||||
if p := strings.TrimSpace(f); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// renderScrapeJob 把一条 ScrapeJob 渲染成 promtail 的 scrape_config。
|
||||
func renderScrapeJob(job models.ScrapeJob, hostname, syslogFormat string) (string, error) {
|
||||
if len(job.Paths) == 0 {
|
||||
return "", fmt.Errorf("采集项 %q 没有配置日志路径", job.Name)
|
||||
}
|
||||
name := job.Name
|
||||
if name == "" {
|
||||
name = "custom"
|
||||
}
|
||||
isCRI := pathsContainCRI(job.Paths)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, " - job_name: %s\n", name)
|
||||
// encoding 是 scrape_config 级字段(不是 static_config),写错位置 promtail 直接起不来
|
||||
if job.Encoding != "" {
|
||||
fmt.Fprintf(&b, " encoding: %s\n", job.Encoding)
|
||||
}
|
||||
// 一个 job 下多条路径要写成 static_configs 的**列表**,
|
||||
// 不能重复出现 static_configs 键(YAML 会报 duplicate key)。
|
||||
b.WriteString(" static_configs:\n")
|
||||
for _, p := range job.Paths {
|
||||
fmt.Fprintf(&b, " - targets: [localhost]\n labels:\n job: %s\n host: %s\n __path__: %s\n",
|
||||
yamlScalar(name), yamlScalar(hostname), yamlScalar(p))
|
||||
for k, v := range job.Labels {
|
||||
fmt.Fprintf(&b, " %s: %s\n", sanitizePromLabel(k), yamlScalar(v))
|
||||
}
|
||||
}
|
||||
|
||||
stages := make([]string, 0, 8)
|
||||
// cri 必须排在最前:容器日志行是 "时间 stdout F 内容",
|
||||
// 不先剥离前缀,后面的 multiline/json/regex 全都匹配不上。
|
||||
if isCRI {
|
||||
stages = append(stages, " - cri: {}")
|
||||
}
|
||||
if job.Multiline != nil && strings.TrimSpace(job.Multiline.FirstLine) != "" {
|
||||
maxLines := job.Multiline.MaxLines
|
||||
if maxLines <= 0 {
|
||||
maxLines = 128
|
||||
}
|
||||
stages = append(stages, fmt.Sprintf(` - multiline:
|
||||
firstline: '%s'
|
||||
max_wait_time: 3s
|
||||
max_lines: %d`, strings.ReplaceAll(job.Multiline.FirstLine, "'", "''"), maxLines))
|
||||
}
|
||||
if job.JSON {
|
||||
stages = append(stages, " - json: {}")
|
||||
}
|
||||
if job.System {
|
||||
switch syslogFormat {
|
||||
case SyslogFormatISO8601:
|
||||
stages = append(stages,
|
||||
` - regex:
|
||||
expression: '^(?P<ts>\d{4}-\d{2}-\d{2}T\S+)'`,
|
||||
` - timestamp:
|
||||
source: ts
|
||||
format: RFC3339Nano`)
|
||||
case SyslogFormatClassic:
|
||||
stages = append(stages,
|
||||
` - regex:
|
||||
expression: '^(?P<ts>[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<hostname>\S+)\s+(?P<unit>[^:\[]+)(?:\[(?P<pid>\d+)\])?:\s*(?P<msg>.*)$'`,
|
||||
` - timestamp:
|
||||
source: ts
|
||||
format: Jan _2 15:04:05
|
||||
location: Local`)
|
||||
}
|
||||
} else if job.Timestamp != nil && strings.TrimSpace(job.Timestamp.Source) != "" {
|
||||
loc := ""
|
||||
if job.Timestamp.Location != "" {
|
||||
loc = fmt.Sprintf("\n location: %s", job.Timestamp.Location)
|
||||
}
|
||||
stages = append(stages, fmt.Sprintf(` - timestamp:
|
||||
source: %s
|
||||
format: '%s'%s`, job.Timestamp.Source, strings.ReplaceAll(job.Timestamp.Format, "'", "''"), loc))
|
||||
} else if isCRI {
|
||||
// CRI 行自带时间字段,不解析它就会用采集时间(回填历史日志时间全错)
|
||||
stages = append(stages, ` - timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano`)
|
||||
}
|
||||
for _, expr := range job.DropIfMatch {
|
||||
if strings.TrimSpace(expr) == "" {
|
||||
continue
|
||||
}
|
||||
stages = append(stages, fmt.Sprintf(` - drop:
|
||||
expression: '%s'
|
||||
drop_counter_reason: drop_if_match`, strings.ReplaceAll(expr, "'", "''")))
|
||||
}
|
||||
|
||||
rate, burst := 2000, 4000
|
||||
if job.RateLimit != nil {
|
||||
if job.RateLimit.Rate > 0 {
|
||||
rate = job.RateLimit.Rate
|
||||
}
|
||||
if job.RateLimit.Burst > 0 {
|
||||
burst = job.RateLimit.Burst
|
||||
}
|
||||
}
|
||||
stages = append(stages, fmt.Sprintf(` - limit:
|
||||
rate: %d
|
||||
burst: %d
|
||||
drop: true`, rate, burst))
|
||||
|
||||
fmt.Fprintf(&b, " pipeline_stages:\n%s\n", strings.Join(stages, "\n"))
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// pathsContainCRI 判断是否有容器运行时日志路径(需要 cri 阶段解析)。
|
||||
func pathsContainCRI(paths []string) bool {
|
||||
for _, p := range paths {
|
||||
if isCRILogPath(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -5,53 +5,183 @@ import (
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// 主机模式(PromtailConfig)生成的 YAML 必须能被解析,
|
||||
// 且系统日志任务不能用 cri 阶段(否则 syslog 格式解析失败丢行)。
|
||||
func TestHostPromtailConfigValid(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki.log.svc:3100", "/var/log/myapp/*.log", "pre", `{"env":"pre"}`)
|
||||
|
||||
func renderHost(t *testing.T, o HostPromtailOptions) string {
|
||||
t.Helper()
|
||||
cfg, err := PromtailConfigForHost(o)
|
||||
if err != nil {
|
||||
t.Fatalf("渲染主机配置失败: %v", err)
|
||||
}
|
||||
var out map[string]interface{}
|
||||
if err := yaml.Unmarshal([]byte(cfg), &out); err != nil {
|
||||
t.Fatalf("生成的 promtail 配置不是合法 YAML: %v\n%s", err, cfg)
|
||||
}
|
||||
if _, ok := out["scrape_configs"]; !ok {
|
||||
t.Fatal("缺少 scrape_configs")
|
||||
t.Fatalf("缺少 scrape_configs\n%s", cfg)
|
||||
}
|
||||
// 必须包含系统日志任务
|
||||
if !strings.Contains(cfg, "job_name: system") {
|
||||
t.Fatal("缺少 system 日志采集任务")
|
||||
return cfg
|
||||
}
|
||||
|
||||
// 默认(未配置采集项)仍要生成 app + system 两个 job,且系统日志不能套 cri
|
||||
func TestHostPromtailConfigDefaultJobs(t *testing.T) {
|
||||
cfg := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki.log.svc:3100",
|
||||
Cluster: "pre",
|
||||
Hostname: "pre-cex-ops-jenkins",
|
||||
LogPath: "/var/log/myapp/*.log",
|
||||
LabelsJSON: `{"env":"pre"}`,
|
||||
})
|
||||
if !strings.Contains(cfg, "job_name: app") || !strings.Contains(cfg, "job_name: system") {
|
||||
t.Fatalf("默认应有 app + system 两个 job:\n%s", cfg)
|
||||
}
|
||||
if !strings.Contains(cfg, "/var/log/{messages,secure,cron,maillog,syslog,dmesg}") {
|
||||
t.Fatal("system 任务没有采集系统日志路径")
|
||||
}
|
||||
// 系统任务段内不能出现 cri 阶段
|
||||
sysIdx := strings.Index(cfg, "job_name: system")
|
||||
if sysIdx < 0 {
|
||||
t.Fatal("找不到 system 任务")
|
||||
t.Fatal("找不到 system job")
|
||||
}
|
||||
if strings.Contains(cfg[sysIdx:], "- cri: {}") {
|
||||
t.Fatal("system 任务里错误地使用了 cri 阶段")
|
||||
if strings.Contains(cfg[sysIdx:], "cri: {}") {
|
||||
t.Fatal("system job 里错误地使用了 cri 阶段")
|
||||
}
|
||||
if !strings.Contains(cfg, "env: pre") {
|
||||
t.Fatal("自定义标签没写进 external_labels")
|
||||
}
|
||||
}
|
||||
|
||||
// 容器/Cri 路径仍应保留 cri 解析与时间戳提取
|
||||
func TestHostPromtailCRIPathKeepsCRIStage(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki:3100", "/var/log/pods/*/*/*.log", "pre", "")
|
||||
if !strings.Contains(cfg, "- cri: {}") {
|
||||
t.Fatal("CRI 路径应保留 cri 阶段")
|
||||
// Bug3 回归:host 标签必须是真实主机名,不能是 ${HOSTNAME} 字面量
|
||||
func TestHostPromtailHostnameIsConcrete(t *testing.T) {
|
||||
cfg := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100",
|
||||
Hostname: "pre-cex-ops-jenkins",
|
||||
LogPath: "/data/app/*.log",
|
||||
})
|
||||
if strings.Contains(cfg, "${HOSTNAME}") {
|
||||
t.Fatalf("host 标签仍是 ${HOSTNAME} 字面量(promtail 不做环境变量展开):\n%s", cfg)
|
||||
}
|
||||
var out map[string]interface{}
|
||||
if err := yaml.Unmarshal([]byte(cfg), &out); err != nil {
|
||||
t.Fatalf("YAML 非法: %v", err)
|
||||
// 断言"每个 static_config 都带真实主机名"而不是写死数量:
|
||||
// 默认系统日志有多个文件(messages/secure/cron…),每个文件一条 static_config,
|
||||
// 数量会随目标机实际存在的日志文件变化。
|
||||
if n := strings.Count(cfg, "host: pre-cex-ops-jenkins"); n < 2 {
|
||||
t.Fatalf("app / system 两个 job 都应带真实主机名(实际 %d 处):\n%s", n, cfg)
|
||||
}
|
||||
// 即使调用方漏传主机名,也不能退回占位符
|
||||
cfg2 := renderHost(t, HostPromtailOptions{Endpoint: "http://loki:3100", LogPath: "/data/app/*.log"})
|
||||
if strings.Contains(cfg2, "${") {
|
||||
t.Fatalf("空主机名时退化成了占位符:\n%s", cfg2)
|
||||
}
|
||||
}
|
||||
|
||||
// 普通应用文件不应套 cri,否则非 CRI 行会被丢弃
|
||||
func TestHostPromtailPlainFileHasNoCRIStage(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki:3100", "/data/app/logs/*.log", "pre", "")
|
||||
if strings.Contains(cfg, "- cri: {}") {
|
||||
t.Fatal("普通应用日志不应使用 cri 阶段")
|
||||
// Bug4 回归:syslog 格式决定正则与 timestamp 布局
|
||||
func TestHostPromtailSyslogFormatSwitch(t *testing.T) {
|
||||
iso := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100", Hostname: "h1", SyslogFormat: SyslogFormatISO8601,
|
||||
Jobs: []models.ScrapeJob{{Name: "system", System: true}},
|
||||
})
|
||||
if !strings.Contains(iso, `\d{4}-\d{2}-\d{2}T`) {
|
||||
t.Fatalf("iso8601 模式下应使用 ISO8601 正则:\n%s", iso)
|
||||
}
|
||||
if !strings.Contains(iso, "format: RFC3339Nano") {
|
||||
t.Fatalf("iso8601 模式下 timestamp 应用 RFC3339Nano:\n%s", iso)
|
||||
}
|
||||
if strings.Contains(iso, "Jan _2 15:04:05") {
|
||||
t.Fatalf("iso8601 模式下不该出现传统 syslog 布局:\n%s", iso)
|
||||
}
|
||||
|
||||
classic := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100", Hostname: "h1", SyslogFormat: SyslogFormatClassic,
|
||||
Jobs: []models.ScrapeJob{{Name: "system", System: true}},
|
||||
})
|
||||
if !strings.Contains(classic, "Jan _2 15:04:05") {
|
||||
t.Fatalf("传统 syslog 模式应使用经典布局:\n%s", classic)
|
||||
}
|
||||
if strings.Contains(classic, `\d{4}-\d{2}-\d{2}T`) {
|
||||
t.Fatalf("传统 syslog 模式不该出现 ISO8601 正则:\n%s", classic)
|
||||
}
|
||||
}
|
||||
|
||||
// 容器路径仍保留 cri 解析;普通文件不套 cri(否则非 CRI 行会被丢)
|
||||
func TestHostPromtailCRIStageOnlyForCRIPaths(t *testing.T) {
|
||||
cri := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100", Hostname: "h1",
|
||||
Jobs: []models.ScrapeJob{{Name: "pods", Paths: []string{"/var/log/pods/*/*/*.log"}}},
|
||||
})
|
||||
if !strings.Contains(cri, "cri: {}") {
|
||||
t.Fatalf("CRI 路径应保留 cri 阶段:\n%s", cri)
|
||||
}
|
||||
plain := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100", Hostname: "h1",
|
||||
Jobs: []models.ScrapeJob{{Name: "app", Paths: []string{"/data/app/logs/*.log"}}},
|
||||
})
|
||||
if strings.Contains(plain, "cri: {}") {
|
||||
t.Fatalf("普通应用日志不应使用 cri 阶段:\n%s", plain)
|
||||
}
|
||||
}
|
||||
|
||||
// B 档:多 job / 多路径 / 多行 / 丢弃 / 限流都要真的渲染出来
|
||||
func TestHostPromtailStructuredJobs(t *testing.T) {
|
||||
cfg := renderHost(t, HostPromtailOptions{
|
||||
Endpoint: "http://loki:3100", Hostname: "h1", SyslogFormat: SyslogFormatISO8601,
|
||||
Jobs: []models.ScrapeJob{
|
||||
{
|
||||
Name: "app-api",
|
||||
Paths: []string{"/data/app/logs/*.log", "/data/app/logs/error/*.log"},
|
||||
Labels: map[string]string{"app": "cex-api"},
|
||||
Multiline: &models.JobMultiline{
|
||||
FirstLine: `^\d{4}-\d{2}-\d{2}`,
|
||||
MaxLines: 500,
|
||||
},
|
||||
JSON: true,
|
||||
Timestamp: &models.JobTimestamp{Source: "time", Format: "RFC3339Nano"},
|
||||
DropIfMatch: []string{`"status":200`},
|
||||
RateLimit: &models.JobRateLimit{Rate: 500, Burst: 900},
|
||||
},
|
||||
{Name: "system", System: true},
|
||||
},
|
||||
})
|
||||
for _, want := range []string{
|
||||
"job_name: app-api",
|
||||
"/data/app/logs/error/*.log",
|
||||
"app: cex-api",
|
||||
"max_lines: 500",
|
||||
"json: {}",
|
||||
"drop_counter_reason: drop_if_match",
|
||||
"rate: 500",
|
||||
"burst: 900",
|
||||
"job_name: system",
|
||||
} {
|
||||
if !strings.Contains(cfg, want) {
|
||||
t.Fatalf("配置缺少 %q:\n%s", want, cfg)
|
||||
}
|
||||
}
|
||||
// 两个路径应渲染成两个 static_configs
|
||||
if strings.Count(cfg, "targets:") != 3 { // app-api 两个路径 + system 默认路径 6 条? 见下
|
||||
// system 默认路径有 6 条,这里只断言 app-api 的两条
|
||||
if strings.Count(cfg, "/data/app/logs/") < 2 {
|
||||
t.Fatalf("多路径未展开成多个 static_configs:\n%s", cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 校验必须拦死敏感路径和坏正则(这两种都会把主机凭据搬进日志平台 / 让 promtail 起不来)
|
||||
func TestValidateScrapeJobsRejectsUnsafe(t *testing.T) {
|
||||
bad := [][]models.ScrapeJob{
|
||||
{{Name: "x", Paths: []string{"/etc/shadow"}}},
|
||||
{{Name: "x", Paths: []string{"/root/.ssh/id_rsa"}}},
|
||||
{{Name: "x", Paths: []string{"relative/path.log"}}},
|
||||
{{Name: "x", Paths: []string{"/var/log/a.log"}, Multiline: &models.JobMultiline{FirstLine: "(["}}},
|
||||
{{Name: "x", Paths: []string{"/var/log/a.log"}, DropIfMatch: []string{"(["}}},
|
||||
{{Name: "x", Paths: []string{"/var/log/a.log"}, Labels: map[string]string{"job": "y"}}},
|
||||
{{Name: "", Paths: []string{"/var/log/a.log"}}},
|
||||
{{Name: "x", Paths: nil}},
|
||||
}
|
||||
for i, jobs := range bad {
|
||||
if err := models.ValidateScrapeJobs(jobs); err == nil {
|
||||
t.Fatalf("第 %d 组非法采集项应被拒绝: %+v", i+1, jobs)
|
||||
}
|
||||
}
|
||||
good := []models.ScrapeJob{{Name: "app", Paths: []string{"/var/log/a.log"}, Labels: map[string]string{"app_name": "x"}}}
|
||||
if err := models.ValidateScrapeJobs(good); err != nil {
|
||||
t.Fatalf("合法采集项被误拒: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,15 @@
|
||||
// 采集器域 API
|
||||
import { http, buildQuery, type ListParams, type RequestOptions } from '../http';
|
||||
import type { PageData, Collector, CreateCollectorRequest } from '@/types';
|
||||
import type { PageData, Collector, CreateCollectorRequest, ScrapeJob } from '@/types';
|
||||
|
||||
export interface CollectorPreview {
|
||||
/** 渲染出来的 promtail.yaml */
|
||||
config: string;
|
||||
/** 平台给出的提示(未安装过 → 主机名/syslog 格式待探测等) */
|
||||
warnings?: string[];
|
||||
hostname?: string;
|
||||
syslogFormat?: string;
|
||||
}
|
||||
|
||||
export const collectorApi = {
|
||||
list: (params: ListParams = {}, options?: RequestOptions) =>
|
||||
@@ -16,9 +25,18 @@ export const collectorApi = {
|
||||
targetNamespaces?: string[];
|
||||
appLabelSource?: string;
|
||||
appInclude?: string[];
|
||||
/** 主机模式的结构化采集项(B 档) */
|
||||
scrapeJobs?: ScrapeJob[];
|
||||
}) =>
|
||||
http.put<{ message: string }>(`/collectors/${id}`, data),
|
||||
|
||||
/** 只预览渲染结果,不落库(结构化编辑器的「预览 YAML」用) */
|
||||
preview: (id: number, data: {
|
||||
logPath?: string;
|
||||
labels?: string;
|
||||
scrapeJobs?: ScrapeJob[];
|
||||
}) => http.post<CollectorPreview>(`/collectors/${id}/preview`, data),
|
||||
|
||||
remove: (id: number) => http.delete<{ message: string }>(`/collectors/${id}`),
|
||||
|
||||
install: (id: number) => http.post<{ message: string }>(`/collectors/${id}/install`),
|
||||
|
||||
@@ -120,6 +120,41 @@ const collectors = {
|
||||
appIncludeHint:
|
||||
'Keeps only logs whose app matches these values (<code>action: keep</code>) — the most direct way to cap log volume.',
|
||||
logPathHint: 'Sidecar needs a shared log volume or the dedicated logs directory; click Deploy after saving.',
|
||||
// ── host mode: structured scrape jobs ──
|
||||
jobs: {
|
||||
title: 'Log scrape jobs',
|
||||
hint: 'One entry = one promtail job; add more paths for more directories. Click Deploy after saving.',
|
||||
add: 'Add job',
|
||||
remove: 'Remove job',
|
||||
namePlaceholder: 'job name, e.g. app-api',
|
||||
paths: 'Log paths (comma separated, glob supported)',
|
||||
systemPathsPlaceholder: 'Empty = use the system logs that actually exist on this host',
|
||||
systemBadge: 'system logs · auto timestamp',
|
||||
labels: 'Extra labels (key=value, comma separated)',
|
||||
advanced: 'Advanced: multiline / JSON / timestamp / drop / rate limit',
|
||||
system: 'System logs (auto ISO8601 / classic)',
|
||||
json: 'Parse JSON',
|
||||
multiline: 'Multiline (Java stack traces)',
|
||||
timestamp: 'Custom timestamp',
|
||||
maxLinesPlaceholder: 'max lines 128',
|
||||
tsSourcePlaceholder: 'field name, e.g. time',
|
||||
tsLocationPlaceholder: 'location, e.g. Asia/Shanghai',
|
||||
dropIfMatch: 'Drop when matching (regex, comma separated)',
|
||||
dropIfMatchPlaceholder: 'Empty = keep all; e.g. "status":200',
|
||||
ratePlaceholder: 'rate limit (default 2000)',
|
||||
burstPlaceholder: 'burst limit (default 4000)',
|
||||
hostInfo: 'Detected on last install: hostname {{host}}, syslog format {{format}}',
|
||||
},
|
||||
preview: {
|
||||
button: 'Preview YAML',
|
||||
loading: 'Rendering…',
|
||||
failed: 'Preview failed',
|
||||
title: 'promtail.yaml that will be written',
|
||||
description: 'Preview only — nothing is saved or deployed. Then click Save → Deploy.',
|
||||
copy: 'Copy',
|
||||
copied: 'Copied',
|
||||
close: 'Close',
|
||||
},
|
||||
save: 'Save',
|
||||
saving: 'Saving…',
|
||||
validation: {
|
||||
|
||||
@@ -124,6 +124,41 @@ const collectors = {
|
||||
appIncludeHint:
|
||||
'只保留 app 命中这些值的日志(<code>action: keep</code>),是控制日志量最直接的手段。',
|
||||
logPathHint: 'Sidecar 需共享日志卷或专用 logs 目录;保存后点「部署」生效。',
|
||||
// ── 主机模式:结构化采集项(B 档) ──
|
||||
jobs: {
|
||||
title: '日志采集项',
|
||||
hint: '一个采集项 = promtail 的一个 job;多个目录就多填几个路径。保存后点「部署」下发。',
|
||||
add: '添加采集项',
|
||||
remove: '删除采集项',
|
||||
namePlaceholder: 'job 名称,如 app-api',
|
||||
paths: '日志路径(逗号分隔,支持 glob)',
|
||||
systemPathsPlaceholder: '留空 = 自动使用该主机实际存在的系统日志',
|
||||
systemBadge: '系统日志 · 自动时间戳',
|
||||
labels: '附加标签(key=value,逗号分隔)',
|
||||
advanced: '高级:多行合并 / JSON / 时间戳 / 丢弃 / 限流',
|
||||
system: '系统日志(自动选 ISO8601/传统格式)',
|
||||
json: 'JSON 解析',
|
||||
multiline: '多行合并(Java 堆栈)',
|
||||
timestamp: '自定义时间戳',
|
||||
maxLinesPlaceholder: '最大行数 128',
|
||||
tsSourcePlaceholder: '字段名,如 time',
|
||||
tsLocationPlaceholder: '时区,如 Asia/Shanghai',
|
||||
dropIfMatch: '命中即丢弃(正则,逗号分隔)',
|
||||
dropIfMatchPlaceholder: '留空不丢;如 "status":200',
|
||||
ratePlaceholder: '限流 rate(默认 2000)',
|
||||
burstPlaceholder: '限流 burst(默认 4000)',
|
||||
hostInfo: '上次安装探测:主机名 {{host}},系统日志格式 {{format}}',
|
||||
},
|
||||
preview: {
|
||||
button: '预览 YAML',
|
||||
loading: '渲染中…',
|
||||
failed: '预览失败',
|
||||
title: '将要下发的 promtail.yaml',
|
||||
description: '只预览,不保存也不下发;确认无误再点「保存」→「部署」。',
|
||||
copy: '复制',
|
||||
copied: '已复制',
|
||||
close: '关闭',
|
||||
},
|
||||
save: '保存',
|
||||
saving: '保存中…',
|
||||
validation: {
|
||||
|
||||
@@ -8,12 +8,15 @@ import type { TFunction } from 'i18next'
|
||||
import '@/i18n/locales/zh/collectors'
|
||||
import '@/i18n/locales/en/collectors'
|
||||
import { toast } from 'sonner'
|
||||
import { clusterApi, collectorApi } from '@/api'
|
||||
import { Eye } from 'lucide-react'
|
||||
import { clusterApi, collectorApi, type CollectorPreview } from '@/api'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
|
||||
import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Checkbox } from '@/components/ui/checkbox'
|
||||
import { ScrapeJobsEditor } from './ScrapeJobsEditor'
|
||||
import { draftIssues, draftToJob, draftsFromCollector, type JobDraft } from './scrape-jobs'
|
||||
import type { Collector } from '@/types'
|
||||
|
||||
// 表单 schema:错误消息经 t 注入(随语言重建,RHF 每次渲染刷新 resolver)
|
||||
@@ -53,6 +56,10 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
const [nsLoading, setNsLoading] = useState(false)
|
||||
const [nsError, setNsError] = useState('')
|
||||
const [nsFilter, setNsFilter] = useState('')
|
||||
/** 主机模式的采集项草稿(结构化编辑器,B 档) */
|
||||
const [jobs, setJobs] = useState<JobDraft[]>([])
|
||||
const [preview, setPreview] = useState<CollectorPreview | null>(null)
|
||||
const [previewing, setPreviewing] = useState(false)
|
||||
|
||||
// schema 随语言重建(t 变化即语言切换),校验消息即时跟随当前语言
|
||||
const schema = useMemo(() => buildSchema(t), [t])
|
||||
@@ -62,7 +69,8 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
})
|
||||
|
||||
const isDaemonSet = collector?.type === 'k8s-daemonset'
|
||||
const supportsPath = !isDaemonSet
|
||||
const isHost = collector?.type === 'host-promtail'
|
||||
const supportsPath = !isDaemonSet && !isHost
|
||||
const selectedNs = form.watch('targetNamespaces')
|
||||
|
||||
useEffect(() => {
|
||||
@@ -75,6 +83,8 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
appIncludeRaw: (collector.appInclude ?? []).join(', '),
|
||||
})
|
||||
setNsFilter('')
|
||||
setPreview(null)
|
||||
setJobs(collector.type === 'host-promtail' ? draftsFromCollector(collector.scrapeJobs, collector.logPath) : [])
|
||||
}, [collector, form])
|
||||
|
||||
// 拉集群命名空间列表(供多选)
|
||||
@@ -108,8 +118,40 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
form.setValue('targetNamespaces', [], { shouldDirty: true })
|
||||
}
|
||||
|
||||
function jobPayload() {
|
||||
return jobs.map(draftToJob).filter((j) => j.name || j.paths.length > 0)
|
||||
}
|
||||
|
||||
const runPreview = async () => {
|
||||
if (!collector) return
|
||||
const issues = draftIssues(jobs)
|
||||
if (issues.length > 0) {
|
||||
toast.error(issues[0])
|
||||
return
|
||||
}
|
||||
setPreviewing(true)
|
||||
try {
|
||||
const data = await collectorApi.preview(collector.id, {
|
||||
logPath: form.getValues('logPath'),
|
||||
scrapeJobs: jobPayload(),
|
||||
})
|
||||
setPreview(data)
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : t('edit.preview.failed'))
|
||||
} finally {
|
||||
setPreviewing(false)
|
||||
}
|
||||
}
|
||||
|
||||
const submit = form.handleSubmit(async (values) => {
|
||||
if (!collector) return
|
||||
if (isHost) {
|
||||
const issues = draftIssues(jobs)
|
||||
if (issues.length > 0) {
|
||||
toast.error(issues[0])
|
||||
return
|
||||
}
|
||||
}
|
||||
setSaving(true)
|
||||
try {
|
||||
await collectorApi.update(collector.id, {
|
||||
@@ -121,6 +163,7 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
.split(/[,\s]+/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
...(isHost ? { scrapeJobs: jobPayload() } : {}),
|
||||
})
|
||||
onOpenChange(false)
|
||||
onSaved()
|
||||
@@ -132,125 +175,184 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
|
||||
})
|
||||
|
||||
return (
|
||||
<Dialog open={collector !== null} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-2xl max-h-[90vh] overflow-y-auto">
|
||||
<DialogHeader><DialogTitle>{t('edit.title')}</DialogTitle></DialogHeader>
|
||||
<Form {...form}>
|
||||
<form onSubmit={submit} className="space-y-4">
|
||||
<FormField control={form.control} name="name" render={({ field }) => (
|
||||
<FormItem><FormLabel>{t('form.name')}</FormLabel><FormControl><Input {...field} /></FormControl><FormMessage /></FormItem>
|
||||
)} />
|
||||
<>
|
||||
<Dialog open={collector !== null} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-2xl max-h-[90vh] overflow-y-auto">
|
||||
<DialogHeader><DialogTitle>{t('edit.title')}</DialogTitle></DialogHeader>
|
||||
<Form {...form}>
|
||||
<form onSubmit={submit} className="space-y-4">
|
||||
<FormField control={form.control} name="name" render={({ field }) => (
|
||||
<FormItem><FormLabel>{t('form.name')}</FormLabel><FormControl><Input {...field} /></FormControl><FormMessage /></FormItem>
|
||||
)} />
|
||||
|
||||
{isDaemonSet && (
|
||||
<>
|
||||
{/* ── 采集命名空间:多选 ── */}
|
||||
{/* ── 主机模式:结构化采集项(多 job / 多路径 / 多行 / 丢弃 / 限流) ── */}
|
||||
{isHost && (
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<FormLabel>{t('edit.namespaces')}</FormLabel>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{allSelected ? t('edit.allNamespaces') : t('edit.selectedCount', { count: selectedNs.length })}
|
||||
</span>
|
||||
<FormLabel>{t('edit.jobs.title')}</FormLabel>
|
||||
<Button type="button" size="sm" variant="outline" className="h-7" onClick={runPreview} disabled={previewing}>
|
||||
<Eye className="mr-1 h-3.5 w-3.5" />
|
||||
{previewing ? t('edit.preview.loading') : t('edit.preview.button')}
|
||||
</Button>
|
||||
</div>
|
||||
{nsError && <p className="text-xs text-destructive">{nsError}</p>}
|
||||
<div className="rounded-md border">
|
||||
<label className="flex items-center gap-2 border-b px-3 py-2 text-sm font-medium cursor-pointer hover:bg-muted/50">
|
||||
<Checkbox checked={allSelected} onCheckedChange={() => setAll()} />
|
||||
{t('edit.allCheckbox')}
|
||||
</label>
|
||||
<div className="p-2">
|
||||
<Input
|
||||
className="mb-2 h-8"
|
||||
placeholder={t('edit.nsSearch')}
|
||||
value={nsFilter}
|
||||
onChange={(e) => setNsFilter(e.target.value)}
|
||||
/>
|
||||
<div className="max-h-44 overflow-auto grid gap-1 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{nsLoading && <p className="p-2 text-xs text-muted-foreground">{t('edit.loading')}</p>}
|
||||
{!nsLoading && filteredNs.length === 0 && (
|
||||
<p className="p-2 text-xs text-muted-foreground">
|
||||
{namespaces.length === 0 ? t('edit.nsListEmpty') : t('edit.nsNoMatch')}
|
||||
</p>
|
||||
)}
|
||||
{filteredNs.map((ns) => (
|
||||
<label key={ns} className="flex items-center gap-2 rounded px-2 py-1 text-sm hover:bg-muted/50 cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
disabled={allSelected}
|
||||
checked={selectedNs.includes(ns)}
|
||||
onChange={() => toggleNs(ns)}
|
||||
/>
|
||||
<span className="truncate">{ns}</span>
|
||||
</label>
|
||||
))}
|
||||
<ScrapeJobsEditor drafts={jobs} onChange={setJobs} />
|
||||
<p className="text-xs text-muted-foreground">{t('edit.jobs.hint')}</p>
|
||||
{collector?.resolvedHostname && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t('edit.jobs.hostInfo', {
|
||||
host: collector.resolvedHostname,
|
||||
format: collector.syslogFormat || 'rfc3164',
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isDaemonSet && (
|
||||
<>
|
||||
{/* ── 采集命名空间:多选 ── */}
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<FormLabel>{t('edit.namespaces')}</FormLabel>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{allSelected ? t('edit.allNamespaces') : t('edit.selectedCount', { count: selectedNs.length })}
|
||||
</span>
|
||||
</div>
|
||||
{nsError && <p className="text-xs text-destructive">{nsError}</p>}
|
||||
<div className="rounded-md border">
|
||||
<label className="flex items-center gap-2 border-b px-3 py-2 text-sm font-medium cursor-pointer hover:bg-muted/50">
|
||||
<Checkbox checked={allSelected} onCheckedChange={() => setAll()} />
|
||||
{t('edit.allCheckbox')}
|
||||
</label>
|
||||
<div className="p-2">
|
||||
<Input
|
||||
className="mb-2 h-8"
|
||||
placeholder={t('edit.nsSearch')}
|
||||
value={nsFilter}
|
||||
onChange={(e) => setNsFilter(e.target.value)}
|
||||
/>
|
||||
<div className="max-h-44 overflow-auto grid gap-1 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{nsLoading && <p className="p-2 text-xs text-muted-foreground">{t('edit.loading')}</p>}
|
||||
{!nsLoading && filteredNs.length === 0 && (
|
||||
<p className="p-2 text-xs text-muted-foreground">
|
||||
{namespaces.length === 0 ? t('edit.nsListEmpty') : t('edit.nsNoMatch')}
|
||||
</p>
|
||||
)}
|
||||
{filteredNs.map((ns) => (
|
||||
<label key={ns} className="flex items-center gap-2 rounded px-2 py-1 text-sm hover:bg-muted/50 cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
disabled={allSelected}
|
||||
checked={selectedNs.includes(ns)}
|
||||
onChange={() => toggleNs(ns)}
|
||||
/>
|
||||
<span className="truncate">{ns}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.nsHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
勾「所有命名空间」= 采集全集群;否则只采集勾选的命名空间。
|
||||
生成 Promtail 的 <code className="font-mono">kubernetes_sd_configs.namespaces</code> 白名单。
|
||||
</Trans>
|
||||
</p>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.nsHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
勾「所有命名空间」= 采集全集群;否则只采集勾选的命名空间。
|
||||
生成 Promtail 的 <code className="font-mono">kubernetes_sd_configs.namespaces</code> 白名单。
|
||||
</Trans>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* ── app 标签来源 ── */}
|
||||
<FormField control={form.control} name="appLabelSource" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('edit.appLabelSource')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input list="app-source-presets" placeholder={t('edit.appSourcePlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<datalist id="app-source-presets">
|
||||
{APP_SOURCE_PRESETS.filter((p) => p.value).map((p) => (
|
||||
<option key={p.value} value={p.value}>{t(p.labelKey)}</option>
|
||||
))}
|
||||
</datalist>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.appSourceHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
默认按 <code className="font-mono">app.kubernetes.io/name → app → controller 名 → pod 名</code> 四级回退。
|
||||
若集群有自己的私有规范标签(如 <code className="font-mono">k8s.kuboard.cn/name</code>),
|
||||
填在这里会作为最高优先级,其余仍兜底。
|
||||
</Trans>
|
||||
</p>
|
||||
</FormItem>
|
||||
)} />
|
||||
{/* ── app 标签来源 ── */}
|
||||
<FormField control={form.control} name="appLabelSource" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('edit.appLabelSource')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input list="app-source-presets" placeholder={t('edit.appSourcePlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<datalist id="app-source-presets">
|
||||
{APP_SOURCE_PRESETS.filter((p) => p.value).map((p) => (
|
||||
<option key={p.value} value={p.value}>{t(p.labelKey)}</option>
|
||||
))}
|
||||
</datalist>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.appSourceHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
默认按 <code className="font-mono">app.kubernetes.io/name → app → controller 名 → pod 名</code> 四级回退。
|
||||
若集群有自己的私有规范标签(如 <code className="font-mono">k8s.kuboard.cn/name</code>),
|
||||
填在这里会作为最高优先级,其余仍兜底。
|
||||
</Trans>
|
||||
</p>
|
||||
</FormItem>
|
||||
)} />
|
||||
|
||||
{/* ── app 白名单 ── */}
|
||||
<FormField control={form.control} name="appIncludeRaw" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('edit.appInclude')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('edit.appIncludePlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.appIncludeHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
填了之后 Promtail 只保留 app 命中这些值的日志(<code className="font-mono">action: keep</code>),
|
||||
是控制日志量最直接的手段。值需与上面「app 标签取值来源」解析出的 app 一致。
|
||||
</Trans>
|
||||
</p>
|
||||
</FormItem>
|
||||
)} />
|
||||
</>
|
||||
)}
|
||||
{/* ── app 白名单 ── */}
|
||||
<FormField control={form.control} name="appIncludeRaw" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('edit.appInclude')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('edit.appIncludePlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">
|
||||
<Trans i18nKey="edit.appIncludeHint" t={t} components={{ code: <code className="font-mono" /> }}>
|
||||
填了之后 Promtail 只保留 app 命中这些值的日志(<code className="font-mono">action: keep</code>),
|
||||
是控制日志量最直接的手段。值需与上面「app 标签取值来源」解析出的 app 一致。
|
||||
</Trans>
|
||||
</p>
|
||||
</FormItem>
|
||||
)} />
|
||||
</>
|
||||
)}
|
||||
|
||||
{supportsPath && <FormField control={form.control} name="logPath" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('form.logPath')}</FormLabel>
|
||||
<FormControl><Input placeholder={collector?.type === 'host-promtail' ? '/var/log/*.log' : '/var/log/app/*.log'} {...field} /></FormControl>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">{t('edit.logPathHint')}</p>
|
||||
</FormItem>
|
||||
)} />}
|
||||
{supportsPath && <FormField control={form.control} name="logPath" render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('form.logPath')}</FormLabel>
|
||||
<FormControl><Input placeholder="/var/log/app/*.log" {...field} /></FormControl>
|
||||
<FormMessage />
|
||||
<p className="text-xs text-muted-foreground">{t('edit.logPathHint')}</p>
|
||||
</FormItem>
|
||||
)} />}
|
||||
|
||||
<div className="flex justify-end gap-2 pt-2">
|
||||
<Button type="button" variant="outline" onClick={() => onOpenChange(false)}>{t('common:dialog.cancel')}</Button>
|
||||
<Button type="submit" disabled={saving}>{saving ? t('edit.saving') : t('edit.save')}</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
<div className="flex justify-end gap-2 pt-2">
|
||||
<Button type="button" variant="outline" onClick={() => onOpenChange(false)}>{t('common:dialog.cancel')}</Button>
|
||||
<Button type="submit" disabled={saving}>{saving ? t('edit.saving') : t('edit.save')}</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* 预览:将要写到目标机的 promtail.yaml(不落库、不下发) */}
|
||||
<Dialog open={preview !== null} onOpenChange={(open) => { if (!open) setPreview(null) }}>
|
||||
<DialogContent className="sm:max-w-3xl max-h-[90vh] overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t('edit.preview.title')}</DialogTitle>
|
||||
<DialogDescription>{t('edit.preview.description')}</DialogDescription>
|
||||
</DialogHeader>
|
||||
{(preview?.warnings ?? []).length > 0 && (
|
||||
<ul className="space-y-1 rounded-md border border-amber-300/60 bg-amber-50 p-2 text-xs text-amber-900 dark:bg-amber-950/30 dark:text-amber-200">
|
||||
{(preview?.warnings ?? []).map((w) => <li key={w}>· {w}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
<pre className="max-h-[55vh] overflow-auto rounded-md bg-muted p-3 text-[11px] leading-relaxed">
|
||||
{preview?.config}
|
||||
</pre>
|
||||
<div className="flex items-center justify-between">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
if (preview?.config) {
|
||||
void navigator.clipboard?.writeText(preview.config)
|
||||
toast.success(t('edit.preview.copied'))
|
||||
}
|
||||
}}
|
||||
>
|
||||
{t('edit.preview.copy')}
|
||||
</Button>
|
||||
<Button type="button" size="sm" onClick={() => setPreview(null)}>{t('edit.preview.close')}</Button>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,193 @@
|
||||
import { memo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import '@/i18n/locales/zh/collectors'
|
||||
import '@/i18n/locales/en/collectors'
|
||||
import { Plus, Trash2 } from 'lucide-react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { emptyJobDraft, type JobDraft } from './scrape-jobs'
|
||||
|
||||
interface Props {
|
||||
drafts: JobDraft[]
|
||||
onChange: (next: JobDraft[]) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* 采集项结构化编辑器(B 档)。
|
||||
*
|
||||
* 只暴露能覆盖常见场景的几个开关,不逼用户写 YAML / Promtail 配置:
|
||||
* 名称、路径、标签常驻;多行合并 / JSON / 自定义时间戳 / 丢弃 / 限流收进「高级」,
|
||||
* 免得默认视图就一屏表单(用户明确要求紧凑、高频项置顶、低频项折叠)。
|
||||
*/
|
||||
export const ScrapeJobsEditor = memo(function ScrapeJobsEditor({ drafts, onChange }: Props) {
|
||||
const { t } = useTranslation('collectors')
|
||||
|
||||
const patch = (index: number, part: Partial<JobDraft>) => {
|
||||
onChange(drafts.map((d, i) => (i === index ? { ...d, ...part } : d)))
|
||||
}
|
||||
|
||||
const add = () => {
|
||||
const n = drafts.length + 1
|
||||
onChange([...drafts, emptyJobDraft(n <= 2 ? `app-${n}` : `job-${n}`)])
|
||||
}
|
||||
|
||||
const remove = (index: number) => onChange(drafts.filter((_, i) => i !== index))
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
{drafts.map((d, i) => (
|
||||
<div key={i} className="space-y-2 rounded-md border p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
className="h-8 w-44 font-mono text-xs"
|
||||
value={d.name}
|
||||
placeholder={t('edit.jobs.namePlaceholder')}
|
||||
onChange={(e) => patch(i, { name: e.target.value })}
|
||||
/>
|
||||
{d.system && (
|
||||
<span className="rounded bg-muted px-1.5 py-0.5 text-[11px] text-muted-foreground">
|
||||
{t('edit.jobs.systemBadge')}
|
||||
</span>
|
||||
)}
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="ml-auto h-7 px-2 text-muted-foreground"
|
||||
onClick={() => remove(i)}
|
||||
aria-label={t('edit.jobs.remove')}
|
||||
>
|
||||
<Trash2 className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-2">
|
||||
<div className="space-y-1">
|
||||
<Label className="text-xs text-muted-foreground">{t('edit.jobs.paths')}</Label>
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.paths}
|
||||
placeholder={d.system ? t('edit.jobs.systemPathsPlaceholder') : '/data/app/logs/*.log, /data/app/logs/error/*.log'}
|
||||
onChange={(e) => patch(i, { paths: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-1">
|
||||
<Label className="text-xs text-muted-foreground">{t('edit.jobs.labels')}</Label>
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.labels}
|
||||
placeholder="app=cex-api, env=pre"
|
||||
onChange={(e) => patch(i, { labels: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details className="group">
|
||||
<summary className="cursor-pointer text-xs text-muted-foreground hover:text-foreground">
|
||||
{t('edit.jobs.advanced')}
|
||||
</summary>
|
||||
<div className="mt-2 space-y-3 rounded-md bg-muted/40 p-2">
|
||||
<div className="flex flex-wrap items-center gap-4">
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
<Switch checked={d.system} onCheckedChange={(v) => patch(i, { system: v })} />
|
||||
{t('edit.jobs.system')}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
<Switch checked={d.json} onCheckedChange={(v) => patch(i, { json: v })} />
|
||||
{t('edit.jobs.json')}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
<Switch
|
||||
checked={d.multilineEnabled}
|
||||
onCheckedChange={(v) => patch(i, { multilineEnabled: v })}
|
||||
/>
|
||||
{t('edit.jobs.multiline')}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
<Switch
|
||||
checked={d.timestampEnabled}
|
||||
onCheckedChange={(v) => patch(i, { timestampEnabled: v })}
|
||||
disabled={d.system}
|
||||
/>
|
||||
{t('edit.jobs.timestamp')}
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{d.multilineEnabled && (
|
||||
<div className="grid gap-2 sm:grid-cols-[1fr_120px]">
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.firstline}
|
||||
placeholder="^\d{4}-\d{2}-\d{2}"
|
||||
onChange={(e) => patch(i, { firstline: e.target.value })}
|
||||
/>
|
||||
<Input
|
||||
className="h-8 text-xs"
|
||||
value={d.maxLines}
|
||||
placeholder={t('edit.jobs.maxLinesPlaceholder')}
|
||||
onChange={(e) => patch(i, { maxLines: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{d.timestampEnabled && !d.system && (
|
||||
<div className="grid gap-2 sm:grid-cols-3">
|
||||
<Input
|
||||
className="h-8 text-xs"
|
||||
value={d.tsSource}
|
||||
placeholder={t('edit.jobs.tsSourcePlaceholder')}
|
||||
onChange={(e) => patch(i, { tsSource: e.target.value })}
|
||||
/>
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.tsFormat}
|
||||
placeholder="RFC3339"
|
||||
onChange={(e) => patch(i, { tsFormat: e.target.value })}
|
||||
/>
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.tsLocation}
|
||||
placeholder={t('edit.jobs.tsLocationPlaceholder')}
|
||||
onChange={(e) => patch(i, { tsLocation: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-1">
|
||||
<Label className="text-xs text-muted-foreground">{t('edit.jobs.dropIfMatch')}</Label>
|
||||
<Input
|
||||
className="h-8 font-mono text-xs"
|
||||
value={d.dropIfMatch}
|
||||
placeholder={t('edit.jobs.dropIfMatchPlaceholder')}
|
||||
onChange={(e) => patch(i, { dropIfMatch: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-2 sm:grid-cols-2">
|
||||
<Input
|
||||
className="h-8 text-xs"
|
||||
value={d.rate}
|
||||
placeholder={t('edit.jobs.ratePlaceholder')}
|
||||
onChange={(e) => patch(i, { rate: e.target.value })}
|
||||
/>
|
||||
<Input
|
||||
className="h-8 text-xs"
|
||||
value={d.burst}
|
||||
placeholder={t('edit.jobs.burstPlaceholder')}
|
||||
onChange={(e) => patch(i, { burst: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
))}
|
||||
|
||||
<Button type="button" size="sm" variant="outline" onClick={add}>
|
||||
<Plus className="mr-1 h-3.5 w-3.5" />
|
||||
{t('edit.jobs.add')}
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,166 @@
|
||||
// 采集项草稿模型:编辑器里全部用字符串保存(输入框友好),
|
||||
// 提交/预览时再转成后端要的 ScrapeJob 结构。
|
||||
import type { ScrapeJob } from '@/types'
|
||||
|
||||
export interface JobDraft {
|
||||
name: string
|
||||
paths: string
|
||||
labels: string
|
||||
system: boolean
|
||||
json: boolean
|
||||
multilineEnabled: boolean
|
||||
firstline: string
|
||||
maxLines: string
|
||||
timestampEnabled: boolean
|
||||
tsSource: string
|
||||
tsFormat: string
|
||||
tsLocation: string
|
||||
dropIfMatch: string
|
||||
rate: string
|
||||
burst: string
|
||||
}
|
||||
|
||||
export const emptyJobDraft = (name = ''): JobDraft => ({
|
||||
name,
|
||||
paths: '',
|
||||
labels: '',
|
||||
system: false,
|
||||
json: false,
|
||||
multilineEnabled: false,
|
||||
firstline: '',
|
||||
maxLines: '',
|
||||
timestampEnabled: false,
|
||||
tsSource: '',
|
||||
tsFormat: 'RFC3339',
|
||||
tsLocation: '',
|
||||
dropIfMatch: '',
|
||||
rate: '',
|
||||
burst: '',
|
||||
})
|
||||
|
||||
export function jobToDraft(j: ScrapeJob): JobDraft {
|
||||
return {
|
||||
name: j.name ?? '',
|
||||
paths: (j.paths ?? []).join(', '),
|
||||
labels: Object.entries(j.labels ?? {})
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(', '),
|
||||
system: Boolean(j.system),
|
||||
json: Boolean(j.json),
|
||||
multilineEnabled: Boolean(j.multiline?.firstline),
|
||||
firstline: j.multiline?.firstline ?? '',
|
||||
maxLines: j.multiline?.maxLines ? String(j.multiline.maxLines) : '',
|
||||
timestampEnabled: Boolean(j.timestamp?.source),
|
||||
tsSource: j.timestamp?.source ?? '',
|
||||
tsFormat: j.timestamp?.format ?? 'RFC3339',
|
||||
tsLocation: j.timestamp?.location ?? '',
|
||||
dropIfMatch: (j.dropIfMatch ?? []).join(', '),
|
||||
rate: j.rateLimit?.rate ? String(j.rateLimit.rate) : '',
|
||||
burst: j.rateLimit?.burst ? String(j.rateLimit.burst) : '',
|
||||
}
|
||||
}
|
||||
|
||||
function splitList(raw: string): string[] {
|
||||
return raw
|
||||
.split(/[,\n]/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
/** "app=cex-api, env=pre" → {app:'cex-api', env:'pre'};不合法键由后端校验给具体报错 */
|
||||
export function parseLabels(raw: string): Record<string, string> {
|
||||
const out: Record<string, string> = {}
|
||||
for (const item of splitList(raw)) {
|
||||
const i = item.indexOf('=')
|
||||
if (i <= 0) continue
|
||||
const k = item.slice(0, i).trim()
|
||||
const v = item.slice(i + 1).trim()
|
||||
if (k) out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const toInt = (raw: string) => {
|
||||
const n = Number.parseInt(raw.trim(), 10)
|
||||
return Number.isFinite(n) && n > 0 ? n : 0
|
||||
}
|
||||
|
||||
export function draftToJob(d: JobDraft): ScrapeJob {
|
||||
const job: ScrapeJob = {
|
||||
name: d.name.trim(),
|
||||
paths: splitList(d.paths),
|
||||
}
|
||||
if (d.system) job.system = true
|
||||
const labels = parseLabels(d.labels)
|
||||
if (Object.keys(labels).length > 0) job.labels = labels
|
||||
if (d.multilineEnabled && d.firstline.trim()) {
|
||||
job.multiline = { firstline: d.firstline.trim() }
|
||||
if (toInt(d.maxLines) > 0) job.multiline.maxLines = toInt(d.maxLines)
|
||||
}
|
||||
if (d.json) job.json = true
|
||||
if (d.timestampEnabled && d.tsSource.trim()) {
|
||||
job.timestamp = {
|
||||
source: d.tsSource.trim(),
|
||||
format: d.tsFormat.trim() || 'RFC3339',
|
||||
}
|
||||
if (d.tsLocation.trim()) job.timestamp.location = d.tsLocation.trim()
|
||||
}
|
||||
const drops = splitList(d.dropIfMatch)
|
||||
if (drops.length > 0) job.dropIfMatch = drops
|
||||
const rate = toInt(d.rate)
|
||||
const burst = toInt(d.burst)
|
||||
if (rate > 0 || burst > 0) job.rateLimit = { rate, burst }
|
||||
return job
|
||||
}
|
||||
|
||||
/** 本地即时校验(后端仍会再校验一次,这边只做「不用等请求就看出填错」) */
|
||||
export function draftIssues(drafts: JobDraft[]): string[] {
|
||||
const issues: string[] = []
|
||||
const seen = new Set<string>()
|
||||
drafts.forEach((d, i) => {
|
||||
const label = d.name.trim() || `第 ${i + 1} 项`
|
||||
if (!d.name.trim()) {
|
||||
issues.push(`第 ${i + 1} 个采集项缺少名称`)
|
||||
} else if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(d.name.trim())) {
|
||||
issues.push(`采集项「${label}」名称只能用字母、数字、点、下划线、中划线`)
|
||||
} else if (seen.has(d.name.trim())) {
|
||||
issues.push(`采集项名称「${label}」重复`)
|
||||
} else {
|
||||
seen.add(d.name.trim())
|
||||
}
|
||||
const paths = splitList(d.paths)
|
||||
if (!d.system && paths.length === 0) {
|
||||
issues.push(`采集项「${label}」至少填一个日志路径`)
|
||||
}
|
||||
for (const p of paths) {
|
||||
if (!p.startsWith('/')) {
|
||||
issues.push(`采集项「${label}」的路径「${p}」必须是绝对路径`)
|
||||
}
|
||||
if (/^\/(etc\/shadow|etc\/gshadow|etc\/sudoers|root\/\.ssh)|id_rsa|id_ed25519|\.pem$/i.test(p)) {
|
||||
issues.push(`采集项「${label}」的路径「${p}」是敏感凭据文件,禁止采集`)
|
||||
}
|
||||
}
|
||||
if (d.multilineEnabled && !d.firstline.trim()) {
|
||||
issues.push(`采集项「${label}」开了多行合并但没填首行正则`)
|
||||
}
|
||||
if (d.timestampEnabled && !d.tsSource.trim()) {
|
||||
issues.push(`采集项「${label}」开了自定义时间戳但没填字段名`)
|
||||
}
|
||||
for (const item of splitList(d.labels)) {
|
||||
if (!item.includes('=')) {
|
||||
issues.push(`采集项「${label}」的标签「${item}」要写成 key=value`)
|
||||
}
|
||||
}
|
||||
})
|
||||
return issues
|
||||
}
|
||||
|
||||
/** 没有任何采集项时,从旧字段 logPath 迁移出默认两项(应用日志 + 系统日志) */
|
||||
export function draftsFromCollector(jobs: ScrapeJob[] | undefined, logPath: string): JobDraft[] {
|
||||
if (jobs && jobs.length > 0) return jobs.map(jobToDraft)
|
||||
const app = emptyJobDraft('app')
|
||||
app.paths = (logPath || '/var/log/*.log').trim()
|
||||
const system = emptyJobDraft('system')
|
||||
system.system = true
|
||||
return [app, system]
|
||||
}
|
||||
@@ -182,6 +182,21 @@ export interface PageData<T> {
|
||||
pageSize: number;
|
||||
}
|
||||
|
||||
// ScrapeJob 主机模式的一个采集项(对应 promtail 的 scrape_configs 一项)
|
||||
export interface ScrapeJob {
|
||||
name: string;
|
||||
paths: string[];
|
||||
labels?: Record<string, string>;
|
||||
encoding?: string;
|
||||
/** 内置系统日志采集项:自动 syslog 正则 + 时间戳解析,paths 留空用探测到的系统日志 */
|
||||
system?: boolean;
|
||||
multiline?: { firstline: string; maxLines?: number };
|
||||
json?: boolean;
|
||||
timestamp?: { source: string; format: string; location?: string };
|
||||
dropIfMatch?: string[];
|
||||
rateLimit?: { rate: number; burst: number };
|
||||
}
|
||||
|
||||
// Collector(日志采集器 / 部署客户端)
|
||||
export interface Collector {
|
||||
id: number;
|
||||
@@ -204,6 +219,12 @@ export interface Collector {
|
||||
appInclude?: string[];
|
||||
logPath: string;
|
||||
labels: string;
|
||||
/** 主机模式的结构化采集项(B 档);空 = 按 logPath 生成 app + system 默认项 */
|
||||
scrapeJobs?: ScrapeJob[];
|
||||
/** 上次安装探测到的真实主机名(host 标签用它,不是 ${HOSTNAME}) */
|
||||
resolvedHostname?: string;
|
||||
/** 上次安装探测到的 syslog 时间戳格式:iso8601 | rfc3164 */
|
||||
syslogFormat?: string;
|
||||
status: string;
|
||||
statusMessage: string;
|
||||
createdAt: string;
|
||||
|
||||
Reference in new issue
Block a user