feat(collector): 采集命名空间多选 + app 标签来源可自定义 + app 白名单

=== 一、采集命名空间改为多选(原来只有文本输入)===
- CollectorEditDialog 重写:勾选式多选面板,从
  GET /clusters/:id/namespaces 拉取真实命名空间列表
- "所有命名空间(全集群采集)"一键全选(targetNamespaces 置空即全集群)
- 支持搜索过滤命名空间;已选数量实时显示
- 保存时为空数组 → Promtail 不加 namespaces 白名单

=== 二、app 标签取值来源可配置 ===
新增 Collector.AppLabelSource:
  留空(默认)= app.kubernetes.io/name → app → controller 名(去hash) → pod 名 四级回退
  可填任意 Pod 标签,如 app.kubernetes.io/name / app / k8s.kuboard.cn/name
  也可直接写 __meta_kubernetes_pod_label_xxx
  → 作为最高优先级插到回退链最前,其余仍兜底

模板侧新增:
  sanitizePromLabel()      标签名 → Prometheus meta label(. / 等转 _)
  promtailAppSourceLabel() 解析用户配置
  promtailAppIncludeFilter() 生成白名单 keep 规则

=== 三、app 白名单(控制采集量)===
新增 Collector.AppInclude []string:
  填了之后生成
    - source_labels: [app]
      regex: ^(app1|app2)$
      action: keep
  只保留匹配的日志流 —— 这是收敛日志量最直接的手段

service.Update 同步支持 appLabelSource / appInclude(兼容 []string 与逗号分隔字符串)。

前端 CollectorEditDialog:
  采集命名空间 = 勾选多选 + 全选
  app 标签取值来源 = 输入框 + 5 个常用值 datalist 提示
  只采集这些 app = 逗号分隔输入

镜像:lghost/lokistack:latest + v0.1 = sha256:43369d06cb28
This commit is contained in:
opsx committed 2026-10-27 03:55:13 +00:00
1 parent e006fc1a0e
commit 41633f0b38
7 files changed
+280 -33

No files matched your search

+7
View File
@@ -16,6 +16,13 @@ type Collector struct {
Cluster *Cluster `json:"cluster,omitempty"`
TargetDeployment string `json:"targetDeployment"` // sidecar 专用:目标 deployment 名
Namespace string `json:"namespace"` // 采集器自身部署所在的 k8s 命名空间
// AppLabelSource 决定 app 标签从哪里取值(可填私有标签,便于按自己的规范匹配)。
// 留空 = 默认:app.kubernetes.io/name → app → Pod 的 controller_name(去 hash) → pod 名
// 可填:controller_name / app.kubernetes.io/name / app / k8s.kuboard.cn/name 等任意 Pod 标签
AppLabelSource string `json:"appLabelSource"`
// AppInclude 只采集这些 app 的日志(白名单,按 AppLabelSource 解析出的 app 值匹配)。
// 留空 = 不限制。配合 TargetNamespaces 可显著收敛日志量。
AppInclude []string `json:"appInclude"`
// TargetNamespaces 只采集这些命名空间的 Pod 日志;为空表示采集全集群。
// 对应 promtail 里 kubernetes_sd_configs.namespaces.names
TargetNamespaces []string `json:"targetNamespaces"`
+37
View File
@@ -248,6 +248,43 @@ func (s *CollectorService) Update(id uint, updates map[string]interface{}) error
}
db.Collectors[i].TargetNamespaces = nss
}
// appLabelSource:app 标签取值来源(空=默认 controller_name 回退链)
if v, ok := updates["appLabelSource"].(string); ok {
if strings.TrimSpace(v) != db.Collectors[i].AppLabelSource {
db.Collectors[i].StatusMessage = "app 标签来源已保存,点击部署以应用新配置"
}
db.Collectors[i].AppLabelSource = strings.TrimSpace(v)
}
// appInclude:只采集这些 app(白名单,空=不限)
if v, ok := updates["appInclude"]; ok {
var apps []string
switch t := v.(type) {
case []interface{}:
for _, item := range t {
if s, ok := item.(string); ok {
if s = strings.TrimSpace(s); s != "" {
apps = append(apps, s)
}
}
}
case []string:
for _, s := range t {
if s = strings.TrimSpace(s); s != "" {
apps = append(apps, s)
}
}
case string:
for _, s := range strings.Split(t, ",") {
if s = strings.TrimSpace(s); s != "" {
apps = append(apps, s)
}
}
}
if len(apps) != len(db.Collectors[i].AppInclude) {
db.Collectors[i].StatusMessage = "采集 app 白名单已保存,点击部署以应用新配置"
}
db.Collectors[i].AppInclude = apps
}
db.Collectors[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
+2 -1
View File
@@ -204,7 +204,8 @@ func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint strin
for _, kind := range []string{"daemonset", "configmap", "serviceaccount"} {
_ = deleteK8sResource(ctx, cluster, kind, ns, "promtail")
}
manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels, d.collector.TargetNamespaces)
manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels,
d.collector.TargetNamespaces, d.collector.AppLabelSource, d.collector.AppInclude)
if err := d.applyYAML(ctx, nil, manifest, "promtail"); err != nil {
return err
}
+62 -4
View File
@@ -3,6 +3,7 @@ package services
import (
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
)
@@ -196,6 +197,55 @@ extraVolumeMounts:
// PromtailDaemonsetYAML DaemonSet 模式纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),
// 取代 helm 安装 —— 后端无需安装 helm 二进制。
// relabel 体系与主流 helm chart 对齐:namespace/pod/container/node_name + __path__ 按 pod uid 精确匹配。
// sanitizePromLabel 把 K8s 标签名转成 Prometheus/Promtail 的 meta label 片段。
//
// k8s.kuboard.cn/name → k8s_kuboard_cn_name
// app.kubernetes.io/name → app_kubernetes_io_name
func sanitizePromLabel(name string) string {
var b strings.Builder
for _, r := range strings.TrimSpace(name) {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_':
b.WriteRune(r)
default:
b.WriteRune('_')
}
}
return b.String()
}
// promtailAppSourceLabel 把用户配置的 AppLabelSource 转成 Promtail 的 meta label 名。
// 返回空串表示「用默认的四级回退链」。
func promtailAppSourceLabel(src string) string {
src = strings.TrimSpace(src)
if src == "" || src == "controller_name" {
return ""
}
if strings.HasPrefix(src, "label:") {
src = strings.TrimPrefix(src, "label:")
}
// 已经是 __meta_ 开头就直接用
if strings.HasPrefix(src, "__meta_kubernetes_") {
return src
}
return "__meta_kubernetes_pod_label_" + sanitizePromLabel(src)
}
// promtailAppIncludeFilter 生成 app 白名单的 keep 规则(为空则不限)。
func promtailAppIncludeFilter(apps []string) string {
clean := make([]string, 0, len(apps))
for _, a := range apps {
if a = strings.TrimSpace(a); a != "" {
clean = append(clean, regexp.QuoteMeta(a))
}
}
if len(clean) == 0 {
return ""
}
return fmt.Sprintf("\n - source_labels: [app]\n regex: ^(%s)$\n action: keep",
strings.Join(clean, "|"))
}
// promtailNamespaceFilter 生成 kubernetes_sd_configs 的命名空间白名单。
// namespaces 为空 → 返回空串(采集全集群所有命名空间)。
func promtailNamespaceFilter(namespaces []string) string {
@@ -218,10 +268,16 @@ func promtailNamespaceFilter(namespaces []string) string {
return b.String()
}
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string, targetNamespaces []string) string {
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string, targetNamespaces []string, appLabelSource string, appInclude []string) string {
promtailImage := ResolveImageFor("promtail", "grafana/promtail", "3.4.0")
extLabels := promtailExternalLabels(clusterName, labelsJSON)
nsFilter := promtailNamespaceFilter(targetNamespaces)
// 自定义 app 标签来源(用户可用自己注入的私有标签),插到回退链最前面
appSourceLine := ""
if lbl := promtailAppSourceLabel(appLabelSource); lbl != "" {
appSourceLine = " - " + lbl + "\n"
}
appIncludeFilter := promtailAppIncludeFilter(appInclude)
return fmt.Sprintf(`apiVersion: v1
kind: ServiceAccount
metadata:
@@ -310,8 +366,10 @@ data:
#
# 不使用 k8s.kuboard.cn/name 之类的平台私有标签:那只是 Kuboard 管理的集群才有,
# 换 Rancher/Lens/Helm/kubectl 部署的集群就不存在,会破坏通用性。
# 如集群确实有私有规范标签,可在「采集器 → 编辑」里配置 AppLabelSource,
# 它会作为最高优先级插到下面列表最前(其余仍兜底)。
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
%s - __meta_kubernetes_pod_label_app_kubernetes_io_name
- __meta_kubernetes_pod_label_app
- __tmp_controller_name
- __meta_kubernetes_pod_name
@@ -350,7 +408,7 @@ data:
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
target_label: __path__
target_label: __path__%s
---
apiVersion: apps/v1
kind: DaemonSet
@@ -424,7 +482,7 @@ spec:
- name: containers
hostPath:
path: /var/lib/docker/containers
`, ns, ns, ns, lokiURL, extLabels, nsFilter, ns, promtailImage)
`, ns, ns, ns, lokiURL, extLabels, nsFilter, appSourceLine, promtailImage, appIncludeFilter)
}
// FluentbitConfig Sidecar 模式 Fluent Bit 配置(采集 logPath,推送至 lokiHost:lokiPort)
+7 -1
View File
@@ -10,7 +10,13 @@ export const collectorApi = {
create: (data: CreateCollectorRequest) => http.post<Collector>('/collectors', data),
update: (id: number, data: { name: string; logPath: string; targetNamespaces?: string[] }) =>
update: (id: number, data: {
name: string;
logPath: string;
targetNamespaces?: string[];
appLabelSource?: string;
appInclude?: string[];
}) =>
http.put<{ message: string }>(`/collectors/${id}`, data),
remove: (id: number) => http.delete<{ message: string }>(`/collectors/${id}`),
@@ -1,9 +1,9 @@
import { memo, useEffect, useState } from 'react'
import { memo, useEffect, useMemo, useState } from 'react'
import { useForm } from 'react-hook-form'
import { zodResolver } from '@hookform/resolvers/zod'
import { z } from 'zod'
import { toast } from 'sonner'
import { collectorApi } from '@/api'
import { clusterApi, collectorApi } from '@/api'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'
import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'
@@ -13,11 +13,24 @@ import type { Collector } from '@/types'
const schema = z.object({
name: z.string().trim().min(1, '请输入采集策略名称'),
logPath: z.string().trim(),
/** 逗号分隔的命名空间白名单;留空 = 采集全集群 */
targetNamespaces: z.string().trim(),
/** 采集的命名空间白名单;空数组 = 全集群 */
targetNamespaces: z.array(z.string()),
/** app 标签取值来源;留空 = 默认 controller_name 四级回退 */
appLabelSource: z.string().trim(),
/** 只采集这些 app(逗号分隔);留空 = 不限 */
appIncludeRaw: z.string().trim(),
})
type Values = z.infer<typeof schema>
/** app 标签来源的常用取值(可自由填写其它 Pod 标签名) */
const APP_SOURCE_PRESETS = [
{ value: '', label: '默认:app.kubernetes.io/name → app → controller 名 → pod 名' },
{ value: 'controller_name', label: 'controller_name(Pod 的 ReplicaSet 名,去 hash = Deployment 名)' },
{ value: 'app.kubernetes.io/name', label: 'app.kubernetes.io/name(K8s 标准标签)' },
{ value: 'app', label: 'app(常见简写)' },
{ value: 'k8s.kuboard.cn/name', label: 'k8s.kuboard.cn/name(Kuboard 注入)' },
]
interface Props {
collector: Collector | null
onOpenChange: (open: boolean) => void
@@ -26,21 +39,63 @@ interface Props {
export const CollectorEditDialog = memo(function CollectorEditDialog({ collector, onOpenChange, onSaved }: Props) {
const [saving, setSaving] = useState(false)
const [namespaces, setNamespaces] = useState<string[]>([])
const [nsLoading, setNsLoading] = useState(false)
const [nsError, setNsError] = useState('')
const [nsFilter, setNsFilter] = useState('')
const form = useForm<Values>({
resolver: zodResolver(schema),
defaultValues: { name: '', logPath: '', targetNamespaces: '' },
defaultValues: { name: '', logPath: '', targetNamespaces: [], appLabelSource: '', appIncludeRaw: '' },
})
const isDaemonSet = collector?.type === 'k8s-daemonset'
const supportsPath = !isDaemonSet
const selectedNs = form.watch('targetNamespaces')
useEffect(() => {
if (collector) {
form.reset({
name: collector.name,
logPath: collector.logPath || '',
targetNamespaces: (collector.targetNamespaces ?? []).join(', '),
})
}
if (!collector) return
form.reset({
name: collector.name,
logPath: collector.logPath || '',
targetNamespaces: collector.targetNamespaces ?? [],
appLabelSource: collector.appLabelSource ?? '',
appIncludeRaw: (collector.appInclude ?? []).join(', '),
})
setNsFilter('')
}, [collector, form])
// 拉集群命名空间列表(供多选)
useEffect(() => {
if (!collector || !isDaemonSet) return
const clusterID = collector.clusterID
if (!clusterID) return
let active = true
setNsLoading(true)
setNsError('')
clusterApi.namespaces(clusterID)
.then((res) => { if (active) setNamespaces(res?.namespaces ?? []) })
.catch((err) => { if (active) setNsError(err instanceof Error ? err.message : '加载命名空间失败') })
.finally(() => { if (active) setNsLoading(false) })
return () => { active = false }
}, [collector, isDaemonSet])
const filteredNs = useMemo(
() => namespaces.filter((n) => n.toLowerCase().includes(nsFilter.toLowerCase())),
[namespaces, nsFilter],
)
const allSelected = selectedNs.length === 0 // 空 = 全集群
function toggleNs(ns: string) {
const next = selectedNs.includes(ns) ? selectedNs.filter((n) => n !== ns) : [...selectedNs, ns]
form.setValue('targetNamespaces', next, { shouldDirty: true })
}
function setAll() {
form.setValue('targetNamespaces', [], { shouldDirty: true })
}
const submit = form.handleSubmit(async (values) => {
if (!collector) return
setSaving(true)
@@ -48,8 +103,9 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
await collectorApi.update(collector.id, {
name: values.name,
logPath: values.logPath,
// 逗号/空格分隔 → 字符串数组;留空数组表示采集全集群
targetNamespaces: values.targetNamespaces
targetNamespaces: values.targetNamespaces,
appLabelSource: values.appLabelSource,
appInclude: values.appIncludeRaw
.split(/[,\s]+/)
.map((s) => s.trim())
.filter(Boolean),
@@ -63,27 +119,104 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
}
})
const supportsPath = collector?.type !== 'k8s-daemonset'
return (
<Dialog open={collector !== null} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-md">
<DialogContent className="sm:max-w-2xl max-h-[90vh] overflow-y-auto">
<DialogHeader><DialogTitle>编辑采集策略</DialogTitle></DialogHeader>
<Form {...form}>
<form onSubmit={submit} className="space-y-4">
<FormField control={form.control} name="name" render={({ field }) => (
<FormItem><FormLabel>名称</FormLabel><FormControl><Input {...field} /></FormControl><FormMessage /></FormItem>
)} />
{collector?.type === 'k8s-daemonset' && <FormField control={form.control} name="targetNamespaces" render={({ field }) => (
<FormItem>
<FormLabel>采集命名空间</FormLabel>
<FormControl><Input placeholder="astralx, log(留空 = 全集群)" {...field} /></FormControl>
<FormMessage />
<p className="text-xs text-muted-foreground">
多个命名空间用逗号分隔;留空则采集全集群所有命名空间。保存后点击「部署」应用新配置,
Promtail 会按此生成 kubernetes_sd_configs.namespaces 白名单。
</p>
</FormItem>
)} />}
{isDaemonSet && (
<>
{/* ── 采集命名空间:多选 ── */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<FormLabel>采集命名空间</FormLabel>
<span className="text-xs text-muted-foreground">
{allSelected ? '全集群所有命名空间' : `已选 ${selectedNs.length} 个`}
</span>
</div>
{nsError && <p className="text-xs text-destructive">{nsError}</p>}
<div className="rounded-md border">
<label className="flex items-center gap-2 border-b px-3 py-2 text-sm font-medium cursor-pointer hover:bg-muted/50">
<input type="checkbox" checked={allSelected} onChange={setAll} />
所有命名空间(全集群采集)
</label>
<div className="p-2">
<Input
className="mb-2 h-8"
placeholder="搜索命名空间…"
value={nsFilter}
onChange={(e) => setNsFilter(e.target.value)}
/>
<div className="max-h-44 overflow-auto grid gap-1 sm:grid-cols-2 lg:grid-cols-3">
{nsLoading && <p className="p-2 text-xs text-muted-foreground">加载中…</p>}
{!nsLoading && filteredNs.length === 0 && (
<p className="p-2 text-xs text-muted-foreground">
{namespaces.length === 0 ? '未取到命名空间列表(可手动输入)' : '无匹配'}
</p>
)}
{filteredNs.map((ns) => (
<label key={ns} className="flex items-center gap-2 rounded px-2 py-1 text-sm hover:bg-muted/50 cursor-pointer">
<input
type="checkbox"
disabled={allSelected}
checked={selectedNs.includes(ns)}
onChange={() => toggleNs(ns)}
/>
<span className="truncate">{ns}</span>
</label>
))}
</div>
</div>
</div>
<p className="text-xs text-muted-foreground">
勾「所有命名空间」= 采集全集群;否则只采集勾选的命名空间。
生成 Promtail 的 <code className="font-mono">kubernetes_sd_configs.namespaces</code> 白名单。
</p>
</div>
{/* ── app 标签来源 ── */}
<FormField control={form.control} name="appLabelSource" render={({ field }) => (
<FormItem>
<FormLabel>app 标签取值来源</FormLabel>
<FormControl>
<Input list="app-source-presets" placeholder="留空 = 默认(controller 名回退链)" {...field} />
</FormControl>
<datalist id="app-source-presets">
{APP_SOURCE_PRESETS.filter((p) => p.value).map((p) => (
<option key={p.value} value={p.value}>{p.label}</option>
))}
</datalist>
<FormMessage />
<p className="text-xs text-muted-foreground">
默认按 <code className="font-mono">app.kubernetes.io/name → app → controller 名 → pod 名</code> 四级回退。
若集群有自己的私有规范标签(如 <code className="font-mono">k8s.kuboard.cn/name</code>),
填在这里会作为最高优先级,其余仍兜底。
</p>
</FormItem>
)} />
{/* ── app 白名单 ── */}
<FormField control={form.control} name="appIncludeRaw" render={({ field }) => (
<FormItem>
<FormLabel>只采集这些 app(可选)</FormLabel>
<FormControl>
<Input placeholder="留空 = 不限;多个用逗号分隔,如 cex-spot-api, cex-asset-api" {...field} />
</FormControl>
<FormMessage />
<p className="text-xs text-muted-foreground">
填了之后 Promtail 只保留 app 命中这些值的日志(<code className="font-mono">action: keep</code>),
是控制日志量最直接的手段。值需与上面「app 标签取值来源」解析出的 app 一致。
</p>
</FormItem>
)} />
</>
)}
{supportsPath && <FormField control={form.control} name="logPath" render={({ field }) => (
<FormItem>
<FormLabel>采集日志路径</FormLabel>
@@ -92,6 +225,7 @@ export const CollectorEditDialog = memo(function CollectorEditDialog({ collector
<p className="text-xs text-muted-foreground">Sidecar 需使用业务容器的共享日志卷,或专用 logs 目录。保存后点击「部署」应用新配置。</p>
</FormItem>
)} />}
<div className="flex justify-end gap-2 pt-2">
<Button type="button" variant="outline" onClick={() => onOpenChange(false)}>取消</Button>
<Button type="submit" disabled={saving}>{saving ? '保存中…' : '保存'}</Button>
+4
View File
@@ -199,6 +199,10 @@ export interface Collector {
namespace: string;
/** 只采集这些命名空间;为空表示采集全集群 */
targetNamespaces?: string[];
/** app 标签取值来源;留空=默认 controller_name 四级回退 */
appLabelSource?: string;
/** 只采集这些 app 的日志(白名单);留空=不限 */
appInclude?: string[];
logPath: string;
labels: string;
status: string;