From 16306a95321ca4dd7f30d2c0c84a6583488ea83f Mon Sep 17 00:00:00 2001 From: opsx Date: Wed, 28 Oct 2026 22:43:25 +0800 Subject: [PATCH] =?UTF-8?q?feat(logs):=20metrics=20=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E7=AB=AF=E7=82=B9(admin)=20+=20=E5=BF=AB?= =?UTF-8?q?=E6=8D=B7=E6=9F=A5=E8=AF=A2=E9=80=89=E4=B8=AD=20App/Pod=20?= =?UTF-8?q?=E5=8D=B3=E6=8F=92=E5=85=A5=E5=B9=B6=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /logs/metrics:LogQL 指标查询代理,[$step] 占位符按跨度自适应步长,返回多序列(httptest 假 Loki 单测) - 快捷查询:选中 App/Pod 即写入输入框并立即查询,面板保持打开支持 Pod 级联;命名空间仅级联筛选不插入,切换时清理失效标签 - 删除冗余的「查询该范围日志」「指定副本日志」按钮;Ingress 按钮仅在选择 ingress 命名空间/app 时显示 - /logs 支持 ?q= 预填自动执行(表格「查看日志」跳转入口) - 清理无引用死组件 LogResults/LogHistogram --- backend/controllers/logs.go | 24 ++ backend/dto/logs.go | 9 + backend/routers/router.go | 1 + backend/services/log_metrics_test.go | 82 +++++++ backend/services/log_query.go | 67 +++++- .../pages/logs/components/LogHistogram.tsx | 70 ------ .../src/pages/logs/components/LogResults.tsx | 215 ------------------ .../src/pages/logs/components/QueryInput.tsx | 8 +- .../logs/components/QuickQueryTemplates.tsx | 50 +++- frontend/src/pages/logs/query-utils.ts | 33 +++ 10 files changed, 264 insertions(+), 295 deletions(-) create mode 100644 backend/services/log_metrics_test.go delete mode 100644 frontend/src/pages/logs/components/LogHistogram.tsx delete mode 100644 frontend/src/pages/logs/components/LogResults.tsx diff --git a/backend/controllers/logs.go b/backend/controllers/logs.go index 2496a2f..1a4c3b3 100644 --- a/backend/controllers/logs.go +++ b/backend/controllers/logs.go @@ -317,3 +317,27 @@ func (ctrl *LogsController) Histogram(c *gin.Context) { } utils.Success(c, buckets) } + +// Metrics 日志分析指标查询(仅 admin,路由层 AdminAuth 拦截) +func (ctrl *LogsController) Metrics(c *gin.Context) { + var req dto.LogMetricQueryRequest + if !utils.BindAndValidate(c, &req) { + return + } + src := services.GetLogSource(req.SourceID) + if src == nil { + utils.BadRequest(c, "数据源不存在") + return + } + query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query) + if err != nil { + utils.Forbidden(c, err.Error()) + return + } + series, err := services.LokiMetrics(c.Request.Context(), src, query, req.Start, req.End) + if err != nil { + utils.Error(c, 502, err.Error()) + return + } + utils.Success(c, series) +} diff --git a/backend/dto/logs.go b/backend/dto/logs.go index d486335..0291952 100644 --- a/backend/dto/logs.go +++ b/backend/dto/logs.go @@ -32,6 +32,15 @@ type LogQueryRequest struct { Cursor string `json:"cursor"` } +// LogMetricQueryRequest 日志分析指标查询:Query 为完整 LogQL 指标表达式, +// 其中的 [$step] 占位符由服务端按时间跨度替换为自适应步长(与直方图同款) +type LogMetricQueryRequest struct { + SourceID uint `json:"sourceId" binding:"required"` + Query string `json:"query" binding:"required"` + Start int64 `json:"start"` // epoch 纳秒 + End int64 `json:"end"` +} + type LogContextRequest struct { SourceID uint `json:"sourceId" binding:"required"` TimestampNs string `json:"timestampNs" binding:"required"` diff --git a/backend/routers/router.go b/backend/routers/router.go index b3da124..651b5d2 100644 --- a/backend/routers/router.go +++ b/backend/routers/router.go @@ -91,6 +91,7 @@ func SetupRouter() *gin.Engine { logs.GET("/label/:name/values", logsCtrl.LabelValues) logs.GET("/my-scope", logsCtrl.MyScope) logs.POST("/histogram", logsCtrl.Histogram) + logs.POST("/metrics", middleware.AdminAuth(), logsCtrl.Metrics) logs.GET("/health", logsCtrl.Health) } diff --git a/backend/services/log_metrics_test.go b/backend/services/log_metrics_test.go new file mode 100644 index 0000000..1af33d6 --- /dev/null +++ b/backend/services/log_metrics_test.go @@ -0,0 +1,82 @@ +package services + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "lokistack/models" +) + +// TestLokiMetricsParsesMatrixSeries 用假 Loki 验证指标代理: +// [$step] 占位符替换 + matrix 多序列解析(按 app/namespace 分组)。 +func TestLokiMetricsParsesMatrixSeries(t *testing.T) { + var gotQuery, gotStep string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query().Get("query") + gotStep = r.URL.Query().Get("step") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {"app": "web", "namespace": "prod"}, + "values": [[1.79e+09, "10"], [1.79e+09, "20"]] + }, + { + "metric": {"app": "api", "namespace": "prod"}, + "values": [[1.79e+09, "5"]] + }, + { + "metric": {"app": "empty", "namespace": "prod"}, + "values": [] + } + ] + } + }`)) + })) + defer server.Close() + + src := &models.LogSource{TargetType: "url", URL: server.URL, AuthType: "none"} + query := `sum by (app, namespace) (count_over_time({namespace=~".+"}[$step]))` + end := time.Now().UnixNano() + start := end - int64(time.Hour) + + series, err := LokiMetrics(context.Background(), src, query, start, end) + if err != nil { + t.Fatalf("LokiMetrics: %v", err) + } + + // 占位符被替换为 1h 跨度对应的步长,且 query_range 带上了同样的 step + if gotQuery == query || !strings.Contains(gotQuery, `count_over_time({namespace=~".+"}[`) { + t.Fatalf("[$step] not replaced, got %q", gotQuery) + } + if gotStep == "" || !strings.Contains(gotQuery, "["+gotStep+"]") { + t.Fatalf("step mismatch: query=%q step=%q", gotQuery, gotStep) + } + + // 空序列被丢弃,两条有效序列带标签返回 + if len(series) != 2 { + t.Fatalf("want 2 series, got %d", len(series)) + } + if series[0].Labels["app"] != "web" || series[0].Labels["namespace"] != "prod" { + t.Fatalf("labels not parsed: %+v", series[0].Labels) + } + if len(series[0].Points) != 2 || series[0].Points[0].Value != 10 || series[0].Points[1].Value != 20 { + t.Fatalf("points not parsed: %+v", series[0].Points) + } + if series[0].Points[0].Timestamp == "" { + t.Fatal("timestamp not formatted") + } + + // 响应可 JSON 序列化(controller 直接 utils.Success 包装) + if _, err := json.Marshal(series); err != nil { + t.Fatalf("marshal: %v", err) + } +} diff --git a/backend/services/log_query.go b/backend/services/log_query.go index 735aef2..bbfb8b9 100644 --- a/backend/services/log_query.go +++ b/backend/services/log_query.go @@ -17,9 +17,9 @@ import ( "strings" "time" + "k8s.io/client-go/rest" "lokistack/models" "lokistack/repository" - "k8s.io/client-go/rest" ) const maxLogQueryLimit = 1000 @@ -449,6 +449,71 @@ func LokiHistogram(ctx context.Context, src *models.LogSource, query string, sta return buckets, nil } +// MetricSeriesPoint 指标序列上的一个采样点 +type MetricSeriesPoint struct { + Timestamp string `json:"timestamp"` + Value float64 `json:"value"` +} + +// MetricSeries 一条按标签分组的指标序列(如 sum by (app)(...) 的每个 app 一条) +type MetricSeries struct { + Labels map[string]string `json:"labels"` + Points []MetricSeriesPoint `json:"points"` +} + +// LokiMetrics 通用 LogQL 指标查询(日志分析页用):query 为完整指标表达式, +// 其中所有 [$step] 占位符替换为按时间跨度自适应的步长后走 query_range, +// 返回矩阵多序列(直方图只取单序列,这里按 by (...) 分组整表返回) +func LokiMetrics(ctx context.Context, src *models.LogSource, query string, start, end int64) ([]MetricSeries, error) { + if end <= 0 { + end = time.Now().UnixNano() + } + if start <= 0 { + start = end - int64(time.Hour) + } + step := pickHistogramStep(end - start) + wrapped := strings.ReplaceAll(query, "[$step]", "["+step+"]") + + q := url.Values{} + q.Set("query", wrapped) + q.Set("start", strconv.FormatInt(start, 10)) + q.Set("end", strconv.FormatInt(end, 10)) + q.Set("step", step) + body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/query_range", q) + if err != nil { + return nil, err + } + var resp struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]interface{} `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return nil, err + } + series := make([]MetricSeries, 0, len(resp.Data.Result)) + for _, item := range resp.Data.Result { + points := make([]MetricSeriesPoint, 0, len(item.Values)) + for _, v := range item.Values { + // metric API 的 ts 为秒级浮点数(如 1.79e+09) + tsf, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[0]), 64) + cnt, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[1]), 64) + points = append(points, MetricSeriesPoint{ + Timestamp: time.Unix(int64(tsf), 0).Format(time.RFC3339), + Value: cnt, + }) + } + if len(points) == 0 { + continue + } + series = append(series, MetricSeries{Labels: item.Metric, Points: points}) + } + return series, nil +} + // pickHistogramStep 按时间跨度自适应步长(aiops 同款) func pickHistogramStep(spanNano int64) string { span := time.Duration(spanNano) diff --git a/frontend/src/pages/logs/components/LogHistogram.tsx b/frontend/src/pages/logs/components/LogHistogram.tsx deleted file mode 100644 index 8d92982..0000000 --- a/frontend/src/pages/logs/components/LogHistogram.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import { memo } from 'react' -import { AreaChart, Area, ResponsiveContainer, XAxis, YAxis, Tooltip } from 'recharts' -import { useTranslation } from 'react-i18next' -import type { HistogramBucket } from '@/api/logs' - -interface LogHistogramProps { - buckets: HistogramBucket[] - loading: boolean - /** 点击柱体 → 缩放到相邻 bucket 时间范围 */ - onZoomRange: (start: number, end: number) => void -} - -/** 日志量直方图(服务端 count_over_time 聚合);点击柱体缩放时间范围 */ -export const LogHistogram = memo(function LogHistogram({ buckets, loading, onZoomRange }: LogHistogramProps) { - const { t } = useTranslation('logs') - const data = buckets.map((b) => ({ - ts: b.timestamp.slice(11, 16), - count: b.count, - raw: new Date(b.timestamp).getTime(), - })) - - if (loading && data.length === 0) { - return
{t('volume.histogramLoading')}
- } - if (data.length === 0) { - return
{t('volume.histogramEmpty')}
- } - - const handleClick = (idx: number) => { - const cur = data[idx]?.raw - if (!cur) return - const step = data.length > 1 ? Math.max(data[1].raw - data[0].raw, 60000) : 300000 - onZoomRange(cur - step / 2, cur + step / 2) - } - - return ( -
- - { - if (state?.activeTooltipIndex !== undefined) handleClick(state.activeTooltipIndex) - }} - > - - - - - - - - - - - - -
- ) -}) diff --git a/frontend/src/pages/logs/components/LogResults.tsx b/frontend/src/pages/logs/components/LogResults.tsx deleted file mode 100644 index 3c852e5..0000000 --- a/frontend/src/pages/logs/components/LogResults.tsx +++ /dev/null @@ -1,215 +0,0 @@ -import { memo, useMemo, useRef, useState } from 'react' -import { ChevronDown, ChevronRight } from 'lucide-react' -import { useTranslation } from 'react-i18next' -import { cn } from '@/lib/utils' -import type { LogEntry } from '@/api/logs' -import { applyLabelFilter } from '../query-utils' - -// 级别 → 颜色(语义 token) -const LEVEL_COLOR: Record = { - error: 'text-destructive', - warn: 'text-warning', - info: 'text-success', - debug: 'text-muted-foreground', -} - -/** 从日志行提取 k=v / JSON 字段(点击过滤用) */ -function detectFields(message: string): Array<[string, string]> { - const out: Array<[string, string]> = [] - const trimmed = message.trim() - if (trimmed.startsWith('{')) { - try { - const obj = JSON.parse(trimmed) - for (const [k, v] of Object.entries(obj)) { - if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { - out.push([k, String(v)]) - } - } - return out.slice(0, 12) - } catch { - /* 非 JSON 继续 */ - } - } - const re = /([a-zA-Z_][\w.-]*)=("[^"]*"|\S+)/g - let m: RegExpExecArray | null - while ((m = re.exec(message)) !== null && out.length < 12) { - out.push([m[1], m[2].replace(/^"|"$/g, '')]) - } - return out -} - -interface LogResultsProps { - entries: LogEntry[] - loading: boolean - hasMore: boolean - onLoadMore: () => void - /** 点击标签过滤(改写查询) */ - onFilter: (nextQuery: string) => void - query: string -} - -/** - * 日志结果区(Grafana 风格裁剪版):级别着色、行展开(原文 + labels + 提取字段, - * 点击即过滤/排除)、去重开关 - */ -export const LogResults = memo(function LogResults({ - entries, - loading, - hasMore, - onLoadMore, - onFilter, - query, -}: LogResultsProps) { - const { t } = useTranslation('logs') - const [expanded, setExpanded] = useState>(new Set()) - const [dedup, setDedup] = useState(false) - const listRef = useRef(null) - - const display = useMemo(() => { - if (!dedup) return entries - const seen = new Set() - return entries.filter((e) => { - const k = e.message.replace(/\d+/g, '#') - if (seen.has(k)) return false - seen.add(k) - return true - }) - }, [entries, dedup]) - - const toggle = (key: string) => { - setExpanded((prev) => { - const next = new Set(prev) - if (next.has(key)) next.delete(key) - else next.add(key) - return next - }) - } - - // 公共标签(所有结果共有) - const commonLabels = useMemo(() => { - if (entries.length === 0) return [] as string[] - let keys = Object.keys(entries[0].labels) - for (const e of entries) { - keys = keys.filter((k) => e.labels[k] !== undefined && e.labels[k] === entries[0].labels[k]) - } - return keys - }, [entries]) - - if (loading && entries.length === 0) { - return
{t('raw.searching')}
- } - - return ( -
-
- - {t('raw.count', { count: display.length })} - {dedup && t('raw.dedupSuffix', { count: entries.length })} - {commonLabels.length > 0 && ( - - {commonLabels.slice(0, 6).map((k) => ( - - {k}={entries[0].labels[k]} - - ))} - - )} - - -
- -
- {display.length === 0 ? ( -
{t('raw.empty')}
- ) : ( - display.map((e, idx) => { - const key = `${e.timestamp}-${idx}` - const open = expanded.has(key) - const fields = open ? detectFields(e.message) : [] - return ( -
- - {open && ( -
-
-

{t('raw.original')}

- 
{e.message}
-
-
-

{t('raw.labelsTitle')}

-
- {Object.entries(e.labels).map(([k, v]) => ( - - - - - ))} -
-
- {fields.length > 0 && ( -
-

{t('raw.fields')}

-
- {fields.map(([k, v]) => ( - - - - ))} -
-
- )} -
- )} -
- ) - }) - )} -
- - {hasMore && ( - - )} -
- ) -}) diff --git a/frontend/src/pages/logs/components/QueryInput.tsx b/frontend/src/pages/logs/components/QueryInput.tsx index 505fbba..8e1f290 100644 --- a/frontend/src/pages/logs/components/QueryInput.tsx +++ b/frontend/src/pages/logs/components/QueryInput.tsx @@ -23,7 +23,8 @@ function colorizedLogQL(query: string) { interface QueryInputProps { value: string onChange: (value: string) => void - onSubmit: () => void + /** 执行查询;传入 query 时直接以其为准(避免 setQuery 异步取到旧值) */ + onSubmit: (query?: string) => void sourceId: number history: string[] } @@ -152,7 +153,10 @@ export const QueryInput = memo(function QueryInput({ value, onChange, onSubmit, {t('editor.escClose')} - {assistantTab === 'quick' ? : + {assistantTab === 'quick' ? { onChange(next); onSubmit(next) }} /> :
{history.length === 0 ?

{t('editor.historyEmpty')}

: history.map((item, index) => )}
} diff --git a/frontend/src/pages/logs/components/QuickQueryTemplates.tsx b/frontend/src/pages/logs/components/QuickQueryTemplates.tsx index aa335e1..056779b 100644 --- a/frontend/src/pages/logs/components/QuickQueryTemplates.tsx +++ b/frontend/src/pages/logs/components/QuickQueryTemplates.tsx @@ -3,9 +3,9 @@ import { useTranslation } from 'react-i18next' import { Button } from '@/components/ui/button' import { logsApi } from '@/api/logs' import { SearchableValueSelect } from './SearchableValueSelect' -import { applyLabelFilter, extractSelector } from '../query-utils' +import { applyLabelFilter, extractSelector, hasLabelFilter, removeLabelFilter, replaceSelector } from '../query-utils' -type Template = 'app' | 'error' | 'warn' | 'pod' | 'ingress' +type Template = 'error' | 'warn' | 'ingress' /** Ingress 访问日志所在命名空间候选（按顺序取第一个实际存在的） */ const INGRESS_NAMESPACE_CANDIDATES = ['ingress-nginx', 'ingress'] @@ -16,10 +16,15 @@ function selectorOf(labels: Record) { return `{${entries.map(([key, value]) => `${key}=${JSON.stringify(value.trim())}`).join(',')}}` } -export function QuickQueryTemplates({ sourceId, query, onQueryChange, compact }: { +export function QuickQueryTemplates({ sourceId, query, onQueryChange, onInsert, onRun, compact }: { sourceId: number query: string + /** 终态动作(模板按钮):写入输入框并关闭助手面板 */ onQueryChange: (query: string) => void + /** 只写入输入框,不执行、不关面板(清理失效标签用) */ + onInsert?: (query: string) => void + /** 写入并立即执行查询,面板保持打开(选中 App/Pod 后直接出结果) */ + onRun?: (query: string) => void compact?: boolean }) { const { t } = useTranslation('logs') @@ -114,11 +119,38 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, compact }: // 换命名空间后旧的 app / pod 条件失效 setApp('') setPod('') + // 命名空间只作级联筛选,不主动插入查询(范围太大); + // 但输入框里已插入的 app/pod 属于旧命名空间,需要清理,namespace 标签同步为新值 + if (!onInsert || !extractSelector(query)) return + let next = removeLabelFilter(removeLabelFilter(query, appLabel), 'pod') + if (value.trim()) { + if (hasLabelFilter(next, 'namespace')) next = applyLabelFilter(next, 'namespace', value.trim()) + } else { + next = removeLabelFilter(next, 'namespace') + } + if (next !== query) onInsert(next) } function chooseApp(value: string) { setApp(value) setPod('') + // 选中 App 即把选择器写入输入框并立即执行(命名空间范围太大故不自动执行,App 级别粒度合适) + if (!value.trim() || !onRun) return + let selector = extractSelector(query) || '{}' + if (namespace.trim()) selector = applyLabelFilter(selector, 'namespace', namespace.trim()) + selector = applyLabelFilter(selector, appLabel, value.trim()) + onRun(replaceSelector(query, selector)) + } + + function choosePod(value: string) { + setPod(value) + // 选 Pod 同样立即执行;级联保证 namespace/app 已选定,合并进选择器 + if (!value.trim() || !onRun) return + let selector = extractSelector(query) || '{}' + if (namespace.trim()) selector = applyLabelFilter(selector, 'namespace', namespace.trim()) + if (app.trim()) selector = applyLabelFilter(selector, appLabel, app.trim()) + selector = applyLabelFilter(selector, 'pod', value.trim()) + onRun(replaceSelector(query, selector)) } function generate(template: Template) { @@ -154,6 +186,12 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, compact }: const hasSelector = extractSelector(query).replace(/[{}\s]/g, '').length > 0 const hasScope = Boolean(app.trim() || namespace.trim() || pod.trim() || hasSelector) + // Ingress 访问日志按钮只在确实指向 ingress controller 时出现: + // 选了 ingress 命名空间,或选中的 app 就是 ingress controller(如 ingress-nginx-controller-nginx), + // 避免在业务命名空间下显示一个查不到东西的入口 + const ingressReady = + INGRESS_NAMESPACE_CANDIDATES.includes(namespace) || + Boolean(app.trim() && app.toLowerCase().includes('ingress-nginx')) return
{!compact &&

{t('templates.title')}

} @@ -183,18 +221,16 @@ export function QuickQueryTemplates({ sourceId, query, onQueryChange, compact }: label={t('templates.pod')} values={pods} value={pod} - onChange={setPod} + onChange={choosePod} placeholder={pods.length ? t('templates.selectCount', { count: pods.length }) : app ? t('templates.noPod') : t('templates.optional')} allowEmpty disabled={!sourceId} />
- - - + {ingressReady && } {!hasScope && {t('templates.scopeHint')}}
diff --git a/frontend/src/pages/logs/query-utils.ts b/frontend/src/pages/logs/query-utils.ts index 222ee9a..873e486 100644 --- a/frontend/src/pages/logs/query-utils.ts +++ b/frontend/src/pages/logs/query-utils.ts @@ -7,6 +7,39 @@ export function extractSelector(query: string): string { return i >= 0 && j > i ? query.slice(i, j + 1) : '' } +/** + * 替换 LogQL 的 {...} 选择器,保留其后的管道过滤(|= "x" 等); + * 原查询没有选择器时把新选择器前置到最前面 + */ +export function replaceSelector(query: string, nextSelector: string): string { + const current = extractSelector(query) + if (!current) return nextSelector + query.trim().replace(/^\{[^}]*\}/, '') + const i = query.indexOf(current) + return query.slice(0, i) + nextSelector + query.slice(i + current.length) +} + +/** 从选择器里删除指定标签的所有匹配条件(=、!=、=~、!~) */ +export function removeLabelFilter(query: string, key: string): string { + const current = extractSelector(query) + if (!current) return query + const inner = current.slice(1, -1).trim() + if (!inner) return query + const pattern = new RegExp(`^${key}(=|!=|=~|!~)`) + const kept = inner.split(',').map((s) => s.trim()).filter(Boolean).filter((p) => !pattern.test(p)) + const next = kept.length ? `{${kept.join(', ')}}` : '{}' + return replaceSelector(query, next) +} + +/** 选择器里是否已包含指定标签的匹配条件 */ +export function hasLabelFilter(query: string, key: string): boolean { + const current = extractSelector(query) + if (!current) return false + return current + .slice(1, -1) + .split(',') + .some((part) => new RegExp(`^\\s*${key}(=|!=|=~|!~)`).test(part)) +} + /** * 把 key="value" 合并进 LogQL 的 {...} 选择器(同名标签替换旧值);exclude=true 用 != 排除 */