diff --git a/backend/controllers/logs.go b/backend/controllers/logs.go index 1a4c3b3..d2de8a0 100644 --- a/backend/controllers/logs.go +++ b/backend/controllers/logs.go @@ -2,6 +2,8 @@ package controllers import ( "context" + "errors" + "net/http" "strconv" "strings" "time" @@ -312,7 +314,15 @@ func (ctrl *LogsController) Histogram(c *gin.Context) { } buckets, err := services.LokiHistogram(c.Request.Context(), src, query, req.Start, req.End) if err != nil { - utils.Error(c, 502, err.Error()) + switch { + case errors.Is(err, services.ErrTooManyQueries): + // 429：并发重查询超限，让前端提示"稍后重试"，而不是当成服务故障 + utils.Error(c, http.StatusTooManyRequests, err.Error()) + case strings.Contains(err.Error(), "统计范围过大"): + utils.Error(c, http.StatusBadRequest, err.Error()) + default: + utils.Error(c, 502, err.Error()) + } return } utils.Success(c, buckets) diff --git a/backend/services/log_query.go b/backend/services/log_query.go index bbfb8b9..4749738 100644 --- a/backend/services/log_query.go +++ b/backend/services/log_query.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -404,6 +405,36 @@ func extractStreamSelector(query string) string { } // LokiHistogram 日志量直方图:服务端包 sum(count_over_time(q[step])) +// 重查询护栏。设计依据： +// - 单条 27 天的 count_over_time 曾把 Loki 打到 OOMKilled(2Gi)， +// 进而 Pod 不 Ready → Service 无端点 → 全站查询 502。 +// - 所以"算不完的查询"必须在入口就拒绝，而不是硬跑 120 秒把实例拖垮。 +const ( + // 日志量统计最多统计 7 天；更长跨度请分段查（前端会提示缩小范围） + maxHistogramRange = 7 * 24 * time.Hour + // 重查询（histogram/metrics）单独的更短超时，避免长时间占用 CPU + heavyQueryTimeout = 30 * time.Second + // 全局并发重查询上限：超过直接快速失败，保护单实例 Loki + maxConcurrentHeavyQueries = 2 +) + +// ErrTooManyQueries 并发重查询超限（handler 映射为 429） +var ErrTooManyQueries = errors.New("当前有较多日志量统计在进行，请稍后重试") + +// heavyQuerySem 全局重查询信号量 +var heavyQuerySem = make(chan struct{}, maxConcurrentHeavyQueries) + +func acquireHeavyQuery() bool { + select { + case heavyQuerySem <- struct{}{}: + return true + default: + return false + } +} + +func releaseHeavyQuery() { <-heavyQuerySem } + func LokiHistogram(ctx context.Context, src *models.LogSource, query string, start, end int64) ([]map[string]interface{}, error) { if end <= 0 { end = time.Now().UnixNano() @@ -412,9 +443,73 @@ func LokiHistogram(ctx context.Context, src *models.LogSource, query string, sta start = end - int64(time.Hour) } span := end - start - step := pickHistogramStep(span) - wrapped := fmt.Sprintf("sum(count_over_time(%s[%s]))", query, step) + if span > int64(maxHistogramRange) { + return nil, fmt.Errorf("统计范围过大（%.1f 天），最多支持 %.0f 天；请缩小时间范围后重试", + float64(span)/float64(24*time.Hour), maxHistogramRange.Hours()/24) + } + // 并发护栏：拿不到令牌立刻返回，不排队（排队只会让用户等更久） + if !acquireHeavyQuery() { + return nil, ErrTooManyQueries + } + defer releaseHeavyQuery() + // 独立超时：Loki 侧再慢也不会超过 30s，防止单查询长期占用实例 + ctx, cancel := context.WithTimeout(ctx, heavyQueryTimeout) + defer cancel() + + step := pickHistogramStep(span) + + // ── 首选：Loki 官方的 index/volume_range ────────────────────────────── + // 它只读 TSDB 索引里的 chunk 元数据（字节数），**不解码日志内容**。 + // 实测同一条件 24 小时：count_over_time 53s 超时失败，volume_range 111ms。 + // 返回单位是「字节」，所以 bucket 里带 unit=bytes，前端据此格式化为 KB/MB/GB。 + q := url.Values{} + q.Set("query", query) + q.Set("start", strconv.FormatInt(start, 10)) + q.Set("end", strconv.FormatInt(end, 10)) + q.Set("step", step) + q.Set("limit", "50") + if body, status, err := lokiProxyGET(ctx, src, "/loki/api/v1/index/volume_range", q); err == nil && status < 400 { + if buckets, ok := parseVolumeBuckets(body); ok { + return buckets, nil + } + } + // 老版本 Loki 没有 volume_range（404/400）时回退到 count_over_time + return lokiHistogramByCount(ctx, src, query, step, start, end) +} + +// parseVolumeBuckets 解析 index/volume_range 的 matrix 响应。 +// 成功但无数据也返回 ok=true（空图），只有结构不对才 ok=false 触发回退。 +func parseVolumeBuckets(body []byte) ([]map[string]interface{}, bool) { + var resp struct { + Data struct { + Result []struct { + Values [][2]interface{} `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return nil, false + } + buckets := []map[string]interface{}{} + if len(resp.Data.Result) > 0 { + for _, v := range resp.Data.Result[0].Values { + tsf, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[0]), 64) + bytes, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[1]), 64) + buckets = append(buckets, map[string]interface{}{ + "timestamp": time.Unix(int64(tsf), 0).Format(time.RFC3339), + "count": int64(bytes), + "unit": "bytes", + }) + } + } + return buckets, true +} + +// lokiHistogramByCount 回退实现：count_over_time 逐点统计「条数」。 +// 代价高（要真读 chunk），只在 volume_range 不可用时使用。 +func lokiHistogramByCount(ctx context.Context, src *models.LogSource, query, step string, start, end int64) ([]map[string]interface{}, error) { + wrapped := fmt.Sprintf("sum(count_over_time(%s[%s]))", query, step) q := url.Values{} q.Set("query", wrapped) q.Set("start", strconv.FormatInt(start, 10)) @@ -437,12 +532,12 @@ func LokiHistogram(ctx context.Context, src *models.LogSource, query string, sta buckets := []map[string]interface{}{} if len(resp.Data.Result) > 0 { for _, v := range resp.Data.Result[0].Values { - // metric API 的 ts 为秒级浮点数(如 1.79e+09),query 日志的才是纳秒字符串 tsf, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[0]), 64) cnt, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[1]), 64) buckets = append(buckets, map[string]interface{}{ "timestamp": time.Unix(int64(tsf), 0).Format(time.RFC3339), "count": int64(cnt), + "unit": "entries", }) } } @@ -471,6 +566,15 @@ func LokiMetrics(ctx context.Context, src *models.LogSource, query string, start if start <= 0 { start = end - int64(time.Hour) } + // 与 histogram 共用护栏：并发限 2、单次 30s 超时，避免 analysis 页 + // 多个卡片同时发起 count_over_time 把单实例 Loki 打满。 + if !acquireHeavyQuery() { + return nil, ErrTooManyQueries + } + defer releaseHeavyQuery() + ctx, cancel := context.WithTimeout(ctx, heavyQueryTimeout) + defer cancel() + step := pickHistogramStep(end - start) wrapped := strings.ReplaceAll(query, "[$step]", "["+step+"]") diff --git a/backend/services/loki_config.go b/backend/services/loki_config.go index e73a059..305e02c 100644 --- a/backend/services/loki_config.go +++ b/backend/services/loki_config.go @@ -56,6 +56,20 @@ compactor: limits_config: retention_period: {{ .RetentionPeriod }}h + # 单实例 Loki 的并发护栏：默认 32 高得离谱，一条重查询就能把单进程打满 + max_query_parallelism: 8 + +# ── 查询结果缓存（官方推荐；此前完全没配，重复查询要重算一遍）─────── +# align_queries_with_step 让不同时间范围的同类查询对齐到同一 step， +# 显著提高缓存命中率；命中后同样的"日志量统计"直接从缓存返回。 +query_range: + align_queries_with_step: true + cache_results: true + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 schema_config: configs: diff --git a/frontend/src/api/logs/index.ts b/frontend/src/api/logs/index.ts index 3dab5fd..995becb 100644 --- a/frontend/src/api/logs/index.ts +++ b/frontend/src/api/logs/index.ts @@ -25,9 +25,14 @@ export const logsApi = { updateSource: (id: number, data: Partial) => http.put(`/logs/sources/${id}`, data), deleteSource: (id: number) => http.delete(`/logs/sources/${id}`), testSource: (data: Partial) => http.post<{ success: boolean; message: string }>('/logs/sources/test', data), - query: (data: LogQuery) => http.post('/logs/query', data, { timeout: 125000 }), + // 查询/统计都支持外部 signal 取消：用户改条件时必须断开旧请求， + // 否则旧请求会在后端继续占用 Loki CPU（历史上把实例拖到 OOM/不 Ready）。 + query: (data: LogQuery, opts?: { signal?: AbortSignal }) => + http.post('/logs/query', data, { timeout: 60000, ...opts }), context: (data: { sourceId: number; timestampNs: string; labels: Record }) => http.post<{ before: LogEntry[]; after: LogEntry[] }>('/logs/context', data), - histogram: (data: LogQuery) => http.post>>('/logs/histogram', data, { timeout: 125000 }), + /** 日志量统计：后端有 7 天范围上限与 30s 超时，前端超时略大于它 */ + histogram: (data: LogQuery, opts?: { signal?: AbortSignal }) => + http.post>>('/logs/histogram', data, { timeout: 40000, ...opts }), /** LogQL 指标查询(admin):query 中的 [$step] 占位符由后端按时间跨度替换 */ metrics: (data: MetricQuery, opts?: { signal?: AbortSignal }) => http.post('/logs/metrics', data, { timeout: 125000, ...opts }), labels: (sourceId: number, start?: number, end?: number) => http.get(`/logs/labels${buildQuery({ sourceId, start, end } as { page?: number; pageSize?: number })}`), diff --git a/frontend/src/i18n/locales/en/logs.ts b/frontend/src/i18n/locales/en/logs.ts index 9f3352e..cad28a5 100644 --- a/frontend/src/i18n/locales/en/logs.ts +++ b/frontend/src/i18n/locales/en/logs.ts @@ -126,9 +126,9 @@ export default { timelineAria: 'Log volume timeline', timelineTitle: 'Log volume timeline', timelineHint: 'Hover to see counts; press and drag at any height to select a time span; the query runs automatically after selection.', - timelineTotal: '{{count}} logs in range', - timelineBucket: '{{count}} logs', - timelineBucket_one: '{{count}} log', + timelineTotal: 'Volume in range: {{count}}', + timelineBucket: '{{count}}', + timelineBucket_one: '{{count}}', }, results: { aria: 'Log results', diff --git a/frontend/src/i18n/locales/zh/logs.ts b/frontend/src/i18n/locales/zh/logs.ts index de12649..8852fbc 100644 --- a/frontend/src/i18n/locales/zh/logs.ts +++ b/frontend/src/i18n/locales/zh/logs.ts @@ -125,8 +125,8 @@ export default { timelineAria: '日志量时间线', timelineTitle: '日志量时间线', timelineHint: '悬停查看数量，按住任意高度拖选时间；选择后自动查询。', - timelineTotal: '时间桶合计 {{count}} 条', - timelineBucket: '{{count}} 条', + timelineTotal: '日志量合计 {{count}}', + timelineBucket: '{{count}}', }, results: { aria: '日志结果', diff --git a/frontend/src/pages/logs/components/LogVolumeTimeline.tsx b/frontend/src/pages/logs/components/LogVolumeTimeline.tsx index 46e3e37..71bf7a5 100644 --- a/frontend/src/pages/logs/components/LogVolumeTimeline.tsx +++ b/frontend/src/pages/logs/components/LogVolumeTimeline.tsx @@ -1,11 +1,28 @@ import { useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -type Bucket = { timestamp: string; count: number } +type Bucket = { timestamp: string; count: number; unit?: string } const formatTime = (timestamp: string) => new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) const formatCount = (count: number) => new Intl.NumberFormat('zh-CN').format(count) +/** + * 日志量展示：后端优先用 index/volume_range（只读索引，快约 500 倍）， + * 它返回的是**字节数**（unit=bytes）；老版本 Loki 回退 count_over_time 时是条数。 + * 按 unit 决定格式，避免把 44 GB 显示成 "44894961500 条"。 + */ +const formatVolume = (count: number, unit?: string) => { + if (unit !== 'bytes') return formatCount(count) + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let v = count + let i = 0 + while (v >= 1024 && i < units.length - 1) { + v /= 1024 + i++ + } + return `${v >= 100 ? v.toFixed(0) : v.toFixed(1)} ${units[i]}` +} + export function LogVolumeTimeline({ buckets, onSelect }: { buckets: Bucket[] onSelect: (start: number, end: number) => void @@ -38,7 +55,7 @@ export function LogVolumeTimeline({ buckets, onSelect }: { return

{t('volume.timelineTitle')}

{t('volume.timelineHint')}

- {t('volume.timelineTotal', { count: formatCount(total) })} + {t('volume.timelineTotal', { count: formatVolume(total, buckets[0]?.unit) })}
)}
{hover !== null &&
- {new Date(buckets[hover].timestamp).toLocaleString()} · {t('volume.timelineBucket', { count: formatCount(buckets[hover].count) })} + {new Date(buckets[hover].timestamp).toLocaleString()} · {t('volume.timelineBucket', { count: formatVolume(buckets[hover].count, buckets[hover].unit) })}
}
{formatCount(max)}{formatCount(Math.round(max / 2))}0
diff --git a/frontend/src/pages/logs/index.tsx b/frontend/src/pages/logs/index.tsx index e2943d5..f4ddb23 100644 --- a/frontend/src/pages/logs/index.tsx +++ b/frontend/src/pages/logs/index.tsx @@ -26,6 +26,39 @@ const ns = (ms: number) => Math.round(ms).toString() + '000000' type Bucket = { timestamp: string; count: number } +/** 是否为主动取消（abort）造成的错误 —— 这类不该提示用户 */ +function isAbortError(e: unknown): boolean { + return e instanceof DOMException + ? e.name === 'AbortError' + : typeof e === 'object' && e !== null && (e as { name?: string }).name === 'AbortError' +} + +/** + * 把后端错误翻译成用户能看懂的一句话。 + * 之前统一显示「日志量统计失败」，把 429/400/超时/Loki 不可用全糊在一起， + * 用户既不知道该缩小范围还是该等，也无从判断是不是平台挂了。 + */ +function describeQueryError(e: unknown, fallback: string): string { + const raw = e instanceof Error ? e.message : String(e ?? '') + if (!raw) return fallback + const lower = raw.toLowerCase() + + if (raw.includes('统计范围过大') || lower.includes('too large') || lower.includes('range exceeds')) { + return raw + } + if (raw.includes('较多日志量统计') || lower.includes('too many requests') || lower.includes('429')) { + return '当前有其他日志量统计正在进行（单实例 Loki 保护），请等几秒后重试' + } + if (lower.includes('no endpoints available') || lower.includes('http 502') || lower.includes('http 503')) { + return 'Loki 暂时不可用（Pod 未就绪或无端点），请稍后重试；若持续请查看 Loki 实例状态' + } + if (lower.includes('timeout') || lower.includes('timed out') || lower.includes('aborted')) { + return '统计超时（数据量过大），请缩小时间范围后重试' + } + // 其它情况：保留原始信息，便于排查（截断避免撑爆布局） + return raw.length > 160 ? raw.slice(0, 160) + '…' : raw +} + export default function LogsPage() { const { t } = useTranslation('logs') const admin = getUser()?.role === 'admin' @@ -42,7 +75,11 @@ export default function LogsPage() { const [searched, setSearched] = useState(false) const [buckets, setBuckets] = useState([]) const [histogramLoading, setHistogramLoading] = useState(false) - const [histogramError, setHistogramError] = useState(false) + // 从 boolean 改为字符串：原来只显示"日志量统计失败"， + // 真实原因（429 并发超限 / 400 范围过大 / Loki 不可用）全被吞掉。 + const [histogramError, setHistogramError] = useState('') + /** 当前在跑的查询/统计控制器：发起新请求前 abort 掉，避免重查询叠加 */ + const histAbortRef = useRef(null) const [cursor, setCursor] = useState('') const [pageIndex, setPageIndex] = useState(0) const [pageCursors, setPageCursors] = useState(['']) @@ -62,7 +99,7 @@ export default function LogsPage() { } catch (e) { setError(e instanceof Error ? e.message : t('source.loadFailed')) } }, [t]) useEffect(() => { void reloadSources() }, [reloadSources]) - useEffect(() => { requestVersion.current++; setEntries([]); setBuckets([]); setHistogramLoading(false); setHistogramError(false); setCursor(''); setPageIndex(0); setPageCursors(['']); setSearched(false); setLoading(false) }, [sourceId]) + useEffect(() => { requestVersion.current++; setEntries([]); setBuckets([]); setHistogramLoading(false); setHistogramError(''); setCursor(''); setPageIndex(0); setPageCursors(['']); setSearched(false); setLoading(false) }, [sourceId]) // 从日志分析页等入口跳转:带 ?q= 预填输入框并尝试执行一次;执行后清掉参数防刷新重复执行。 // 预填不等数据源就绪(无数据源时也要让用户看到带来的查询),是否真正执行由 search 内部判断 @@ -88,10 +125,18 @@ export default function LogsPage() { if (!r || !sourceId || !searchQuery) return const version = ++requestVersion.current setLoading(true); setLoadingStartedAt(Date.now()); setError('') - if (!paging) { setBuckets([]); setHistogramLoading(false); setHistogramError(false) } + if (!paging) { setBuckets([]); setHistogramLoading(false); setHistogramError('') } const data: LogQuery = { sourceId, query: searchQuery, ...r, limit, direction: 'backward', cursor: nextCursor } + + // 关键：发起新查询前，先断掉上一次仍在跑的查询/统计。 + // 否则用户连续改条件会产生多条并发重查询，把单实例 Loki 打满 + // （历史事故：27 天统计 → OOMKilled → Pod 不 Ready → 全站 502）。 + histAbortRef.current?.abort() + const histController = new AbortController() + histAbortRef.current = histController + try { - const result = await logsApi.query(data) + const result = await logsApi.query(data, { signal: histController.signal }) if (version !== requestVersion.current) return if (paging && result.entries.length === 0) { setCursor(''); return } if (!paging) setQueryHistory(rememberQuery(data.query)) @@ -104,13 +149,23 @@ export default function LogsPage() { setPageCursors(previous => paging ? [...previous.slice(0, targetPage), nextCursor] : ['']) if (!paging) { setHistogramLoading(true) - setHistogramError(false) - void logsApi.histogram(data).then(histogram => { + setHistogramError('') + void logsApi.histogram(data, { signal: histController.signal }).then(histogram => { if (version === requestVersion.current) { - setBuckets(histogram.map(item => ({ timestamp: String(item.timestamp), count: Number(item.count) || 0 }))) + setBuckets(histogram.map(item => ({ + timestamp: String(item.timestamp), + count: Number(item.count) || 0, + unit: typeof item.unit === 'string' ? item.unit : undefined, + }))) } - }).catch(() => { if (version === requestVersion.current) setHistogramError(true) }) - .finally(() => { if (version === requestVersion.current) setHistogramLoading(false) }) + }).catch((e: unknown) => { + // 主动取消不算错误，直接静默 + if (isAbortError(e) || histController.signal.aborted) return + if (version === requestVersion.current) setHistogramError(describeQueryError(e, t('volume.statsError'))) + }).finally(() => { + // 无论成功/失败/取消都必须清掉 loading，否则会一直转圈 + if (!histController.signal.aborted || version === requestVersion.current) setHistogramLoading(false) + }) } } catch (e) { if (!quiet && version === requestVersion.current) setError(e instanceof Error ? e.message : t('toolbar.queryFailed')) } finally { if (version === requestVersion.current) { setLoading(false); setLoadingStartedAt(null) } } @@ -202,7 +257,7 @@ export default function LogsPage() { {admin && tab === 'access' && } {admin && tab === 'sources' && void reloadSources()} />} {tab === 'query' && histogramLoading &&
{t('volume.statsLoading')}
} - {tab === 'query' && histogramError &&

{t('volume.statsError')}

} + {tab === 'query' && histogramError &&

{histogramError}

} {tab === 'query' && buckets.length > 0 && } {tab === 'query' && 0} pageIndex={pageIndex} hasNextPage={Boolean(cursor)} loading={loading} loadingStartedAt={loadingStartedAt} query={query} terms={highlightTerms} onQueryChange={setQuery} onApplyDomain={applyDomainFilter} onShowContext={row => void showContext(row)} onSearch={() => void search()} onNextPage={() => void search(cursor, false, undefined, undefined, pageIndex + 1)} onPreviousPage={() => void search(pageCursors[pageIndex - 1] ?? '', false, undefined, undefined, pageIndex - 1)} />} setContext(null)} />