Files
LokiStack/frontend/src/api/logs/index.ts
T
opsx 5f42d9f696
CI / Backend (go vet + test) (push) Canceled after 0s
CI / Frontend (lint + build + test) (push) Canceled after 0s
perf(logs): label 缓存 TTL 30s + 平台变更即时失效(回答"新 app 会不会延迟")
问题:加了 60 秒缓存后,新增 app / 新副本 / 新主机采集器要等最久 60 秒
才出现在下拉里,抢修时"刚上线的服务查不到"是不可接受的。

两层处理:

1) TTL 60s → 30s(前后端一致)
   纯 Loki 侧的变更(新 Pod 开始打日志、新副本、主机上新进程写日志)
   最长 30 秒可见。这类变化平台无从感知,只能靠 TTL 兜住。

2) 平台侧变更**立即失效**(不等 TTL)
   新增 InvalidateLabelCache(),在下列入口 defer 调用:
     CollectorService.Create / Update        (主机模式 app 标签来源)
     DeploymentService.Delete                (连带清理托管数据源)
     CreateLogSource / UpdateLogSource / DeleteLogSource
   也就是"用户在平台上做了什么导致候选值集合变化",缓存当场清掉,
   下一个请求必然回源。这正好覆盖"主机采集的 app 标签"这个场景 ——
   采集器一改,下拉里马上就有。

关于「要不要用定时任务预热」:
   不做。预热只能省掉一次 25~40ms 的网络往返,却要多一个常驻任务和
   一份额外的失败路径;而"时效性"问题已经由上面的即时失效解决了。
   如果将来真要削掉这 40ms,更划算的做法是把 label 值随部署一起下发缓存,
   而不是定时轮询 Loki。
2026-10-31 18:27:17 +00:00

142 lines
8.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { http, buildQuery } from '../http'
import type { PageData } from '@/types'
export interface LogSourceAccess { userId: number; labels: Record<string, string> }
export interface LogPolicyGroup { id: number; name: string; sourceId: number; labels: Record<string, string>; userIds: number[]; createdAt?: string; updatedAt?: string }
export interface LogSource {
id: number; name: string; targetType: 'url' | 'cluster-svc'; clusterID?: number
namespace: string; service: string; port: number; url: string
authType: string; basicUser: string; hasBasicPass?: boolean
headers?: Record<string, string>; headerNames?: string[]; skipTLSVerify?: boolean; hasBearerToken?: boolean
isDefault: boolean; managed?: boolean; access?: LogSourceAccess[]
}
export interface LogEntry { timestamp: string; timestampNs: string; level: string; message: string; labels: Record<string, string> }
export interface HistogramBucket { timestamp: string; count: number }
export interface LogResult { entries: LogEntry[]; hasMore: boolean; nextCursor: string }
export interface LogQuery { sourceId: number; query: string; start: number; end: number; limit?: number; direction?: 'backward' | 'forward'; cursor?: string }
/** 指标序列采样点(日志分析页) */
export interface MetricPoint { timestamp: string; value: number }
/** 一条按标签分组的指标序列(如 sum by (app)(...) 中每个 app 一条) */
export interface MetricSeries { labels: Record<string, string>; points: MetricPoint[] }
export interface MetricQuery { sourceId: number; query: string; start: number; end: number }
/** 每日日志量(平台留存历史,不受 Loki retention 限制) */
export interface DailyVolumePoint {
date: string
bytes: number
entries: number
/** 当天未结束或抓取时数据不全 */
partial: boolean
/** 该天平台没有留存快照(超过留存期/当天平台没运行)——与"日志量真的是 0"不同 */
missing?: boolean
}
/**
* label 候选值的前端缓存(60 秒 + 同请求合并)。
*
* 为什么前端也要缓存:快捷 LogQL 的下拉在「切数据源 / 切命名空间 / 重开面板」
* 时都会重新走一遍请求链,后端虽然只要 25~40ms,但串起来仍是三四次往返,
* 用户看到的就是转圈。缓存后同一组合在 30 秒内直接命中内存,体感是"打开就有"。
*
* 时效性说明:新增 app / 新副本最长 30 秒后出现在下拉里。平台侧的变更
* (采集器增删改、部署删除、数据源增删改)会由后端立即清缓存,
* 所以"刚配好的 app 找不到"这类问题不会被缓存放大。
*
* 用 Promise 作为缓存值:并发的相同请求共享同一个飞行中的请求(去重),
* 失败时立刻从缓存剔除,避免把错误结果缓存住。
*/
// 与后端 TTL 对齐(30 秒):新上线的 app 要在半分钟内可被选中,
// 同时又能挡住"切命名空间/重开面板"造成的重复请求。
const LABEL_CACHE_TTL = 30_000
const labelValueCache = new Map<string, { expires: number; data: Promise<unknown> }>()
function cachedRequest<T>(key: string, run: () => Promise<T>): Promise<T> {
const now = Date.now()
const hit = labelValueCache.get(key)
if (hit && hit.expires > now) return hit.data as Promise<T>
const promise = run().catch((err: unknown) => {
labelValueCache.delete(key)
throw err
})
labelValueCache.set(key, { expires: now + LABEL_CACHE_TTL, data: promise })
return promise
}
/** 手动失效(数据源被删除/重建后调用,避免拿到旧源的候选值) */
export function invalidateLabelCache(): void {
labelValueCache.clear()
}
export const logsApi = {
sources: () => http.get<PageData<LogSource>>('/logs/sources'),
createSource: (source: Partial<LogSource> & { basicPass?: string; bearerToken?: string }) => http.post<LogSource>('/logs/sources', source),
updateSource: (id: number, data: Partial<LogSource>) => http.put(`/logs/sources/${id}`, data),
deleteSource: (id: number) => http.delete(`/logs/sources/${id}`),
testSource: (data: Partial<LogSource>) => http.post<{ success: boolean; message: string }>('/logs/sources/test', data),
// 查询/统计都支持外部 signal 取消:用户改条件时必须断开旧请求,
// 否则旧请求会在后端继续占用 Loki CPU(历史上把实例拖到 OOM/不 Ready)。
query: (data: LogQuery, opts?: { signal?: AbortSignal }) =>
http.post<LogResult>('/logs/query', data, { timeout: 60000, ...opts }),
/** 日志上下文:before/after 为前后各取几条(后端默认 2、上限 50,原来写死 5) */
context: (data: { sourceId: number; timestampNs: string; labels: Record<string, string>; before?: number; after?: number }) =>
http.post<{ before: LogEntry[]; after: LogEntry[] }>('/logs/context', data),
/** 总量统计(Loki index/stats,只读索引):用于计算总页数 */
stats: (data: LogQuery, opts?: { signal?: AbortSignal }) =>
http.post<{ streams: number; chunks: number; bytes: number; entries: number }>(
'/logs/stats', data, { timeout: 25000, ...opts }),
/** 日志量统计:后端有 7 天范围上限与 30s 超时,前端超时略大于它 */
/** 应用排行(长范围读平台预聚合快照,毫秒级,不扫日志内容) */
appRanking: (sourceId: number, from: number, to: number, limit = 10, opts?: { signal?: AbortSignal; sort?: 'bytes' | 'entries' }) => {
const { signal, sort = 'bytes' } = opts ?? {}
return http.get<{
list: Array<{ app: string; namespace: string; entries: number; bytes: number; fromHistory: boolean }>
/** 快照覆盖的小时数 */
hours: number
/** 其中带体积的小时数(体积列是本轮新增,早期快照只有条数) */
bytesHours: number
}>(
`/logs/app-ranking?sourceId=${sourceId}&from=${from}&to=${to}&limit=${limit}&sort=${sort}`,
signal ? { signal } : undefined,
)
},
/** 每日日志量历史(读平台库,毫秒级) */
dailyVolume: (sourceId: number, days: number, opts?: { signal?: AbortSignal }) =>
http.get<DailyVolumePoint[]>(`/logs/daily-volume?sourceId=${sourceId}&days=${days}`, opts),
histogram: (data: LogQuery, opts?: { signal?: AbortSignal }) =>
http.post<Array<Record<string, unknown>>>('/logs/histogram', data, { timeout: 40000, ...opts }),
/** LogQL 指标查询(admin):query 中的 [$step] 占位符由后端按时间跨度替换 */
metrics: (data: MetricQuery, opts?: { signal?: AbortSignal }) => http.post<MetricSeries[]>('/logs/metrics', data, { timeout: 125000, ...opts }),
labels: (sourceId: number, start?: number, end?: number) => http.get<string[]>(`/logs/labels${buildQuery({ sourceId, start, end } as { page?: number; pageSize?: number })}`),
labelValues: (sourceId: number, name: string, query = '', start?: number, end?: number) =>
cachedRequest<string[]>(
`lv|${sourceId}|${name}|${query}|${start ?? 0}|${end ?? 0}`,
() => http.get<string[]>(`/logs/label/${encodeURIComponent(name)}/values${buildQuery({ sourceId, query, start, end } as { page?: number; pageSize?: number })}`),
),
/**
* 批量取多个 label 的候选值:`/logs/label-values?names=app,service_name,container&query={...}`
*
* 快捷查询的 app 下拉要做 app → service_name → container 三级回退,
* 逐个请求要 3 个往返(每个都经 K8s API 代理到 Loki 索引);
* 批量接口后端内部并发,一次往返拿全,前端只看"有没有值"。
* 返回里缺某个 label 或取失败 = 空数组(不会抛错)。
*/
labelValuesBatch: (sourceId: number, names: string[], query = '', start?: number, end?: number, opts?: { signal?: AbortSignal }) =>
// 带 signal 的调用(用户主动取消)不参与缓存,避免把"被取消"的请求共享给别人
opts?.signal
? http.get<Record<string, string[]>>(
`/logs/label-values${buildQuery({ sourceId, names: names.join(','), query, start, end } as { page?: number; pageSize?: number })}`,
opts,
)
: cachedRequest<Record<string, string[]>>(
`lvb|${sourceId}|${names.join(',')}|${query}|${start ?? 0}|${end ?? 0}`,
() => http.get<Record<string, string[]>>(
`/logs/label-values${buildQuery({ sourceId, names: names.join(','), query, start, end } as { page?: number; pageSize?: number })}`,
),
),
/** 当前用户在某数据源上的授权范围(策略 labels;管理员 unrestricted=true) */
myScope: (sourceId: number) => http.get<{ labels: Record<string, string>; unrestricted: boolean }>(`/logs/my-scope?sourceId=${sourceId}`),
policyGroups: () => http.get<LogPolicyGroup[]>('/logs/policy-groups'),
createPolicyGroup: (data: Omit<LogPolicyGroup, 'id'>) => http.post<LogPolicyGroup>('/logs/policy-groups', data),
updatePolicyGroup: (id: number, data: Omit<LogPolicyGroup, 'id'>) => http.put<LogPolicyGroup>(`/logs/policy-groups/${id}`, data),
deletePolicyGroup: (id: number) => http.delete(`/logs/policy-groups/${id}`),
}