diff --git a/backend/services/deployment_config_test.go b/backend/services/deployment_config_test.go index 6211e19..cc9bada 100644 --- a/backend/services/deployment_config_test.go +++ b/backend/services/deployment_config_test.go @@ -67,12 +67,15 @@ func TestGeneratedLokiConfigKeepsRequiredSettingsWithoutDefaults(t *testing.T) { // loki_config.go 里对 s3/oss/minio/cos/obs 刻意把 compactor 的删除请求存到 // 本地盘（见该处注释），避免各云 S3 兼容实现差异导致 compactor 启动失败。 // loki_object_store_test.go 断言的也是 filesystem —— 两处必须一致。 - for _, required := range []string{"retention_period: 2160h", "retention_enabled: true", "delete_request_store: filesystem", "bucketnames: logs", "store: tsdb", "schema: v13"} { + for _, required := range []string{"retention_period: 2160h", "retention_enabled: true", "delete_request_store: filesystem", "retention_delete_delay: 10m", "bucketnames: logs", "store: tsdb", "schema: v13"} { if !strings.Contains(config, required) { t.Fatalf("required setting %q missing", required) } } - for _, defaultSetting := range []string{"http_listen_port:", "grpc_listen_port:", "chunk_idle_period:", "compaction_interval:", "retention_delete_delay:", "max_entries_limit_per_query:", "frontend_worker:", "chunk_store_config:", "autoforget_unhealthy:"} { + // 注意：retention_delete_delay 已从"不该出现"清单移除 —— 现在**有意**配置为 10m。 + // 默认 2h 会让对象存储里多留 2 小时数据（实测桶内超期对象一度占 24.6GB）， + // 预发环境用 10m 既保留安全窗口又不白付存储费。 + for _, defaultSetting := range []string{"http_listen_port:", "grpc_listen_port:", "chunk_idle_period:", "compaction_interval:", "max_entries_limit_per_query:", "frontend_worker:", "chunk_store_config:", "autoforget_unhealthy:"} { if strings.Contains(config, defaultSetting) { t.Fatalf("unnecessary setting %q remained", defaultSetting) } diff --git a/backend/services/log_query.go b/backend/services/log_query.go index 4e027c1..07f8964 100644 --- a/backend/services/log_query.go +++ b/backend/services/log_query.go @@ -23,10 +23,11 @@ import ( "lokistack/repository" ) -// maxLogQueryLimit 单页硬上限。依据：实测每条日志约 671B—— -// 1000 条 ≈ 655KB 响应；而前端结果表**没有虚拟滚动**，1000 行会全部渲染成 -// DOM 节点导致卡顿。前端下拉只给到 1000，超过会被这里静默收敛。 -const maxLogQueryLimit = 1000 +// maxLogQueryLimit 单页硬上限，与前端下拉选项保持**完全一致**， +// 避免"前端能选 5000、后端静默截成 1000"这种假选项。 +// 取值 5000 = Loki limits_config.max_entries_limit_per_query 的默认上限； +// 前端结果表已加虚拟滚动（>100 条只渲染可视区），5000 行不会卡。 +const maxLogQueryLimit = 5000 const logQueryTimeout = 120 * time.Second var lokiHTTPClients = [2]*http.Client{ @@ -417,8 +418,13 @@ const ( maxHistogramRange = 7 * 24 * time.Hour // 重查询（histogram/metrics）单独的更短超时，避免长时间占用 CPU heavyQueryTimeout = 30 * time.Second - // 全局并发重查询上限：超过直接快速失败，保护单实例 Loki - maxConcurrentHeavyQueries = 2 + // 全局并发重查询上限。取 4 而不是 2： + // 日志分析页一次会并发 5 条 count_over_time，限制 2 会让大部分请求拿不到令牌。 + maxConcurrentHeavyQueries = 4 + // 拿不到令牌时最多排队 10 秒再拒绝。 + // 原来"立即 429"会把分析页 5 条并发里的 3 条直接打回， + // 用户看到的就是"当前有较多日志量统计在进行"。 + heavyQueryWait = 10 * time.Second ) // ErrTooManyQueries 并发重查询超限（handler 映射为 429） @@ -427,11 +433,20 @@ var ErrTooManyQueries = errors.New("当前有较多日志量统计在进行， // heavyQuerySem 全局重查询信号量 var heavyQuerySem = make(chan struct{}, maxConcurrentHeavyQueries) +// acquireHeavyQuery 先尝试立即获取；拿不到则最多排队 heavyQueryWait。 +// 突发（如分析页并发 5 条）会自然排队消化，只有持续超载才返回 429。 func acquireHeavyQuery() bool { select { case heavyQuerySem <- struct{}{}: return true default: + } + timer := time.NewTimer(heavyQueryWait) + defer timer.Stop() + select { + case heavyQuerySem <- struct{}{}: + return true + case <-timer.C: return false } } diff --git a/backend/services/loki_config.go b/backend/services/loki_config.go index 305e02c..afdf261 100644 --- a/backend/services/loki_config.go +++ b/backend/services/loki_config.go @@ -53,6 +53,10 @@ compactor: working_directory: /var/lib/loki/compactor retention_enabled: true delete_request_store: {{ .CompactorStoreType }} + # 过期 chunk 的「标记 → 真正删除」间隔。默认 2h 会让对象存储里 + # 多留 2 小时的数据（实测桶内超过保留期的对象一度占到 24.6 GB）。 + # 预发环境取 10m：既保留短暂的安全窗口，又不会长期占用存储成本。 + retention_delete_delay: 10m limits_config: retention_period: {{ .RetentionPeriod }}h diff --git a/frontend/src/i18n/locales/en/analysis.ts b/frontend/src/i18n/locales/en/analysis.ts index ca86865..b0e295d 100644 --- a/frontend/src/i18n/locales/en/analysis.ts +++ b/frontend/src/i18n/locales/en/analysis.ts @@ -14,6 +14,9 @@ export default { filter: { source: 'Source', sourceEmpty: 'No sources', + namespace: 'Namespace', + namespaceEmpty: 'Loading namespaces…', + namespaceAll: 'All namespaces (slow)', refresh: 'Refresh', }, stats: { diff --git a/frontend/src/i18n/locales/en/deployments.ts b/frontend/src/i18n/locales/en/deployments.ts index 3e058f8..bfbfc70 100644 --- a/frontend/src/i18n/locales/en/deployments.ts +++ b/frontend/src/i18n/locales/en/deployments.ts @@ -89,26 +89,25 @@ export default { storageSizeMin: 'Alibaba Cloud ESSD requires at least 20Gi — enter 20Gi or more', }, retention: { - title: 'Edit Loki Config · {{name}}', + title: 'Edit retention policy · {{name}}', locationLabel: 'Config location: ', locationHostBinary: '/etc/loki/loki.yaml → limits_config.retention_period', locationHostDocker: 'Host /etc/lokistack/loki-{{id}}/loki.yaml → limits_config.retention_period', locationK8s: 'ConfigMap {{namespace}}/loki-config → loki.yaml', compactorNote: 'Retention is enforced by the Loki compactor; MinIO only stores the log objects.', - liveTitle: 'Full live loki.yaml', - liveHint: - 'Read from the deployment target with storage credentials masked; retention days below are edited separately.', - view: 'View Full Config', - refresh: 'Refresh', + liveHint: 'Read from the deployment target with storage credentials masked.', + daysLabel: 'Retention days', + daysUnit: 'days', + daysConvert: '= {{hours}} hours', + daysHint: 'Just enter a number — no YAML needed. Range 1–3650 days.', + daysInteger: 'Enter a whole number of days', + daysRange: 'Days must be between 1 and 3650', + costWarning: 'Keeping {{days}} days grows object-storage usage linearly with retention.', loading: 'Loading…', - templateNote: 'Generated by the deployment template', - yamlLabel: 'Retention Config YAML', - yamlHint: 'Edit retention_period, e.g. 90d / 2160h; only this setting is accepted.', - applyHintRunning: - 'Save only updates the deployment record; Save & Apply redeploys Loki and queries may be briefly interrupted.', - applyHintIdle: 'Takes effect on the next deployment.', + liveToggle: 'Show the full live loki.yaml (read-only)', + applyHintRunning: 'Save & Apply redeploys Loki; log queries may be briefly interrupted (~1 min).', + applyHintIdle: 'Saves and deploys this Loki instance immediately.', cancel: 'Cancel', - saveOnly: 'Save Only', saveApply: 'Save & Apply', processing: 'Processing…', savedApplied: 'Saved {{days}} days — Loki is being redeployed', diff --git a/frontend/src/i18n/locales/zh/analysis.ts b/frontend/src/i18n/locales/zh/analysis.ts index d823d70..4d57f1f 100644 --- a/frontend/src/i18n/locales/zh/analysis.ts +++ b/frontend/src/i18n/locales/zh/analysis.ts @@ -13,6 +13,9 @@ export default { filter: { source: '数据源', sourceEmpty: '暂无数据源', + namespace: '命名空间', + namespaceEmpty: '正在加载命名空间…', + namespaceAll: '全部命名空间（较慢）', refresh: '刷新', }, stats: { diff --git a/frontend/src/i18n/locales/zh/deployments.ts b/frontend/src/i18n/locales/zh/deployments.ts index d8b9876..1f5caff 100644 --- a/frontend/src/i18n/locales/zh/deployments.ts +++ b/frontend/src/i18n/locales/zh/deployments.ts @@ -88,7 +88,7 @@ export default { storageSizeMin: '阿里云 ESSD 最小 20Gi，请填 20Gi 或以上', }, retention: { - title: '编辑 Loki 配置 · {{name}}', + title: '编辑保留策略 · {{name}}', locationLabel: '配置位置：', locationHostBinary: '/etc/loki/loki.yaml → limits_config.retention_period', locationHostDocker: '主机 /etc/lokistack/loki-{{id}}/loki.yaml → limits_config.retention_period', @@ -96,16 +96,18 @@ export default { compactorNote: '保留策略由 Loki compactor 执行；MinIO 只负责存储日志对象。', liveTitle: '运行中的完整 loki.yaml', liveHint: '从部署目标读取，存储凭证已遮盖；下方保留天数仍单独编辑。', - view: '查看完整配置', - refresh: '刷新配置', + daysLabel: '保留天数', + daysUnit: '天', + daysConvert: '= {{hours}} 小时', + daysHint: '填一个数字即可，无需写 YAML。范围 1–3650 天。', + daysInteger: '请填写整数天数', + daysRange: '天数需在 1–3650 之间', + costWarning: '保留 {{days}} 天会让对象存储占用随天数线性增长；清空历史日志请谨慎。', loading: '读取中…', - templateNote: '由部署模板生成', - yamlLabel: '保留配置 YAML', - yamlHint: '编辑 retention_period，如 90d / 2160h；仅支持这一项。', - applyHintRunning: '仅保存会更新部署记录；保存并应用会重新部署 Loki，查询可能短暂中断。', - applyHintIdle: '保存后在下次部署时生效。', + liveToggle: '查看运行中的完整 loki.yaml（只读）', + applyHintRunning: '保存并应用会重新部署 Loki，查询可能短暂中断（约 1 分钟）。', + applyHintIdle: '保存后会立即部署该 Loki 实例。', cancel: '取消', - saveOnly: '仅保存', saveApply: '保存并应用', processing: '处理中…', savedApplied: '已保存 {{days}} 天，Loki 正在重新部署', diff --git a/frontend/src/pages/analysis/hooks/useAnalysisMetrics.ts b/frontend/src/pages/analysis/hooks/useAnalysisMetrics.ts index 120d415..c9bd66a 100644 --- a/frontend/src/pages/analysis/hooks/useAnalysisMetrics.ts +++ b/frontend/src/pages/analysis/hooks/useAnalysisMetrics.ts @@ -95,16 +95,40 @@ export function useAnalysisMetrics({ sourceId, start, end, namespace }: Analysis const range = { sourceId, start: Number(ns(from)), end: Number(ns(to)) } const fetchMetrics = (query: string) => logsApi.metrics({ ...range, query }, { signal: controller.signal }) + // 遇到 429（并发重查询已满）等一小会儿重试一次，避免分析页 5 条并发时 + // 部分卡片直接报"当前有较多日志量统计在进行"。 + const fetchRetry = async (query: string) => { + try { + return await fetchMetrics(query) + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + if (!msg.includes('较多日志量统计') && !msg.includes('429')) throw e + await new Promise(resolve => setTimeout(resolve, 1500)) + return await fetchMetrics(query) + } + } + + // 分批（每批 2 条）而不是一次性 5 条并发： + // Loki 单实例被 5 条 count_over_time 同时打会互相拖慢， + // 而且会撞后端并发护栏。分批后总耗时接近，但不会互相踩。 + const runInBatches = async (tasks: Array<() => Promise>, size = 2): Promise => { + const out: T[] = [] + for (let index = 0; index < tasks.length; index += size) { + out.push(...await Promise.all(tasks.slice(index, index + size).map(task => task()))) + } + return out + } + setLoading(true) setError('') void (async () => { try { - const [totalSeries, errorSeries, warnSeries, byAppSeries, errByAppSeries] = await Promise.all([ - fetchMetrics(`sum(count_over_time(${selector}[$step]))`), - fetchMetrics(`sum(count_over_time(${selector} |= "error"[$step]))`), - fetchMetrics(`sum(count_over_time(${selector} |= "warn"[$step]))`), - fetchMetrics(`sum by (${appLabel}, namespace) (count_over_time(${selector}[$step]))`), - fetchMetrics(`sum by (${appLabel}, namespace) (count_over_time(${selector} |= "error"[$step]))`), + const [totalSeries, errorSeries, warnSeries, byAppSeries, errByAppSeries] = await runInBatches([ + () => fetchRetry(`sum(count_over_time(${selector}[$step]))`), + () => fetchRetry(`sum(count_over_time(${selector} |= "error"[$step]))`), + () => fetchRetry(`sum(count_over_time(${selector} |= "warn"[$step]))`), + () => fetchRetry(`sum by (${appLabel}, namespace) (count_over_time(${selector}[$step]))`), + () => fetchRetry(`sum by (${appLabel}, namespace) (count_over_time(${selector} |= "error"[$step]))`), ]) if (version !== versionRef.current) return diff --git a/frontend/src/pages/analysis/index.tsx b/frontend/src/pages/analysis/index.tsx index d7e4826..d509788 100644 --- a/frontend/src/pages/analysis/index.tsx +++ b/frontend/src/pages/analysis/index.tsx @@ -25,6 +25,9 @@ import { IngressAnalysisSection } from './components/IngressAnalysisSection' const toLocal = (date: Date) => new Date(date.getTime() - date.getTimezoneOffset() * 60000).toISOString().slice(0, 19) +/** "全部命名空间"哨兵值：对应后端 {namespace=~".+"}，代价高，仅作显式选项 */ +const ALL_NAMESPACES = '__all__' + /** * 日志分析页(admin 专用):SLS 风格仪表盘 — 统计卡 + 趋势折线 + 命名空间饼图 + * 应用流图 + 应用排行表;数据来自 5 条 LogQL 指标查询(POST /logs/metrics)。 @@ -39,6 +42,16 @@ const AnalysisPage = memo(function AnalysisPage() { const [start, setStart] = useState(() => toLocal(new Date(Date.now() - 3600000))) const [end, setEnd] = useState(() => toLocal(new Date())) + /** + * 命名空间筛选。 + * 之前这里写死 namespace: ''，等价于 {namespace=~".+"} —— 一次分析就扫**全集群** + * （实测约 370 GB/天），5 条 count_over_time 跑 30 秒都出不来，页面永远转圈。 + * 现在默认只查业务命名空间，用户可下拉切换；"全部命名空间"作为显式选项并列提示较慢。 + */ + const [namespaces, setNamespaces] = useState([]) + const [namespace, setNamespace] = useState(() => localStorage.getItem('analysisNamespace') ?? '') + const [namespaceReady, setNamespaceReady] = useState(false) + useEffect(() => { logsApi.sources() .then(result => { @@ -52,9 +65,32 @@ const AnalysisPage = memo(function AnalysisPage() { .catch(() => setSources([])) }, []) - const { data, loading, error, appLabel, refresh } = useAnalysisMetrics({ sourceId, start, end, namespace: '' }) + // 拉取可用命名空间；默认优先业务命名空间 astralx，其次列表第一个 + useEffect(() => { + if (!sourceId) return + let cancelled = false + logsApi.labelValues(sourceId, 'namespace') + .then(list => { + if (cancelled) return + const values = (list || []).filter(Boolean).sort() + setNamespaces(values) + setNamespace(current => { + if (current && (current === ALL_NAMESPACES || values.includes(current))) return current + const saved = localStorage.getItem('analysisNamespace') + if (saved && (saved === ALL_NAMESPACES || values.includes(saved))) return saved + return values.includes('astralx') ? 'astralx' : values[0] ?? '' + }) + }) + .catch(() => { if (!cancelled) setNamespaces([]) }) + .finally(() => { if (!cancelled) setNamespaceReady(true) }) + return () => { cancelled = true } + }, [sourceId]) + + // hook 内部把空串当作 {namespace=~".+"}（全部），这里做哨兵转换 + const scopedNamespace = namespace === ALL_NAMESPACES ? '' : namespace + const { data, loading, error, appLabel, refresh } = useAnalysisMetrics({ sourceId, start, end, namespace: scopedNamespace }) // 每日统计固定近 7 天,不随上方时间范围变化;错误就地展示,不影响主区 - const daily = useDailyVolume({ sourceId, namespace: '', appLabel }) + const daily = useDailyVolume({ sourceId, namespace: scopedNamespace, appLabel }) // 非 admin:菜单不可见,直访时给明确提示(后端 AdminAuth 403 兜底) if (!isAdmin) { @@ -92,10 +128,28 @@ const AnalysisPage = memo(function AnalysisPage() { ))} + { setStart(nextStart); setEnd(nextEnd) }} /> - {sources.length === 0 ? ( + {!namespaceReady && sourceId ? ( + /* 命名空间列表未就绪先等待：否则会先用空串发起一次"全集群"查询，非常慢 */ + + ) : sources.length === 0 ? ( d` 生成（后端只认这一项）。 + * 2. 完整运行配置改为**默认折叠、展开即自动加载**，去掉多余的「查看完整配置」按钮。 + * 3. 按钮只保留 **取消 / 保存并应用** —— 去掉「仅保存」： + * 它只改部署记录不重载 Loki，容易造成「记录 7 天、运行中 24h」的配置漂移。 + * 4. 删掉硬编码的 `retention_delete_delay: 2h` 提示块（实际已改为 10m，会误导）。 + */ +// 注意：不要用 z.coerce.number() —— 它会把表单输入类型推成 unknown， +// 与 react-hook-form 的 useForm 泛型冲突（TS2322）。 +// 这里用原生 number 校验，输入框里做字符串↔数字转换。 +const schema = z.object({ + days: z + .number({ error: i18n.t('deployments:retention.daysInteger') }) + .int(i18n.t('deployments:retention.daysInteger')) + .min(1, i18n.t('deployments:retention.daysRange')) + .max(3650, i18n.t('deployments:retention.daysRange')), +}) type Values = z.infer +const PRESETS = [1, 7, 14, 30, 90] + interface Props { deployment: LokiDeployment | null onClose: () => void @@ -35,19 +57,19 @@ export const RetentionDialog = memo(function RetentionDialog({ deployment, onClo const [configError, setConfigError] = useState('') const form = useForm({ resolver: zodResolver(schema), - defaultValues: { yaml: 'limits_config:\n retention_period: 30d\n' }, + defaultValues: { days: 30 }, }) + const days = form.watch('days') useEffect(() => { - if (deployment) { - form.reset({ yaml: `limits_config:\n retention_period: ${deployment.retentionDays || 30}d\n` }) - } + if (deployment) form.reset({ days: deployment.retentionDays || 30 }) setLiveConfig(null) setConfigError('') }, [deployment, form]) + /** 展开「运行中的完整配置」时才去读，避免一打开弹框就打后端 */ const loadConfig = async () => { - if (!deployment) return + if (!deployment || liveConfig || configLoading) return setConfigLoading(true) setConfigError('') try { @@ -59,14 +81,16 @@ export const RetentionDialog = memo(function RetentionDialog({ deployment, onClo } } - const save = (apply: boolean) => form.handleSubmit(async ({ yaml }) => { + const submit = form.handleSubmit(async ({ days: value }) => { if (!deployment) return setSaving(true) try { - const result = await deploymentApi.updateRetention(deployment.id, yaml, apply) + // 后端只认 limits_config.retention_period 这一项，YAML 由前端拼，用户不用写 + const yaml = `limits_config:\n retention_period: ${value}d\n` + const result = await deploymentApi.updateRetention(deployment.id, yaml, true) onSaved() onClose() - if (apply) toast.info(t('retention.savedApplied', { days: result.retentionDays })) + toast.info(t('retention.savedApplied', { days: result.retentionDays })) } catch (err) { toast.error(err instanceof Error ? err.message : t('retention.errSave')) onSaved() @@ -77,63 +101,99 @@ export const RetentionDialog = memo(function RetentionDialog({ deployment, onClo return ( { if (!open && !saving) onClose() }}> - + {deployment ? t('retention.title', { name: deployment.name }) : ''} + + {t('retention.locationLabel')}{deployment ? configLocation(deployment, t) : ''} + -
-

{t('retention.locationLabel')}{deployment ? configLocation(deployment, t) : ''}

-

{t('retention.compactorNote')}

-
-
-
-
-

{t('retention.liveTitle')}

-

{t('retention.liveHint')}

-
- -
- {configError &&

{configError}

} - {liveConfig && ( -
-

{liveConfig.location}

- 
{liveConfig.yaml}
-
- )} -
-
-

{t('retention.templateNote')}

- 
{'compactor:\n  retention_enabled: true\n  retention_delete_delay: 2h'}
-
+
- + ( - {t('retention.yamlLabel')} - -