Files
LokiStack/backend/controllers/logs.go
T
opsx de376a1284 perf(logs): 日志量统计改用 index/volume_range(快 ~1300 倍)+ 查询护栏 + 真实错误提示
=== 一、现象 ===
- /logs 页「正在统计整个时间范围的日志量」24 小时查询转圈 5 分钟不出结果
- /analysis 页默认 1 小时也一直「加载中...」
- 期间 Loki 被 OOMKilled(2Gi) → Pod not Ready → Service 无端点 → 全站查询 502

=== 二、根因(实测数据) ===
同一 query {namespace="astralx", app="cex-ipublic-gateway"}:
  24h count_over_time[15m] step=15m  → 53.7s 后连接断开(算不完)
  24h count_over_time[1h]  step=1h   → 60.1s 失败
  24h index/stats(只读索引)        → 1.3s  → bytes 44.9GB / entries 3.33亿 / chunks 2474
  24h index/volume_range step=1h     → 111ms → 25 个点
  1h  index/volume_range step=5m     → 115ms → 13 个点

原因:count_over_time 必须**下载并解码所有 chunk**;volume_range 只读 TSDB 索引里的
chunk 元数据(字节数),根本不碰日志内容。该 app 一天就有 3.33 亿条 / 44.9 GB。

=== 三、改造 ===
1. LokiHistogram 改用官方 /loki/api/v1/index/volume_range(带 step)
   - 返回单位是「字节」,bucket 带 unit=bytes,前端格式化为 KB/MB/GB
   - 老版本 Loki 没有该端点时自动回退 count_over_time(unit=entries)
2. 查询护栏(防止单条查询拖垮单实例 Loki)
   - 统计范围上限 7 天,超出直接 400「统计范围过大(27.0 天),最多支持 7 天」
   - 重查询单独 30s 超时(原来 120s)
   - 全局并发重查询上限 2,超出立刻 429「请稍后重试」(不排队)
   - /logs/metrics(analysis 页)共用同一套护栏
3. 前端
   - 发起新查询前 abort 上一次请求(原来只用版本号丢弃结果,旧请求仍在后端跑)
   - histogram/query 支持 signal;超时 40s/60s(原 125s)
   - 错误不再统一显示「日志量统计失败」,按 429/400/502/超时给不同文案
   - 时间线按 unit 格式化(字节 → GB),i18n 去掉「条」
4. Loki 侧官方优化(此前 limits_config 只有 retention_period)
   - query_range: align_queries_with_step + cache_results + embedded_cache 100MB
   - limits_config: max_query_parallelism 8(默认 32,单实例过高)
   - 已用 `loki -verify-config` 校验通过(config is valid)

=== 四、实测(平台 API) ===
  1 分钟  0.02s  2 桶
  1 小时  0.05s  61 桶  3.8 GB
  6 小时  0.04s  73 桶  12.3 GB
  24 小时 0.04s  97 桶  41.8 GB   ← 改造前 53.7s 失败
  7 天    0.05s  37 桶  58.2 GB
  27 天   0.01s  400「最多支持 7 天」

镜像:lghost/lokistack:latest + v0.1 = sha256:9fa1c4597533
2026-10-28 15:54:55 +00:00

354 lines
11 KiB
Go

package controllers
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"time"
"lokistack/dto"
"lokistack/models"
"lokistack/services"
"lokistack/utils"
"github.com/gin-gonic/gin"
)
func (ctrl *LogsController) Context(c *gin.Context) {
var req dto.LogContextRequest
if !utils.BindAndValidate(c, &req) {
return
}
src := services.GetLogSource(req.SourceID)
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
ts, err := strconv.ParseInt(req.TimestampNs, 10, 64)
if err != nil || ts < 1 {
utils.BadRequest(c, "时间戳无效")
return
}
selector, err := services.ContextLogSelector(req.Labels)
if err != nil {
utils.BadRequest(c, err.Error())
return
}
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", selector)
if err != nil {
utils.Forbidden(c, err.Error())
return
}
span := int64(5 * time.Minute)
before, err := services.QueryLokiLogs(c.Request.Context(), src, query, ts-span, ts-1, 5, "backward", "")
if err != nil {
utils.Error(c, 502, err.Error())
return
}
after, err := services.QueryLokiLogs(c.Request.Context(), src, query, ts+1, ts+span, 5, "forward", "")
if err != nil {
utils.Error(c, 502, err.Error())
return
}
utils.Success(c, gin.H{"before": before.Entries, "after": after.Entries})
}
// LogsController 日志查询代理 + 数据源管理
type LogsController struct{}
func NewLogsController() *LogsController {
return &LogsController{}
}
// ==================== 数据源 ====================
func (ctrl *LogsController) ListSources(c *gin.Context) {
if c.GetString("role") == "admin" {
services.EnsureLogSourcesForRunningDeployments()
}
list := services.ListVisibleLogSources(c.GetUint("userId"), c.GetString("role") == "admin")
utils.Success(c, utils.PageData{List: list, Total: int64(len(list)), Page: 1, PageSize: 100})
}
func (ctrl *LogsController) CreateSource(c *gin.Context) {
var req dto.LogSourceRequest
if !utils.BindAndValidate(c, &req) {
return
}
if req.Name == "" || req.TargetType == "" {
utils.BadRequest(c, "名称与类型必填")
return
}
if err := services.ValidateSourceAccess(req.Access); err != nil {
utils.BadRequest(c, err.Error())
return
}
src := &models.LogSource{
Name: req.Name, TargetType: req.TargetType,
Namespace: req.Namespace, Service: req.Service, Port: req.Port,
URL: req.URL, AuthType: req.AuthType, BasicUser: req.BasicUser,
BasicPass: req.BasicPass, BearerToken: req.BearerToken, Headers: req.Headers, SkipTLSVerify: req.SkipTLSVerify, IsDefault: req.IsDefault, Access: req.Access,
}
if req.TargetType == "cluster-svc" {
if req.ClusterID > 0 {
id := req.ClusterID
src.ClusterID = &id
} else {
utils.BadRequest(c, "cluster-svc 类型必须选择集群")
return
}
}
created, err := services.CreateLogSource(src)
if err != nil {
utils.Error(c, 400, err.Error())
return
}
utils.Success(c, created.Sanitized())
}
func (ctrl *LogsController) UpdateSource(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
utils.BadRequest(c, "ID 无效")
return
}
var updates map[string]interface{}
if !utils.BindAndValidate(c, &updates) {
return
}
if src := services.GetLogSource(uint(id)); src != nil && src.Managed && updates["isDefault"] == nil {
utils.BadRequest(c, "平台部署的数据源由系统管理,不可编辑")
return
}
if err := services.UpdateLogSource(uint(id), updates); err != nil {
utils.Error(c, 400, err.Error())
return
}
utils.Success(c, gin.H{"message": "updated"})
}
func (ctrl *LogsController) DeleteSource(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
utils.BadRequest(c, "ID 无效")
return
}
if src := services.GetLogSource(uint(id)); src != nil && src.Managed {
utils.BadRequest(c, "平台部署的数据源随部署管理,请先删除对应部署")
return
}
if err := services.DeleteLogSource(uint(id)); err != nil {
utils.Error(c, 400, err.Error())
return
}
utils.Success(c, gin.H{"message": "deleted"})
}
// TestSource 数据源连通性测试(按 id 或表单参数)
func (ctrl *LogsController) TestSource(c *gin.Context) {
var req dto.LogSourceRequest
if !utils.BindAndValidate(c, &req) {
return
}
var src *models.LogSource
if req.ID > 0 {
src = services.GetLogSource(req.ID)
if src == nil {
utils.NotFound(c, "数据源不存在")
return
}
} else {
src = &models.LogSource{
TargetType: req.TargetType, Namespace: req.Namespace, Service: req.Service,
Port: req.Port, URL: req.URL, AuthType: req.AuthType,
BasicUser: req.BasicUser, BasicPass: req.BasicPass, BearerToken: req.BearerToken, Headers: req.Headers, SkipTLSVerify: req.SkipTLSVerify,
}
if req.TargetType == "cluster-svc" && req.ClusterID > 0 {
id := req.ClusterID
src.ClusterID = &id
}
}
utils.Success(c, services.TestLogSource(c.Request.Context(), src))
}
// ==================== 查询代理 ====================
func (ctrl *LogsController) Query(c *gin.Context) {
var req dto.LogQueryRequest
if !utils.BindAndValidate(c, &req) {
return
}
src := services.GetLogSource(req.SourceID)
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
if err != nil {
utils.Forbidden(c, err.Error())
return
}
result, err := services.QueryLokiLogs(c.Request.Context(), src, query,
req.Start, req.End, req.Limit, req.Direction, req.Cursor)
if err != nil {
utils.Error(c, 502, err.Error())
return
}
utils.Success(c, result)
}
func (ctrl *LogsController) Labels(c *gin.Context) {
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
src := services.GetLogSource(uint(id))
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
if _, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", "{}"); err != nil {
utils.Forbidden(c, err.Error())
return
}
start, _ := strconv.ParseInt(c.Query("start"), 10, 64)
end, _ := strconv.ParseInt(c.Query("end"), 10, 64)
labels, err := services.LokiLabels(c.Request.Context(), src, start, end)
if err != nil {
utils.Error(c, 502, err.Error())
return
}
utils.Success(c, labels)
}
func (ctrl *LogsController) LabelValues(c *gin.Context) {
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
src := services.GetLogSource(uint(id))
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
start, _ := strconv.ParseInt(c.Query("start"), 10, 64)
end, _ := strconv.ParseInt(c.Query("end"), 10, 64)
requestedQuery := c.Query("query")
if strings.TrimSpace(requestedQuery) == "" {
requestedQuery = "{}"
}
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", requestedQuery)
if err != nil {
utils.Forbidden(c, err.Error())
return
}
values, err := services.LokiLabelValues(c.Request.Context(), src,
c.Param("name"), query, start, end)
if err != nil {
utils.Error(c, 502, err.Error())
return
}
utils.Success(c, values)
}
// MyScope 返回当前登录用户在某数据源上的授权范围。
//
// 快捷 LogQL 用它做两件事:
// 1. 把授权策略里配置的 namespace 作为默认选中值(管理员配了 default/astralx 后开箱即用)
// 2. 把该值合并进 namespace 候选列表 —— 否则像 default 这种「策略里有、Loki 里没日志」
// 的命名空间不会出现在下拉里,用户会以为策略没生效
func (ctrl *LogsController) MyScope(c *gin.Context) {
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
src := services.GetLogSource(uint(id))
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
if c.GetString("role") == "admin" {
// 管理员的查询范围不受策略限制(unrestricted=true),
// 但仍返回策略里的 labels 作为快捷 LogQL 的默认选中值——
// 否则管理员在小黑框里看到的下拉永远没有策略配的那个 namespace,
// 会误以为「默认策略没生效」。
labels := services.UserAccessLabels(src, c.GetUint("userId"))
if labels == nil {
labels = map[string]string{}
}
utils.Success(c, gin.H{"labels": labels, "unrestricted": true})
return
}
labels := services.UserAccessLabels(src, c.GetUint("userId"))
if labels == nil {
utils.Forbidden(c, "无权查询此数据源")
return
}
utils.Success(c, gin.H{"labels": labels, "unrestricted": len(labels) == 0})
}
// Health Loki /ready 探活(查询页状态点)
func (ctrl *LogsController) Health(c *gin.Context) {
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
src := services.GetLogSource(uint(id))
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
if _, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", "{}"); err != nil {
utils.Forbidden(c, err.Error())
return
}
ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second)
defer cancel()
utils.Success(c, gin.H{"ready": services.LokiReady(ctx, src)})
}
func (ctrl *LogsController) Histogram(c *gin.Context) {
var req dto.LogQueryRequest
if !utils.BindAndValidate(c, &req) {
return
}
src := services.GetLogSource(req.SourceID)
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
if err != nil {
utils.Forbidden(c, err.Error())
return
}
buckets, err := services.LokiHistogram(c.Request.Context(), src, query, req.Start, req.End)
if err != nil {
switch {
case errors.Is(err, services.ErrTooManyQueries):
// 429:并发重查询超限,让前端提示"稍后重试",而不是当成服务故障
utils.Error(c, http.StatusTooManyRequests, err.Error())
case strings.Contains(err.Error(), "统计范围过大"):
utils.Error(c, http.StatusBadRequest, err.Error())
default:
utils.Error(c, 502, err.Error())
}
return
}
utils.Success(c, buckets)
}
// Metrics 日志分析指标查询(仅 admin,路由层 AdminAuth 拦截)
func (ctrl *LogsController) Metrics(c *gin.Context) {
var req dto.LogMetricQueryRequest
if !utils.BindAndValidate(c, &req) {
return
}
src := services.GetLogSource(req.SourceID)
if src == nil {
utils.BadRequest(c, "数据源不存在")
return
}
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
if err != nil {
utils.Forbidden(c, err.Error())
return
}
series, err := services.LokiMetrics(c.Request.Context(), src, query, req.Start, req.End)
if err != nil {
utils.Error(c, 502, err.Error())
return
}
utils.Success(c, series)
}