=== 一、现象 ===
- /logs 页「正在统计整个时间范围的日志量」24 小时查询转圈 5 分钟不出结果
- /analysis 页默认 1 小时也一直「加载中...」
- 期间 Loki 被 OOMKilled(2Gi) → Pod not Ready → Service 无端点 → 全站查询 502
=== 二、根因(实测数据) ===
同一 query {namespace="astralx", app="cex-ipublic-gateway"}:
24h count_over_time[15m] step=15m → 53.7s 后连接断开(算不完)
24h count_over_time[1h] step=1h → 60.1s 失败
24h index/stats(只读索引) → 1.3s → bytes 44.9GB / entries 3.33亿 / chunks 2474
24h index/volume_range step=1h → 111ms → 25 个点
1h index/volume_range step=5m → 115ms → 13 个点
原因:count_over_time 必须**下载并解码所有 chunk**;volume_range 只读 TSDB 索引里的
chunk 元数据(字节数),根本不碰日志内容。该 app 一天就有 3.33 亿条 / 44.9 GB。
=== 三、改造 ===
1. LokiHistogram 改用官方 /loki/api/v1/index/volume_range(带 step)
- 返回单位是「字节」,bucket 带 unit=bytes,前端格式化为 KB/MB/GB
- 老版本 Loki 没有该端点时自动回退 count_over_time(unit=entries)
2. 查询护栏(防止单条查询拖垮单实例 Loki)
- 统计范围上限 7 天,超出直接 400「统计范围过大(27.0 天),最多支持 7 天」
- 重查询单独 30s 超时(原来 120s)
- 全局并发重查询上限 2,超出立刻 429「请稍后重试」(不排队)
- /logs/metrics(analysis 页)共用同一套护栏
3. 前端
- 发起新查询前 abort 上一次请求(原来只用版本号丢弃结果,旧请求仍在后端跑)
- histogram/query 支持 signal;超时 40s/60s(原 125s)
- 错误不再统一显示「日志量统计失败」,按 429/400/502/超时给不同文案
- 时间线按 unit 格式化(字节 → GB),i18n 去掉「条」
4. Loki 侧官方优化(此前 limits_config 只有 retention_period)
- query_range: align_queries_with_step + cache_results + embedded_cache 100MB
- limits_config: max_query_parallelism 8(默认 32,单实例过高)
- 已用 `loki -verify-config` 校验通过(config is valid)
=== 四、实测(平台 API) ===
1 分钟 0.02s 2 桶
1 小时 0.05s 61 桶 3.8 GB
6 小时 0.04s 73 桶 12.3 GB
24 小时 0.04s 97 桶 41.8 GB ← 改造前 53.7s 失败
7 天 0.05s 37 桶 58.2 GB
27 天 0.01s 400「最多支持 7 天」
镜像:lghost/lokistack:latest + v0.1 = sha256:9fa1c4597533
354 lines
11 KiB
Go
354 lines
11 KiB
Go
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"lokistack/dto"
|
|
"lokistack/models"
|
|
"lokistack/services"
|
|
"lokistack/utils"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
func (ctrl *LogsController) Context(c *gin.Context) {
|
|
var req dto.LogContextRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
src := services.GetLogSource(req.SourceID)
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
ts, err := strconv.ParseInt(req.TimestampNs, 10, 64)
|
|
if err != nil || ts < 1 {
|
|
utils.BadRequest(c, "时间戳无效")
|
|
return
|
|
}
|
|
selector, err := services.ContextLogSelector(req.Labels)
|
|
if err != nil {
|
|
utils.BadRequest(c, err.Error())
|
|
return
|
|
}
|
|
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", selector)
|
|
if err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
span := int64(5 * time.Minute)
|
|
before, err := services.QueryLokiLogs(c.Request.Context(), src, query, ts-span, ts-1, 5, "backward", "")
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
after, err := services.QueryLokiLogs(c.Request.Context(), src, query, ts+1, ts+span, 5, "forward", "")
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, gin.H{"before": before.Entries, "after": after.Entries})
|
|
}
|
|
|
|
// LogsController 日志查询代理 + 数据源管理
|
|
type LogsController struct{}
|
|
|
|
func NewLogsController() *LogsController {
|
|
return &LogsController{}
|
|
}
|
|
|
|
// ==================== 数据源 ====================
|
|
|
|
func (ctrl *LogsController) ListSources(c *gin.Context) {
|
|
if c.GetString("role") == "admin" {
|
|
services.EnsureLogSourcesForRunningDeployments()
|
|
}
|
|
list := services.ListVisibleLogSources(c.GetUint("userId"), c.GetString("role") == "admin")
|
|
utils.Success(c, utils.PageData{List: list, Total: int64(len(list)), Page: 1, PageSize: 100})
|
|
}
|
|
|
|
func (ctrl *LogsController) CreateSource(c *gin.Context) {
|
|
var req dto.LogSourceRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
if req.Name == "" || req.TargetType == "" {
|
|
utils.BadRequest(c, "名称与类型必填")
|
|
return
|
|
}
|
|
if err := services.ValidateSourceAccess(req.Access); err != nil {
|
|
utils.BadRequest(c, err.Error())
|
|
return
|
|
}
|
|
src := &models.LogSource{
|
|
Name: req.Name, TargetType: req.TargetType,
|
|
Namespace: req.Namespace, Service: req.Service, Port: req.Port,
|
|
URL: req.URL, AuthType: req.AuthType, BasicUser: req.BasicUser,
|
|
BasicPass: req.BasicPass, BearerToken: req.BearerToken, Headers: req.Headers, SkipTLSVerify: req.SkipTLSVerify, IsDefault: req.IsDefault, Access: req.Access,
|
|
}
|
|
if req.TargetType == "cluster-svc" {
|
|
if req.ClusterID > 0 {
|
|
id := req.ClusterID
|
|
src.ClusterID = &id
|
|
} else {
|
|
utils.BadRequest(c, "cluster-svc 类型必须选择集群")
|
|
return
|
|
}
|
|
}
|
|
created, err := services.CreateLogSource(src)
|
|
if err != nil {
|
|
utils.Error(c, 400, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, created.Sanitized())
|
|
}
|
|
|
|
func (ctrl *LogsController) UpdateSource(c *gin.Context) {
|
|
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
|
if err != nil {
|
|
utils.BadRequest(c, "ID 无效")
|
|
return
|
|
}
|
|
var updates map[string]interface{}
|
|
if !utils.BindAndValidate(c, &updates) {
|
|
return
|
|
}
|
|
if src := services.GetLogSource(uint(id)); src != nil && src.Managed && updates["isDefault"] == nil {
|
|
utils.BadRequest(c, "平台部署的数据源由系统管理,不可编辑")
|
|
return
|
|
}
|
|
if err := services.UpdateLogSource(uint(id), updates); err != nil {
|
|
utils.Error(c, 400, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, gin.H{"message": "updated"})
|
|
}
|
|
|
|
func (ctrl *LogsController) DeleteSource(c *gin.Context) {
|
|
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
|
if err != nil {
|
|
utils.BadRequest(c, "ID 无效")
|
|
return
|
|
}
|
|
if src := services.GetLogSource(uint(id)); src != nil && src.Managed {
|
|
utils.BadRequest(c, "平台部署的数据源随部署管理,请先删除对应部署")
|
|
return
|
|
}
|
|
if err := services.DeleteLogSource(uint(id)); err != nil {
|
|
utils.Error(c, 400, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, gin.H{"message": "deleted"})
|
|
}
|
|
|
|
// TestSource 数据源连通性测试(按 id 或表单参数)
|
|
func (ctrl *LogsController) TestSource(c *gin.Context) {
|
|
var req dto.LogSourceRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
var src *models.LogSource
|
|
if req.ID > 0 {
|
|
src = services.GetLogSource(req.ID)
|
|
if src == nil {
|
|
utils.NotFound(c, "数据源不存在")
|
|
return
|
|
}
|
|
} else {
|
|
src = &models.LogSource{
|
|
TargetType: req.TargetType, Namespace: req.Namespace, Service: req.Service,
|
|
Port: req.Port, URL: req.URL, AuthType: req.AuthType,
|
|
BasicUser: req.BasicUser, BasicPass: req.BasicPass, BearerToken: req.BearerToken, Headers: req.Headers, SkipTLSVerify: req.SkipTLSVerify,
|
|
}
|
|
if req.TargetType == "cluster-svc" && req.ClusterID > 0 {
|
|
id := req.ClusterID
|
|
src.ClusterID = &id
|
|
}
|
|
}
|
|
utils.Success(c, services.TestLogSource(c.Request.Context(), src))
|
|
}
|
|
|
|
// ==================== 查询代理 ====================
|
|
|
|
func (ctrl *LogsController) Query(c *gin.Context) {
|
|
var req dto.LogQueryRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
src := services.GetLogSource(req.SourceID)
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
|
|
if err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
result, err := services.QueryLokiLogs(c.Request.Context(), src, query,
|
|
req.Start, req.End, req.Limit, req.Direction, req.Cursor)
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, result)
|
|
}
|
|
|
|
func (ctrl *LogsController) Labels(c *gin.Context) {
|
|
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
|
|
src := services.GetLogSource(uint(id))
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
if _, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", "{}"); err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
start, _ := strconv.ParseInt(c.Query("start"), 10, 64)
|
|
end, _ := strconv.ParseInt(c.Query("end"), 10, 64)
|
|
labels, err := services.LokiLabels(c.Request.Context(), src, start, end)
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, labels)
|
|
}
|
|
|
|
func (ctrl *LogsController) LabelValues(c *gin.Context) {
|
|
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
|
|
src := services.GetLogSource(uint(id))
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
start, _ := strconv.ParseInt(c.Query("start"), 10, 64)
|
|
end, _ := strconv.ParseInt(c.Query("end"), 10, 64)
|
|
requestedQuery := c.Query("query")
|
|
if strings.TrimSpace(requestedQuery) == "" {
|
|
requestedQuery = "{}"
|
|
}
|
|
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", requestedQuery)
|
|
if err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
values, err := services.LokiLabelValues(c.Request.Context(), src,
|
|
c.Param("name"), query, start, end)
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, values)
|
|
}
|
|
|
|
// MyScope 返回当前登录用户在某数据源上的授权范围。
|
|
//
|
|
// 快捷 LogQL 用它做两件事:
|
|
// 1. 把授权策略里配置的 namespace 作为默认选中值(管理员配了 default/astralx 后开箱即用)
|
|
// 2. 把该值合并进 namespace 候选列表 —— 否则像 default 这种「策略里有、Loki 里没日志」
|
|
// 的命名空间不会出现在下拉里,用户会以为策略没生效
|
|
func (ctrl *LogsController) MyScope(c *gin.Context) {
|
|
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
|
|
src := services.GetLogSource(uint(id))
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
if c.GetString("role") == "admin" {
|
|
// 管理员的查询范围不受策略限制(unrestricted=true),
|
|
// 但仍返回策略里的 labels 作为快捷 LogQL 的默认选中值——
|
|
// 否则管理员在小黑框里看到的下拉永远没有策略配的那个 namespace,
|
|
// 会误以为「默认策略没生效」。
|
|
labels := services.UserAccessLabels(src, c.GetUint("userId"))
|
|
if labels == nil {
|
|
labels = map[string]string{}
|
|
}
|
|
utils.Success(c, gin.H{"labels": labels, "unrestricted": true})
|
|
return
|
|
}
|
|
labels := services.UserAccessLabels(src, c.GetUint("userId"))
|
|
if labels == nil {
|
|
utils.Forbidden(c, "无权查询此数据源")
|
|
return
|
|
}
|
|
utils.Success(c, gin.H{"labels": labels, "unrestricted": len(labels) == 0})
|
|
}
|
|
|
|
// Health Loki /ready 探活(查询页状态点)
|
|
func (ctrl *LogsController) Health(c *gin.Context) {
|
|
id, _ := strconv.ParseUint(c.Query("sourceId"), 10, 64)
|
|
src := services.GetLogSource(uint(id))
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
if _, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", "{}"); err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second)
|
|
defer cancel()
|
|
utils.Success(c, gin.H{"ready": services.LokiReady(ctx, src)})
|
|
}
|
|
|
|
func (ctrl *LogsController) Histogram(c *gin.Context) {
|
|
var req dto.LogQueryRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
src := services.GetLogSource(req.SourceID)
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
|
|
if err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
buckets, err := services.LokiHistogram(c.Request.Context(), src, query, req.Start, req.End)
|
|
if err != nil {
|
|
switch {
|
|
case errors.Is(err, services.ErrTooManyQueries):
|
|
// 429:并发重查询超限,让前端提示"稍后重试",而不是当成服务故障
|
|
utils.Error(c, http.StatusTooManyRequests, err.Error())
|
|
case strings.Contains(err.Error(), "统计范围过大"):
|
|
utils.Error(c, http.StatusBadRequest, err.Error())
|
|
default:
|
|
utils.Error(c, 502, err.Error())
|
|
}
|
|
return
|
|
}
|
|
utils.Success(c, buckets)
|
|
}
|
|
|
|
// Metrics 日志分析指标查询(仅 admin,路由层 AdminAuth 拦截)
|
|
func (ctrl *LogsController) Metrics(c *gin.Context) {
|
|
var req dto.LogMetricQueryRequest
|
|
if !utils.BindAndValidate(c, &req) {
|
|
return
|
|
}
|
|
src := services.GetLogSource(req.SourceID)
|
|
if src == nil {
|
|
utils.BadRequest(c, "数据源不存在")
|
|
return
|
|
}
|
|
query, err := services.AuthorizedLogQuery(src, c.GetUint("userId"), c.GetString("role") == "admin", req.Query)
|
|
if err != nil {
|
|
utils.Forbidden(c, err.Error())
|
|
return
|
|
}
|
|
series, err := services.LokiMetrics(c.Request.Context(), src, query, req.Start, req.End)
|
|
if err != nil {
|
|
utils.Error(c, 502, err.Error())
|
|
return
|
|
}
|
|
utils.Success(c, series)
|
|
}
|