LokiStack/backend/pkg/metrics/metrics.go

// Package metrics 定义 LokiStack 自身的 Prometheus 指标体系。
//
// 为什么单独成包：这些指标要同时被 repository（DB 落盘耗时）和
// services（业务计数）以及 middleware（HTTP）使用，
// 而 services 依赖 repository —— 指标定义必须放在最底层，否则循环依赖。
package metrics

import (
	"net/http"
	"runtime"
	"strconv"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/collectors"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	// HTTP 层
	HTTPRequestsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "lokistack_http_requests_total",
			Help: "HTTP 请求总数（按方法/路由模板/状态码）",
		},
		[]string{"method", "path", "status"},
	)
	HTTPRequestDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "lokistack_http_request_duration_seconds",
			Help:    "HTTP 请求耗时分布",
			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},
		},
		[]string{"method", "path"},
	)
	HTTPInFlight = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "lokistack_http_in_flight_requests",
			Help: "当前正在处理的请求数",
		},
	)

	// 存储层（JSON DB 是平台最大风险点：全量读内存、全量写盘）
	DBSizeBytes = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "lokistack_db_size_bytes",
			Help: "JSON 数据库文件大小",
		},
	)
	DBSaveDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "lokistack_db_save_duration_seconds",
			Help:    "数据库落盘耗时",
			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 3, 5, 10},
		},
	)
	DBSaveTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "lokistack_db_save_total",
			Help: "数据库保存次数（按结果）",
		},
		[]string{"result"},
	)
	DBRecordsTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "lokistack_db_records",
			Help: "各集合的记录数",
		},
		[]string{"collection"},
	)

	// 业务层
	DeploymentsTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "lokistack_deployments",
			Help: "Loki 部署数（按状态/运行模式）",
		},
		[]string{"status", "mode"},
	)
	CollectorsTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "lokistack_collectors",
			Help: "采集器数（按状态/类型/托管方式）",
		},
		[]string{"status", "type", "run_mode"},
	)

	// 后台任务层
	AlertEvaluationsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "lokistack_alert_evaluations_total",
			Help: "告警规则评估次数（按结果）",
		},
		[]string{"result"},
	)
	AlertEvaluationDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "lokistack_alert_evaluation_duration_seconds",
			Help:    "单轮告警评估耗时",
			Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 3, 5, 10, 30},
		},
	)
	LokiQueriesTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "lokistack_loki_queries_total",
			Help: "对后端 Loki 的查询次数（按结果）",
		},
		[]string{"result"},
	)

	BuildInfo = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "lokistack_build_info",
			Help: "构建信息（恒为 1，标签里带版本与 Go 版本）",
		},
		[]string{"version", "go_version"},
	)

	// registry 用独立实例，不用 prometheus.DefaultRegisterer。
	// 原因：client_golang 在 init() 里已经把 GoCollector / ProcessCollector
	// 注册进默认 registry，再用 MustRegister 注册同一批 collector 会
	// **运行时 panic: duplicate metrics collector registration attempted**
	// （编译期和单测都发现不了，只在进程启动时炸）。
	registry = prometheus.NewRegistry()

	once sync.Once
)

// Register 注册全部指标（幂等）。version 一般传 main.version。
func Register(version string) {
	once.Do(func() {
		registry.MustRegister(
			HTTPRequestsTotal, HTTPRequestDuration, HTTPInFlight,
			DBSizeBytes, DBSaveDuration, DBSaveTotal, DBRecordsTotal,
			DeploymentsTotal, CollectorsTotal,
			AlertEvaluationsTotal, AlertEvaluationDuration, LokiQueriesTotal,
			BuildInfo,
			collectors.NewGoCollector(),
			collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
		)
		BuildInfo.WithLabelValues(version, runtime.Version()).Set(1)
	})
}

// Handler 返回 Prometheus 抓取端点（读自有 registry，不含默认 registry 的重复项）。
func Handler() http.Handler {
	return promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
}

// ── 埋点辅助（各层直接调） ──

// ObserveHTTP 记录一次 HTTP 请求。pathLabel 必须是路由模板，不能用原始 URL。
func ObserveHTTP(method, pathLabel string, status int, seconds float64) {
	HTTPRequestsTotal.WithLabelValues(method, pathLabel, strconv.Itoa(status)).Inc()
	HTTPRequestDuration.WithLabelValues(method, pathLabel).Observe(seconds)
}

// IncInFlight / DecInFlight 在途请求计数。
func IncInFlight() { HTTPInFlight.Inc() }
func DecInFlight() { HTTPInFlight.Dec() }

// ObserveDBSave 记录一次落盘结果。
func ObserveDBSave(seconds float64, ok bool) {
	DBSaveDuration.Observe(seconds)
	DBSaveTotal.WithLabelValues(resultLabel(ok)).Inc()
}

// ObserveAlertEvaluation 记录一轮告警评估。
func ObserveAlertEvaluation(seconds float64, ok bool) {
	AlertEvaluationDuration.Observe(seconds)
	AlertEvaluationsTotal.WithLabelValues(resultLabel(ok)).Inc()
}

// ObserveLokiQuery 记录一次后端 Loki 查询。
func ObserveLokiQuery(ok bool) {
	LokiQueriesTotal.WithLabelValues(resultLabel(ok)).Inc()
}

func resultLabel(ok bool) string {
	if ok {
		return "success"
	}
	return "error"
}