logan
2d73661dd1
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application. Validated with go test ./..., npm run build, and npm run lint.
506 lines
14 KiB
Go
506 lines
14 KiB
Go
package services
|
||
|
||
// 日志查询代理 — 后端代表前端访问 Loki HTTP API。
|
||
// cluster-svc 型经 kube-apiserver service proxy(不依赖 ClusterIP 直通与集群 DNS);
|
||
// url 型直连。契约参照 aiops 日志中心(query/labels/values/histogram + cursor 翻页)。
|
||
import (
|
||
"context"
|
||
"crypto/tls"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"opsxloki/models"
|
||
"opsxloki/repository"
|
||
"k8s.io/client-go/rest"
|
||
)
|
||
|
||
const maxLogQueryLimit = 1000
|
||
const logQueryTimeout = 120 * time.Second
|
||
|
||
var lokiHTTPClients = [2]*http.Client{
|
||
{Timeout: logQueryTimeout, Transport: newLokiTransport(false)},
|
||
{Timeout: logQueryTimeout, Transport: newLokiTransport(true)},
|
||
}
|
||
|
||
func newLokiTransport(skipTLSVerify bool) *http.Transport {
|
||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: skipTLSVerify} //nolint:gosec // 数据源显式配置
|
||
return transport
|
||
}
|
||
|
||
// ==================== Loki HTTP 访问 ====================
|
||
|
||
// lokiProxyGET 对数据源执行 GET；cluster-svc 使用 client-go 的认证传输访问 API Server Service Proxy。
|
||
func lokiProxyGET(ctx context.Context, src *models.LogSource, path string, query url.Values) ([]byte, int, error) {
|
||
if src.TargetType == "cluster-svc" {
|
||
db := repository.GetDB()
|
||
db.RLock()
|
||
var cluster *models.Cluster
|
||
for i := range db.Clusters {
|
||
if src.ClusterID != nil && db.Clusters[i].ID == *src.ClusterID {
|
||
c := db.Clusters[i]
|
||
cluster = &c
|
||
break
|
||
}
|
||
}
|
||
db.RUnlock()
|
||
if cluster == nil {
|
||
return nil, 500, fmt.Errorf("数据源关联的集群不存在")
|
||
}
|
||
if !kubernetesNamespacePattern.MatchString(src.Namespace) || !kubernetesNamespacePattern.MatchString(src.Service) || src.Port < 1 || src.Port > 65535 {
|
||
return nil, 500, fmt.Errorf("Loki 数据源的命名空间、Service 或端口无效")
|
||
}
|
||
config, err := clusterRESTConfig(cluster, logQueryTimeout)
|
||
if err != nil {
|
||
return nil, 500, err
|
||
}
|
||
client, err := rest.HTTPClientFor(config)
|
||
if err != nil {
|
||
return nil, 500, fmt.Errorf("创建 Kubernetes API 客户端失败: %w", err)
|
||
}
|
||
full := strings.TrimRight(config.Host, "/") + fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy%s",
|
||
src.Namespace, src.Service, src.Port, path)
|
||
if qs := query.Encode(); qs != "" {
|
||
full += "?" + qs
|
||
}
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil)
|
||
if err != nil {
|
||
return nil, 500, err
|
||
}
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
return nil, 502, fmt.Errorf("Kubernetes API 代理查询失败: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
|
||
if err != nil {
|
||
return nil, 502, err
|
||
}
|
||
if resp.StatusCode >= 400 {
|
||
return body, resp.StatusCode, fmt.Errorf("Kubernetes API 代理返回 HTTP %d: %s", resp.StatusCode, truncateStr(string(body), 300))
|
||
}
|
||
return body, resp.StatusCode, nil
|
||
}
|
||
|
||
// url 直连
|
||
if src.URL == "" {
|
||
return nil, 500, fmt.Errorf("数据源未配置访问地址")
|
||
}
|
||
full := strings.TrimSuffix(src.URL, "/") + path
|
||
if qs := query.Encode(); qs != "" {
|
||
full += "?" + qs
|
||
}
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil)
|
||
if err != nil {
|
||
return nil, 500, err
|
||
}
|
||
applyLokiAuth(req, src)
|
||
client := lokiHTTPClients[0]
|
||
if src.SkipTLSVerify {
|
||
client = lokiHTTPClients[1]
|
||
}
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
return nil, 502, err
|
||
}
|
||
defer resp.Body.Close()
|
||
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
|
||
if err != nil {
|
||
return nil, 502, err
|
||
}
|
||
if resp.StatusCode >= 400 {
|
||
return body, resp.StatusCode, fmt.Errorf("Loki 返回 HTTP %d: %s", resp.StatusCode, truncateStr(string(body), 300))
|
||
}
|
||
return body, resp.StatusCode, nil
|
||
}
|
||
|
||
func applyLokiAuth(req *http.Request, src *models.LogSource) {
|
||
switch src.AuthType {
|
||
case "basic":
|
||
if pass, err := DecryptCred(src.BasicPass); err == nil {
|
||
req.Header.Set("Authorization", "Basic "+
|
||
base64.StdEncoding.EncodeToString([]byte(src.BasicUser+":"+pass)))
|
||
}
|
||
case "bearer":
|
||
if token, err := DecryptCred(src.BearerToken); err == nil {
|
||
req.Header.Set("Authorization", "Bearer "+token)
|
||
}
|
||
}
|
||
for name, encrypted := range src.Headers {
|
||
if validSourceHeader(name) {
|
||
if value, err := DecryptCred(encrypted); err == nil {
|
||
req.Header.Set(name, value)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func truncateStr(s string, n int) string {
|
||
if len(s) <= n {
|
||
return s
|
||
}
|
||
return s[:n] + "..."
|
||
}
|
||
|
||
// ==================== 查询实现(契约对齐 aiops) ====================
|
||
|
||
type LogEntryRow struct {
|
||
Timestamp string `json:"timestamp"`
|
||
TimestampNs string `json:"timestampNs"`
|
||
Level string `json:"level"`
|
||
Message string `json:"message"`
|
||
Labels map[string]string `json:"labels"`
|
||
}
|
||
|
||
type LogQueryResult struct {
|
||
Entries []LogEntryRow `json:"entries"`
|
||
HasMore bool `json:"hasMore"`
|
||
NextCursor string `json:"nextCursor"`
|
||
}
|
||
|
||
// ContextLogSelector uses stable stream identity labels. Loki can add per-entry
|
||
// labels such as detected_level to query results; matching those would hide
|
||
// adjacent lines whose detected level differs from the selected line.
|
||
func ContextLogSelector(labels map[string]string) (string, error) {
|
||
for key := range labels {
|
||
if !ValidLogLabelName(key) {
|
||
return "", fmt.Errorf("标签名无效")
|
||
}
|
||
}
|
||
keys := []string{"cluster", "namespace", "app", "pod", "container", "filename"}
|
||
if labels["pod"] == "" {
|
||
keys = append(keys, "job", "instance", "host")
|
||
}
|
||
parts := make([]string, 0, len(keys))
|
||
for _, key := range keys {
|
||
if value := labels[key]; value != "" {
|
||
encoded, _ := json.Marshal(value)
|
||
parts = append(parts, key+"="+string(encoded))
|
||
}
|
||
}
|
||
if len(parts) == 0 {
|
||
for key, value := range labels {
|
||
if value == "" || key == "detected_level" || key == "level" || strings.HasPrefix(key, "__") {
|
||
continue
|
||
}
|
||
encoded, _ := json.Marshal(value)
|
||
parts = append(parts, key+"="+string(encoded))
|
||
}
|
||
sort.Strings(parts)
|
||
}
|
||
if len(parts) == 0 {
|
||
return "", fmt.Errorf("缺少可用于定位日志流的标签")
|
||
}
|
||
return "{" + strings.Join(parts, ",") + "}", nil
|
||
}
|
||
|
||
// QueryLokiLogs 日志查询:flatten streams → 统一行;cursor(最后一条纳秒时间戳)向后翻页
|
||
func QueryLokiLogs(ctx context.Context, src *models.LogSource, query string, start, end int64, limit int, direction, cursor string) (*LogQueryResult, error) {
|
||
if limit <= 0 {
|
||
limit = 200
|
||
}
|
||
if limit > maxLogQueryLimit {
|
||
limit = maxLogQueryLimit
|
||
}
|
||
if direction == "" {
|
||
direction = "backward"
|
||
}
|
||
if end <= 0 {
|
||
end = time.Now().UnixNano()
|
||
}
|
||
if start <= 0 {
|
||
start = end - int64(time.Hour) // 默认最近 1 小时
|
||
}
|
||
if cursor != "" {
|
||
if c, err := strconv.ParseInt(cursor, 10, 64); err == nil {
|
||
if direction == "forward" {
|
||
start = c + 1
|
||
} else {
|
||
end = c - 1
|
||
}
|
||
}
|
||
}
|
||
|
||
q := url.Values{}
|
||
q.Set("query", query)
|
||
q.Set("start", strconv.FormatInt(start, 10))
|
||
q.Set("end", strconv.FormatInt(end, 10))
|
||
q.Set("limit", strconv.Itoa(limit))
|
||
q.Set("direction", direction)
|
||
|
||
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/query_range", q)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var resp struct {
|
||
Status string `json:"status"`
|
||
Data struct {
|
||
ResultType string `json:"resultType"`
|
||
Result []struct {
|
||
Stream map[string]string `json:"stream"`
|
||
Values [][2]string `json:"values"`
|
||
} `json:"result"`
|
||
} `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(body, &resp); err != nil {
|
||
return nil, fmt.Errorf("解析 Loki 响应失败: %w", err)
|
||
}
|
||
|
||
entries := make([]LogEntryRow, 0, limit)
|
||
var lastTs string
|
||
for _, stream := range resp.Data.Result {
|
||
for _, v := range stream.Values {
|
||
entries = append(entries, LogEntryRow{
|
||
Timestamp: formatLokiTimestamp(v[0]),
|
||
TimestampNs: v[0],
|
||
Level: detectLevel(stream.Stream, v[1]),
|
||
Message: v[1],
|
||
Labels: stream.Stream,
|
||
})
|
||
}
|
||
}
|
||
sort.SliceStable(entries, func(i, j int) bool {
|
||
a, _ := strconv.ParseInt(entries[i].TimestampNs, 10, 64)
|
||
b, _ := strconv.ParseInt(entries[j].TimestampNs, 10, 64)
|
||
if direction == "forward" {
|
||
return a < b
|
||
}
|
||
return a > b
|
||
})
|
||
if len(entries) > limit {
|
||
entries = entries[:limit]
|
||
}
|
||
if len(entries) > 0 {
|
||
lastTs = entries[len(entries)-1].TimestampNs
|
||
}
|
||
|
||
nextCursor := ""
|
||
if len(entries) >= limit && lastTs != "" {
|
||
if ts, err := strconv.ParseInt(strings.Split(lastTs, ".")[0], 10, 64); err == nil {
|
||
nextCursor = strconv.FormatInt(ts, 10)
|
||
}
|
||
}
|
||
return &LogQueryResult{Entries: entries, HasMore: nextCursor != "", NextCursor: nextCursor}, nil
|
||
}
|
||
|
||
// formatLokiTimestamp Loki 时间串(纳秒字符串/科学计数法)→ RFC3339
|
||
func formatLokiTimestamp(ts string) string {
|
||
if i := strings.IndexAny(ts, ".eE"); i > 0 {
|
||
ts = ts[:i]
|
||
}
|
||
n, err := strconv.ParseInt(ts, 10, 64)
|
||
if err != nil {
|
||
return ts
|
||
}
|
||
// 自适应纳秒/微秒/毫秒/秒
|
||
switch {
|
||
case n > 1e18:
|
||
return time.Unix(0, n).Format(time.RFC3339)
|
||
case n > 1e15:
|
||
return time.Unix(0, n*1e3).Format(time.RFC3339)
|
||
case n > 1e12:
|
||
return time.Unix(0, n*1e6).Format(time.RFC3339)
|
||
case n > 1e9:
|
||
return time.Unix(0, n*1e9).Format(time.RFC3339)
|
||
default:
|
||
return time.Unix(n, 0).Format(time.RFC3339)
|
||
}
|
||
}
|
||
|
||
// detectLevel 优先取 level 标签;否则按行内容推断
|
||
func detectLevel(labels map[string]string, line string) string {
|
||
if lv, ok := labels["level"]; ok && lv != "" {
|
||
return strings.ToLower(lv)
|
||
}
|
||
l := strings.ToLower(line)
|
||
for _, p := range []string{"error", "err", "fatal", "panic"} {
|
||
if strings.Contains(l, p) {
|
||
return "error"
|
||
}
|
||
}
|
||
if strings.Contains(l, "warn") {
|
||
return "warn"
|
||
}
|
||
return "info"
|
||
}
|
||
|
||
// LokiLabels 标签名列表
|
||
func LokiLabels(ctx context.Context, src *models.LogSource, start, end int64) ([]string, error) {
|
||
q := url.Values{}
|
||
if end > 0 {
|
||
q.Set("end", strconv.FormatInt(end, 10))
|
||
}
|
||
if start > 0 {
|
||
q.Set("start", strconv.FormatInt(start, 10))
|
||
}
|
||
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/labels", q)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var resp struct {
|
||
Data []string `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(body, &resp); err != nil {
|
||
return nil, err
|
||
}
|
||
return resp.Data, nil
|
||
}
|
||
|
||
// LokiLabelValues 标签值列表(query 只取 {...} selector 部分)
|
||
func LokiLabelValues(ctx context.Context, src *models.LogSource, name, selector string, start, end int64) ([]string, error) {
|
||
q := url.Values{}
|
||
if s := extractStreamSelector(selector); s != "" && s != "{}" {
|
||
q.Set("query", s)
|
||
}
|
||
if end > 0 {
|
||
q.Set("end", strconv.FormatInt(end, 10))
|
||
}
|
||
if start > 0 {
|
||
q.Set("start", strconv.FormatInt(start, 10))
|
||
}
|
||
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/label/"+url.PathEscape(name)+"/values", q)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var resp struct {
|
||
Data []string `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(body, &resp); err != nil {
|
||
return nil, err
|
||
}
|
||
return resp.Data, nil
|
||
}
|
||
|
||
// extractStreamSelector 从 LogQL 中提取 {...} 部分(labels API 仅接受 stream selector)
|
||
func extractStreamSelector(query string) string {
|
||
i := strings.Index(query, "{")
|
||
if i < 0 {
|
||
return ""
|
||
}
|
||
j := strings.Index(query[i:], "}")
|
||
if j < 0 {
|
||
return ""
|
||
}
|
||
return query[i : i+j+1]
|
||
}
|
||
|
||
// LokiHistogram 日志量直方图:服务端包 sum(count_over_time(q[step]))
|
||
func LokiHistogram(ctx context.Context, src *models.LogSource, query string, start, end int64) ([]map[string]interface{}, error) {
|
||
if end <= 0 {
|
||
end = time.Now().UnixNano()
|
||
}
|
||
if start <= 0 {
|
||
start = end - int64(time.Hour)
|
||
}
|
||
span := end - start
|
||
step := pickHistogramStep(span)
|
||
wrapped := fmt.Sprintf("sum(count_over_time(%s[%s]))", query, step)
|
||
|
||
q := url.Values{}
|
||
q.Set("query", wrapped)
|
||
q.Set("start", strconv.FormatInt(start, 10))
|
||
q.Set("end", strconv.FormatInt(end, 10))
|
||
q.Set("step", step)
|
||
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/query_range", q)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var resp struct {
|
||
Data struct {
|
||
Result []struct {
|
||
Values [][2]interface{} `json:"values"`
|
||
} `json:"result"`
|
||
} `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(body, &resp); err != nil {
|
||
return nil, err
|
||
}
|
||
buckets := []map[string]interface{}{}
|
||
if len(resp.Data.Result) > 0 {
|
||
for _, v := range resp.Data.Result[0].Values {
|
||
// metric API 的 ts 为秒级浮点数(如 1.79e+09),query 日志的才是纳秒字符串
|
||
tsf, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[0]), 64)
|
||
cnt, _ := strconv.ParseFloat(fmt.Sprintf("%v", v[1]), 64)
|
||
buckets = append(buckets, map[string]interface{}{
|
||
"timestamp": time.Unix(int64(tsf), 0).Format(time.RFC3339),
|
||
"count": int64(cnt),
|
||
})
|
||
}
|
||
}
|
||
return buckets, nil
|
||
}
|
||
|
||
// pickHistogramStep 按时间跨度自适应步长(aiops 同款)
|
||
func pickHistogramStep(spanNano int64) string {
|
||
span := time.Duration(spanNano)
|
||
switch {
|
||
case span <= 15*time.Minute:
|
||
return "30s"
|
||
case span <= time.Hour:
|
||
return "1m"
|
||
case span <= 6*time.Hour:
|
||
return "5m"
|
||
case span <= 24*time.Hour:
|
||
return "15m"
|
||
case span <= 7*24*time.Hour:
|
||
return "1h"
|
||
default:
|
||
return "6h"
|
||
}
|
||
}
|
||
|
||
// DeploymentReady 部署级就绪探测:按部署名匹配自动注册的数据源,经其访问 Loki /ready
|
||
func DeploymentReady(ctx context.Context, deploymentName string) (bool, string) {
|
||
if deploymentName == "" {
|
||
return false, ""
|
||
}
|
||
db := repository.GetDB()
|
||
db.RLock()
|
||
var src *models.LogSource
|
||
for i := range db.LogSources {
|
||
if db.LogSources[i].Name == deploymentName {
|
||
s := db.LogSources[i]
|
||
src = &s
|
||
break
|
||
}
|
||
}
|
||
db.RUnlock()
|
||
if src == nil {
|
||
return false, "未注册数据源"
|
||
}
|
||
if LokiReady(ctx, src) {
|
||
return true, ""
|
||
}
|
||
return false, "Loki 未就绪"
|
||
}
|
||
|
||
// LokiReady Loki /ready 探活(部署卡片/查询页状态点)
|
||
func LokiReady(ctx context.Context, src *models.LogSource) bool {
|
||
body, code, err := lokiProxyGET(ctx, src, "/ready", nil)
|
||
if err != nil || code != 200 {
|
||
return false
|
||
}
|
||
return strings.Contains(strings.ToLower(string(body)), "ready")
|
||
}
|
||
|
||
// TestLogSource 数据源连通性(/loki/api/v1/labels 探活)
|
||
func TestLogSource(ctx context.Context, src *models.LogSource) map[string]interface{} {
|
||
labels, err := LokiLabels(ctx, src, 0, 0)
|
||
if err != nil {
|
||
return map[string]interface{}{"success": false, "message": err.Error()}
|
||
}
|
||
return map[string]interface{}{
|
||
"success": true,
|
||
"message": fmt.Sprintf("连接成功,发现 %d 个标签", len(labels)),
|
||
"labels": labels,
|
||
}
|
||
}
|