rlwj-lab/LokiStack
1
0 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: complete Loki query, insights, settings and deployment workflows

Browse Source 
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application.

Validated with go test ./..., npm run build, and npm run lint.
This commit is contained in:
logan committed 2026-10-25 21:28:32 +08:00
1 parent f9c6c1d826
commit 2d73661dd1
127 files changed
+8117 -2876
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/services/alert.go
+265
View File
@@ -0,0 +1,265 @@
package services
// 日志告警引擎:周期执行 AlertRule 的 LogQL 计数查询,超阈值发送飞书卡片。
// 设计参照运维 app-alert.py:即时查询 sum(count_over_time(q[窗口])) 精确计数 +
// 按 app/pod 分组定位 + 日志样例 + 告警冷却(同规则冷却期内不重复发)。
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strings"
"time"
"opsxloki/models"
"opsxloki/repository"
)
const alertCheckInterval = time.Minute
var alertTicker *time.Ticker
// StartAlertEngine 启动周期检查(main 调用)
func StartAlertEngine() {
if alertTicker != nil {
return
}
alertTicker = time.NewTicker(alertCheckInterval)
go func() {
for range alertTicker.C {
RunAlertCheck(context.Background())
}
}()
}
// StopAlertEngine 优雅关停
func StopAlertEngine() {
if alertTicker != nil {
alertTicker.Stop()
alertTicker = nil
}
}
// lokiInstantCount 即时计数:sum(count_over_time(q[window]))
func lokiInstantCount(ctx context.Context, src *models.LogSource, query string, windowMin int) (int, error) {
wrapped := fmt.Sprintf("sum(count_over_time(%s[%dm]))", query, windowMin)
q := url.Values{}
q.Set("query", wrapped)
q.Set("time", fmt.Sprintf("%d", time.Now().UnixNano()))
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/query", q)
if err != nil {
return 0, err
}
var resp struct {
Data struct {
Result []struct {
Value [2]interface{} `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return 0, err
}
if len(resp.Data.Result) == 0 {
return 0, nil
}
count := 0
switch v := resp.Data.Result[0].Value[1].(type) {
case string:
fmt.Sscanf(v, "%d", &count)
case float64:
count = int(v)
}
return count, nil
}
// lokiGroupTop 按 app/pod 分组定位(取前 10)
func lokiGroupTop(ctx context.Context, src *models.LogSource, query string, windowMin int) []string {
wrapped := fmt.Sprintf("sum by (app, pod) (count_over_time(%s[%dm]))", query, windowMin)
q := url.Values{}
q.Set("query", wrapped)
q.Set("time", fmt.Sprintf("%d", time.Now().UnixNano()))
body, _, err := lokiProxyGET(ctx, src, "/loki/api/v1/query", q)
if err != nil {
return nil
}
var resp struct {
Data struct {
Result []struct {
Metric map[string]string `json:"metric"`
Value [2]interface{} `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil
}
lines := []string{}
for _, r := range resp.Data.Result {
count := 0
if v, ok := r.Value[1].(string); ok {
fmt.Sscanf(v, "%d", &count)
} else if v, ok := r.Value[1].(float64); ok {
count = int(v)
}
app := r.Metric["app"]
if app == "" {
app = "?"
}
pod := r.Metric["pod"]
if pod == "" {
pod = "?"
}
lines = append(lines, fmt.Sprintf("- %s / %s(%d 条)", app, pod, count))
if len(lines) >= 10 {
break
}
}
return lines
}
// lokiSampleLines 日志样例(最多 3 条,每条截 220)
func lokiSampleLines(ctx context.Context, src *models.LogSource, query string, windowMin int) []string {
res, err := QueryLokiLogs(ctx, src, query, 0, 0, 3, "backward", "")
if err != nil {
return nil
}
out := []string{}
for _, e := range res.Entries {
line := e.Message
if len(line) > 220 {
line = line[:220]
}
out = append(out, "> "+line)
if len(out) >= 3 {
break
}
}
return out
}
func recordDelivery(ruleID uint, ruleName string, count int, status, reason string) {
db := repository.GetDB()
newID := db.NextID("alertDeliveries")
db.Lock()
db.AlertDeliveries = append(db.AlertDeliveries, models.AlertDelivery{
ID: newID, RuleID: ruleID, RuleName: ruleName,
Count: count, Status: status, Reason: reason,
})
// 只保留最近 50 条
if len(db.AlertDeliveries) > 50 {
db.AlertDeliveries = db.AlertDeliveries[len(db.AlertDeliveries)-50:]
}
db.RequestSave()
db.Unlock()
}
// RunAlertCheck 执行一次全部启用规则的检查(定时器与手动触发共用)
func RunAlertCheck(ctx context.Context) {
db := repository.GetDB()
db.RLock()
rules := make([]models.AlertRule, 0, len(db.AlertRules))
for i := range db.AlertRules {
if db.AlertRules[i].Enabled {
rules = append(rules, db.AlertRules[i])
}
}
db.RUnlock()
for i := range rules {
checkRule(ctx, &rules[i])
}
}
func checkRule(ctx context.Context, rule *models.AlertRule) {
src := GetLogSource(rule.SourceID)
if src == nil {
return
}
count, err := lokiInstantCount(ctx, src, rule.Query, rule.WindowMin)
if err != nil {
recordDelivery(rule.ID, rule.Name, 0, "failed", err.Error())
return
}
if count <= rule.Threshold {
return
}
// 冷却检查(同规则)
db := repository.GetDB()
db.RLock()
lastFired := rule.LastFiredAt
cooldown := rule.CooldownMin
if cooldown <= 0 {
cooldown = 60
}
db.RUnlock()
if !lastFired.IsZero() && time.Since(lastFired) < time.Duration(cooldown)*time.Minute {
recordDelivery(rule.ID, rule.Name, count, "cooldown", "冷却中,未重复发送")
return
}
// 组装卡片
groups := lokiGroupTop(ctx, src, rule.Query, rule.WindowMin)
samples := lokiSampleLines(ctx, src, rule.Query, rule.WindowMin)
ts := time.Now().Format("2006-01-02 15:04:05")
lines := []string{
fmt.Sprintf("**告警时间**: %s", ts),
fmt.Sprintf("**规则**: %s", rule.Name),
fmt.Sprintf("**命中数量**: %d 条(%d 分钟内)", count, rule.WindowMin),
fmt.Sprintf("**触发阈值**: > %d 条", rule.Threshold),
fmt.Sprintf("**查询**: `%s`", rule.Query),
}
if len(groups) > 0 {
lines = append(lines, "**来源 app/pod**:\n"+strings.Join(groups, "\n"))
}
if len(samples) > 0 {
lines = append(lines, "**日志样例**:\n"+strings.Join(samples, "\n"))
}
// 扇出到全部关联通道;部分失败记录失败原因(成功任一即更新冷却)
title := "🔴 日志告警 " + rule.Name
var failMsgs []string
if len(rule.ChannelIDs) > 0 {
for _, chID := range rule.ChannelIDs {
if err := SendLarkCardToChannel(chID, title, "red", lines); err != nil {
chName := fmt.Sprintf("通道%d", chID)
if ch := GetNotifyChannel(chID); ch != nil {
chName = ch.Name
}
failMsgs = append(failMsgs, fmt.Sprintf("%s: %v", chName, err))
}
}
} else if cfg := GetNotifyConfig(); cfg.Enabled && cfg.Webhook != "" {
// 兼容:未关联通道回退旧全局配置
if err := SendLarkCard(title, "red", lines); err != nil {
failMsgs = append(failMsgs, err.Error())
}
} else {
failMsgs = append(failMsgs, "规则未关联通知通道")
}
if len(failMsgs) > 0 {
recordDelivery(rule.ID, rule.Name, count, "failed", strings.Join(failMsgs, "; "))
return
}
recordDelivery(rule.ID, rule.Name, count, "sent", fmt.Sprintf("已扇出到 %d 个通道", maxInt(len(rule.ChannelIDs), 1)))
// 更新 LastFiredAt
db.Lock()
for i := range db.AlertRules {
if db.AlertRules[i].ID == rule.ID {
db.AlertRules[i].LastFiredAt = time.Now()
break
}
}
db.RequestSave()
db.Unlock()
recordDelivery(rule.ID, rule.Name, count, "sent", "")
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}
backend/services/cluster.go
+8
View File
@@ -3,6 +3,7 @@ package services
import (
"context"
"fmt"
"opsxloki/pkg/crypto"
"os"
"path/filepath"
"time"
@@ -143,6 +144,13 @@ func (s *ClusterService) Update(id uint, updates map[string]interface{}) error {
if v, ok := updates["apiServer"].(string); ok {
db.Clusters[i].APIServer = v
}
// kubeconfig 编辑约定:非空才更新(留空=保留旧密文)
if v, ok := updates["kubeconfig"].(string); ok && v != "" {
if enc, err := crypto.EncryptField(v); err == nil {
db.Clusters[i].KubeConfig = enc
db.Clusters[i].KubeConfigPath = ""
}
}
// 凭据字段：空值不覆盖，避免编辑其它项时清空
if v, ok := updates["kubeconfig"].(string); ok && v != "" {
db.Clusters[i].KubeConfig = v
backend/services/cluster_info.go
+38 -104
View File
@@ -2,13 +2,13 @@ package services
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/clientcmd"
"opsxloki/models"
"sort"
"strings"
"time"
)
type ClusterInfo struct {
@@ -30,72 +30,21 @@ type NodeInfo struct {
Labels map[string]string `json:"labels"`
}
// GetClusterInfo 通过 kubectl 查询集群信息。
// Token 接入的集群在创建时已构造等价 kubeconfig，因此 kubectl 同样适用。
// GetClusterInfo 从 API Server 读取集群信息，context 列表由 kubeconfig 本地解析。
func GetClusterInfo(ctx context.Context, cluster *models.Cluster) (*ClusterInfo, error) {
kubeconfig, err := resolveKubeconfig(cluster)
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return nil, err
}
kubeFile, err := os.CreateTemp("", "kube-*.yaml")
info := &ClusterInfo{Nodes: []NodeInfo{}, Namespaces: []string{}, Contexts: []string{}}
if version, err := client.Discovery().ServerVersion(); err == nil {
info.Version = version.GitVersion
}
nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
return nil, fmt.Errorf("获取节点列表失败: %w", err)
}
defer os.Remove(kubeFile.Name())
kubeFile.WriteString(kubeconfig)
kubeFile.Close()
env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile.Name()))
info := &ClusterInfo{
Nodes: []NodeInfo{},
Namespaces: []string{},
Contexts: []string{},
}
// Version（best-effort）
if versionOut, err := runCmd(ctx, env, "kubectl", "version", "-o", "json"); err == nil {
var versionData struct {
ServerVersion struct {
GitVersion string `json:"gitVersion"`
} `json:"serverVersion"`
}
json.Unmarshal(versionOut, &versionData)
info.Version = versionData.ServerVersion.GitVersion
}
// Nodes
nodesOut, err := runCmd(ctx, env, "kubectl", "get", "nodes", "-o", "json")
if err != nil {
return nil, fmt.Errorf("获取节点列表失败: %v", err)
}
var nodesData struct {
Items []struct {
Metadata struct {
Name string `json:"name"`
Labels map[string]string `json:"labels"`
} `json:"metadata"`
Status struct {
Conditions []struct {
Type string `json:"type"`
Status string `json:"status"`
} `json:"conditions"`
NodeInfo struct {
KubeletVersion string `json:"kubeletVersion"`
OperatingSystem string `json:"operatingSystem"`
Architecture string `json:"architecture"`
} `json:"nodeInfo"`
Addresses []struct {
Type string `json:"type"`
Address string `json:"address"`
} `json:"addresses"`
} `json:"status"`
} `json:"items"`
}
json.Unmarshal(nodesOut, &nodesData)
for _, item := range nodesData.Items {
for _, item := range nodes.Items {
status := "Unknown"
for _, cond := range item.Status.Conditions {
if cond.Type == "Ready" {
@@ -107,58 +56,43 @@ func GetClusterInfo(ctx context.Context, cluster *models.Cluster) (*ClusterInfo,
break
}
}
var roles []string
for label := range item.Metadata.Labels {
roles := []string{}
for label := range item.Labels {
if strings.HasPrefix(label, "node-role.kubernetes.io/") {
roles = append(roles, label[len("node-role.kubernetes.io/"):])
roles = append(roles, strings.TrimPrefix(label, "node-role.kubernetes.io/"))
}
}
if len(roles) == 0 {
roles = []string{"worker"}
}
internalIP := ""
for _, addr := range item.Status.Addresses {
if addr.Type == "InternalIP" {
internalIP = addr.Address
sort.Strings(roles)
ip := ""
for _, address := range item.Status.Addresses {
if address.Type == "InternalIP" {
ip = address.Address
break
}
}
info.Nodes = append(info.Nodes, NodeInfo{
Name: item.Metadata.Name,
Status: status,
Roles: roles,
Version: item.Status.NodeInfo.KubeletVersion,
OS: item.Status.NodeInfo.OperatingSystem,
Arch: item.Status.NodeInfo.Architecture,
InternalIP: internalIP,
Labels: item.Metadata.Labels,
})
info.Nodes = append(info.Nodes, NodeInfo{Name: item.Name, Status: status, Roles: roles, Version: item.Status.NodeInfo.KubeletVersion, OS: item.Status.NodeInfo.OperatingSystem, Arch: item.Status.NodeInfo.Architecture, InternalIP: ip, Labels: item.Labels})
}
// Namespaces（best-effort）
if nsOut, err := runCmd(ctx, env, "kubectl", "get", "ns", "-o", "jsonpath={.items[*].metadata.name}"); err == nil {
info.Namespaces = strings.Fields(string(nsOut))
if namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}); err == nil {
for _, item := range namespaces.Items {
info.Namespaces = append(info.Namespaces, item.Name)
}
sort.Strings(info.Namespaces)
}
// Contexts（best-effort）
if ctxOut, err := runCmd(ctx, env, "kubectl", "config", "get-contexts", "-o", "name"); err == nil {
for _, line := range strings.Split(strings.TrimSpace(string(ctxOut)), "\n") {
if line = strings.TrimSpace(line); line != "" {
info.Contexts = append(info.Contexts, line)
source, err := resolveKubeconfig(cluster)
if err == nil {
if raw, err := clientcmd.Load([]byte(source)); err == nil {
for name := range raw.Contexts {
info.Contexts = append(info.Contexts, name)
}
sort.Strings(info.Contexts)
info.CurrentCtx = raw.CurrentContext
if cluster.Context != "" {
info.CurrentCtx = cluster.Context
}
}
}
// Current context（best-effort）
if currentCtxOut, err := runCmd(ctx, env, "kubectl", "config", "current-context"); err == nil {
info.CurrentCtx = strings.TrimSpace(string(currentCtxOut))
}
return info, nil
}
func runCmd(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
cmd.Env = env
return cmd.CombinedOutput()
}
backend/services/cluster_workloads.go
+63
View File
@@ -0,0 +1,63 @@
package services
import (
"context"
"fmt"
"regexp"
"sort"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"opsxloki/models"
)
type ClusterDeploymentOption struct {
Name string `json:"name"`
Replicas int `json:"replicas"`
ReadyReplicas int `json:"readyReplicas"`
}
var kubernetesNamespacePattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`)
func ListClusterNamespaces(ctx context.Context, cluster *models.Cluster) ([]string, error) {
client, err := clusterClient(cluster, 10*time.Second)
if err != nil {
return nil, err
}
items, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("读取集群命名空间失败: %w", err)
}
result := make([]string, 0, len(items.Items))
for _, item := range items.Items {
result = append(result, item.Name)
}
sort.Strings(result)
return result, nil
}
func ListClusterDeployments(ctx context.Context, cluster *models.Cluster, namespace string) ([]ClusterDeploymentOption, error) {
if len(namespace) > 63 || !kubernetesNamespacePattern.MatchString(namespace) {
return nil, fmt.Errorf("命名空间名称无效")
}
client, err := clusterClient(cluster, 10*time.Second)
if err != nil {
return nil, err
}
items, err := client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("读取集群 Deployment 失败: %w", err)
}
result := make([]ClusterDeploymentOption, 0, len(items.Items))
for _, item := range items.Items {
replicas := int32(1)
if item.Spec.Replicas != nil {
replicas = *item.Spec.Replicas
}
result = append(result, ClusterDeploymentOption{
Name: item.Name, Replicas: int(replicas), ReadyReplicas: int(item.Status.ReadyReplicas),
})
}
sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name })
return result, nil
}
backend/services/cluster_workloads_test.go
+38
View File
@@ -0,0 +1,38 @@
package services
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"opsxloki/models"
"testing"
)
func TestClusterWorkloadSelectorsUseSelectedNamespace(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v1/namespaces":
fmt.Fprint(w, `{"apiVersion":"v1","kind":"NamespaceList","items":[{"metadata":{"name":"prod"}},{"metadata":{"name":"default"}}]}`)
case "/apis/apps/v1/namespaces/prod/deployments":
fmt.Fprint(w, `{"apiVersion":"apps/v1","kind":"DeploymentList","items":[{"metadata":{"name":"api"},"spec":{"replicas":3},"status":{"readyReplicas":2}}]}`)
default:
t.Errorf("unexpected Kubernetes API path %s", r.URL.Path)
http.NotFound(w, r)
}
}))
defer server.Close()
cluster := &models.Cluster{KubeConfig: fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: test\n cluster:\n server: %s\ncontexts:\n- name: test\n context:\n cluster: test\n user: test\ncurrent-context: test\nusers:\n- name: test\n user: {}\n", server.URL)}
namespaces, err := ListClusterNamespaces(context.Background(), cluster)
if err != nil || len(namespaces) != 2 || namespaces[0] != "default" || namespaces[1] != "prod" {
t.Fatalf("unexpected namespaces: %v, %v", namespaces, err)
}
deployments, err := ListClusterDeployments(context.Background(), cluster, "prod")
if err != nil || len(deployments) != 1 || deployments[0].Name != "api" || deployments[0].Replicas != 3 || deployments[0].ReadyReplicas != 2 {
t.Fatalf("unexpected deployments: %v, %v", deployments, err)
}
if _, err := ListClusterDeployments(context.Background(), cluster, "-bad"); err == nil {
t.Fatal("expected invalid namespace to be rejected")
}
}
backend/services/collector.go
+3
View File
@@ -201,6 +201,9 @@ func (s *CollectorService) Update(id uint, updates map[string]interface{}) error
db.Collectors[i].StatusMessage = v
}
if v, ok := updates["logPath"].(string); ok {
if v != db.Collectors[i].LogPath {
db.Collectors[i].StatusMessage = "采集路径已保存，点击部署以应用新配置"
}
db.Collectors[i].LogPath = v
}
if v, ok := updates["labels"].(string); ok {
backend/services/collector_deployer.go
+109 -159
View File
@@ -2,10 +2,11 @@ package services
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"strings"
"time"
"opsxloki/models"
)
@@ -27,28 +28,6 @@ func (d *CollectorDeployer) log(level, message, detail string) {
}
}
// kubeconfigEnv 构造 kubectl/helm 所需的 KUBECONFIG 环境变量，返回 cleanup 删除临时文件
func kubeconfigEnv(cluster *models.Cluster) (env []string, cleanup func(), err error) {
kubeconfig, err := resolveKubeconfig(cluster)
if err != nil {
return nil, nil, err
}
kubeFile, err := os.CreateTemp("", "kube-*.yaml")
if err != nil {
return nil, nil, fmt.Errorf("create kubeconfig temp file failed: %w", err)
}
if _, err := kubeFile.WriteString(kubeconfig); err != nil {
kubeFile.Close()
os.Remove(kubeFile.Name())
return nil, nil, fmt.Errorf("write kubeconfig temp file failed: %w", err)
}
kubeFile.Close()
os.Chmod(kubeFile.Name(), 0600)
env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile.Name()))
cleanup = func() { os.Remove(kubeFile.Name()) }
return env, cleanup, nil
}
// Install 按采集器类型分发安装
func (d *CollectorDeployer) Install(ctx context.Context) error {
endpoint := d.loki.Endpoint
@@ -206,11 +185,6 @@ func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint strin
if cluster == nil {
return fmt.Errorf("集群采集器缺少目标集群")
}
env, cleanup, err := kubeconfigEnv(cluster)
if err != nil {
return err
}
defer cleanup()
// 默认部署到关联 Loki 的命名空间(统一 log 命名空间策略)
ns := d.collector.Namespace
@@ -226,102 +200,43 @@ func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint strin
clusterName = cluster.Name
}
// 幂等:先清掉同名残留(如历史失败安装留下的无注解资源)
del := exec.CommandContext(ctx, "kubectl", "delete",
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
"-n", ns, "--ignore-not-found=true", "--wait=false")
del.Env = env
del.CombinedOutput()
// 纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),不依赖后端安装 helm
// 删除历史残留并通过 Kubernetes API 提交资源。
for _, kind := range []string{"daemonset", "configmap", "serviceaccount"} {
_ = deleteK8sResource(ctx, cluster, kind, ns, "promtail")
}
manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels)
if err := d.applyYAML(ctx, env, manifest, "promtail"); err != nil {
if err := d.applyYAML(ctx, nil, manifest, "promtail"); err != nil {
return err
}
d.log("info", fmt.Sprintf("等待 DaemonSet promtail 就绪(命名空间 %s)...", ns), "")
waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "daemonset/promtail",
"-n", ns, "--timeout=5m")
waitCmd.Env = env
waitOut, _ := waitCmd.CombinedOutput()
d.log("info", string(waitOut), "")
// DNS 自愈:CoreDNS 异常的集群上服务名不可解析,回退 Loki ClusterIP 重建配置
svcURL := endpoint
if strings.Contains(endpoint, ".svc") {
probe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns,
"daemonset/promtail", "--", "wget", "-q", "-O", "/dev/null", "-T", "5", endpoint+"/ready")
probe.Env = env
if pout, perr := probe.CombinedOutput(); perr != nil {
d.log("warn", fmt.Sprintf("promtail 无法解析 Loki 服务名(%s),回退 ClusterIP...", strings.TrimSpace(string(pout))), "")
ip := d.lokiClusterIP(ctx, env, endpoint)
if ip != "" {
// 只替换 host 部分,保留 scheme/port/path: http://loki.ns.svc:3100/... → http://IP:3100/...
afterScheme := strings.SplitN(endpoint, "//", 2)
if len(afterScheme) == 2 {
rest := afterScheme[1]
host := rest
if i := strings.IndexAny(rest, ":/"); i >= 0 {
host = rest[:i]
}
fallback := afterScheme[0] + "//" + strings.Replace(rest, host, ip, 1)
if err := d.applyYAML(ctx, env, PromtailDaemonsetYAML(ns, fallback, clusterName, d.collector.Labels), "promtail"); err == nil {
rs := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "daemonset/promtail", "-n", ns)
rs.Env = env
rs.CombinedOutput()
d.log("info", fmt.Sprintf("promtail 推送地址已回退为 ClusterIP: %s", fallback), "")
svcURL = fallback
}
}
}
}
if err := waitDaemonSet(ctx, cluster, ns, "promtail"); err != nil {
return fmt.Errorf("Promtail DaemonSet 未就绪: %w", err)
}
svcURL := endpoint
d.collector.Config = svcURL
d.log("success", fmt.Sprintf("Promtail DaemonSet 已部署到 %s(推送至 %s)", ns, svcURL), "")
return nil
}
// applyYAML kubectl apply 一个 manifest(临时文件)
func (d *CollectorDeployer) applyYAML(ctx context.Context, env []string, manifest, label string) error {
f, cleanup, err := writeTempFile("collector-*.yaml", manifest, 0600)
if err != nil {
// applyYAML 通过 API Server 应用采集器清单。
func (d *CollectorDeployer) applyYAML(ctx context.Context, _ []string, manifest, label string) error {
if d.collector.Cluster == nil {
return fmt.Errorf("采集器缺少集群")
}
if err := applyManifest(ctx, d.collector.Cluster, manifest); err != nil {
d.log("error", fmt.Sprintf("应用 %s 失败: %v", label, err), "")
return err
}
defer cleanup()
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", f)
cmd.Env = env
out, err := cmd.CombinedOutput()
if err != nil {
d.log("error", fmt.Sprintf("kubectl apply %s failed: %v", label, err), string(out))
return fmt.Errorf("kubectl apply %s failed: %w\n%s", label, err, string(out))
}
return nil
}
// lokiClusterIP 从 endpoint 的命名空间取 loki Service 的 ClusterIP(不经 DNS)
func (d *CollectorDeployer) lokiClusterIP(ctx context.Context, env []string, endpoint string) string {
ns := "log"
if d.loki != nil && d.loki.Namespace != "" {
ns = d.loki.Namespace
}
cmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
"-o", "jsonpath={.spec.clusterIP}")
cmd.Env = env
out, _ := cmd.CombinedOutput()
return strings.TrimSpace(string(out))
}
func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error {
cluster := d.collector.Cluster
if cluster == nil {
return nil
}
env, cleanup, err := kubeconfigEnv(cluster)
if err != nil {
return err
}
defer cleanup()
ns := d.collector.Namespace
if ns == "" && d.loki != nil {
ns = d.loki.Namespace
@@ -329,12 +244,11 @@ func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error {
if ns == "" {
ns = "log"
}
cmd := exec.CommandContext(ctx, "kubectl", "delete",
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
"clusterrole/opsxloki-promtail", "clusterrolebinding/opsxloki-promtail",
"-n", ns, "--ignore-not-found=true")
cmd.Env = env
cmd.CombinedOutput()
for _, item := range [][2]string{{"daemonset", "promtail"}, {"configmap", "promtail"}, {"serviceaccount", "promtail"}, {"clusterrole", "opsxloki-promtail"}, {"clusterrolebinding", "opsxloki-promtail"}} {
if err := deleteK8sResource(ctx, cluster, item[0], ns, item[1]); err != nil {
return err
}
}
return nil
}
@@ -349,11 +263,10 @@ func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string)
if target == "" {
return fmt.Errorf("Sidecar 采集器缺少目标 Deployment")
}
env, cleanup, err := kubeconfigEnv(cluster)
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
defer cleanup()
ns := d.collector.Namespace
if ns == "" {
@@ -367,57 +280,65 @@ func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string)
// 1. 检查目标 deployment 是否存在
d.log("info", fmt.Sprintf("检查目标 Deployment %s/%s", ns, target), "")
check := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns)
check.Env = env
if out, err := check.CombinedOutput(); err != nil {
return fmt.Errorf("目标 Deployment 不存在: %s/%s\n%s", ns, target, string(out))
deployment, err := client.AppsV1().Deployments(ns).Get(ctx, target, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("读取目标 Deployment %s/%s: %w", ns, target, err)
}
// 2. 幂等检查：是否已有 fluent-bit 容器
getNames := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns, "-o", "jsonpath={.spec.template.spec.containers[*].name}")
getNames.Env = env
namesOut, _ := getNames.CombinedOutput()
if strings.Contains(string(namesOut), "fluent-bit") {
d.log("warn", "目标 Deployment 已含 fluent-bit 容器，跳过注入", "")
return nil
deploymentJSON, err := json.Marshal(deployment)
if err != nil {
return err
}
mount, err := resolveSidecarLogMount(deploymentJSON, target, logPath, d.collector.ID)
if err != nil {
return err
}
hasSidecar := false
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == "fluent-bit" {
hasSidecar = true
break
}
}
// 3. 部署 ConfigMap
lokiHost, lokiPort := parseLokiURL(endpoint)
conf := FluentbitConfig(lokiHost, lokiPort, logPath, app, ns)
cm := FluentbitConfigMapYAML(conf, ns)
cmFile, cmCleanup, err := writeTempFile("fluent-bit-cm-*.yaml", cm, 0600)
if err != nil {
return err
}
defer cmCleanup()
configMapName := fmt.Sprintf("fluent-bit-config-%d", d.collector.ID)
cm := FluentbitConfigMapYAML(conf, ns, configMapName)
d.log("info", "部署 Fluent Bit ConfigMap", "")
applyCm := exec.CommandContext(ctx, "kubectl", "apply", "-f", cmFile)
applyCm.Env = env
if out, err := applyCm.CombinedOutput(); err != nil {
return fmt.Errorf("ConfigMap 部署失败: %w\n%s", err, string(out))
if err := applyManifest(ctx, cluster, cm); err != nil {
return fmt.Errorf("ConfigMap 部署失败: %w", err)
}
// 4. patch 注入 sidecar
d.log("info", fmt.Sprintf("为 %s 注入 Fluent Bit Sidecar", target), "")
patchFile, patchCleanup, err := writeTempFile("sidecar-patch-*.yaml", SidecarPatchYAML, 0600)
if err != nil {
return err
d.log("info", fmt.Sprintf("为 %s 配置 Fluent Bit Sidecar", target), "")
if err := patchDeployment(ctx, cluster, ns, target, SidecarPatchYAML(configMapName, mount)); err != nil {
return fmt.Errorf("Sidecar 注入失败: %w", err)
}
defer patchCleanup()
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", target, "-n", ns, "--type=strategic", "--patch-file", patchFile)
patch.Env = env
if out, err := patch.CombinedOutput(); err != nil {
return fmt.Errorf("Sidecar 注入失败: %w\n%s", err, string(out))
if hasSidecar {
if err := restartDeployment(ctx, cluster, ns, target); err != nil {
return fmt.Errorf("Sidecar 配置已更新，但重启 Deployment 失败: %w", err)
}
}
// 5. 等待 rollout
d.log("info", "等待 Pod 重建", "")
rollout := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/"+target, "-n", ns, "--timeout=5m")
rollout.Env = env
rollout.CombinedOutput()
if err := waitDeployment(ctx, cluster, ns, target); err != nil {
return fmt.Errorf("Sidecar Deployment 未就绪: %w", err)
}
// DNS 自愈:CoreDNS 异常集群上 svc 域名不可解析 → 用 Loki ClusterIP 重建 ConfigMap 并滚动重启
if strings.Contains(endpoint, ".svc") {
host := strings.Split(strings.Split(endpoint, "//")[1], ":")[0]
if podDNSBroken(ctx, cluster, ns, target, "fluent-bit", host) {
d.log("warn", "fluent-bit 无法解析 Loki 服务名，回退 ClusterIP…", "")
if ip, ierr := serviceClusterIP(ctx, cluster, lokiNamespaceOf(d.loki, endpoint), "loki"); ierr == nil && ip != "" {
conf2 := FluentbitConfig(ip, lokiPortOf(endpoint), logPath, target, ns)
cm2 := FluentbitConfigMapYAML(conf2, ns, configMapName)
if err := applyManifest(ctx, cluster, cm2); err == nil {
_ = restartDeployment(ctx, cluster, ns, target)
d.log("info", "fluent-bit 推送地址已回退为 ClusterIP: "+ip, "")
}
}
}
}
return nil
}
@@ -426,23 +347,52 @@ func (d *CollectorDeployer) uninstallSidecar(ctx context.Context) error {
if cluster == nil || d.collector.TargetDeployment == "" {
return nil
}
env, cleanup, err := kubeconfigEnv(cluster)
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
defer cleanup()
ns := d.collector.Namespace
if ns == "" {
ns = "default"
}
patchFile, patchCleanup, err := writeTempFile("sidecar-rm-*.yaml", SidecarRemovePatchYAML, 0600)
deployment, err := client.AppsV1().Deployments(ns).Get(ctx, d.collector.TargetDeployment, metav1.GetOptions{})
if err != nil {
return err
}
defer patchCleanup()
deploymentJSON, err := json.Marshal(deployment)
if err != nil {
return err
}
removePatch, err := sidecarRemovePatchYAML(deploymentJSON, d.collector.ID)
if err != nil {
return err
}
if err := patchDeployment(ctx, cluster, ns, d.collector.TargetDeployment, removePatch); err != nil {
return fmt.Errorf("移除 Sidecar 失败: %w", err)
}
if err := deleteK8sResource(ctx, cluster, "configmap", ns, fmt.Sprintf("fluent-bit-config-%d", d.collector.ID)); err != nil {
return err
}
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", d.collector.TargetDeployment, "-n", ns, "--type=strategic", "--patch-file", patchFile)
patch.Env = env
patch.CombinedOutput()
return nil
}
// lokiPortOf 从 endpoint 提取端口(默认 3100)
func lokiPortOf(endpoint string) string {
rest := strings.SplitN(endpoint, "//", 2)
if len(rest) != 2 {
return "3100"
}
hostPart := rest[1]
if i := strings.Index(hostPart, ":"); i >= 0 {
port := hostPart[i+1:]
if j := strings.IndexAny(port, "/"); j >= 0 {
port = port[:j]
}
if port != "" {
return port
}
}
return "3100"
}
backend/services/collector_sidecar_mount.go
+150
View File
@@ -0,0 +1,150 @@
package services
import (
"encoding/json"
"fmt"
"path"
"strings"
)
type sidecarLogMount struct {
containerName string
volumeName string
mountPath string
subPath string
createVolume bool
mountApplication bool
}
type sidecarDeployment struct {
Spec struct {
Template struct {
Spec struct {
Containers []struct {
Name string `json:"name"`
VolumeMounts []struct {
Name string `json:"name"`
MountPath string `json:"mountPath"`
SubPath string `json:"subPath"`
SubPathExpr string `json:"subPathExpr"`
ReadOnly bool `json:"readOnly"`
} `json:"volumeMounts"`
} `json:"containers"`
Volumes []struct {
Name string `json:"name"`
} `json:"volumes"`
} `json:"spec"`
} `json:"template"`
} `json:"spec"`
}
// resolveSidecarLogMount 优先复用业务容器已有的卷；否则仅在专用日志目录创建共享 emptyDir。
func resolveSidecarLogMount(deploymentJSON []byte, target, logPath string, collectorID uint) (sidecarLogMount, error) {
var deployment sidecarDeployment
if err := json.Unmarshal(deploymentJSON, &deployment); err != nil {
return sidecarLogMount{}, fmt.Errorf("读取目标 Deployment 失败: %w", err)
}
logDir := path.Dir(strings.TrimSpace(logPath))
if !path.IsAbs(logPath) || logDir == "/" || strings.ContainsAny(logDir, "*?[]\n\r") {
return sidecarLogMount{}, fmt.Errorf("Sidecar 采集路径需为专用目录下的绝对文件路径或 glob，例如 /var/log/app/*.log")
}
containers := deployment.Spec.Template.Spec.Containers
var appIndex = -1
var candidate = -1
for i, container := range containers {
if container.Name == target {
appIndex = i
break
}
if container.Name != "fluent-bit" {
if candidate >= 0 {
candidate = -2
} else if candidate == -1 {
candidate = i
}
}
}
if appIndex < 0 {
appIndex = candidate
}
if appIndex < 0 {
return sidecarLogMount{}, fmt.Errorf("目标 Deployment 中无法确定业务容器，请使容器名与 Deployment 名一致，或仅保留一个业务容器")
}
app := containers[appIndex]
mount := sidecarLogMount{containerName: app.Name}
longest := -1
for _, item := range app.VolumeMounts {
prefix := strings.TrimSuffix(item.MountPath, "/")
if (logDir == prefix || strings.HasPrefix(logDir, prefix+"/")) && len(prefix) > longest {
if item.ReadOnly || item.SubPathExpr != "" {
return sidecarLogMount{}, fmt.Errorf("业务容器的日志卷必须可写，且不能使用 subPathExpr")
}
mount.volumeName = item.Name
mount.mountPath = item.MountPath
mount.subPath = item.SubPath
longest = len(prefix)
}
}
if mount.volumeName != "" {
if mount.mountPath == "/tmp" {
return sidecarLogMount{}, fmt.Errorf("日志卷不能挂载在 /tmp：Fluent Bit 用此目录保存读取位置；请改用专用日志目录")
}
return mount, nil
}
// 自动挂载会遮蔽目录原有内容，仅对明确的专用日志目录执行。
if logDir != "/var/log/app" && path.Base(logDir) != "logs" && path.Base(logDir) != "log" {
return sidecarLogMount{}, fmt.Errorf("业务容器未共享 %s；请将采集路径改为 /var/log/app/*.log 或已有日志卷中的文件", logDir)
}
volumeName := fmt.Sprintf("loki-collector-logs-%d", collectorID)
for _, volume := range deployment.Spec.Template.Spec.Volumes {
if volume.Name == volumeName {
return sidecarLogMount{}, fmt.Errorf("卷 %s 已存在但业务容器未挂载，请先检查目标 Deployment", volumeName)
}
}
mount.volumeName = volumeName
mount.mountPath = logDir
mount.createVolume = true
mount.mountApplication = true
return mount, nil
}
// sidecarRemovePatchYAML 只清理该采集策略注入的卷，保留业务原有的日志卷。
func sidecarRemovePatchYAML(deploymentJSON []byte, collectorID uint) (string, error) {
var deployment sidecarDeployment
if err := json.Unmarshal(deploymentJSON, &deployment); err != nil {
return "", fmt.Errorf("读取目标 Deployment 失败: %w", err)
}
injectedName := fmt.Sprintf("loki-collector-logs-%d", collectorID)
appPatch := ""
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == "fluent-bit" {
continue
}
for _, mount := range container.VolumeMounts {
if mount.Name == injectedName {
appPatch += fmt.Sprintf(" - name: %s\n volumeMounts:\n - mountPath: %q\n $patch: delete\n", container.Name, mount.MountPath)
}
}
}
volumePatch := ""
for _, volume := range deployment.Spec.Template.Spec.Volumes {
if volume.Name == injectedName {
volumePatch = fmt.Sprintf(" - name: %s\n $patch: delete\n", injectedName)
break
}
}
return fmt.Sprintf(`spec:
template:
spec:
containers:
- name: fluent-bit
$patch: delete
%s volumes:
- name: fluent-bit-config
$patch: delete
- name: fluent-bit-state
$patch: delete
%s`, appPatch, volumePatch), nil
}
backend/services/collector_sidecar_mount_test.go
+64
View File
@@ -0,0 +1,64 @@
package services
import (
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestSidecarReusesApplicationLogVolume(t *testing.T) {
deployment := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"st-bos-forex-order","volumeMounts":[{"name":"app-logs","mountPath":"/bitcloud/app/st-bos-forex-order/logs"}]},{"name":"fluent-bit"}],"volumes":[{"name":"app-logs"}]}}}}`)
mount, err := resolveSidecarLogMount(deployment, "st-bos-forex-order", "/bitcloud/app/st-bos-forex-order/logs/*.log", 42)
if err != nil {
t.Fatal(err)
}
if mount.volumeName != "app-logs" || mount.mountPath != "/bitcloud/app/st-bos-forex-order/logs" || mount.createVolume || mount.mountApplication {
t.Fatalf("should reuse the existing application mount: %+v", mount)
}
patch := SidecarPatchYAML("fluent-bit-config-42", mount)
var parsed map[string]any
if err := yaml.Unmarshal([]byte(patch), &parsed); err != nil {
t.Fatalf("invalid strategic merge patch: %v", err)
}
if !strings.Contains(patch, "name: fluent-bit-config-42") || !strings.Contains(patch, "name: app-logs") {
t.Fatalf("patch does not reference the existing log volume and dedicated config: %s", patch)
}
}
func TestSidecarCreatesSharedVolumeOnlyForDedicatedDirectory(t *testing.T) {
deployment := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"jar-monitor"}]}}}}`)
mount, err := resolveSidecarLogMount(deployment, "jar-monitor", "/var/log/app/*.log", 7)
if err != nil {
t.Fatal(err)
}
if !mount.createVolume || !mount.mountApplication || mount.mountPath != "/var/log/app" || mount.volumeName != "loki-collector-logs-7" {
t.Fatalf("expected a shared volume for both containers: %+v", mount)
}
patch := SidecarPatchYAML("fluent-bit-config-7", mount)
var parsed map[string]any
if err := yaml.Unmarshal([]byte(patch), &parsed); err != nil {
t.Fatalf("invalid strategic merge patch: %v", err)
}
if !strings.Contains(patch, "name: jar-monitor\n volumeMounts:") {
t.Fatalf("application container is missing its log mount: %s", patch)
}
if _, err := resolveSidecarLogMount(deployment, "jar-monitor", "/tmp/test.log", 7); err == nil {
t.Fatal("/tmp must not be covered by a new emptyDir")
}
}
func TestSidecarRemovalKeepsApplicationOwnedVolume(t *testing.T) {
deployment := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"app","volumeMounts":[{"name":"app-logs","mountPath":"/business/logs"},{"name":"loki-collector-logs-7","mountPath":"/var/log/app"}]},{"name":"fluent-bit"}],"volumes":[{"name":"app-logs"},{"name":"loki-collector-logs-7"},{"name":"fluent-bit-config"}]}}}}`)
patch, err := sidecarRemovePatchYAML(deployment, 7)
if err != nil {
t.Fatal(err)
}
var parsed map[string]any
if err := yaml.Unmarshal([]byte(patch), &parsed); err != nil {
t.Fatalf("invalid removal patch: %v", err)
}
if strings.Contains(patch, "name: app-logs") || !strings.Contains(patch, "name: loki-collector-logs-7") || !strings.Contains(patch, `mountPath: "/var/log/app"`) {
t.Fatalf("removal must only clean collector-owned mounts: %s", patch)
}
}
backend/services/collector_templates.go
+39 -54
View File
@@ -194,6 +194,7 @@ extraVolumeMounts:
// 取代 helm 安装 —— 后端无需安装 helm 二进制。
// relabel 体系与主流 helm chart 对齐:namespace/pod/container/node_name + __path__ 按 pod uid 精确匹配。
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string) string {
promtailImage := ResolveImage("docker.io/grafana/promtail:3.4.0")
extLabels := promtailExternalLabels(clusterName, labelsJSON)
return fmt.Sprintf(`apiVersion: v1
kind: ServiceAccount
@@ -292,7 +293,7 @@ spec:
operator: Exists
containers:
- name: promtail
image: grafana/promtail:3.4.0
image: %s
imagePullPolicy: IfNotPresent
args: ["-config.file=/etc/promtail/promtail.yaml"]
env:
@@ -340,11 +341,11 @@ spec:
- name: containers
hostPath:
path: /var/lib/docker/containers
`, ns, ns, ns, lokiURL, extLabels, ns)
`, ns, ns, ns, lokiURL, extLabels, ns, promtailImage)
}
// FluentbitConfig Sidecar 模式 Fluent Bit 配置（采集 logPath，推送至 lokiHost:lokiPort）
// 生产级配置：kubernetes过滤器、内存缓冲、健康检查
// Sidecar 已明确知道应用与命名空间，无需 Kubernetes 元数据过滤器。
func FluentbitConfig(lokiHost, lokiPort, logPath, app, namespace string) string {
return fmt.Sprintf(`[SERVICE]
Flush 1
@@ -352,11 +353,9 @@ func FluentbitConfig(lokiHost, lokiPort, logPath, app, namespace string) string
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
Parsers_File parsers.conf
storage.path /tmp/flb-storage/
storage.sync normal
storage.checksum off
storage.max_chunks_pause On
storage.backlog.mem_limit 5M
Health_Check On
HC_Errors_Count 5
@@ -383,70 +382,69 @@ func FluentbitConfig(lokiHost, lokiPort, logPath, app, namespace string) string
Add app %s
Add source fluent-bit
[FILTER]
Name kubernetes
Match app.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
Labels On
Annotations Off
[OUTPUT]
Name loki
Match *
Host %s
Port %s
Labels job=fluent-bit, app=%s, namespace=%s
Drop_Records_Without_Labels On
Line_Format json
Auto_Kubernetes_Labels On
Retry_Limit 10
`, logPath, app, namespace, app, lokiHost, lokiPort, app, namespace)
}
// FluentbitConfigMapYAML 包装 fluent-bit.conf 为 ConfigMap YAML（部署到 namespace）
func FluentbitConfigMapYAML(conf, namespace string) string {
// FluentbitConfigMapYAML 为每个采集策略生成独立 ConfigMap，避免同命名空间的 Sidecar 相互覆盖。
func FluentbitConfigMapYAML(conf, namespace, name string) string {
return fmt.Sprintf(`apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
name: %s
namespace: %s
data:
fluent-bit.conf: |
%s
`, namespace, indentBlock(conf, " "))
`, name, namespace, indentBlock(conf, " "))
}
// SidecarPatchYAML strategic merge patch：为 Deployment 注入 Fluent Bit sidecar 容器
// 生产级配置：资源限制、健康检查、安全上下文
const SidecarPatchYAML = `spec:
// SidecarPatchYAML 将业务容器与 Fluent Bit 挂到同一日志卷和目录。
func SidecarPatchYAML(configMapName string, mount sidecarLogMount) string {
volume := ""
if mount.createVolume {
volume = fmt.Sprintf(" - name: %s\n emptyDir: {}\n", mount.volumeName)
}
appMount := ""
if mount.mountApplication {
appMount = fmt.Sprintf(" - name: %s\n volumeMounts:\n - name: %s\n mountPath: %q\n", mount.containerName, mount.volumeName, mount.mountPath)
}
subPath := ""
if mount.subPath != "" {
subPath = fmt.Sprintf(" subPath: %q\n", mount.subPath)
}
return fmt.Sprintf(`spec:
template:
spec:
volumes:
- name: app-logs
emptyDir: {}
- name: fluent-bit-config
%s - name: fluent-bit-config
configMap:
name: fluent-bit-config
name: %s
- name: fluent-bit-state
emptyDir: {}
containers:
- name: fluent-bit
image: cr.fluentbit.io/fluent/fluent-bit:3.1.1
%s - name: fluent-bit
image: %s
imagePullPolicy: IfNotPresent
ports:
- name: http
- name: fluent-bit-http
containerPort: 2020
protocol: TCP
volumeMounts:
- name: app-logs
mountPath: /var/log/app
readOnly: true
- name: %s
mountPath: %q
%s readOnly: true
- name: fluent-bit-config
mountPath: /fluent-bit/etc/
- name: fluent-bit-state
mountPath: /tmp
resources:
limits:
cpu: 200m
@@ -463,30 +461,17 @@ const SidecarPatchYAML = `spec:
livenessProbe:
httpGet:
path: /
port: http
port: fluent-bit-http
initialDelaySeconds: 10
periodSeconds: 10
readinessProbe:
httpGet:
path: /
port: http
port: fluent-bit-http
initialDelaySeconds: 5
periodSeconds: 5
`
// SidecarRemovePatchYAML strategic merge patch：从 Deployment 移除 Fluent Bit sidecar（卸载）
const SidecarRemovePatchYAML = `spec:
template:
spec:
containers:
- name: fluent-bit
$patch: delete
volumes:
- name: app-logs
$patch: delete
- name: fluent-bit-config
$patch: delete
`
`, volume, configMapName, appMount, ResolveImage("docker.io/fluent/fluent-bit:5.1.5"), mount.volumeName, mount.mountPath, subPath)
}
// indentBlock 给每行加前缀（ConfigMap 嵌入配置需缩进）
func indentBlock(s, pad string) string {
backend/services/deployment.go
+98 -27
View File
@@ -1,9 +1,9 @@
package services
import (
"os/exec"
"context"
"fmt"
"net"
"strconv"
"strings"
"sync"
@@ -11,6 +11,7 @@ import (
"opsxloki/models"
"opsxloki/repository"
"opsxloki/seed"
"opsxloki/utils"
)
@@ -29,6 +30,8 @@ var LokiVersionList = []string{
// minSupportedMajorVersion 最低支持的主版本号
const minSupportedMajorVersion = 2
const DefaultRetentionDays = 30
// DeploymentService 部署业务逻辑层
type DeploymentService struct{}
@@ -43,6 +46,8 @@ type CreateDeploymentRequest struct {
Mode string `json:"mode" binding:"required"`
ClusterID *uint `json:"clusterID"`
HostID *uint `json:"hostID"`
ListenIP string `json:"listenIP"`
ListenPort int `json:"listenPort"`
Namespace string `json:"namespace"`
StorageConfigID uint `json:"storageConfigID"`
AuthEnabled bool `json:"authEnabled"`
@@ -56,6 +61,41 @@ type CreateDeploymentRequest struct {
}
func (s *DeploymentService) Create(req *CreateDeploymentRequest) (*models.LokiDeployment, error) {
if req.Mode != "k8s-yaml" && req.Mode != "host-docker" && req.Mode != "host-binary" {
return nil, fmt.Errorf("不支持的部署方式: %s", req.Mode)
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("Grafana 自动部署已移除，请使用独立 Grafana 服务")
}
if req.Mode == "host-docker" {
if _, err := dockerLokiImage(req.Version); err != nil {
return nil, err
}
if req.HostID == nil || *req.HostID == 0 {
return nil, fmt.Errorf("请选择目标主机")
}
if req.ListenPort == 0 {
req.ListenPort = 3100
}
if req.ListenPort < 1 || req.ListenPort > 65535 {
return nil, fmt.Errorf("监听端口须在 1 到 65535 之间")
}
if req.Replicas > 1 {
return nil, fmt.Errorf("主机 Docker 部署仅支持单副本")
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("主机 Docker 模式暂不支持同时部署 Grafana")
}
if req.ListenIP != "" && net.ParseIP(req.ListenIP).To4() == nil {
return nil, fmt.Errorf("监听地址须为 IPv4 地址")
}
}
if req.RetentionDays < 0 {
return nil, fmt.Errorf("保留天数必须是正整数")
}
if req.RetentionDays == 0 {
req.RetentionDays = DefaultRetentionDays
}
// 版本兼容性预检查
if req.Version != "" {
if err := ValidateLokiVersion(req.Version); err != nil {
@@ -72,6 +112,8 @@ func (s *DeploymentService) Create(req *CreateDeploymentRequest) (*models.LokiDe
Mode: req.Mode,
ClusterID: req.ClusterID,
HostID: req.HostID,
ListenIP: req.ListenIP,
ListenPort: req.ListenPort,
Namespace: req.Namespace,
StorageConfigID: req.StorageConfigID,
AuthEnabled: req.AuthEnabled,
@@ -107,6 +149,17 @@ func (s *DeploymentService) Create(req *CreateDeploymentRequest) (*models.LokiDe
}
}
}
if d.Mode == "host-docker" {
if d.Host == nil {
return nil, fmt.Errorf("目标主机不存在")
}
if d.ListenIP == "" {
d.ListenIP = d.Host.HostIP
}
if net.ParseIP(d.ListenIP).To4() == nil {
return nil, fmt.Errorf("主机 IP 格式无效，请填写监听内网 IPv4 地址")
}
}
if d.StorageConfigID > 0 {
for i := range db.Storages {
if db.Storages[i].ID == d.StorageConfigID {
@@ -169,6 +222,12 @@ func (s *DeploymentService) List(page, pageSize int) ([]models.LokiDeployment, i
}
func (s *DeploymentService) Update(id uint, updates map[string]interface{}) error {
if value, exists := updates["retentionDays"]; exists {
days, ok := value.(float64)
if !ok || days < 1 || days != float64(int(days)) {
return fmt.Errorf("保留天数必须是正整数")
}
}
db := repository.GetDB()
db.Lock()
defer db.Unlock()
@@ -243,6 +302,22 @@ func (s *DeploymentService) Update(id uint, updates map[string]interface{}) erro
}
func (s *DeploymentService) Delete(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Mode == "host-docker" {
if deployment.Status == "deploying" {
return fmt.Errorf("部署正在进行中，请等待完成后删除")
}
if deployment.Status != "pending" {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if err := deleteDockerLoki(ctx, deployment); err != nil {
return err
}
}
}
db := repository.GetDB()
db.Lock()
defer db.Unlock()
@@ -323,10 +398,11 @@ func (s *DeploymentService) StartDeploy(id uint) error {
"statusMessage": "Deployment completed successfully",
"endpoint": s.getEndpoint(deployment),
})
// 部署成功自动注册日志查询数据源 + 记录 Grafana 访问地址
// 部署成功自动注册日志查询数据源 + 记录 Grafana 访问地址 + 补内置告警模板
if fresh, gerr := s.Get(id); gerr == nil {
EnsureLogSourceForDeployment(fresh)
go s.recordGrafanaEndpoint(fresh)
go seed.SeedAlertRules()
}
s.AddLog(id, "success", "Deployment completed successfully", "")
}
@@ -353,7 +429,13 @@ func (s *DeploymentService) StopDeployment(id uint) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
CleanupLoki(ctx, deployment)
if deployment.Mode == "host-docker" {
if err := stopDockerLoki(ctx, deployment); err != nil {
return err
}
} else {
CleanupLoki(ctx, deployment)
}
return s.Update(id, map[string]interface{}{
"status": "stopped",
"statusMessage": "Deployment stopped by user",
@@ -499,6 +581,8 @@ func (s *DeploymentService) getEndpoint(d *models.LokiDeployment) string {
if d.Host != nil {
return fmt.Sprintf("http://%s:3100", d.Host.HostIP)
}
case "host-docker":
return fmt.Sprintf("http://%s:%d", d.ListenIP, d.ListenPort)
}
return ""
}
@@ -599,12 +683,12 @@ func (s *DeploymentService) Preflight() map[string]interface{} {
d := &db.Deployments[i]
if d.Status == "running" {
loki := map[string]interface{}{
"id": d.ID,
"name": d.Name,
"mode": d.Mode,
"endpoint": d.Endpoint,
"namespace": d.Namespace,
"grafana": d.GrafanaEnabled,
"id": d.ID,
"name": d.Name,
"mode": d.Mode,
"endpoint": d.Endpoint,
"namespace": d.Namespace,
"grafana": d.GrafanaEnabled,
"grafanaEndpoint": d.GrafanaEndpoint,
}
result["runningLoki"] = loki
@@ -638,29 +722,16 @@ func (s *DeploymentService) recordGrafanaEndpoint(d *models.LokiDeployment) {
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
env, cleanup, err := kubeconfigEnv(d.Cluster)
if err != nil {
return
}
defer cleanup()
ns := d.Namespace
if ns == "" {
ns = "log"
}
ipCmd := exec.CommandContext(ctx, "kubectl", "get", "nodes",
"-o", "jsonpath={.items[0].status.addresses[0].address}")
ipCmd.Env = env
ipOut, _ := ipCmd.CombinedOutput()
portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
"-o", "jsonpath={.spec.ports[0].nodePort}")
portCmd.Env = env
portOut, _ := portCmd.CombinedOutput()
ip := strings.TrimSpace(string(ipOut))
port := strings.TrimSpace(string(portOut))
s.AddLog(d.ID, "info", fmt.Sprintf("Grafana 地址探测: node=%s port=%s", ip, port), "")
if ip != "" && port != "" {
endpoint = fmt.Sprintf("http://%s:%s", ip, port)
ip, ipErr := firstNodeIP(ctx, d.Cluster)
port, portErr := serviceNodePort(ctx, d.Cluster, ns, "grafana")
if ipErr == nil && portErr == nil && port > 0 {
endpoint = fmt.Sprintf("http://%s:%d", ip, port)
}
case "host-binary":
if d.Host != nil {
endpoint = fmt.Sprintf("http://%s:3000", d.Host.HostIP)
backend/services/deployment_config.go
+120
View File
@@ -0,0 +1,120 @@
package services
import (
"context"
"fmt"
"strings"
"gopkg.in/yaml.v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"time"
)
type LiveLokiConfig struct {
Location string `json:"location"`
YAML string `json:"yaml"`
}
// GetLiveConfig 读取实际运行的 Loki 配置。仅返回 loki.yaml，不返回整个 ConfigMap 元数据。
func (s *DeploymentService) GetLiveConfig(ctx context.Context, id uint) (*LiveLokiConfig, error) {
d, err := s.Get(id)
if err != nil {
return nil, err
}
var raw, location string
switch d.Mode {
case "k8s-yaml":
if d.Cluster == nil {
return nil, fmt.Errorf("部署未关联 Kubernetes 集群")
}
client, err := clusterClient(d.Cluster, 30*time.Second)
if err != nil {
return nil, err
}
ns := d.Namespace
if ns == "" {
ns = "log"
}
configMap, err := client.CoreV1().ConfigMaps(ns).Get(ctx, "loki-config", metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("无法读取 %s/loki-config: %w", ns, err)
}
raw = configMap.Data["loki.yaml"]
location = fmt.Sprintf("ConfigMap %s/loki-config → loki.yaml", ns)
case "host-binary":
if d.Host == nil {
return nil, fmt.Errorf("部署未关联主机")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
if err != nil {
return nil, fmt.Errorf("连接部署主机失败")
}
defer client.Close()
output, _, err := client.Execute(ctx, "cat /etc/loki/loki.yaml")
if err != nil {
return nil, fmt.Errorf("无法读取主机上的 /etc/loki/loki.yaml")
}
raw = output
location = "/etc/loki/loki.yaml"
case "host-docker":
if d.Host == nil {
return nil, fmt.Errorf("部署未关联主机")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
if err != nil {
return nil, fmt.Errorf("连接部署主机失败")
}
defer client.Close()
location = dockerConfigPath(d.ID)
output, _, err := client.Execute(ctx, "cat "+location)
if err != nil {
return nil, fmt.Errorf("无法读取主机上的 Loki 配置")
}
raw = output
default:
return nil, fmt.Errorf("当前部署模式暂不支持查看运行配置")
}
if strings.TrimSpace(raw) == "" {
return nil, fmt.Errorf("运行配置为空或尚未部署")
}
redacted, err := redactLokiYAML(raw)
if err != nil {
return nil, fmt.Errorf("运行配置无法安全展示：YAML 格式无效")
}
return &LiveLokiConfig{Location: location, YAML: redacted}, nil
}
func redactLokiYAML(source string) (string, error) {
var document yaml.Node
if err := yaml.Unmarshal([]byte(source), &document); err != nil {
return "", err
}
redactYAMLNode(&document)
output, err := yaml.Marshal(&document)
return string(output), err
}
func redactYAMLNode(node *yaml.Node) {
if node.Kind == yaml.ScalarNode && strings.HasPrefix(node.Value, "ENC:") {
node.Value = "********"
return
}
if node.Kind == yaml.MappingNode {
for i := 0; i+1 < len(node.Content); i += 2 {
key, value := node.Content[i], node.Content[i+1]
switch strings.ToLower(key.Value) {
case "access_key_id", "secret_access_key", "account_key", "password", "token", "client_secret":
value.Kind = yaml.ScalarNode
value.Tag = "!!str"
value.Value = "********"
value.Content = nil
continue
}
redactYAMLNode(value)
}
return
}
for _, child := range node.Content {
redactYAMLNode(child)
}
}
backend/services/deployment_config_test.go
+138
View File
@@ -0,0 +1,138 @@
package services
import (
"context"
"encoding/base64"
"io"
"regexp"
"strings"
"testing"
"gopkg.in/yaml.v3"
"opsxloki/models"
)
func TestRedactLokiYAML(t *testing.T) {
source := "storage_config:\n aws:\n bucketnames: loki\n access_key_id: ENC:private-access\n secret_access_key: private-secret\nlimits_config:\n retention_period: 720h\n"
got, err := redactLokiYAML(source)
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{"private-access", "private-secret", "ENC:"} {
if strings.Contains(got, secret) {
t.Fatalf("sensitive value remains in config: %s", secret)
}
}
for _, want := range []string{"bucketnames: loki", "retention_period: 720h", "access_key_id: '********'"} {
if !strings.Contains(got, want) {
t.Fatalf("missing %q in sanitized config: %s", want, got)
}
}
}
func TestMinIOConfigUsesBucketNameNotPathPrefix(t *testing.T) {
d := NewLokiDeployer(&models.LokiDeployment{StorageConfig: &models.StorageConfig{
StorageType: "minio", Bucket: "logs-bucket", PathPrefix: "wrong-prefix", Endpoint: "http://minio:9000",
AccessKey: "key", SecretKey: "secret", Insecure: true,
}}, nil)
config := d.generateLokiConfig()
if count := strings.Count(config, "bucketnames: logs-bucket"); count != 2 {
t.Fatalf("bucket name should appear in storage and ruler config; got %d", count)
}
if strings.Contains(config, "wrong-prefix") || strings.Contains(config, "s3://") {
t.Fatalf("invalid S3 URL or path prefix in generated config")
}
}
func TestDeployRejectsUndecryptedStorageCredential(t *testing.T) {
d := NewLokiDeployer(&models.LokiDeployment{StorageConfig: &models.StorageConfig{
StorageType: "minio", AccessKey: "ENC:unavailable", SecretKey: "secret",
}}, nil)
if err := d.Deploy(context.Background()); err == nil || !strings.Contains(err.Error(), "存储凭证解密失败") {
t.Fatalf("expected credential failure, got %v", err)
}
}
func TestGeneratedLokiConfigKeepsRequiredSettingsWithoutDefaults(t *testing.T) {
d := NewLokiDeployer(&models.LokiDeployment{RetentionDays: 90, Replicas: 1, StorageConfig: &models.StorageConfig{
StorageType: "minio", Bucket: "logs", Endpoint: "http://minio:9000",
AccessKey: "key", SecretKey: "secret", Insecure: true,
}}, nil)
config := d.generateLokiConfig()
var parsed map[string]interface{}
if err := yaml.Unmarshal([]byte(config), &parsed); err != nil {
t.Fatalf("invalid Loki YAML: %v", err)
}
for _, required := range []string{"retention_period: 2160h", "retention_enabled: true", "delete_request_store: s3", "bucketnames: logs", "store: tsdb", "schema: v13"} {
if !strings.Contains(config, required) {
t.Fatalf("required setting %q missing", required)
}
}
for _, defaultSetting := range []string{"http_listen_port:", "grpc_listen_port:", "chunk_idle_period:", "compaction_interval:", "retention_delete_delay:", "max_entries_limit_per_query:", "frontend_worker:", "chunk_store_config:", "autoforget_unhealthy:"} {
if strings.Contains(config, defaultSetting) {
t.Fatalf("unnecessary setting %q remained", defaultSetting)
}
}
d.deployment.Replicas = 2
if !strings.Contains(d.generateLokiConfig(), "autoforget_unhealthy: true") {
t.Fatal("multi-replica ring recovery setting missing")
}
}
func TestK8sManifestKeepsStorageCredentialsInSecret(t *testing.T) {
d := NewLokiDeployer(&models.LokiDeployment{Mode: "k8s-yaml", RetentionDays: 60, Replicas: 1, StorageConfig: &models.StorageConfig{
StorageType: "minio", Bucket: "loki", Endpoint: "http://minio:9000",
AccessKey: "test-access", SecretKey: "test-secret", Insecure: true,
}}, nil)
manifest := d.generateManifest("log")
decoder := yaml.NewDecoder(strings.NewReader(manifest))
var configYAML string
var secret map[string]interface{}
for {
var doc map[string]interface{}
if err := decoder.Decode(&doc); err != nil {
if err == io.EOF {
break
}
t.Fatalf("invalid manifest YAML: %v", err)
}
switch doc["kind"] {
case "ConfigMap":
configYAML = doc["data"].(map[string]interface{})["loki.yaml"].(string)
case "Secret":
secret = doc["data"].(map[string]interface{})
}
}
if configYAML == "" || secret == nil {
t.Fatal("ConfigMap or Secret missing")
}
if strings.Contains(configYAML, "test-access") || strings.Contains(configYAML, "test-secret") {
t.Fatal("storage credential leaked into ConfigMap")
}
for _, marker := range []string{"${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}", "retention_period: 1440h", "retention_enabled: true"} {
if !strings.Contains(configYAML, marker) {
t.Fatalf("required Loki setting %q missing", marker)
}
}
if !strings.Contains(manifest, "-config.expand-env=true") || !strings.Contains(manifest, "secretKeyRef:") {
t.Fatal("StatefulSet does not reference storage Secret")
}
for key, want := range map[string]string{"accessKey": "test-access", "secretKey": "test-secret"} {
decoded, err := base64.StdEncoding.DecodeString(secret[key].(string))
if err != nil || string(decoded) != want {
t.Fatalf("Secret %s data mismatch", key)
}
}
}
func TestK8sManifestChangesPodTemplateWhenRetentionChanges(t *testing.T) {
deployment := &models.LokiDeployment{RetentionDays: 30, Replicas: 1}
d := NewLokiDeployer(deployment, nil)
checksum := regexp.MustCompile(`checksum/loki-config: "[a-f0-9]{64}"`)
first := checksum.FindString(d.generateManifest("log"))
deployment.RetentionDays = 70
second := checksum.FindString(d.generateManifest("log"))
if first == "" || second == "" || first == second {
t.Fatalf("Pod config checksum must change with retention: %q -> %q", first, second)
}
}
backend/services/deployment_retention.go
+79
View File
@@ -0,0 +1,79 @@
package services
import (
"fmt"
"io"
"regexp"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
var retentionPeriodPattern = regexp.MustCompile(`^([1-9][0-9]*)(d|h)$`)
// parseRetentionYAML 仅接收 limits_config.retention_period，避免把任意配置或凭证写入部署。
func parseRetentionYAML(source string) (int, error) {
var fragment struct {
LimitsConfig *struct {
RetentionPeriod string `yaml:"retention_period"`
} `yaml:"limits_config"`
}
decoder := yaml.NewDecoder(strings.NewReader(source))
decoder.KnownFields(true)
if err := decoder.Decode(&fragment); err != nil {
return 0, fmt.Errorf("YAML 格式无效: %w", err)
}
var extra interface{}
if err := decoder.Decode(&extra); err != io.EOF {
return 0, fmt.Errorf("只允许一段 limits_config YAML")
}
if fragment.LimitsConfig == nil {
return 0, fmt.Errorf("缺少 limits_config.retention_period")
}
match := retentionPeriodPattern.FindStringSubmatch(fragment.LimitsConfig.RetentionPeriod)
if match == nil {
return 0, fmt.Errorf("retention_period 须使用正整数天数，如 90d；也支持 2160h")
}
value, err := strconv.Atoi(match[1])
if err != nil {
return 0, fmt.Errorf("保留时长超出范围")
}
if match[2] == "h" {
if value%24 != 0 {
return 0, fmt.Errorf("小时数必须是 24 的倍数")
}
value /= 24
}
if value < 1 || value > 36500 {
return 0, fmt.Errorf("保留天数须在 1 到 36500 之间")
}
return value, nil
}
// UpdateRetention 保存 Loki 的保留时长；apply 为 true 时触发当前部署重新生成并应用配置。
func (s *DeploymentService) UpdateRetention(id uint, source string, apply bool) (int, error) {
days, err := parseRetentionYAML(source)
if err != nil {
return 0, err
}
deployment, err := s.Get(id)
if err != nil {
return 0, err
}
if deployment.Status == "deploying" {
return 0, fmt.Errorf("部署正在进行中，请稍后编辑")
}
if apply && deployment.Status != "running" {
return 0, fmt.Errorf("当前部署未运行，请仅保存后再部署")
}
if err := s.Update(id, map[string]interface{}{"retentionDays": float64(days)}); err != nil {
return 0, err
}
if apply {
if err := s.StartDeploy(id); err != nil {
return days, fmt.Errorf("保留配置已保存，但重新部署失败: %w", err)
}
}
return days, nil
}
backend/services/deployment_retention_test.go
+68
View File
@@ -0,0 +1,68 @@
package services
import (
"testing"
"opsxloki/repository"
)
func TestParseRetentionYAML(t *testing.T) {
for _, tc := range []struct {
name string
source string
want int
}{
{"days", "limits_config:\n retention_period: 90d\n", 90},
{"hours", "limits_config:\n retention_period: 2160h\n", 90},
{"unknown field", "limits_config:\n retention_period: 90d\n ingestion_rate_mb: 16\n", 0},
{"fractional day", "limits_config:\n retention_period: 25h\n", 0},
{"extra document", "limits_config:\n retention_period: 90d\n---\nfoo: bar\n", 0},
{"missing field", "limits_config: {}\n", 0},
} {
t.Run(tc.name, func(t *testing.T) {
got, err := parseRetentionYAML(tc.source)
if tc.want == 0 && err == nil {
t.Fatalf("accepted invalid YAML: %q", tc.source)
}
if tc.want != 0 && (err != nil || got != tc.want) {
t.Fatalf("got %d, err %v; want %d", got, err, tc.want)
}
})
}
}
func TestDeploymentRetentionPersistsAndRejectsInvalidValues(t *testing.T) {
t.Setenv("DATA_DIR", t.TempDir())
t.Setenv("JWT_SECRET", "test-jwt-secret-with-at-least-32-characters")
t.Setenv("ENCRYPTION_KEY", "test-encryption-key-with-at-least-32-characters")
if err := repository.InitDB(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = repository.Shutdown() })
service := NewDeploymentService()
deployment, err := service.Create(&CreateDeploymentRequest{
Name: "loki", Mode: "k8s-yaml", Version: "3.0.0",
})
if err != nil {
t.Fatal(err)
}
if deployment.RetentionDays != DefaultRetentionDays {
t.Fatalf("default retention = %d, want %d", deployment.RetentionDays, DefaultRetentionDays)
}
for _, invalid := range []float64{0, -1, 1.5} {
if err := service.Update(deployment.ID, map[string]interface{}{"retentionDays": invalid}); err == nil {
t.Fatalf("accepted invalid retention %v", invalid)
}
}
if err := service.Update(deployment.ID, map[string]interface{}{"retentionDays": float64(90)}); err != nil {
t.Fatal(err)
}
updated, err := service.Get(deployment.ID)
if err != nil || updated.RetentionDays != 90 {
t.Fatalf("updated retention = %v, err = %v", updated, err)
}
if err := repository.GetDB().RequestSaveSync(); err != nil {
t.Fatal(err)
}
}
backend/services/grafana_deployer.go
-438
View File
@@ -1,438 +0,0 @@
package services
import (
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
"os/exec"
"strings"
"time"
"opsxloki/models"
)
// GrafanaDeployer 与 Loki 同环境部署 Grafana 并自动配置 Loki 数据源。
// k8s 模式:manifest(Deployment + NodePort Service + 数据源 ConfigMap),与 Loki 同 namespace;
// host-binary 模式:tarball + systemd,与 Loki 同机。
type GrafanaDeployer struct {
deployment *models.LokiDeployment
log func(level, message, detail string)
}
const defaultGrafanaVersion = "11.3.0"
func (d *GrafanaDeployer) version() string {
if d.deployment.GrafanaVersion != "" {
return d.deployment.GrafanaVersion
}
return defaultGrafanaVersion
}
// lokiPushURL 数据源指向的 Loki 地址(k8s 走集群内服务名,主机走本机)
func (d *GrafanaDeployer) lokiPushURL() string {
if d.deployment.Mode == "host-binary" {
return "http://127.0.0.1:3100"
}
ns := d.deployment.Namespace
if ns == "" {
ns = "log"
}
return fmt.Sprintf("http://loki.%s.svc.cluster.local:3100", ns)
}
// Deploy 部署入口:按模式分发
func (d *GrafanaDeployer) Deploy(ctx context.Context) error {
d.log("info", fmt.Sprintf("Deploying Grafana %s (mode: %s)...", d.version(), d.deployment.Mode), "")
switch d.deployment.Mode {
case "k8s-helm", "k8s-yaml":
return d.deployK8s(ctx)
case "host-binary":
return d.deployHost(ctx)
default:
return fmt.Errorf("unsupported mode for grafana: %s", d.deployment.Mode)
}
}
// ==================== K8s manifest 部署 ====================
// grafanaManifest 生成 Grafana 资源;lokiURL 显式传入(默认服务名,DNS 异常的集群可回退 ClusterIP)。
// 数据卷用 emptyDir:数据源/配置由 ConfigMap 供给,重启即恢复;避免无默认
// StorageClass 的集群上 PVC Pending 阻塞(UI 内自建的面板需重新导入,见已知限制)。
func (d *GrafanaDeployer) grafanaManifest(ns, lokiURL string) string {
version := d.version()
datasources := fmt.Sprintf(`apiVersion: 1
datasources:
- name: Loki
type: loki
uid: loki-ds
access: proxy
url: %s
isDefault: true
editable: false`, lokiURL)
return fmt.Sprintf(`apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-datasources
namespace: %s
data:
datasources.yaml: |
%s
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
namespace: %s
labels:
app: grafana
spec:
replicas: 1
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
spec:
containers:
- name: grafana
image: grafana/grafana:%s
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3000
env:
- name: GF_SECURITY_ADMIN_USER
value: admin
- name: GF_SECURITY_ADMIN_PASSWORD
value: admin
readinessProbe:
httpGet:
path: /api/health
port: http
initialDelaySeconds: 10
periodSeconds: 10
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: datasources
mountPath: /etc/grafana/provisioning/datasources
volumes:
- name: datasources
configMap:
name: grafana-datasources
---
apiVersion: v1
kind: Service
metadata:
name: grafana
namespace: %s
labels:
app: grafana
spec:
# NodePort 便于集群外直接访问(端口由集群自动分配,实际端口见部署日志)
type: NodePort
ports:
- name: http
port: 3000
targetPort: 3000
selector:
app: grafana
`, ns, indentBlock(datasources, " "), ns, version, ns)
}
func (d *GrafanaDeployer) deployK8s(ctx context.Context) error {
if d.deployment.Cluster == nil {
return fmt.Errorf("please select target cluster")
}
env, cleanup, err := kubeconfigEnv(d.deployment.Cluster)
if err != nil {
return err
}
defer cleanup()
ns := d.deployment.Namespace
if ns == "" {
ns = "log"
}
lokiURL := d.lokiPushURL()
manifestFile, manifestCleanup, err := writeTempFile("grafana-*.yaml", d.grafanaManifest(ns, lokiURL), 0600)
if err != nil {
return err
}
defer manifestCleanup()
d.log("info", "Applying Grafana manifest (Deployment + NodePort Service + datasource ConfigMap)...", "")
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", manifestFile)
cmd.Env = env
out, err := cmd.CombinedOutput()
if err != nil {
d.log("error", fmt.Sprintf("kubectl apply grafana failed: %v", err), string(out))
return fmt.Errorf("kubectl apply grafana failed: %w\n%s", err, string(out))
}
waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
"-n", ns, "--timeout=5m")
waitCmd.Env = env
waitOut, _ := waitCmd.CombinedOutput()
d.log("info", string(waitOut), "")
// DNS 自愈:部分集群 CoreDNS 对 Service 记录异常(如 watch 失效),
// 经 grafana 容器探测 Loki 服务名,不通则回退 ClusterIP 并热更新数据源。
dnsProbe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, "deploy/grafana", "--",
"wget", "-q", "-O", "/dev/null", "-T", "5", lokiURL+"/ready")
dnsProbe.Env = env
if probeOut, probeErr := dnsProbe.CombinedOutput(); probeErr != nil {
d.log("warn", fmt.Sprintf("Grafana 无法解析 Loki 服务名(%s),回退 ClusterIP…", strings.TrimSpace(string(probeOut))), "")
ipCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
"-o", "jsonpath={.spec.clusterIP}")
ipCmd.Env = env
ipOut, ipErr := ipCmd.CombinedOutput()
clusterIP := strings.TrimSpace(string(ipOut))
if ipErr != nil || clusterIP == "" {
d.log("error", "获取 Loki ClusterIP 失败,数据源保持服务名形式", "")
} else {
fallbackURL := fmt.Sprintf("http://%s:3100", clusterIP)
fbFile, fbCleanup, ferr := writeTempFile("grafana-fb-*.yaml", d.grafanaManifest(ns, fallbackURL), 0600)
if ferr == nil {
fbApply := exec.CommandContext(ctx, "kubectl", "apply", "-f", fbFile)
fbApply.Env = env
fbApply.CombinedOutput()
fbCleanup()
restart := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "deployment/grafana", "-n", ns)
restart.Env = env
restart.CombinedOutput()
wait2 := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
"-n", ns, "--timeout=5m")
wait2.Env = env
w2, _ := wait2.CombinedOutput()
d.log("info", string(w2), "")
d.log("info", fmt.Sprintf("数据源已回退为 ClusterIP: %s", fallbackURL), "")
}
}
}
// 回读实际分配的 NodePort(固定端口易与集群已有服务冲突)
portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
"-o", "jsonpath={.spec.ports[0].nodePort}")
portCmd.Env = env
portOut, _ := portCmd.CombinedOutput()
nodePort := strings.TrimSpace(string(portOut))
access := fmt.Sprintf("http://<节点IP>:%s", nodePort)
if nodePort == "" {
access = "kubectl get svc grafana -n " + ns + " 查看访问端口"
}
d.log("success", fmt.Sprintf("Grafana deployed. 访问: %s (NodePort, 账号 admin/admin), 数据源已指向 %s", access, d.lokiPushURL()), "")
// 自动导入 Loki 日志面板(打开即可看图;密码被用户修改过则跳过并提示)
if err := d.importDashboard(ctx, env, ns); err != nil {
d.log("warn", fmt.Sprintf("面板自动导入跳过: %v", err), "")
} else {
d.log("info", "已导入「Loki 日志」面板(Explore 可用,默认账号 admin/admin)", "")
}
return nil
}
// importDashboard 通过 Grafana API 导入预置的 Loki 日志面板
func (d *GrafanaDeployer) importDashboard(ctx context.Context, env []string, ns string) error {
payload := grafanaDashboardPayload()
postDashboard := func(baseURL string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
strings.TrimSuffix(baseURL, "/")+"/api/dashboards/db", strings.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("admin:admin")))
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
return nil
}
switch d.deployment.Mode {
case "k8s-helm", "k8s-yaml":
// 经 NodePort 直连(kube service proxy 的 POST 需额外 RBAC,直连最稳)
ipCmd := exec.CommandContext(ctx, "kubectl", "get", "nodes",
"-o", "jsonpath={.items[0].status.addresses[0].address}")
ipCmd.Env = env
ipOut, _ := ipCmd.CombinedOutput()
portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
"-o", "jsonpath={.spec.ports[0].nodePort}")
portCmd.Env = env
portOut, _ := portCmd.CombinedOutput()
ip, port := strings.TrimSpace(string(ipOut)), strings.TrimSpace(string(portOut))
if ip == "" || port == "" {
return fmt.Errorf("获取 NodePort 失败")
}
return postDashboard(fmt.Sprintf("http://%s:%s", ip, port))
case "host-binary":
if d.deployment.Host == nil {
return fmt.Errorf("no host")
}
return postDashboard(fmt.Sprintf("http://%s:3000", d.deployment.Host.HostIP))
}
return fmt.Errorf("unsupported mode")
}
// ==================== 主机 tarball 部署 ====================
func (d *GrafanaDeployer) deployHost(ctx context.Context) error {
host := d.deployment.Host
if host == nil {
return fmt.Errorf("please select target host")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(host))
if err != nil {
return err
}
defer client.Close()
arch, _, _ := client.Execute(ctx, "uname -m")
grafanaArch := "amd64"
if strings.Contains(arch, "aarch64") || strings.Contains(arch, "arm64") {
grafanaArch = "arm64"
}
version := d.version()
d.log("info", fmt.Sprintf("Downloading Grafana %s (%s)...", version, grafanaArch), "")
url := fmt.Sprintf("https://dl.grafana.com/oss/release/grafana-%s.linux-%s.tar.gz", version, grafanaArch)
dl := fmt.Sprintf("curl -fsSL -o /tmp/grafana.tar.gz %s && tar -xzf /tmp/grafana.tar.gz -C /opt/ && rm -rf /opt/grafana && ln -sfn /opt/grafana-v%s /opt/grafana && rm -f /tmp/grafana.tar.gz", url, version)
if out, _, err := client.Execute(ctx, dl); err != nil {
return fmt.Errorf("download grafana failed: %w\n%s", err, out)
}
// 数据源 provisioning(指向同机 Loki)
dsYAML := fmt.Sprintf(`apiVersion: 1
datasources:
- name: Loki
type: loki
access: proxy
url: %s
isDefault: true
`, d.lokiPushURL())
client.Execute(ctx, "mkdir -p /etc/grafana/provisioning/datasources /var/lib/grafana /var/log/grafana")
writeCmd := fmt.Sprintf("cat > /etc/grafana/provisioning/datasources/loki.yaml <<'EOF'\n%s\nEOF", dsYAML)
if out, _, err := client.Execute(ctx, writeCmd); err != nil {
return fmt.Errorf("write datasource config failed: %w\n%s", err, out)
}
unit := `[Unit]
Description=Grafana
After=network.target
[Service]
Type=simple
User=root
WorkingDirectory=/opt/grafana
ExecStart=/opt/grafana/bin/grafana server --homepath /opt/grafana --config /opt/grafana/conf/defaults.ini
Restart=always
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target`
writeUnit := fmt.Sprintf("cat > /etc/systemd/system/grafana.service <<'EOF'\n%s\nEOF", unit)
if out, _, err := client.Execute(ctx, writeUnit); err != nil {
return fmt.Errorf("write systemd unit failed: %w\n%s", err, out)
}
if out, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable grafana && systemctl restart grafana"); err != nil {
return fmt.Errorf("start grafana failed: %w\n%s", err, out)
}
// 健康检查(最多 10 次)
for i := 0; i < 10; i++ {
out, _, _ := client.Execute(ctx, "curl -fsSL http://localhost:3000/api/health")
if strings.Contains(out, "\"version\"") {
d.log("success", fmt.Sprintf("Grafana running on %s:3000 (admin/admin), 数据源已指向本机 Loki", host.HostIP), "")
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
d.log("warn", "Grafana 服务已启动但健康检查未通过,请稍后确认 http://<主机IP>:3000", "")
return nil
}
// CleanupGrafana 卸载 Grafana(停止部署时同步调用)
func CleanupGrafana(ctx context.Context, deployment *models.LokiDeployment) {
if deployment == nil {
return
}
switch deployment.Mode {
case "k8s-helm", "k8s-yaml":
if deployment.Cluster == nil {
return
}
env, cleanup, err := kubeconfigEnv(deployment.Cluster)
if err != nil {
return
}
defer cleanup()
ns := deployment.Namespace
if ns == "" {
ns = "log"
}
cmd := exec.CommandContext(ctx, "kubectl", "delete",
"deployment/grafana", "service/grafana", "configmap/grafana-datasources",
"-n", ns, "--ignore-not-found=true")
cmd.Env = env
cmd.CombinedOutput()
case "host-binary":
if deployment.Host == nil {
return
}
client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
if err != nil {
return
}
defer client.Close()
client.Execute(ctx, "systemctl stop grafana 2>/dev/null; systemctl disable grafana 2>/dev/null; rm -f /etc/systemd/system/grafana.service; systemctl daemon-reload")
}
}
// grafanaDashboardPayload 预置「Loki 日志」面板:日志量(按命名空间)/错误日志/最近日志
func grafanaDashboardPayload() string {
return `{"dashboard":{
"uid":"loki-logs","title":"Loki 日志","tags":["lokideploy"],
"timezone":"browser","schemaVersion":39,"refresh":"30s","version":1,
"time":{"from":"now-1h","to":"now"},
"panels":[
{"id":1,"type":"timeseries","title":"日志量(按命名空间)",
"datasource":{"type":"loki","uid":"loki-ds"},
"gridPos":{"h":8,"w":12,"x":0,"y":0},
"fieldConfig":{"defaults":{"unit":"short"},"overrides":[]},
"targets":[{"refId":"A","expr":"sum by (namespace) (count_over_time({namespace=\".+\"}[5m]))","queryType":"range"}]},
{"id":2,"type":"timeseries","title":"错误日志",
"datasource":{"type":"loki","uid":"loki-ds"},
"gridPos":{"h":8,"w":12,"x":12,"y":0},
"fieldConfig":{"defaults":{"unit":"short","color":{"mode":"palette-classic"}},"overrides":[]},
"targets":[{"refId":"A","expr":"sum(count_over_time({namespace=\".+\"} |~ \"(?i)error\" [5m]))","queryType":"range"}]},
{"id":3,"type":"logs","title":"最近日志",
"datasource":{"type":"loki","uid":"loki-ds"},
"gridPos":{"h":10,"w":24,"x":0,"y":8},
"targets":[{"refId":"A","expr":"{namespace=\".+\"}","queryType":"range"}]}
]
},"overwrite":true}`
}
backend/services/host.go
+27
View File
@@ -120,6 +120,33 @@ func (s *HostService) Update(id uint, updates map[string]interface{}) error {
if v, ok := updates["arch"].(string); ok {
db.Hosts[i].Arch = v
}
if v, ok := updates["hostIP"].(string); ok && v != "" {
db.Hosts[i].HostIP = v
}
if v, ok := updates["port"].(float64); ok && v > 0 {
db.Hosts[i].Port = int(v)
}
if v, ok := updates["user"].(string); ok && v != "" {
db.Hosts[i].User = v
}
if v, ok := updates["labels"].(string); ok {
db.Hosts[i].Labels = v
}
// 凭证编辑约定:非空才更新(留空=保留旧密文)
for _, field := range []string{"password", "privateKey", "keyPhrase"} {
if v, ok := updates[field].(string); ok && v != "" {
if enc, err := crypto.EncryptField(v); err == nil {
switch field {
case "password":
db.Hosts[i].Password = enc
case "privateKey":
db.Hosts[i].PrivateKey = enc
case "keyPhrase":
db.Hosts[i].KeyPhrase = enc
}
}
}
}
db.Hosts[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
backend/services/image_registry.go
+131
View File
@@ -0,0 +1,131 @@
package services
// 镜像仓库源 — 全局设置(参照 kuboard 的国内/海外镜像选择):
// overseas = docker.io 官方源;cn = daocloud 国内镜像;custom = 用户自填 registry 前缀。
// 影响平台拉取的全部镜像:Loki / Grafana / Promtail / Fluent Bit。
import (
"encoding/json"
"fmt"
"strings"
"opsxloki/models"
"opsxloki/repository"
)
const (
imageRegistrySettingKey = "image_registry"
RegistryOverseas = "overseas" // docker.io
RegistryCN = "cn" // docker.m.daocloud.io(国内加速)
RegistryCustom = "custom" // 用户私有仓库前缀
daocloudPrefix = "docker.m.daocloud.io"
)
// ImageRegistryConfig 镜像源配置
type ImageRegistryConfig struct {
Mode string `json:"mode"` // overseas | cn | custom
CustomHost string `json:"customHost"` // custom 模式下的仓库前缀,如 swr.cn-east-2.myhuaweicloud.com/xxx
}
type ImageRegistryPublic struct {
Mode string `json:"mode"`
CustomHost string `json:"customHost"`
// 示例预览(前端展示生效镜像)
SampleLoki string `json:"sampleLoki"`
SampleGrafana string `json:"sampleGrafana"`
SamplePromtail string `json:"samplePromtail"`
SampleFluentBit string `json:"sampleFluentBit"`
}
func GetImageRegistry() *ImageRegistryConfig {
db := repository.GetDB()
if db == nil {
return &ImageRegistryConfig{Mode: RegistryOverseas}
}
db.RLock()
var raw string
for i := range db.Settings {
if db.Settings[i].Key == imageRegistrySettingKey {
raw = db.Settings[i].Value
break
}
}
db.RUnlock()
cfg := &ImageRegistryConfig{Mode: RegistryOverseas}
if raw != "" {
_ = json.Unmarshal([]byte(raw), cfg)
}
if cfg.Mode == "" {
cfg.Mode = RegistryOverseas
}
return cfg
}
func SaveImageRegistry(mode, customHost string) error {
if mode != RegistryOverseas && mode != RegistryCN && mode != RegistryCustom {
return fmt.Errorf("无效的镜像源模式")
}
if mode == RegistryCustom {
customHost = strings.TrimSuffix(strings.TrimSpace(customHost), "/")
if customHost == "" || !strings.Contains(customHost, ".") {
return fmt.Errorf("自定义镜像仓库地址无效(需含域名,如 swr.cn-east-2.myhuaweicloud.com/your-ns)")
}
}
data, _ := json.Marshal(&ImageRegistryConfig{Mode: mode, CustomHost: customHost})
db := repository.GetDB()
newID := db.NextID("settings")
db.Lock()
defer db.Unlock()
for i := range db.Settings {
if db.Settings[i].Key == imageRegistrySettingKey {
db.Settings[i].Value = string(data)
db.RequestSave()
return nil
}
}
db.Settings = append(db.Settings, models.SystemSetting{ID: newID, Key: imageRegistrySettingKey, Value: string(data)})
db.RequestSave()
return nil
}
// ResolveImage 按当前镜像源解析镜像地址:docker.io/grafana/loki:3.4.0 形式的官方镜像
// 会按 mode 重写前缀;已带私有仓库前缀(含镜像源域名)的地址原样返回。
func ResolveImage(official string) string {
cfg := GetImageRegistry()
switch cfg.Mode {
case RegistryCN:
if strings.HasPrefix(official, "docker.io/") {
return daocloudPrefix + "/" + strings.TrimPrefix(official, "docker.io/")
}
if !strings.Contains(strings.SplitN(official, "/", 2)[0], ".") {
// 无仓库前缀的官方镜像(grafana/loki:tag)
return daocloudPrefix + "/" + official
}
return official
case RegistryCustom:
if strings.HasPrefix(official, "docker.io/") {
return cfg.CustomHost + "/" + strings.TrimPrefix(official, "docker.io/")
}
if !strings.Contains(strings.SplitN(official, "/", 2)[0], ".") {
return cfg.CustomHost + "/" + official
}
return official
default:
if strings.HasPrefix(official, "docker.io/") {
return official // 官方模式保留全限定名
}
return official
}
}
// ImageRegistryPublicConfig 对外输出(含示例预览)
func ImageRegistryPublicConfig() *ImageRegistryPublic {
cfg := GetImageRegistry()
return &ImageRegistryPublic{
Mode: cfg.Mode,
CustomHost: cfg.CustomHost,
SampleLoki: ResolveImage("docker.io/grafana/loki:3.4.0"),
SampleGrafana: ResolveImage("docker.io/grafana/grafana:11.3.0"),
SamplePromtail: ResolveImage("docker.io/grafana/promtail:3.4.0"),
SampleFluentBit: ResolveImage("docker.io/fluent/fluent-bit:5.1.5"),
}
}
backend/services/ingress_access.go
+184
View File
@@ -0,0 +1,184 @@
package services
import (
"context"
"fmt"
"strings"
"time"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"opsxloki/models"
)
const ingressNamespace = "ingress-nginx"
const ingressControllerName = "ingress-nginx-controller"
// The field names are shared with the JSON parser on the Loki query page.
const ingressAccessFormat = `{"time":"$time_iso8601","remote_addr":"$remote_addr","host":"$host","http_host":"$http_host","request":"$request","method":"$request_method","uri":"$request_uri","status":$status,"body_bytes_sent":$body_bytes_sent,"http_referer":"$http_referer","http_user_agent":"$http_user_agent","request_length":$request_length,"request_time":$request_time,"proxy_upstream_name":"$proxy_upstream_name","upstream_addr":"$upstream_addr","upstream_status":"$upstream_status","upstream_response_time":"$upstream_response_time"}`
type IngressAccessStatus struct {
Namespace string `json:"namespace"`
Deployment string `json:"deployment"`
Image string `json:"image"`
ConfigMap string `json:"configMap"`
ConfigMapAttached bool `json:"configMapAttached"`
CurrentFormat string `json:"currentFormat"`
TargetFormat string `json:"targetFormat"`
Standardized bool `json:"standardized"`
ManagedBy string `json:"managedBy"`
}
func ingressConfigArg(args []string) string {
for i, arg := range args {
if strings.HasPrefix(arg, "--configmap=") {
return strings.TrimPrefix(arg, "--configmap=")
}
if arg == "--configmap" && i+1 < len(args) {
return args[i+1]
}
}
return ""
}
func ingressAccessState(ctx context.Context, cluster *models.Cluster) (IngressAccessStatus, error) {
client, err := clusterClient(cluster, 10*time.Second)
if err != nil {
return IngressAccessStatus{}, err
}
dep, err := client.AppsV1().Deployments(ingressNamespace).Get(ctx, ingressControllerName, metav1.GetOptions{})
if err != nil {
return IngressAccessStatus{}, fmt.Errorf("读取 ingress-nginx 控制器失败: %w", err)
}
index := -1
for i, container := range dep.Spec.Template.Spec.Containers {
if strings.Contains(container.Image, "ingress-nginx/controller:") {
index = i
break
}
}
if index < 0 {
return IngressAccessStatus{}, fmt.Errorf("未识别到社区 ingress-nginx controller 容器")
}
container := dep.Spec.Template.Spec.Containers[index]
configName := ingressConfigArg(container.Args)
if configName == "" {
configName = ingressNamespace + "/" + ingressControllerName
}
parts := strings.Split(configName, "/")
if len(parts) != 2 || parts[0] != ingressNamespace || parts[1] == "" {
return IngressAccessStatus{}, fmt.Errorf("控制器使用其他命名空间的 ConfigMap: %s", configName)
}
cm, err := client.CoreV1().ConfigMaps(parts[0]).Get(ctx, parts[1], metav1.GetOptions{})
if err != nil {
return IngressAccessStatus{}, fmt.Errorf("读取 ingress-nginx ConfigMap 失败: %w", err)
}
format := cm.Data["log-format-upstream"]
return IngressAccessStatus{
Namespace: ingressNamespace, Deployment: dep.Name, Image: container.Image,
ConfigMap: configName, ConfigMapAttached: ingressConfigArg(container.Args) != "",
CurrentFormat: format, TargetFormat: ingressAccessFormat,
Standardized: format == ingressAccessFormat && cm.Data["log-format-escape-json"] == "true" && ingressConfigArg(container.Args) != "",
ManagedBy: dep.Labels["app.kubernetes.io/managed-by"],
}, nil
}
func InspectIngressAccess(ctx context.Context, cluster *models.Cluster) (IngressAccessStatus, error) {
return ingressAccessState(ctx, cluster)
}
// ApplyIngressAccess changes only the two log keys, preserving unrelated ingress settings.
// Explicitly attaching the ConfigMap is necessary for controllers launched without --configmap.
func ApplyIngressAccess(ctx context.Context, cluster *models.Cluster) (IngressAccessStatus, error) {
state, err := ingressAccessState(ctx, cluster)
if err != nil {
return state, err
}
client, err := clusterClient(cluster, 10*time.Second)
if err != nil {
return state, err
}
parts := strings.Split(state.ConfigMap, "/")
cm, err := client.CoreV1().ConfigMaps(parts[0]).Get(ctx, parts[1], metav1.GetOptions{})
if err != nil {
return state, err
}
if !state.ConfigMapAttached {
for key := range cm.Data {
if key != "log-format-upstream" && key != "log-format-escape-json" {
return state, fmt.Errorf("控制器尚未关联 ConfigMap，且其中已有 %s 等设置；为避免意外启用原有配置，已跳过自动配置", key)
}
}
}
previousFormat := strings.Replace(ingressAccessFormat, `"request":"$request",`, "", 1)
if current := cm.Data["log-format-upstream"]; current != "" && current != ingressAccessFormat && current != previousFormat {
return state, fmt.Errorf("已有自定义 access log 格式；为避免覆盖，已跳过自动配置")
}
if state.Standardized {
if err := ensureIngressRolloutStrategy(ctx, cluster); err != nil {
return state, err
}
return state, nil
}
if cm.Data == nil {
cm.Data = make(map[string]string)
}
cm.Data["log-format-upstream"] = ingressAccessFormat
cm.Data["log-format-escape-json"] = "true"
if _, err = client.CoreV1().ConfigMaps(parts[0]).Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
return state, fmt.Errorf("更新 ingress-nginx ConfigMap 失败: %w", err)
}
if !state.ConfigMapAttached {
dep, err := client.AppsV1().Deployments(ingressNamespace).Get(ctx, ingressControllerName, metav1.GetOptions{})
if err != nil {
return state, err
}
for i := range dep.Spec.Template.Spec.Containers {
container := &dep.Spec.Template.Spec.Containers[i]
if strings.Contains(container.Image, "ingress-nginx/controller:") {
container.Args = append(container.Args, "--configmap="+state.ConfigMap)
break
}
}
setIngressRolloutStrategy(dep)
if _, err = client.AppsV1().Deployments(ingressNamespace).Update(ctx, dep, metav1.UpdateOptions{}); err != nil {
return state, fmt.Errorf("ConfigMap 已更新，但控制器尚未关联配置；更新 Deployment 失败: %w", err)
}
}
return ingressAccessState(ctx, cluster)
}
// Host-network replicas occupying every ingress node cannot use a surge pod:
// the new pod would need the same 80/443 ports as an existing replica.
func setIngressRolloutStrategy(dep *appsv1.Deployment) bool {
if !dep.Spec.Template.Spec.HostNetwork || dep.Spec.Replicas == nil || *dep.Spec.Replicas < 2 {
return false
}
zero, one := intstr.FromInt32(0), intstr.FromInt32(1)
strategy := &dep.Spec.Strategy
if strategy.Type == appsv1.RecreateDeploymentStrategyType {
return false
}
if strategy.RollingUpdate != nil && strategy.RollingUpdate.MaxSurge != nil && *strategy.RollingUpdate.MaxSurge == zero && strategy.RollingUpdate.MaxUnavailable != nil && *strategy.RollingUpdate.MaxUnavailable == one {
return false
}
strategy.Type = appsv1.RollingUpdateDeploymentStrategyType
strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{MaxSurge: &zero, MaxUnavailable: &one}
return true
}
func ensureIngressRolloutStrategy(ctx context.Context, cluster *models.Cluster) error {
client, err := clusterClient(cluster, 10*time.Second)
if err != nil {
return err
}
dep, err := client.AppsV1().Deployments(ingressNamespace).Get(ctx, ingressControllerName, metav1.GetOptions{})
if err != nil {
return err
}
if setIngressRolloutStrategy(dep) {
_, err = client.AppsV1().Deployments(ingressNamespace).Update(ctx, dep, metav1.UpdateOptions{})
}
return err
}
backend/services/ingress_access_test.go
+37
View File
@@ -0,0 +1,37 @@
package services
import (
"strings"
"testing"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
)
func TestIngressAccessConfiguration(t *testing.T) {
if got := ingressConfigArg([]string{"/nginx-ingress-controller", "--configmap=ingress-nginx/ingress-nginx-controller"}); got != "ingress-nginx/ingress-nginx-controller" {
t.Fatalf("unexpected configmap: %q", got)
}
if got := ingressConfigArg([]string{"--configmap", "custom/controller"}); got != "custom/controller" {
t.Fatalf("unexpected separate configmap flag: %q", got)
}
for _, field := range []string{`"host":"$host"`, `"http_host":"$http_host"`, `"request":"$request"`, `"http_referer":"$http_referer"`, `"method":"$request_method"`, `"status":$status`} {
if !strings.Contains(ingressAccessFormat, field) {
t.Errorf("missing access log field %s", field)
}
}
}
func TestIngressHostNetworkRolloutStrategy(t *testing.T) {
replicas := int32(2)
dep := &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: &replicas, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{HostNetwork: true}}}}
if !setIngressRolloutStrategy(dep) {
t.Fatal("expected host-network rollout adjustment")
}
if dep.Spec.Strategy.RollingUpdate.MaxSurge.IntVal != 0 || dep.Spec.Strategy.RollingUpdate.MaxUnavailable.IntVal != 1 {
t.Fatalf("unexpected rollout strategy: %+v", dep.Spec.Strategy.RollingUpdate)
}
if setIngressRolloutStrategy(dep) {
t.Fatal("strategy update must be idempotent")
}
}
backend/services/k8s_client.go
+35
View File
@@ -5,8 +5,12 @@ import (
"fmt"
"os"
"regexp"
"time"
"opsxloki/models"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
// resolveKubeconfig 取集群最终的 kubeconfig 文本（优先内存字段，其次磁盘文件）
@@ -24,6 +28,37 @@ func resolveKubeconfig(cluster *models.Cluster) (string, error) {
return "", fmt.Errorf("集群未配置 kubeconfig")
}
// clusterRESTConfig 使用集群保存的 kubeconfig 构造原生 API Server 客户端配置。
func clusterRESTConfig(cluster *models.Cluster, timeout time.Duration) (*rest.Config, error) {
source, err := resolveKubeconfig(cluster)
if err != nil {
return nil, err
}
config, err := clientcmd.Load([]byte(source))
if err != nil {
return nil, fmt.Errorf("解析集群 kubeconfig 失败: %w", err)
}
overrides := &clientcmd.ConfigOverrides{}
if cluster.Context != "" {
overrides.CurrentContext = cluster.Context
}
result, err := clientcmd.NewNonInteractiveClientConfig(*config, config.CurrentContext, overrides, nil).ClientConfig()
if err != nil {
return nil, fmt.Errorf("构造 Kubernetes 客户端失败: %w", err)
}
result.Timeout = timeout
result.UserAgent = "LokiDeploy"
return result, nil
}
func clusterClient(cluster *models.Cluster, timeout time.Duration) (kubernetes.Interface, error) {
config, err := clusterRESTConfig(cluster, timeout)
if err != nil {
return nil, err
}
return kubernetes.NewForConfig(config)
}
// BuildKubeconfigFromToken 用 API Server + Token 构造最小 kubeconfig
// 优先使用 CA 证书验证 TLS，仅在用户显式要求时跳过证书验证
// Parameters:
backend/services/k8s_resources.go
+331
View File
@@ -0,0 +1,331 @@
package services
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
yamlutil "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
kcScheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/remotecommand"
"opsxloki/models"
)
func ensureNamespace(ctx context.Context, cluster *models.Cluster, namespace string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
_, err = client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
_, err = client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{})
}
return err
}
// applyManifest 使用 API Server 的 server-side apply，保留已有 YAML 模板与字段所有权。
func applyManifest(ctx context.Context, cluster *models.Cluster, manifest string) error {
cfg, err := clusterRESTConfig(cluster, 30*time.Second)
if err != nil {
return err
}
dc, err := discovery.NewDiscoveryClientForConfig(cfg)
if err != nil {
return err
}
groups, err := restmapper.GetAPIGroupResources(dc)
if err != nil {
return fmt.Errorf("读取 Kubernetes API 资源列表: %w", err)
}
mapper := restmapper.NewDiscoveryRESTMapper(groups)
client, err := dynamic.NewForConfig(cfg)
if err != nil {
return err
}
decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(manifest), 4096)
for {
var raw map[string]interface{}
if err := decoder.Decode(&raw); err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("解析 Kubernetes YAML: %w", err)
}
if len(raw) == 0 {
continue
}
obj := &unstructured.Unstructured{Object: raw}
mapping, err := mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version)
if err != nil {
return err
}
name := obj.GetName()
if name == "" {
return fmt.Errorf("Kubernetes %s 缺少 metadata.name", obj.GetKind())
}
data, err := obj.MarshalJSON()
if err != nil {
return err
}
var resource dynamic.ResourceInterface = client.Resource(mapping.Resource)
if mapping.Scope.Name() == "namespace" {
resource = client.Resource(mapping.Resource).Namespace(obj.GetNamespace())
}
force := true
if _, err := resource.Patch(ctx, name, types.ApplyPatchType, data, metav1.PatchOptions{FieldManager: "lokideploy", Force: &force}); err != nil {
return fmt.Errorf("应用 %s %s/%s: %w", obj.GetKind(), obj.GetNamespace(), name, err)
}
}
return nil
}
func waitStatefulSet(ctx context.Context, cluster *models.Cluster, namespace, name string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
limit, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) {
item, err := client.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return false, err
}
desired := int32(1)
if item.Spec.Replicas != nil {
desired = *item.Spec.Replicas
}
return item.Status.ObservedGeneration >= item.Generation && item.Status.ReadyReplicas >= desired && item.Status.UpdatedReplicas >= desired, nil
})
}
func waitDeployment(ctx context.Context, cluster *models.Cluster, namespace, name string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
limit, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) {
item, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return false, err
}
desired := int32(1)
if item.Spec.Replicas != nil {
desired = *item.Spec.Replicas
}
return item.Status.ObservedGeneration >= item.Generation && item.Status.ReadyReplicas >= desired && item.Status.UpdatedReplicas >= desired, nil
})
}
func waitDaemonSet(ctx context.Context, cluster *models.Cluster, namespace, name string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
limit, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
return wait.PollUntilContextCancel(limit, 3*time.Second, true, func(ctx context.Context) (bool, error) {
item, err := client.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return false, err
}
return item.Status.ObservedGeneration >= item.Generation && item.Status.NumberReady >= item.Status.DesiredNumberScheduled && item.Status.UpdatedNumberScheduled >= item.Status.DesiredNumberScheduled, nil
})
}
func restartDeployment(ctx context.Context, cluster *models.Cluster, namespace, name string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
item, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return err
}
item.Spec.Template.Annotations = copyAnnotations(item.Spec.Template.Annotations)
item.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().UTC().Format(time.RFC3339)
_, err = client.AppsV1().Deployments(namespace).Update(ctx, item, metav1.UpdateOptions{})
return err
}
func copyAnnotations(source map[string]string) map[string]string {
result := map[string]string{}
for k, v := range source {
result[k] = v
}
return result
}
func deleteK8sResource(ctx context.Context, cluster *models.Cluster, kind, namespace, name string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
var result error
switch kind {
case "statefulset":
result = client.AppsV1().StatefulSets(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "deployment":
result = client.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "daemonset":
result = client.AppsV1().DaemonSets(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "service":
result = client.CoreV1().Services(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "configmap":
result = client.CoreV1().ConfigMaps(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "serviceaccount":
result = client.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "persistentvolumeclaim":
result = client.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, name, metav1.DeleteOptions{})
case "clusterrole":
result = client.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
case "clusterrolebinding":
result = client.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
default:
return fmt.Errorf("未知 Kubernetes 资源: %s", kind)
}
if apierrors.IsNotFound(result) {
return nil
}
return result
}
func serviceClusterIP(ctx context.Context, cluster *models.Cluster, namespace, name string) (string, error) {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return "", err
}
svc, err := client.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return "", err
}
return svc.Spec.ClusterIP, nil
}
func serviceNodePort(ctx context.Context, cluster *models.Cluster, namespace, name string) (int32, error) {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return 0, err
}
svc, err := client.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return 0, err
}
if len(svc.Spec.Ports) == 0 {
return 0, fmt.Errorf("Service %s/%s 没有端口", namespace, name)
}
return svc.Spec.Ports[0].NodePort, nil
}
func firstNodeIP(ctx context.Context, cluster *models.Cluster) (string, error) {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return "", err
}
nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1})
if err != nil {
return "", err
}
for _, node := range nodes.Items {
for _, address := range node.Status.Addresses {
if address.Type == corev1.NodeInternalIP {
return address.Address, nil
}
}
}
return "", fmt.Errorf("集群无可用节点内网地址")
}
func patchDeployment(ctx context.Context, cluster *models.Cluster, namespace, name, yamlPatch string) error {
client, err := clusterClient(cluster, 30*time.Second)
if err != nil {
return err
}
patch, err := yamlutil.ToJSON([]byte(yamlPatch))
if err != nil {
return err
}
_, err = client.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{})
return err
}
// podDNSBroken 在指定 deployment 的容器里探测 hostname 是否可解析(wget /ready 连通性)。
func podDNSBroken(ctx context.Context, cluster *models.Cluster, namespace, deployment, container, host string) bool {
pods, err := listDeploymentPods(ctx, cluster, namespace, deployment)
if err != nil || len(pods) == 0 {
return false // 拿不到 pod 时不做自愈,交给后续日志排查
}
client, err := clusterClient(cluster, 15*time.Second)
if err != nil {
return false
}
restCfg, err := clusterRESTConfig(cluster, 15*time.Second)
if err != nil {
return false
}
execReq := client.CoreV1().RESTClient().Post().
Namespace(namespace).Resource("pods").Name(pods[0]).
SubResource("exec").
VersionedParams(&corev1.PodExecOptions{
Command: []string{"sh", "-c", "wget -q -O /dev/null -T 3 http://" + host + ":3100/ready"},
Container: container,
Stdout: true, Stderr: true,
}, kcScheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", execReq.URL())
if err != nil {
return false
}
var out bytes.Buffer
err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &out, Stderr: &out})
return err != nil // 执行失败(连接/解析不通)视为 DNS 故障
}
// listDeploymentPods 按 deployment selector 列出 pod 名。
func listDeploymentPods(ctx context.Context, cluster *models.Cluster, namespace, deployment string) ([]string, error) {
client, err := clusterClient(cluster, 15*time.Second)
if err != nil {
return nil, err
}
dep, err := client.AppsV1().Deployments(namespace).Get(ctx, deployment, metav1.GetOptions{})
if err != nil {
return nil, err
}
selector := metav1.FormatLabelSelector(dep.Spec.Selector)
podList, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
if err != nil {
return nil, err
}
names := make([]string, 0, len(podList.Items))
for i := range podList.Items {
names = append(names, podList.Items[i].Name)
}
return names, nil
}
// lokiNamespaceOf 从 endpoint 的 svc FQDN 提取 Loki 命名空间(log.ns.svc → ns)。
func lokiNamespaceOf(loki *models.LokiDeployment, endpoint string) string {
if loki != nil && loki.Namespace != "" {
return loki.Namespace
}
parts := strings.SplitN(endpoint, "//", 2)
if len(parts) == 2 {
host := strings.SplitN(parts[1], ".", 3)
if len(host) >= 2 {
return host[1]
}
}
return "log"
}
backend/services/ldap.go
+11 -11
View File
@@ -27,17 +27,17 @@ func netDialer() *net.Dialer {
// LDAPConfig LDAP 配置(存储形态;BindPassword 以 ENC: 前缀 AES-GCM 加密落库)
type LDAPConfig struct {
Enabled bool `json:"enabled"`
Host string `json:"host"`
Port int `json:"port"`
UseSSL bool `json:"useSSL"`
BindDN string `json:"bindDN"`
BindPassword string `json:"bindPassword"` // 存储为 ENC:密文;对外输出时剔除
BaseDN string `json:"baseDN"`
UserFilter string `json:"userFilter"`
UsernameAttr string `json:"usernameAttr"`
NameAttr string `json:"nameAttr"`
EmailAttr string `json:"emailAttr"`
Enabled bool `json:"enabled"`
Host string `json:"host"`
Port int `json:"port"`
UseSSL bool `json:"useSSL"`
BindDN string `json:"bindDN"`
BindPassword string `json:"bindPassword"` // 存储为 ENC:密文;对外输出时剔除
BaseDN string `json:"baseDN"`
UserFilter string `json:"userFilter"`
UsernameAttr string `json:"usernameAttr"`
NameAttr string `json:"nameAttr"`
EmailAttr string `json:"emailAttr"`
}
// LDAPPublicConfig 对外输出形态(密码剔除,附 hasPassword)
backend/services/log_policy.go
+3 -1
View File
@@ -18,7 +18,9 @@ func ListLogPolicyGroups() []models.LogPolicyGroup {
groups[i] = group
groups[i].UserIDs = append([]uint(nil), group.UserIDs...)
groups[i].Labels = make(map[string]string, len(group.Labels))
for key, value := range group.Labels { groups[i].Labels[key] = value }
for key, value := range group.Labels {
groups[i].Labels[key] = value
}
}
return groups
}
backend/services/log_query.go
+74 -349
View File
@@ -12,360 +12,34 @@ import (
"io"
"net/http"
"net/url"
"os/exec"
"regexp"
"sort"
"strconv"
"strings"
"time"
"opsxloki/models"
"opsxloki/pkg/crypto"
"opsxloki/repository"
"k8s.io/client-go/rest"
)
const maxLogQueryLimit = 1000
const logQueryTimeout = 120 * time.Second
// ==================== 数据源 CRUD ====================
func GetLogSource(id uint) *models.LogSource {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
for i := range db.LogSources {
if db.LogSources[i].ID == id {
s := db.LogSources[i]
return &s
}
}
return nil
var lokiHTTPClients = [2]*http.Client{
{Timeout: logQueryTimeout, Transport: newLokiTransport(false)},
{Timeout: logQueryTimeout, Transport: newLokiTransport(true)},
}
func ListLogSources() []map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
out := make([]map[string]interface{}, 0, len(db.LogSources))
for i := range db.LogSources {
out = append(out, db.LogSources[i].Sanitized())
}
return out
}
func ListVisibleLogSources(userID uint, admin bool) []map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
out := make([]map[string]interface{}, 0)
for i := range db.LogSources {
s := &db.LogSources[i]
if admin || policyGroupAccess(db.LogPolicyGroups, s.ID, userID) != nil || sourceAccess(s, userID) != nil {
item := s.Sanitized()
if !admin {
item = map[string]interface{}{"id": s.ID, "name": s.Name, "isDefault": s.IsDefault}
}
out = append(out, item)
}
}
return out
}
func sourceAccess(src *models.LogSource, userID uint) *models.LogSourceAccess {
for i := range src.Access {
if src.Access[i].UserID == userID {
return &src.Access[i]
}
}
return nil
}
var labelNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
func ValidLogLabelName(name string) bool { return labelNamePattern.MatchString(name) }
// AuthorizedLogQuery constrains the Loki stream selector server-side. Non-admins must be explicitly granted access.
func AuthorizedLogQuery(src *models.LogSource, userID uint, admin bool, query string) (string, error) {
if admin {
return query, nil
}
db := repository.GetDB()
var access *models.LogSourceAccess
if db != nil {
db.RLock()
access = policyGroupAccess(db.LogPolicyGroups, src.ID, userID)
db.RUnlock()
}
if access == nil {
access = sourceAccess(src, userID)
}
if access == nil {
return "", fmt.Errorf("无权查询此数据源")
}
q := strings.TrimSpace(query)
if !strings.HasPrefix(q, "{") {
return "", fmt.Errorf("普通用户查询必须以日志流选择器开头")
}
close := strings.IndexByte(q, '}')
if close < 0 || strings.Contains(q[close+1:], "{") {
return "", fmt.Errorf("查询只能包含一个日志流选择器")
}
if len(access.Labels) == 0 {
return q, nil
}
keys := make([]string, 0, len(access.Labels))
for k := range access.Labels {
if !labelNamePattern.MatchString(k) {
return "", fmt.Errorf("访问策略包含无效标签名")
}
keys = append(keys, k)
}
sort.Strings(keys)
matchers := make([]string, 0, len(keys))
for _, k := range keys {
v, _ := json.Marshal(access.Labels[k])
matchers = append(matchers, k+"="+string(v))
}
inner := strings.TrimSpace(q[1:close])
if inner != "" {
inner += ","
}
return "{" + inner + strings.Join(matchers, ",") + "}" + q[close+1:], nil
}
func CreateLogSource(s *models.LogSource) (*models.LogSource, error) {
if err := encryptSourceCredentials(s); err != nil {
return nil, err
}
db := repository.GetDB()
newID := db.NextID("logSources")
db.Lock()
defer db.Unlock()
s.ID = newID
if s.IsDefault {
for i := range db.LogSources {
db.LogSources[i].IsDefault = false
}
}
// 首个数据源自动设默认
if len(db.LogSources) == 0 {
s.IsDefault = true
}
db.LogSources = append(db.LogSources, *s)
db.RequestSave()
return s, nil
}
func encryptSourceCredentials(s *models.LogSource) error {
if s.BasicPass != "" {
v, err := crypto.EncryptField(s.BasicPass)
if err != nil {
return err
}
s.BasicPass = v
}
if s.BearerToken != "" {
v, err := crypto.EncryptField(s.BearerToken)
if err != nil {
return err
}
s.BearerToken = v
}
for name, value := range s.Headers {
if !validSourceHeader(name) {
return fmt.Errorf("自定义 Header 名称无效: %s", name)
}
v, err := crypto.EncryptField(value)
if err != nil {
return err
}
s.Headers[name] = v
}
return nil
}
func validSourceHeader(name string) bool {
if name == "" || strings.EqualFold(name, "Host") || strings.EqualFold(name, "Authorization") {
return false
}
for _, c := range name {
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
return false
}
}
return true
}
func UpdateLogSource(id uint, patch map[string]interface{}) error {
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.LogSources {
if db.LogSources[i].ID != id {
continue
}
if v, ok := patch["name"].(string); ok && v != "" {
db.LogSources[i].Name = v
}
if v, ok := patch["namespace"].(string); ok {
db.LogSources[i].Namespace = v
}
if v, ok := patch["service"].(string); ok {
db.LogSources[i].Service = v
}
if v, ok := patch["port"].(float64); ok {
db.LogSources[i].Port = int(v)
}
if v, ok := patch["url"].(string); ok {
db.LogSources[i].URL = v
}
if v, ok := patch["authType"].(string); ok {
db.LogSources[i].AuthType = v
}
if v, ok := patch["basicUser"].(string); ok {
db.LogSources[i].BasicUser = v
}
if v, ok := patch["bearerToken"].(string); ok {
if v != "" {
enc, err := crypto.EncryptField(v)
if err != nil {
return err
}
db.LogSources[i].BearerToken = enc
}
}
if v, ok := patch["skipTLSVerify"].(bool); ok {
db.LogSources[i].SkipTLSVerify = v
}
if raw, ok := patch["headers"]; ok {
data, err := json.Marshal(raw)
if err != nil {
return err
}
var headers map[string]string
if err := json.Unmarshal(data, &headers); err != nil {
return err
}
for name, value := range headers {
if !validSourceHeader(name) {
return fmt.Errorf("自定义 Header 名称无效: %s", name)
}
if value == "" {
if old, ok := db.LogSources[i].Headers[name]; ok {
headers[name] = old
}
continue
}
enc, err := crypto.EncryptField(value)
if err != nil {
return err
}
headers[name] = enc
}
db.LogSources[i].Headers = headers
}
if v, ok := patch["basicPass"].(string); ok && v != "" {
if enc, err := crypto.EncryptField(v); err == nil {
db.LogSources[i].BasicPass = enc
}
}
if v, ok := patch["isDefault"].(bool); ok && v {
for j := range db.LogSources {
db.LogSources[j].IsDefault = db.LogSources[j].ID == id
}
}
if raw, ok := patch["access"]; ok {
data, err := json.Marshal(raw)
if err != nil {
return err
}
var access []models.LogSourceAccess
if err := json.Unmarshal(data, &access); err != nil {
return err
}
if err := ValidateSourceAccess(access); err != nil {
return err
}
db.LogSources[i].Access = access
}
db.LogSources[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
}
return fmt.Errorf("数据源不存在")
}
func ValidateSourceAccess(access []models.LogSourceAccess) error {
seen := map[uint]bool{}
for _, rule := range access {
if rule.UserID == 0 || seen[rule.UserID] {
return fmt.Errorf("访问策略的用户 ID 无效或重复")
}
seen[rule.UserID] = true
for k := range rule.Labels {
if !labelNamePattern.MatchString(k) {
return fmt.Errorf("访问策略包含无效标签名: %s", k)
}
}
}
return nil
}
func DeleteLogSource(id uint) error {
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.LogSources {
if db.LogSources[i].ID == id {
db.LogSources = append(db.LogSources[:i], db.LogSources[i+1:]...)
groups := db.LogPolicyGroups[:0]
for _, group := range db.LogPolicyGroups {
if group.SourceID != id {
groups = append(groups, group)
}
}
db.LogPolicyGroups = groups
db.RequestSave()
return nil
}
}
return fmt.Errorf("数据源不存在")
}
// EnsureLogSourceForDeployment 部署成功后自动注册查询数据源(host-binary→url;k8s→cluster-svc)
func EnsureLogSourceForDeployment(d *models.LokiDeployment) {
if d == nil || d.Endpoint == "" {
return
}
db := repository.GetDB()
db.RLock()
for i := range db.LogSources {
if db.LogSources[i].Name == d.Name {
db.RUnlock()
return
}
}
db.RUnlock()
svc := "loki"
port := 3100
src := &models.LogSource{
Name: d.Name, TargetType: "cluster-svc",
ClusterID: d.ClusterID, Namespace: d.Namespace, Service: svc, Port: port,
AuthType: "none",
}
if d.Mode == "host-binary" {
src.TargetType = "url"
src.URL = d.Endpoint
src.ClusterID = nil
}
_, _ = CreateLogSource(src)
func newLokiTransport(skipTLSVerify bool) *http.Transport {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: skipTLSVerify} //nolint:gosec // 数据源显式配置
return transport
}
// ==================== Loki HTTP 访问 ====================
// lokiProxyGET 对数据源执行 GET;cluster-svc 走 kubectl --raw service proxy。
// lokiProxyGET 对数据源执行 GET；cluster-svc 使用 client-go 的认证传输访问 API Server Service Proxy。
func lokiProxyGET(ctx context.Context, src *models.LogSource, path string, query url.Values) ([]byte, int, error) {
// cluster-svc:kubectl get --raw /api/v1/namespaces/{ns}/services/{svc}:{port}/proxy{path}?q
if src.TargetType == "cluster-svc" {
db := repository.GetDB()
db.RLock()
@@ -381,25 +55,39 @@ func lokiProxyGET(ctx context.Context, src *models.LogSource, path string, query
if cluster == nil {
return nil, 500, fmt.Errorf("数据源关联的集群不存在")
}
env, cleanup, err := kubeconfigEnv(cluster)
if !kubernetesNamespacePattern.MatchString(src.Namespace) || !kubernetesNamespacePattern.MatchString(src.Service) || src.Port < 1 || src.Port > 65535 {
return nil, 500, fmt.Errorf("Loki 数据源的命名空间、Service 或端口无效")
}
config, err := clusterRESTConfig(cluster, logQueryTimeout)
if err != nil {
return nil, 500, err
}
defer cleanup()
proxyPath := fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy%s",
client, err := rest.HTTPClientFor(config)
if err != nil {
return nil, 500, fmt.Errorf("创建 Kubernetes API 客户端失败: %w", err)
}
full := strings.TrimRight(config.Host, "/") + fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy%s",
src.Namespace, src.Service, src.Port, path)
full := proxyPath
if qs := query.Encode(); qs != "" {
full += "?" + qs
}
cmd := exec.CommandContext(ctx, "kubectl", "get", "--raw", full)
cmd.Env = env
out, err := cmd.CombinedOutput()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil)
if err != nil {
return nil, 502, fmt.Errorf("kubectl 代理查询失败: %v\n%s", err, truncateStr(string(out), 300))
return nil, 500, err
}
return out, 200, nil
resp, err := client.Do(req)
if err != nil {
return nil, 502, fmt.Errorf("Kubernetes API 代理查询失败: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
if err != nil {
return nil, 502, err
}
if resp.StatusCode >= 400 {
return body, resp.StatusCode, fmt.Errorf("Kubernetes API 代理返回 HTTP %d: %s", resp.StatusCode, truncateStr(string(body), 300))
}
return body, resp.StatusCode, nil
}
// url 直连
@@ -415,9 +103,10 @@ func lokiProxyGET(ctx context.Context, src *models.LogSource, path string, query
return nil, 500, err
}
applyLokiAuth(req, src)
client := &http.Client{Timeout: 60 * time.Second, Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: src.SkipTLSVerify},
}}
client := lokiHTTPClients[0]
if src.SkipTLSVerify {
client = lokiHTTPClients[1]
}
resp, err := client.Do(req)
if err != nil {
return nil, 502, err
@@ -477,6 +166,42 @@ type LogQueryResult struct {
NextCursor string `json:"nextCursor"`
}
// ContextLogSelector uses stable stream identity labels. Loki can add per-entry
// labels such as detected_level to query results; matching those would hide
// adjacent lines whose detected level differs from the selected line.
func ContextLogSelector(labels map[string]string) (string, error) {
for key := range labels {
if !ValidLogLabelName(key) {
return "", fmt.Errorf("标签名无效")
}
}
keys := []string{"cluster", "namespace", "app", "pod", "container", "filename"}
if labels["pod"] == "" {
keys = append(keys, "job", "instance", "host")
}
parts := make([]string, 0, len(keys))
for _, key := range keys {
if value := labels[key]; value != "" {
encoded, _ := json.Marshal(value)
parts = append(parts, key+"="+string(encoded))
}
}
if len(parts) == 0 {
for key, value := range labels {
if value == "" || key == "detected_level" || key == "level" || strings.HasPrefix(key, "__") {
continue
}
encoded, _ := json.Marshal(value)
parts = append(parts, key+"="+string(encoded))
}
sort.Strings(parts)
}
if len(parts) == 0 {
return "", fmt.Errorf("缺少可用于定位日志流的标签")
}
return "{" + strings.Join(parts, ",") + "}", nil
}
// QueryLokiLogs 日志查询:flatten streams → 统一行;cursor(最后一条纳秒时间戳)向后翻页
func QueryLokiLogs(ctx context.Context, src *models.LogSource, query string, start, end int64, limit int, direction, cursor string) (*LogQueryResult, error) {
if limit <= 0 {
backend/services/log_query_test.go
+14
View File
@@ -40,3 +40,17 @@ func TestPolicyGroupAccess(t *testing.T) {
t.Fatal("nonmember gained access")
}
}
func TestContextLogSelectorIgnoresDetectedLevel(t *testing.T) {
selector, err := ContextLogSelector(map[string]string{
"cluster": "demo-prod", "namespace": "monitor", "app": "consul",
"pod": "consul-0", "container": "consul", "detected_level": "info",
"filename": "/var/log/pods/consul/1.log",
})
if err != nil {
t.Fatal(err)
}
if selector != `{cluster="demo-prod",namespace="monitor",app="consul",pod="consul-0",container="consul",filename="/var/log/pods/consul/1.log"}` {
t.Fatalf("unexpected context selector: %s", selector)
}
}
backend/services/log_source.go
+350
View File
@@ -0,0 +1,350 @@
// 日志数据源管理 — CRUD、默认源、部署自动注册(Managed 标记)。
package services
import (
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
"time"
"opsxloki/models"
"opsxloki/pkg/crypto"
"opsxloki/repository"
)
// ==================== 数据源 CRUD ====================
func GetLogSource(id uint) *models.LogSource {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
for i := range db.LogSources {
if db.LogSources[i].ID == id {
s := db.LogSources[i]
return &s
}
}
return nil
}
func ListLogSources() []map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
out := make([]map[string]interface{}, 0, len(db.LogSources))
for i := range db.LogSources {
out = append(out, db.LogSources[i].Sanitized())
}
return out
}
func ListVisibleLogSources(userID uint, admin bool) []map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
out := make([]map[string]interface{}, 0)
for i := range db.LogSources {
s := &db.LogSources[i]
if admin || policyGroupAccess(db.LogPolicyGroups, s.ID, userID) != nil || sourceAccess(s, userID) != nil {
item := s.Sanitized()
if !admin {
item = map[string]interface{}{"id": s.ID, "name": s.Name, "isDefault": s.IsDefault}
}
out = append(out, item)
}
}
return out
}
func sourceAccess(src *models.LogSource, userID uint) *models.LogSourceAccess {
for i := range src.Access {
if src.Access[i].UserID == userID {
return &src.Access[i]
}
}
return nil
}
var labelNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
func ValidLogLabelName(name string) bool { return labelNamePattern.MatchString(name) }
// AuthorizedLogQuery constrains the Loki stream selector server-side. Non-admins must be explicitly granted access.
func AuthorizedLogQuery(src *models.LogSource, userID uint, admin bool, query string) (string, error) {
if admin {
return query, nil
}
db := repository.GetDB()
var access *models.LogSourceAccess
if db != nil {
db.RLock()
access = policyGroupAccess(db.LogPolicyGroups, src.ID, userID)
db.RUnlock()
}
if access == nil {
access = sourceAccess(src, userID)
}
if access == nil {
return "", fmt.Errorf("无权查询此数据源")
}
q := strings.TrimSpace(query)
if !strings.HasPrefix(q, "{") {
return "", fmt.Errorf("普通用户查询必须以日志流选择器开头")
}
close := strings.IndexByte(q, '}')
if close < 0 || strings.Contains(q[close+1:], "{") {
return "", fmt.Errorf("查询只能包含一个日志流选择器")
}
if len(access.Labels) == 0 {
return q, nil
}
keys := make([]string, 0, len(access.Labels))
for k := range access.Labels {
if !labelNamePattern.MatchString(k) {
return "", fmt.Errorf("访问策略包含无效标签名")
}
keys = append(keys, k)
}
sort.Strings(keys)
matchers := make([]string, 0, len(keys))
for _, k := range keys {
v, _ := json.Marshal(access.Labels[k])
matchers = append(matchers, k+"="+string(v))
}
inner := strings.TrimSpace(q[1:close])
if inner != "" {
inner += ","
}
return "{" + inner + strings.Join(matchers, ",") + "}" + q[close+1:], nil
}
func CreateLogSource(s *models.LogSource) (*models.LogSource, error) {
if err := encryptSourceCredentials(s); err != nil {
return nil, err
}
db := repository.GetDB()
newID := db.NextID("logSources")
db.Lock()
defer db.Unlock()
s.ID = newID
if s.IsDefault {
for i := range db.LogSources {
db.LogSources[i].IsDefault = false
}
}
// 首个数据源自动设默认
if len(db.LogSources) == 0 {
s.IsDefault = true
}
db.LogSources = append(db.LogSources, *s)
db.RequestSave()
return s, nil
}
func encryptSourceCredentials(s *models.LogSource) error {
if s.BasicPass != "" {
v, err := crypto.EncryptField(s.BasicPass)
if err != nil {
return err
}
s.BasicPass = v
}
if s.BearerToken != "" {
v, err := crypto.EncryptField(s.BearerToken)
if err != nil {
return err
}
s.BearerToken = v
}
for name, value := range s.Headers {
if !validSourceHeader(name) {
return fmt.Errorf("自定义 Header 名称无效: %s", name)
}
v, err := crypto.EncryptField(value)
if err != nil {
return err
}
s.Headers[name] = v
}
return nil
}
func validSourceHeader(name string) bool {
if name == "" || strings.EqualFold(name, "Host") || strings.EqualFold(name, "Authorization") {
return false
}
for _, c := range name {
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
return false
}
}
return true
}
func UpdateLogSource(id uint, patch map[string]interface{}) error {
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.LogSources {
if db.LogSources[i].ID != id {
continue
}
if v, ok := patch["name"].(string); ok && v != "" {
db.LogSources[i].Name = v
}
if v, ok := patch["namespace"].(string); ok {
db.LogSources[i].Namespace = v
}
if v, ok := patch["service"].(string); ok {
db.LogSources[i].Service = v
}
if v, ok := patch["port"].(float64); ok {
db.LogSources[i].Port = int(v)
}
if v, ok := patch["url"].(string); ok {
db.LogSources[i].URL = v
}
if v, ok := patch["authType"].(string); ok {
db.LogSources[i].AuthType = v
}
if v, ok := patch["basicUser"].(string); ok {
db.LogSources[i].BasicUser = v
}
if v, ok := patch["bearerToken"].(string); ok {
if v != "" {
enc, err := crypto.EncryptField(v)
if err != nil {
return err
}
db.LogSources[i].BearerToken = enc
}
}
if v, ok := patch["skipTLSVerify"].(bool); ok {
db.LogSources[i].SkipTLSVerify = v
}
if raw, ok := patch["headers"]; ok {
data, err := json.Marshal(raw)
if err != nil {
return err
}
var headers map[string]string
if err := json.Unmarshal(data, &headers); err != nil {
return err
}
for name, value := range headers {
if !validSourceHeader(name) {
return fmt.Errorf("自定义 Header 名称无效: %s", name)
}
if value == "" {
if old, ok := db.LogSources[i].Headers[name]; ok {
headers[name] = old
}
continue
}
enc, err := crypto.EncryptField(value)
if err != nil {
return err
}
headers[name] = enc
}
db.LogSources[i].Headers = headers
}
if v, ok := patch["basicPass"].(string); ok && v != "" {
if enc, err := crypto.EncryptField(v); err == nil {
db.LogSources[i].BasicPass = enc
}
}
if v, ok := patch["isDefault"].(bool); ok && v {
for j := range db.LogSources {
db.LogSources[j].IsDefault = db.LogSources[j].ID == id
}
}
if raw, ok := patch["access"]; ok {
data, err := json.Marshal(raw)
if err != nil {
return err
}
var access []models.LogSourceAccess
if err := json.Unmarshal(data, &access); err != nil {
return err
}
if err := ValidateSourceAccess(access); err != nil {
return err
}
db.LogSources[i].Access = access
}
db.LogSources[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
}
return fmt.Errorf("数据源不存在")
}
func ValidateSourceAccess(access []models.LogSourceAccess) error {
seen := map[uint]bool{}
for _, rule := range access {
if rule.UserID == 0 || seen[rule.UserID] {
return fmt.Errorf("访问策略的用户 ID 无效或重复")
}
seen[rule.UserID] = true
for k := range rule.Labels {
if !labelNamePattern.MatchString(k) {
return fmt.Errorf("访问策略包含无效标签名: %s", k)
}
}
}
return nil
}
func DeleteLogSource(id uint) error {
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.LogSources {
if db.LogSources[i].ID == id {
db.LogSources = append(db.LogSources[:i], db.LogSources[i+1:]...)
groups := db.LogPolicyGroups[:0]
for _, group := range db.LogPolicyGroups {
if group.SourceID != id {
groups = append(groups, group)
}
}
db.LogPolicyGroups = groups
db.RequestSave()
return nil
}
}
return fmt.Errorf("数据源不存在")
}
// EnsureLogSourceForDeployment 部署成功后自动注册查询数据源(host-binary→url;k8s→cluster-svc)
func EnsureLogSourceForDeployment(d *models.LokiDeployment) {
if d == nil || d.Endpoint == "" {
return
}
db := repository.GetDB()
db.RLock()
for i := range db.LogSources {
if db.LogSources[i].Name == d.Name {
db.RUnlock()
return
}
}
db.RUnlock()
svc := "loki"
port := 3100
src := &models.LogSource{
Name: d.Name, TargetType: "cluster-svc",
ClusterID: d.ClusterID, Namespace: d.Namespace, Service: svc, Port: port,
AuthType: "none",
}
if d.Mode == "host-binary" || d.Mode == "host-docker" {
src.TargetType = "url"
src.URL = d.Endpoint
src.ClusterID = nil
}
_, _ = CreateLogSource(src)
}
backend/services/loki_config.go
+370
View File
@@ -0,0 +1,370 @@
// LokiDeployer — loki.yaml 配置生成:模板 + 对象存储/规则存储/ring 拓扑。
package services
import (
"bytes"
"fmt"
"text/template"
)
const lokiConfigTemplate = `auth_enabled: {{ .AuthEnabled }}
common:
path_prefix: /var/lib/loki
{{- if .FilesystemStorage }}
storage:
filesystem:
chunks_directory: /var/lib/loki/chunks
rules_directory: /var/lib/loki/rules
{{- end }}
replication_factor: {{ .ReplicationFactor }}
ring:
instance_addr: {{ .RingInstanceAddr }}
kvstore:
store: {{ .RingKVStore }}
{{- if .StorageConfigBlock }}
storage_config:
{{ .StorageConfigBlock }}
{{- end }}
{{- if .MemberlistBlock }}
{{ .MemberlistBlock }}
{{- end }}
ingester:
wal:
dir: /var/lib/loki/wal
flush_on_shutdown: true
replay_memory_ceiling: 1GB
lifecycler:
ring:
kvstore:
store: {{ .RingKVStore }}
replication_factor: {{ .ReplicationFactor }}
{{- if .MultiReplica }}
autoforget_unhealthy: true
{{- end }}
compactor:
working_directory: /var/lib/loki/compactor
retention_enabled: true
delete_request_store: {{ .CompactorStoreType }}
limits_config:
retention_period: {{ .RetentionPeriod }}h
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: {{ .ObjectStore }}
schema: v13
index:
prefix: tsdb_index_
period: 24h
ruler:
storage:
type: {{ .RulerStoreType }}
{{- if .RulerStorageBlock }}
{{ .RulerStorageBlock }}
{{- end }}
{{- if eq .RulerStoreType "local" }}
local:
directory: /var/lib/loki/rules
{{- end }}
rule_path: /var/lib/loki/rules-temp
ring:
kvstore:
store: {{ .RingKVStore }}
analytics:
reporting_enabled: false
`
type lokiConfigData struct {
AuthEnabled bool
ReplicationFactor int
RetentionPeriod int
StorageConfigBlock string
ObjectStore string
RulerStoreType string
CompactorStoreType string
RingInstanceAddr string // 单副本 127.0.0.1；多副本 0.0.0.0（须可被对端寻址）
RingKVStore string // 单副本 inmemory；多副本 memberlist
MemberlistBlock string // 多副本时非空，注入 memberlist 顶级配置
RulerStorageBlock string // 对象存储时非空，注入 ruler.storage.s3/gcs/azure 嵌套配置
MultiReplica bool
FilesystemStorage bool
}
func (d *LokiDeployer) generateLokiConfig() string {
retentionHours := d.deployment.RetentionDays * 24
if retentionHours <= 0 {
retentionHours = DefaultRetentionDays * 24
}
storageType := d.effectiveStorageType()
storageBlock := d.buildStorageConfigBlock()
objectStore := "filesystem"
rulerStore := "local"
compactorStore := "filesystem"
switch storageType {
case "s3", "oss", "minio":
objectStore = "s3"
rulerStore = "s3"
compactorStore = "s3"
case "gcs":
objectStore = "gcs"
rulerStore = "gcs"
compactorStore = "gcs"
case "azure":
objectStore = "azure"
rulerStore = "azure"
compactorStore = "azure"
}
// Ring 拓扑：单副本用 inmemory + 127.0.0.1（无需跨实例通信）；
// 多副本必须切 memberlist + 0.0.0.0，否则 ring 无法收敛、写入失败。
ringAddr := "127.0.0.1"
ringStore := "inmemory"
memberlistBlock := ""
if d.deployment.Replicas >= 2 {
ringAddr = "0.0.0.0"
ringStore = "memberlist"
ns := d.deployment.Namespace
if ns == "" {
ns = "log"
}
memberlistBlock = fmt.Sprintf(`memberlist:
join_members:
- loki-headless.%s.svc.cluster.local
bind_port: 7946
dead_node_reclaim_time: 30s
gossip_to_dead_nodes_time: 2m
left_ingesters_timeout: 5m`, ns)
}
data := lokiConfigData{
AuthEnabled: d.deployment.AuthEnabled,
ReplicationFactor: d.replicationFactor(),
RetentionPeriod: retentionHours,
StorageConfigBlock: storageBlock,
ObjectStore: objectStore,
RulerStoreType: rulerStore,
CompactorStoreType: compactorStore,
RingInstanceAddr: ringAddr,
RingKVStore: ringStore,
MemberlistBlock: memberlistBlock,
RulerStorageBlock: d.buildRulerStorageBlock(),
MultiReplica: d.deployment.Replicas >= 2,
FilesystemStorage: objectStore == "filesystem",
}
tmpl, err := template.New("loki-config").Parse(lokiConfigTemplate)
if err != nil {
return d.generateLokiConfigFallback(retentionHours)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil {
return d.generateLokiConfigFallback(retentionHours)
}
return buf.String()
}
func (d *LokiDeployer) buildStorageConfigBlock() string {
if d.deployment.StorageConfig == nil {
return ""
}
sc := d.deployment.StorageConfig
accessKey, secretKey := d.storageCredentialValues()
switch sc.StorageType {
case "s3":
endpoint := sc.Endpoint
if endpoint == "" {
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
}
insecure := "false"
if sc.Insecure {
insecure = "true"
}
return fmt.Sprintf(` aws:
bucketnames: %s
endpoint: %s
region: %s
access_key_id: %s
secret_access_key: %s
insecure: %s
s3forcepathstyle: true`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
case "oss":
return fmt.Sprintf(` alibabacloud:
bucket: %s
endpoint: %s
region: %s
access_key_id: %s
secret_access_key: %s`, sc.Bucket, sc.Endpoint, sc.Region, accessKey, secretKey)
case "minio":
insecure := "false"
if sc.Insecure {
insecure = "true"
}
return fmt.Sprintf(` aws:
bucketnames: %s
endpoint: %s
access_key_id: %s
secret_access_key: %s
insecure: %s
s3forcepathstyle: true`, sc.Bucket, sc.Endpoint, accessKey, secretKey, insecure)
case "gcs":
return fmt.Sprintf(` gcs:
bucket_name: %s`, sc.Bucket)
case "azure":
return fmt.Sprintf(` azure:
container_name: %s
account_name: %s
account_key: %s`, sc.Bucket, accessKey, secretKey)
default:
return ""
}
}
// buildRulerStorageBlock 生成 ruler.storage 下与存储类型同名的嵌套配置
// (Loki 3.x 的 ruler 存储客户端不回退读取 storage_config,缺少此块会以空 bucket 启动失败)
func (d *LokiDeployer) buildRulerStorageBlock() string {
if d.deployment.StorageConfig == nil {
return ""
}
sc := d.deployment.StorageConfig
accessKey, secretKey := d.storageCredentialValues()
switch sc.StorageType {
case "s3":
endpoint := sc.Endpoint
if endpoint == "" {
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", sc.Region)
}
insecure := "false"
if sc.Insecure {
insecure = "true"
}
return fmt.Sprintf(` s3:
bucketnames: %s
endpoint: %s
region: %s
access_key_id: %s
secret_access_key: %s
insecure: %s
s3forcepathstyle: true`, sc.Bucket, endpoint, sc.Region, accessKey, secretKey, insecure)
case "oss":
return fmt.Sprintf(` s3:
bucketnames: %s
endpoint: %s
access_key_id: %s
secret_access_key: %s
s3forcepathstyle: true`, sc.Bucket, sc.Endpoint, accessKey, secretKey)
case "minio":
insecure := "false"
if sc.Insecure {
insecure = "true"
}
return fmt.Sprintf(` s3:
bucketnames: %s
endpoint: %s
access_key_id: %s
secret_access_key: %s
insecure: %s
s3forcepathstyle: true`, sc.Bucket, sc.Endpoint, accessKey, secretKey, insecure)
case "gcs":
return fmt.Sprintf(` gcs:
bucket_name: %s`, sc.Bucket)
case "azure":
return fmt.Sprintf(` azure:
container_name: %s
account_name: %s
account_key: %s`, sc.Bucket, accessKey, secretKey)
default:
return ""
}
}
func (d *LokiDeployer) storageCredentialValues() (string, string) {
if d.deployment.StorageConfig == nil {
return "", ""
}
if d.deployment.Mode == "k8s-yaml" || d.deployment.Mode == "host-docker" {
return "${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}"
}
return d.deployment.StorageConfig.AccessKey, d.deployment.StorageConfig.SecretKey
}
// generateLokiConfigFallback 当模板解析失败时的降级配置
func (d *LokiDeployer) generateLokiConfigFallback(retentionHours int) string {
return fmt.Sprintf(`auth_enabled: %v
server:
http_listen_port: 3100
grpc_listen_port: 9095
http_server_read_timeout: 3m
http_server_write_timeout: 3m
log_level: info
common:
path_prefix: /var/lib/loki
storage:
filesystem:
chunks_directory: /var/lib/loki/chunks
rules_directory: /var/lib/loki/rules
replication_factor: %d
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
ingester:
wal:
enabled: true
dir: /var/lib/loki/wal
lifecycler:
ring:
kvstore:
store: inmemory
replication_factor: %d
final_sleep: 0s
chunk_idle_period: 30m
max_chunk_age: 2h
querier:
max_concurrent: 10
compactor:
working_directory: /var/lib/loki/compactor
retention_enabled: true
retention_delete_delay: 2h
limits_config:
retention_period: %dh
ingestion_rate_mb: 16
ingestion_burst_size_mb: 32
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: tsdb_index_
period: 24h
analytics:
reporting_enabled: false
`, d.deployment.AuthEnabled, d.replicationFactor(), d.replicationFactor(), retentionHours)
}
// ==================== 辅助函数 ====================
func (d *LokiDeployer) replicationFactor() int {
if d.deployment.Replicas >= 3 {
return 3
}
if d.deployment.Replicas >= 2 {
return 2
}
return 1
}
backend/services/loki_deployer.go
+32 -998
View File
File diff suppressed because it is too large. Load diff
backend/services/loki_docker.go
+149
View File
@@ -0,0 +1,149 @@
package services
import (
"context"
"encoding/base64"
"fmt"
"strings"
"time"
"opsxloki/models"
)
func dockerContainerName(id uint) string { return fmt.Sprintf("lokideploy-loki-%d", id) }
func dockerConfigDir(id uint) string { return fmt.Sprintf("/etc/lokideploy/loki-%d", id) }
func dockerConfigPath(id uint) string { return dockerConfigDir(id) + "/loki.yaml" }
func dockerDataDir(id uint) string { return fmt.Sprintf("/var/lib/lokideploy/loki-%d", id) }
func dockerLokiImage(version string) (string, error) {
if version == "" {
version = "3.0.0"
}
for _, supported := range LokiVersionList {
if supported == version && strings.HasPrefix(version, "3.") {
return "grafana/loki:" + version, nil
}
}
return "", fmt.Errorf("主机 Docker 部署请选择支持的 Loki 3.x 版本")
}
func stopDockerLoki(ctx context.Context, deployment *models.LokiDeployment) error {
if deployment.Host == nil {
return fmt.Errorf("部署未关联主机")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
if err != nil {
return fmt.Errorf("连接部署主机失败: %w", err)
}
defer client.Close()
if _, stderr, err := client.Execute(ctx, "docker stop "+dockerContainerName(deployment.ID)); err != nil {
return fmt.Errorf("停止 Loki 容器失败: %v: %s", err, strings.TrimSpace(stderr))
}
return nil
}
func deleteDockerLoki(ctx context.Context, deployment *models.LokiDeployment) error {
if deployment.Host == nil {
return fmt.Errorf("部署未关联主机")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
if err != nil {
return fmt.Errorf("连接部署主机失败: %w", err)
}
defer client.Close()
name := dockerContainerName(deployment.ID)
if _, _, err := client.Execute(ctx, "docker info >/dev/null 2>&1"); err != nil {
return fmt.Errorf("目标主机 Docker 不可用，无法安全清理部署")
}
// 仅删除该部署的容器和配置；日志数据目录保留，以便人工恢复。
cmd := "if docker container inspect " + name + " >/dev/null 2>&1; then docker rm -f " + name + " >/dev/null || exit 1; fi; rm -rf -- " + dockerConfigDir(deployment.ID)
if _, stderr, err := client.Execute(ctx, cmd); err != nil {
return fmt.Errorf("清理 Loki 容器与配置失败: %v: %s", err, strings.TrimSpace(stderr))
}
return nil
}
// uploadDockerFile 通过 SSH 写入固定路径。内容仅以 base64 传给远程 shell，避免 YAML 或凭证参与 shell 解析。
func uploadDockerFile(ctx context.Context, client *SSHClient, path, content, mode string) error {
encoded := base64.StdEncoding.EncodeToString([]byte(content))
cmd := fmt.Sprintf("umask 077 && printf '%%s' '%s' | base64 -d > %s && chmod %s %s", encoded, path, mode, path)
_, _, err := client.Execute(ctx, cmd)
if err != nil {
return fmt.Errorf("写入主机配置失败: %w", err)
}
return nil
}
func (d *LokiDeployer) deployDocker(ctx context.Context) error {
deployment := d.deployment
image, err := dockerLokiImage(deployment.Version)
if err != nil {
return err
}
if deployment.Host == nil {
return fmt.Errorf("请选择目标主机")
}
client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
if err != nil {
return err
}
defer client.Close()
if _, _, err := client.Execute(ctx, "docker info >/dev/null 2>&1"); err != nil {
return fmt.Errorf("目标主机 Docker 不可用，请检查安装状态及 SSH 用户权限")
}
name := dockerContainerName(deployment.ID)
configDir := dockerConfigDir(deployment.ID)
dataDir := dockerDataDir(deployment.ID)
if _, _, err := client.Execute(ctx, "mkdir -p "+configDir+" "+dataDir+" && chmod 755 "+configDir+" && chown 10001:10001 "+dataDir); err != nil {
return fmt.Errorf("创建主机持久化目录失败: %w", err)
}
config := d.generateLokiConfig()
if err := uploadDockerFile(ctx, client, dockerConfigPath(deployment.ID), config, "644"); err != nil {
return err
}
// env 文件由 Docker daemon 读取；Loki 配置只保留变量引用。
env := ""
if sc := deployment.StorageConfig; sc != nil {
for _, value := range []string{sc.AccessKey, sc.SecretKey} {
if strings.ContainsAny(value, "\r\n") {
return fmt.Errorf("存储凭证包含不支持的换行字符")
}
}
env = "LOKI_STORAGE_ACCESS_KEY=" + sc.AccessKey + "\nLOKI_STORAGE_SECRET_KEY=" + sc.SecretKey + "\n"
}
if err := uploadDockerFile(ctx, client, configDir+"/storage.env", env, "600"); err != nil {
return err
}
d.log("info", "拉取 Docker 镜像 "+image, "")
if _, stderr, err := client.Execute(ctx, "docker pull "+image); err != nil {
return fmt.Errorf("拉取 Loki 镜像失败: %v: %s", err, strings.TrimSpace(stderr))
}
// 同一部署 ID 的重部署只替换容器，配置和日志数据目录保留。
client.Execute(ctx, "docker rm -f "+name+" >/dev/null 2>&1 || true")
port := deployment.ListenPort
if port == 0 {
port = 3100
}
listenIP := deployment.ListenIP
if listenIP == "" {
listenIP = deployment.Host.HostIP
}
cmd := fmt.Sprintf("docker run -d --name %s --restart unless-stopped --env-file %s/storage.env -p %s:%d:3100 -v %s:/etc/loki:ro -v %s:/var/lib/loki %s -config.file=/etc/loki/loki.yaml -config.expand-env=true", name, configDir, listenIP, port, configDir, dataDir, image)
if _, stderr, err := client.Execute(ctx, cmd); err != nil {
return fmt.Errorf("启动 Loki 容器失败: %v: %s", err, strings.TrimSpace(stderr))
}
d.log("info", "等待 Loki 容器就绪", "")
readyCmd := fmt.Sprintf("curl -fsS --max-time 2 http://%s:%d/ready >/dev/null", listenIP, port)
for i := 0; i < 30; i++ {
if _, _, err := client.Execute(ctx, readyCmd); err == nil {
d.log("success", fmt.Sprintf("Loki 已启动：http://%s:%d", listenIP, port), "")
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
return fmt.Errorf("Loki 容器启动后未就绪，请在目标主机执行 docker logs %s", name)
}
backend/services/loki_docker_test.go
+41
View File
@@ -0,0 +1,41 @@
package services
import (
"strings"
"testing"
"opsxloki/models"
)
func TestDockerLokiConfigUsesEnvironmentForStorageCredentials(t *testing.T) {
deployment := &models.LokiDeployment{
ID: 7, Mode: "host-docker", RetentionDays: 60, Replicas: 1,
StorageConfig: &models.StorageConfig{
StorageType: "minio", Endpoint: "http://10.10.10.88:9000", Bucket: "loki",
AccessKey: "test-access-secret", SecretKey: "test-secret-secret",
},
}
config := NewLokiDeployer(deployment, nil).generateLokiConfig()
if strings.Contains(config, "test-access-secret") || strings.Contains(config, "test-secret-secret") {
t.Fatal("Docker Loki config embedded storage credentials")
}
for _, expected := range []string{"${LOKI_STORAGE_ACCESS_KEY}", "${LOKI_STORAGE_SECRET_KEY}", "retention_period: 1440h"} {
if !strings.Contains(config, expected) {
t.Errorf("config missing %q", expected)
}
}
if got := dockerContainerName(deployment.ID); got != "lokideploy-loki-7" {
t.Fatalf("unexpected container name: %s", got)
}
}
func TestDockerLokiImageRejectsShellInputAndOldVersions(t *testing.T) {
for _, version := range []string{"2.9.8", "3.0.0;touch /tmp/example", "3.9.9"} {
if _, err := dockerLokiImage(version); err == nil {
t.Errorf("accepted unsupported version %q", version)
}
}
if image, err := dockerLokiImage("3.4.0"); err != nil || image != "grafana/loki:3.4.0" {
t.Fatalf("unexpected image %q, error %v", image, err)
}
}
backend/services/loki_manifest.go
+243
View File
@@ -0,0 +1,243 @@
// LokiDeployer — k8s-yaml 模式 manifest 生成(StatefulSet/PVC/ConfigMap/Service)。
package services
import (
"crypto/sha256"
"encoding/base64"
"fmt"
)
func (d *LokiDeployer) generateManifest(ns string) string {
version := d.effectiveVersion()
image := ResolveImage("docker.io/grafana/loki:" + version)
replicas := d.deployment.Replicas
if replicas < 1 {
replicas = 1
}
storageSize := d.deployment.StorageSize
if storageSize == "" {
storageSize = "10Gi"
}
cpu := d.deployment.ResourcesCPU
mem := d.deployment.ResourcesMem
if cpu == "" {
cpu = "1000m"
}
if mem == "" {
mem = "1Gi"
}
configData := d.generateLokiConfig()
secretManifest := ""
configExpandArg := ""
storageEnvBlock := ""
if sc := d.deployment.StorageConfig; sc != nil {
switch sc.StorageType {
case "s3", "oss", "minio", "azure":
secretManifest = fmt.Sprintf(`---
apiVersion: v1
kind: Secret
metadata:
name: loki-storage-credentials
namespace: %s
type: Opaque
data:
accessKey: %s
secretKey: %s`, ns,
base64.StdEncoding.EncodeToString([]byte(sc.AccessKey)),
base64.StdEncoding.EncodeToString([]byte(sc.SecretKey)))
configExpandArg = "\n - -config.expand-env=true"
storageEnvBlock = `
env:
- name: LOKI_STORAGE_ACCESS_KEY
valueFrom:
secretKeyRef:
name: loki-storage-credentials
key: accessKey
- name: LOKI_STORAGE_SECRET_KEY
valueFrom:
secretKeyRef:
name: loki-storage-credentials
key: secretKey`
}
}
// ConfigMap/Secret 更新本身不会重建 Pod。把内容摘要写入 Pod 模板以触发滚动更新。
configHash := sha256.Sum256([]byte(configData + "\n" + secretManifest))
manifest := fmt.Sprintf(`apiVersion: v1
kind: Namespace
metadata:
name: %s
---
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-config
namespace: %s
data:
loki.yaml: |
%s
%s
---
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: %s
labels:
app: loki
spec:
type: ClusterIP
ports:
- name: http
port: 3100
targetPort: 3100
protocol: TCP
- name: grpc
port: 9095
targetPort: 9095
protocol: TCP
selector:
app: loki
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: loki-data
namespace: %s
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: %s
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: loki
namespace: %s
labels:
app: loki
spec:
serviceName: loki
replicas: %d
selector:
matchLabels:
app: loki
template:
metadata:
labels:
app: loki
annotations:
checksum/loki-config: "%x"
spec:
securityContext:
runAsUser: 0
runAsGroup: 0
fsGroup: 0
containers:
- name: loki
image: grafana/loki:%s
imagePullPolicy: IfNotPresent
args:
- -config.file=/etc/loki/loki.yaml%s%s
ports:
- name: http
containerPort: 3100
protocol: TCP
- name: grpc
containerPort: 9095
protocol: TCP
resources:
limits:
cpu: "%s"
memory: "%s"
requests:
cpu: "200m"
memory: "256Mi"
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop:
- ALL
livenessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
startupProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 30
volumeMounts:
- name: config
mountPath: /etc/loki
readOnly: true
- name: data
mountPath: /var/lib/loki
- name: wal
mountPath: /var/lib/loki/wal
volumes:
- name: config
configMap:
name: loki-config
- name: data
persistentVolumeClaim:
claimName: loki-data
- name: wal
emptyDir: {}
`, ns, ns, indentYAML(configData, 4), secretManifest, ns, ns, storageSize, ns, replicas, configHash, image, configExpandArg, storageEnvBlock, cpu, mem)
// 多副本：追加 Headless Service（clusterIP: None + selector app:loki），
// 使 memberlist 的 join_members (loki-headless.<ns>.svc.cluster.local) 能解析到所有 Pod IP。
if replicas >= 2 {
manifest += fmt.Sprintf(`
---
apiVersion: v1
kind: Service
metadata:
name: loki-headless
namespace: %s
labels:
app: loki
spec:
clusterIP: None
ports:
- name: http
port: 3100
targetPort: 3100
protocol: TCP
- name: grpc
port: 9095
targetPort: 9095
protocol: TCP
- name: memberlist
port: 7946
targetPort: 7946
protocol: TCP
selector:
app: loki
`, ns)
}
return manifest
}
// ==================== Loki 本地配置生成 (text/template) ====================
backend/services/notifier.go
+146
View File
@@ -0,0 +1,146 @@
package services
// 飞书/Lark 群机器人通知 — 卡片消息(参照 opsx-sec notifier 与运维告警脚本的卡片结构),
// webhook 存 SystemSetting(AES 加密),对外只回 hasWebhook。
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"opsxloki/models"
"opsxloki/pkg/crypto"
"opsxloki/repository"
)
const (
notifySettingKey = "notify_settings"
notifyHTTPTimeout = 10 * time.Second
)
// NotifyConfig 通知配置(webhook/签名 加密存储)
type NotifyConfig struct {
Enabled bool `json:"enabled"`
Webhook string `json:"webhook"` // ENC: 密文存储
SigningSecret string `json:"signingSecret"` // 飞书签名校验(可选)
}
// NotifyPublicConfig 对外输出形态
type NotifyPublicConfig struct {
Enabled bool `json:"enabled"`
HasWebhook bool `json:"hasWebhook"`
HasSecret bool `json:"hasSecret"`
}
func GetNotifyConfig() *NotifyConfig {
db := repository.GetDB()
db.RLock()
var raw string
for i := range db.Settings {
if db.Settings[i].Key == notifySettingKey {
raw = db.Settings[i].Value
break
}
}
db.RUnlock()
cfg := &NotifyConfig{}
if raw != "" {
_ = json.Unmarshal([]byte(raw), cfg)
}
if c, err := DecryptCred(cfg.Webhook); err == nil {
cfg.Webhook = c
}
if c, err := DecryptCred(cfg.SigningSecret); err == nil {
cfg.SigningSecret = c
}
return cfg
}
func SaveNotifyConfig(enabled bool, webhook, secret string) error {
// 留空 = 保留旧值
old := GetNotifyConfig()
if webhook == "" {
webhook = old.Webhook
} else if enc, err := crypto.EncryptField(webhook); err == nil {
webhook = enc
}
if secret == "" {
secret = old.SigningSecret
} else if enc, err := crypto.EncryptField(secret); err == nil {
secret = enc
}
data, _ := json.Marshal(&NotifyConfig{Enabled: enabled, Webhook: webhook, SigningSecret: secret})
db := repository.GetDB()
newID := db.NextID("settings")
db.Lock()
defer db.Unlock()
for i := range db.Settings {
if db.Settings[i].Key == notifySettingKey {
db.Settings[i].Value = string(data)
db.RequestSave()
return nil
}
}
db.Settings = append(db.Settings, models.SystemSetting{ID: newID, Key: notifySettingKey, Value: string(data)})
db.RequestSave()
return nil
}
// larkSign 飞书签名:timestamp + "\n" + secret 的 HMAC-SHA256,base64
func larkSign(secret string, ts int64) string {
stringToSign := fmt.Sprintf("%d\n%s", ts, secret)
h := hmac.New(sha256.New, []byte(stringToSign))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// SendLarkCard 发送飞书卡片;title 带颜色(red/orange),lines 为 markdown 内容行
func SendLarkCard(title, color string, lines []string) error {
cfg := GetNotifyConfig()
if !cfg.Enabled || cfg.Webhook == "" {
return fmt.Errorf("通知未启用或未配置 webhook")
}
content := strings.Join(lines, "\n")
payload := map[string]interface{}{
"msg_type": "interactive",
"card": map[string]interface{}{
"config": map[string]interface{}{"wide_screen_mode": true},
"header": map[string]interface{}{
"title": map[string]string{"tag": "plain_text", "content": title},
"template": color,
},
"elements": []interface{}{
map[string]interface{}{"tag": "div", "text": map[string]string{"tag": "lark_md", "content": content}},
map[string]interface{}{"tag": "note", "elements": []interface{}{map[string]string{"tag": "plain_text", "content": "opsx-loki 日志告警"}}},
},
},
}
if cfg.SigningSecret != "" {
ts := time.Now().Unix()
payload["timestamp"] = fmt.Sprintf("%d", ts)
payload["sign"] = larkSign(cfg.SigningSecret, ts)
}
body, _ := json.Marshal(payload)
resp, err := (&http.Client{Timeout: notifyHTTPTimeout}).Post(cfg.Webhook, "application/json", bytes.NewReader(body))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("飞书返回 HTTP %d", resp.StatusCode)
}
var out struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
_ = json.NewDecoder(resp.Body).Decode(&out)
if out.Code != 0 {
return fmt.Errorf("飞书拒绝: %s", out.Msg)
}
return nil
}
backend/services/notify_channel.go
+226
View File
@@ -0,0 +1,226 @@
package services
// 多通知通道(飞书/Lark 群机器人)CRUD 与按通道发送。
// 取代旧的单 webhook 全局配置:每个告警群一个通道,告警规则各自关联。
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"time"
"opsxloki/models"
"opsxloki/pkg/crypto"
"opsxloki/repository"
)
const channelHTTPTimeout = 10 * time.Second
func ListNotifyChannels() []map[string]interface{} {
db := repository.GetDB()
db.RLock()
out := make([]map[string]interface{}, 0, len(db.NotifyChannels))
for i := range db.NotifyChannels {
out = append(out, db.NotifyChannels[i].Sanitized())
}
db.RUnlock()
return out
}
func GetNotifyChannel(id uint) *models.NotifyChannel {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
for i := range db.NotifyChannels {
if db.NotifyChannels[i].ID == id {
c := db.NotifyChannels[i]
return &c
}
}
return nil
}
// GetNotifyChannelDecrypted 取通道并解密凭证(发送用)
func GetNotifyChannelDecrypted(id uint) *models.NotifyChannel {
c := GetNotifyChannel(id)
if c == nil {
return nil
}
if w, err := DecryptCred(c.Webhook); err == nil {
c.Webhook = w
}
if s, err := DecryptCred(c.SigningSecret); err == nil {
c.SigningSecret = s
}
return c
}
// SaveNotifyChannel 新建(id=0)或更新;webhook/secret 非空才更新(留空=保留旧值)
func SaveNotifyChannel(id uint, name, webhook, secret string, enabled bool) (*models.NotifyChannel, error) {
db := repository.GetDB()
// 更新:留空字段回填旧值(旧值已是密文)
if id > 0 {
db.RLock()
var old models.NotifyChannel
found := false
for i := range db.NotifyChannels {
if db.NotifyChannels[i].ID == id {
old = db.NotifyChannels[i]
found = true
break
}
}
db.RUnlock()
if !found {
return nil, fmt.Errorf("通道不存在")
}
if webhook == "" {
webhook = old.Webhook
} else if enc, err := crypto.EncryptField(webhook); err == nil {
webhook = enc
}
if secret == "" {
secret = old.SigningSecret
} else if enc, err := crypto.EncryptField(secret); err == nil {
secret = enc
}
db.Lock()
defer db.Unlock()
for i := range db.NotifyChannels {
if db.NotifyChannels[i].ID == id {
db.NotifyChannels[i].Name = name
db.NotifyChannels[i].Webhook = webhook
db.NotifyChannels[i].SigningSecret = secret
db.NotifyChannels[i].Enabled = enabled
db.NotifyChannels[i].UpdatedAt = time.Now()
db.RequestSave()
c := db.NotifyChannels[i]
return &c, nil
}
}
return nil, fmt.Errorf("通道不存在")
}
// 新建
if name == "" {
return nil, fmt.Errorf("通道名称不能为空")
}
if webhook == "" {
return nil, fmt.Errorf("webhook 不能为空")
}
if enc, err := crypto.EncryptField(webhook); err == nil {
webhook = enc
}
if secret != "" {
if enc, err := crypto.EncryptField(secret); err == nil {
secret = enc
}
}
newID := db.NextID("notifyChannels")
ch := &models.NotifyChannel{
ID: newID, Name: name, Webhook: webhook, SigningSecret: secret, Enabled: enabled,
}
db.Lock()
defer db.Unlock()
db.NotifyChannels = append(db.NotifyChannels, *ch)
db.RequestSave()
return ch, nil
}
func DeleteNotifyChannel(id uint) error {
// 有关联规则时拒绝删除
db := repository.GetDB()
db.RLock()
for i := range db.AlertRules {
for _, chID := range db.AlertRules[i].ChannelIDs {
if chID == id {
db.RUnlock()
return fmt.Errorf("仍有告警规则关联此通道,请先解除关联")
}
}
}
db.RUnlock()
db.Lock()
defer db.Unlock()
for i := range db.NotifyChannels {
if db.NotifyChannels[i].ID == id {
db.NotifyChannels = append(db.NotifyChannels[:i], db.NotifyChannels[i+1:]...)
db.RequestSave()
return nil
}
}
return fmt.Errorf("通道不存在")
}
// larkSignChannel 飞书签名
func larkSignChannel(secret string, ts int64) string {
stringToSign := fmt.Sprintf("%d\n%s", ts, secret)
h := hmac.New(sha256.New, []byte(stringToSign))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// SendLarkCardToChannel 向指定通道发送卡片
func SendLarkCardToChannel(channelID uint, title, color string, lines []string) error {
ch := GetNotifyChannelDecrypted(channelID)
if ch == nil {
return fmt.Errorf("通知通道不存在")
}
if !ch.Enabled {
return fmt.Errorf("通道「%s」已禁用", ch.Name)
}
content := joinNonEmpty(lines, "\n")
payload := map[string]interface{}{
"msg_type": "interactive",
"card": map[string]interface{}{
"config": map[string]interface{}{"wide_screen_mode": true},
"header": map[string]interface{}{
"title": map[string]string{"tag": "plain_text", "content": title},
"template": color,
},
"elements": []interface{}{
map[string]interface{}{"tag": "div", "text": map[string]string{"tag": "lark_md", "content": content}},
map[string]interface{}{"tag": "note", "elements": []interface{}{map[string]string{"tag": "plain_text", "content": "opsx-loki 日志告警"}}},
},
},
}
if ch.SigningSecret != "" {
ts := time.Now().Unix()
payload["timestamp"] = fmt.Sprintf("%d", ts)
payload["sign"] = larkSignChannel(ch.SigningSecret, ts)
}
body, _ := json.Marshal(payload)
resp, err := (&http.Client{Timeout: channelHTTPTimeout}).Post(ch.Webhook, "application/json", bytes.NewReader(body))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("飞书返回 HTTP %d", resp.StatusCode)
}
var out struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
_ = json.NewDecoder(resp.Body).Decode(&out)
if out.Code != 0 {
return fmt.Errorf("飞书拒绝: %s", out.Msg)
}
return nil
}
func joinNonEmpty(items []string, sep string) string {
result := ""
for _, item := range items {
if item == "" {
continue
}
if result != "" {
result += sep
}
result += item
}
return result
}
backend/services/storage.go
+21
View File
@@ -108,6 +108,27 @@ func (s *StorageService) Update(id uint, updates map[string]interface{}) error {
if v, ok := updates["region"].(string); ok {
db.Storages[i].Region = v
}
if v, ok := updates["pathPrefix"].(string); ok {
db.Storages[i].PathPrefix = v
}
if v, ok := updates["insecure"].(bool); ok {
db.Storages[i].Insecure = v
}
// 凭证编辑约定:传入非空才更新(留空=保留旧密文)
if v, ok := updates["accessKey"].(string); ok && v != "" {
enc, err := crypto.EncryptField(v)
if err != nil {
return fmt.Errorf("AccessKey 加密失败: %w", err)
}
db.Storages[i].AccessKey = enc
}
if v, ok := updates["secretKey"].(string); ok && v != "" {
enc, err := crypto.EncryptField(v)
if err != nil {
return fmt.Errorf("SecretKey 加密失败: %w", err)
}
db.Storages[i].SecretKey = enc
}
if v, ok := updates["isDefault"].(bool); ok {
if v {
for j := range db.Storages {
backend/services/storage_check.go
+7 -7
View File
@@ -24,13 +24,13 @@ type StorageTestResult struct {
// StorageTestParams 测试参数(与创建存储配置一致,额外支持自动建桶)
type StorageTestParams struct {
StorageType string
Endpoint string
Bucket string
Region string
AccessKey string
SecretKey string
Insecure bool
StorageType string
Endpoint string
Bucket string
Region string
AccessKey string
SecretKey string
Insecure bool
CreateIfMissing bool
}
backend/services/storage_update_test.go
+34
View File
@@ -0,0 +1,34 @@
package services
import (
"testing"
"opsxloki/repository"
)
func TestStorageUpdateReplacesBothCredentials(t *testing.T) {
t.Setenv("DATA_DIR", t.TempDir())
t.Setenv("JWT_SECRET", "test-jwt-secret-with-at-least-32-characters")
t.Setenv("ENCRYPTION_KEY", "test-encryption-key-with-at-least-32-characters")
if err := repository.InitDB(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = repository.Shutdown() })
svc := NewStorageService()
storage, err := svc.Create("test", "minio", "http://localhost:9000", "loki", "", "old-access", "old-secret", "", true, true)
if err != nil {
t.Fatal(err)
}
if err := svc.Update(storage.ID, map[string]interface{}{"accessKey": "new-access", "secretKey": "new-secret"}); err != nil {
t.Fatal(err)
}
updated, err := svc.Get(storage.ID)
if err != nil {
t.Fatal(err)
}
access, accessErr := DecryptCred(updated.AccessKey)
secret, secretErr := DecryptCred(updated.SecretKey)
if accessErr != nil || secretErr != nil || access != "new-access" || secret != "new-secret" {
t.Fatal("storage credentials were not both updated and decryptable")
}
}