feat: opsx-loki 全面升级 — 前端重构/用户与LDAP/部署向导/日志查询
基于 WEB-UI-Development-Guide 标准的整体升级,并更名 LokiDeploy → opsx-loki。 前端(Gin+React+shadcn/ui): - 按 02/07 章重构:页面域化拆分(无超 400 行文件)、PageContainer/ListQuery、 反馈分级(成功静默/失败 toast/危险确认)、401 单飞刷新+队列重放 - 暗色模式(next-themes)+ 语义 Token(--primary 品牌色/success/warning/info) - 部署向导(状态推导四步:环境→存储→Loki→采集,内嵌表单闭环) - 日志查询页:LogQL 标签构建器/拖拽直方图/日志上下文/Live Tail/查询历史 - 按用户+标签策略的查询授权(服务端强制合并策略) 后端(Go+Gin+JSON DB): - 简单用户管理(仅 admin,无 RBAC)+ TokenVersion 会话吊销 + 登录 IP 限流 - LDAP 接入:配置(AES 加密存储/hasPassword 脱敏)/测试连接/批量导入/登录 - 存储配置测试连接(SigV4 零依赖)+ 桶不存在自动创建 - Loki 三模式部署修复:SSH 密钥认证、3.x 配置模板、ruler 对象存储嵌套块、 凭证解密、孤儿 deploying 状态恢复、Grafana 部署(可选默认关)+面板导入 - DaemonSet 采集纯 YAML(去 helm 依赖)+ DNS 自愈回退 ClusterIP + 集群名标签 - 日志查询代理:cluster-svc 经 kube-apiserver proxy/url 直连 + cursor 翻页 - 统一 log 命名空间;部署就绪探活;Grafana 访问地址自动记录 修复:run.sh 增量编译检测、Sanitized 脱敏补全、LokiDeployment Update 白名单、 健康检查 CreatedAt 零值等 10+ 存量 bug。 数据兼容:DB 沿用旧 lokideploy.json(新名优先);localStorage 一次性迁移; HKDF 加密前缀保留(已加密凭证不受影响)。
This commit is contained in:
logan committed
1 parent
327e282356
commit
f9c6c1d826
204 files changed
+14107
-9103
No files matched your search
@@ -0,0 +1,438 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"opsxloki/models"
|
||||
)
|
||||
|
||||
// GrafanaDeployer 与 Loki 同环境部署 Grafana 并自动配置 Loki 数据源。
|
||||
// k8s 模式:manifest(Deployment + NodePort Service + 数据源 ConfigMap),与 Loki 同 namespace;
|
||||
// host-binary 模式:tarball + systemd,与 Loki 同机。
|
||||
type GrafanaDeployer struct {
|
||||
deployment *models.LokiDeployment
|
||||
log func(level, message, detail string)
|
||||
}
|
||||
|
||||
const defaultGrafanaVersion = "11.3.0"
|
||||
|
||||
func (d *GrafanaDeployer) version() string {
|
||||
if d.deployment.GrafanaVersion != "" {
|
||||
return d.deployment.GrafanaVersion
|
||||
}
|
||||
return defaultGrafanaVersion
|
||||
}
|
||||
|
||||
// lokiPushURL 数据源指向的 Loki 地址(k8s 走集群内服务名,主机走本机)
|
||||
func (d *GrafanaDeployer) lokiPushURL() string {
|
||||
if d.deployment.Mode == "host-binary" {
|
||||
return "http://127.0.0.1:3100"
|
||||
}
|
||||
ns := d.deployment.Namespace
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
return fmt.Sprintf("http://loki.%s.svc.cluster.local:3100", ns)
|
||||
}
|
||||
|
||||
// Deploy 部署入口:按模式分发
|
||||
func (d *GrafanaDeployer) Deploy(ctx context.Context) error {
|
||||
d.log("info", fmt.Sprintf("Deploying Grafana %s (mode: %s)...", d.version(), d.deployment.Mode), "")
|
||||
switch d.deployment.Mode {
|
||||
case "k8s-helm", "k8s-yaml":
|
||||
return d.deployK8s(ctx)
|
||||
case "host-binary":
|
||||
return d.deployHost(ctx)
|
||||
default:
|
||||
return fmt.Errorf("unsupported mode for grafana: %s", d.deployment.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== K8s manifest 部署 ====================
|
||||
|
||||
// grafanaManifest 生成 Grafana 资源;lokiURL 显式传入(默认服务名,DNS 异常的集群可回退 ClusterIP)。
|
||||
// 数据卷用 emptyDir:数据源/配置由 ConfigMap 供给,重启即恢复;避免无默认
|
||||
// StorageClass 的集群上 PVC Pending 阻塞(UI 内自建的面板需重新导入,见已知限制)。
|
||||
func (d *GrafanaDeployer) grafanaManifest(ns, lokiURL string) string {
|
||||
version := d.version()
|
||||
|
||||
datasources := fmt.Sprintf(`apiVersion: 1
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki-ds
|
||||
access: proxy
|
||||
url: %s
|
||||
isDefault: true
|
||||
editable: false`, lokiURL)
|
||||
|
||||
return fmt.Sprintf(`apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-datasources
|
||||
namespace: %s
|
||||
data:
|
||||
datasources.yaml: |
|
||||
%s
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: %s
|
||||
labels:
|
||||
app: grafana
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: grafana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: grafana
|
||||
spec:
|
||||
containers:
|
||||
- name: grafana
|
||||
image: grafana/grafana:%s
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: GF_SECURITY_ADMIN_USER
|
||||
value: admin
|
||||
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||
value: admin
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
volumeMounts:
|
||||
- name: datasources
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
volumes:
|
||||
- name: datasources
|
||||
configMap:
|
||||
name: grafana-datasources
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: %s
|
||||
labels:
|
||||
app: grafana
|
||||
spec:
|
||||
# NodePort 便于集群外直接访问(端口由集群自动分配,实际端口见部署日志)
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: http
|
||||
port: 3000
|
||||
targetPort: 3000
|
||||
selector:
|
||||
app: grafana
|
||||
`, ns, indentBlock(datasources, " "), ns, version, ns)
|
||||
}
|
||||
|
||||
func (d *GrafanaDeployer) deployK8s(ctx context.Context) error {
|
||||
if d.deployment.Cluster == nil {
|
||||
return fmt.Errorf("please select target cluster")
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(d.deployment.Cluster)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ns := d.deployment.Namespace
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
|
||||
lokiURL := d.lokiPushURL()
|
||||
manifestFile, manifestCleanup, err := writeTempFile("grafana-*.yaml", d.grafanaManifest(ns, lokiURL), 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer manifestCleanup()
|
||||
|
||||
d.log("info", "Applying Grafana manifest (Deployment + NodePort Service + datasource ConfigMap)...", "")
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", manifestFile)
|
||||
cmd.Env = env
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
d.log("error", fmt.Sprintf("kubectl apply grafana failed: %v", err), string(out))
|
||||
return fmt.Errorf("kubectl apply grafana failed: %w\n%s", err, string(out))
|
||||
}
|
||||
|
||||
waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
|
||||
"-n", ns, "--timeout=5m")
|
||||
waitCmd.Env = env
|
||||
waitOut, _ := waitCmd.CombinedOutput()
|
||||
d.log("info", string(waitOut), "")
|
||||
|
||||
// DNS 自愈:部分集群 CoreDNS 对 Service 记录异常(如 watch 失效),
|
||||
// 经 grafana 容器探测 Loki 服务名,不通则回退 ClusterIP 并热更新数据源。
|
||||
dnsProbe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, "deploy/grafana", "--",
|
||||
"wget", "-q", "-O", "/dev/null", "-T", "5", lokiURL+"/ready")
|
||||
dnsProbe.Env = env
|
||||
if probeOut, probeErr := dnsProbe.CombinedOutput(); probeErr != nil {
|
||||
d.log("warn", fmt.Sprintf("Grafana 无法解析 Loki 服务名(%s),回退 ClusterIP…", strings.TrimSpace(string(probeOut))), "")
|
||||
ipCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
|
||||
"-o", "jsonpath={.spec.clusterIP}")
|
||||
ipCmd.Env = env
|
||||
ipOut, ipErr := ipCmd.CombinedOutput()
|
||||
clusterIP := strings.TrimSpace(string(ipOut))
|
||||
if ipErr != nil || clusterIP == "" {
|
||||
d.log("error", "获取 Loki ClusterIP 失败,数据源保持服务名形式", "")
|
||||
} else {
|
||||
fallbackURL := fmt.Sprintf("http://%s:3100", clusterIP)
|
||||
fbFile, fbCleanup, ferr := writeTempFile("grafana-fb-*.yaml", d.grafanaManifest(ns, fallbackURL), 0600)
|
||||
if ferr == nil {
|
||||
fbApply := exec.CommandContext(ctx, "kubectl", "apply", "-f", fbFile)
|
||||
fbApply.Env = env
|
||||
fbApply.CombinedOutput()
|
||||
fbCleanup()
|
||||
restart := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "deployment/grafana", "-n", ns)
|
||||
restart.Env = env
|
||||
restart.CombinedOutput()
|
||||
wait2 := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/grafana",
|
||||
"-n", ns, "--timeout=5m")
|
||||
wait2.Env = env
|
||||
w2, _ := wait2.CombinedOutput()
|
||||
d.log("info", string(w2), "")
|
||||
d.log("info", fmt.Sprintf("数据源已回退为 ClusterIP: %s", fallbackURL), "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 回读实际分配的 NodePort(固定端口易与集群已有服务冲突)
|
||||
portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
|
||||
"-o", "jsonpath={.spec.ports[0].nodePort}")
|
||||
portCmd.Env = env
|
||||
portOut, _ := portCmd.CombinedOutput()
|
||||
nodePort := strings.TrimSpace(string(portOut))
|
||||
access := fmt.Sprintf("http://<节点IP>:%s", nodePort)
|
||||
if nodePort == "" {
|
||||
access = "kubectl get svc grafana -n " + ns + " 查看访问端口"
|
||||
}
|
||||
|
||||
d.log("success", fmt.Sprintf("Grafana deployed. 访问: %s (NodePort, 账号 admin/admin), 数据源已指向 %s", access, d.lokiPushURL()), "")
|
||||
|
||||
// 自动导入 Loki 日志面板(打开即可看图;密码被用户修改过则跳过并提示)
|
||||
if err := d.importDashboard(ctx, env, ns); err != nil {
|
||||
d.log("warn", fmt.Sprintf("面板自动导入跳过: %v", err), "")
|
||||
} else {
|
||||
d.log("info", "已导入「Loki 日志」面板(Explore 可用,默认账号 admin/admin)", "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importDashboard 通过 Grafana API 导入预置的 Loki 日志面板
|
||||
func (d *GrafanaDeployer) importDashboard(ctx context.Context, env []string, ns string) error {
|
||||
payload := grafanaDashboardPayload()
|
||||
|
||||
postDashboard := func(baseURL string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
strings.TrimSuffix(baseURL, "/")+"/api/dashboards/db", strings.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("admin:admin")))
|
||||
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
switch d.deployment.Mode {
|
||||
case "k8s-helm", "k8s-yaml":
|
||||
// 经 NodePort 直连(kube service proxy 的 POST 需额外 RBAC,直连最稳)
|
||||
ipCmd := exec.CommandContext(ctx, "kubectl", "get", "nodes",
|
||||
"-o", "jsonpath={.items[0].status.addresses[0].address}")
|
||||
ipCmd.Env = env
|
||||
ipOut, _ := ipCmd.CombinedOutput()
|
||||
portCmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "grafana", "-n", ns,
|
||||
"-o", "jsonpath={.spec.ports[0].nodePort}")
|
||||
portCmd.Env = env
|
||||
portOut, _ := portCmd.CombinedOutput()
|
||||
ip, port := strings.TrimSpace(string(ipOut)), strings.TrimSpace(string(portOut))
|
||||
if ip == "" || port == "" {
|
||||
return fmt.Errorf("获取 NodePort 失败")
|
||||
}
|
||||
return postDashboard(fmt.Sprintf("http://%s:%s", ip, port))
|
||||
case "host-binary":
|
||||
if d.deployment.Host == nil {
|
||||
return fmt.Errorf("no host")
|
||||
}
|
||||
return postDashboard(fmt.Sprintf("http://%s:3000", d.deployment.Host.HostIP))
|
||||
}
|
||||
return fmt.Errorf("unsupported mode")
|
||||
}
|
||||
|
||||
// ==================== 主机 tarball 部署 ====================
|
||||
|
||||
func (d *GrafanaDeployer) deployHost(ctx context.Context) error {
|
||||
host := d.deployment.Host
|
||||
if host == nil {
|
||||
return fmt.Errorf("please select target host")
|
||||
}
|
||||
client, err := NewSSHClientWithAuth(HostSSHAuth(host))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
arch, _, _ := client.Execute(ctx, "uname -m")
|
||||
grafanaArch := "amd64"
|
||||
if strings.Contains(arch, "aarch64") || strings.Contains(arch, "arm64") {
|
||||
grafanaArch = "arm64"
|
||||
}
|
||||
version := d.version()
|
||||
|
||||
d.log("info", fmt.Sprintf("Downloading Grafana %s (%s)...", version, grafanaArch), "")
|
||||
url := fmt.Sprintf("https://dl.grafana.com/oss/release/grafana-%s.linux-%s.tar.gz", version, grafanaArch)
|
||||
dl := fmt.Sprintf("curl -fsSL -o /tmp/grafana.tar.gz %s && tar -xzf /tmp/grafana.tar.gz -C /opt/ && rm -rf /opt/grafana && ln -sfn /opt/grafana-v%s /opt/grafana && rm -f /tmp/grafana.tar.gz", url, version)
|
||||
if out, _, err := client.Execute(ctx, dl); err != nil {
|
||||
return fmt.Errorf("download grafana failed: %w\n%s", err, out)
|
||||
}
|
||||
|
||||
// 数据源 provisioning(指向同机 Loki)
|
||||
dsYAML := fmt.Sprintf(`apiVersion: 1
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: %s
|
||||
isDefault: true
|
||||
`, d.lokiPushURL())
|
||||
client.Execute(ctx, "mkdir -p /etc/grafana/provisioning/datasources /var/lib/grafana /var/log/grafana")
|
||||
writeCmd := fmt.Sprintf("cat > /etc/grafana/provisioning/datasources/loki.yaml <<'EOF'\n%s\nEOF", dsYAML)
|
||||
if out, _, err := client.Execute(ctx, writeCmd); err != nil {
|
||||
return fmt.Errorf("write datasource config failed: %w\n%s", err, out)
|
||||
}
|
||||
|
||||
unit := `[Unit]
|
||||
Description=Grafana
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/grafana
|
||||
ExecStart=/opt/grafana/bin/grafana server --homepath /opt/grafana --config /opt/grafana/conf/defaults.ini
|
||||
Restart=always
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target`
|
||||
writeUnit := fmt.Sprintf("cat > /etc/systemd/system/grafana.service <<'EOF'\n%s\nEOF", unit)
|
||||
if out, _, err := client.Execute(ctx, writeUnit); err != nil {
|
||||
return fmt.Errorf("write systemd unit failed: %w\n%s", err, out)
|
||||
}
|
||||
if out, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable grafana && systemctl restart grafana"); err != nil {
|
||||
return fmt.Errorf("start grafana failed: %w\n%s", err, out)
|
||||
}
|
||||
|
||||
// 健康检查(最多 10 次)
|
||||
for i := 0; i < 10; i++ {
|
||||
out, _, _ := client.Execute(ctx, "curl -fsSL http://localhost:3000/api/health")
|
||||
if strings.Contains(out, "\"version\"") {
|
||||
d.log("success", fmt.Sprintf("Grafana running on %s:3000 (admin/admin), 数据源已指向本机 Loki", host.HostIP), "")
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
d.log("warn", "Grafana 服务已启动但健康检查未通过,请稍后确认 http://<主机IP>:3000", "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupGrafana 卸载 Grafana(停止部署时同步调用)
|
||||
func CleanupGrafana(ctx context.Context, deployment *models.LokiDeployment) {
|
||||
if deployment == nil {
|
||||
return
|
||||
}
|
||||
switch deployment.Mode {
|
||||
case "k8s-helm", "k8s-yaml":
|
||||
if deployment.Cluster == nil {
|
||||
return
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(deployment.Cluster)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer cleanup()
|
||||
ns := deployment.Namespace
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete",
|
||||
"deployment/grafana", "service/grafana", "configmap/grafana-datasources",
|
||||
"-n", ns, "--ignore-not-found=true")
|
||||
cmd.Env = env
|
||||
cmd.CombinedOutput()
|
||||
case "host-binary":
|
||||
if deployment.Host == nil {
|
||||
return
|
||||
}
|
||||
client, err := NewSSHClientWithAuth(HostSSHAuth(deployment.Host))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
client.Execute(ctx, "systemctl stop grafana 2>/dev/null; systemctl disable grafana 2>/dev/null; rm -f /etc/systemd/system/grafana.service; systemctl daemon-reload")
|
||||
}
|
||||
}
|
||||
|
||||
// grafanaDashboardPayload 预置「Loki 日志」面板:日志量(按命名空间)/错误日志/最近日志
|
||||
func grafanaDashboardPayload() string {
|
||||
return `{"dashboard":{
|
||||
"uid":"loki-logs","title":"Loki 日志","tags":["lokideploy"],
|
||||
"timezone":"browser","schemaVersion":39,"refresh":"30s","version":1,
|
||||
"time":{"from":"now-1h","to":"now"},
|
||||
"panels":[
|
||||
{"id":1,"type":"timeseries","title":"日志量(按命名空间)",
|
||||
"datasource":{"type":"loki","uid":"loki-ds"},
|
||||
"gridPos":{"h":8,"w":12,"x":0,"y":0},
|
||||
"fieldConfig":{"defaults":{"unit":"short"},"overrides":[]},
|
||||
"targets":[{"refId":"A","expr":"sum by (namespace) (count_over_time({namespace=\".+\"}[5m]))","queryType":"range"}]},
|
||||
{"id":2,"type":"timeseries","title":"错误日志",
|
||||
"datasource":{"type":"loki","uid":"loki-ds"},
|
||||
"gridPos":{"h":8,"w":12,"x":12,"y":0},
|
||||
"fieldConfig":{"defaults":{"unit":"short","color":{"mode":"palette-classic"}},"overrides":[]},
|
||||
"targets":[{"refId":"A","expr":"sum(count_over_time({namespace=\".+\"} |~ \"(?i)error\" [5m]))","queryType":"range"}]},
|
||||
{"id":3,"type":"logs","title":"最近日志",
|
||||
"datasource":{"type":"loki","uid":"loki-ds"},
|
||||
"gridPos":{"h":10,"w":24,"x":0,"y":8},
|
||||
"targets":[{"refId":"A","expr":"{namespace=\".+\"}","queryType":"range"}]}
|
||||
]
|
||||
},"overwrite":true}`
|
||||
}
|
||||
Reference in new issue
Block a user