logan
f9c6c1d826
基于 WEB-UI-Development-Guide 标准的整体升级,并更名 LokiDeploy → opsx-loki。 前端(Gin+React+shadcn/ui): - 按 02/07 章重构:页面域化拆分(无超 400 行文件)、PageContainer/ListQuery、 反馈分级(成功静默/失败 toast/危险确认)、401 单飞刷新+队列重放 - 暗色模式(next-themes)+ 语义 Token(--primary 品牌色/success/warning/info) - 部署向导(状态推导四步:环境→存储→Loki→采集,内嵌表单闭环) - 日志查询页:LogQL 标签构建器/拖拽直方图/日志上下文/Live Tail/查询历史 - 按用户+标签策略的查询授权(服务端强制合并策略) 后端(Go+Gin+JSON DB): - 简单用户管理(仅 admin,无 RBAC)+ TokenVersion 会话吊销 + 登录 IP 限流 - LDAP 接入:配置(AES 加密存储/hasPassword 脱敏)/测试连接/批量导入/登录 - 存储配置测试连接(SigV4 零依赖)+ 桶不存在自动创建 - Loki 三模式部署修复:SSH 密钥认证、3.x 配置模板、ruler 对象存储嵌套块、 凭证解密、孤儿 deploying 状态恢复、Grafana 部署(可选默认关)+面板导入 - DaemonSet 采集纯 YAML(去 helm 依赖)+ DNS 自愈回退 ClusterIP + 集群名标签 - 日志查询代理:cluster-svc 经 kube-apiserver proxy/url 直连 + cursor 翻页 - 统一 log 命名空间;部署就绪探活;Grafana 访问地址自动记录 修复:run.sh 增量编译检测、Sanitized 脱敏补全、LokiDeployment Update 白名单、 健康检查 CreatedAt 零值等 10+ 存量 bug。 数据兼容:DB 沿用旧 lokideploy.json(新名优先);localStorage 一次性迁移; HKDF 加密前缀保留(已加密凭证不受影响)。
449 lines
14 KiB
Go
449 lines
14 KiB
Go
package services
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"os/exec"
|
||
"strings"
|
||
|
||
"opsxloki/models"
|
||
)
|
||
|
||
// CollectorDeployer 采集器部署器，按 type 分发到三种安装/卸载流程
|
||
type CollectorDeployer struct {
|
||
collector *models.Collector
|
||
loki *models.LokiDeployment
|
||
logFunc func(level, message, detail string)
|
||
}
|
||
|
||
func NewCollectorDeployer(c *models.Collector, l *models.LokiDeployment, logFunc func(level, message, detail string)) *CollectorDeployer {
|
||
return &CollectorDeployer{collector: c, loki: l, logFunc: logFunc}
|
||
}
|
||
|
||
func (d *CollectorDeployer) log(level, message, detail string) {
|
||
if d.logFunc != nil {
|
||
d.logFunc(level, message, detail)
|
||
}
|
||
}
|
||
|
||
// kubeconfigEnv 构造 kubectl/helm 所需的 KUBECONFIG 环境变量，返回 cleanup 删除临时文件
|
||
func kubeconfigEnv(cluster *models.Cluster) (env []string, cleanup func(), err error) {
|
||
kubeconfig, err := resolveKubeconfig(cluster)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
kubeFile, err := os.CreateTemp("", "kube-*.yaml")
|
||
if err != nil {
|
||
return nil, nil, fmt.Errorf("create kubeconfig temp file failed: %w", err)
|
||
}
|
||
if _, err := kubeFile.WriteString(kubeconfig); err != nil {
|
||
kubeFile.Close()
|
||
os.Remove(kubeFile.Name())
|
||
return nil, nil, fmt.Errorf("write kubeconfig temp file failed: %w", err)
|
||
}
|
||
kubeFile.Close()
|
||
os.Chmod(kubeFile.Name(), 0600)
|
||
env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile.Name()))
|
||
cleanup = func() { os.Remove(kubeFile.Name()) }
|
||
return env, cleanup, nil
|
||
}
|
||
|
||
// Install 按采集器类型分发安装
|
||
func (d *CollectorDeployer) Install(ctx context.Context) error {
|
||
endpoint := d.loki.Endpoint
|
||
if endpoint == "" {
|
||
return fmt.Errorf("关联的 Loki 尚未部署，缺少推送地址（endpoint），请先成功部署该 Loki")
|
||
}
|
||
d.log("info", fmt.Sprintf("开始部署采集器 %s（类型 %s）", d.collector.Name, d.collector.Type), "")
|
||
|
||
var err error
|
||
switch d.collector.Type {
|
||
case "host-promtail":
|
||
err = d.installHost(ctx, endpoint)
|
||
case "k8s-daemonset":
|
||
err = d.installDaemonset(ctx, endpoint)
|
||
case "k8s-fluentbit-sidecar":
|
||
err = d.installSidecar(ctx, endpoint)
|
||
default:
|
||
return fmt.Errorf("不支持的采集器类型: %s", d.collector.Type)
|
||
}
|
||
if err != nil {
|
||
d.log("error", fmt.Sprintf("采集器部署失败: %v", err), "")
|
||
return err
|
||
}
|
||
d.log("success", "采集器部署完成", "")
|
||
return nil
|
||
}
|
||
|
||
// Uninstall 按采集器类型分发卸载
|
||
func (d *CollectorDeployer) Uninstall(ctx context.Context) error {
|
||
d.log("info", fmt.Sprintf("开始卸载采集器 %s（类型 %s）", d.collector.Name, d.collector.Type), "")
|
||
switch d.collector.Type {
|
||
case "host-promtail":
|
||
return d.uninstallHost(ctx)
|
||
case "k8s-daemonset":
|
||
return d.uninstallDaemonset(ctx)
|
||
case "k8s-fluentbit-sidecar":
|
||
return d.uninstallSidecar(ctx)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ===== host-promtail：SSH 安装 Promtail 二进制 + systemd =====
|
||
|
||
func (d *CollectorDeployer) installHost(ctx context.Context, endpoint string) error {
|
||
host := d.collector.Host
|
||
if host == nil {
|
||
return fmt.Errorf("主机采集器缺少目标主机")
|
||
}
|
||
client, err := NewSSHClientWithAuth(HostSSHAuth(host))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer client.Close()
|
||
|
||
// 检测架构
|
||
archOut, _, _ := client.Execute(ctx, "uname -m")
|
||
arch := strings.TrimSpace(archOut)
|
||
switch arch {
|
||
case "x86_64":
|
||
arch = "amd64"
|
||
case "aarch64", "arm64":
|
||
arch = "arm64"
|
||
default:
|
||
return fmt.Errorf("不支持的主机架构: %s", arch)
|
||
}
|
||
|
||
// 确定 Promtail 版本（优先使用关联的 Loki 版本）
|
||
version := "3.0.0"
|
||
if d.loki != nil && d.loki.Version != "" {
|
||
version = d.loki.Version
|
||
}
|
||
|
||
// 创建必要目录
|
||
d.log("info", "准备目录 /opt/promtail /etc/promtail /var/lib/promtail", "")
|
||
if _, _, err := client.Execute(ctx, "mkdir -p /opt/promtail /etc/promtail /var/lib/promtail"); err != nil {
|
||
return fmt.Errorf("创建 Promtail 目录失败: %w", err)
|
||
}
|
||
|
||
// 下载 Promtail
|
||
d.log("info", fmt.Sprintf("下载 Promtail %s（%s）", version, arch), "")
|
||
url := fmt.Sprintf("https://github.com/grafana/loki/releases/download/v%s/promtail-linux-%s.zip", version, arch)
|
||
dl := fmt.Sprintf("cd /tmp && curl -fsSL -o p.zip '%s' && unzip -o p.zip -d /opt/promtail/ && chmod +x /opt/promtail/promtail-linux-%s", url, arch)
|
||
if _, _, err := client.Execute(ctx, dl); err != nil {
|
||
return fmt.Errorf("下载 Promtail 失败: %w", err)
|
||
}
|
||
client.Execute(ctx, "ln -sf /opt/promtail/promtail-linux-* /usr/local/bin/promtail 2>/dev/null || true")
|
||
|
||
// 生成并上传配置
|
||
logPath := d.collector.LogPath
|
||
if logPath == "" {
|
||
logPath = "/var/log/*.log"
|
||
}
|
||
conf := PromtailConfig(endpoint, logPath, "host", d.collector.Labels)
|
||
d.log("info", "上传 Promtail 配置", "")
|
||
if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/promtail/promtail.yaml << 'EOF'\n%s\nEOF", conf)); err != nil {
|
||
return fmt.Errorf("上传配置失败: %w", err)
|
||
}
|
||
|
||
// systemd 服务（生产级）
|
||
service := `[Unit]
|
||
Description=Promtail Log Collector
|
||
Wants=network-online.target
|
||
After=network-online.target
|
||
|
||
[Service]
|
||
Type=simple
|
||
User=root
|
||
Group=root
|
||
ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml -positions.enabled=true
|
||
ExecReload=/bin/kill -HUP $MAINPID
|
||
Restart=always
|
||
RestartSec=10
|
||
LimitNOFILE=65536
|
||
StandardOutput=append:/var/log/promtail.log
|
||
StandardError=append:/var/log/promtail.log
|
||
|
||
[Install]
|
||
WantedBy=multi-user.target
|
||
`
|
||
d.log("info", "安装 systemd 服务", "")
|
||
if _, _, err := client.Execute(ctx, fmt.Sprintf("cat > /etc/systemd/system/promtail.service << 'EOF'\n%s\nEOF", service)); err != nil {
|
||
return fmt.Errorf("创建 systemd 服务失败: %w", err)
|
||
}
|
||
if _, _, err := client.Execute(ctx, "systemctl daemon-reload && systemctl enable promtail && systemctl restart promtail"); err != nil {
|
||
return fmt.Errorf("启动 Promtail 失败: %w", err)
|
||
}
|
||
|
||
// 保存配置快照
|
||
d.collector.Config = conf
|
||
d.log("success", "Promtail 部署完成", "")
|
||
return nil
|
||
}
|
||
|
||
func (d *CollectorDeployer) uninstallHost(ctx context.Context) error {
|
||
host := d.collector.Host
|
||
if host == nil {
|
||
return nil
|
||
}
|
||
client, err := NewSSHClientWithAuth(HostSSHAuth(host))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer client.Close()
|
||
client.Execute(ctx, "systemctl stop promtail 2>/dev/null || true")
|
||
client.Execute(ctx, "systemctl disable promtail 2>/dev/null || true")
|
||
client.Execute(ctx, "rm -rf /etc/systemd/system/promtail.service /usr/local/bin/promtail /etc/promtail /opt/promtail /var/lib/promtail")
|
||
client.Execute(ctx, "systemctl daemon-reload")
|
||
return nil
|
||
}
|
||
|
||
// ===== k8s-daemonset：helm 安装 Promtail DaemonSet =====
|
||
|
||
func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint string) error {
|
||
cluster := d.collector.Cluster
|
||
if cluster == nil {
|
||
return fmt.Errorf("集群采集器缺少目标集群")
|
||
}
|
||
env, cleanup, err := kubeconfigEnv(cluster)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cleanup()
|
||
|
||
// 默认部署到关联 Loki 的命名空间(统一 log 命名空间策略)
|
||
ns := d.collector.Namespace
|
||
if ns == "" && d.loki != nil {
|
||
ns = d.loki.Namespace
|
||
}
|
||
if ns == "" {
|
||
ns = "log"
|
||
}
|
||
|
||
clusterName := "unknown"
|
||
if cluster.Name != "" {
|
||
clusterName = cluster.Name
|
||
}
|
||
|
||
// 幂等:先清掉同名残留(如历史失败安装留下的无注解资源)
|
||
del := exec.CommandContext(ctx, "kubectl", "delete",
|
||
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
|
||
"-n", ns, "--ignore-not-found=true", "--wait=false")
|
||
del.Env = env
|
||
del.CombinedOutput()
|
||
|
||
// 纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),不依赖后端安装 helm
|
||
manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels)
|
||
if err := d.applyYAML(ctx, env, manifest, "promtail"); err != nil {
|
||
return err
|
||
}
|
||
|
||
d.log("info", fmt.Sprintf("等待 DaemonSet promtail 就绪(命名空间 %s)...", ns), "")
|
||
waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "daemonset/promtail",
|
||
"-n", ns, "--timeout=5m")
|
||
waitCmd.Env = env
|
||
waitOut, _ := waitCmd.CombinedOutput()
|
||
d.log("info", string(waitOut), "")
|
||
|
||
// DNS 自愈:CoreDNS 异常的集群上服务名不可解析,回退 Loki ClusterIP 重建配置
|
||
svcURL := endpoint
|
||
if strings.Contains(endpoint, ".svc") {
|
||
probe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns,
|
||
"daemonset/promtail", "--", "wget", "-q", "-O", "/dev/null", "-T", "5", endpoint+"/ready")
|
||
probe.Env = env
|
||
if pout, perr := probe.CombinedOutput(); perr != nil {
|
||
d.log("warn", fmt.Sprintf("promtail 无法解析 Loki 服务名(%s),回退 ClusterIP...", strings.TrimSpace(string(pout))), "")
|
||
ip := d.lokiClusterIP(ctx, env, endpoint)
|
||
if ip != "" {
|
||
// 只替换 host 部分,保留 scheme/port/path: http://loki.ns.svc:3100/... → http://IP:3100/...
|
||
afterScheme := strings.SplitN(endpoint, "//", 2)
|
||
if len(afterScheme) == 2 {
|
||
rest := afterScheme[1]
|
||
host := rest
|
||
if i := strings.IndexAny(rest, ":/"); i >= 0 {
|
||
host = rest[:i]
|
||
}
|
||
fallback := afterScheme[0] + "//" + strings.Replace(rest, host, ip, 1)
|
||
if err := d.applyYAML(ctx, env, PromtailDaemonsetYAML(ns, fallback, clusterName, d.collector.Labels), "promtail"); err == nil {
|
||
rs := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "daemonset/promtail", "-n", ns)
|
||
rs.Env = env
|
||
rs.CombinedOutput()
|
||
d.log("info", fmt.Sprintf("promtail 推送地址已回退为 ClusterIP: %s", fallback), "")
|
||
svcURL = fallback
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
d.collector.Config = svcURL
|
||
d.log("success", fmt.Sprintf("Promtail DaemonSet 已部署到 %s(推送至 %s)", ns, svcURL), "")
|
||
return nil
|
||
}
|
||
|
||
// applyYAML kubectl apply 一个 manifest(临时文件)
|
||
func (d *CollectorDeployer) applyYAML(ctx context.Context, env []string, manifest, label string) error {
|
||
f, cleanup, err := writeTempFile("collector-*.yaml", manifest, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cleanup()
|
||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", f)
|
||
cmd.Env = env
|
||
out, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
d.log("error", fmt.Sprintf("kubectl apply %s failed: %v", label, err), string(out))
|
||
return fmt.Errorf("kubectl apply %s failed: %w\n%s", label, err, string(out))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// lokiClusterIP 从 endpoint 的命名空间取 loki Service 的 ClusterIP(不经 DNS)
|
||
func (d *CollectorDeployer) lokiClusterIP(ctx context.Context, env []string, endpoint string) string {
|
||
ns := "log"
|
||
if d.loki != nil && d.loki.Namespace != "" {
|
||
ns = d.loki.Namespace
|
||
}
|
||
cmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
|
||
"-o", "jsonpath={.spec.clusterIP}")
|
||
cmd.Env = env
|
||
out, _ := cmd.CombinedOutput()
|
||
return strings.TrimSpace(string(out))
|
||
}
|
||
|
||
func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error {
|
||
cluster := d.collector.Cluster
|
||
if cluster == nil {
|
||
return nil
|
||
}
|
||
env, cleanup, err := kubeconfigEnv(cluster)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cleanup()
|
||
ns := d.collector.Namespace
|
||
if ns == "" && d.loki != nil {
|
||
ns = d.loki.Namespace
|
||
}
|
||
if ns == "" {
|
||
ns = "log"
|
||
}
|
||
cmd := exec.CommandContext(ctx, "kubectl", "delete",
|
||
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
|
||
"clusterrole/opsxloki-promtail", "clusterrolebinding/opsxloki-promtail",
|
||
"-n", ns, "--ignore-not-found=true")
|
||
cmd.Env = env
|
||
cmd.CombinedOutput()
|
||
return nil
|
||
}
|
||
|
||
// ===== k8s-fluentbit-sidecar：kubectl patch 注入 Fluent Bit sidecar =====
|
||
|
||
func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string) error {
|
||
cluster := d.collector.Cluster
|
||
if cluster == nil {
|
||
return fmt.Errorf("Sidecar 采集器缺少目标集群")
|
||
}
|
||
target := d.collector.TargetDeployment
|
||
if target == "" {
|
||
return fmt.Errorf("Sidecar 采集器缺少目标 Deployment")
|
||
}
|
||
env, cleanup, err := kubeconfigEnv(cluster)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cleanup()
|
||
|
||
ns := d.collector.Namespace
|
||
if ns == "" {
|
||
ns = "default"
|
||
}
|
||
logPath := d.collector.LogPath
|
||
if logPath == "" {
|
||
logPath = "/var/log/app/*.log"
|
||
}
|
||
app := target
|
||
|
||
// 1. 检查目标 deployment 是否存在
|
||
d.log("info", fmt.Sprintf("检查目标 Deployment %s/%s", ns, target), "")
|
||
check := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns)
|
||
check.Env = env
|
||
if out, err := check.CombinedOutput(); err != nil {
|
||
return fmt.Errorf("目标 Deployment 不存在: %s/%s\n%s", ns, target, string(out))
|
||
}
|
||
|
||
// 2. 幂等检查：是否已有 fluent-bit 容器
|
||
getNames := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns, "-o", "jsonpath={.spec.template.spec.containers[*].name}")
|
||
getNames.Env = env
|
||
namesOut, _ := getNames.CombinedOutput()
|
||
if strings.Contains(string(namesOut), "fluent-bit") {
|
||
d.log("warn", "目标 Deployment 已含 fluent-bit 容器，跳过注入", "")
|
||
return nil
|
||
}
|
||
|
||
// 3. 部署 ConfigMap
|
||
lokiHost, lokiPort := parseLokiURL(endpoint)
|
||
conf := FluentbitConfig(lokiHost, lokiPort, logPath, app, ns)
|
||
cm := FluentbitConfigMapYAML(conf, ns)
|
||
cmFile, cmCleanup, err := writeTempFile("fluent-bit-cm-*.yaml", cm, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cmCleanup()
|
||
|
||
d.log("info", "部署 Fluent Bit ConfigMap", "")
|
||
applyCm := exec.CommandContext(ctx, "kubectl", "apply", "-f", cmFile)
|
||
applyCm.Env = env
|
||
if out, err := applyCm.CombinedOutput(); err != nil {
|
||
return fmt.Errorf("ConfigMap 部署失败: %w\n%s", err, string(out))
|
||
}
|
||
|
||
// 4. patch 注入 sidecar
|
||
d.log("info", fmt.Sprintf("为 %s 注入 Fluent Bit Sidecar", target), "")
|
||
patchFile, patchCleanup, err := writeTempFile("sidecar-patch-*.yaml", SidecarPatchYAML, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer patchCleanup()
|
||
|
||
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", target, "-n", ns, "--type=strategic", "--patch-file", patchFile)
|
||
patch.Env = env
|
||
if out, err := patch.CombinedOutput(); err != nil {
|
||
return fmt.Errorf("Sidecar 注入失败: %w\n%s", err, string(out))
|
||
}
|
||
|
||
// 5. 等待 rollout
|
||
d.log("info", "等待 Pod 重建", "")
|
||
rollout := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/"+target, "-n", ns, "--timeout=5m")
|
||
rollout.Env = env
|
||
rollout.CombinedOutput()
|
||
return nil
|
||
}
|
||
|
||
func (d *CollectorDeployer) uninstallSidecar(ctx context.Context) error {
|
||
cluster := d.collector.Cluster
|
||
if cluster == nil || d.collector.TargetDeployment == "" {
|
||
return nil
|
||
}
|
||
env, cleanup, err := kubeconfigEnv(cluster)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer cleanup()
|
||
ns := d.collector.Namespace
|
||
if ns == "" {
|
||
ns = "default"
|
||
}
|
||
patchFile, patchCleanup, err := writeTempFile("sidecar-rm-*.yaml", SidecarRemovePatchYAML, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer patchCleanup()
|
||
|
||
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", d.collector.TargetDeployment, "-n", ns, "--type=strategic", "--patch-file", patchFile)
|
||
patch.Env = env
|
||
patch.CombinedOutput()
|
||
return nil
|
||
}
|