feat: complete Loki query, insights, settings and deployment workflows
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application. Validated with go test ./..., npm run build, and npm run lint.
This commit is contained in:
logan committed
1 parent
f9c6c1d826
commit
2d73661dd1
127 files changed
+8117
-2876
No files matched your search
@@ -2,10 +2,11 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"opsxloki/models"
|
||||
)
|
||||
@@ -27,28 +28,6 @@ func (d *CollectorDeployer) log(level, message, detail string) {
|
||||
}
|
||||
}
|
||||
|
||||
// kubeconfigEnv 构造 kubectl/helm 所需的 KUBECONFIG 环境变量，返回 cleanup 删除临时文件
|
||||
func kubeconfigEnv(cluster *models.Cluster) (env []string, cleanup func(), err error) {
|
||||
kubeconfig, err := resolveKubeconfig(cluster)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
kubeFile, err := os.CreateTemp("", "kube-*.yaml")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create kubeconfig temp file failed: %w", err)
|
||||
}
|
||||
if _, err := kubeFile.WriteString(kubeconfig); err != nil {
|
||||
kubeFile.Close()
|
||||
os.Remove(kubeFile.Name())
|
||||
return nil, nil, fmt.Errorf("write kubeconfig temp file failed: %w", err)
|
||||
}
|
||||
kubeFile.Close()
|
||||
os.Chmod(kubeFile.Name(), 0600)
|
||||
env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeFile.Name()))
|
||||
cleanup = func() { os.Remove(kubeFile.Name()) }
|
||||
return env, cleanup, nil
|
||||
}
|
||||
|
||||
// Install 按采集器类型分发安装
|
||||
func (d *CollectorDeployer) Install(ctx context.Context) error {
|
||||
endpoint := d.loki.Endpoint
|
||||
@@ -206,11 +185,6 @@ func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint strin
|
||||
if cluster == nil {
|
||||
return fmt.Errorf("集群采集器缺少目标集群")
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(cluster)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
// 默认部署到关联 Loki 的命名空间(统一 log 命名空间策略)
|
||||
ns := d.collector.Namespace
|
||||
@@ -226,102 +200,43 @@ func (d *CollectorDeployer) installDaemonset(ctx context.Context, endpoint strin
|
||||
clusterName = cluster.Name
|
||||
}
|
||||
|
||||
// 幂等:先清掉同名残留(如历史失败安装留下的无注解资源)
|
||||
del := exec.CommandContext(ctx, "kubectl", "delete",
|
||||
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
|
||||
"-n", ns, "--ignore-not-found=true", "--wait=false")
|
||||
del.Env = env
|
||||
del.CombinedOutput()
|
||||
|
||||
// 纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),不依赖后端安装 helm
|
||||
// 删除历史残留并通过 Kubernetes API 提交资源。
|
||||
for _, kind := range []string{"daemonset", "configmap", "serviceaccount"} {
|
||||
_ = deleteK8sResource(ctx, cluster, kind, ns, "promtail")
|
||||
}
|
||||
manifest := PromtailDaemonsetYAML(ns, endpoint, clusterName, d.collector.Labels)
|
||||
if err := d.applyYAML(ctx, env, manifest, "promtail"); err != nil {
|
||||
if err := d.applyYAML(ctx, nil, manifest, "promtail"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.log("info", fmt.Sprintf("等待 DaemonSet promtail 就绪(命名空间 %s)...", ns), "")
|
||||
waitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "daemonset/promtail",
|
||||
"-n", ns, "--timeout=5m")
|
||||
waitCmd.Env = env
|
||||
waitOut, _ := waitCmd.CombinedOutput()
|
||||
d.log("info", string(waitOut), "")
|
||||
|
||||
// DNS 自愈:CoreDNS 异常的集群上服务名不可解析,回退 Loki ClusterIP 重建配置
|
||||
svcURL := endpoint
|
||||
if strings.Contains(endpoint, ".svc") {
|
||||
probe := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns,
|
||||
"daemonset/promtail", "--", "wget", "-q", "-O", "/dev/null", "-T", "5", endpoint+"/ready")
|
||||
probe.Env = env
|
||||
if pout, perr := probe.CombinedOutput(); perr != nil {
|
||||
d.log("warn", fmt.Sprintf("promtail 无法解析 Loki 服务名(%s),回退 ClusterIP...", strings.TrimSpace(string(pout))), "")
|
||||
ip := d.lokiClusterIP(ctx, env, endpoint)
|
||||
if ip != "" {
|
||||
// 只替换 host 部分,保留 scheme/port/path: http://loki.ns.svc:3100/... → http://IP:3100/...
|
||||
afterScheme := strings.SplitN(endpoint, "//", 2)
|
||||
if len(afterScheme) == 2 {
|
||||
rest := afterScheme[1]
|
||||
host := rest
|
||||
if i := strings.IndexAny(rest, ":/"); i >= 0 {
|
||||
host = rest[:i]
|
||||
}
|
||||
fallback := afterScheme[0] + "//" + strings.Replace(rest, host, ip, 1)
|
||||
if err := d.applyYAML(ctx, env, PromtailDaemonsetYAML(ns, fallback, clusterName, d.collector.Labels), "promtail"); err == nil {
|
||||
rs := exec.CommandContext(ctx, "kubectl", "rollout", "restart", "daemonset/promtail", "-n", ns)
|
||||
rs.Env = env
|
||||
rs.CombinedOutput()
|
||||
d.log("info", fmt.Sprintf("promtail 推送地址已回退为 ClusterIP: %s", fallback), "")
|
||||
svcURL = fallback
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := waitDaemonSet(ctx, cluster, ns, "promtail"); err != nil {
|
||||
return fmt.Errorf("Promtail DaemonSet 未就绪: %w", err)
|
||||
}
|
||||
svcURL := endpoint
|
||||
|
||||
d.collector.Config = svcURL
|
||||
d.log("success", fmt.Sprintf("Promtail DaemonSet 已部署到 %s(推送至 %s)", ns, svcURL), "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyYAML kubectl apply 一个 manifest(临时文件)
|
||||
func (d *CollectorDeployer) applyYAML(ctx context.Context, env []string, manifest, label string) error {
|
||||
f, cleanup, err := writeTempFile("collector-*.yaml", manifest, 0600)
|
||||
if err != nil {
|
||||
// applyYAML 通过 API Server 应用采集器清单。
|
||||
func (d *CollectorDeployer) applyYAML(ctx context.Context, _ []string, manifest, label string) error {
|
||||
if d.collector.Cluster == nil {
|
||||
return fmt.Errorf("采集器缺少集群")
|
||||
}
|
||||
if err := applyManifest(ctx, d.collector.Cluster, manifest); err != nil {
|
||||
d.log("error", fmt.Sprintf("应用 %s 失败: %v", label, err), "")
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", f)
|
||||
cmd.Env = env
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
d.log("error", fmt.Sprintf("kubectl apply %s failed: %v", label, err), string(out))
|
||||
return fmt.Errorf("kubectl apply %s failed: %w\n%s", label, err, string(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lokiClusterIP 从 endpoint 的命名空间取 loki Service 的 ClusterIP(不经 DNS)
|
||||
func (d *CollectorDeployer) lokiClusterIP(ctx context.Context, env []string, endpoint string) string {
|
||||
ns := "log"
|
||||
if d.loki != nil && d.loki.Namespace != "" {
|
||||
ns = d.loki.Namespace
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "get", "svc", "loki", "-n", ns,
|
||||
"-o", "jsonpath={.spec.clusterIP}")
|
||||
cmd.Env = env
|
||||
out, _ := cmd.CombinedOutput()
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error {
|
||||
cluster := d.collector.Cluster
|
||||
if cluster == nil {
|
||||
return nil
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(cluster)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ns := d.collector.Namespace
|
||||
if ns == "" && d.loki != nil {
|
||||
ns = d.loki.Namespace
|
||||
@@ -329,12 +244,11 @@ func (d *CollectorDeployer) uninstallDaemonset(ctx context.Context) error {
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete",
|
||||
"daemonset/promtail", "configmap/promtail", "serviceaccount/promtail",
|
||||
"clusterrole/opsxloki-promtail", "clusterrolebinding/opsxloki-promtail",
|
||||
"-n", ns, "--ignore-not-found=true")
|
||||
cmd.Env = env
|
||||
cmd.CombinedOutput()
|
||||
for _, item := range [][2]string{{"daemonset", "promtail"}, {"configmap", "promtail"}, {"serviceaccount", "promtail"}, {"clusterrole", "opsxloki-promtail"}, {"clusterrolebinding", "opsxloki-promtail"}} {
|
||||
if err := deleteK8sResource(ctx, cluster, item[0], ns, item[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -349,11 +263,10 @@ func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string)
|
||||
if target == "" {
|
||||
return fmt.Errorf("Sidecar 采集器缺少目标 Deployment")
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(cluster)
|
||||
client, err := clusterClient(cluster, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ns := d.collector.Namespace
|
||||
if ns == "" {
|
||||
@@ -367,57 +280,65 @@ func (d *CollectorDeployer) installSidecar(ctx context.Context, endpoint string)
|
||||
|
||||
// 1. 检查目标 deployment 是否存在
|
||||
d.log("info", fmt.Sprintf("检查目标 Deployment %s/%s", ns, target), "")
|
||||
check := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns)
|
||||
check.Env = env
|
||||
if out, err := check.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("目标 Deployment 不存在: %s/%s\n%s", ns, target, string(out))
|
||||
deployment, err := client.AppsV1().Deployments(ns).Get(ctx, target, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取目标 Deployment %s/%s: %w", ns, target, err)
|
||||
}
|
||||
|
||||
// 2. 幂等检查：是否已有 fluent-bit 容器
|
||||
getNames := exec.CommandContext(ctx, "kubectl", "get", "deployment", target, "-n", ns, "-o", "jsonpath={.spec.template.spec.containers[*].name}")
|
||||
getNames.Env = env
|
||||
namesOut, _ := getNames.CombinedOutput()
|
||||
if strings.Contains(string(namesOut), "fluent-bit") {
|
||||
d.log("warn", "目标 Deployment 已含 fluent-bit 容器，跳过注入", "")
|
||||
return nil
|
||||
deploymentJSON, err := json.Marshal(deployment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mount, err := resolveSidecarLogMount(deploymentJSON, target, logPath, d.collector.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hasSidecar := false
|
||||
for _, container := range deployment.Spec.Template.Spec.Containers {
|
||||
if container.Name == "fluent-bit" {
|
||||
hasSidecar = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 部署 ConfigMap
|
||||
lokiHost, lokiPort := parseLokiURL(endpoint)
|
||||
conf := FluentbitConfig(lokiHost, lokiPort, logPath, app, ns)
|
||||
cm := FluentbitConfigMapYAML(conf, ns)
|
||||
cmFile, cmCleanup, err := writeTempFile("fluent-bit-cm-*.yaml", cm, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cmCleanup()
|
||||
|
||||
configMapName := fmt.Sprintf("fluent-bit-config-%d", d.collector.ID)
|
||||
cm := FluentbitConfigMapYAML(conf, ns, configMapName)
|
||||
d.log("info", "部署 Fluent Bit ConfigMap", "")
|
||||
applyCm := exec.CommandContext(ctx, "kubectl", "apply", "-f", cmFile)
|
||||
applyCm.Env = env
|
||||
if out, err := applyCm.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("ConfigMap 部署失败: %w\n%s", err, string(out))
|
||||
if err := applyManifest(ctx, cluster, cm); err != nil {
|
||||
return fmt.Errorf("ConfigMap 部署失败: %w", err)
|
||||
}
|
||||
|
||||
// 4. patch 注入 sidecar
|
||||
d.log("info", fmt.Sprintf("为 %s 注入 Fluent Bit Sidecar", target), "")
|
||||
patchFile, patchCleanup, err := writeTempFile("sidecar-patch-*.yaml", SidecarPatchYAML, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
d.log("info", fmt.Sprintf("为 %s 配置 Fluent Bit Sidecar", target), "")
|
||||
if err := patchDeployment(ctx, cluster, ns, target, SidecarPatchYAML(configMapName, mount)); err != nil {
|
||||
return fmt.Errorf("Sidecar 注入失败: %w", err)
|
||||
}
|
||||
defer patchCleanup()
|
||||
|
||||
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", target, "-n", ns, "--type=strategic", "--patch-file", patchFile)
|
||||
patch.Env = env
|
||||
if out, err := patch.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("Sidecar 注入失败: %w\n%s", err, string(out))
|
||||
if hasSidecar {
|
||||
if err := restartDeployment(ctx, cluster, ns, target); err != nil {
|
||||
return fmt.Errorf("Sidecar 配置已更新，但重启 Deployment 失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 等待 rollout
|
||||
d.log("info", "等待 Pod 重建", "")
|
||||
rollout := exec.CommandContext(ctx, "kubectl", "rollout", "status", "deployment/"+target, "-n", ns, "--timeout=5m")
|
||||
rollout.Env = env
|
||||
rollout.CombinedOutput()
|
||||
if err := waitDeployment(ctx, cluster, ns, target); err != nil {
|
||||
return fmt.Errorf("Sidecar Deployment 未就绪: %w", err)
|
||||
}
|
||||
|
||||
// DNS 自愈:CoreDNS 异常集群上 svc 域名不可解析 → 用 Loki ClusterIP 重建 ConfigMap 并滚动重启
|
||||
if strings.Contains(endpoint, ".svc") {
|
||||
host := strings.Split(strings.Split(endpoint, "//")[1], ":")[0]
|
||||
if podDNSBroken(ctx, cluster, ns, target, "fluent-bit", host) {
|
||||
d.log("warn", "fluent-bit 无法解析 Loki 服务名，回退 ClusterIP…", "")
|
||||
if ip, ierr := serviceClusterIP(ctx, cluster, lokiNamespaceOf(d.loki, endpoint), "loki"); ierr == nil && ip != "" {
|
||||
conf2 := FluentbitConfig(ip, lokiPortOf(endpoint), logPath, target, ns)
|
||||
cm2 := FluentbitConfigMapYAML(conf2, ns, configMapName)
|
||||
if err := applyManifest(ctx, cluster, cm2); err == nil {
|
||||
_ = restartDeployment(ctx, cluster, ns, target)
|
||||
d.log("info", "fluent-bit 推送地址已回退为 ClusterIP: "+ip, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -426,23 +347,52 @@ func (d *CollectorDeployer) uninstallSidecar(ctx context.Context) error {
|
||||
if cluster == nil || d.collector.TargetDeployment == "" {
|
||||
return nil
|
||||
}
|
||||
env, cleanup, err := kubeconfigEnv(cluster)
|
||||
client, err := clusterClient(cluster, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ns := d.collector.Namespace
|
||||
if ns == "" {
|
||||
ns = "default"
|
||||
}
|
||||
patchFile, patchCleanup, err := writeTempFile("sidecar-rm-*.yaml", SidecarRemovePatchYAML, 0600)
|
||||
deployment, err := client.AppsV1().Deployments(ns).Get(ctx, d.collector.TargetDeployment, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer patchCleanup()
|
||||
deploymentJSON, err := json.Marshal(deployment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removePatch, err := sidecarRemovePatchYAML(deploymentJSON, d.collector.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := patchDeployment(ctx, cluster, ns, d.collector.TargetDeployment, removePatch); err != nil {
|
||||
return fmt.Errorf("移除 Sidecar 失败: %w", err)
|
||||
}
|
||||
if err := deleteK8sResource(ctx, cluster, "configmap", ns, fmt.Sprintf("fluent-bit-config-%d", d.collector.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
patch := exec.CommandContext(ctx, "kubectl", "patch", "deployment", d.collector.TargetDeployment, "-n", ns, "--type=strategic", "--patch-file", patchFile)
|
||||
patch.Env = env
|
||||
patch.CombinedOutput()
|
||||
return nil
|
||||
}
|
||||
|
||||
// lokiPortOf 从 endpoint 提取端口(默认 3100)
|
||||
func lokiPortOf(endpoint string) string {
|
||||
rest := strings.SplitN(endpoint, "//", 2)
|
||||
if len(rest) != 2 {
|
||||
return "3100"
|
||||
}
|
||||
hostPart := rest[1]
|
||||
if i := strings.Index(hostPart, ":"); i >= 0 {
|
||||
port := hostPart[i+1:]
|
||||
if j := strings.IndexAny(port, "/"); j >= 0 {
|
||||
port = port[:j]
|
||||
}
|
||||
if port != "" {
|
||||
return port
|
||||
}
|
||||
}
|
||||
return "3100"
|
||||
}
|
||||
Reference in new issue
Block a user