Files
LokiStack/backend/services/deployment.go
T
opsx bad61e5027 fix(deploy): 删除部署时同步清理自动注册的查询数据源(孤儿数据源)
=== 现象(用户实测)===
在 /deployments 页面删掉两个主机模式部署后,本机(172.19.16.8)容器确实清干净了:
  docker ps -a 无任何 lokistack-* 容器
  /etc/promtail、/var/lib/promtail 等目录均不存在
  无 promtail systemd 单元
但**平台里残留一个数据源** host-verify-loki(url=http://172.19.16.8:3100),
「日志查询」下拉里还能选到它,选中必然连接失败。

=== 根因 ===
DeploymentService.Delete 只做了两件事:删容器、删部署记录。
部署创建时由 EnsureLogSourceForDeployment 自动注册的托管数据源(Managed=true)
没有任何清理逻辑 —— 删除动作只完成了一半。

=== 修法 ===
1) DeploymentService.Delete 删完记录后调用 CleanupManagedLogSource(部署名)
   - 删掉同名且 Managed=true 的数据源(手填的 Managed=false 不动)
   - DeleteLogSource 内部会连带删除指向它的查询授权策略组
   - 绑定该源的告警规则置为禁用(保留配置便于审计,但不让它每次评估都失败)
   - 注意锁顺序:必须先 db.Unlock() 再调用,CleanupManagedLogSource 内部会自己加锁,
     持有锁再调用会死锁
2) schemaVersion v6→v7 迁移 removeOrphanManagedLogSources()
   清理历史遗留:Managed=true 但没有任何同名部署的数据源 → 删除,
   连带删策略组、禁用告警规则

=== 实测(d9487694ece6)===
  v7 迁移日志:移除孤儿数据源 "host-verify-loki"(id=2):对应部署已删除
  数据源:1 个(只剩 loki-pre,managed=true)
  部署:1 / 采集器:1 / 策略组:1
  本机:docker ps -a 无 lokistack-* 容器,/etc/promtail 等目录不存在,无 systemd 单元
2026-10-28 23:18:43 +00:00

755 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package services
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"sync"
"time"
"lokistack/models"
"lokistack/repository"
"lokistack/seed"
"lokistack/utils"
)
// LokiVersionList 支持的 Loki 版本列表(按推荐程度排序)
var LokiVersionList = []string{
"3.4.0",
"3.3.0",
"3.2.0",
"3.1.0",
"3.0.0",
"2.9.8",
"2.9.0",
"2.8.9",
}
// minSupportedMajorVersion 最低支持的主版本号
const minSupportedMajorVersion = 2
const DefaultRetentionDays = 30
// DeploymentService 部署业务逻辑层
type DeploymentService struct{}
func NewDeploymentService() *DeploymentService {
return &DeploymentService{}
}
type CreateDeploymentRequest struct {
Name string `json:"name" binding:"required"`
Description string `json:"description"`
Version string `json:"version"`
Mode string `json:"mode" binding:"required"`
ClusterID *uint `json:"clusterID"`
HostID *uint `json:"hostID"`
ListenIP string `json:"listenIP"`
ListenPort int `json:"listenPort"`
Namespace string `json:"namespace"`
StorageConfigID uint `json:"storageConfigID"`
AuthEnabled bool `json:"authEnabled"`
RetentionDays int `json:"retentionDays"`
Replicas int `json:"replicas"`
ResourcesCPU string `json:"resourcesCPU"`
ResourcesMem string `json:"resourcesMem"`
StorageSize string `json:"storageSize"`
GrafanaEnabled bool `json:"grafanaEnabled"`
GrafanaVersion string `json:"grafanaVersion"`
}
func (s *DeploymentService) Create(req *CreateDeploymentRequest) (*models.LokiDeployment, error) {
if req.Mode != "k8s-yaml" && req.Mode != "host-docker" && req.Mode != "host-binary" {
return nil, fmt.Errorf("不支持的部署方式: %s", req.Mode)
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("Grafana 自动部署已移除,请使用独立 Grafana 服务")
}
if req.Mode == "host-docker" {
if _, err := dockerLokiImage(req.Version); err != nil {
return nil, err
}
if req.HostID == nil || *req.HostID == 0 {
return nil, fmt.Errorf("请选择目标主机")
}
if req.ListenPort == 0 {
req.ListenPort = 3100
}
if req.ListenPort < 1 || req.ListenPort > 65535 {
return nil, fmt.Errorf("监听端口须在 1 到 65535 之间")
}
if req.Replicas > 1 {
return nil, fmt.Errorf("主机 Docker 部署仅支持单副本")
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("主机 Docker 模式暂不支持同时部署 Grafana")
}
if req.ListenIP != "" && net.ParseIP(req.ListenIP).To4() == nil {
return nil, fmt.Errorf("监听地址须为 IPv4 地址")
}
}
if req.RetentionDays < 0 {
return nil, fmt.Errorf("保留天数必须是正整数")
}
if req.RetentionDays == 0 {
req.RetentionDays = DefaultRetentionDays
}
// 命名空间固定为 log:平台自身(lokistack)、Loki 实例(loki)、Promtail
// 全部收敛在同一个 log 命名空间,未显式传值时一律兜底为 log。
if strings.TrimSpace(req.Namespace) == "" && req.Mode == "k8s-yaml" {
req.Namespace = "log"
}
// 版本兼容性预检查
if req.Version != "" {
if err := ValidateLokiVersion(req.Version); err != nil {
return nil, err
}
}
db := repository.GetDB()
d := models.LokiDeployment{
ID: db.NextID("deployments"),
Name: req.Name,
Description: req.Description,
Version: req.Version,
Mode: req.Mode,
ClusterID: req.ClusterID,
HostID: req.HostID,
ListenIP: req.ListenIP,
ListenPort: req.ListenPort,
Namespace: req.Namespace,
StorageConfigID: req.StorageConfigID,
AuthEnabled: req.AuthEnabled,
RetentionDays: req.RetentionDays,
Replicas: req.Replicas,
ResourcesCPU: req.ResourcesCPU,
ResourcesMem: req.ResourcesMem,
StorageSize: req.StorageSize,
GrafanaEnabled: req.GrafanaEnabled,
GrafanaVersion: req.GrafanaVersion,
Status: "pending",
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
d.BeforeCreate()
// Load relations
if d.ClusterID != nil && *d.ClusterID > 0 {
for i := range db.Clusters {
if db.Clusters[i].ID == *d.ClusterID {
c := db.Clusters[i]
d.Cluster = &c
break
}
}
}
if d.HostID != nil && *d.HostID > 0 {
for i := range db.Hosts {
if db.Hosts[i].ID == *d.HostID {
h := db.Hosts[i]
d.Host = &h
break
}
}
}
if d.Mode == "host-docker" {
if d.Host == nil {
return nil, fmt.Errorf("目标主机不存在")
}
if d.ListenIP == "" {
d.ListenIP = d.Host.HostIP
}
if net.ParseIP(d.ListenIP).To4() == nil {
return nil, fmt.Errorf("主机 IP 格式无效,请填写监听内网 IPv4 地址")
}
}
if d.StorageConfigID > 0 {
for i := range db.Storages {
if db.Storages[i].ID == d.StorageConfigID {
st := db.Storages[i]
// 模型内为 ENC: 密文,生成 Loki 配置前解密(否则密文进 ConfigMap 且无法通过对象存储认证)
if ak, err := DecryptCred(st.AccessKey); err == nil && ak != "" {
st.AccessKey = ak
}
if sk, err := DecryptCred(st.SecretKey); err == nil && sk != "" {
st.SecretKey = sk
}
d.StorageConfig = &st
break
}
}
}
db.Lock()
db.Deployments = append(db.Deployments, d)
db.Unlock()
db.RequestSave()
return &d, nil
}
func (s *DeploymentService) Get(id uint) (*models.LokiDeployment, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
for i := range db.Deployments {
if db.Deployments[i].ID == id {
d := db.Deployments[i]
s.loadRelations(&d)
return &d, nil
}
}
return nil, utils.ErrNotFound
}
func (s *DeploymentService) List(page, pageSize int) ([]models.LokiDeployment, int64, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
total := int64(len(db.Deployments))
start := (page - 1) * pageSize
if start < 0 || start >= len(db.Deployments) {
return []models.LokiDeployment{}, total, nil
}
end := start + pageSize
if end > len(db.Deployments) {
end = len(db.Deployments)
}
result := make([]models.LokiDeployment, end-start)
copy(result, db.Deployments[start:end])
for i := range result {
s.loadRelations(&result[i])
}
return result, total, nil
}
func (s *DeploymentService) Update(id uint, updates map[string]interface{}) error {
if value, exists := updates["retentionDays"]; exists {
days, ok := value.(float64)
if !ok || days < 1 || days != float64(int(days)) {
return fmt.Errorf("保留天数必须是正整数")
}
}
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.Deployments {
if db.Deployments[i].ID == id {
if v, ok := updates["status"].(string); ok {
db.Deployments[i].Status = v
}
if v, ok := updates["statusMessage"].(string); ok {
db.Deployments[i].StatusMessage = v
}
if v, ok := updates["endpoint"].(string); ok {
db.Deployments[i].Endpoint = v
}
// 业务字段(来自 PUT /deployments/:id;JSON 数字解码为 float64)
if v, ok := updates["name"].(string); ok && v != "" {
db.Deployments[i].Name = v
}
if v, ok := updates["description"].(string); ok {
db.Deployments[i].Description = v
}
if v, ok := updates["version"].(string); ok && v != "" {
db.Deployments[i].Version = v
}
if v, ok := updates["namespace"].(string); ok && v != "" {
db.Deployments[i].Namespace = v
}
if v, ok := updates["resourcesCPU"].(string); ok {
db.Deployments[i].ResourcesCPU = v
}
if v, ok := updates["resourcesMem"].(string); ok {
db.Deployments[i].ResourcesMem = v
}
if v, ok := updates["storageSize"].(string); ok && v != "" {
db.Deployments[i].StorageSize = v
}
if v, ok := updates["authEnabled"].(bool); ok {
db.Deployments[i].AuthEnabled = v
}
if v, ok := updates["grafanaEnabled"].(bool); ok {
db.Deployments[i].GrafanaEnabled = v
}
if v, ok := updates["grafanaVersion"].(string); ok && v != "" {
db.Deployments[i].GrafanaVersion = v
}
if v, ok := updates["grafanaEndpoint"].(string); ok {
db.Deployments[i].GrafanaEndpoint = v
}
if v, ok := updates["storageConfigID"].(float64); ok && v > 0 {
db.Deployments[i].StorageConfigID = uint(v)
}
if v, ok := updates["clusterID"].(float64); ok && v > 0 {
id := uint(v)
db.Deployments[i].ClusterID = &id
}
if v, ok := updates["hostID"].(float64); ok && v > 0 {
id := uint(v)
db.Deployments[i].HostID = &id
}
if v, ok := updates["retentionDays"].(float64); ok && v > 0 {
db.Deployments[i].RetentionDays = int(v)
}
if v, ok := updates["replicas"].(float64); ok && v > 0 {
db.Deployments[i].Replicas = int(v)
}
db.Deployments[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
}
}
return utils.ErrNotFound
}
func (s *DeploymentService) Delete(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Mode == "host-docker" {
if deployment.Status == "deploying" {
return fmt.Errorf("部署正在进行中,请等待完成后删除")
}
if deployment.Status != "pending" {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if err := deleteDockerLoki(ctx, deployment); err != nil {
return err
}
}
}
db := repository.GetDB()
db.Lock()
removedName := ""
for i := range db.Deployments {
if db.Deployments[i].ID == id {
removedName = db.Deployments[i].Name
db.Deployments = append(db.Deployments[:i], db.Deployments[i+1:]...)
db.RequestSave()
break
}
}
// 必须先解锁再清理数据源:CleanupManagedLogSource 内部会自己加锁,
// 持有 db.Lock() 时再调用会死锁。
db.Unlock()
if removedName == "" {
return utils.ErrNotFound
}
// 同步清理该部署自动注册的查询数据源。
// 之前漏了这一步:删完部署后「日志查询」里还留着一个指向已删实例的数据源,
// 选中它必然连接失败(线上就残留过 host-verify-loki)。
CleanupManagedLogSource(removedName)
return nil
}
// ==================== Goroutine 管理改进 ====================
// deployCtxEntry 保存正在进行的部署上下文
type deployCtxEntry struct {
cancel context.CancelFunc
done chan struct{}
}
var (
deployCtxMap = make(map[uint]*deployCtxEntry)
deployCtxMu sync.Mutex
)
func (s *DeploymentService) StartDeploy(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Status == "deploying" {
return fmt.Errorf("部署正在进行中,请勿重复操作")
}
// 检查已有 goroutine 状态,防止重复部署
deployCtxMu.Lock()
if entry, exists := deployCtxMap[id]; exists {
select {
case <-entry.done:
// 已完成,可以重新部署
delete(deployCtxMap, id)
default:
deployCtxMu.Unlock()
return fmt.Errorf("部署正在进行中,请勿重复操作")
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
done := make(chan struct{})
deployCtxMap[id] = &deployCtxEntry{cancel: cancel, done: done}
deployCtxMu.Unlock()
s.Update(id, map[string]interface{}{"status": "deploying"})
logFunc := func(level, message, detail string) {
s.AddLog(id, level, message, detail)
}
go func() {
defer close(done)
defer func() {
deployCtxMu.Lock()
delete(deployCtxMap, id)
deployCtxMu.Unlock()
}()
deployer := NewLokiDeployer(deployment, logFunc)
if err := deployer.Deploy(ctx); err != nil {
s.Update(id, map[string]interface{}{
"status": "failed",
"statusMessage": fmt.Sprintf("Deployment failed: %v", err),
})
s.AddLog(id, "error", fmt.Sprintf("Deployment failed: %v", err), "")
} else {
s.Update(id, map[string]interface{}{
"status": "running",
"statusMessage": "Deployment completed successfully",
"endpoint": s.getEndpoint(deployment),
})
// 部署成功自动注册日志查询数据源 + 记录 Grafana 访问地址 + 补内置告警模板
if fresh, gerr := s.Get(id); gerr == nil {
EnsureLogSourceForDeployment(fresh)
go s.recordGrafanaEndpoint(fresh)
go seed.SeedAlertRules()
}
s.AddLog(id, "success", "Deployment completed successfully", "")
}
}()
return nil
}
func (s *DeploymentService) StopDeployment(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Status != "running" && deployment.Status != "deploying" {
return fmt.Errorf("部署未在运行,无法停止")
}
// 取消正在进行的部署 goroutine
deployCtxMu.Lock()
if entry, exists := deployCtxMap[id]; exists {
entry.cancel()
}
deployCtxMu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if deployment.Mode == "host-docker" {
if err := stopDockerLoki(ctx, deployment); err != nil {
return err
}
} else {
CleanupLoki(ctx, deployment)
}
return s.Update(id, map[string]interface{}{
"status": "stopped",
"statusMessage": "Deployment stopped by user",
})
}
func (s *DeploymentService) RestartDeployment(id uint) error {
if err := s.StopDeployment(id); err != nil {
// 如果停止失败但不是因为部署未运行,返回错误
if !strings.Contains(err.Error(), "部署未在运行") {
return err
}
}
time.Sleep(2 * time.Second)
return s.StartDeploy(id)
}
func (s *DeploymentService) AddLog(deploymentID uint, level, message, detail string) {
db := repository.GetDB()
log := models.DeploymentLog{
ID: db.NextID("logs"),
DeploymentID: deploymentID,
Level: level,
Message: message,
Detail: detail,
CreatedAt: time.Now(),
}
db.Lock()
db.Logs = append(db.Logs, log)
db.Unlock()
db.RequestSave()
}
func (s *DeploymentService) GetLogs(deploymentID uint, page, pageSize int) ([]models.DeploymentLog, int64, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
var filtered []models.DeploymentLog
for i := len(db.Logs) - 1; i >= 0; i-- {
if db.Logs[i].DeploymentID == deploymentID {
filtered = append(filtered, db.Logs[i])
}
}
total := int64(len(filtered))
start := (page - 1) * pageSize
if start >= len(filtered) {
return []models.DeploymentLog{}, total, nil
}
end := start + pageSize
if end > len(filtered) {
end = len(filtered)
}
return filtered[start:end], total, nil
}
func (s *DeploymentService) GetStats() (map[string]interface{}, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
var running, failed, pending, stopped int64
for _, d := range db.Deployments {
switch d.Status {
case "running":
running++
case "failed":
failed++
case "pending":
pending++
case "stopped":
stopped++
}
}
return map[string]interface{}{
"totalDeployments": int64(len(db.Deployments)),
"running": running,
"failed": failed,
"pending": pending,
"stopped": stopped,
"clusterCount": int64(len(db.Clusters)),
"hostCount": int64(len(db.Hosts)),
"supportedVersions": LokiVersionList,
}, nil
}
func (s *DeploymentService) loadRelations(d *models.LokiDeployment) {
db := repository.GetDB()
if d.ClusterID != nil && *d.ClusterID > 0 {
for i := range db.Clusters {
if db.Clusters[i].ID == *d.ClusterID {
c := db.Clusters[i]
d.Cluster = &c
break
}
}
}
if d.HostID != nil && *d.HostID > 0 {
for i := range db.Hosts {
if db.Hosts[i].ID == *d.HostID {
h := db.Hosts[i]
d.Host = &h
break
}
}
}
if d.StorageConfigID > 0 {
for i := range db.Storages {
if db.Storages[i].ID == d.StorageConfigID {
st := db.Storages[i]
// 模型内为 ENC: 密文,生成 Loki 配置前解密(否则密文进 ConfigMap 且无法通过对象存储认证)
if ak, err := DecryptCred(st.AccessKey); err == nil && ak != "" {
st.AccessKey = ak
}
if sk, err := DecryptCred(st.SecretKey); err == nil && sk != "" {
st.SecretKey = sk
}
d.StorageConfig = &st
break
}
}
}
}
func (s *DeploymentService) getEndpoint(d *models.LokiDeployment) string {
switch d.Mode {
case "k8s-yaml":
ns := d.Namespace
if ns == "" {
ns = "log"
}
// k8s-yaml manifest 固定创建名为 loki 的 Service。
// 注意:不使用 loki-gateway,gateway 仅在 Distributed+gateway 启用时存在。
return fmt.Sprintf("http://loki.%s.svc.cluster.local:3100", ns)
case "host-binary":
if d.Host != nil {
return fmt.Sprintf("http://%s:3100", d.Host.HostIP)
}
case "host-docker":
return fmt.Sprintf("http://%s:%d", d.ListenIP, d.ListenPort)
}
return ""
}
// ==================== 版本兼容性检查 ====================
// ValidateLokiVersion 检查 Loki 版本是否在支持列表中且满足最低版本要求
func ValidateLokiVersion(version string) error {
if version == "" {
return nil // 使用默认值,不报错
}
// 基本格式验证
parts := strings.Split(version, ".")
if len(parts) < 2 {
return fmt.Errorf("版本号格式无效: %s,期望格式如 3.0.0", version)
}
major, err := strconv.Atoi(parts[0])
if err != nil {
return fmt.Errorf("无法解析主版本号: %s", version)
}
minor, err := strconv.Atoi(parts[1])
if err != nil {
return fmt.Errorf("无法解析次版本号: %s", version)
}
// 最低版本检查
if major < minSupportedMajorVersion {
return fmt.Errorf("Loki %s 版本过低,最低支持 %d.x 版本", version, minSupportedMajorVersion)
}
if major == 2 && minor < 8 {
return fmt.Errorf("Loki %s 版本过低,2.x 最低支持 2.8.x 版本", version)
}
if major >= 4 {
return fmt.Errorf("Loki %s 是未发布的版本,请使用已知稳定版本", version)
}
return nil
}
// GetRecommendedVersions 返回推荐的 Loki 版本列表
func GetRecommendedVersions() []string {
result := make([]string, len(LokiVersionList))
copy(result, LokiVersionList)
return result
}
// Preflight 向导预检:聚合各资源真实状态,前端据此推导步骤完成态(状态推导式向导)
func (s *DeploymentService) Preflight() map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
result := map[string]interface{}{
"hasStorage": len(db.Storages) > 0,
"hasActiveCluster": false,
"clusterNames": []string{},
"hasActiveHost": false,
"hostNames": []string{},
"runningLoki": nil,
"activeCollectors": 0,
"grafanaReady": false,
}
var defaultStorage string
for i := range db.Storages {
if db.Storages[i].IsDefault {
defaultStorage = db.Storages[i].Name
break
}
}
if defaultStorage == "" && len(db.Storages) > 0 {
defaultStorage = db.Storages[0].Name
}
result["defaultStorage"] = defaultStorage
clusters := []string{}
for i := range db.Clusters {
clusters = append(clusters, db.Clusters[i].Name)
if db.Clusters[i].Status == "active" || db.Clusters[i].Status == "running" {
result["hasActiveCluster"] = true
}
}
result["clusterNames"] = clusters
hosts := []string{}
for i := range db.Hosts {
hosts = append(hosts, db.Hosts[i].Name)
if db.Hosts[i].Status == "active" {
result["hasActiveHost"] = true
}
}
result["hostNames"] = hosts
for i := range db.Deployments {
d := &db.Deployments[i]
if d.Status == "running" {
loki := map[string]interface{}{
"id": d.ID,
"name": d.Name,
"mode": d.Mode,
"endpoint": d.Endpoint,
"namespace": d.Namespace,
"grafana": d.GrafanaEnabled,
"grafanaEndpoint": d.GrafanaEndpoint,
}
result["runningLoki"] = loki
if d.GrafanaEnabled {
result["grafanaReady"] = true
}
break
}
}
for i := range db.Collectors {
if db.Collectors[i].Status == "active" {
result["activeCollectors"] = result["activeCollectors"].(int) + 1
}
}
return result
}
// recordGrafanaEndpoint 记录 Grafana 可点击访问地址(k8s: 首节点 IP + NodePort;host: 主机 IP:3000),
// 供部署卡片/向导直接展示,用户不用翻日志找端口。
func (s *DeploymentService) recordGrafanaEndpoint(d *models.LokiDeployment) {
if d == nil || !d.GrafanaEnabled {
return
}
var endpoint string
switch d.Mode {
case "k8s-yaml":
if d.Cluster == nil {
s.AddLog(d.ID, "warn", "Grafana 地址记录跳过:未关联集群", "")
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ns := d.Namespace
if ns == "" {
ns = "log"
}
ip, ipErr := firstNodeIP(ctx, d.Cluster)
port, portErr := serviceNodePort(ctx, d.Cluster, ns, "grafana")
if ipErr == nil && portErr == nil && port > 0 {
endpoint = fmt.Sprintf("http://%s:%d", ip, port)
}
case "host-binary":
if d.Host != nil {
endpoint = fmt.Sprintf("http://%s:3000", d.Host.HostIP)
}
}
if endpoint != "" {
_ = s.Update(d.ID, map[string]interface{}{"grafanaEndpoint": endpoint})
}
}