fix(reliability): 采集器并发去重 + 备份恢复即时重载 + context 贯穿请求链

- CollectorService 安装/卸载按采集器互斥去重(重复触发不再并发 docker rm/run 互踩);删除后卸载失败留日志
- ApplyStagedBackup 替换文件后立即 Reload 内存态,消除「重启前旧内存落盘覆盖恢复文件」竞态;暂存文件名升级 lokistack.json(兼容旧名)
- GetUsage/Validate 全链路 context 贯穿:请求取消即中止 SSH/k8s 查询(异步调用点显式 Background)
This commit is contained in:
opsx committed 2026-10-29 13:29:25 +08:00
1 parent 9daaf7f403
commit 572efc89a4
9 files changed
+180 -29

No files matched your search

+42 -4
View File
@@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"sync"
"time"
"lokistack/models"
@@ -12,10 +14,36 @@ import (
"lokistack/utils"
)
type CollectorService struct{}
type CollectorService struct {
// opsMu 保护 opsInflight:同一采集器的安装/卸载互斥去重。
// 采集器是 fire-and-forget(接口立即返回),没有去重时连点「部署」
// 会对同一主机并发 docker rm -f + docker run,容器名冲突/配置互踩。
opsMu sync.Mutex
opsInflight map[uint]bool
}
func NewCollectorService() *CollectorService {
return &CollectorService{}
return &CollectorService{opsInflight: map[uint]bool{}}
}
// beginOp 声明开始一次安装/卸载;已在进行返回 false(直接忽略重复触发)
func (s *CollectorService) beginOp(id uint) bool {
s.opsMu.Lock()
defer s.opsMu.Unlock()
if s.opsInflight == nil {
s.opsInflight = map[uint]bool{}
}
if s.opsInflight[id] {
return false
}
s.opsInflight[id] = true
return true
}
func (s *CollectorService) endOp(id uint) {
s.opsMu.Lock()
delete(s.opsInflight, id)
s.opsMu.Unlock()
}
type CreateCollectorRequest struct {
@@ -475,17 +503,23 @@ func (s *CollectorService) Delete(id uint) error {
}
db.Unlock()
db.RequestSave()
// 异步卸载(用拷贝的 collector,避免删除后取不到)
// 异步卸载(用拷贝的 collector,避免删除后取不到;失败留日志,远端不留孤儿容器也无提示)
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx)
if err := NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx); err != nil {
log.Printf("[collector] 删除后卸载 %q(目标 %s)失败: %v", collector.Name, collector.Name, err)
}
}()
return nil
}
// Install 异步部署采集器
func (s *CollectorService) Install(id uint) {
if !s.beginOp(id) {
return // 同一采集器的安装/卸载正在进行,忽略重复触发
}
defer s.endOp(id)
collector, err := s.Get(id)
if err != nil {
return
@@ -522,6 +556,10 @@ func (s *CollectorService) Install(id uint) {
// Uninstall 异步卸载采集器
func (s *CollectorService) Uninstall(id uint) {
if !s.beginOp(id) {
return
}
defer s.endOp(id)
collector, err := s.Get(id)
if err != nil {
return