fix(reliability): 采集器并发去重 + 备份恢复即时重载 + context 贯穿请求链
- CollectorService 安装/卸载按采集器互斥去重(重复触发不再并发 docker rm/run 互踩);删除后卸载失败留日志 - ApplyStagedBackup 替换文件后立即 Reload 内存态,消除「重启前旧内存落盘覆盖恢复文件」竞态;暂存文件名升级 lokistack.json(兼容旧名) - GetUsage/Validate 全链路 context 贯穿:请求取消即中止 SSH/k8s 查询(异步调用点显式 Background)
This commit is contained in:
1 parent
9daaf7f403
commit
572efc89a4
9 files changed
+180
-29
No files matched your search
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lokistack/models"
|
||||
@@ -12,10 +14,36 @@ import (
|
||||
"lokistack/utils"
|
||||
)
|
||||
|
||||
type CollectorService struct{}
|
||||
type CollectorService struct {
|
||||
// opsMu 保护 opsInflight:同一采集器的安装/卸载互斥去重。
|
||||
// 采集器是 fire-and-forget(接口立即返回),没有去重时连点「部署」
|
||||
// 会对同一主机并发 docker rm -f + docker run,容器名冲突/配置互踩。
|
||||
opsMu sync.Mutex
|
||||
opsInflight map[uint]bool
|
||||
}
|
||||
|
||||
func NewCollectorService() *CollectorService {
|
||||
return &CollectorService{}
|
||||
return &CollectorService{opsInflight: map[uint]bool{}}
|
||||
}
|
||||
|
||||
// beginOp 声明开始一次安装/卸载;已在进行返回 false(直接忽略重复触发)
|
||||
func (s *CollectorService) beginOp(id uint) bool {
|
||||
s.opsMu.Lock()
|
||||
defer s.opsMu.Unlock()
|
||||
if s.opsInflight == nil {
|
||||
s.opsInflight = map[uint]bool{}
|
||||
}
|
||||
if s.opsInflight[id] {
|
||||
return false
|
||||
}
|
||||
s.opsInflight[id] = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *CollectorService) endOp(id uint) {
|
||||
s.opsMu.Lock()
|
||||
delete(s.opsInflight, id)
|
||||
s.opsMu.Unlock()
|
||||
}
|
||||
|
||||
type CreateCollectorRequest struct {
|
||||
@@ -475,17 +503,23 @@ func (s *CollectorService) Delete(id uint) error {
|
||||
}
|
||||
db.Unlock()
|
||||
db.RequestSave()
|
||||
// 异步卸载(用拷贝的 collector,避免删除后取不到)
|
||||
// 异步卸载(用拷贝的 collector,避免删除后取不到;失败留日志,远端不留孤儿容器也无提示)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx)
|
||||
if err := NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx); err != nil {
|
||||
log.Printf("[collector] 删除后卸载 %q(目标 %s)失败: %v", collector.Name, collector.Name, err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Install 异步部署采集器
|
||||
func (s *CollectorService) Install(id uint) {
|
||||
if !s.beginOp(id) {
|
||||
return // 同一采集器的安装/卸载正在进行,忽略重复触发
|
||||
}
|
||||
defer s.endOp(id)
|
||||
collector, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -522,6 +556,10 @@ func (s *CollectorService) Install(id uint) {
|
||||
|
||||
// Uninstall 异步卸载采集器
|
||||
func (s *CollectorService) Uninstall(id uint) {
|
||||
if !s.beginOp(id) {
|
||||
return
|
||||
}
|
||||
defer s.endOp(id)
|
||||
collector, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user