fix(reliability): 采集器并发去重 + 备份恢复即时重载 + context 贯穿请求链
- CollectorService 安装/卸载按采集器互斥去重(重复触发不再并发 docker rm/run 互踩);删除后卸载失败留日志 - ApplyStagedBackup 替换文件后立即 Reload 内存态,消除「重启前旧内存落盘覆盖恢复文件」竞态;暂存文件名升级 lokistack.json(兼容旧名) - GetUsage/Validate 全链路 context 贯穿:请求取消即中止 SSH/k8s 查询(异步调用点显式 Background)
This commit is contained in:
1 parent
9daaf7f403
commit
572efc89a4
9 files changed
+180
-29
No files matched your search
@@ -281,7 +281,12 @@ func encryptionKeyFingerprint() string {
|
||||
|
||||
// ApplyStagedBackup 把暂存文件应用为正式 DB(调用方负责重启进程)。
|
||||
func ApplyStagedBackup() (string, error) {
|
||||
staged := filepath.Join(dataDirPath(), ".restore-pending", "opsxloki.json")
|
||||
restoreDir := filepath.Join(dataDirPath(), ".restore-pending")
|
||||
staged := filepath.Join(restoreDir, "lokistack.json")
|
||||
if _, err := os.Stat(staged); err != nil {
|
||||
// 兼容旧版暂存文件名
|
||||
staged = filepath.Join(restoreDir, "opsxloki.json")
|
||||
}
|
||||
b, err := os.ReadFile(staged)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("没有待应用的恢复文件: %w", err)
|
||||
@@ -305,6 +310,16 @@ func ApplyStagedBackup() (string, error) {
|
||||
return "", err
|
||||
}
|
||||
_ = os.Remove(staged)
|
||||
// 关键:立即把内存态切换为恢复后的数据。否则「替换文件 → 重启」之间,
|
||||
// 任何一次防抖落盘都会用旧内存态把恢复文件覆盖回去,恢复静默失效。
|
||||
if db := repository.GetDB(); db != nil {
|
||||
db.Lock()
|
||||
if err := db.Reload(); err != nil {
|
||||
db.Unlock()
|
||||
return dbPath, fmt.Errorf("恢复文件已生效但内存重载失败(%v),请立即重启进程", err)
|
||||
}
|
||||
db.Unlock()
|
||||
}
|
||||
return dbPath, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ func (s *ClusterService) Create(name, description, configType, apiServer, kubeco
|
||||
db.Unlock()
|
||||
db.RequestSave()
|
||||
|
||||
go s.Validate(cluster.ID)
|
||||
go s.Validate(context.Background(), cluster.ID)
|
||||
return &cluster, nil
|
||||
}
|
||||
|
||||
@@ -222,12 +222,12 @@ func (s *ClusterService) Delete(id uint) error {
|
||||
return utils.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *ClusterService) Validate(id uint) {
|
||||
func (s *ClusterService) Validate(ctx context.Context, id uint) {
|
||||
cluster, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
info, err := GetClusterInfo(ctx, cluster)
|
||||
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lokistack/models"
|
||||
@@ -12,10 +14,36 @@ import (
|
||||
"lokistack/utils"
|
||||
)
|
||||
|
||||
type CollectorService struct{}
|
||||
type CollectorService struct {
|
||||
// opsMu 保护 opsInflight:同一采集器的安装/卸载互斥去重。
|
||||
// 采集器是 fire-and-forget(接口立即返回),没有去重时连点「部署」
|
||||
// 会对同一主机并发 docker rm -f + docker run,容器名冲突/配置互踩。
|
||||
opsMu sync.Mutex
|
||||
opsInflight map[uint]bool
|
||||
}
|
||||
|
||||
func NewCollectorService() *CollectorService {
|
||||
return &CollectorService{}
|
||||
return &CollectorService{opsInflight: map[uint]bool{}}
|
||||
}
|
||||
|
||||
// beginOp 声明开始一次安装/卸载;已在进行返回 false(直接忽略重复触发)
|
||||
func (s *CollectorService) beginOp(id uint) bool {
|
||||
s.opsMu.Lock()
|
||||
defer s.opsMu.Unlock()
|
||||
if s.opsInflight == nil {
|
||||
s.opsInflight = map[uint]bool{}
|
||||
}
|
||||
if s.opsInflight[id] {
|
||||
return false
|
||||
}
|
||||
s.opsInflight[id] = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *CollectorService) endOp(id uint) {
|
||||
s.opsMu.Lock()
|
||||
delete(s.opsInflight, id)
|
||||
s.opsMu.Unlock()
|
||||
}
|
||||
|
||||
type CreateCollectorRequest struct {
|
||||
@@ -475,17 +503,23 @@ func (s *CollectorService) Delete(id uint) error {
|
||||
}
|
||||
db.Unlock()
|
||||
db.RequestSave()
|
||||
// 异步卸载(用拷贝的 collector,避免删除后取不到)
|
||||
// 异步卸载(用拷贝的 collector,避免删除后取不到;失败留日志,远端不留孤儿容器也无提示)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx)
|
||||
if err := NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx); err != nil {
|
||||
log.Printf("[collector] 删除后卸载 %q(目标 %s)失败: %v", collector.Name, collector.Name, err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Install 异步部署采集器
|
||||
func (s *CollectorService) Install(id uint) {
|
||||
if !s.beginOp(id) {
|
||||
return // 同一采集器的安装/卸载正在进行,忽略重复触发
|
||||
}
|
||||
defer s.endOp(id)
|
||||
collector, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -522,6 +556,10 @@ func (s *CollectorService) Install(id uint) {
|
||||
|
||||
// Uninstall 异步卸载采集器
|
||||
func (s *CollectorService) Uninstall(id uint) {
|
||||
if !s.beginOp(id) {
|
||||
return
|
||||
}
|
||||
defer s.endOp(id)
|
||||
collector, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -69,18 +69,18 @@ type podMetricsResponse struct {
|
||||
}
|
||||
|
||||
// GetUsage 采集某个 Loki 部署的资源用量(按部署模式分派)。
|
||||
func (s *DeploymentService) GetUsage(id uint) (*ResourceUsage, error) {
|
||||
func (s *DeploymentService) GetUsage(ctx context.Context, id uint) (*ResourceUsage, error) {
|
||||
d, err := s.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch d.Mode {
|
||||
case "k8s-yaml":
|
||||
return s.k8sUsage(d)
|
||||
return s.k8sUsage(ctx, d)
|
||||
case "host-docker":
|
||||
return s.dockerUsage(d)
|
||||
return s.dockerUsage(ctx, d)
|
||||
case "host-binary":
|
||||
return s.processUsage(d)
|
||||
return s.processUsage(ctx, d)
|
||||
default:
|
||||
return nil, fmt.Errorf("不支持的部署模式:%s", d.Mode)
|
||||
}
|
||||
@@ -88,7 +88,7 @@ func (s *DeploymentService) GetUsage(id uint) (*ResourceUsage, error) {
|
||||
|
||||
// ---------- Kubernetes ----------
|
||||
|
||||
func (s *DeploymentService) k8sUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
func (s *DeploymentService) k8sUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Cluster == nil {
|
||||
return nil, fmt.Errorf("部署未关联 Kubernetes 集群")
|
||||
}
|
||||
@@ -96,7 +96,7 @@ func (s *DeploymentService) k8sUsage(d *models.LokiDeployment) (*ResourceUsage,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ns := d.Namespace
|
||||
@@ -192,7 +192,7 @@ func (s *DeploymentService) applyPrometheus(u *ResourceUsage) {
|
||||
|
||||
// ---------- host-docker ----------
|
||||
|
||||
func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
func (s *DeploymentService) dockerUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Host == nil {
|
||||
return nil, fmt.Errorf("部署未关联主机")
|
||||
}
|
||||
@@ -202,7 +202,7 @@ func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsag
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
name := dockerContainerName(d.ID)
|
||||
@@ -254,7 +254,7 @@ func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsag
|
||||
|
||||
// ---------- host-binary ----------
|
||||
|
||||
func (s *DeploymentService) processUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
func (s *DeploymentService) processUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Host == nil {
|
||||
return nil, fmt.Errorf("部署未关联主机")
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func (s *DeploymentService) processUsage(d *models.LokiDeployment) (*ResourceUsa
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
usage := &ResourceUsage{Mode: d.Mode, Source: "binary", Pod: "loki", Node: d.Host.HostIP}
|
||||
|
||||
@@ -88,7 +88,7 @@ func (s *HostService) Create(name, hostIP string, port int, user, authType, pass
|
||||
db.Unlock()
|
||||
db.RequestSave()
|
||||
|
||||
go s.Validate(host.ID)
|
||||
go s.Validate(context.Background(), host.ID)
|
||||
return &host, nil
|
||||
}
|
||||
|
||||
@@ -194,12 +194,12 @@ func (s *HostService) Delete(id uint) error {
|
||||
return utils.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *HostService) Validate(id uint) {
|
||||
func (s *HostService) Validate(ctx context.Context, id uint) {
|
||||
host, err := s.Get(id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := NewSSHClientWithAuth(HostSSHAuth(host))
|
||||
|
||||
Reference in new issue
Block a user