fix(reliability): 采集器并发去重 + 备份恢复即时重载 + context 贯穿请求链

- CollectorService 安装/卸载按采集器互斥去重(重复触发不再并发 docker rm/run 互踩);删除后卸载失败留日志
- ApplyStagedBackup 替换文件后立即 Reload 内存态,消除「重启前旧内存落盘覆盖恢复文件」竞态;暂存文件名升级 lokistack.json(兼容旧名)
- GetUsage/Validate 全链路 context 贯穿:请求取消即中止 SSH/k8s 查询(异步调用点显式 Background)
This commit is contained in:
opsx committed 2026-10-29 13:29:25 +08:00
1 parent 9daaf7f403
commit 572efc89a4
9 files changed
+180 -29

No files matched your search

+16 -1
View File
@@ -281,7 +281,12 @@ func encryptionKeyFingerprint() string {
// ApplyStagedBackup 把暂存文件应用为正式 DB(调用方负责重启进程)。
func ApplyStagedBackup() (string, error) {
staged := filepath.Join(dataDirPath(), ".restore-pending", "opsxloki.json")
restoreDir := filepath.Join(dataDirPath(), ".restore-pending")
staged := filepath.Join(restoreDir, "lokistack.json")
if _, err := os.Stat(staged); err != nil {
// 兼容旧版暂存文件名
staged = filepath.Join(restoreDir, "opsxloki.json")
}
b, err := os.ReadFile(staged)
if err != nil {
return "", fmt.Errorf("没有待应用的恢复文件: %w", err)
@@ -305,6 +310,16 @@ func ApplyStagedBackup() (string, error) {
return "", err
}
_ = os.Remove(staged)
// 关键:立即把内存态切换为恢复后的数据。否则「替换文件 → 重启」之间,
// 任何一次防抖落盘都会用旧内存态把恢复文件覆盖回去,恢复静默失效。
if db := repository.GetDB(); db != nil {
db.Lock()
if err := db.Reload(); err != nil {
db.Unlock()
return dbPath, fmt.Errorf("恢复文件已生效但内存重载失败(%v),请立即重启进程", err)
}
db.Unlock()
}
return dbPath, nil
}
+3 -3
View File
@@ -107,7 +107,7 @@ func (s *ClusterService) Create(name, description, configType, apiServer, kubeco
db.Unlock()
db.RequestSave()
go s.Validate(cluster.ID)
go s.Validate(context.Background(), cluster.ID)
return &cluster, nil
}
@@ -222,12 +222,12 @@ func (s *ClusterService) Delete(id uint) error {
return utils.ErrNotFound
}
func (s *ClusterService) Validate(id uint) {
func (s *ClusterService) Validate(ctx context.Context, id uint) {
cluster, err := s.Get(id)
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
info, err := GetClusterInfo(ctx, cluster)
+42 -4
View File
@@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"sync"
"time"
"lokistack/models"
@@ -12,10 +14,36 @@ import (
"lokistack/utils"
)
type CollectorService struct{}
type CollectorService struct {
// opsMu 保护 opsInflight:同一采集器的安装/卸载互斥去重。
// 采集器是 fire-and-forget(接口立即返回),没有去重时连点「部署」
// 会对同一主机并发 docker rm -f + docker run,容器名冲突/配置互踩。
opsMu sync.Mutex
opsInflight map[uint]bool
}
func NewCollectorService() *CollectorService {
return &CollectorService{}
return &CollectorService{opsInflight: map[uint]bool{}}
}
// beginOp 声明开始一次安装/卸载;已在进行返回 false(直接忽略重复触发)
func (s *CollectorService) beginOp(id uint) bool {
s.opsMu.Lock()
defer s.opsMu.Unlock()
if s.opsInflight == nil {
s.opsInflight = map[uint]bool{}
}
if s.opsInflight[id] {
return false
}
s.opsInflight[id] = true
return true
}
func (s *CollectorService) endOp(id uint) {
s.opsMu.Lock()
delete(s.opsInflight, id)
s.opsMu.Unlock()
}
type CreateCollectorRequest struct {
@@ -475,17 +503,23 @@ func (s *CollectorService) Delete(id uint) error {
}
db.Unlock()
db.RequestSave()
// 异步卸载(用拷贝的 collector,避免删除后取不到)
// 异步卸载(用拷贝的 collector,避免删除后取不到;失败留日志,远端不留孤儿容器也无提示)
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx)
if err := NewCollectorDeployer(collector, collector.LokiDeployment, nil).Uninstall(ctx); err != nil {
log.Printf("[collector] 删除后卸载 %q(目标 %s)失败: %v", collector.Name, collector.Name, err)
}
}()
return nil
}
// Install 异步部署采集器
func (s *CollectorService) Install(id uint) {
if !s.beginOp(id) {
return // 同一采集器的安装/卸载正在进行,忽略重复触发
}
defer s.endOp(id)
collector, err := s.Get(id)
if err != nil {
return
@@ -522,6 +556,10 @@ func (s *CollectorService) Install(id uint) {
// Uninstall 异步卸载采集器
func (s *CollectorService) Uninstall(id uint) {
if !s.beginOp(id) {
return
}
defer s.endOp(id)
collector, err := s.Get(id)
if err != nil {
return
+10 -10
View File
@@ -69,18 +69,18 @@ type podMetricsResponse struct {
}
// GetUsage 采集某个 Loki 部署的资源用量(按部署模式分派)。
func (s *DeploymentService) GetUsage(id uint) (*ResourceUsage, error) {
func (s *DeploymentService) GetUsage(ctx context.Context, id uint) (*ResourceUsage, error) {
d, err := s.Get(id)
if err != nil {
return nil, err
}
switch d.Mode {
case "k8s-yaml":
return s.k8sUsage(d)
return s.k8sUsage(ctx, d)
case "host-docker":
return s.dockerUsage(d)
return s.dockerUsage(ctx, d)
case "host-binary":
return s.processUsage(d)
return s.processUsage(ctx, d)
default:
return nil, fmt.Errorf("不支持的部署模式:%s", d.Mode)
}
@@ -88,7 +88,7 @@ func (s *DeploymentService) GetUsage(id uint) (*ResourceUsage, error) {
// ---------- Kubernetes ----------
func (s *DeploymentService) k8sUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
func (s *DeploymentService) k8sUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
if d.Cluster == nil {
return nil, fmt.Errorf("部署未关联 Kubernetes 集群")
}
@@ -96,7 +96,7 @@ func (s *DeploymentService) k8sUsage(d *models.LokiDeployment) (*ResourceUsage,
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
ns := d.Namespace
@@ -192,7 +192,7 @@ func (s *DeploymentService) applyPrometheus(u *ResourceUsage) {
// ---------- host-docker ----------
func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
func (s *DeploymentService) dockerUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
if d.Host == nil {
return nil, fmt.Errorf("部署未关联主机")
}
@@ -202,7 +202,7 @@ func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsag
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
name := dockerContainerName(d.ID)
@@ -254,7 +254,7 @@ func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsag
// ---------- host-binary ----------
func (s *DeploymentService) processUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
func (s *DeploymentService) processUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
if d.Host == nil {
return nil, fmt.Errorf("部署未关联主机")
}
@@ -264,7 +264,7 @@ func (s *DeploymentService) processUsage(d *models.LokiDeployment) (*ResourceUsa
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
usage := &ResourceUsage{Mode: d.Mode, Source: "binary", Pod: "loki", Node: d.Host.HostIP}
+3 -3
View File
@@ -88,7 +88,7 @@ func (s *HostService) Create(name, hostIP string, port int, user, authType, pass
db.Unlock()
db.RequestSave()
go s.Validate(host.ID)
go s.Validate(context.Background(), host.ID)
return &host, nil
}
@@ -194,12 +194,12 @@ func (s *HostService) Delete(id uint) error {
return utils.ErrNotFound
}
func (s *HostService) Validate(id uint) {
func (s *HostService) Validate(ctx context.Context, id uint) {
host, err := s.Get(id)
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
client, err := NewSSHClientWithAuth(HostSSHAuth(host))