feat: opsx-loki 全面升级 — 前端重构/用户与LDAP/部署向导/日志查询
基于 WEB-UI-Development-Guide 标准的整体升级,并更名 LokiDeploy → opsx-loki。 前端(Gin+React+shadcn/ui): - 按 02/07 章重构:页面域化拆分(无超 400 行文件)、PageContainer/ListQuery、 反馈分级(成功静默/失败 toast/危险确认)、401 单飞刷新+队列重放 - 暗色模式(next-themes)+ 语义 Token(--primary 品牌色/success/warning/info) - 部署向导(状态推导四步:环境→存储→Loki→采集,内嵌表单闭环) - 日志查询页:LogQL 标签构建器/拖拽直方图/日志上下文/Live Tail/查询历史 - 按用户+标签策略的查询授权(服务端强制合并策略) 后端(Go+Gin+JSON DB): - 简单用户管理(仅 admin,无 RBAC)+ TokenVersion 会话吊销 + 登录 IP 限流 - LDAP 接入:配置(AES 加密存储/hasPassword 脱敏)/测试连接/批量导入/登录 - 存储配置测试连接(SigV4 零依赖)+ 桶不存在自动创建 - Loki 三模式部署修复:SSH 密钥认证、3.x 配置模板、ruler 对象存储嵌套块、 凭证解密、孤儿 deploying 状态恢复、Grafana 部署(可选默认关)+面板导入 - DaemonSet 采集纯 YAML(去 helm 依赖)+ DNS 自愈回退 ClusterIP + 集群名标签 - 日志查询代理:cluster-svc 经 kube-apiserver proxy/url 直连 + cursor 翻页 - 统一 log 命名空间;部署就绪探活;Grafana 访问地址自动记录 修复:run.sh 增量编译检测、Sanitized 脱敏补全、LokiDeployment Update 白名单、 健康检查 CreatedAt 零值等 10+ 存量 bug。 数据兼容:DB 沿用旧 lokideploy.json(新名优先);localStorage 一次性迁移; HKDF 加密前缀保留(已加密凭证不受影响)。
This commit is contained in:
logan committed
1 parent
327e282356
commit
f9c6c1d826
204 files changed
+14107
-9103
No files matched your search
+123
-30
@@ -8,8 +8,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lokideploy/config"
|
||||
"lokideploy/models"
|
||||
"opsxloki/config"
|
||||
"opsxloki/models"
|
||||
)
|
||||
|
||||
// saveRequest 表示一个保存请求，通过channel序列化
|
||||
@@ -20,17 +20,19 @@ type saveRequest struct {
|
||||
// JsonDB 使用基于channel的序列化保存机制，防止并发数据丢失。
|
||||
// 该包只负责持久化（落盘 + 并发安全），领域实体定义在 models 包。
|
||||
type JsonDB struct {
|
||||
mu sync.RWMutex
|
||||
dataFile string
|
||||
Users []models.User `json:"users"`
|
||||
Clusters []models.Cluster `json:"clusters"`
|
||||
Hosts []models.Host `json:"hosts"`
|
||||
Storages []models.StorageConfig `json:"storages"`
|
||||
Deployments []models.LokiDeployment `json:"deployments"`
|
||||
Collectors []models.Collector `json:"collectors"`
|
||||
Logs []models.DeploymentLog `json:"logs"`
|
||||
Settings []models.SystemSetting `json:"settings"`
|
||||
sequences map[string]uint
|
||||
mu sync.RWMutex
|
||||
dataFile string
|
||||
Users []models.User `json:"users"`
|
||||
Clusters []models.Cluster `json:"clusters"`
|
||||
Hosts []models.Host `json:"hosts"`
|
||||
Storages []models.StorageConfig `json:"storages"`
|
||||
Deployments []models.LokiDeployment `json:"deployments"`
|
||||
Collectors []models.Collector `json:"collectors"`
|
||||
Logs []models.DeploymentLog `json:"logs"`
|
||||
Settings []models.SystemSetting `json:"settings"`
|
||||
LogSources []models.LogSource `json:"logSources"`
|
||||
LogPolicyGroups []models.LogPolicyGroup `json:"logPolicyGroups"`
|
||||
sequences map[string]uint
|
||||
|
||||
// 防抖保存机制
|
||||
saveTimer *time.Timer
|
||||
@@ -51,23 +53,26 @@ func InitDB() error {
|
||||
_ = os.MkdirAll(cfg.DataDir, 0755)
|
||||
|
||||
d := &JsonDB{
|
||||
dataFile: cfg.DatabasePath,
|
||||
Users: []models.User{},
|
||||
Clusters: []models.Cluster{},
|
||||
Hosts: []models.Host{},
|
||||
Storages: []models.StorageConfig{},
|
||||
Deployments: []models.LokiDeployment{},
|
||||
Collectors: []models.Collector{},
|
||||
Logs: []models.DeploymentLog{},
|
||||
Settings: []models.SystemSetting{},
|
||||
dataFile: cfg.DatabasePath,
|
||||
Users: []models.User{},
|
||||
Clusters: []models.Cluster{},
|
||||
Hosts: []models.Host{},
|
||||
Storages: []models.StorageConfig{},
|
||||
Deployments: []models.LokiDeployment{},
|
||||
Collectors: []models.Collector{},
|
||||
Logs: []models.DeploymentLog{},
|
||||
Settings: []models.SystemSetting{},
|
||||
LogSources: []models.LogSource{},
|
||||
LogPolicyGroups: []models.LogPolicyGroup{},
|
||||
sequences: map[string]uint{
|
||||
"users": 0,
|
||||
"clusters": 0,
|
||||
"hosts": 0,
|
||||
"storages": 0,
|
||||
"deployments": 0,
|
||||
"collectors": 0,
|
||||
"logs": 0,
|
||||
"users": 0,
|
||||
"clusters": 0,
|
||||
"hosts": 0,
|
||||
"storages": 0,
|
||||
"deployments": 0,
|
||||
"collectors": 0,
|
||||
"logs": 0,
|
||||
"logPolicyGroups": 0,
|
||||
},
|
||||
saveChan: make(chan saveRequest, 10),
|
||||
stopChan: make(chan struct{}),
|
||||
@@ -111,17 +116,81 @@ func InitDB() error {
|
||||
d.sequences["logs"] = l.ID
|
||||
}
|
||||
}
|
||||
for _, group := range d.LogPolicyGroups {
|
||||
if group.ID > d.sequences["logPolicyGroups"] {
|
||||
d.sequences["logPolicyGroups"] = group.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
migratedLogAccess := migrateLogAccessToGroups(d)
|
||||
// 启动序列化保存goroutine
|
||||
d.wg.Add(1)
|
||||
go d.saveWorker()
|
||||
|
||||
db = d
|
||||
if migratedLogAccess {
|
||||
d.RequestSave()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Legacy per-user grants are grouped by identical source and label scope.
|
||||
func migrateLogAccessToGroups(d *JsonDB) bool {
|
||||
changed := false
|
||||
for i := range d.LogSources {
|
||||
source := &d.LogSources[i]
|
||||
if len(source.Access) == 0 {
|
||||
continue
|
||||
}
|
||||
byLabels := map[string]*models.LogPolicyGroup{}
|
||||
for _, rule := range source.Access {
|
||||
validUser := false
|
||||
for _, user := range d.Users {
|
||||
if user.ID == rule.UserID && user.Role != "admin" {
|
||||
validUser = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !validUser {
|
||||
continue
|
||||
}
|
||||
alreadyGrouped := false
|
||||
for _, group := range d.LogPolicyGroups {
|
||||
if group.SourceID != source.ID {
|
||||
continue
|
||||
}
|
||||
for _, id := range group.UserIDs {
|
||||
if id == rule.UserID {
|
||||
alreadyGrouped = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if alreadyGrouped {
|
||||
continue
|
||||
}
|
||||
keyBytes, _ := json.Marshal(rule.Labels)
|
||||
key := string(keyBytes)
|
||||
group := byLabels[key]
|
||||
if group == nil {
|
||||
d.sequences["logPolicyGroups"]++
|
||||
id := d.sequences["logPolicyGroups"]
|
||||
group = &models.LogPolicyGroup{ID: id, Name: fmt.Sprintf("旧授权 · %s · %d", source.Name, id), SourceID: source.ID, Labels: rule.Labels, CreatedAt: time.Now(), UpdatedAt: time.Now()}
|
||||
byLabels[key] = group
|
||||
}
|
||||
group.UserIDs = append(group.UserIDs, rule.UserID)
|
||||
}
|
||||
for _, group := range byLabels {
|
||||
d.LogPolicyGroups = append(d.LogPolicyGroups, *group)
|
||||
}
|
||||
source.Access = nil
|
||||
changed = true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Shutdown 优雅关闭数据库，确保所有数据已落盘。
|
||||
// 顺序很关键：先停防抖定时器（不再产生新请求）→ 关闭 worker 并等其退出
|
||||
// （worker 可能正在处理已入队的请求）→ 此时已无并发，做最后一次 doSave 把
|
||||
@@ -181,7 +250,7 @@ func (d *JsonDB) doSave() error {
|
||||
|
||||
// 原子写入：先写入临时文件，再重命名
|
||||
dir := filepath.Dir(d.dataFile)
|
||||
tmpFile := filepath.Join(dir, fmt.Sprintf(".lokideploy-%d.tmp", time.Now().UnixNano()))
|
||||
tmpFile := filepath.Join(dir, fmt.Sprintf(".opsxloki-%d.tmp", time.Now().UnixNano()))
|
||||
|
||||
if err := os.WriteFile(tmpFile, data, 0644); err != nil {
|
||||
return err
|
||||
@@ -275,7 +344,31 @@ func (d *JsonDB) CreateUser(user *models.User) error {
|
||||
user.ID = d.sequences["users"] + 1
|
||||
d.sequences["users"] = user.ID
|
||||
}
|
||||
if user.CreatedAt.IsZero() {
|
||||
now := time.Now()
|
||||
user.CreatedAt = now
|
||||
user.UpdatedAt = now
|
||||
}
|
||||
d.Users = append(d.Users, *user)
|
||||
d.RequestSave()
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverOrphanDeployments 启动时把残留的 deploying 状态部署标记为失败:
|
||||
// 部署 goroutine 随进程消亡,不恢复会永久卡在「部署中」(UI 无停止入口)。
|
||||
func (d *JsonDB) RecoverOrphanDeployments() {
|
||||
d.Lock()
|
||||
defer d.Unlock()
|
||||
recovered := 0
|
||||
for i := range d.Deployments {
|
||||
if d.Deployments[i].Status == "deploying" {
|
||||
d.Deployments[i].Status = "failed"
|
||||
d.Deployments[i].StatusMessage = "服务重启导致部署中断，请重新部署"
|
||||
d.Deployments[i].UpdatedAt = time.Now()
|
||||
recovered++
|
||||
}
|
||||
}
|
||||
if recovered > 0 {
|
||||
d.RequestSave()
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"lokideploy/models"
|
||||
"opsxloki/models"
|
||||
)
|
||||
|
||||
// TestConcurrentSaveNoDataLoss 是并发写安全（B1/B2）的回归测试：
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"opsxloki/models"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMigrateLogAccessToGroups(t *testing.T) {
|
||||
d := &JsonDB{
|
||||
Users: []models.User{{ID: 1, Role: "user"}, {ID: 2, Role: "user"}},
|
||||
LogSources: []models.LogSource{{ID: 9, Name: "loki", Access: []models.LogSourceAccess{
|
||||
{UserID: 1, Labels: map[string]string{"namespace": "log"}},
|
||||
{UserID: 2, Labels: map[string]string{"namespace": "log"}},
|
||||
}}},
|
||||
sequences: map[string]uint{"logPolicyGroups": 0},
|
||||
}
|
||||
if !migrateLogAccessToGroups(d) {
|
||||
t.Fatal("expected migration")
|
||||
}
|
||||
if len(d.LogSources[0].Access) != 0 {
|
||||
t.Fatal("legacy grants remain")
|
||||
}
|
||||
if len(d.LogPolicyGroups) != 1 || len(d.LogPolicyGroups[0].UserIDs) != 2 {
|
||||
t.Fatalf("expected one shared group, got %#v", d.LogPolicyGroups)
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,13 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"lokideploy/config"
|
||||
"opsxloki/config"
|
||||
)
|
||||
|
||||
// TestMain 为 repository 包预置安全环境变量与临时 DATA_DIR，并管理 JsonDB 生命周期。
|
||||
// config.GetConfig() 在 JWT_SECRET/ENCRYPTION_KEY 缺失时会 log.Fatal，故须先设置。
|
||||
func TestMain(m *testing.M) {
|
||||
dir, err := os.MkdirTemp("", "lokideploy-repository-test")
|
||||
dir, err := os.MkdirTemp("", "opsxloki-repository-test")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user