Files
LokiStack/backend/services/deployment.go
T
logan 2d73661dd1 feat: complete Loki query, insights, settings and deployment workflows
Add the Loki query UI, access policies, ingress insights, alert and registry settings, and their backend APIs. Include the related Kubernetes deployment and collector updates required by the integrated application.

Validated with go test ./..., npm run build, and npm run lint.
2026-10-25 21:28:32 +08:00

744 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package services
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"sync"
"time"
"opsxloki/models"
"opsxloki/repository"
"opsxloki/seed"
"opsxloki/utils"
)
// LokiVersionList 支持的 Loki 版本列表(按推荐程度排序)
var LokiVersionList = []string{
"3.4.0",
"3.3.0",
"3.2.0",
"3.1.0",
"3.0.0",
"2.9.8",
"2.9.0",
"2.8.9",
}
// minSupportedMajorVersion 最低支持的主版本号
const minSupportedMajorVersion = 2
const DefaultRetentionDays = 30
// DeploymentService 部署业务逻辑层
type DeploymentService struct{}
func NewDeploymentService() *DeploymentService {
return &DeploymentService{}
}
type CreateDeploymentRequest struct {
Name string `json:"name" binding:"required"`
Description string `json:"description"`
Version string `json:"version"`
Mode string `json:"mode" binding:"required"`
ClusterID *uint `json:"clusterID"`
HostID *uint `json:"hostID"`
ListenIP string `json:"listenIP"`
ListenPort int `json:"listenPort"`
Namespace string `json:"namespace"`
StorageConfigID uint `json:"storageConfigID"`
AuthEnabled bool `json:"authEnabled"`
RetentionDays int `json:"retentionDays"`
Replicas int `json:"replicas"`
ResourcesCPU string `json:"resourcesCPU"`
ResourcesMem string `json:"resourcesMem"`
StorageSize string `json:"storageSize"`
GrafanaEnabled bool `json:"grafanaEnabled"`
GrafanaVersion string `json:"grafanaVersion"`
}
func (s *DeploymentService) Create(req *CreateDeploymentRequest) (*models.LokiDeployment, error) {
if req.Mode != "k8s-yaml" && req.Mode != "host-docker" && req.Mode != "host-binary" {
return nil, fmt.Errorf("不支持的部署方式: %s", req.Mode)
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("Grafana 自动部署已移除,请使用独立 Grafana 服务")
}
if req.Mode == "host-docker" {
if _, err := dockerLokiImage(req.Version); err != nil {
return nil, err
}
if req.HostID == nil || *req.HostID == 0 {
return nil, fmt.Errorf("请选择目标主机")
}
if req.ListenPort == 0 {
req.ListenPort = 3100
}
if req.ListenPort < 1 || req.ListenPort > 65535 {
return nil, fmt.Errorf("监听端口须在 1 到 65535 之间")
}
if req.Replicas > 1 {
return nil, fmt.Errorf("主机 Docker 部署仅支持单副本")
}
if req.GrafanaEnabled {
return nil, fmt.Errorf("主机 Docker 模式暂不支持同时部署 Grafana")
}
if req.ListenIP != "" && net.ParseIP(req.ListenIP).To4() == nil {
return nil, fmt.Errorf("监听地址须为 IPv4 地址")
}
}
if req.RetentionDays < 0 {
return nil, fmt.Errorf("保留天数必须是正整数")
}
if req.RetentionDays == 0 {
req.RetentionDays = DefaultRetentionDays
}
// 版本兼容性预检查
if req.Version != "" {
if err := ValidateLokiVersion(req.Version); err != nil {
return nil, err
}
}
db := repository.GetDB()
d := models.LokiDeployment{
ID: db.NextID("deployments"),
Name: req.Name,
Description: req.Description,
Version: req.Version,
Mode: req.Mode,
ClusterID: req.ClusterID,
HostID: req.HostID,
ListenIP: req.ListenIP,
ListenPort: req.ListenPort,
Namespace: req.Namespace,
StorageConfigID: req.StorageConfigID,
AuthEnabled: req.AuthEnabled,
RetentionDays: req.RetentionDays,
Replicas: req.Replicas,
ResourcesCPU: req.ResourcesCPU,
ResourcesMem: req.ResourcesMem,
StorageSize: req.StorageSize,
GrafanaEnabled: req.GrafanaEnabled,
GrafanaVersion: req.GrafanaVersion,
Status: "pending",
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
d.BeforeCreate()
// Load relations
if d.ClusterID != nil && *d.ClusterID > 0 {
for i := range db.Clusters {
if db.Clusters[i].ID == *d.ClusterID {
c := db.Clusters[i]
d.Cluster = &c
break
}
}
}
if d.HostID != nil && *d.HostID > 0 {
for i := range db.Hosts {
if db.Hosts[i].ID == *d.HostID {
h := db.Hosts[i]
d.Host = &h
break
}
}
}
if d.Mode == "host-docker" {
if d.Host == nil {
return nil, fmt.Errorf("目标主机不存在")
}
if d.ListenIP == "" {
d.ListenIP = d.Host.HostIP
}
if net.ParseIP(d.ListenIP).To4() == nil {
return nil, fmt.Errorf("主机 IP 格式无效,请填写监听内网 IPv4 地址")
}
}
if d.StorageConfigID > 0 {
for i := range db.Storages {
if db.Storages[i].ID == d.StorageConfigID {
st := db.Storages[i]
// 模型内为 ENC: 密文,生成 Loki 配置前解密(否则密文进 ConfigMap 且无法通过对象存储认证)
if ak, err := DecryptCred(st.AccessKey); err == nil && ak != "" {
st.AccessKey = ak
}
if sk, err := DecryptCred(st.SecretKey); err == nil && sk != "" {
st.SecretKey = sk
}
d.StorageConfig = &st
break
}
}
}
db.Lock()
db.Deployments = append(db.Deployments, d)
db.Unlock()
db.RequestSave()
return &d, nil
}
func (s *DeploymentService) Get(id uint) (*models.LokiDeployment, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
for i := range db.Deployments {
if db.Deployments[i].ID == id {
d := db.Deployments[i]
s.loadRelations(&d)
return &d, nil
}
}
return nil, utils.ErrNotFound
}
func (s *DeploymentService) List(page, pageSize int) ([]models.LokiDeployment, int64, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
total := int64(len(db.Deployments))
start := (page - 1) * pageSize
if start < 0 || start >= len(db.Deployments) {
return []models.LokiDeployment{}, total, nil
}
end := start + pageSize
if end > len(db.Deployments) {
end = len(db.Deployments)
}
result := make([]models.LokiDeployment, end-start)
copy(result, db.Deployments[start:end])
for i := range result {
s.loadRelations(&result[i])
}
return result, total, nil
}
func (s *DeploymentService) Update(id uint, updates map[string]interface{}) error {
if value, exists := updates["retentionDays"]; exists {
days, ok := value.(float64)
if !ok || days < 1 || days != float64(int(days)) {
return fmt.Errorf("保留天数必须是正整数")
}
}
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i := range db.Deployments {
if db.Deployments[i].ID == id {
if v, ok := updates["status"].(string); ok {
db.Deployments[i].Status = v
}
if v, ok := updates["statusMessage"].(string); ok {
db.Deployments[i].StatusMessage = v
}
if v, ok := updates["endpoint"].(string); ok {
db.Deployments[i].Endpoint = v
}
// 业务字段(来自 PUT /deployments/:id;JSON 数字解码为 float64)
if v, ok := updates["name"].(string); ok && v != "" {
db.Deployments[i].Name = v
}
if v, ok := updates["description"].(string); ok {
db.Deployments[i].Description = v
}
if v, ok := updates["version"].(string); ok && v != "" {
db.Deployments[i].Version = v
}
if v, ok := updates["namespace"].(string); ok && v != "" {
db.Deployments[i].Namespace = v
}
if v, ok := updates["resourcesCPU"].(string); ok {
db.Deployments[i].ResourcesCPU = v
}
if v, ok := updates["resourcesMem"].(string); ok {
db.Deployments[i].ResourcesMem = v
}
if v, ok := updates["storageSize"].(string); ok && v != "" {
db.Deployments[i].StorageSize = v
}
if v, ok := updates["authEnabled"].(bool); ok {
db.Deployments[i].AuthEnabled = v
}
if v, ok := updates["grafanaEnabled"].(bool); ok {
db.Deployments[i].GrafanaEnabled = v
}
if v, ok := updates["grafanaVersion"].(string); ok && v != "" {
db.Deployments[i].GrafanaVersion = v
}
if v, ok := updates["grafanaEndpoint"].(string); ok {
db.Deployments[i].GrafanaEndpoint = v
}
if v, ok := updates["storageConfigID"].(float64); ok && v > 0 {
db.Deployments[i].StorageConfigID = uint(v)
}
if v, ok := updates["clusterID"].(float64); ok && v > 0 {
id := uint(v)
db.Deployments[i].ClusterID = &id
}
if v, ok := updates["hostID"].(float64); ok && v > 0 {
id := uint(v)
db.Deployments[i].HostID = &id
}
if v, ok := updates["retentionDays"].(float64); ok && v > 0 {
db.Deployments[i].RetentionDays = int(v)
}
if v, ok := updates["replicas"].(float64); ok && v > 0 {
db.Deployments[i].Replicas = int(v)
}
db.Deployments[i].UpdatedAt = time.Now()
db.RequestSave()
return nil
}
}
return utils.ErrNotFound
}
func (s *DeploymentService) Delete(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Mode == "host-docker" {
if deployment.Status == "deploying" {
return fmt.Errorf("部署正在进行中,请等待完成后删除")
}
if deployment.Status != "pending" {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if err := deleteDockerLoki(ctx, deployment); err != nil {
return err
}
}
}
db := repository.GetDB()
db.Lock()
defer db.Unlock()
for i, d := range db.Deployments {
if d.ID == id {
db.Deployments = append(db.Deployments[:i], db.Deployments[i+1:]...)
db.RequestSave()
return nil
}
}
return utils.ErrNotFound
}
// ==================== Goroutine 管理改进 ====================
// deployCtxEntry 保存正在进行的部署上下文
type deployCtxEntry struct {
cancel context.CancelFunc
done chan struct{}
}
var (
deployCtxMap = make(map[uint]*deployCtxEntry)
deployCtxMu sync.Mutex
)
func (s *DeploymentService) StartDeploy(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Status == "deploying" {
return fmt.Errorf("部署正在进行中,请勿重复操作")
}
// 检查已有 goroutine 状态,防止重复部署
deployCtxMu.Lock()
if entry, exists := deployCtxMap[id]; exists {
select {
case <-entry.done:
// 已完成,可以重新部署
delete(deployCtxMap, id)
default:
deployCtxMu.Unlock()
return fmt.Errorf("部署正在进行中,请勿重复操作")
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
done := make(chan struct{})
deployCtxMap[id] = &deployCtxEntry{cancel: cancel, done: done}
deployCtxMu.Unlock()
s.Update(id, map[string]interface{}{"status": "deploying"})
logFunc := func(level, message, detail string) {
s.AddLog(id, level, message, detail)
}
go func() {
defer close(done)
defer func() {
deployCtxMu.Lock()
delete(deployCtxMap, id)
deployCtxMu.Unlock()
}()
deployer := NewLokiDeployer(deployment, logFunc)
if err := deployer.Deploy(ctx); err != nil {
s.Update(id, map[string]interface{}{
"status": "failed",
"statusMessage": fmt.Sprintf("Deployment failed: %v", err),
})
s.AddLog(id, "error", fmt.Sprintf("Deployment failed: %v", err), "")
} else {
s.Update(id, map[string]interface{}{
"status": "running",
"statusMessage": "Deployment completed successfully",
"endpoint": s.getEndpoint(deployment),
})
// 部署成功自动注册日志查询数据源 + 记录 Grafana 访问地址 + 补内置告警模板
if fresh, gerr := s.Get(id); gerr == nil {
EnsureLogSourceForDeployment(fresh)
go s.recordGrafanaEndpoint(fresh)
go seed.SeedAlertRules()
}
s.AddLog(id, "success", "Deployment completed successfully", "")
}
}()
return nil
}
func (s *DeploymentService) StopDeployment(id uint) error {
deployment, err := s.Get(id)
if err != nil {
return err
}
if deployment.Status != "running" && deployment.Status != "deploying" {
return fmt.Errorf("部署未在运行,无法停止")
}
// 取消正在进行的部署 goroutine
deployCtxMu.Lock()
if entry, exists := deployCtxMap[id]; exists {
entry.cancel()
}
deployCtxMu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if deployment.Mode == "host-docker" {
if err := stopDockerLoki(ctx, deployment); err != nil {
return err
}
} else {
CleanupLoki(ctx, deployment)
}
return s.Update(id, map[string]interface{}{
"status": "stopped",
"statusMessage": "Deployment stopped by user",
})
}
func (s *DeploymentService) RestartDeployment(id uint) error {
if err := s.StopDeployment(id); err != nil {
// 如果停止失败但不是因为部署未运行,返回错误
if !strings.Contains(err.Error(), "部署未在运行") {
return err
}
}
time.Sleep(2 * time.Second)
return s.StartDeploy(id)
}
func (s *DeploymentService) AddLog(deploymentID uint, level, message, detail string) {
db := repository.GetDB()
log := models.DeploymentLog{
ID: db.NextID("logs"),
DeploymentID: deploymentID,
Level: level,
Message: message,
Detail: detail,
CreatedAt: time.Now(),
}
db.Lock()
db.Logs = append(db.Logs, log)
db.Unlock()
db.RequestSave()
}
func (s *DeploymentService) GetLogs(deploymentID uint, page, pageSize int) ([]models.DeploymentLog, int64, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
var filtered []models.DeploymentLog
for i := len(db.Logs) - 1; i >= 0; i-- {
if db.Logs[i].DeploymentID == deploymentID {
filtered = append(filtered, db.Logs[i])
}
}
total := int64(len(filtered))
start := (page - 1) * pageSize
if start >= len(filtered) {
return []models.DeploymentLog{}, total, nil
}
end := start + pageSize
if end > len(filtered) {
end = len(filtered)
}
return filtered[start:end], total, nil
}
func (s *DeploymentService) GetStats() (map[string]interface{}, error) {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
var running, failed, pending, stopped int64
for _, d := range db.Deployments {
switch d.Status {
case "running":
running++
case "failed":
failed++
case "pending":
pending++
case "stopped":
stopped++
}
}
return map[string]interface{}{
"totalDeployments": int64(len(db.Deployments)),
"running": running,
"failed": failed,
"pending": pending,
"stopped": stopped,
"clusterCount": int64(len(db.Clusters)),
"hostCount": int64(len(db.Hosts)),
"supportedVersions": LokiVersionList,
}, nil
}
func (s *DeploymentService) loadRelations(d *models.LokiDeployment) {
db := repository.GetDB()
if d.ClusterID != nil && *d.ClusterID > 0 {
for i := range db.Clusters {
if db.Clusters[i].ID == *d.ClusterID {
c := db.Clusters[i]
d.Cluster = &c
break
}
}
}
if d.HostID != nil && *d.HostID > 0 {
for i := range db.Hosts {
if db.Hosts[i].ID == *d.HostID {
h := db.Hosts[i]
d.Host = &h
break
}
}
}
if d.StorageConfigID > 0 {
for i := range db.Storages {
if db.Storages[i].ID == d.StorageConfigID {
st := db.Storages[i]
// 模型内为 ENC: 密文,生成 Loki 配置前解密(否则密文进 ConfigMap 且无法通过对象存储认证)
if ak, err := DecryptCred(st.AccessKey); err == nil && ak != "" {
st.AccessKey = ak
}
if sk, err := DecryptCred(st.SecretKey); err == nil && sk != "" {
st.SecretKey = sk
}
d.StorageConfig = &st
break
}
}
}
}
func (s *DeploymentService) getEndpoint(d *models.LokiDeployment) string {
switch d.Mode {
case "k8s-helm", "k8s-yaml":
ns := d.Namespace
if ns == "" {
ns = "loki"
}
// k8s-yaml manifest 固定创建名为 loki 的 Service;
// k8s-helm (grafana/loki) 以 release 名(=部署名,默认 loki)暴露 Service。
// 注意:不使用 loki-gateway,gateway 仅在 Distributed+gateway 启用时存在。
svc := "loki"
if d.Mode == "k8s-helm" && d.Name != "" {
svc = d.Name
}
return fmt.Sprintf("http://%s.%s.svc.cluster.local:3100", svc, ns)
case "host-binary":
if d.Host != nil {
return fmt.Sprintf("http://%s:3100", d.Host.HostIP)
}
case "host-docker":
return fmt.Sprintf("http://%s:%d", d.ListenIP, d.ListenPort)
}
return ""
}
// ==================== 版本兼容性检查 ====================
// ValidateLokiVersion 检查 Loki 版本是否在支持列表中且满足最低版本要求
func ValidateLokiVersion(version string) error {
if version == "" {
return nil // 使用默认值,不报错
}
// 基本格式验证
parts := strings.Split(version, ".")
if len(parts) < 2 {
return fmt.Errorf("版本号格式无效: %s,期望格式如 3.0.0", version)
}
major, err := strconv.Atoi(parts[0])
if err != nil {
return fmt.Errorf("无法解析主版本号: %s", version)
}
minor, err := strconv.Atoi(parts[1])
if err != nil {
return fmt.Errorf("无法解析次版本号: %s", version)
}
// 最低版本检查
if major < minSupportedMajorVersion {
return fmt.Errorf("Loki %s 版本过低,最低支持 %d.x 版本", version, minSupportedMajorVersion)
}
if major == 2 && minor < 8 {
return fmt.Errorf("Loki %s 版本过低,2.x 最低支持 2.8.x 版本", version)
}
if major >= 4 {
return fmt.Errorf("Loki %s 是未发布的版本,请使用已知稳定版本", version)
}
return nil
}
// GetRecommendedVersions 返回推荐的 Loki 版本列表
func GetRecommendedVersions() []string {
result := make([]string, len(LokiVersionList))
copy(result, LokiVersionList)
return result
}
// Preflight 向导预检:聚合各资源真实状态,前端据此推导步骤完成态(状态推导式向导)
func (s *DeploymentService) Preflight() map[string]interface{} {
db := repository.GetDB()
db.RLock()
defer db.RUnlock()
result := map[string]interface{}{
"hasStorage": len(db.Storages) > 0,
"hasActiveCluster": false,
"clusterNames": []string{},
"hasActiveHost": false,
"hostNames": []string{},
"runningLoki": nil,
"activeCollectors": 0,
"grafanaReady": false,
}
var defaultStorage string
for i := range db.Storages {
if db.Storages[i].IsDefault {
defaultStorage = db.Storages[i].Name
break
}
}
if defaultStorage == "" && len(db.Storages) > 0 {
defaultStorage = db.Storages[0].Name
}
result["defaultStorage"] = defaultStorage
clusters := []string{}
for i := range db.Clusters {
clusters = append(clusters, db.Clusters[i].Name)
if db.Clusters[i].Status == "active" || db.Clusters[i].Status == "running" {
result["hasActiveCluster"] = true
}
}
result["clusterNames"] = clusters
hosts := []string{}
for i := range db.Hosts {
hosts = append(hosts, db.Hosts[i].Name)
if db.Hosts[i].Status == "active" {
result["hasActiveHost"] = true
}
}
result["hostNames"] = hosts
for i := range db.Deployments {
d := &db.Deployments[i]
if d.Status == "running" {
loki := map[string]interface{}{
"id": d.ID,
"name": d.Name,
"mode": d.Mode,
"endpoint": d.Endpoint,
"namespace": d.Namespace,
"grafana": d.GrafanaEnabled,
"grafanaEndpoint": d.GrafanaEndpoint,
}
result["runningLoki"] = loki
if d.GrafanaEnabled {
result["grafanaReady"] = true
}
break
}
}
for i := range db.Collectors {
if db.Collectors[i].Status == "active" {
result["activeCollectors"] = result["activeCollectors"].(int) + 1
}
}
return result
}
// recordGrafanaEndpoint 记录 Grafana 可点击访问地址(k8s: 首节点 IP + NodePort;host: 主机 IP:3000),
// 供部署卡片/向导直接展示,用户不用翻日志找端口。
func (s *DeploymentService) recordGrafanaEndpoint(d *models.LokiDeployment) {
if d == nil || !d.GrafanaEnabled {
return
}
var endpoint string
switch d.Mode {
case "k8s-helm", "k8s-yaml":
if d.Cluster == nil {
s.AddLog(d.ID, "warn", "Grafana 地址记录跳过:未关联集群", "")
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ns := d.Namespace
if ns == "" {
ns = "log"
}
ip, ipErr := firstNodeIP(ctx, d.Cluster)
port, portErr := serviceNodePort(ctx, d.Cluster, ns, "grafana")
if ipErr == nil && portErr == nil && port > 0 {
endpoint = fmt.Sprintf("http://%s:%d", ip, port)
}
case "host-binary":
if d.Host != nil {
endpoint = fmt.Sprintf("http://%s:3000", d.Host.HostIP)
}
}
if endpoint != "" {
_ = s.Update(d.ID, map[string]interface{}{"grafanaEndpoint": endpoint})
}
}