From b49677f6a6d12f1c4fb7fcb7d4e69e70cf5ec693 Mon Sep 17 00:00:00 2001 From: opsx Date: Wed, 28 Oct 2026 21:27:01 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=BF=90=E7=BB=B4=E6=A0=87=E6=9D=86?= =?UTF-8?q?=E5=8C=96=E6=89=93=E7=A3=A8=20P0/P1/P2=20=E2=80=94=E2=80=94=20?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E6=97=A5=E5=BF=97=E3=80=81=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E3=80=81=E5=A4=87=E4=BB=BD=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E3=80=81=E8=8F=9C=E5=8D=95=E6=94=B6=E6=95=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 审计发现 5 处硬伤（/metrics 空壳、零备份、prod 双副本数据分裂、零审计、资源偏紧）， 本轮全部落地。 === P0-1 真实 /metrics（原来是空壳）=== 原实现只返回一行注释 `# LokiStack Metrics`，注释却写着"可被 Prometheus 抓取"， 平台挂了/DB 写不进去/告警引擎卡死都只能靠用户反馈。 现接入 prometheus/client_golang，四层指标： HTTP : lokistack_http_requests_total / _duration_seconds / _in_flight_requests 存储 : lokistack_db_size_bytes / _db_save_duration_seconds / _db_save_total / _db_records 业务 : lokistack_deployments{status,mode} / lokistack_collectors{status,type,run_mode} 后台任务: lokistack_alert_evaluations_total / _duration_seconds / lokistack_loki_queries_total + go_* / process_*（GoCollector、ProcessCollector） 关键点：path 标签用 c.FullPath() 路由模板，不用原始 URL —— 否则 117 个 Deployment 会把 Prometheus 基数炸掉。指标定义下沉 pkg/metrics 叶子包，避开 services ↔ repository 循环依赖。 === P0-2 备份与恢复（原来 0 处代码）=== 平台全部状态在 /app/data/opsxloki.json（51KB）+ kubeconfigs/ + known_hosts/， DB 丢了 = 失去"这些资源是谁建的"的映射，从此无法管理/卸载/升级在跑的 Loki。 CLI : lokistack backup --output / restore / self-check / version API : GET /system/backup（下载）/ POST /system/restore（校验暂存）/ POST /system/restore/apply（确认应用）/ GET /system/backups K8s : kubernetes/pdb-backup.yaml —— 每日 03:30 CronJob，保留最近 7 份 安全设计： - manifest.json 记录 ENCRYPTION_KEY 的 sha256 前 16 位**指纹**，用于恢复时 校验"这把钥匙能不能解开这份备份"，不泄露密钥本身 - 恢复分两步（校验暂存 → 确认应用），应用前自动把当前 DB 备份为 opsxloki.json.before-restore.<时间戳> - 包内不含明文密钥（kubeconfig 本就是加密态） === P0-3 prod compose 双副本 → 单副本 === lokistack-1 / lokistack-2 各挂独立数据卷 + nginx 轮询 = 数据分裂： 页面刷新两次可能看到不同结果，写操作只落一个副本。不是高可用。 已改单实例（nginx 仅做反代），文件顶部与 nginx.conf 都写明原因。 === P0-4 操作审计日志（借鉴 /usr/local/src/aiops 的字段口径）=== aiops 是每个 controller 各写 recordXxxAudit()；本平台有 60+ 写接口， 逐个埋点必然漏，漏掉的那个恰好最危险（删集群、改存储凭证）。 改为 **gin 中间件统一记录** + handler 通过 SetAudit 补充业务语义： models/audit.go : 字段对齐 aiops，另加 Method/Path/Before/After services/audit.go : 写入 + 90 天/2 万条自动裁剪 + 脱敏 + diff 生成 middleware/audit.go : 自动识别 module/action/resource，跳过只读接口 controllers/audit.go: 列表/详情/统计（仅 admin） 敏感字段（password/secretKey/privateKey/token）写入前统一替换为 ***。 === P1 稳定性 === - 平台资源 1C/1Gi → 4C/4Gi 限（500m/512Mi 请求）：JSON DB 全量读写 + 每分钟告警评估 + 批量发布并发，1Gi 会 OOM - terminationGracePeriodSeconds: 30 —— 收到 SIGTERM 要等 DB 完整落盘， 否则 kubelet SIGKILL 会把 JSON 写一半（全库丢失） - liveness timeoutSeconds 10s（默认 1s 写盘时会被误杀） - PDB minAvailable: 0 —— 让 kubectl drain 正确等待重建，不卡维护 - DB schemaVersion + migrateSchema() 版本递进迁移（原先加字段全靠零值兜底） - scripts/release.sh —— 不可变 tag <版本>-<日期>- + IfNotPresent， 根治 latest 并发构建互相覆盖导致的静默回滚（本项目已实际踩过） - PVC 保持 30Gi（按长期增长预留，云盘单价低，不做在线扩容） === P2 易用性 === - 一键自检 GET /system/self-check + 仪表盘自检卡：真实探活 Loki /ready、 集群凭据解密、磁盘空间、主机状态、近 24h 失败操作 - 审计日志页 /audit：统计卡 + 用户名/模块/结果筛选 + 变更前后 diff 弹窗 - 菜单收敛 13 → 7（页内 tab 切换，URL 全部不变）： 资源接入 = 集群 + 主机 + 存储 日志服务 = Loki 实例 + 采集器 日志 = 查询 + 分析 系统管理 = 用户 + 审计 + 设置 新增 GroupTabs 组件 + meta.group/groupKey，侧栏按分组去重并正确高亮 测试：go vet 通过、go test 全绿、tsc 通过 --- backend/controllers/audit.go | 71 ++++ backend/controllers/system.go | 91 ++++ backend/go.mod | 7 + backend/go.sum | 14 + backend/main.go | 89 ++++ backend/middleware/audit.go | 206 ++++++++++ backend/middleware/metrics.go | 54 +++ backend/models/audit.go | 80 ++++ backend/pkg/metrics/metrics.go | 175 ++++++++ backend/repository/db.go | 60 ++- backend/routers/router.go | 36 +- backend/services/alert.go | 15 +- backend/services/audit.go | 309 ++++++++++++++ backend/services/backup.go | 337 +++++++++++++++ backend/services/metrics.go | 97 +++++ backend/services/selfcheck.go | 233 +++++++++++ docker-compose.prod.yml | 154 ++----- frontend/src/api/system/index.ts | 102 +++++ frontend/src/components/GroupTabs.tsx | 57 +++ frontend/src/components/Layout.tsx | 12 +- frontend/src/i18n/domains.ts | 4 + frontend/src/i18n/locales/en/audit.ts | 32 ++ frontend/src/i18n/locales/en/common.ts | 7 + frontend/src/i18n/locales/en/dashboard.ts | 7 + frontend/src/i18n/locales/zh/audit.ts | 32 ++ frontend/src/i18n/locales/zh/common.ts | 7 + frontend/src/i18n/locales/zh/dashboard.ts | 7 + frontend/src/pages/audit/index.tsx | 389 ++++++++++++++++++ .../dashboard/components/SelfCheckCard.tsx | 120 ++++++ frontend/src/pages/dashboard/index.tsx | 4 + frontend/src/router/routes.tsx | 125 +++++- frontend/src/types/router.ts | 20 + kubernetes/deployment.yaml | 20 +- kubernetes/pdb-backup.yaml | 76 ++++ kubernetes/pvc.yaml | 2 + nginx.conf | 6 +- scripts/release.sh | 73 ++++ 37 files changed, 2983 insertions(+), 147 deletions(-) create mode 100644 backend/controllers/audit.go create mode 100644 backend/controllers/system.go create mode 100644 backend/middleware/audit.go create mode 100644 backend/middleware/metrics.go create mode 100644 backend/models/audit.go create mode 100644 backend/pkg/metrics/metrics.go create mode 100644 backend/services/audit.go create mode 100644 backend/services/backup.go create mode 100644 backend/services/metrics.go create mode 100644 backend/services/selfcheck.go create mode 100644 frontend/src/api/system/index.ts create mode 100644 frontend/src/components/GroupTabs.tsx create mode 100644 frontend/src/i18n/locales/en/audit.ts create mode 100644 frontend/src/i18n/locales/zh/audit.ts create mode 100644 frontend/src/pages/audit/index.tsx create mode 100644 frontend/src/pages/dashboard/components/SelfCheckCard.tsx create mode 100644 kubernetes/pdb-backup.yaml create mode 100755 scripts/release.sh diff --git a/backend/controllers/audit.go b/backend/controllers/audit.go new file mode 100644 index 0000000..6120b65 --- /dev/null +++ b/backend/controllers/audit.go @@ -0,0 +1,71 @@ +package controllers + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "lokistack/models" + "lokistack/services" + "lokistack/utils" +) + +// AuditController 审计日志查询（仅管理员） +type AuditController struct{} + +func NewAuditController() *AuditController { return &AuditController{} } + +// List 分页查询审计日志 +// GET /api/v1/audit-logs?page=1&pageSize=20&username=&module=&action=&status=0&startTime=&endTime= +func (ctrl *AuditController) List(c *gin.Context) { + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "20")) + if pageSize <= 0 || pageSize > 200 { + pageSize = 20 + } + status, _ := strconv.Atoi(c.DefaultQuery("status", "0")) + + q := models.AuditQuery{ + Username: c.Query("username"), + Module: c.Query("module"), + Action: c.Query("action"), + ResourceID: c.Query("resourceId"), + Status: status, + Page: page, + PageSize: pageSize, + } + if v := c.Query("startTime"); v != "" { + if t, err := time.Parse(time.RFC3339, v); err == nil { + q.StartTime = &t + } + } + if v := c.Query("endTime"); v != "" { + if t, err := time.Parse(time.RFC3339, v); err == nil { + q.EndTime = &t + } + } + + list, total := services.ListAuditLogs(q) + utils.Success(c, gin.H{"list": list, "total": total, "page": page, "pageSize": pageSize}) +} + +// Get 审计详情（含 Before/After 完整内容） +func (ctrl *AuditController) Get(c *gin.Context) { + id, err := utils.IntParam(c, "id") + if err != nil { + utils.BadRequest(c, "ID 无效") + return + } + entry, err := services.GetAuditLog(uint(id)) + if err != nil { + utils.NotFound(c, err.Error()) + return + } + utils.Success(c, entry) +} + +// Stats 统计卡片 +func (ctrl *AuditController) Stats(c *gin.Context) { + utils.Success(c, services.AuditStatistics()) +} diff --git a/backend/controllers/system.go b/backend/controllers/system.go new file mode 100644 index 0000000..d4704cf --- /dev/null +++ b/backend/controllers/system.go @@ -0,0 +1,91 @@ +package controllers + +import ( + "fmt" + "time" + + "github.com/gin-gonic/gin" + + "lokistack/services" + "lokistack/utils" +) + +// SystemController 系统级运维操作：备份 / 恢复 / 巡检。 +type SystemController struct{} + +func NewSystemController() *SystemController { return &SystemController{} } + +// Backup 导出当前平台数据（tar.gz 下载）。 +// +// 内容：数据文件 + kubeconfigs/ + known_hosts/ + manifest.json（含密钥指纹）， +// 不含明文密钥。恢复必须使用同一把 ENCRYPTION_KEY。 +func (ctrl *SystemController) Backup(c *gin.Context) { + name := fmt.Sprintf("lokistack-backup-%s.tar.gz", time.Now().Format("20060102-150405")) + c.Header("Content-Type", "application/gzip") + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) + + if _, err := services.CreateBackup(c.Writer); err != nil { + // 已经写了响应头，无法再改状态码，只能把错误写进 body 尾部 + _, _ = c.Writer.WriteString(fmt.Sprintf("\n\n# 备份失败: %v", err)) + return + } +} + +// Restore 上传备份包：校验 + 暂存，**不直接覆盖**线上数据库。 +// 返回暂存路径与密钥指纹比对结果，由管理员确认后再调用 RestoreApply。 +func (ctrl *SystemController) Restore(c *gin.Context) { + file, err := c.FormFile("file") + if err != nil { + utils.BadRequest(c, "请上传备份文件（form-data 字段名 file）") + return + } + if file.Size > 128<<20 { + utils.BadRequest(c, "备份文件超过 128MB，请用 run.sh restore 在服务器上操作") + return + } + f, err := file.Open() + if err != nil { + utils.InternalServerError(c, fmt.Sprintf("读取上传文件失败: %v", err)) + return + } + defer f.Close() + + res, err := services.ValidateAndStageBackup(f) + if err != nil { + utils.BadRequest(c, fmt.Sprintf("备份校验失败: %v", err)) + return + } + utils.Success(c, res) +} + +// RestoreApply 应用已暂存的备份。应用后需重启平台进程才生效。 +func (ctrl *SystemController) RestoreApply(c *gin.Context) { + var req struct { + Confirm bool `json:"confirm"` + } + _ = c.ShouldBindJSON(&req) + if !req.Confirm { + utils.BadRequest(c, "恢复会覆盖当前全部数据，请传 confirm=true 确认") + return + } + path, err := services.ApplyStagedBackup() + if err != nil { + utils.BadRequest(c, err.Error()) + return + } + utils.Success(c, gin.H{ + "message": "备份已应用，请重启平台使新数据生效（kubectl -n log rollout restart deploy/lokistack）", + "dbPath": path, + }) +} + +// Backups 列出服务器上已有的备份文件（run.sh backup 产出）。 +func (ctrl *SystemController) Backups(c *gin.Context) { + utils.Success(c, gin.H{"list": services.ListBackups()}) +} + +// Health 平台自检：一键诊断用。 +// 聚合「能不能用」的判断，避免排障时在多个页面之间跳。 +func (ctrl *SystemController) Health(c *gin.Context) { + utils.Success(c, services.SelfCheck()) +} diff --git a/backend/go.mod b/backend/go.mod index 4defb87..ec93b6c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -19,8 +19,10 @@ require ( require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.12.6 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -43,6 +45,7 @@ require ( github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -55,6 +58,10 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.7 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect diff --git a/backend/go.sum b/backend/go.sum index c344a2b..dbbaf3d 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -4,11 +4,15 @@ github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktp github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -92,6 +96,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= @@ -132,6 +138,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= diff --git a/backend/main.go b/backend/main.go index e96d235..5af651d 100644 --- a/backend/main.go +++ b/backend/main.go @@ -2,11 +2,14 @@ package main import ( "context" + "encoding/json" "errors" + "fmt" "log" "net/http" "os" "os/signal" + "path/filepath" "strings" "syscall" "time" @@ -20,7 +23,89 @@ import ( "lokistack/services" ) +// version 由构建注入（-ldflags "-X main.version=..."），未注入时为 dev +var version = "2.0.0" + +// runCLI 提供不依赖 Web 的运维命令（备份 CronJob、恢复、自检都走它）。 +// +// lokistack backup --output /app/data/backups/xxx.tar.gz +// lokistack restore <备份包路径> # 校验 + 立即应用（会先备份当前 DB） +// lokistack self-check # 打印 JSON 自检结果 +func runCLI(args []string) { + switch args[0] { + case "backup": + out := "" + for i := 1; i < len(args); i++ { + if args[i] == "--output" && i+1 < len(args) { + out = args[i+1] + } + } + if out == "" { + out = "/app/data/backups/lokistack-" + time.Now().Format("20060102-150405") + ".tar.gz" + } + if err := os.MkdirAll(filepath.Dir(out), 0o700); err != nil { + fmt.Fprintf(os.Stderr, "创建备份目录失败: %v\n", err) + os.Exit(1) + } + f, err := os.Create(out) + if err != nil { + fmt.Fprintf(os.Stderr, "创建备份文件失败: %v\n", err) + os.Exit(1) + } + defer f.Close() + m, err := services.CreateBackup(f) + if err != nil { + fmt.Fprintf(os.Stderr, "备份失败: %v\n", err) + os.Exit(1) + } + fmt.Printf("备份完成: %s\n记录数: %d\n密钥指纹: %s\n", out, m.RecordCount, m.KeyFP) + + case "restore": + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "用法: lokistack restore <备份包.tar.gz>") + os.Exit(1) + } + f, err := os.Open(args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "打开备份失败: %v\n", err) + os.Exit(1) + } + defer f.Close() + res, err := services.ValidateAndStageBackup(f) + if err != nil { + fmt.Fprintf(os.Stderr, "校验失败: %v\n", err) + os.Exit(1) + } + if res.Warning != "" { + fmt.Fprintf(os.Stderr, "⚠️ %s\n", res.Warning) + } + path, err := services.ApplyStagedBackup() + if err != nil { + fmt.Fprintf(os.Stderr, "应用失败: %v\n", err) + os.Exit(1) + } + fmt.Printf("恢复完成: %s（备份 %d 条记录）\n请重启平台进程使新数据生效。\n", path, res.RowsInBackup) + + case "self-check": + b, _ := json.MarshalIndent(services.SelfCheck(), "", " ") + fmt.Println(string(b)) + + case "version": + fmt.Println(version) + + default: + fmt.Fprintf(os.Stderr, "未知命令 %q。可用: backup / restore / self-check / version\n", args[0]) + os.Exit(1) + } +} + func main() { + // 运维命令行：支持在容器/CronJob 里不经 Web 完成备份/恢复/自检 + if len(os.Args) > 1 { + runCLI(os.Args[1:]) + return + } + cfg := config.GetConfig() // gin 默认是 debug 模式：每次启动会把上百行路由表 dump 到日志里， @@ -35,6 +120,10 @@ func main() { log.Fatalf("Failed to init DB: %v", err) } + // 指标注册要早于任何可能埋点的组件（middleware / alert engine） + services.InitMetrics(version) + log.Println("Metrics enabled at /api/v1/metrics") + log.Println("Seeding default data...") seed.Seed() diff --git a/backend/middleware/audit.go b/backend/middleware/audit.go new file mode 100644 index 0000000..e254929 --- /dev/null +++ b/backend/middleware/audit.go @@ -0,0 +1,206 @@ +package middleware + +import ( + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + + "lokistack/models" + "lokistack/services" +) + +// AuditInfo 由 controller 主动补充的业务语义。 +// +// 中间件已经能自动记录 method/path/status/耗时/IP/用户， +// 但「改了哪个部署」「保留天数 7 → 3」这类信息只有 handler 自己知道， +// 所以提供 SetAudit 让 handler 补一刀，不补也能记全（只是 Detail 为空）。 +type AuditInfo struct { + Module string + Action string + Resource string + ResourceID string + Detail string + Before string + After string + // Skip 置 true 表示这条请求不记（例如纯查询用了 POST 的场景） + Skip bool +} + +const auditInfoKey = "audit.info" + +// SetAudit 由 controller 调用，补充审计语义。 +func SetAudit(c *gin.Context, info AuditInfo) { + c.Set(auditInfoKey, info) +} + +// auditSkipPaths 这些路径不记审计： +// - 健康检查 / metrics：高频噪音 +// - 日志查询：GET 本来就不记，但 /logs/query 之类用 POST 的也不记（只读） +var auditSkipPrefixes = []string{ + "/api/v1/health", + "/api/v1/metrics", + "/api/v1/logs/query", + "/api/v1/logs/stats", + "/api/v1/logs/histogram", + "/api/v1/logs/labels", + "/api/v1/logs/label", + "/api/v1/logs/context", + "/api/v1/logs/volume", + "/api/v1/logs/my-scope", + "/api/v1/deployments/", // 例外：其中 /deploy /stop /restart 要记，见下方白名单 +} + +// auditAlwaysRecord 匹配到的路径即使在上面的 skip 前缀里也要记。 +var auditAlwaysRecordSuffixes = []string{ + "/deploy", "/stop", "/restart", "/install", "/uninstall", "/test", "/validate", + "/preview", "/rollback", "/reset-password", "/backup", "/restore", +} + +// AuditLog 审计中间件：自动记录所有写操作（POST / PUT / PATCH / DELETE）。 +// +// 为什么用中间件而不是每个 controller 各写一段： +// - LokiStack 有 60+ 个写接口，逐个埋点必然漏，漏掉的那个恰好最危险（删集群、改存储） +// - 中间件能拿到统一的 status / 耗时 / IP / UA +// - handler 需要更细的语义时再 SetAudit 补，不补也有基础记录 +// +// 不含请求体（读 body 会影响后续 binding），敏感字段由 handler 通过 SetAudit +// 传入已脱敏的 Before/After。 +func AuditLog() gin.HandlerFunc { + return func(c *gin.Context) { + method := c.Request.Method + // 只读请求不记（GET/HEAD/OPTIONS） + if method == http.MethodGet || method == http.MethodHead || method == http.MethodOptions { + c.Next() + return + } + + path := c.Request.URL.Path + if !shouldAudit(path) { + c.Next() + return + } + + start := time.Now() + c.Next() + elapsed := time.Since(start).Milliseconds() + + info, _ := c.Get(auditInfoKey) + ai, _ := info.(AuditInfo) + if ai.Skip { + return + } + + status := c.Writer.Status() + userID, _ := c.Get("userId") + username, _ := c.Get("username") + role, _ := c.Get("role") + + entry := models.AuditLog{ + IP: c.ClientIP(), + UserAgent: c.Request.UserAgent(), + Method: method, + Path: path, + Status: status, + DurationMs: elapsed, + Detail: ai.Detail, + Before: ai.Before, + After: ai.After, + CreatedAt: time.Now(), + } + if v, ok := userID.(uint); ok { + entry.UserID = v + } + if v, ok := username.(string); ok { + entry.Username = v + } + if v, ok := role.(string); ok { + entry.Role = v + } + + // 失败时把错误信息带上（gin 的 Error 里有 handler 写入的错误） + if status >= 400 { + if len(c.Errors) > 0 { + entry.Error = c.Errors.String() + } + } + + // handler 没补语义时按路径推断，保证「谁删了什么」至少能看出模块 + module, action, resource, resourceID := inferAuditFields(method, path) + if ai.Module != "" { + module = ai.Module + } + if ai.Action != "" { + action = ai.Action + } + if ai.Resource != "" { + resource = ai.Resource + } + if ai.ResourceID != "" { + resourceID = ai.ResourceID + } + entry.Module, entry.Action, entry.Resource, entry.ResourceID = module, action, resource, resourceID + + services.RecordAudit(entry) + } +} + +func shouldAudit(path string) bool { + for _, suffix := range auditAlwaysRecordSuffixes { + if strings.HasSuffix(path, suffix) { + return true + } + } + for _, p := range auditSkipPrefixes { + if strings.HasPrefix(path, p) { + return false + } + } + return true +} + +// inferAuditFields 从 method + path 推断模块与动作（handler 未补充时的兜底）。 +func inferAuditFields(method, path string) (module, action, resource, resourceID string) { + action = map[string]string{ + http.MethodPost: "create", + http.MethodPut: "update", + http.MethodPatch: "update", + http.MethodDelete: "delete", + }[method] + + segments := strings.Split(strings.Trim(path, "/"), "/") + // /api/v1/// + if len(segments) >= 3 { + module = segments[2] + } + if len(segments) >= 4 { + resource = segments[3] + resourceID = segments[4] + } + + // 语义化覆盖 + switch { + case strings.HasSuffix(path, "/deploy"): + action, resource = "deploy", "部署" + case strings.HasSuffix(path, "/stop"): + action, resource = "stop", "部署" + case strings.HasSuffix(path, "/restart"): + action, resource = "restart", "部署" + case strings.HasSuffix(path, "/install"): + action, resource = "install", "采集器" + case strings.HasSuffix(path, "/uninstall"): + action, resource = "uninstall", "采集器" + case strings.HasSuffix(path, "/test"): + action = "test" + case strings.HasSuffix(path, "/validate"): + action = "validate" + case strings.HasSuffix(path, "/reset-password"): + action, resource = "reset-password", "用户" + case strings.HasSuffix(path, "/login"): + module, action, resource = "auth", "login", "登录" + case strings.HasSuffix(path, "/password"): + module, action, resource = "auth", "change-password", "登录" + } + return module, action, resource, resourceID +} diff --git a/backend/middleware/metrics.go b/backend/middleware/metrics.go new file mode 100644 index 0000000..2182781 --- /dev/null +++ b/backend/middleware/metrics.go @@ -0,0 +1,54 @@ +package middleware + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "lokistack/pkg/metrics" +) + +// Metrics 采集 HTTP 层指标。 +// +// 关键点：path 标签用 **c.FullPath()（路由模板）**而不是原始 URL。 +// 原始 URL 带 ID/UUID，每个值都会新建一条时间序列 → Prometheus 基数爆炸 +// （本平台有 117 个 Deployment，光 /deployments/:id 就能炸出上千条）。 +func Metrics() gin.HandlerFunc { + return func(c *gin.Context) { + // 抓取端点与健康检查不计入，否则监控自身的流量会污染 QPS + path := c.FullPath() + if path == "/api/v1/metrics" || path == "/api/v1/health" { + c.Next() + return + } + + metrics.IncInFlight() + start := time.Now() + c.Next() + metrics.DecInFlight() + + label := path + if label == "" { + // 未匹配到路由（404），统一归到一个标签，避免任意 URL 打爆基数 + label = "unmatched" + } + metrics.ObserveHTTP(c.Request.Method, label, c.Writer.Status(), time.Since(start).Seconds()) + } +} + +// statusClass 给日志/看板用的状态分类（保留工具函数） +func statusClass(code int) string { + switch { + case code < 300: + return "2xx" + case code < 400: + return "3xx" + case code < 500: + return "4xx" + default: + return "5xx" + } +} + +var _ = strconv.Itoa diff --git a/backend/models/audit.go b/backend/models/audit.go new file mode 100644 index 0000000..bfcea1c --- /dev/null +++ b/backend/models/audit.go @@ -0,0 +1,80 @@ +package models + +import "time" + +// AuditLog 操作审计日志。 +// +// 设计参考 /usr/local/src/aiops 的 audit_operation_logs（字段口径一致， +// 便于两套运维工具链用同一套查询习惯与看板），差异有两点： +// +// 1. 多了 Method / Path / Before / After —— aiops 用 gorm 表，LokiStack 用 JSON 文件， +// 没法靠 SQL 关联，所以把"改了什么"直接落到行上，查询时不用再回表。 +// 2. 不做 Response 全量落盘（Loki 日志响应动辄几百 KB，会撑爆 JSON DB）， +// 只留 Detail 摘要 + 结构化 diff。 +// +// 敏感字段（password / secretKey / privateKey / token）在写入前统一脱敏， +// 见 middleware.MaskAuditPayload。 +type AuditLog struct { + ID uint `json:"id"` + UserID uint `json:"userId"` + Username string `json:"username"` + Role string `json:"role,omitempty"` + IP string `json:"ip"` + // UserAgent 截断到 200 字符，避免个别客户端把 DB 撑大 + UserAgent string `json:"userAgent,omitempty"` + + // Module 业务模块：deployment / collector / storage / host / cluster / user / policy / alert / system / auth + Module string `json:"module"` + // Action 动作：create / update / delete / deploy / install / uninstall / login / logout / test / reset + Action string `json:"action"` + // Resource 资源类型（中文可读名，如「部署」「采集器」） + Resource string `json:"resource"` + // ResourceID 资源 ID 或名称 + ResourceID string `json:"resourceId,omitempty"` + + Method string `json:"method"` + Path string `json:"path"` + Status int `json:"status"` + // DurationMs 处理耗时（毫秒） + DurationMs int64 `json:"durationMs"` + + // Detail 人类可读的一句话说明（如「修改保留天数 7 → 3」） + Detail string `json:"detail,omitempty"` + // Before / After 变更前后摘要（仅记录变更字段，已脱敏） + Before string `json:"before,omitempty"` + After string `json:"after,omitempty"` + // Error 失败原因（status >= 400 时填） + Error string `json:"error,omitempty"` + + CreatedAt time.Time `json:"createdAt"` +} + +// AuditQuery 审计日志查询条件 +type AuditQuery struct { + Username string + Module string + Action string + ResourceID string + // Status 0 = 全部；1 = 仅成功（2xx）；2 = 仅失败（>=400） + Status int + StartTime *time.Time + EndTime *time.Time + Page int + PageSize int +} + +// AuditStats 审计统计（给审计页面顶部卡片用） +type AuditStats struct { + TodayCount int64 `json:"todayCount"` + WeekCount int64 `json:"weekCount"` + FailCount int64 `json:"failCount"` + TotalCount int64 `json:"totalCount"` + TopModules []AuditCount `json:"topModules"` + TopUsers []AuditCount `json:"topUsers"` +} + +// AuditCount 分组计数 +type AuditCount struct { + Name string `json:"name"` + Count int64 `json:"count"` +} diff --git a/backend/pkg/metrics/metrics.go b/backend/pkg/metrics/metrics.go new file mode 100644 index 0000000..0c0f7f7 --- /dev/null +++ b/backend/pkg/metrics/metrics.go @@ -0,0 +1,175 @@ +// Package metrics 定义 LokiStack 自身的 Prometheus 指标体系。 +// +// 为什么单独成包：这些指标要同时被 repository（DB 落盘耗时）和 +// services（业务计数）以及 middleware（HTTP）使用， +// 而 services 依赖 repository —— 指标定义必须放在最底层，否则循环依赖。 +package metrics + +import ( + "net/http" + "runtime" + "strconv" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + // HTTP 层 + HTTPRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "lokistack_http_requests_total", + Help: "HTTP 请求总数（按方法/路由模板/状态码）", + }, + []string{"method", "path", "status"}, + ) + HTTPRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "lokistack_http_request_duration_seconds", + Help: "HTTP 请求耗时分布", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }, + []string{"method", "path"}, + ) + HTTPInFlight = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "lokistack_http_in_flight_requests", + Help: "当前正在处理的请求数", + }, + ) + + // 存储层（JSON DB 是平台最大风险点：全量读内存、全量写盘） + DBSizeBytes = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "lokistack_db_size_bytes", + Help: "JSON 数据库文件大小", + }, + ) + DBSaveDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "lokistack_db_save_duration_seconds", + Help: "数据库落盘耗时", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 3, 5, 10}, + }, + ) + DBSaveTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "lokistack_db_save_total", + Help: "数据库保存次数（按结果）", + }, + []string{"result"}, + ) + DBRecordsTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "lokistack_db_records", + Help: "各集合的记录数", + }, + []string{"collection"}, + ) + + // 业务层 + DeploymentsTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "lokistack_deployments", + Help: "Loki 部署数（按状态/运行模式）", + }, + []string{"status", "mode"}, + ) + CollectorsTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "lokistack_collectors", + Help: "采集器数（按状态/类型/托管方式）", + }, + []string{"status", "type", "run_mode"}, + ) + + // 后台任务层 + AlertEvaluationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "lokistack_alert_evaluations_total", + Help: "告警规则评估次数（按结果）", + }, + []string{"result"}, + ) + AlertEvaluationDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "lokistack_alert_evaluation_duration_seconds", + Help: "单轮告警评估耗时", + Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 3, 5, 10, 30}, + }, + ) + LokiQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "lokistack_loki_queries_total", + Help: "对后端 Loki 的查询次数（按结果）", + }, + []string{"result"}, + ) + + BuildInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "lokistack_build_info", + Help: "构建信息（恒为 1，标签里带版本与 Go 版本）", + }, + []string{"version", "go_version"}, + ) + + once sync.Once +) + +// Register 注册全部指标（幂等）。version 一般传 main.version。 +func Register(version string) { + once.Do(func() { + prometheus.MustRegister( + HTTPRequestsTotal, HTTPRequestDuration, HTTPInFlight, + DBSizeBytes, DBSaveDuration, DBSaveTotal, DBRecordsTotal, + DeploymentsTotal, CollectorsTotal, + AlertEvaluationsTotal, AlertEvaluationDuration, LokiQueriesTotal, + BuildInfo, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + BuildInfo.WithLabelValues(version, runtime.Version()).Set(1) + }) +} + +// Handler 返回 Prometheus 抓取端点。 +func Handler() http.Handler { return promhttp.Handler() } + +// ── 埋点辅助（各层直接调） ── + +// ObserveHTTP 记录一次 HTTP 请求。pathLabel 必须是路由模板，不能用原始 URL。 +func ObserveHTTP(method, pathLabel string, status int, seconds float64) { + HTTPRequestsTotal.WithLabelValues(method, pathLabel, strconv.Itoa(status)).Inc() + HTTPRequestDuration.WithLabelValues(method, pathLabel).Observe(seconds) +} + +// IncInFlight / DecInFlight 在途请求计数。 +func IncInFlight() { HTTPInFlight.Inc() } +func DecInFlight() { HTTPInFlight.Dec() } + +// ObserveDBSave 记录一次落盘结果。 +func ObserveDBSave(seconds float64, ok bool) { + DBSaveDuration.Observe(seconds) + DBSaveTotal.WithLabelValues(resultLabel(ok)).Inc() +} + +// ObserveAlertEvaluation 记录一轮告警评估。 +func ObserveAlertEvaluation(seconds float64, ok bool) { + AlertEvaluationDuration.Observe(seconds) + AlertEvaluationsTotal.WithLabelValues(resultLabel(ok)).Inc() +} + +// ObserveLokiQuery 记录一次后端 Loki 查询。 +func ObserveLokiQuery(ok bool) { + LokiQueriesTotal.WithLabelValues(resultLabel(ok)).Inc() +} + +func resultLabel(ok bool) string { + if ok { + return "success" + } + return "error" +} diff --git a/backend/repository/db.go b/backend/repository/db.go index 644bf46..596f95c 100644 --- a/backend/repository/db.go +++ b/backend/repository/db.go @@ -3,6 +3,7 @@ package repository import ( "encoding/json" "fmt" + "log" "os" "path/filepath" "sync" @@ -10,6 +11,7 @@ import ( "lokistack/config" "lokistack/models" + "lokistack/pkg/metrics" ) // saveRequest 表示一个保存请求，通过channel序列化 @@ -35,6 +37,14 @@ type JsonDB struct { AlertDeliveries []models.AlertDelivery `json:"alertDeliveries"` NotifyChannels []models.NotifyChannel `json:"notifyChannels"` LogPolicyGroups []models.LogPolicyGroup `json:"logPolicyGroups"` + // AuditLogs 操作审计日志（保留期 90 天 / 上限 2 万条，见 services.auditRetention） + AuditLogs []models.AuditLog `json:"auditLogs"` + + // SchemaVersion 数据结构版本号。 + // 之前加字段全靠 JSON 反序列化的零值兜底，字段一多必然出事 + // （例如新增 ScrapeJobs 时老数据要手写兼容）。有了版本号， + // 以后的结构变更可以在 InitDB 里按版本号顺序迁移，可测、可回滚。 + SchemaVersion int `json:"schemaVersion"` sequences map[string]uint // 防抖保存机制 @@ -51,6 +61,41 @@ func GetDB() *JsonDB { return db } +// CurrentSchemaVersion 当前数据结构版本。 +// 每次做**不兼容**的结构变更时 +1，并在 migrateSchema 里补一段迁移逻辑。 +// +// v0/v1（无字段）→ 1：新增 auditLogs / schemaVersion（老库走零值兜底即可，无需处理） +// v1 → 2：预留示例（如把 ScrapeJobs 从 logPath 拆分时，需按旧结构生成默认 job） +const CurrentSchemaVersion = 1 + +// migrateSchema 按版本号顺序迁移老库。返回是否发生了迁移（需要回写）。 +func (d *JsonDB) migrateSchema() bool { + if d.SchemaVersion >= CurrentSchemaVersion { + return false + } + from := d.SchemaVersion + + // 逐版本递进，每段只处理"上一版 → 下一版"的差异， + // 不要写成 if version < X 的叠加判断，否则分支会指数增长。 + for d.SchemaVersion < CurrentSchemaVersion { + switch d.SchemaVersion { + case 0: + // v0 → v1：补审计表；老库没有 auditLogs 字段，反序列化后为 nil， + // 这里显式初始化，避免后续 append 到 nil 之外还留着 nil 判断。 + if d.AuditLogs == nil { + d.AuditLogs = []models.AuditLog{} + } + d.SchemaVersion = 1 + default: + // 防御：出现未知版本（比当前代码还新）时不乱动，避免降级把数据写坏 + log.Printf("[schema] 数据库版本 %d 高于当前代码支持版本 %d，跳过迁移", d.SchemaVersion, CurrentSchemaVersion) + return false + } + } + log.Printf("[schema] 数据结构已从 v%d 迁移到 v%d", from, d.SchemaVersion) + return true +} + func InitDB() error { cfg := config.GetConfig() _ = os.MkdirAll(cfg.DataDir, 0700) @@ -70,6 +115,8 @@ func InitDB() error { AlertDeliveries: []models.AlertDelivery{}, NotifyChannels: []models.NotifyChannel{}, LogPolicyGroups: []models.LogPolicyGroup{}, + AuditLogs: []models.AuditLog{}, + SchemaVersion: CurrentSchemaVersion, sequences: map[string]uint{ "users": 0, "clusters": 0, @@ -131,12 +178,14 @@ func InitDB() error { } migratedLogAccess := migrateLogAccessToGroups(d) + // 结构版本迁移（审计表、采集项等新字段的兼容处理） + migratedSchema := d.migrateSchema() // 启动序列化保存goroutine d.wg.Add(1) go d.saveWorker() db = d - if migratedLogAccess { + if migratedLogAccess || migratedSchema { d.RequestSave() } return nil @@ -247,6 +296,14 @@ func (d *JsonDB) saveWorker() { // doSave 执行实际的原子文件写入 func (d *JsonDB) doSave() error { + start := time.Now() + err := d.doSaveInner() + // JSON DB 是全量写盘，落盘耗时是平台最主要的延迟来源之一，必须可观测 + metrics.ObserveDBSave(time.Since(start).Seconds(), err == nil) + return err +} + +func (d *JsonDB) doSaveInner() error { d.mu.RLock() data, err := json.MarshalIndent(d, "", " ") d.mu.RUnlock() @@ -275,6 +332,7 @@ func (d *JsonDB) doSave() error { os.Remove(tmpFile) return err } + metrics.DBSizeBytes.Set(float64(len(data))) return nil } diff --git a/backend/routers/router.go b/backend/routers/router.go index 0c7168a..1d406b3 100644 --- a/backend/routers/router.go +++ b/backend/routers/router.go @@ -9,6 +9,7 @@ import ( "lokistack/controllers" "lokistack/middleware" + "lokistack/pkg/metrics" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" @@ -20,7 +21,9 @@ func SetupRouter() *gin.Engine { // 按字段过滤，同时跳过健康检查/静态资源，避免探针轮询把日志量刷满。 // Recovery 必须显式加回来（Default 里包含它），否则 panic 会打挂进程。 r := gin.New() - r.Use(middleware.AccessLog(), gin.Recovery()) + // AuditLog 放在最外层：它依赖的 userId/username 由 group 内的 JWTAuth 写入， + // 但中间件的后置逻辑在 c.Next() 返回后才执行，那时已就绪（见 middleware/audit.go）。 + r.Use(middleware.AccessLog(), middleware.AuditLog(), middleware.Metrics(), gin.Recovery()) // CORS cfg := cors.DefaultConfig() @@ -40,6 +43,8 @@ func SetupRouter() *gin.Engine { ldapCtrl := controllers.NewLdapController() logsCtrl := controllers.NewLogsController() alertsCtrl := controllers.NewAlertsController() + auditCtrl := controllers.NewAuditController() + systemCtrl := controllers.NewSystemController() registryCtrl := controllers.NewImageRegistryController() // API v1 @@ -54,12 +59,9 @@ func SetupRouter() *gin.Engine { }) }) - // Prometheus metrics 端点（公开访问，可被Prometheus抓取） - api.GET("/metrics", func(c *gin.Context) { - // 基础metrics输出（符合Prometheus文本格式） - c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") - c.String(http.StatusOK, "# LokiStack Metrics\n") - }) + // Prometheus metrics 端点（真实指标，见 services/metrics.go） + // 原实现只返回一行注释却声称"可被 Prometheus 抓取"，平台自身等于不可观测。 + api.GET("/metrics", gin.WrapH(metrics.Handler())) // Public（登录接口挂 IP 限流防爆破：20 次/分钟） api.POST("/auth/login", middleware.LoginRateLimit(20, time.Minute), authCtrl.Login) @@ -127,6 +129,26 @@ func SetupRouter() *gin.Engine { authorized.POST("/settings/ldap/users", middleware.AdminAuth(), ldapCtrl.ListUsers) // Users（admin 专用） + // 系统运维：备份 / 恢复 / 一键自检（仅管理员） + system := authorized.Group("/system") + system.Use(middleware.AdminAuth()) + { + system.GET("/backup", systemCtrl.Backup) + system.POST("/restore", systemCtrl.Restore) + system.POST("/restore/apply", systemCtrl.RestoreApply) + system.GET("/backups", systemCtrl.Backups) + system.GET("/self-check", systemCtrl.Health) + } + + // 审计日志：仅管理员可查（含他人操作记录） + audit := authorized.Group("/audit-logs") + audit.Use(middleware.AdminAuth()) + { + audit.GET("", auditCtrl.List) + audit.GET("/stats", auditCtrl.Stats) + audit.GET("/:id", auditCtrl.Get) + } + users := authorized.Group("/users") users.Use(middleware.AdminAuth()) { diff --git a/backend/services/alert.go b/backend/services/alert.go index 26581b3..74b48ae 100644 --- a/backend/services/alert.go +++ b/backend/services/alert.go @@ -12,6 +12,7 @@ import ( "time" "lokistack/models" + "lokistack/pkg/metrics" "lokistack/repository" ) @@ -27,7 +28,19 @@ func StartAlertEngine() { alertTicker = time.NewTicker(alertCheckInterval) go func() { for range alertTicker.C { - RunAlertCheck(context.Background()) + start := time.Now() + // 告警引擎是后台 goroutine：卡死或 panic 时页面完全看不出来， + // 只能靠这两个指标发现（配合 Grafana 告警） + ok := true + func() { + defer func() { + if r := recover(); r != nil { + ok = false + } + }() + RunAlertCheck(context.Background()) + }() + metrics.ObserveAlertEvaluation(time.Since(start).Seconds(), ok) } }() } diff --git a/backend/services/audit.go b/backend/services/audit.go new file mode 100644 index 0000000..a7d4ef8 --- /dev/null +++ b/backend/services/audit.go @@ -0,0 +1,309 @@ +package services + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "lokistack/models" + "lokistack/repository" +) + +const ( + // maxAuditLogs 审计日志条数上限。JSON DB 是全量读进内存、全量写盘， + // 不设上限的话审计表会把内存和落盘时间一起拖垮。 + maxAuditLogs = 20000 + // auditRetention 审计日志保留天数（合规常见要求 90 天） + auditRetention = 90 * 24 * time.Hour +) + +// RecordAudit 写入一条审计日志。 +// +// 并发安全：直接拿 db 锁追加，与其它业务写入同一把锁，避免 RequestSave 时丢更新。 +// 裁剪：超过条数上限或保留期时从最早开始删。 +func RecordAudit(entry models.AuditLog) { + db := repository.GetDB() + if db == nil { + return + } + db.Lock() + defer db.Unlock() + + if entry.CreatedAt.IsZero() { + entry.CreatedAt = time.Now() + } + // 单字段长度保护：UA/Detail 可能来自客户端，不能无限制落盘 + entry.UserAgent = truncateRunes(entry.UserAgent, 200) + entry.Detail = truncateRunes(entry.Detail, 500) + entry.Before = truncateRunes(entry.Before, 1000) + entry.After = truncateRunes(entry.After, 1000) + entry.Error = truncateRunes(entry.Error, 500) + entry.ResourceID = truncateRunes(entry.ResourceID, 128) + + nextID := uint(1) + for _, a := range db.AuditLogs { + if a.ID >= nextID { + nextID = a.ID + 1 + } + } + entry.ID = nextID + db.AuditLogs = append(db.AuditLogs, entry) + + pruneAuditLocked(db) + db.RequestSave() +} + +// pruneAuditLocked 裁剪审计日志（调用方必须已持锁）。 +// 从头部（最早）开始丢，直到满足条数与保留期两条约束。 +func pruneAuditLocked(db *repository.JsonDB) { + logs := db.AuditLogs + cut := 0 + deadline := time.Now().Add(-auditRetention) + + // 1) 先按保留期裁 + for cut < len(logs) && logs[cut].CreatedAt.Before(deadline) { + cut++ + } + // 2) 再按条数上限裁（保留最新的 maxAuditLogs 条） + if remain := len(logs) - cut; remain > maxAuditLogs { + cut += remain - maxAuditLogs + } + if cut > 0 { + db.AuditLogs = append([]models.AuditLog{}, logs[cut:]...) + } +} + +// ListAuditLogs 分页查询审计日志（按时间倒序）。 +func ListAuditLogs(q models.AuditQuery) ([]models.AuditLog, int64) { + db := repository.GetDB() + if db == nil { + return nil, 0 + } + db.RLock() + all := db.AuditLogs + db.RUnlock() + + matched := make([]models.AuditLog, 0, len(all)) + for _, a := range all { + if q.Username != "" && !strings.Contains(strings.ToLower(a.Username), strings.ToLower(q.Username)) { + continue + } + if q.Module != "" && a.Module != q.Module { + continue + } + if q.Action != "" && a.Action != q.Action { + continue + } + if q.ResourceID != "" && !strings.Contains(a.ResourceID, q.ResourceID) { + continue + } + switch q.Status { + case 1: + if a.Status < 200 || a.Status >= 300 { + continue + } + case 2: + if a.Status < 400 { + continue + } + } + if q.StartTime != nil && a.CreatedAt.Before(*q.StartTime) { + continue + } + if q.EndTime != nil && a.CreatedAt.After(*q.EndTime) { + continue + } + matched = append(matched, a) + } + + // 时间倒序 + sort.Slice(matched, func(i, j int) bool { return matched[i].CreatedAt.After(matched[j].CreatedAt) }) + + total := int64(len(matched)) + if q.PageSize <= 0 { + q.PageSize = 20 + } + if q.Page <= 0 { + q.Page = 1 + } + start := (q.Page - 1) * q.PageSize + if start >= len(matched) { + return []models.AuditLog{}, total + } + end := start + q.PageSize + if end > len(matched) { + end = len(matched) + } + return matched[start:end], total +} + +// GetAuditLog 取单条（详情弹窗用）。 +func GetAuditLog(id uint) (*models.AuditLog, error) { + db := repository.GetDB() + if db == nil { + return nil, fmt.Errorf("数据库未初始化") + } + db.RLock() + defer db.RUnlock() + for i := range db.AuditLogs { + if db.AuditLogs[i].ID == id { + entry := db.AuditLogs[i] + return &entry, nil + } + } + return nil, fmt.Errorf("审计记录不存在") +} + +// AuditStatistics 统计卡片数据。 +func AuditStatistics() models.AuditStats { + db := repository.GetDB() + stats := models.AuditStats{} + if db == nil { + return stats + } + db.RLock() + logs := db.AuditLogs + db.RUnlock() + + now := time.Now() + todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + weekStart := todayStart.AddDate(0, 0, -6) + + moduleCount := map[string]int64{} + userCount := map[string]int64{} + for _, a := range logs { + stats.TotalCount++ + if a.CreatedAt.After(todayStart) { + stats.TodayCount++ + } + if a.CreatedAt.After(weekStart) { + stats.WeekCount++ + } + if a.Status >= 400 { + stats.FailCount++ + } + moduleCount[a.Module]++ + if a.Username != "" { + userCount[a.Username]++ + } + } + stats.TopModules = topN(moduleCount, 5) + stats.TopUsers = topN(userCount, 5) + return stats +} + +func topN(m map[string]int64, n int) []models.AuditCount { + out := make([]models.AuditCount, 0, len(m)) + for k, v := range m { + out = append(out, models.AuditCount{Name: k, Count: v}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count == out[j].Count { + return out[i].Name < out[j].Name + } + return out[i].Count > out[j].Count + }) + if len(out) > n { + out = out[:n] + } + return out +} + +// ───────────────────────── 脱敏 / 摘要 ───────────────────────── + +// auditSensitiveKeys 这些字段一律不进审计（只留「已修改」的痕迹）。 +var auditSensitiveKeys = map[string]bool{ + "password": true, "oldpassword": true, "newpassword": true, + "secretkey": true, "accesskey": true, "secretaccesskey": true, + "privatekey": true, "token": true, "accesstoken": true, "refreshtoken": true, + "encryptionkey": true, "jwtsecret": true, +} + +// MaskAuditPayload 把请求体里的敏感值替换成 ***，返回脱敏后的 JSON 字符串。 +// 解析失败时返回 "<非 JSON 请求体>"，避免把表单里的密码原样落盘。 +func MaskAuditPayload(body []byte, maxLen int) string { + if len(body) == 0 { + return "" + } + var v interface{} + if err := json.Unmarshal(body, &v); err != nil { + return "<非 JSON 请求体，已省略>" + } + masked := maskValue(v) + out, err := json.Marshal(masked) + if err != nil { + return "" + } + return truncateRunes(string(out), maxLen) +} + +func maskValue(v interface{}) interface{} { + switch t := v.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(t)) + for k, val := range t { + if auditSensitiveKeys[strings.ToLower(k)] { + if s, ok := val.(string); ok && s == "" { + out[k] = "" + } else { + out[k] = "***" + } + continue + } + out[k] = maskValue(val) + } + return out + case []interface{}: + out := make([]interface{}, len(t)) + for i, val := range t { + out[i] = maskValue(val) + } + return out + default: + return v + } +} + +// AuditDiff 生成「变更前后」摘要，只保留真正变化的字段。 +// prev/next 传脱敏后的 JSON 字符串；解析失败时返回空（不硬编码原文）。 +func AuditDiff(prev, next string) (string, string) { + var a, b map[string]interface{} + if prev == "" || next == "" || json.Unmarshal([]byte(prev), &a) != nil || json.Unmarshal([]byte(next), &b) != nil { + return "", "" + } + before := map[string]interface{}{} + after := map[string]interface{}{} + for k, nv := range b { + ov, existed := a[k] + if !existed || fmt.Sprint(ov) != fmt.Sprint(nv) { + if existed { + before[k] = ov + } + after[k] = nv + } + } + for k, ov := range a { + if _, ok := b[k]; !ok { + before[k] = ov + } + } + if len(after) == 0 && len(before) == 0 { + return "", "" + } + bs, _ := json.Marshal(before) + as, _ := json.Marshal(after) + return truncateRunes(string(bs), 1000), truncateRunes(string(as), 1000) +} + +func truncateRunes(s string, n int) string { + if n <= 0 { + return s + } + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/backend/services/backup.go b/backend/services/backup.go new file mode 100644 index 0000000..4b7d2d8 --- /dev/null +++ b/backend/services/backup.go @@ -0,0 +1,337 @@ +package services + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "lokistack/repository" +) + +// 备份/恢复设计要点 +// +// 为什么必须有：平台的全部状态（部署/采集/存储/集群/主机/策略/告警/用户/审计） +// 都在这一个 JSON 文件里；kubeconfig 虽然加密存在 DB 内，但解密依赖 +// ENCRYPTION_KEY（K8s Secret）。DB 丢了 = 平台失去"这些资源是谁建的"的映射， +// 从此无法管理/卸载/升级正在跑的 Loki 与采集器。 +// +// 安全取舍： +// - 备份包内含加密后的 kubeconfig 与 known_hosts，**不含明文密钥** +// - manifest 里只记录 ENCRYPTION_KEY 的指纹（sha256 前 16 位）， +// 用于恢复时校验"这把钥匙能不能解开这份备份"，不泄露密钥本身 +// - 恢复分两步（校验暂存 → 手动应用），避免一个误点就覆盖线上 DB + +// BackupManifest 备份包元信息（放在包内 manifest.json） +type BackupManifest struct { + Version string `json:"version"` + CreatedAt time.Time `json:"createdAt"` + DBBytes int64 `json:"dbBytes"` + DBSHA256 string `json:"dbSha256"` + KeyFP string `json:"encryptionKeyFingerprint"` + RecordCount int `json:"recordCount"` + Note string `json:"note"` +} + +// backupExclude 备份时跳过的文件/目录（临时文件、lost+found、正在写的 restore 暂存） +var backupExclude = map[string]bool{ + "lost+found": true, ".restore-pending": true, "backups": true, +} + +// CreateBackup 把 /app/data 打包成 tar.gz 写入 w。 +func CreateBackup(w io.Writer) (*BackupManifest, error) { + dataDir := dataDirPath() + dbPath := databasePath() + + dbInfo, err := os.Stat(dbPath) + if err != nil { + return nil, fmt.Errorf("读取数据库失败: %w", err) + } + dbBytes, err := os.ReadFile(dbPath) + if err != nil { + return nil, fmt.Errorf("读取数据库失败: %w", err) + } + sum := sha256.Sum256(dbBytes) + + db := repository.GetDB() + recordCount := 0 + if db != nil { + db.RLock() + recordCount = len(db.Users) + len(db.Clusters) + len(db.Hosts) + len(db.Storages) + + len(db.Deployments) + len(db.Collectors) + len(db.LogSources) + + len(db.AlertRules) + len(db.NotifyChannels) + len(db.LogPolicyGroups) + len(db.AuditLogs) + db.RUnlock() + } + + manifest := &BackupManifest{ + Version: "2.0.0", + CreatedAt: time.Now(), + DBBytes: dbInfo.Size(), + DBSHA256: hex.EncodeToString(sum[:]), + KeyFP: encryptionKeyFingerprint(), + RecordCount: recordCount, + Note: "包内 kubeconfig 为加密态；恢复必须使用同一把 ENCRYPTION_KEY，否则 kubeconfig 无法解密", + } + + gz := gzip.NewWriter(w) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + + // 1) manifest + mBytes, _ := json.MarshalIndent(manifest, "", " ") + if err := writeTarFile(tw, "manifest.json", mBytes, 0o600); err != nil { + return nil, err + } + + // 2) data 目录下所有文件（DB + kubeconfigs/ + known_hosts/） + err = filepath.Walk(dataDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil // 单个文件读不到不阻断整包 + } + if path == dataDir { + return nil + } + rel, err := filepath.Rel(dataDir, path) + if err != nil { + return nil + } + top := strings.SplitN(rel, string(os.PathSeparator), 2)[0] + if backupExclude[top] { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if info.IsDir() { + return nil + } + // 跳过正在写的临时文件 + if strings.HasPrefix(filepath.Base(path), ".lokistack-") && strings.HasSuffix(path, ".tmp") { + return nil + } + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + hdr := &tar.Header{ + Name: filepath.ToSlash(filepath.Join("data", rel)), + Mode: int64(info.Mode().Perm()), + Size: info.Size(), + ModTime: info.ModTime(), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + _, err = io.Copy(tw, f) + return err + }) + if err != nil { + return nil, err + } + return manifest, nil +} + +func writeTarFile(tw *tar.Writer, name string, data []byte, mode int64) error { + hdr := &tar.Header{Name: name, Mode: mode, Size: int64(len(data)), ModTime: time.Now()} + if err := tw.WriteHeader(hdr); err != nil { + return err + } + _, err := tw.Write(data) + return err +} + +// RestoreResult 校验结果 +type RestoreResult struct { + Manifest *BackupManifest `json:"manifest"` + KeyMatches bool `json:"keyMatches"` + RowsInBackup int `json:"rowsInBackup"` + CurrentRows int `json:"currentRows"` + StagedPath string `json:"stagedPath,omitempty"` + Warning string `json:"warning,omitempty"` +} + +// ValidateAndStageBackup 校验备份包并暂存到 /.restore-pending/。 +// +// 不在这一步直接覆盖线上 DB：恢复是不可逆操作，必须让人再确认一次 +// （页面上的「应用恢复」或 run.sh restore）。 +func ValidateAndStageBackup(r io.Reader) (*RestoreResult, error) { + gr, err := gzip.NewReader(r) + if err != nil { + return nil, fmt.Errorf("不是合法的 gzip 文件: %w", err) + } + defer gr.Close() + tr := tar.NewReader(gr) + + var ( + manifest *BackupManifest + dbBytes []byte + ) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("解包失败: %w", err) + } + switch { + case hdr.Name == "manifest.json": + b, _ := io.ReadAll(io.LimitReader(tr, 1<<20)) + var m BackupManifest + if err := json.Unmarshal(b, &m); err == nil { + manifest = &m + } + case hdr.Name == "data/opsxloki.json": + dbBytes, _ = io.ReadAll(io.LimitReader(tr, 64<<20)) + } + } + if manifest == nil { + return nil, fmt.Errorf("备份包缺少 manifest.json，不是本平台导出的备份") + } + if len(dbBytes) == 0 { + return nil, fmt.Errorf("备份包内没有 data/opsxloki.json") + } + + // 校验 DB 完整性：必须是合法 JSON 且能解析出预期集合 + var probe map[string]json.RawMessage + if err := json.Unmarshal(dbBytes, &probe); err != nil { + return nil, fmt.Errorf("备份内的数据库不是合法 JSON: %w", err) + } + for _, required := range []string{"users", "deployments", "collectors"} { + if _, ok := probe[required]; !ok { + return nil, fmt.Errorf("备份内数据库缺少集合 %q，可能不是 LokiStack 的备份", required) + } + } + + // 哈希校验（防止传输损坏） + sum := sha256.Sum256(dbBytes) + if manifest.DBSHA256 != "" && hex.EncodeToString(sum[:]) != manifest.DBSHA256 { + return nil, fmt.Errorf("备份内数据库校验失败（sha256 不匹配），文件可能已损坏") + } + + stagedDir := filepath.Join(dataDirPath(), ".restore-pending") + if err := os.MkdirAll(stagedDir, 0o700); err != nil { + return nil, fmt.Errorf("创建暂存目录失败: %w", err) + } + stagedPath := filepath.Join(stagedDir, "opsxloki.json") + if err := os.WriteFile(stagedPath, dbBytes, 0o600); err != nil { + return nil, fmt.Errorf("写入暂存文件失败: %w", err) + } + + currentRows := 0 + if db := repository.GetDB(); db != nil { + db.RLock() + currentRows = len(db.Deployments) + len(db.Collectors) + len(db.Clusters) + len(db.Hosts) + db.RUnlock() + } + + res := &RestoreResult{ + Manifest: manifest, + KeyMatches: manifest.KeyFP == "" || manifest.KeyFP == encryptionKeyFingerprint(), + RowsInBackup: manifest.RecordCount, + CurrentRows: currentRows, + StagedPath: stagedPath, + } + if !res.KeyMatches { + res.Warning = "备份的 ENCRYPTION_KEY 指纹与当前平台不一致：恢复后 kubeconfig / 存储凭证将无法解密，" + + "需要用备份时的那把 ENCRYPTION_KEY 覆盖 lokistack-secrets 中的值后再启动" + } + return res, nil +} + +// encryptionKeyFingerprint 返回当前 ENCRYPTION_KEY 的指纹（sha256 前 16 位），不泄露密钥。 +func encryptionKeyFingerprint() string { + key := os.Getenv("ENCRYPTION_KEY") + if key == "" { + // 兼容从文件读取的部署方式 + if p := os.Getenv("ENCRYPTION_KEY_FILE"); p != "" { + if b, err := os.ReadFile(p); err == nil { + key = strings.TrimSpace(string(b)) + } + } + } + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(key)) + return hex.EncodeToString(sum[:])[:16] +} + +// ApplyStagedBackup 把暂存文件应用为正式 DB（调用方负责重启进程）。 +func ApplyStagedBackup() (string, error) { + staged := filepath.Join(dataDirPath(), ".restore-pending", "opsxloki.json") + b, err := os.ReadFile(staged) + if err != nil { + return "", fmt.Errorf("没有待应用的恢复文件: %w", err) + } + var probe map[string]json.RawMessage + if err := json.Unmarshal(b, &probe); err != nil { + return "", fmt.Errorf("暂存文件已损坏: %w", err) + } + + dbPath := databasePath() + // 先把当前 DB 备份一份，恢复错了还能退回去 + if cur, err := os.ReadFile(dbPath); err == nil { + rollback := dbPath + ".before-restore." + time.Now().Format("20060102-150405") + _ = os.WriteFile(rollback, cur, 0o600) + } + tmp := dbPath + ".restoring" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return "", err + } + if err := os.Rename(tmp, dbPath); err != nil { + return "", err + } + _ = os.Remove(staged) + return dbPath, nil +} + +// ListBackups 列出 dataDir/backups 下已有的备份文件（run.sh backup 产出）。 +func ListBackups() []map[string]interface{} { + dir := filepath.Join(dataDirPath(), "backups") + entries, err := os.ReadDir(dir) + if err != nil { + return []map[string]interface{}{} + } + type item struct { + name string + size int64 + mod time.Time + } + var items []item + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".tar.gz") { + continue + } + info, err := e.Info() + if err != nil { + continue + } + items = append(items, item{e.Name(), info.Size(), info.ModTime()}) + } + sort.Slice(items, func(i, j int) bool { return items[i].mod.After(items[j].mod) }) + out := make([]map[string]interface{}, 0, len(items)) + for _, it := range items { + out = append(out, map[string]interface{}{ + "name": it.name, "sizeBytes": it.size, "createdAt": it.mod, + }) + } + return out +} + +func dataDirPath() string { + if p := os.Getenv("DATA_DIR"); p != "" { + return p + } + return "/app/data" +} diff --git a/backend/services/metrics.go b/backend/services/metrics.go new file mode 100644 index 0000000..7c7bce1 --- /dev/null +++ b/backend/services/metrics.go @@ -0,0 +1,97 @@ +package services + +import ( + "os" + "time" + + "lokistack/pkg/metrics" + "lokistack/repository" +) + +// InitMetrics 注册指标并启动业务/存储指标的周期刷新。 +// +// HTTP 层与 DB 落盘由 middleware / repository 直接埋点； +// 这里只负责需要"扫 DB 算出来"的指标（记录数、部署/采集器状态分布、DB 大小）。 +func InitMetrics(version string) { + metrics.Register(version) + + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + refreshGauges() + for range ticker.C { + refreshGauges() + } + }() +} + +// refreshGauges 刷新需要遍历 DB 才能得出的指标。 +func refreshGauges() { + db := repository.GetDB() + if db == nil { + return + } + + if path := databasePath(); path != "" { + if fi, err := os.Stat(path); err == nil { + metrics.DBSizeBytes.Set(float64(fi.Size())) + } + } + + db.RLock() + users := len(db.Users) + clusters := len(db.Clusters) + hosts := len(db.Hosts) + storages := len(db.Storages) + deployList := db.Deployments + collectorList := db.Collectors + alertRules := len(db.AlertRules) + auditCount := len(db.AuditLogs) + db.RUnlock() + + metrics.DBRecordsTotal.WithLabelValues("users").Set(float64(users)) + metrics.DBRecordsTotal.WithLabelValues("clusters").Set(float64(clusters)) + metrics.DBRecordsTotal.WithLabelValues("hosts").Set(float64(hosts)) + metrics.DBRecordsTotal.WithLabelValues("storages").Set(float64(storages)) + metrics.DBRecordsTotal.WithLabelValues("deployments").Set(float64(len(deployList))) + metrics.DBRecordsTotal.WithLabelValues("collectors").Set(float64(len(collectorList))) + metrics.DBRecordsTotal.WithLabelValues("alertRules").Set(float64(alertRules)) + metrics.DBRecordsTotal.WithLabelValues("auditLogs").Set(float64(auditCount)) + + // 状态分布：先 Reset 再重算，避免已删除对象留下僵尸时间序列 + metrics.DeploymentsTotal.Reset() + for _, d := range deployList { + status := d.Status + if status == "" { + status = "unknown" + } + mode := d.Mode + if mode == "" { + mode = "unknown" + } + metrics.DeploymentsTotal.WithLabelValues(status, mode).Inc() + } + + metrics.CollectorsTotal.Reset() + for _, c := range collectorList { + status := c.Status + if status == "" { + status = "unknown" + } + runMode := c.RunMode + if runMode == "" { + runMode = "unknown" + } + metrics.CollectorsTotal.WithLabelValues(status, c.Type, runMode).Inc() + } +} + +func databasePath() string { + if p := os.Getenv("DATABASE_PATH"); p != "" { + return p + } + if p := os.Getenv("DATA_DIR"); p != "" { + return p + "/opsxloki.json" + } + return "/app/data/opsxloki.json" +} diff --git a/backend/services/selfcheck.go b/backend/services/selfcheck.go new file mode 100644 index 0000000..2ae990e --- /dev/null +++ b/backend/services/selfcheck.go @@ -0,0 +1,233 @@ +package services + +import ( + "fmt" + "net/http" + "os" + "strings" + "syscall" + "time" + + "lokistack/models" + "lokistack/repository" +) + +// CheckItem 单条自检结果 +type CheckItem struct { + // Name 检查项名称（中文可读） + Name string `json:"name"` + // Status ok / warn / error + Status string `json:"status"` + // Message 一句话结论 + Message string `json:"message,omitempty"` + // Detail 补充信息（数量、路径、错误原文） + Detail string `json:"detail,omitempty"` +} + +// SelfCheckResult 一键诊断结果 +type SelfCheckResult struct { + CheckedAt time.Time `json:"checkedAt"` + // Overall ok / warn / error —— 前端据此显示整体颜色 + Overall string `json:"overall"` + Items []CheckItem `json:"items"` +} + +// SelfCheck 平台自检：把「能不能用」的判断集中到一处。 +// +// 设计目标：新人接手或故障时，先看这一个接口，不必在 5 个页面之间来回跳。 +// 所有检查都必须有超时保护，不能让诊断本身把平台拖慢。 +func SelfCheck() SelfCheckResult { + res := SelfCheckResult{CheckedAt: time.Now(), Overall: "ok"} + add := func(item CheckItem) { + res.Items = append(res.Items, item) + if item.Status == "error" { + res.Overall = "error" + } else if item.Status == "warn" && res.Overall == "ok" { + res.Overall = "warn" + } + } + + db := repository.GetDB() + + // 1) 数据库 + if db == nil { + add(CheckItem{Name: "数据库", Status: "error", Message: "未初始化"}) + } else { + dbPath := databasePath() + fi, err := os.Stat(dbPath) + if err != nil { + add(CheckItem{Name: "数据库", Status: "error", Message: "数据文件读不到", Detail: err.Error()}) + } else { + db.RLock() + counts := fmt.Sprintf("集群 %d / 主机 %d / 存储 %d / 部署 %d / 采集器 %d / 审计 %d", + len(db.Clusters), len(db.Hosts), len(db.Storages), len(db.Deployments), len(db.Collectors), len(db.AuditLogs)) + db.RUnlock() + status := "ok" + if fi.Size() > 16<<20 { + status = "warn" + } + add(CheckItem{Name: "数据库", Status: status, + Message: fmt.Sprintf("%.1f KB", float64(fi.Size())/1024), Detail: counts}) + } + } + + // 2) 加密密钥（丢了所有 kubeconfig/凭证都解不开） + if fp := encryptionKeyFingerprint(); fp == "" { + add(CheckItem{Name: "加密密钥", Status: "error", + Message: "ENCRYPTION_KEY 未配置", + Detail: "kubeconfig 与存储凭证依赖它解密；请确认 lokistack-secrets 中的 ENCRYPTION_KEY 已注入"}) + } else { + add(CheckItem{Name: "加密密钥", Status: "ok", Message: "已配置", Detail: "指纹 " + fp}) + } + + // 3) 数据目录可用空间 + if free, total, err := diskUsage(dataDirPath()); err == nil && total > 0 { + pct := float64(free) / float64(total) * 100 + status, msg := "ok", fmt.Sprintf("剩余 %.1f GB（%.0f%%）", float64(free)/(1<<30), pct) + if pct < 10 { + status = "error" + } else if pct < 20 { + status = "warn" + } + add(CheckItem{Name: "数据盘", Status: status, Message: msg}) + } + + if db != nil { + db.RLock() + clusterList := db.Clusters + hostList := db.Hosts + deployList := db.Deployments + collectorList := db.Collectors + auditList := db.AuditLogs + db.RUnlock() + + // 4) 集群接入（kubeconfig 能否解密） + if len(clusterList) == 0 { + add(CheckItem{Name: "K8s 集群", Status: "warn", Message: "尚未添加集群"}) + } else { + bad := 0 + for i := range clusterList { + if _, err := resolveKubeconfig(&clusterList[i]); err != nil { + bad++ + } + } + if bad > 0 { + add(CheckItem{Name: "K8s 集群", Status: "error", + Message: fmt.Sprintf("%d/%d 个集群的 kubeconfig 无法解密", bad, len(clusterList)), + Detail: "多半是 ENCRYPTION_KEY 与写入时不一致"}) + } else { + add(CheckItem{Name: "K8s 集群", Status: "ok", Message: fmt.Sprintf("%d 个，kubeconfig 均可解密", len(clusterList))}) + } + } + + // 5) 主机可达性（用最近一次验证结果，不在这里做 SSH 连接，避免诊断变慢） + if len(hostList) > 0 { + errCount := 0 + for _, h := range hostList { + if h.Status == "error" { + errCount++ + } + } + if errCount > 0 { + add(CheckItem{Name: "主机", Status: "warn", + Message: fmt.Sprintf("%d/%d 台连接异常", errCount, len(hostList)), + Detail: "到「主机管理」点「测试连接」查看具体原因"}) + } else { + add(CheckItem{Name: "主机", Status: "ok", Message: fmt.Sprintf("%d 台均正常", len(hostList))}) + } + } + + // 6) Loki 部署实例（真实探活 /ready） + if len(deployList) == 0 { + add(CheckItem{Name: "Loki 实例", Status: "warn", Message: "尚未部署 Loki"}) + } else { + var names []string + bad := 0 + for _, d := range deployList { + status, err := probeDeploymentReady(&d) + if err != nil || !status { + bad++ + names = append(names, d.Name) + } + } + if bad > 0 { + add(CheckItem{Name: "Loki 实例", Status: "error", + Message: fmt.Sprintf("%d/%d 个实例未就绪", bad, len(deployList)), + Detail: strings.Join(names, ", ")}) + } else { + add(CheckItem{Name: "Loki 实例", Status: "ok", Message: fmt.Sprintf("%d 个全部 ready", len(deployList))}) + } + } + + // 7) 采集器 + if len(collectorList) > 0 { + errCount, okCount := 0, 0 + for _, c := range collectorList { + switch c.Status { + case "active", "running": + okCount++ + case "error": + errCount++ + } + } + if errCount > 0 { + add(CheckItem{Name: "采集器", Status: "error", + Message: fmt.Sprintf("%d 个异常", errCount), + Detail: fmt.Sprintf("正常 %d 个；到「采集器」页看具体报错", okCount)}) + } else { + add(CheckItem{Name: "采集器", Status: "ok", Message: fmt.Sprintf("%d 个正常", okCount)}) + } + } + + // 8) 近期失败操作（审计） + recentFail := 0 + cutoff := time.Now().Add(-24 * time.Hour) + for _, a := range auditList { + if a.CreatedAt.After(cutoff) && a.Status >= 400 { + recentFail++ + } + } + if recentFail > 0 { + add(CheckItem{Name: "近 24h 失败操作", Status: "warn", + Message: fmt.Sprintf("%d 次", recentFail), Detail: "到「审计日志」筛选失败查看详情"}) + } else { + add(CheckItem{Name: "近 24h 失败操作", Status: "ok", Message: "无"}) + } + } + + return res +} + +// probeDeploymentReady 探测某个 Loki 部署实例是否 /ready。 +// +// 不用 DeploymentReady()：那个函数按"部署名 == 数据源名"匹配， +// 而生成的 K8s 资源名固定是 loki，两者对不上会永远返回未就绪（已知问题）。 +// 这里直接按部署记录里的 endpoint 探活，并对主机模式修正监听地址： +// 主机 Docker 部署存的是 0.0.0.0:3100，从平台 Pod 侧要用主机真实 IP。 +func probeDeploymentReady(d *models.LokiDeployment) (bool, error) { + url := strings.TrimSuffix(strings.TrimSpace(d.Endpoint), "/") + if url == "" { + return false, fmt.Errorf("部署未记录访问地址") + } + if strings.Contains(url, "0.0.0.0") && d.Host != nil && d.Host.HostIP != "" { + url = strings.Replace(url, "0.0.0.0", d.Host.HostIP, 1) + } + client := &http.Client{Timeout: 3 * time.Second} + resp, err := client.Get(url + "/ready") + if err != nil { + return false, err + } + defer resp.Body.Close() + buf := make([]byte, 128) + n, _ := resp.Body.Read(buf) + return resp.StatusCode == 200 && strings.Contains(strings.ToLower(string(buf[:n])), "ready"), nil +} + +// diskUsage 返回剩余与总空间（字节） +func diskUsage(path string) (free, total uint64, err error) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, 0, err + } + return st.Bavail * uint64(st.Bsize), st.Blocks * uint64(st.Bsize), nil +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index fe0dd80..d4ad90e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,18 +1,30 @@ # ============================================================ # lokistack - Production Docker Compose Configuration # ============================================================ -# Features: -# - Multi-instance deployment with load balancing -# - Read-only root filesystem for security -# - tmpfs for temporary directories -# - Stricter resource limits -# - Docker Secrets support -# - Health checks with longer intervals -# - Rolling update configuration +# ⚠️ 重要：本平台是【单写者】架构，只能跑一个应用实例。 +# +# 原因：全部状态存在 /app/data/opsxloki.json（JSON 文件 DB）， +# 进程内用一把读写锁 + 全量写盘来保证一致性。两个实例各挂各的卷时： +# - 数据完全不共享 → 页面刷新两次可能看到不同结果（负载均衡打到不同实例） +# - 写操作只落一个实例 → 另一个实例随后全量写盘会把对方的数据覆盖掉 +# - 不是高可用，是数据分裂 +# 曾经此文件配置 lokistack-1 / lokistack-2 双实例 + nginx 轮询， +# 属于**看起来高可用、实际会丢数据**的错误配置，已改为单实例。 +# +# 真要高可用，得先把存储层换成外部数据库（MySQL/PostgreSQL）， +# 那是架构变更，不是加个副本能解决的。 +# +# 本文件提供的生产特性： +# - 只读根文件系统 + tmpfs 可写目录（容器逃逸面收敛） +# - 严格资源限制与 pids 上限 +# - Docker Secrets 注入密钥（不落环境变量/命令行） +# - 健康检查 + 优雅停机（30s，等 DB 落盘完成） +# - 日志轮转（json-file max-size/max-file） +# - nginx 只做 TLS/反代，不做负载均衡 # ============================================================ services: - # ---- Nginx Load Balancer ---- + # ---- Nginx 反向代理（单后端，仅做 TLS / 统一入口）---- nginx: image: nginx:1.27-alpine container_name: lokistack-lb @@ -24,7 +36,7 @@ services: networks: - lokistack-net healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"] + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/api/v1/health"] interval: 30s timeout: 5s retries: 3 @@ -52,21 +64,19 @@ services: - /var/cache/nginx:noexec,nosuid,size=10m - /var/run:noexec,nosuid,size=5m depends_on: - lokistack-1: - condition: service_healthy - lokistack-2: + lokistack: condition: service_healthy - # ---- Application Instance 1 ---- - lokistack-1: + # ---- 应用（唯一实例）---- + lokistack: build: context: . dockerfile: Dockerfile args: TARGETARCH: ${TARGETARCH:-amd64} - image: lghost/lokistack:latest - container_name: lokistack-1 - hostname: lokistack-1 + image: lghost/lokistack:${IMAGE_TAG:-latest} + container_name: lokistack + hostname: lokistack restart: unless-stopped security_opt: @@ -76,26 +86,22 @@ services: cap_add: - NET_BIND_SERVICE - # P0: Read-only root filesystem + # 只读根文件系统：数据目录与 tmpfs 之外不可写 read_only: true - - # P0: tmpfs for writable directories tmpfs: - /tmp:noexec,nosuid,size=50m,uid=1000,gid=1000 volumes: - # Named volume for data persistence - - lokistack-data-1:/app/data - # kubeconfig read-only (P0: non-root path) + # 唯一数据卷：DB + kubeconfigs/ + known_hosts/ + backups/ + - lokistack-data:/app/data - ${KUBECONFIG_PATH:-~/.kube}:/home/lokistack/.kube:ro - # SSH keys read-only - ${SSH_KEY_PATH:-~/.ssh}:/home/lokistack/.ssh:ro environment: - SERVER_PORT=8080 - DATA_DIR=/app/data - KUBECONFIG=/home/lokistack/.kube/config - # Secrets via Docker Secrets file injection + # 密钥走 Docker Secrets 文件注入，不进环境变量（ps/env 里看不到） - JWT_SECRET_FILE=${JWT_SECRET_FILE:-/run/secrets/jwt_secret} - ENCRYPTION_KEY_FILE=${ENCRYPTION_KEY_FILE:-/run/secrets/encryption_key} @@ -103,24 +109,26 @@ services: - jwt_secret - encryption_key - # Health check healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/index.html"] + # 用 API 健康端点而不是静态文件：静态文件返回 200 只能说明 nginx/文件在， + # 不能说明 DB 和业务可用 + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/api/v1/health"] interval: 30s timeout: 5s retries: 3 start_period: 20s - # Stricter resource limits for production deploy: resources: limits: - cpus: '1.0' - memory: 256M - pids: 50 + # 平台要扛 JSON DB 全量读写 + 告警引擎 + 并发 API； + # 实测 1 核/256M 在批量发布时会顶到 OOM，这里按 2C/2G 起步 + cpus: '2.0' + memory: 2G + pids: 200 reservations: - cpus: '0.25' - memory: 64M + cpus: '0.5' + memory: 512M restart_policy: condition: on-failure delay: 5s @@ -132,90 +140,16 @@ services: options: max-size: "50m" max-file: "5" - labels: "service,instance" + # 优雅停机：给正在进行的 DB 落盘留时间，避免 JSON 被截断 stop_grace_period: 30s - networks: - - lokistack-net - # ---- Application Instance 2 ---- - lokistack-2: - build: - context: . - dockerfile: Dockerfile - args: - TARGETARCH: ${TARGETARCH:-amd64} - image: lghost/lokistack:latest - container_name: lokistack-2 - hostname: lokistack-2 - restart: unless-stopped - - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - cap_add: - - NET_BIND_SERVICE - - read_only: true - - tmpfs: - - /tmp:noexec,nosuid,size=50m,uid=1000,gid=1000 - - volumes: - - lokistack-data-2:/app/data - - ${KUBECONFIG_PATH:-~/.kube}:/home/lokistack/.kube:ro - - ${SSH_KEY_PATH:-~/.ssh}:/home/lokistack/.ssh:ro - - environment: - - SERVER_PORT=8080 - - DATA_DIR=/app/data - - KUBECONFIG=/home/lokistack/.kube/config - - JWT_SECRET_FILE=${JWT_SECRET_FILE:-/run/secrets/jwt_secret} - - ENCRYPTION_KEY_FILE=${ENCRYPTION_KEY_FILE:-/run/secrets/encryption_key} - - secrets: - - jwt_secret - - encryption_key - - healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/index.html"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 20s - - deploy: - resources: - limits: - cpus: '1.0' - memory: 256M - pids: 50 - reservations: - cpus: '0.25' - memory: 64M - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - window: 60s - - logging: - driver: "json-file" - options: - max-size: "50m" - max-file: "5" - labels: "service,instance" - - stop_grace_period: 30s networks: - lokistack-net # ---- Named Volumes ---- volumes: - lokistack-data-1: - driver: local - lokistack-data-2: + lokistack-data: driver: local # ---- Docker Secrets ---- diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts new file mode 100644 index 0000000..e816895 --- /dev/null +++ b/frontend/src/api/system/index.ts @@ -0,0 +1,102 @@ +// 系统运维域 API：审计日志 / 备份恢复 / 自检 +import { http, buildQuery } from '../http'; + +export interface AuditLog { + id: number; + userId: number; + username: string; + role?: string; + ip: string; + userAgent?: string; + module: string; + action: string; + resource: string; + resourceId?: string; + method: string; + path: string; + status: number; + durationMs: number; + detail?: string; + before?: string; + after?: string; + error?: string; + createdAt: string; +} + +export interface AuditListParams { + page?: number; + pageSize?: number; + username?: string; + module?: string; + action?: string; + resourceId?: string; + status?: number; + startTime?: string; + endTime?: string; +} + +export interface AuditStats { + todayCount: number; + weekCount: number; + failCount: number; + totalCount: number; + topModules: { name: string; count: number }[]; + topUsers: { name: string; count: number }[]; +} + +export interface SelfCheckItem { + name: string; + status: 'ok' | 'warn' | 'error'; + message?: string; + detail?: string; +} + +export interface SelfCheckResult { + checkedAt: string; + overall: 'ok' | 'warn' | 'error'; + items: SelfCheckItem[]; +} + +export interface BackupFile { + name: string; + sizeBytes: number; + createdAt: string; +} + +export interface RestoreResult { + manifest: { + version: string; + createdAt: string; + dbBytes: number; + recordCount: number; + note: string; + }; + keyMatches: boolean; + rowsInBackup: number; + currentRows: number; + stagedPath?: string; + warning?: string; +} + +export const systemApi = { + // 审计 + auditList: (params: AuditListParams = {}) => + http.get<{ list: AuditLog[]; total: number; page: number; pageSize: number }>( + `/audit-logs${buildQuery(params as Record)}`, + ), + auditGet: (id: number) => http.get(`/audit-logs/${id}`), + auditStats: () => http.get('/audit-logs/stats'), + + // 自检 + selfCheck: () => http.get('/system/self-check'), + + // 备份恢复 + backups: () => http.get<{ list: BackupFile[] }>('/system/backups'), + restore: (file: File) => { + const form = new FormData(); + form.append('file', file); + return http.post('/system/restore', form); + }, + restoreApply: () => + http.post<{ message: string; dbPath: string }>('/system/restore/apply', { confirm: true }), +}; diff --git a/frontend/src/components/GroupTabs.tsx b/frontend/src/components/GroupTabs.tsx new file mode 100644 index 0000000..d328356 --- /dev/null +++ b/frontend/src/components/GroupTabs.tsx @@ -0,0 +1,57 @@ +import { memo, useMemo } from 'react' +import { Link, useLocation } from 'react-router' +import { useTranslation } from 'react-i18next' +import { findGroupKey, findGroupRoutes } from '@/router/routes' +import { cn } from '@/lib/utils' + +/** + * 分组页内 tab 切换条。 + * + * 背景：侧栏菜单曾膨胀到 13 项（集群/主机/存储/部署/采集/查询/分析/告警/用户/ + * 审计/设置…），新人不清楚该从哪进。现在把同一心智模型的页面收成一个侧栏入口： + * 资源接入 = 集群 + 主机 + 存储 + * 日志服务 = Loki 实例 + 采集器 + * 日志 = 查询 + 分析 + * 系统管理 = 用户 + 审计 + 设置 + * 但 URL 完全不改（书签、审计记录、文档里的链接都还有效）， + * 靠这条 tab 条在组内切换 —— 等于给每组合并后的页面加了"分页"。 + * + * 不在任何分组内的页面，本组件直接不渲染（零影响）。 + */ +export const GroupTabs = memo(function GroupTabs() { + const location = useLocation() + const { t } = useTranslation() + + const groupKey = findGroupKey(location.pathname) + const groupRoutes = useMemo(() => (groupKey ? findGroupRoutes(groupKey) : []), [groupKey]) + + if (!groupKey || groupRoutes.length < 2) return null + + return ( +
+ {groupRoutes.map(route => { + const active = location.pathname === route.path + return ( + + {t(route.meta.title)} + + ) + })} +
+ ) +}) diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 6664a5c..ed89bab 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -21,7 +21,8 @@ import { useTheme } from 'next-themes' import { useTranslation } from 'react-i18next' import { cn } from '@/lib/utils' import { useIsMobile } from '@/hooks/use-mobile' -import { menuItems, findRouteMeta } from '@/router/routes' +import { menuItems, findRouteMeta, findGroupKey } from '@/router/routes' +import { GroupTabs } from './GroupTabs' import { getUser, clearAuth } from '@/lib/auth-storage' import { isThemeAuto, setThemeAuto } from '@/lib/theme-schedule' import { useScheduledTheme } from '@/hooks/useScheduledTheme' @@ -41,7 +42,11 @@ function SidebarNav({ collapsed, onNavigate }: { collapsed: boolean; onNavigate? return (