rlwj-lab/opsxsql
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7dfbc0b6800145001fc767417775c99133fa8009
opsxsql/backend/cmd/server/main.go
T
opsx 7dfbc0b680 feat(redis): 大 key 分页 + 流式导出（含票据鉴权）+ 查询行数上限默认 5000 
=== 一、问题：大 key 会把内存打爆，且查询结果没有导出 ===
RedisGetKeyDetail 原来对每种类型都是全量读取：
  string → GET        list → LRANGE 0 -1     set → SMEMBERS
  zset   → ZRANGE 0 -1                        hash → HGETALL
然后整包序列化返回。像 PERP:market:* 这类盘口/kline key，
一个 key 几十万条就会：Go 侧整值入内存 → JSON 序列化翻倍 → 浏览器渲染卡死。

=== 二、后端：键详情分页（统一令牌，兼容四种类型）===
GET /redis/:id/keys/:key?db=&count=&token=
  count  默认 200，硬上限 2000
  token  语义按类型区分，前端无感知：
           list / zset  → 十进制 offset（LRANGE / ZRANGE 区间）
           hash / set   → Redis SCAN 游标（HSCAN / SSCAN）
           string       → 忽略
  返回   total / count / nextToken / truncated / previewCap

string 特殊处理：不能分页，改为预览上限 256KB
  - 用 STRLEN 拿真实大小（O(1)）
  - 超过上限时用 GETRANGE 只取前 256KB，truncated=true
  - 完整内容只能靠导出

新增 wrapper 方法：HScan / ZScan / HLen / ZCard / StrLen / GetRange
（HGETALL、SMEMBERS、ZRANGE 0 -1 一律不再用于详情读取）

=== 三、后端：流式导出（两段式票据，避免令牌进 URL 日志）===
① POST /redis/:id/keys/:key/export-ticket   （走 Bearer 鉴权）
     → HMAC-SHA256(JWTSecret, instance|db|key|exp) 生成票据，2 分钟过期，
       绑定具体 instance/db/key，不能跨 key 复用
② GET  /redis/:id/keys/:key/export?db=&format=csv|jsonl&ticket=
     → 注册在鉴权组【之外】，靠票据校验
     为什么这么做：浏览器原生下载（<a download>/window.open）带不了
     Authorization 头；若前端用 axios 取 blob，整份导出会进浏览器内存，
     与「大 key 不爆内存」的目标相悖。

导出实现全程流式（服务端内存 = 单批 + 64KB 缓冲）：
  string → GETRANGE 256KB 分块
  list   → LRANGE 每批 1000
  zset   → ZRANGE WITHSCORES 每批 1000
  hash   → HSCAN 游标每批 1000
  set    → SSCAN 游标每批 1000
单次上限 200 万行；CSV 带 UTF-8 BOM（Excel 直接打开不乱码）；
列顺：hash(field,value) / list(index,value) / set(member) / zset(rank,member,score)

=== 四、前端：详情面板加分页条 + 4 个导出按钮 ===
RedisKeyDetailPanel：
  - 「共 N 条 · 第 X 页 · 每页 [100/200/500/1000/2000]」+ 上一页/下一页
  - SCAN 游标不可逆，上一页用令牌栈回放
  - string 超预览上限时显示黄色提示「仅预览前 256KB，共 N 字节」
  - 导出四档：当前页 CSV / 当前页 JSON（纯前端，瞬时）
             导出全部 CSV / 导出全部 JSONL（服务端流式，边读边写）
  顺带修掉头注释里「已知限制：大值不分页」的过时说明。

=== 五、查询行数上限：默认 1000 → 5000，并修掉硬编码 ===
用户要求：「默认设置里面有查询最大上限吧 比如 mysql pg 查询最大Limit限制 5000」

发现两个问题：
1. seed 默认值只有 1000（main.go query_limit_max）
2. ExecuteQuery 里还有一段硬编码：
     if input.Limit <= 0 { input.Limit = maxLimit; if input.Limit > 1000 { input.Limit = 1000 } }
   → 管理员把设置改成 5000 也白搭，前端不传 limit 时永远被压回 1000

修复：
  - seed 默认值 1000 → 5000（Value + DefaultValue）
  - getSettingInt("query_limit_max", 1000) → 5000
  - 删掉硬编码 1000 封顶，唯一口径就是设置值
  - 线上已有记录的 value 已通过 PUT /settings 改成 5000（seed 只在缺失时插入，不会自动更新旧值）

对 MySQL / PostgreSQL / Redis / MongoDB 统一生效（ExecuteQuery 按 db_type 分发，
limit 在入口处统一 clamp）。

=== 六、实测结果（线上）===
分页：
  zset PERP:market:kline:symbol_kline_12h_AAVEUSDT_PERP_MARK
        total=83，count=5 → 第1页 5 条 nextToken=5 → 第2页 5 条 nextToken=10，数据不同 ✅
  hash ST:market:ticker:symbol_ticker
        total=27，count=3 → 第1页 3 字段 nextToken=24 → 第2页 3 字段（完全不同）✅
  string PERP:market:depth:symbol_depth_BTCUSDT_PERP
        STRLEN=14192 → 导出 14192 字节，完全一致 ✅（实时 key 两次采样有差异属正常）
导出：
  zset CSV  200 / 30970 字节 / 84 行（header+83）✅
  zset JSONL 200 / 84 行 ✅
  string CSV 200 / 分块 GETRANGE 拼接完整 ✅
  无票据 → 403 ✅   伪造票据 → 403 ✅
查询上限：
  limit=99999 → 5000 行 ✅    limit=0（默认）→ 5000 行 ✅
SQL 导出 CSV（原已存在 ExportQueryCSV，支持 mysql/pg/redis/mongo）200 / 101 行 ✅

镜像：lghost/opsxsql:latest + v0.1 = sha256:68fc996681c6
2026-10-27 09:14:02 +00:00

181 lines
11 KiB
Go
Raw Blame History

package main
import (
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"sql-platform/config"
"sql-platform/internal/middleware"
"sql-platform/internal/model"
"sql-platform/router"
"github.com/gin-gonic/gin"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// AppVersion 与前端 package.json 版本保持同步
const AppVersion = "0.4.0"
func main() {
// Load config
config.Init()
if !config.C.Debug {
gin.SetMode(gin.ReleaseMode)
}
// Connect to SQLite
db, err := gorm.Open(sqlite.Open(config.C.DBPath), &gorm.Config{})
if err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}
model.DB = db
// Auto-migrate (add missing tables/columns, won\'t drop existing data)
_ = db.AutoMigrate(
&model.User{},
&model.Instance{},
&model.Ticket{},
&model.TicketApproval{},
&model.TicketSQLItem{},
&model.AuditRule{},
&model.AuditLog{},
&model.QueryHistory{},
&model.WorkflowDefinition{},
&model.WorkflowStep{},
&model.WorkflowVersion{},
&model.WorkflowExecutionLog{},
&model.Announcement{},
&model.InstanceGroup{},
&model.InstanceGroupMember{},
&model.LDAPConfig{},
&model.NotificationChannel{},
&model.NotificationTemplate{},
&model.SystemSetting{},
&model.OperationLog{},
&model.MaskingRule{},
&model.Backup{},
&model.SchemaRevision{},
&model.SlowLog{},
&model.ScheduledTask{},
&model.ScheduledTaskLog{},
&model.UserGroup{},
&model.RefreshToken{},
)
// Seed default data (idempotent): users, workflow, settings
seedAll(db)
// 治理默认 JWT 密钥（生产模式自动生成并持久化）
config.EnsureJWTSecret()
// Setup Gin
r := gin.Default()
r.Use(middleware.CORSMiddleware())
// Health check
r.GET("/health", func(c *gin.Context) {
c.JSON(200, gin.H{"status": "ok", "version": AppVersion, "backend": "go"})
})
// Routes
router.Setup(r)
// 单镜像部署:直接托管前端产物(DIST_DIR 为空则跳过,开发模式由 vite 独立提供)
if distDir := os.Getenv("DIST_DIR"); distDir != "" {
indexPath := filepath.Join(distDir, "index.html")
if _, err := os.Stat(indexPath); err == nil {
r.NoRoute(func(c *gin.Context) {
p := c.Request.URL.Path
if strings.HasPrefix(p, "/api") {
c.JSON(http.StatusNotFound, gin.H{"detail": "Not found"})
return
}
// 目录穿越防护后按文件直出(assets/db-logos/favicon),其余回落 SPA index
full := filepath.Join(distDir, filepath.Clean("/"+p))
if strings.HasPrefix(full, filepath.Clean(distDir)) {
if info, err := os.Stat(full); err == nil && !info.IsDir() {
c.File(full)
return
}
}
c.File(indexPath)
})
log.Printf("serving frontend from %s", distDir)
} else {
log.Printf("DIST_DIR=%s 无 index.html,跳过静态托管", distDir)
}
}
// Start server
addr := fmt.Sprintf(":%s", config.C.Port)
log.Printf("SQL Platform Go backend starting on %s", addr)
if err := r.Run(addr); err != nil {
log.Fatalf("Failed to start server: %v", err)
}
}
func seedDefaultSettings(db *gorm.DB) {
defaults := []model.SystemSetting{
// ── 查询控制 ──
{Key: "query_limit_max", Value: "5000", ValueType: "number", Group: "query", Label: "LIMIT 上限", Desc: strPtr("SELECT 查询返回的最大行数（默认 5000），超过自动截断；对 MySQL / PostgreSQL / Redis / MongoDB 统一生效"), DefaultValue: strPtr("5000"), SortOrder: 1},
{Key: "query_timeout", Value: "30", ValueType: "number", Group: "query", Label: "查询超时(秒)", Desc: strPtr("单次查询最长执行时间，超时自动终止"), DefaultValue: strPtr("30"), SortOrder: 2},
{Key: "query_auto_limit", Value: "true", ValueType: "boolean", Group: "query", Label: "自动加 LIMIT", Desc: strPtr("用户未写 LIMIT 时自动追加，防止全表扫描"), DefaultValue: strPtr("true"), SortOrder: 3},
{Key: "query_max_rows_examined", Value: "1000000", ValueType: "number", Group: "query", Label: "最大扫描行数", Desc: strPtr("单次查询允许扫描的最大行数"), DefaultValue: strPtr("1000000"), SortOrder: 4},
{Key: "query_dangerous_pattern", Value: "true", ValueType: "boolean", Group: "query", Label: "危险模式检测", Desc: strPtr("检测不带 WHERE 的 DELETE/UPDATE 等高危操作"), DefaultValue: strPtr("true"), SortOrder: 5},
// ── 安全策略 ──
{Key: "login_max_attempts", Value: "5", ValueType: "number", Group: "security", Label: "登录失败锁定次数", Desc: strPtr("连续登录失败达到此次数后锁定账号"), DefaultValue: strPtr("5"), SortOrder: 1},
{Key: "login_lockout_minutes", Value: "30", ValueType: "number", Group: "security", Label: "锁定时长(分钟)", Desc: strPtr("账号被锁定后的持续时长"), DefaultValue: strPtr("30"), SortOrder: 2},
{Key: "password_min_length", Value: "8", ValueType: "number", Group: "security", Label: "密码最小长度", Desc: strPtr("用户密码的最短字符数"), DefaultValue: strPtr("8"), SortOrder: 3},
{Key: "password_require_complex", Value: "true", ValueType: "boolean", Group: "security", Label: "密码复杂度", Desc: strPtr("要求包含大小写字母、数字和特殊字符"), DefaultValue: strPtr("true"), SortOrder: 4},
{Key: "session_timeout_hours", Value: "24", ValueType: "number", Group: "security", Label: "会话超时(小时)", Desc: strPtr("JWT Token 有效期"), DefaultValue: strPtr("24"), SortOrder: 5},
// ── 数据脱敏 ──
{Key: "data_masking_enabled", Value: "true", ValueType: "boolean", Group: "data_masking", Label: "启用数据脱敏", Desc: strPtr("启用后，查询结果中的敏感字段将被替换为 *"), DefaultValue: strPtr("true"), SortOrder: 1},
{Key: "data_masking_sensitive_fields", Value: "mobile,concat_mobile,email_alias,email,password,ga_key,private_key,secret,secret_key,key_snow,telephone,username,real_name,token,two_factor_secret,content,phone", ValueType: "string", Group: "data_masking", Label: "默认敏感字段列表", Desc: strPtr("全局敏感字段，逗号分隔。匹配列名（不区分大小写）"), DefaultValue: strPtr("mobile,concat_mobile,email_alias,email,password,ga_key,private_key,secret,secret_key,key_snow,telephone,username,real_name,token,two_factor_secret,content,phone"), SortOrder: 2},
// ── 日志清理 ──
{Key: "ticket_retention_days", Value: "180", ValueType: "number", Group: "cleanup", Label: "工单保留天数", Desc: strPtr("超过此天数的已完成/已关闭工单将被清理"), DefaultValue: strPtr("180"), SortOrder: 1},
{Key: "audit_log_retention_days", Value: "365", ValueType: "number", Group: "cleanup", Label: "审计日志保留天数", Desc: strPtr("超过此天数的审计日志将被清理"), DefaultValue: strPtr("365"), SortOrder: 2},
{Key: "query_history_retention_days", Value: "90", ValueType: "number", Group: "cleanup", Label: "查询历史保留天数", Desc: strPtr("超过此天数的查询历史将被清理"), DefaultValue: strPtr("90"), SortOrder: 3},
{Key: "slow_log_retention_days", Value: "180", ValueType: "number", Group: "cleanup", Label: "慢日志保留天数", Desc: strPtr("超过此天数的慢查询日志将被清理"), DefaultValue: strPtr("180"), SortOrder: 4},
// ── 执行器 ──
{Key: "executor_timeout", Value: "300", ValueType: "number", Group: "executor", Label: "执行超时(秒)", Desc: strPtr("工单中 SQL 语句执行的最大等待时间"), DefaultValue: strPtr("300"), SortOrder: 1},
{Key: "executor_batch_size", Value: "100", ValueType: "number", Group: "executor", Label: "批量执行大小", Desc: strPtr("DML 批量执行时每批的行数"), DefaultValue: strPtr("100"), SortOrder: 2},
{Key: "executor_dry_run", Value: "true", ValueType: "boolean", Group: "executor", Label: "执行前预检查", Desc: strPtr("执行前先 EXPLAIN 验证 SQL 语法正确性"), DefaultValue: strPtr("true"), SortOrder: 3},
{Key: "executor_backup", Value: "true", ValueType: "boolean", Group: "executor", Label: "执行前备份", Desc: strPtr("DML 执行前自动备份受影响的数据"), DefaultValue: strPtr("true"), SortOrder: 4},
// ── Ghost/OSC (gh-ost 在线DDL) ──
{Key: "ghost_enabled", Value: "false", ValueType: "boolean", Group: "ghost", Label: "启用 gh-ost", Desc: strPtr("启用 gh-ost 在线 DDL 变更工具"), DefaultValue: strPtr("false"), SortOrder: 1},
{Key: "ghost_path", Value: "gh-ost", ValueType: "string", Group: "ghost", Label: "gh-ost 路径", Desc: strPtr("gh-ost 二进制文件路径"), DefaultValue: strPtr("gh-ost"), SortOrder: 2},
{Key: "ghost_default_mode", Value: "dry_run", ValueType: "string", Group: "ghost", Label: "默认执行模式", Desc: strPtr("dry_run=预演 / execute=直接执行"), DefaultValue: strPtr("dry_run"), SortOrder: 3},
{Key: "ghost_osc_priority", Value: "native_first", ValueType: "string", Group: "ghost", Label: "OSC 优先级", Desc: strPtr("native_first / ghost_first / ghost_only"), DefaultValue: strPtr("native_first"), SortOrder: 4},
{Key: "ghost_env_mode", Value: "development", ValueType: "string", Group: "ghost", Label: "环境模式", Desc: strPtr("development / staging / production"), DefaultValue: strPtr("development"), SortOrder: 5},
{Key: "ghost_cloud_type", Value: "self_hosted", ValueType: "string", Group: "ghost", Label: "部署类型", Desc: strPtr("self_hosted / rds / cloud"), DefaultValue: strPtr("self_hosted"), SortOrder: 6},
{Key: "ghost_auto_trigger_rows", Value: "10", ValueType: "number", Group: "ghost", Label: "自动触发行数(万)", Desc: strPtr("表行数超过此阈值(万)自动推荐 OSC"), DefaultValue: strPtr("10"), SortOrder: 7},
{Key: "ghost_auto_trigger_ddl", Value: "ADD COLUMN,MODIFY COLUMN,ADD INDEX", ValueType: "string", Group: "ghost", Label: "自动触发DDL类型", Desc: strPtr("自动推荐 OSC 的 DDL 操作类型列表"), DefaultValue: strPtr("ADD COLUMN,MODIFY COLUMN,ADD INDEX"), SortOrder: 8},
{Key: "ghost_default_chunk_size", Value: "1000", ValueType: "number", Group: "ghost", Label: "默认 chunk 大小", Desc: strPtr("每次拷贝的行数"), DefaultValue: strPtr("1000"), SortOrder: 9},
{Key: "ghost_max_load", Value: "Threads_running=25", ValueType: "string", Group: "ghost", Label: "最大负载阈值", Desc: strPtr("超过此负载暂停拷贝"), DefaultValue: strPtr("Threads_running=25"), SortOrder: 10},
{Key: "ghost_critical_load", Value: "Threads_running=50", ValueType: "string", Group: "ghost", Label: "临界负载阈值", Desc: strPtr("超过此负载立即停止"), DefaultValue: strPtr("Threads_running=50"), SortOrder: 11},
{Key: "ghost_allow_on_master", Value: "true", ValueType: "boolean", Group: "ghost", Label: "允许在主库执行", Desc: strPtr("是否允许直接在主库运行 gh-ost"), DefaultValue: strPtr("true"), SortOrder: 12},
{Key: "ghost_max_lag_millis", Value: "1500", ValueType: "number", Group: "ghost", Label: "最大复制延迟(ms)", Desc: strPtr("从库复制延迟超过此值暂停拷贝"), DefaultValue: strPtr("1500"), SortOrder: 13},
}
for _, s := range defaults {
var existing model.SystemSetting
if db.Where("`key` = ?", s.Key).First(&existing).Error != nil {
db.Create(&s)
}
}
}
func strPtr(s string) *string {
return &s
}
Reference in new issue View Git Blame Copy Permalink