From 5ea86fe33403621fc07b030e9a66e766f2bfad27 Mon Sep 17 00:00:00 2001 From: logan Date: Thu, 28 May 2026 15:51:22 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9F=A5=E8=AF=A2=E5=88=86=E9=A1=B5(li?= =?UTF-8?q?mit/offset)+=E6=B5=81=E5=BC=8FCSV=E5=AF=BC=E5=87=BA+=E5=8E=BB?= =?UTF-8?q?=E5=A4=8D=E5=88=B6=E6=8C=89=E9=92=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 后端: - ExecuteQuery 添加 limit/offset 参数，MySQL 自动注入 LIMIT/OFFSET - 首页自动 COUNT 查总数(total)，翻页页不再重复 count - 新增 POST /query/export 流式 CSV 导出端点，游标分批写不占内存 - 默认 limit=100, max=1000，防止大数据量 OOM 前端: - 查询结果加分页控件(首页/上一页/下一页/末页 + 每页条数选择) - CSV 导出改走流式端点(全量数据)，不限于当前页 - 去掉复制按钮(Copy/Check/coped state)，Redis 单值也不显示复制 - 工具栏显示 总行数/行每页/列数 --- backend/internal/handler/instance_detail.go | 162 +++++++++++++++++++- backend/router/router.go | 1 + frontend/src/api/query.ts | 2 +- frontend/src/pages/QueriesPage.tsx | 128 +++++++++------- 4 files changed, 232 insertions(+), 61 deletions(-) diff --git a/backend/internal/handler/instance_detail.go b/backend/internal/handler/instance_detail.go index 3eb1c0a..24021de 100644 --- a/backend/internal/handler/instance_detail.go +++ b/backend/internal/handler/instance_detail.go @@ -505,6 +505,8 @@ func ExecuteQuery(c *gin.Context) { InstanceID int `json:"instance_id"` Database string `json:"database"` SQL string `json:"sql"` + Limit int `json:"limit"` + Offset int `json:"offset"` } if err := c.ShouldBindJSON(&input); err != nil { c.JSON(http.StatusBadRequest, gin.H{"detail": err.Error()}) @@ -514,6 +516,16 @@ func ExecuteQuery(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"detail": "instance_id and sql are required"}) return } + // Default pagination + if input.Limit <= 0 { + input.Limit = 100 + } + if input.Limit > 1000 { + input.Limit = 1000 + } + if input.Offset < 0 { + input.Offset = 0 + } var instance model.Instance if err := model.DB.First(&instance, input.InstanceID).Error; err != nil { @@ -525,7 +537,7 @@ func ExecuteQuery(c *gin.Context) { switch instance.DBType { case "mysql", "postgresql", "postgres", "pg": - result, err := executeSQLQuery(&instance, input.Database, input.SQL) + result, err := executeSQLQuery(&instance, input.Database, input.SQL, input.Limit, input.Offset) costMs := time.Since(start).Milliseconds() if err != nil { c.JSON(http.StatusOK, gin.H{"columns": []string{}, "rows": []interface{}{}, "total": 0, "cost_ms": costMs, "error": err.Error()}) @@ -551,13 +563,41 @@ func ExecuteQuery(c *gin.Context) { } } -func executeSQLQuery(instance *model.Instance, dbName string, sqlStr string) (gin.H, error) { +func executeSQLQuery(instance *model.Instance, dbName string, sqlStr string, limit int, offset int) (gin.H, error) { db, err := getDBConnection(instance, dbName) if err != nil { return nil, fmt.Errorf("connection failed: %v", err) } defer db.Close() + upperSQL := strings.ToUpper(strings.TrimSpace(sqlStr)) + + // Count total (only when no user LIMIT and offset=0) + total := 0 + hasLimit := strings.Contains(upperSQL, " LIMIT ") + + if !hasLimit && offset == 0 { + if strings.HasPrefix(upperSQL, "SELECT") { + fromIdx := strings.Index(upperSQL, " FROM ") + if fromIdx > 0 { + countSQL := "SELECT COUNT(*) " + sqlStr[fromIdx:] + upperCount := strings.ToUpper(countSQL) + if idx := strings.Index(upperCount, " ORDER BY "); idx > 0 { + countSQL = countSQL[:idx] + } + if idx := strings.Index(upperCount, " GROUP BY "); idx > 0 { + countSQL = countSQL[:idx] + } + db.QueryRow(countSQL).Scan(&total) + } + } + } + + // Inject LIMIT/OFFSET if not already present + if !hasLimit { + sqlStr = fmt.Sprintf("%s LIMIT %d OFFSET %d", sqlStr, limit, offset) + } + rows, err := db.Query(sqlStr) if err != nil { return nil, fmt.Errorf("query failed: %v", err) @@ -594,7 +634,8 @@ func executeSQLQuery(instance *model.Instance, dbName string, sqlStr string) (gi return gin.H{ "columns": colNames, "rows": resultRows, - "total": len(resultRows), + "total": total, + "count": len(resultRows), }, nil } @@ -742,6 +783,121 @@ func executeRedisQuery(instance *model.Instance, dbSelect string, command string }, nil } +// POST /api/v1/query/export — Streaming CSV export for large datasets +func ExportQueryCSV(c *gin.Context) { + var input struct { + InstanceID int `json:"instance_id"` + Database string `json:"database"` + SQL string `json:"sql"` + Limit int `json:"limit"` + } + if err := c.ShouldBindJSON(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"detail": err.Error()}) + return + } + if input.InstanceID == 0 || input.SQL == "" { + c.JSON(http.StatusBadRequest, gin.H{"detail": "instance_id and sql are required"}) + return + } + if input.Limit <= 0 { + input.Limit = 10000 // Export max 10k rows + } + if input.Limit > 50000 { + input.Limit = 50000 + } + + var instance model.Instance + if err := model.DB.First(&instance, input.InstanceID).Error; err != nil { + c.JSON(http.StatusNotFound, gin.H{"detail": "Instance not found"}) + return + } + + if instance.DBType != "mysql" && instance.DBType != "postgresql" && instance.DBType != "postgres" && instance.DBType != "pg" { + c.JSON(http.StatusBadRequest, gin.H{"detail": "CSV export only supports MySQL/PostgreSQL"}) + return + } + + db, err := getDBConnection(&instance, input.Database) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"detail": "Connection failed: " + err.Error()}) + return + } + defer db.Close() + + // Inject LIMIT if not present + upperSQL := strings.ToUpper(strings.TrimSpace(input.SQL)) + if !strings.Contains(upperSQL, " LIMIT ") { + input.SQL = fmt.Sprintf("%s LIMIT %d", input.SQL, input.Limit) + } + + rows, err := db.Query(input.SQL) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"detail": "Query failed: " + err.Error()}) + return + } + defer rows.Close() + + cols, _ := rows.Columns() + colNames := make([]string, len(cols)) + for i, col := range cols { + colNames[i] = col + } + + // Stream CSV response + filename := fmt.Sprintf("query_export_%s.csv", time.Now().Format("20060102_150405")) + c.Header("Content-Type", "text/csv; charset=utf-8") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename)) + // BOM for Excel UTF-8 + c.Writer.Write([]byte("\xEF\xBB\xBF")) + + // Header row + c.Writer.Write([]byte(strings.Join(colNames, ",") + "\n")) + + // Data rows — write in chunks to avoid OOM + batchSize := 500 + batch := make([]string, 0, batchSize) + rowCount := 0 + + for rows.Next() { + values := make([]interface{}, len(cols)) + ptrValues := make([]interface{}, len(cols)) + for i := range values { + ptrValues[i] = &values[i] + } + rows.Scan(ptrValues...) + + cells := make([]string, len(cols)) + for i, val := range values { + s := "" + if b, ok := val.([]byte); ok { + s = string(b) + } else if val != nil { + s = fmt.Sprintf("%v", val) + } + // CSV escape: wrap in quotes if contains comma, quote, or newline + if strings.ContainsAny(s, ",\"\n\r") { + s = `"` + strings.ReplaceAll(s, `"`, `""`) + `"` + } + cells[i] = s + } + batch = append(batch, strings.Join(cells, ",")) + rowCount++ + + if len(batch) >= batchSize { + c.Writer.Write([]byte(strings.Join(batch, "\n") + "\n")) + c.Writer.(http.Flusher).Flush() + batch = batch[:0] + } + } + // Flush remaining + if len(batch) > 0 { + c.Writer.Write([]byte(strings.Join(batch, "\n") + "\n")) + } + c.Writer.(http.Flusher).Flush() + + saveQueryHistory(0, input.InstanceID, input.Database, input.SQL, 0, rowCount) +} + func saveQueryHistory(userID int, instanceID int, database string, sqlStr string, costMs int64, rowCount int) { var user model.User var inst model.Instance diff --git a/backend/router/router.go b/backend/router/router.go index 561d971..a49adcd 100644 --- a/backend/router/router.go +++ b/backend/router/router.go @@ -155,6 +155,7 @@ func Setup(r *gin.Engine) { // Query execution auth.POST("/query/execute", handler.ExecuteQuery) auth.POST("/query", handler.ExecuteQuery) + auth.POST("/query/export", handler.ExportQueryCSV) // SQL Assistant diff --git a/frontend/src/api/query.ts b/frontend/src/api/query.ts index adfb6e7..16d772f 100644 --- a/frontend/src/api/query.ts +++ b/frontend/src/api/query.ts @@ -1,7 +1,7 @@ import request from './request'; export const queryApi = { - execute: (data: { instance_id: number; database?: string; sql: string }) => + execute: (data: { instance_id: number; database?: string; sql: string; limit?: number; offset?: number }) => request.post('/query', data), formatSQL: (sql: string) => diff --git a/frontend/src/pages/QueriesPage.tsx b/frontend/src/pages/QueriesPage.tsx index e5f8d23..656f7db 100644 --- a/frontend/src/pages/QueriesPage.tsx +++ b/frontend/src/pages/QueriesPage.tsx @@ -5,7 +5,7 @@ import { StarOff, Download, Search, ArrowUpDown, Database, Table2, ArrowUp, ArrowDown, Eye, Hash, PanelLeftClose, Columns3, KeyRound, FolderOpen, Server, Zap, ClipboardPaste, - History, PanelRightClose, PanelRightOpen, Copy, Check, + History, PanelRightClose, PanelRightOpen, ChevronLeft, ChevronLast, } from 'lucide-react'; // Card components replaced by design system utility classes import { useActionPermission } from '@/hooks/useAuth'; @@ -23,6 +23,8 @@ import type { DBType, QueryTemplate } from '@/lib/queryTemplates'; interface QueryResult { columns: string[]; rows: Record[]; + total: number; + count: number; affected_rows: number; execution_time_ms: number; cost_ms?: number; @@ -123,6 +125,10 @@ export default function QueriesPage() { const [error, setError] = useState(''); const [result, setResult] = useState(null); + // -- Pagination -- + const [pageSize, setPageSize] = useState(100); + const [currentPage, setCurrentPage] = useState(1); + // -- UI state -- const [showTemplates, setShowTemplates] = useState(false); const [templateSearch, setTemplateSearch] = useState(''); @@ -209,12 +215,14 @@ export default function QueriesPage() { const catLabel: Record = { all: '全部', '表结构': '表结构', '性能分析': '性能分析', '数据统计': '数据统计', '基础查询': '基础查询' }; /* ---- Execute ---- */ - const handleExecute = useCallback(async () => { + const handleExecute = useCallback(async (page = 1) => { if (!sql || !instanceId) { setError('请选择实例并输入查询语句'); return; } if (/\{(?:table|db|collection)\}/.test(sql)) { setError('SQL 中含有未替换的占位符，请先替换或删除'); return; } - setExecuting(true); setError(''); setResult(null); + setExecuting(true); setError(''); setCurrentPage(page); + if (page === 1) setResult(null); try { - const res = await queryApi.execute({ instance_id: Number(instanceId), database: database || undefined, sql }) as unknown as QueryResult; + const offset = (page - 1) * pageSize; + const res = await queryApi.execute({ instance_id: Number(instanceId), database: database || undefined, sql, limit: pageSize, offset }) as unknown as QueryResult; const normalized = { ...res, execution_time_ms: res.execution_time_ms || res.cost_ms || 0 }; if (normalized.error) { setError(normalized.error); @@ -222,14 +230,18 @@ export default function QueriesPage() { } else { setResult(normalized); } - setHistory(prev => [{ id: Date.now(), sql, instance_id: Number(instanceId), database, db_type: dbType, status: 'success', execution_time_ms: normalized.execution_time_ms, result_preview: JSON.stringify(normalized.rows?.[0] || {}).substring(0, 100), created_at: new Date().toISOString(), favorite: false }, ...prev]); + if (page === 1) { + setHistory(prev => [{ id: Date.now(), sql, instance_id: Number(instanceId), database, db_type: dbType, status: 'success', execution_time_ms: normalized.execution_time_ms, result_preview: JSON.stringify(normalized.rows?.[0] || {}).substring(0, 100), created_at: new Date().toISOString(), favorite: false }, ...prev]); + } } catch (e: unknown) { const raw = (e as { response?: { data?: { detail?: string | Array<{ msg: string }> } } })?.response?.data?.detail; const msg = Array.isArray(raw) ? raw.map(v => v.msg).join('; ') : (raw || '查询失败'); setError(msg); - setHistory(prev => [{ id: Date.now(), sql, instance_id: Number(instanceId), database, db_type: dbType, status: 'failed', execution_time_ms: 0, result_preview: msg, created_at: new Date().toISOString(), favorite: false }, ...prev]); + if (page === 1) { + setHistory(prev => [{ id: Date.now(), sql, instance_id: Number(instanceId), database, db_type: dbType, status: 'failed', execution_time_ms: 0, result_preview: msg, created_at: new Date().toISOString(), favorite: false }, ...prev]); + } } finally { setExecuting(false); } - }, [sql, instanceId, database, dbType]); + }, [sql, instanceId, database, dbType, pageSize]); /* ---- Result Sorting ---- */ const sortedResult = useMemo(() => { @@ -264,38 +276,26 @@ export default function QueriesPage() { }, [sortedResult, rowHeight]); const totalResultHeight = (sortedResult?.rows?.length || 0) * rowHeight; - const handleExportResult = useCallback(() => { - if (!result) return; - downloadCSV(`query_result_${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.csv`, exportToCSV(result.columns, result.rows)); - }, [result]); - - /* ---- Copy Result ---- */ - const [copied, setCopied] = useState(false); - const handleCopyResult = useCallback(() => { - if (!result) return; - let text = ''; - if (result.rows?.length && result.columns?.length) { - // Format as readable text - const lines = result.rows.map((row: Record) => - result.columns.map((col: string) => { - const val = String(row[col] ?? ''); - // Try to pretty-format JSON values - try { - const parsed = JSON.parse(val); - if (typeof parsed === 'object') return JSON.stringify(parsed, null, 2); - } catch { /* not JSON */ } - return val; - }).join('\t') - ); - text = [result.columns.join('\t'), ...lines].join('\n'); - } else { - text = JSON.stringify(result, null, 2); + const handleExportResult = useCallback(async () => { + if (!sql || !instanceId) return; + try { + const resp = await fetch('/api/v1/query/export', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('token')}` }, + body: JSON.stringify({ instance_id: Number(instanceId), database: database || undefined, sql }), + }); + if (!resp.ok) throw new Error('导出失败'); + const blob = await resp.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `query_result_${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.csv`; + a.click(); + URL.revokeObjectURL(url); + } catch (e) { + setError((e as Error).message || '导出失败'); } - navigator.clipboard.writeText(text).then(() => { - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }); - }, [result]); + }, [sql, instanceId, database]); /* ---- Smart cell value formatter (detect JSON) ---- */ const formatCellValue = useCallback((val: unknown) => { @@ -486,24 +486,19 @@ export default function QueriesPage() {
{result.execution_time_ms || result.cost_ms || 0}ms {result.raw_text !== undefined ? ( - {result.raw_value ? 'JSON' : typeof result.raw_text === 'string' ? 'String' : 'Value'} + {result.raw_value ? 'JSON' : 'String'} ) : ( <> - {result.rows?.length || 0} 行 + {result.total || result.rows?.length || 0} 总行 + {result.rows?.length || 0} 行/页 {result.columns?.length || 0} 列 )} - {(result.rows?.length > 0 || result.raw_text !== undefined) && ( - <> - - - + {result.rows?.length > 0 && dbType !== 'redis' && ( + )}
@@ -524,12 +519,6 @@ export default function QueriesPage() { ? (() => { try { return JSON.stringify(JSON.parse(String(result.raw_text)), null, 2); } catch { return String(result.raw_text); } })() : String(result.raw_text) } - ) : result.columns && result.columns.length > 0 ? ( @@ -576,6 +565,31 @@ export default function QueriesPage() { + {/* Pagination */} + {result.total > pageSize && ( +
+
+ 每页 + + 条 +
+
+ + + {currentPage} / {Math.ceil(result.total / pageSize)} + + +
+ 共 {result.total} 条 +
+ )} ) : (