feat: run.sh restart/stop/status + real DB connector + GitLab CI

1. run.sh: start/stop/restart/status/logs/build/test
   - PID file management in .pids/
   - Colored output, health check wait, log files
   - Auto alembic migrate on start

2. DB Connector (replaces mock aiomysql):
   - MySQL: aiomysql (connect/query/dml/test)
   - PostgreSQL: asyncpg (connect/query/dml/test)
   - SQLite: sqlite3 via executor (connect/query/dml/test)
   - Unified interface: DBConnector.test_connection/execute_query/execute_dml
   - Connection test returns version + latency

3. GitLab CI/CD pipeline:
   - test: backend pytest + frontend build
   - build: Docker images (manual trigger)
   - deploy: SSH deploy (manual trigger)
   - Cache: pip + node_modules

Tests: 128 passing, frontend build: OK
This commit is contained in:
logan committed 2026-05-22 15:16:59 +08:00
1 parent 64e0c37972
commit 2cfd45c9d1
10 files changed
+712 -126

No files matched your search

+2 -11
View File
@@ -129,17 +129,8 @@ async def test_connection(
raise HTTPException(status_code=404, detail="数据源不存在")
# Decrypt password for connection test
from app.models.instance import Instance as InstModel
decrypted = instance.__class__(
name=instance.name, db_type=instance.db_type, host=instance.host,
port=instance.port, username=instance.username,
password=decrypt_password(instance.password),
database=instance.database, env=instance.env,
description=instance.description, status=instance.status,
version=instance.version,
)
decrypted.id = instance.id
return await InstanceService.test_connection(decrypted)
instance.password = decrypt_password(instance.password)
return await InstanceService.test_connection(instance)
@router.get("/{instance_id}/databases")
+2 -2
View File
@@ -8,7 +8,7 @@ from app.models.user import User
from app.models.instance import Instance
from app.models.audit_log import AuditLog
from app.schemas.query import QueryRequest, QueryResult
from app.services.query_service import QueryService
from app.services.db_connector import DBConnector
from app.core.crypto import decrypt_password
router = APIRouter(prefix="/query", tags=["SQL 查询"])
@@ -32,7 +32,7 @@ async def execute_query(
instance.password = decrypt_password(instance.password)
try:
query_result = await QueryService.execute_query(instance, req.database, req.sql, current_user.id)
query_result = await DBConnector.execute_query(instance, req.database, req.sql, current_user.id)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
+2 -2
View File
@@ -239,9 +239,9 @@ async def execute_ticket(
if not instance:
raise HTTPException(status_code=400, detail="数据源不存在")
from app.services.query_service import QueryService
from app.services.db_connector import DBConnector
instance.password = decrypt_password(instance.password)
exec_result = await QueryService.execute_dml(instance, ticket.target_database, ticket.sql_content)
exec_result = await DBConnector.execute_dml(instance, ticket.target_database, ticket.sql_content)
ticket.status = "success"
ticket.execute_result = exec_result
+288
View File
@@ -0,0 +1,288 @@
"""Database connector — unified interface for MySQL/PostgreSQL/SQLite.
Provides real database connections for query execution and connection testing.
Replaces mock aiomysql calls with actual driver connections.
"""
import asyncio
import time
import sqlite3
from typing import Optional, List, Dict, Any, Tuple
from app.models.instance import Instance
from app.services.sql_analyzer import SQLAnalyzer
MAX_ROWS = 10000
QUERY_TIMEOUT = 30
DML_TIMEOUT = 60
class ConnectionResult:
"""Result of a connection test."""
def __init__(self, success: bool, message: str, version: str = None, latency_ms: int = 0):
self.success = success
self.message = message
self.version = version
self.latency_ms = latency_ms
def to_dict(self):
return {
"success": self.success,
"message": self.message,
"version": self.version,
"latency_ms": self.latency_ms,
}
class QueryResult:
"""Result of a SQL query execution."""
def __init__(self, columns: List[str], rows: List[Dict[str, Any]],
total: int, elapsed_ms: int, truncated: bool = False):
self.columns = columns
self.rows = rows
self.total = total
self.elapsed_ms = elapsed_ms
self.truncated = truncated
class DBConnector:
"""Unified database connector supporting MySQL, PostgreSQL, SQLite."""
_analyzer = SQLAnalyzer()
@staticmethod
async def test_connection(instance: Instance) -> ConnectionResult:
"""Test database connectivity and return version info."""
start = time.time()
try:
if instance.db_type == "mysql":
return await DBConnector._test_mysql(instance, start)
elif instance.db_type in ("postgres", "postgresql"):
return await DBConnector._test_postgres(instance, start)
elif instance.db_type == "sqlite":
return await DBConnector._test_sqlite(instance, start)
else:
return ConnectionResult(False, f"暂不支持 {instance.db_type} 类型数据源")
except Exception as e:
elapsed = int((time.time() - start) * 1000)
return ConnectionResult(False, f"连接失败: {str(e)}", latency_ms=elapsed)
@staticmethod
async def execute_query(
instance: Instance, database: Optional[str], sql: str, user_id: int
) -> 'QueryResult':
"""Execute a read-only SQL query on the target database."""
# Validate SQL is read-only
is_safe, error_msg = DBConnector._analyzer.validate_readonly(sql)
if not is_safe:
raise ValueError(error_msg)
# Auto-add LIMIT
upper_sql = sql.strip().upper()
import re
if not re.search(r"LIMIT\s+\d+", upper_sql, re.IGNORECASE):
sql = sql.rstrip(";") + f" LIMIT {MAX_ROWS};"
start = time.time()
if instance.db_type == "mysql":
rows, columns = await DBConnector._query_mysql(instance, database, sql)
elif instance.db_type in ("postgres", "postgresql"):
rows, columns = await DBConnector._query_postgres(instance, database, sql)
elif instance.db_type == "sqlite":
rows, columns = await DBConnector._query_sqlite(instance, sql)
else:
raise ValueError(f"暂不支持 {instance.db_type} 查询")
elapsed = int((time.time() - start) * 1000)
return QueryResult(
columns=columns,
rows=rows,
total=len(rows),
elapsed_ms=elapsed,
truncated=len(rows) >= MAX_ROWS,
)
@staticmethod
async def execute_dml(instance: Instance, database: str, sql: str) -> str:
"""Execute a DML statement (INSERT/UPDATE/DELETE)."""
if instance.db_type == "mysql":
return await DBConnector._dml_mysql(instance, database, sql)
elif instance.db_type in ("postgres", "postgresql"):
return await DBConnector._dml_postgres(instance, database, sql)
elif instance.db_type == "sqlite":
return await DBConnector._dml_sqlite(instance, sql)
else:
raise ValueError(f"暂不支持 {instance.db_type} 执行")
# ===== MySQL =====
@staticmethod
async def _test_mysql(instance: Instance, start: float) -> ConnectionResult:
import aiomysql
conn = await asyncio.wait_for(
aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
db=instance.database,
),
timeout=10,
)
try:
async with conn.cursor() as cur:
await cur.execute("SELECT VERSION()")
version = (await cur.fetchone())[0]
elapsed = int((time.time() - start) * 1000)
return ConnectionResult(True, "连接成功", version=version, latency_ms=elapsed)
finally:
conn.close()
@staticmethod
async def _query_mysql(instance: Instance, database: Optional[str], sql: str) -> Tuple[List[Dict], List[str]]:
import aiomysql
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
db=database or instance.database,
)
try:
async with conn.cursor(aiomysql.DictCursor) as cur:
await asyncio.wait_for(cur.execute(sql), timeout=QUERY_TIMEOUT)
rows = await cur.fetchall()
columns = [desc[0] for desc in cur.description] if cur.description else []
return rows, columns
except asyncio.TimeoutError:
raise ValueError("查询超时(30秒)")
finally:
conn.close()
@staticmethod
async def _dml_mysql(instance: Instance, database: str, sql: str) -> str:
import aiomysql
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
db=database, autocommit=True,
)
try:
async with conn.cursor() as cur:
await asyncio.wait_for(cur.execute(sql), timeout=DML_TIMEOUT)
affected = cur.rowcount
return f"执行成功,影响 {affected} 行"
except asyncio.TimeoutError:
raise ValueError("执行超时(60秒)")
finally:
conn.close()
# ===== PostgreSQL =====
@staticmethod
async def _test_postgres(instance: Instance, start: float) -> ConnectionResult:
import asyncpg
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{instance.database or 'postgres'}"
conn = await asyncio.wait_for(asyncpg.connect(dsn), timeout=10)
try:
version = await conn.fetchval("SELECT version()")
elapsed = int((time.time() - start) * 1000)
# version string is long, take first part
ver_short = version.split(",")[0] if version else "unknown"
return ConnectionResult(True, "连接成功", version=ver_short, latency_ms=elapsed)
finally:
await conn.close()
@staticmethod
async def _query_postgres(instance: Instance, database: Optional[str], sql: str) -> Tuple[List[Dict], List[str]]:
import asyncpg
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database or instance.database or 'postgres'}"
conn = await asyncpg.connect(dsn)
try:
# asyncpg returns Records, convert to dicts
stmt = await asyncio.wait_for(conn.fetch(sql), timeout=QUERY_TIMEOUT)
rows = [dict(r) for r in stmt]
columns = list(stmt[0].keys()) if stmt else []
return rows, columns
except asyncio.TimeoutError:
raise ValueError("查询超时(30秒)")
finally:
await conn.close()
@staticmethod
async def _dml_postgres(instance: Instance, database: str, sql: str) -> str:
import asyncpg
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database}"
conn = await asyncpg.connect(dsn)
try:
status = await asyncio.wait_for(conn.execute(sql), timeout=DML_TIMEOUT)
# status like "UPDATE 3"
parts = status.split()
affected = int(parts[1]) if len(parts) >= 2 else 0
return f"执行成功,影响 {affected} 行"
except asyncio.TimeoutError:
raise ValueError("执行超时(60秒)")
finally:
await conn.close()
# ===== SQLite (remote instance) =====
@staticmethod
async def _test_sqlite(instance: Instance, start: float) -> ConnectionResult:
# SQLite uses host as file path
db_path = instance.host
loop = asyncio.get_event_loop()
try:
def _connect():
conn = sqlite3.connect(db_path)
cur = conn.execute("SELECT sqlite_version()")
ver = cur.fetchone()[0]
conn.close()
return ver
version = await loop.run_in_executor(None, _connect)
elapsed = int((time.time() - start) * 1000)
return ConnectionResult(True, "连接成功", version=f"SQLite {version}", latency_ms=elapsed)
except Exception as e:
elapsed = int((time.time() - start) * 1000)
return ConnectionResult(False, f"连接失败: {e}", latency_ms=elapsed)
@staticmethod
async def _query_sqlite(instance: Instance, sql: str) -> Tuple[List[Dict], List[str]]:
db_path = instance.host
loop = asyncio.get_event_loop()
def _query():
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
cur = conn.execute(sql)
rows = [dict(r) for r in cur.fetchall()]
columns = [desc[0] for desc in cur.description] if cur.description else []
conn.close()
return rows, columns
try:
return await asyncio.wait_for(
loop.run_in_executor(None, _query),
timeout=QUERY_TIMEOUT,
)
except asyncio.TimeoutError:
raise ValueError("查询超时(30秒)")
@staticmethod
async def _dml_sqlite(instance: Instance, sql: str) -> str:
db_path = instance.host
loop = asyncio.get_event_loop()
def _execute():
conn = sqlite3.connect(db_path)
cur = conn.execute(sql)
affected = cur.rowcount
conn.commit()
conn.close()
return affected
try:
affected = await asyncio.wait_for(
loop.run_in_executor(None, _execute),
timeout=DML_TIMEOUT,
)
return f"执行成功,影响 {affected} 行"
except asyncio.TimeoutError:
raise ValueError("执行超时(60秒)")
+79 -66
View File
@@ -1,79 +1,92 @@
import asyncio
from typing import List, Dict, Any
from typing import List
import aiomysql
from app.models.instance import Instance
from app.schemas.instance import ConnectionTestResult
from app.services.db_connector import DBConnector
class InstanceService:
@staticmethod
async def test_connection(instance: Instance) -> ConnectionTestResult:
try:
if instance.db_type == "mysql":
import aiomysql
conn = await asyncio.wait_for(
aiomysql.connect(
host=instance.host,
port=instance.port,
user=instance.username,
password=instance.password,
database=instance.database or None,
),
timeout=10,
)
async with conn.cursor() as cur:
await cur.execute("SELECT VERSION()")
version = (await cur.fetchone())[0]
conn.close()
return ConnectionTestResult(success=True, message="连接成功", version=str(version))
return ConnectionTestResult(success=False, message=f"暂不支持 {instance.db_type} 连接测试")
except asyncio.TimeoutError:
return ConnectionTestResult(success=False, message="连接超时")
except Exception as e:
return ConnectionTestResult(success=False, message=f"连接失败: {str(e)}")
@staticmethod
async def test_connection(instance: Instance) -> dict:
"""Test database connection using DBConnector."""
result = await DBConnector.test_connection(instance)
# Update instance status based on result
if result.success:
instance.status = "connected"
instance.version = result.version
else:
instance.status = "error"
return result.to_dict()
@staticmethod
async def list_databases(instance: Instance) -> List[str]:
if instance.db_type != "mysql":
"""List databases for a MySQL instance."""
if instance.db_type == "mysql":
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
)
try:
async with conn.cursor() as cur:
await cur.execute("SHOW DATABASES")
rows = await cur.fetchall()
return [row[0] for row in rows if row[0] not in ("information_schema", "mysql", "performance_schema", "sys")]
finally:
conn.close()
elif instance.db_type in ("postgres", "postgresql"):
import asyncpg
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/postgres"
conn = await asyncpg.connect(dsn)
try:
rows = await conn.fetch("SELECT datname FROM pg_database WHERE datistemplate = false")
return [r["datname"] for r in rows]
finally:
await conn.close()
elif instance.db_type == "sqlite":
return [instance.database or "main"]
else:
return []
import aiomysql
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
)
async with conn.cursor() as cur:
await cur.execute("SHOW DATABASES")
rows = await cur.fetchall()
conn.close()
return [row[0] for row in rows if row[0] not in ("information_schema", "mysql", "performance_schema", "sys")]
@staticmethod
async def list_tables(instance: Instance, database: str) -> List[Dict[str, Any]]:
if instance.db_type != "mysql":
async def list_tables(instance: Instance, database: str) -> List[dict]:
"""List tables in a database."""
if instance.db_type == "mysql":
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
db=database,
)
try:
async with conn.cursor(aiomysql.DictCursor) as cur:
await cur.execute(
"SELECT TABLE_NAME as name, TABLE_ROWS as rows, TABLE_TYPE as type "
"FROM information_schema.TABLES WHERE TABLE_SCHEMA = %s",
(database,)
)
return await cur.fetchall()
finally:
conn.close()
elif instance.db_type in ("postgres", "postgresql"):
import asyncpg
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database}"
conn = await asyncpg.connect(dsn)
try:
rows = await conn.fetch(
"SELECT tablename as name FROM pg_tables WHERE schemaname = 'public'"
)
return [dict(r) for r in rows]
finally:
await conn.close()
elif instance.db_type == "sqlite":
import sqlite3
db_path = instance.host
conn = sqlite3.connect(db_path)
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
tables = [{"name": row[0]} for row in cur.fetchall()]
conn.close()
return tables
else:
return []
import aiomysql
conn = await aiomysql.connect(
host=instance.host, port=instance.port,
user=instance.username, password=instance.password,
database=database,
)
async with conn.cursor() as cur:
await cur.execute("SHOW TABLES")
tables = [row[0] for row in await cur.fetchall()]
result = []
for table in tables:
await cur.execute(f"SHOW TABLE STATUS LIKE '{table}'")
status = await cur.fetchone()
result.append({
"name": table,
"engine": status[1],
"rows": status[4],
"size": status[6],
"comment": status[17],
})
conn.close()
return result