feat: run.sh restart/stop/status + real DB connector + GitLab CI
1. run.sh: start/stop/restart/status/logs/build/test - PID file management in .pids/ - Colored output, health check wait, log files - Auto alembic migrate on start 2. DB Connector (replaces mock aiomysql): - MySQL: aiomysql (connect/query/dml/test) - PostgreSQL: asyncpg (connect/query/dml/test) - SQLite: sqlite3 via executor (connect/query/dml/test) - Unified interface: DBConnector.test_connection/execute_query/execute_dml - Connection test returns version + latency 3. GitLab CI/CD pipeline: - test: backend pytest + frontend build - build: Docker images (manual trigger) - deploy: SSH deploy (manual trigger) - Cache: pip + node_modules Tests: 128 passing, frontend build: OK
This commit is contained in:
1 parent
64e0c37972
commit
2cfd45c9d1
10 files changed
+712
-126
No files matched your search
@@ -129,17 +129,8 @@ async def test_connection(
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
|
||||
# Decrypt password for connection test
|
||||
from app.models.instance import Instance as InstModel
|
||||
decrypted = instance.__class__(
|
||||
name=instance.name, db_type=instance.db_type, host=instance.host,
|
||||
port=instance.port, username=instance.username,
|
||||
password=decrypt_password(instance.password),
|
||||
database=instance.database, env=instance.env,
|
||||
description=instance.description, status=instance.status,
|
||||
version=instance.version,
|
||||
)
|
||||
decrypted.id = instance.id
|
||||
return await InstanceService.test_connection(decrypted)
|
||||
instance.password = decrypt_password(instance.password)
|
||||
return await InstanceService.test_connection(instance)
|
||||
|
||||
|
||||
@router.get("/{instance_id}/databases")
|
||||
|
||||
@@ -8,7 +8,7 @@ from app.models.user import User
|
||||
from app.models.instance import Instance
|
||||
from app.models.audit_log import AuditLog
|
||||
from app.schemas.query import QueryRequest, QueryResult
|
||||
from app.services.query_service import QueryService
|
||||
from app.services.db_connector import DBConnector
|
||||
from app.core.crypto import decrypt_password
|
||||
|
||||
router = APIRouter(prefix="/query", tags=["SQL 查询"])
|
||||
@@ -32,7 +32,7 @@ async def execute_query(
|
||||
instance.password = decrypt_password(instance.password)
|
||||
|
||||
try:
|
||||
query_result = await QueryService.execute_query(instance, req.database, req.sql, current_user.id)
|
||||
query_result = await DBConnector.execute_query(instance, req.database, req.sql, current_user.id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@@ -239,9 +239,9 @@ async def execute_ticket(
|
||||
if not instance:
|
||||
raise HTTPException(status_code=400, detail="数据源不存在")
|
||||
|
||||
from app.services.query_service import QueryService
|
||||
from app.services.db_connector import DBConnector
|
||||
instance.password = decrypt_password(instance.password)
|
||||
exec_result = await QueryService.execute_dml(instance, ticket.target_database, ticket.sql_content)
|
||||
exec_result = await DBConnector.execute_dml(instance, ticket.target_database, ticket.sql_content)
|
||||
|
||||
ticket.status = "success"
|
||||
ticket.execute_result = exec_result
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Database connector — unified interface for MySQL/PostgreSQL/SQLite.
|
||||
|
||||
Provides real database connections for query execution and connection testing.
|
||||
Replaces mock aiomysql calls with actual driver connections.
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import sqlite3
|
||||
from typing import Optional, List, Dict, Any, Tuple
|
||||
|
||||
from app.models.instance import Instance
|
||||
from app.services.sql_analyzer import SQLAnalyzer
|
||||
|
||||
MAX_ROWS = 10000
|
||||
QUERY_TIMEOUT = 30
|
||||
DML_TIMEOUT = 60
|
||||
|
||||
|
||||
class ConnectionResult:
|
||||
"""Result of a connection test."""
|
||||
def __init__(self, success: bool, message: str, version: str = None, latency_ms: int = 0):
|
||||
self.success = success
|
||||
self.message = message
|
||||
self.version = version
|
||||
self.latency_ms = latency_ms
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"success": self.success,
|
||||
"message": self.message,
|
||||
"version": self.version,
|
||||
"latency_ms": self.latency_ms,
|
||||
}
|
||||
|
||||
|
||||
class QueryResult:
|
||||
"""Result of a SQL query execution."""
|
||||
def __init__(self, columns: List[str], rows: List[Dict[str, Any]],
|
||||
total: int, elapsed_ms: int, truncated: bool = False):
|
||||
self.columns = columns
|
||||
self.rows = rows
|
||||
self.total = total
|
||||
self.elapsed_ms = elapsed_ms
|
||||
self.truncated = truncated
|
||||
|
||||
|
||||
class DBConnector:
|
||||
"""Unified database connector supporting MySQL, PostgreSQL, SQLite."""
|
||||
|
||||
_analyzer = SQLAnalyzer()
|
||||
|
||||
@staticmethod
|
||||
async def test_connection(instance: Instance) -> ConnectionResult:
|
||||
"""Test database connectivity and return version info."""
|
||||
start = time.time()
|
||||
|
||||
try:
|
||||
if instance.db_type == "mysql":
|
||||
return await DBConnector._test_mysql(instance, start)
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
return await DBConnector._test_postgres(instance, start)
|
||||
elif instance.db_type == "sqlite":
|
||||
return await DBConnector._test_sqlite(instance, start)
|
||||
else:
|
||||
return ConnectionResult(False, f"暂不支持 {instance.db_type} 类型数据源")
|
||||
except Exception as e:
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
return ConnectionResult(False, f"连接失败: {str(e)}", latency_ms=elapsed)
|
||||
|
||||
@staticmethod
|
||||
async def execute_query(
|
||||
instance: Instance, database: Optional[str], sql: str, user_id: int
|
||||
) -> 'QueryResult':
|
||||
"""Execute a read-only SQL query on the target database."""
|
||||
# Validate SQL is read-only
|
||||
is_safe, error_msg = DBConnector._analyzer.validate_readonly(sql)
|
||||
if not is_safe:
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Auto-add LIMIT
|
||||
upper_sql = sql.strip().upper()
|
||||
import re
|
||||
if not re.search(r"LIMIT\s+\d+", upper_sql, re.IGNORECASE):
|
||||
sql = sql.rstrip(";") + f" LIMIT {MAX_ROWS};"
|
||||
|
||||
start = time.time()
|
||||
|
||||
if instance.db_type == "mysql":
|
||||
rows, columns = await DBConnector._query_mysql(instance, database, sql)
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
rows, columns = await DBConnector._query_postgres(instance, database, sql)
|
||||
elif instance.db_type == "sqlite":
|
||||
rows, columns = await DBConnector._query_sqlite(instance, sql)
|
||||
else:
|
||||
raise ValueError(f"暂不支持 {instance.db_type} 查询")
|
||||
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
return QueryResult(
|
||||
columns=columns,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
truncated=len(rows) >= MAX_ROWS,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def execute_dml(instance: Instance, database: str, sql: str) -> str:
|
||||
"""Execute a DML statement (INSERT/UPDATE/DELETE)."""
|
||||
if instance.db_type == "mysql":
|
||||
return await DBConnector._dml_mysql(instance, database, sql)
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
return await DBConnector._dml_postgres(instance, database, sql)
|
||||
elif instance.db_type == "sqlite":
|
||||
return await DBConnector._dml_sqlite(instance, sql)
|
||||
else:
|
||||
raise ValueError(f"暂不支持 {instance.db_type} 执行")
|
||||
|
||||
# ===== MySQL =====
|
||||
|
||||
@staticmethod
|
||||
async def _test_mysql(instance: Instance, start: float) -> ConnectionResult:
|
||||
import aiomysql
|
||||
conn = await asyncio.wait_for(
|
||||
aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
db=instance.database,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
try:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute("SELECT VERSION()")
|
||||
version = (await cur.fetchone())[0]
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
return ConnectionResult(True, "连接成功", version=version, latency_ms=elapsed)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
async def _query_mysql(instance: Instance, database: Optional[str], sql: str) -> Tuple[List[Dict], List[str]]:
|
||||
import aiomysql
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
db=database or instance.database,
|
||||
)
|
||||
try:
|
||||
async with conn.cursor(aiomysql.DictCursor) as cur:
|
||||
await asyncio.wait_for(cur.execute(sql), timeout=QUERY_TIMEOUT)
|
||||
rows = await cur.fetchall()
|
||||
columns = [desc[0] for desc in cur.description] if cur.description else []
|
||||
return rows, columns
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("查询超时(30秒)")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
async def _dml_mysql(instance: Instance, database: str, sql: str) -> str:
|
||||
import aiomysql
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
db=database, autocommit=True,
|
||||
)
|
||||
try:
|
||||
async with conn.cursor() as cur:
|
||||
await asyncio.wait_for(cur.execute(sql), timeout=DML_TIMEOUT)
|
||||
affected = cur.rowcount
|
||||
return f"执行成功,影响 {affected} 行"
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("执行超时(60秒)")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ===== PostgreSQL =====
|
||||
|
||||
@staticmethod
|
||||
async def _test_postgres(instance: Instance, start: float) -> ConnectionResult:
|
||||
import asyncpg
|
||||
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{instance.database or 'postgres'}"
|
||||
conn = await asyncio.wait_for(asyncpg.connect(dsn), timeout=10)
|
||||
try:
|
||||
version = await conn.fetchval("SELECT version()")
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
# version string is long, take first part
|
||||
ver_short = version.split(",")[0] if version else "unknown"
|
||||
return ConnectionResult(True, "连接成功", version=ver_short, latency_ms=elapsed)
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
@staticmethod
|
||||
async def _query_postgres(instance: Instance, database: Optional[str], sql: str) -> Tuple[List[Dict], List[str]]:
|
||||
import asyncpg
|
||||
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database or instance.database or 'postgres'}"
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
# asyncpg returns Records, convert to dicts
|
||||
stmt = await asyncio.wait_for(conn.fetch(sql), timeout=QUERY_TIMEOUT)
|
||||
rows = [dict(r) for r in stmt]
|
||||
columns = list(stmt[0].keys()) if stmt else []
|
||||
return rows, columns
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("查询超时(30秒)")
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
@staticmethod
|
||||
async def _dml_postgres(instance: Instance, database: str, sql: str) -> str:
|
||||
import asyncpg
|
||||
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database}"
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
status = await asyncio.wait_for(conn.execute(sql), timeout=DML_TIMEOUT)
|
||||
# status like "UPDATE 3"
|
||||
parts = status.split()
|
||||
affected = int(parts[1]) if len(parts) >= 2 else 0
|
||||
return f"执行成功,影响 {affected} 行"
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("执行超时(60秒)")
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
# ===== SQLite (remote instance) =====
|
||||
|
||||
@staticmethod
|
||||
async def _test_sqlite(instance: Instance, start: float) -> ConnectionResult:
|
||||
# SQLite uses host as file path
|
||||
db_path = instance.host
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
def _connect():
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.execute("SELECT sqlite_version()")
|
||||
ver = cur.fetchone()[0]
|
||||
conn.close()
|
||||
return ver
|
||||
version = await loop.run_in_executor(None, _connect)
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
return ConnectionResult(True, "连接成功", version=f"SQLite {version}", latency_ms=elapsed)
|
||||
except Exception as e:
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
return ConnectionResult(False, f"连接失败: {e}", latency_ms=elapsed)
|
||||
|
||||
@staticmethod
|
||||
async def _query_sqlite(instance: Instance, sql: str) -> Tuple[List[Dict], List[str]]:
|
||||
db_path = instance.host
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _query():
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.execute(sql)
|
||||
rows = [dict(r) for r in cur.fetchall()]
|
||||
columns = [desc[0] for desc in cur.description] if cur.description else []
|
||||
conn.close()
|
||||
return rows, columns
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
loop.run_in_executor(None, _query),
|
||||
timeout=QUERY_TIMEOUT,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("查询超时(30秒)")
|
||||
|
||||
@staticmethod
|
||||
async def _dml_sqlite(instance: Instance, sql: str) -> str:
|
||||
db_path = instance.host
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _execute():
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.execute(sql)
|
||||
affected = cur.rowcount
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return affected
|
||||
|
||||
try:
|
||||
affected = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, _execute),
|
||||
timeout=DML_TIMEOUT,
|
||||
)
|
||||
return f"执行成功,影响 {affected} 行"
|
||||
except asyncio.TimeoutError:
|
||||
raise ValueError("执行超时(60秒)")
|
||||
@@ -1,79 +1,92 @@
|
||||
import asyncio
|
||||
from typing import List, Dict, Any
|
||||
from typing import List
|
||||
import aiomysql
|
||||
|
||||
from app.models.instance import Instance
|
||||
from app.schemas.instance import ConnectionTestResult
|
||||
from app.services.db_connector import DBConnector
|
||||
|
||||
|
||||
class InstanceService:
|
||||
@staticmethod
|
||||
async def test_connection(instance: Instance) -> ConnectionTestResult:
|
||||
try:
|
||||
if instance.db_type == "mysql":
|
||||
import aiomysql
|
||||
conn = await asyncio.wait_for(
|
||||
aiomysql.connect(
|
||||
host=instance.host,
|
||||
port=instance.port,
|
||||
user=instance.username,
|
||||
password=instance.password,
|
||||
database=instance.database or None,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute("SELECT VERSION()")
|
||||
version = (await cur.fetchone())[0]
|
||||
conn.close()
|
||||
return ConnectionTestResult(success=True, message="连接成功", version=str(version))
|
||||
|
||||
return ConnectionTestResult(success=False, message=f"暂不支持 {instance.db_type} 连接测试")
|
||||
except asyncio.TimeoutError:
|
||||
return ConnectionTestResult(success=False, message="连接超时")
|
||||
except Exception as e:
|
||||
return ConnectionTestResult(success=False, message=f"连接失败: {str(e)}")
|
||||
@staticmethod
|
||||
async def test_connection(instance: Instance) -> dict:
|
||||
"""Test database connection using DBConnector."""
|
||||
result = await DBConnector.test_connection(instance)
|
||||
|
||||
# Update instance status based on result
|
||||
if result.success:
|
||||
instance.status = "connected"
|
||||
instance.version = result.version
|
||||
else:
|
||||
instance.status = "error"
|
||||
|
||||
return result.to_dict()
|
||||
|
||||
@staticmethod
|
||||
async def list_databases(instance: Instance) -> List[str]:
|
||||
if instance.db_type != "mysql":
|
||||
"""List databases for a MySQL instance."""
|
||||
if instance.db_type == "mysql":
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
)
|
||||
try:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute("SHOW DATABASES")
|
||||
rows = await cur.fetchall()
|
||||
return [row[0] for row in rows if row[0] not in ("information_schema", "mysql", "performance_schema", "sys")]
|
||||
finally:
|
||||
conn.close()
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
import asyncpg
|
||||
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/postgres"
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
rows = await conn.fetch("SELECT datname FROM pg_database WHERE datistemplate = false")
|
||||
return [r["datname"] for r in rows]
|
||||
finally:
|
||||
await conn.close()
|
||||
elif instance.db_type == "sqlite":
|
||||
return [instance.database or "main"]
|
||||
else:
|
||||
return []
|
||||
|
||||
import aiomysql
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
)
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute("SHOW DATABASES")
|
||||
rows = await cur.fetchall()
|
||||
conn.close()
|
||||
return [row[0] for row in rows if row[0] not in ("information_schema", "mysql", "performance_schema", "sys")]
|
||||
|
||||
@staticmethod
|
||||
async def list_tables(instance: Instance, database: str) -> List[Dict[str, Any]]:
|
||||
if instance.db_type != "mysql":
|
||||
async def list_tables(instance: Instance, database: str) -> List[dict]:
|
||||
"""List tables in a database."""
|
||||
if instance.db_type == "mysql":
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
db=database,
|
||||
)
|
||||
try:
|
||||
async with conn.cursor(aiomysql.DictCursor) as cur:
|
||||
await cur.execute(
|
||||
"SELECT TABLE_NAME as name, TABLE_ROWS as rows, TABLE_TYPE as type "
|
||||
"FROM information_schema.TABLES WHERE TABLE_SCHEMA = %s",
|
||||
(database,)
|
||||
)
|
||||
return await cur.fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
import asyncpg
|
||||
dsn = f"postgresql://{instance.username}:{instance.password}@{instance.host}:{instance.port}/{database}"
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
rows = await conn.fetch(
|
||||
"SELECT tablename as name FROM pg_tables WHERE schemaname = 'public'"
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
await conn.close()
|
||||
elif instance.db_type == "sqlite":
|
||||
import sqlite3
|
||||
db_path = instance.host
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
tables = [{"name": row[0]} for row in cur.fetchall()]
|
||||
conn.close()
|
||||
return tables
|
||||
else:
|
||||
return []
|
||||
|
||||
import aiomysql
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host, port=instance.port,
|
||||
user=instance.username, password=instance.password,
|
||||
database=database,
|
||||
)
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute("SHOW TABLES")
|
||||
tables = [row[0] for row in await cur.fetchall()]
|
||||
|
||||
result = []
|
||||
for table in tables:
|
||||
await cur.execute(f"SHOW TABLE STATUS LIKE '{table}'")
|
||||
status = await cur.fetchone()
|
||||
result.append({
|
||||
"name": table,
|
||||
"engine": status[1],
|
||||
"rows": status[4],
|
||||
"size": status[6],
|
||||
"comment": status[17],
|
||||
})
|
||||
conn.close()
|
||||
return result
|
||||
Reference in new issue
Block a user