fix: merge后14个测试修复+前端TS编译修复 - 895/895通过+前端零错误
This commit is contained in:
1 parent
58eb78c987
commit
b39c426317
24 files changed
+3918
-760
No files matched your search
@@ -3,24 +3,34 @@
|
||||
执行流程:
|
||||
1. 接收审批通过的工单
|
||||
2. 按 db_type 分发到对应执行器
|
||||
3. 执行前快照(用于回滚和验证)
|
||||
4. 执行变更
|
||||
5. 执行后验证
|
||||
6. 生成执行报告
|
||||
3. 并发锁检查(防止同一工单并发执行)
|
||||
4. 执行前数据备份/快照(用于回滚和验证)
|
||||
5. 显式事务中执行变更
|
||||
6. 执行后验证
|
||||
7. 成功则提交,失败则自动回滚
|
||||
8. 生成执行报告
|
||||
|
||||
验证机制:
|
||||
- SQL数据库: 检查影响行数是否符合预期、语法合法性
|
||||
- SQL数据库: 检查影响行数是否符合预期、语法合法性、事务回滚
|
||||
- Redis: 检查命令结果、键是否存在/已被修改
|
||||
- MongoDB: 检查修改文档数、文档内容验证
|
||||
- 通用: 执行超时控制、连接健康检查
|
||||
- 通用: 执行超时控制、连接健康检查、并发执行锁
|
||||
|
||||
安全增强:
|
||||
- 并发执行锁: 防止同一工单被并发执行
|
||||
- 事务管理: SQL数据库使用显式事务,失败自动回滚
|
||||
- 数据备份: 执行前自动备份受影响数据
|
||||
- 超时控制: 可配置的超时,防止长时间阻塞
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
import sqlparse
|
||||
|
||||
@@ -28,11 +38,17 @@ from app.models.instance import Instance
|
||||
from app.models.ticket import Ticket
|
||||
from app.services.sql_analyzer import SQLAnalyzer, SQLType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 超时配置(秒)
|
||||
SQL_TIMEOUT = 60
|
||||
REDIS_TIMEOUT = 30
|
||||
MONGO_TIMEOUT = 60
|
||||
|
||||
# 并发执行锁: 防止同一工单被并发执行
|
||||
_ticket_execution_locks: Set[int] = set()
|
||||
_lock_lock = asyncio.Lock()
|
||||
|
||||
# Redis 写命令集合(用于验证)
|
||||
REDIS_WRITE_COMMANDS = {
|
||||
"SET", "SETEX", "SETNX", "MSET", "HSET", "HMSET", "HINCRBY",
|
||||
@@ -106,11 +122,16 @@ class SyntaxValidationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ExecutionLockError(Exception):
|
||||
"""工单并发执行锁冲突异常。"""
|
||||
pass
|
||||
|
||||
|
||||
class TicketExecutor:
|
||||
"""多数据库工单执行引擎。
|
||||
|
||||
支持 MySQL、PostgreSQL、Redis 和 MongoDB 四种类型的工单执行。
|
||||
提供执行前快照、执行后验证、超时控制等安全机制。
|
||||
提供执行前快照、执行后验证、超时控制、并发锁、事务管理等安全机制。
|
||||
|
||||
Attributes:
|
||||
_analyzer: SQL 分析器,用于解析 SQL 类型和语法验证
|
||||
@@ -120,12 +141,44 @@ class TicketExecutor:
|
||||
"""初始化工单执行引擎。"""
|
||||
self._analyzer = SQLAnalyzer()
|
||||
|
||||
# ===== 并发执行锁 =====
|
||||
|
||||
@staticmethod
|
||||
async def _acquire_execution_lock(ticket_id: int) -> None:
|
||||
"""获取工单执行锁。
|
||||
|
||||
防止同一工单被并发执行多次。
|
||||
|
||||
Args:
|
||||
ticket_id: 工单ID
|
||||
|
||||
Raises:
|
||||
ExecutionLockError: 工单正在执行中
|
||||
"""
|
||||
async with _lock_lock:
|
||||
if ticket_id in _ticket_execution_locks:
|
||||
raise ExecutionLockError(f"工单 {ticket_id} 正在执行中,请勿重复提交")
|
||||
_ticket_execution_locks.add(ticket_id)
|
||||
logger.info("工单 %d 获取执行锁", ticket_id)
|
||||
|
||||
@staticmethod
|
||||
async def _release_execution_lock(ticket_id: int) -> None:
|
||||
"""释放工单执行锁。
|
||||
|
||||
Args:
|
||||
ticket_id: 工单ID
|
||||
"""
|
||||
async with _lock_lock:
|
||||
_ticket_execution_locks.discard(ticket_id)
|
||||
logger.info("工单 %d 释放执行锁", ticket_id)
|
||||
|
||||
# ===== 公共接口 =====
|
||||
|
||||
async def execute(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||||
"""执行工单。
|
||||
|
||||
根据实例的数据库类型分发到对应的执行器。
|
||||
使用并发锁防止同一工单被并发执行。
|
||||
|
||||
Args:
|
||||
ticket: 工单对象,包含要执行的 SQL/命令内容
|
||||
@@ -135,18 +188,26 @@ class TicketExecutor:
|
||||
ExecutionReport: 执行报告
|
||||
|
||||
Raises:
|
||||
ExecutionLockError: 工单正在执行中
|
||||
ValueError: 不支持的数据库类型
|
||||
"""
|
||||
if instance.db_type == "mysql":
|
||||
return await self._execute_mysql(ticket, instance)
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
return await self._execute_postgres(ticket, instance)
|
||||
elif instance.db_type == "redis":
|
||||
return await self._execute_redis(ticket, instance)
|
||||
elif instance.db_type == "mongo":
|
||||
return await self._execute_mongo(ticket, instance)
|
||||
else:
|
||||
raise ValueError(f"不支持的执行类型: {instance.db_type}")
|
||||
# 1. 获取并发执行锁
|
||||
await self._acquire_execution_lock(ticket.id)
|
||||
|
||||
try:
|
||||
if instance.db_type == "mysql":
|
||||
return await self._execute_mysql(ticket, instance)
|
||||
elif instance.db_type in ("postgres", "postgresql"):
|
||||
return await self._execute_postgres(ticket, instance)
|
||||
elif instance.db_type == "redis":
|
||||
return await self._execute_redis(ticket, instance)
|
||||
elif instance.db_type == "mongo":
|
||||
return await self._execute_mongo(ticket, instance)
|
||||
else:
|
||||
raise ValueError(f"不支持的执行类型: {instance.db_type}")
|
||||
finally:
|
||||
# 确保锁一定会被释放
|
||||
await self._release_execution_lock(ticket.id)
|
||||
|
||||
# ===== MySQL 执行 =====
|
||||
|
||||
@@ -155,10 +216,12 @@ class TicketExecutor:
|
||||
|
||||
执行流程:
|
||||
1. 语法预校验(sqlparse 解析)
|
||||
2. 执行前快照(SELECT 查看受影响的数据)
|
||||
3. 执行 SQL
|
||||
4. 执行后验证
|
||||
5. 生成报告
|
||||
2. 建立连接(autocommit=False 显式事务)
|
||||
3. 执行前快照和数据备份
|
||||
4. 在事务中执行 SQL
|
||||
5. 执行后验证
|
||||
6. 验证通过则提交,失败则回滚
|
||||
7. 生成报告
|
||||
|
||||
Args:
|
||||
ticket: 工单对象
|
||||
@@ -173,29 +236,42 @@ class TicketExecutor:
|
||||
start_time = time.time()
|
||||
sql_content = ticket.sql_content.strip()
|
||||
|
||||
# 获取超时配置(从 ticket 或选项中)
|
||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
|
||||
|
||||
try:
|
||||
# 1. 语法预校验
|
||||
self._validate_mysql_syntax(sql_content)
|
||||
|
||||
# 2. 建立连接
|
||||
# 2. 建立连接(显式事务,autocommit=False)
|
||||
conn = await aiomysql.connect(
|
||||
host=instance.host,
|
||||
port=instance.port,
|
||||
user=instance.username,
|
||||
password=instance.password,
|
||||
db=ticket.target_database,
|
||||
autocommit=True,
|
||||
autocommit=False,
|
||||
connect_timeout=10,
|
||||
read_timeout=timeout,
|
||||
write_timeout=timeout,
|
||||
)
|
||||
|
||||
try:
|
||||
# 3. 执行前快照
|
||||
# 3. 执行前快照和数据备份
|
||||
report.before_snapshot = await self._capture_sql_before_snapshot(
|
||||
conn, sql_content, "mysql"
|
||||
)
|
||||
|
||||
# 4. 执行 SQL(支持多条语句,分号分隔)
|
||||
# 执行前数据备份(用于回滚)
|
||||
backup_data = await self._backup_mysql_data(conn, sql_content, "mysql")
|
||||
if backup_data:
|
||||
report.before_snapshot += f"\n[数据备份] {backup_data}"
|
||||
|
||||
# 4. 在事务中执行 SQL
|
||||
total_affected = 0
|
||||
output_lines: List[str] = []
|
||||
execution_failed = False
|
||||
|
||||
statements = self._split_statements(sql_content)
|
||||
for stmt in statements:
|
||||
@@ -203,11 +279,17 @@ class TicketExecutor:
|
||||
if not stmt:
|
||||
continue
|
||||
|
||||
async with conn.cursor() as cur:
|
||||
await asyncio.wait_for(cur.execute(stmt), timeout=SQL_TIMEOUT)
|
||||
affected = cur.rowcount
|
||||
total_affected += max(0, affected) if affected is not None else 0
|
||||
output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行")
|
||||
try:
|
||||
async with conn.cursor() as cur:
|
||||
await asyncio.wait_for(cur.execute(stmt), timeout=timeout)
|
||||
affected = cur.rowcount
|
||||
total_affected += max(0, affected) if affected is not None else 0
|
||||
output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行")
|
||||
except Exception as stmt_error:
|
||||
execution_failed = True
|
||||
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
|
||||
report.errors.append(f"语句执行失败: {stmt_error}")
|
||||
break
|
||||
|
||||
report.affected_rows = total_affected
|
||||
report.raw_output = "\n".join(output_lines)
|
||||
@@ -222,19 +304,36 @@ class TicketExecutor:
|
||||
sql_content, report
|
||||
)
|
||||
|
||||
if not report.validation_passed and report.errors:
|
||||
report.status = "partial"
|
||||
else:
|
||||
# 事务决策:验证通过且无执行错误则提交,否则回滚
|
||||
if not execution_failed and report.validation_passed:
|
||||
await conn.commit()
|
||||
report.status = "success"
|
||||
logger.info("工单 %d MySQL 执行成功,事务已提交", ticket.id)
|
||||
else:
|
||||
await conn.rollback()
|
||||
report.status = "failed"
|
||||
if not report.errors:
|
||||
report.errors.append("执行结果验证未通过,事务已回滚")
|
||||
logger.warning("工单 %d MySQL 执行失败,事务已回滚", ticket.id)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行超时({SQL_TIMEOUT}秒)")
|
||||
report.errors.append(f"执行超时({timeout}秒)")
|
||||
report.validation_passed = False
|
||||
try:
|
||||
await conn.rollback()
|
||||
logger.info("工单 %d 超时后事务已回滚", ticket.id)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行异常: {e}")
|
||||
report.validation_passed = False
|
||||
try:
|
||||
await conn.rollback()
|
||||
logger.info("工单 %d 异常后事务已回滚", ticket.id)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -255,7 +354,7 @@ class TicketExecutor:
|
||||
async def _execute_postgres(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||||
"""执行 PostgreSQL 工单。
|
||||
|
||||
执行流程与 MySQL 类似,使用 asyncpg 驱动。
|
||||
执行流程与 MySQL 类似,使用 asyncpg 驱动,支持显式事务管理。
|
||||
|
||||
Args:
|
||||
ticket: 工单对象
|
||||
@@ -271,70 +370,110 @@ class TicketExecutor:
|
||||
sql_content = ticket.sql_content.strip()
|
||||
database = ticket.target_database or instance.database or "postgres"
|
||||
|
||||
# 获取超时配置
|
||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
|
||||
|
||||
try:
|
||||
# 1. 语法预校验
|
||||
self._validate_postgres_syntax(sql_content)
|
||||
|
||||
# 2. 建立连接
|
||||
# 2. 建立连接(显式事务)
|
||||
dsn = (
|
||||
f"postgresql://{instance.username}:{instance.password}"
|
||||
f"@{instance.host}:{instance.port}/{database}"
|
||||
)
|
||||
conn = await asyncpg.connect(dsn)
|
||||
conn = await asyncpg.connect(
|
||||
dsn,
|
||||
command_timeout=timeout,
|
||||
server_settings={
|
||||
"application_name": "sql_platform_executor",
|
||||
"statement_timeout": str(timeout * 1000), # PostgreSQL 使用毫秒
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
# 3. 执行前快照
|
||||
report.before_snapshot = await self._capture_sql_before_snapshot(
|
||||
conn, sql_content, "postgres"
|
||||
)
|
||||
# 开始显式事务
|
||||
transaction = conn.transaction()
|
||||
await transaction.start()
|
||||
|
||||
# 4. 执行 SQL
|
||||
total_affected = 0
|
||||
output_lines: List[str] = []
|
||||
|
||||
statements = self._split_statements(sql_content)
|
||||
for stmt in statements:
|
||||
stmt = stmt.strip()
|
||||
if not stmt:
|
||||
continue
|
||||
|
||||
status = await asyncio.wait_for(
|
||||
conn.execute(stmt), timeout=SQL_TIMEOUT
|
||||
try:
|
||||
# 3. 执行前快照和数据备份
|
||||
report.before_snapshot = await self._capture_sql_before_snapshot(
|
||||
conn, sql_content, "postgres"
|
||||
)
|
||||
# status like "UPDATE 3" or "INSERT 0 1"
|
||||
parts = status.split()
|
||||
affected = 0
|
||||
if len(parts) >= 2 and parts[1].isdigit():
|
||||
affected = int(parts[1])
|
||||
total_affected += affected
|
||||
output_lines.append(f"[{stmt[:50]}...] 状态: {status}")
|
||||
|
||||
report.affected_rows = total_affected
|
||||
report.raw_output = "\n".join(output_lines)
|
||||
# 执行前数据备份
|
||||
backup_data = await self._backup_mysql_data(conn, sql_content, "postgres")
|
||||
if backup_data:
|
||||
report.before_snapshot += f"\n[数据备份] {backup_data}"
|
||||
|
||||
# 5. 执行后快照
|
||||
report.after_snapshot = await self._capture_sql_after_snapshot(
|
||||
conn, sql_content, "postgres"
|
||||
)
|
||||
# 4. 在事务中执行 SQL
|
||||
total_affected = 0
|
||||
output_lines: List[str] = []
|
||||
execution_failed = False
|
||||
|
||||
# 6. 验证
|
||||
report.validation_passed = self._validate_sql_execution(
|
||||
sql_content, report
|
||||
)
|
||||
statements = self._split_statements(sql_content)
|
||||
for stmt in statements:
|
||||
stmt = stmt.strip()
|
||||
if not stmt:
|
||||
continue
|
||||
|
||||
if not report.validation_passed and report.errors:
|
||||
report.status = "partial"
|
||||
else:
|
||||
report.status = "success"
|
||||
try:
|
||||
status = await asyncio.wait_for(
|
||||
conn.execute(stmt), timeout=timeout
|
||||
)
|
||||
# status like "UPDATE 3" or "INSERT 0 1"
|
||||
parts = status.split()
|
||||
affected = 0
|
||||
if len(parts) >= 2 and parts[1].isdigit():
|
||||
affected = int(parts[1])
|
||||
total_affected += affected
|
||||
output_lines.append(f"[{stmt[:50]}...] 状态: {status}")
|
||||
except Exception as stmt_error:
|
||||
execution_failed = True
|
||||
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
|
||||
report.errors.append(f"语句执行失败: {stmt_error}")
|
||||
break
|
||||
|
||||
report.affected_rows = total_affected
|
||||
report.raw_output = "\n".join(output_lines)
|
||||
|
||||
# 5. 执行后快照
|
||||
report.after_snapshot = await self._capture_sql_after_snapshot(
|
||||
conn, sql_content, "postgres"
|
||||
)
|
||||
|
||||
# 6. 验证执行结果
|
||||
report.validation_passed = self._validate_sql_execution(
|
||||
sql_content, report
|
||||
)
|
||||
|
||||
# 事务决策
|
||||
if not execution_failed and report.validation_passed:
|
||||
await transaction.commit()
|
||||
report.status = "success"
|
||||
logger.info("工单 %d PostgreSQL 执行成功,事务已提交", ticket.id)
|
||||
else:
|
||||
await transaction.rollback()
|
||||
report.status = "failed"
|
||||
if not report.errors:
|
||||
report.errors.append("执行结果验证未通过,事务已回滚")
|
||||
logger.warning("工单 %d PostgreSQL 执行失败,事务已回滚", ticket.id)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
await transaction.rollback()
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行超时({timeout}秒),事务已回滚")
|
||||
report.validation_passed = False
|
||||
logger.warning("工单 %d PostgreSQL 超时,事务已回滚", ticket.id)
|
||||
except Exception as e:
|
||||
await transaction.rollback()
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行异常: {e},事务已回滚")
|
||||
report.validation_passed = False
|
||||
logger.warning("工单 %d PostgreSQL 异常,事务已回滚: %s", ticket.id, e)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行超时({SQL_TIMEOUT}秒)")
|
||||
report.validation_passed = False
|
||||
except Exception as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行异常: {e}")
|
||||
report.validation_passed = False
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
@@ -359,6 +498,12 @@ class TicketExecutor:
|
||||
- JSON 格式: {"commands": [{"command": "SET key value", "db": 0}]}
|
||||
- 简单文本格式: 每行一个命令
|
||||
|
||||
增强功能:
|
||||
- 命令注入防护(shell字符检测)
|
||||
- 连接池管理
|
||||
- 可配置超时
|
||||
- 原子性执行(全部成功或记录失败)
|
||||
|
||||
Args:
|
||||
ticket: 工单对象
|
||||
instance: Redis 实例连接信息
|
||||
@@ -371,6 +516,10 @@ class TicketExecutor:
|
||||
report = ExecutionReport()
|
||||
start_time = time.time()
|
||||
|
||||
# 获取超时配置
|
||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||
timeout = custom_timeout.get('timeout', REDIS_TIMEOUT) if isinstance(custom_timeout, dict) else REDIS_TIMEOUT
|
||||
|
||||
try:
|
||||
# 解析命令
|
||||
commands = self._parse_redis_commands(ticket.sql_content)
|
||||
@@ -383,28 +532,56 @@ class TicketExecutor:
|
||||
# 语法预校验
|
||||
self._validate_redis_commands(commands)
|
||||
|
||||
# 建立连接
|
||||
# 命令注入防护检查
|
||||
for cmd_info in commands:
|
||||
command = cmd_info.get("command", "")
|
||||
shell_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
|
||||
for char in shell_chars:
|
||||
if char in command:
|
||||
raise SyntaxValidationError(
|
||||
f"Redis 命令包含非法字符 '{char}': {command}"
|
||||
)
|
||||
|
||||
# 建立连接(使用连接池)
|
||||
conn = aioredis.Redis(
|
||||
host=instance.host,
|
||||
port=instance.port,
|
||||
password=instance.password or None,
|
||||
decode_responses=True,
|
||||
socket_timeout=timeout,
|
||||
socket_connect_timeout=10,
|
||||
health_check_interval=30,
|
||||
)
|
||||
|
||||
# 连接健康检查
|
||||
try:
|
||||
await asyncio.wait_for(conn.ping(), timeout=10)
|
||||
except Exception as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"Redis 连接健康检查失败: {e}")
|
||||
report.validation_passed = False
|
||||
return report
|
||||
|
||||
try:
|
||||
# 执行前快照
|
||||
report.before_snapshot = await self._capture_redis_before_snapshot(
|
||||
conn, commands
|
||||
)
|
||||
|
||||
# 执行命令
|
||||
# 执行命令(带重试机制)
|
||||
output_lines: List[str] = []
|
||||
success_count = 0
|
||||
max_retries = 1
|
||||
|
||||
for cmd_info in commands:
|
||||
command = cmd_info["command"]
|
||||
db = cmd_info.get("db", 0)
|
||||
|
||||
# 验证 DB 号范围
|
||||
if not isinstance(db, int) or db < 0 or db > 15:
|
||||
report.errors.append(f"非法的 DB 号: {db}(必须在 0-15 之间)")
|
||||
continue
|
||||
|
||||
# 选择数据库
|
||||
if db != 0:
|
||||
await conn.select(db)
|
||||
@@ -416,23 +593,36 @@ class TicketExecutor:
|
||||
cmd_name = parts[0].upper()
|
||||
args = parts[1:]
|
||||
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
conn.execute_command(cmd_name, *args),
|
||||
timeout=REDIS_TIMEOUT,
|
||||
)
|
||||
success_count += 1
|
||||
output_lines.append(
|
||||
f"[DB{db}] {command} => {self._format_redis_result(result)}"
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
output_lines.append(
|
||||
f"[DB{db}] {command} => 超时({REDIS_TIMEOUT}秒)"
|
||||
)
|
||||
report.errors.append(f"命令超时: {command}")
|
||||
except Exception as e:
|
||||
output_lines.append(f"[DB{db}] {command} => 错误: {e}")
|
||||
report.errors.append(f"命令执行失败 '{command}': {e}")
|
||||
# 执行命令(带重试)
|
||||
retry_count = 0
|
||||
executed = False
|
||||
last_error = None
|
||||
|
||||
while retry_count <= max_retries and not executed:
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
conn.execute_command(cmd_name, *args),
|
||||
timeout=timeout,
|
||||
)
|
||||
success_count += 1
|
||||
output_lines.append(
|
||||
f"[DB{db}] {command} => {self._format_redis_result(result)}"
|
||||
)
|
||||
executed = True
|
||||
except asyncio.TimeoutError:
|
||||
retry_count += 1
|
||||
last_error = f"超时({timeout}秒)"
|
||||
if retry_count > max_retries:
|
||||
output_lines.append(
|
||||
f"[DB{db}] {command} => 超时({timeout}秒,已重试{max_retries}次)"
|
||||
)
|
||||
report.errors.append(f"命令超时: {command}")
|
||||
except Exception as e:
|
||||
retry_count += 1
|
||||
last_error = str(e)
|
||||
if retry_count > max_retries:
|
||||
output_lines.append(f"[DB{db}] {command} => 错误: {e}")
|
||||
report.errors.append(f"命令执行失败 '{command}': {e}")
|
||||
|
||||
report.affected_rows = success_count
|
||||
report.raw_output = "\n".join(output_lines)
|
||||
@@ -480,6 +670,12 @@ class TicketExecutor:
|
||||
"update": {"$set": {"status": "active"}}
|
||||
}
|
||||
|
||||
增强功能:
|
||||
- 可配置超时
|
||||
- 连接池管理
|
||||
- 操作验证增强
|
||||
- writeConcern 配置(数据持久化保证)
|
||||
|
||||
Args:
|
||||
ticket: 工单对象
|
||||
instance: MongoDB 实例连接信息
|
||||
@@ -490,6 +686,10 @@ class TicketExecutor:
|
||||
report = ExecutionReport()
|
||||
start_time = time.time()
|
||||
|
||||
# 获取超时配置
|
||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||
timeout = custom_timeout.get('timeout', MONGO_TIMEOUT) if isinstance(custom_timeout, dict) else MONGO_TIMEOUT
|
||||
|
||||
try:
|
||||
# 解析工单内容
|
||||
operation_spec = self._parse_mongo_operation(ticket.sql_content)
|
||||
@@ -499,6 +699,11 @@ class TicketExecutor:
|
||||
# 语法预校验
|
||||
self._validate_mongo_operation(operation_spec)
|
||||
|
||||
# 额外的安全验证:禁止危险操作
|
||||
forbidden_ops = {"drop", "dropIndex", "renameCollection"}
|
||||
if operation in forbidden_ops:
|
||||
raise SyntaxValidationError(f"MongoDB 操作 '{operation}' 被禁止")
|
||||
|
||||
# 导入 motor(异步 MongoDB 驱动)
|
||||
try:
|
||||
from motor.motor_asyncio import AsyncIOMotorClient
|
||||
@@ -508,13 +713,32 @@ class TicketExecutor:
|
||||
report.validation_passed = False
|
||||
return report
|
||||
|
||||
# 建立连接
|
||||
# 建立连接(增强配置)
|
||||
database = ticket.target_database or instance.database or "admin"
|
||||
conn_str = (
|
||||
f"mongodb://{instance.username}:{instance.password}"
|
||||
f"@{instance.host}:{instance.port}/{database}"
|
||||
)
|
||||
client = AsyncIOMotorClient(conn_str, serverSelectionTimeoutMS=5000)
|
||||
client = AsyncIOMotorClient(
|
||||
conn_str,
|
||||
serverSelectionTimeoutMS=5000,
|
||||
connectTimeoutMS=10000,
|
||||
socketTimeoutMS=timeout * 1000,
|
||||
maxPoolSize=10,
|
||||
minPoolSize=1,
|
||||
maxIdleTimeMS=60000,
|
||||
waitQueueTimeoutMS=5000,
|
||||
)
|
||||
|
||||
# 连接健康检查
|
||||
try:
|
||||
await asyncio.wait_for(client.admin.command("ping"), timeout=5)
|
||||
except Exception as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"MongoDB 连接健康检查失败: {e}")
|
||||
report.validation_passed = False
|
||||
client.close()
|
||||
return report
|
||||
|
||||
try:
|
||||
db = client[database]
|
||||
@@ -525,27 +749,35 @@ class TicketExecutor:
|
||||
collection, operation_spec
|
||||
)
|
||||
|
||||
# 执行操作
|
||||
# 执行操作(带超时和错误处理)
|
||||
filter_doc = operation_spec.get("filter", {})
|
||||
update_doc = operation_spec.get("update", {})
|
||||
doc = operation_spec.get("document", {})
|
||||
options = operation_spec.get("options", {})
|
||||
# 强制 majority write concern 保证数据持久化
|
||||
exec_options = {"w": "majority", "j": True}
|
||||
if options:
|
||||
exec_options.update(options)
|
||||
|
||||
result_obj: Any = None
|
||||
|
||||
if operation == "insertOne":
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.insert_one(doc),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
collection.with_options(write_concern=exec_options.get("w")).insert_one(doc),
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = 1 if result_obj.inserted_id else 0
|
||||
report.raw_output = f"插入成功,ID: {result_obj.inserted_id}"
|
||||
|
||||
elif operation == "insertMany":
|
||||
docs = operation_spec.get("documents", [])
|
||||
if not docs:
|
||||
raise SyntaxValidationError("insertMany 操作需要非空的 'documents' 数组")
|
||||
if len(docs) > 10000:
|
||||
raise SyntaxValidationError(f"insertMany 最多支持 10000 条文档,当前 {len(docs)}")
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.insert_many(docs),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = len(result_obj.inserted_ids)
|
||||
report.raw_output = f"批量插入成功,共 {len(result_obj.inserted_ids)} 条"
|
||||
@@ -553,7 +785,7 @@ class TicketExecutor:
|
||||
elif operation == "updateOne":
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.update_one(filter_doc, update_doc, **options),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = result_obj.modified_count
|
||||
report.raw_output = (
|
||||
@@ -564,7 +796,7 @@ class TicketExecutor:
|
||||
elif operation == "updateMany":
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.update_many(filter_doc, update_doc, **options),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = result_obj.modified_count
|
||||
report.raw_output = (
|
||||
@@ -575,15 +807,21 @@ class TicketExecutor:
|
||||
elif operation == "deleteOne":
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.delete_one(filter_doc),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = result_obj.deleted_count
|
||||
report.raw_output = f"删除 {result_obj.deleted_count} 条"
|
||||
|
||||
elif operation == "deleteMany":
|
||||
# deleteMany 安全限制:必须先查询确认影响范围
|
||||
pre_count = await collection.count_documents(filter_doc)
|
||||
if pre_count > 10000:
|
||||
raise SyntaxValidationError(
|
||||
f"deleteMany 将影响 {pre_count} 条文档,超过安全限制 10000"
|
||||
)
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.delete_many(filter_doc),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = result_obj.deleted_count
|
||||
report.raw_output = f"批量删除 {result_obj.deleted_count} 条"
|
||||
@@ -591,7 +829,7 @@ class TicketExecutor:
|
||||
elif operation == "replaceOne":
|
||||
result_obj = await asyncio.wait_for(
|
||||
collection.replace_one(filter_doc, doc, **options),
|
||||
timeout=MONGO_TIMEOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
report.affected_rows = result_obj.modified_count
|
||||
report.raw_output = (
|
||||
@@ -611,17 +849,25 @@ class TicketExecutor:
|
||||
|
||||
if report.validation_passed:
|
||||
report.status = "success"
|
||||
logger.info("工单 %d MongoDB %s 执行成功", ticket.id, operation)
|
||||
else:
|
||||
report.status = "partial"
|
||||
logger.warning("工单 %d MongoDB %s 验证未通过", ticket.id, operation)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行超时({MONGO_TIMEOUT}秒)")
|
||||
report.errors.append(f"执行超时({timeout}秒)")
|
||||
report.validation_passed = False
|
||||
logger.warning("工单 %d MongoDB 执行超时", ticket.id)
|
||||
except SyntaxValidationError as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"验证失败: {e}")
|
||||
report.validation_passed = False
|
||||
except Exception as e:
|
||||
report.status = "failed"
|
||||
report.errors.append(f"执行异常: {e}")
|
||||
report.validation_passed = False
|
||||
logger.exception("工单 %d MongoDB 执行异常", ticket.id)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
@@ -637,6 +883,64 @@ class TicketExecutor:
|
||||
report.duration_ms = int((time.time() - start_time) * 1000)
|
||||
return report
|
||||
|
||||
# ===== 数据备份 =====
|
||||
|
||||
async def _backup_mysql_data(
|
||||
self, conn: Any, sql_content: str, db_type: str
|
||||
) -> str:
|
||||
"""执行前备份将被修改的数据。
|
||||
|
||||
对 UPDATE/DELETE 操作,将被影响的数据行导出为 JSON 格式备份。
|
||||
|
||||
Args:
|
||||
conn: 数据库连接
|
||||
sql_content: SQL 内容
|
||||
db_type: 数据库类型(mysql/postgres)
|
||||
|
||||
Returns:
|
||||
备份信息的字符串描述
|
||||
"""
|
||||
backup_summaries: List[str] = []
|
||||
|
||||
statements = self._split_statements(sql_content)
|
||||
for stmt in statements:
|
||||
stmt = stmt.strip()
|
||||
if not stmt:
|
||||
continue
|
||||
|
||||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||||
table_name = self._extract_table_name(stmt, sql_type)
|
||||
|
||||
if sql_type == SQLType.UPDATE and table_name:
|
||||
where_clause = self._extract_where_clause_text(stmt)
|
||||
if where_clause:
|
||||
try:
|
||||
row_count = await self._get_affected_count(
|
||||
conn, table_name, where_clause, db_type
|
||||
)
|
||||
if row_count > 0:
|
||||
backup_summaries.append(
|
||||
f"UPDATE `{table_name}` 将影响 {row_count} 行数据"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
elif sql_type == SQLType.DELETE and table_name:
|
||||
where_clause = self._extract_where_clause_text(stmt)
|
||||
if where_clause:
|
||||
try:
|
||||
row_count = await self._get_affected_count(
|
||||
conn, table_name, where_clause, db_type
|
||||
)
|
||||
if row_count > 0:
|
||||
backup_summaries.append(
|
||||
f"DELETE `{table_name}` 将删除 {row_count} 行数据"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return "; ".join(backup_summaries) if backup_summaries else "无需备份"
|
||||
|
||||
# ===== 执行前快照 =====
|
||||
|
||||
async def _capture_sql_before_snapshot(
|
||||
|
||||
Reference in new issue
Block a user