新增: - explain_preview.py: EXPLAIN预检服务(MySQL/PG), 影响行数预估, 全表扫描检测 - ticket.executor: 三种执行策略(stop_on_error/continue/transaction) - risk_evaluator: table_sizes参数, 大表ALTER/UPDATE/DELETE自动升级high - test_phase2_features.py: 22个测试(EXPLAIN+执行策略+大表检测) 简化: - 砍掉CTX-001上下文感知规则(复杂二次扫描), 合并大表检测到risk_evaluator - 审核引擎保持纯静态分析, 不再导入_RULE_REGISTRY做二次扫描 模型: - Ticket加exec_strategy字段 - TicketSqlItem模型 - schema加sql_type/exec_strategy/risk_level/workflow_id
1706 lines
63 KiB
Python
1706 lines
63 KiB
Python
"""多数据库工单执行引擎 — 支持 MySQL/PostgreSQL/Redis/MongoDB 的变更执行。
|
||
|
||
执行流程:
|
||
1. 接收审批通过的工单
|
||
2. 按 db_type 分发到对应执行器
|
||
3. 并发锁检查(防止同一工单并发执行)
|
||
4. 执行前数据备份/快照(用于回滚和验证)
|
||
5. 显式事务中执行变更
|
||
6. 执行后验证
|
||
7. 成功则提交,失败则自动回滚
|
||
8. 生成执行报告
|
||
|
||
验证机制:
|
||
- SQL数据库: 检查影响行数是否符合预期、语法合法性、事务回滚
|
||
- Redis: 检查命令结果、键是否存在/已被修改
|
||
- MongoDB: 检查修改文档数、文档内容验证
|
||
- 通用: 执行超时控制、连接健康检查、并发执行锁
|
||
|
||
安全增强:
|
||
- 并发执行锁: 防止同一工单被并发执行
|
||
- 事务管理: SQL数据库使用显式事务,失败自动回滚
|
||
- 数据备份: 执行前自动备份受影响数据
|
||
- 超时控制: 可配置的超时,防止长时间阻塞
|
||
"""
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Dict, List, Optional, Set
|
||
|
||
import sqlparse
|
||
|
||
from app.models.instance import Instance
|
||
from app.models.ticket import Ticket
|
||
from app.services.sql_analyzer import SQLAnalyzer, SQLType
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 超时配置(秒)—— 默认值,优先从 system_settings 读取
|
||
SQL_TIMEOUT = 60
|
||
REDIS_TIMEOUT = 30
|
||
MONGO_TIMEOUT = 60
|
||
|
||
|
||
def _get_executor_timeout(db_type: str) -> int:
|
||
"""从 query_service 配置缓存获取执行超时。"""
|
||
from app.services.query_service import current_config
|
||
cfg = current_config()
|
||
key = {"mysql": "sql_timeout", "postgres": "sql_timeout", "redis": "redis_timeout", "mongodb": "mongo_timeout"}.get(db_type, "sql_timeout")
|
||
return cfg.get(key, SQL_TIMEOUT)
|
||
|
||
# 并发执行锁: 防止同一工单被并发执行
|
||
_ticket_execution_locks: Set[int] = set()
|
||
_lock_lock = asyncio.Lock()
|
||
|
||
# Redis 写命令集合(用于验证)
|
||
REDIS_WRITE_COMMANDS = {
|
||
"SET", "SETEX", "SETNX", "MSET", "HSET", "HMSET", "HINCRBY",
|
||
"LPUSH", "RPUSH", "LPOP", "RPOP", "LREM", "LSET", "LTRIM",
|
||
"SADD", "SREM", "SPOP", "SMOVE", "ZADD", "ZREM", "ZINCRBY",
|
||
"ZREMRANGEBYSCORE", "ZREMRANGEBYRANK", "DEL", "HDEL",
|
||
"EXPIRE", "EXPIREAT", "PERSIST", "RENAME", "RENAMENX",
|
||
"INCR", "INCRBY", "DECR", "DECRBY", "APPEND", "SETBIT",
|
||
"FLUSHDB", "FLUSHALL",
|
||
}
|
||
|
||
# MongoDB 支持的操作类型
|
||
MONGO_SUPPORTED_OPERATIONS = {
|
||
"insertOne", "insertMany",
|
||
"updateOne", "updateMany",
|
||
"deleteOne", "deleteMany",
|
||
"replaceOne",
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class ExecutionReport:
|
||
"""执行报告,记录工单执行的完整结果。"""
|
||
|
||
status: str = "success" # success / failed / partial
|
||
affected_rows: int = 0 # 影响行数
|
||
duration_ms: int = 0 # 执行耗时
|
||
before_snapshot: str = "" # 执行前快照
|
||
after_snapshot: str = "" # 执行后快照
|
||
validation_passed: bool = True # 验证是否通过
|
||
warnings: List[str] = field(default_factory=list) # 警告信息
|
||
errors: List[str] = field(default_factory=list) # 错误信息
|
||
raw_output: str = "" # 原始输出
|
||
|
||
def to_json(self) -> str:
|
||
"""将执行报告序列化为 JSON 字符串。"""
|
||
return json.dumps(
|
||
{
|
||
"status": self.status,
|
||
"affected_rows": self.affected_rows,
|
||
"duration_ms": self.duration_ms,
|
||
"before_snapshot": self.before_snapshot,
|
||
"after_snapshot": self.after_snapshot,
|
||
"validation_passed": self.validation_passed,
|
||
"warnings": self.warnings,
|
||
"errors": self.errors,
|
||
"raw_output": self.raw_output,
|
||
},
|
||
ensure_ascii=False,
|
||
default=str,
|
||
)
|
||
|
||
@classmethod
|
||
def from_exception(cls, exc: Exception, stage: str = "执行") -> "ExecutionReport":
|
||
"""从异常创建失败的执行报告。"""
|
||
return cls(
|
||
status="failed",
|
||
errors=[f"{stage}失败: {type(exc).__name__}: {exc}"],
|
||
duration_ms=0,
|
||
validation_passed=False,
|
||
)
|
||
|
||
|
||
class TicketExecutorError(Exception):
|
||
"""工单执行引擎异常。"""
|
||
pass
|
||
|
||
|
||
class SyntaxValidationError(Exception):
|
||
"""语法预校验异常。"""
|
||
pass
|
||
|
||
|
||
class ExecutionLockError(Exception):
|
||
"""工单并发执行锁冲突异常。"""
|
||
pass
|
||
|
||
|
||
class TicketExecutor:
|
||
"""多数据库工单执行引擎。
|
||
|
||
支持 MySQL、PostgreSQL、Redis 和 MongoDB 四种类型的工单执行。
|
||
提供执行前快照、执行后验证、超时控制、并发锁、事务管理等安全机制。
|
||
|
||
Attributes:
|
||
_analyzer: SQL 分析器,用于解析 SQL 类型和语法验证
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
"""初始化工单执行引擎。"""
|
||
self._analyzer = SQLAnalyzer()
|
||
|
||
# ===== 并发执行锁 =====
|
||
|
||
@staticmethod
|
||
async def _acquire_execution_lock(ticket_id: int) -> None:
|
||
"""获取工单执行锁。
|
||
|
||
防止同一工单被并发执行多次。
|
||
|
||
Args:
|
||
ticket_id: 工单ID
|
||
|
||
Raises:
|
||
ExecutionLockError: 工单正在执行中
|
||
"""
|
||
async with _lock_lock:
|
||
if ticket_id in _ticket_execution_locks:
|
||
raise ExecutionLockError(f"工单 {ticket_id} 正在执行中,请勿重复提交")
|
||
_ticket_execution_locks.add(ticket_id)
|
||
logger.info("工单 %d 获取执行锁", ticket_id)
|
||
|
||
@staticmethod
|
||
async def _release_execution_lock(ticket_id: int) -> None:
|
||
"""释放工单执行锁。
|
||
|
||
Args:
|
||
ticket_id: 工单ID
|
||
"""
|
||
async with _lock_lock:
|
||
_ticket_execution_locks.discard(ticket_id)
|
||
logger.info("工单 %d 释放执行锁", ticket_id)
|
||
|
||
# ===== 公共接口 =====
|
||
|
||
async def execute(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||
"""执行工单。
|
||
|
||
根据实例的数据库类型分发到对应的执行器。
|
||
使用并发锁防止同一工单被并发执行。
|
||
|
||
Args:
|
||
ticket: 工单对象,包含要执行的 SQL/命令内容
|
||
instance: 数据源实例,包含连接信息
|
||
|
||
Returns:
|
||
ExecutionReport: 执行报告
|
||
|
||
Raises:
|
||
ExecutionLockError: 工单正在执行中
|
||
ValueError: 不支持的数据库类型
|
||
"""
|
||
# 1. 获取并发执行锁
|
||
await self._acquire_execution_lock(ticket.id)
|
||
|
||
try:
|
||
if instance.db_type == "mysql":
|
||
return await self._execute_mysql(ticket, instance)
|
||
elif instance.db_type in ("postgres", "postgresql"):
|
||
return await self._execute_postgres(ticket, instance)
|
||
elif instance.db_type == "redis":
|
||
return await self._execute_redis(ticket, instance)
|
||
elif instance.db_type == "mongo":
|
||
return await self._execute_mongo(ticket, instance)
|
||
else:
|
||
raise ValueError(f"不支持的执行类型: {instance.db_type}")
|
||
finally:
|
||
# 确保锁一定会被释放
|
||
await self._release_execution_lock(ticket.id)
|
||
|
||
# ===== MySQL 执行 =====
|
||
|
||
async def _execute_mysql(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||
"""执行 MySQL 工单。"""
|
||
import aiomysql
|
||
|
||
report = ExecutionReport()
|
||
start_time = time.time()
|
||
sql_content = ticket.sql_content.strip()
|
||
exec_strategy = getattr(ticket, "exec_strategy", "stop_on_error") or "stop_on_error"
|
||
|
||
# 获取超时配置(从 ticket 或选项中)
|
||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||
_default_timeout = _get_executor_timeout('mysql')
|
||
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||
|
||
try:
|
||
# 1. 语法预校验
|
||
self._validate_mysql_syntax(sql_content)
|
||
|
||
# 2. 建立连接(显式事务,autocommit=False)
|
||
conn = await aiomysql.connect(
|
||
host=instance.host,
|
||
port=instance.port,
|
||
user=instance.username,
|
||
password=instance.password,
|
||
db=ticket.target_database,
|
||
autocommit=False,
|
||
connect_timeout=10,
|
||
)
|
||
|
||
try:
|
||
# 3. 执行前快照和数据备份
|
||
report.before_snapshot = await self._capture_sql_before_snapshot(
|
||
conn, sql_content, "mysql"
|
||
)
|
||
|
||
# 执行前数据备份(用于回滚)
|
||
backup_data = await self._backup_mysql_data(conn, sql_content, "mysql")
|
||
if backup_data:
|
||
report.before_snapshot += f"\n[数据备份] {backup_data}"
|
||
|
||
# 4. 在事务中执行 SQL
|
||
total_affected = 0
|
||
output_lines: List[str] = []
|
||
execution_failed = False
|
||
failed_count = 0
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
try:
|
||
async with conn.cursor() as cur:
|
||
await asyncio.wait_for(cur.execute(stmt), timeout=timeout)
|
||
affected = cur.rowcount
|
||
total_affected += max(0, affected) if affected is not None else 0
|
||
output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行")
|
||
except Exception as stmt_error:
|
||
failed_count += 1
|
||
execution_failed = True
|
||
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
|
||
report.errors.append(f"语句执行失败: {stmt_error}")
|
||
|
||
if exec_strategy == "stop_on_error":
|
||
break
|
||
# continue: 跳过失败继续执行
|
||
# transaction: 任何失败都回滚(默认行为,break 后走 rollback)
|
||
|
||
report.affected_rows = total_affected
|
||
report.raw_output = "\n".join(output_lines)
|
||
|
||
# 5. 执行后快照
|
||
report.after_snapshot = await self._capture_sql_after_snapshot(
|
||
conn, sql_content, "mysql"
|
||
)
|
||
|
||
# 6. 验证执行结果
|
||
report.validation_passed = self._validate_sql_execution(
|
||
sql_content, report
|
||
)
|
||
|
||
# 事务决策:根据执行策略处理
|
||
if exec_strategy == "continue" and failed_count > 0 and failed_count < len(statements):
|
||
# continue 模式:部分失败也算成功,提交已执行的语句
|
||
await conn.commit()
|
||
report.status = "partial_success"
|
||
report.errors.insert(0, f"部分失败: {failed_count}/{len(statements)} 条语句失败")
|
||
logger.warning("工单 %d MySQL 部分执行成功: %d 条失败", ticket.id, failed_count)
|
||
elif not execution_failed and report.validation_passed:
|
||
await conn.commit()
|
||
report.status = "success"
|
||
logger.info("工单 %d MySQL 执行成功,事务已提交", ticket.id)
|
||
else:
|
||
await conn.rollback()
|
||
report.status = "failed"
|
||
if not report.errors:
|
||
report.errors.append("执行结果验证未通过,事务已回滚")
|
||
logger.warning("工单 %d MySQL 执行失败,事务已回滚", ticket.id)
|
||
|
||
except asyncio.TimeoutError:
|
||
report.status = "failed"
|
||
report.errors.append(f"执行超时({timeout}秒)")
|
||
report.validation_passed = False
|
||
try:
|
||
await conn.rollback()
|
||
logger.info("工单 %d 超时后事务已回滚", ticket.id)
|
||
except Exception:
|
||
pass
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"执行异常: {e}")
|
||
report.validation_passed = False
|
||
try:
|
||
await conn.rollback()
|
||
logger.info("工单 %d 异常后事务已回滚", ticket.id)
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
conn.close()
|
||
|
||
except SyntaxValidationError as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"语法校验失败: {e}")
|
||
report.validation_passed = False
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"连接异常: {e}")
|
||
report.validation_passed = False
|
||
|
||
report.duration_ms = int((time.time() - start_time) * 1000)
|
||
return report
|
||
|
||
# ===== PostgreSQL 执行 =====
|
||
|
||
async def _execute_postgres(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||
"""执行 PostgreSQL 工单。
|
||
|
||
执行流程与 MySQL 类似,使用 asyncpg 驱动,支持显式事务管理。
|
||
|
||
Args:
|
||
ticket: 工单对象
|
||
instance: PostgreSQL 实例连接信息
|
||
|
||
Returns:
|
||
ExecutionReport: 执行报告
|
||
"""
|
||
import asyncpg
|
||
|
||
report = ExecutionReport()
|
||
start_time = time.time()
|
||
sql_content = ticket.sql_content.strip()
|
||
database = ticket.target_database or instance.database or "postgres"
|
||
|
||
# 获取超时配置
|
||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||
_default_timeout = _get_executor_timeout('postgres')
|
||
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||
|
||
try:
|
||
# 1. 语法预校验
|
||
self._validate_postgres_syntax(sql_content)
|
||
|
||
# 2. 建立连接(显式事务)
|
||
dsn = (
|
||
f"postgresql://{instance.username}:{instance.password}"
|
||
f"@{instance.host}:{instance.port}/{database}"
|
||
)
|
||
conn = await asyncpg.connect(
|
||
dsn,
|
||
command_timeout=timeout,
|
||
server_settings={
|
||
"application_name": "sql_platform_executor",
|
||
"statement_timeout": str(timeout * 1000), # PostgreSQL 使用毫秒
|
||
},
|
||
)
|
||
|
||
try:
|
||
# 开始显式事务
|
||
transaction = conn.transaction()
|
||
await transaction.start()
|
||
|
||
try:
|
||
# 3. 执行前快照和数据备份
|
||
report.before_snapshot = await self._capture_sql_before_snapshot(
|
||
conn, sql_content, "postgres"
|
||
)
|
||
|
||
# 执行前数据备份
|
||
backup_data = await self._backup_mysql_data(conn, sql_content, "postgres")
|
||
if backup_data:
|
||
report.before_snapshot += f"\n[数据备份] {backup_data}"
|
||
|
||
# 4. 在事务中执行 SQL
|
||
total_affected = 0
|
||
output_lines: List[str] = []
|
||
execution_failed = False
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
try:
|
||
status = await asyncio.wait_for(
|
||
conn.execute(stmt), timeout=timeout
|
||
)
|
||
# status like "UPDATE 3" or "INSERT 0 1"
|
||
parts = status.split()
|
||
affected = 0
|
||
if len(parts) >= 2 and parts[1].isdigit():
|
||
affected = int(parts[1])
|
||
total_affected += affected
|
||
output_lines.append(f"[{stmt[:50]}...] 状态: {status}")
|
||
except Exception as stmt_error:
|
||
execution_failed = True
|
||
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
|
||
report.errors.append(f"语句执行失败: {stmt_error}")
|
||
break
|
||
|
||
report.affected_rows = total_affected
|
||
report.raw_output = "\n".join(output_lines)
|
||
|
||
# 5. 执行后快照
|
||
report.after_snapshot = await self._capture_sql_after_snapshot(
|
||
conn, sql_content, "postgres"
|
||
)
|
||
|
||
# 6. 验证执行结果
|
||
report.validation_passed = self._validate_sql_execution(
|
||
sql_content, report
|
||
)
|
||
|
||
# 事务决策
|
||
if not execution_failed and report.validation_passed:
|
||
await transaction.commit()
|
||
report.status = "success"
|
||
logger.info("工单 %d PostgreSQL 执行成功,事务已提交", ticket.id)
|
||
else:
|
||
await transaction.rollback()
|
||
report.status = "failed"
|
||
if not report.errors:
|
||
report.errors.append("执行结果验证未通过,事务已回滚")
|
||
logger.warning("工单 %d PostgreSQL 执行失败,事务已回滚", ticket.id)
|
||
|
||
except asyncio.TimeoutError:
|
||
await transaction.rollback()
|
||
report.status = "failed"
|
||
report.errors.append(f"执行超时({timeout}秒),事务已回滚")
|
||
report.validation_passed = False
|
||
logger.warning("工单 %d PostgreSQL 超时,事务已回滚", ticket.id)
|
||
except Exception as e:
|
||
await transaction.rollback()
|
||
report.status = "failed"
|
||
report.errors.append(f"执行异常: {e},事务已回滚")
|
||
report.validation_passed = False
|
||
logger.warning("工单 %d PostgreSQL 异常,事务已回滚: %s", ticket.id, e)
|
||
|
||
finally:
|
||
await conn.close()
|
||
|
||
except SyntaxValidationError as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"语法校验失败: {e}")
|
||
report.validation_passed = False
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"连接异常: {e}")
|
||
report.validation_passed = False
|
||
|
||
report.duration_ms = int((time.time() - start_time) * 1000)
|
||
return report
|
||
|
||
# ===== Redis 执行 =====
|
||
|
||
async def _execute_redis(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||
"""执行 Redis 工单。
|
||
|
||
Redis 工单格式支持:
|
||
- JSON 格式: {"commands": [{"command": "SET key value", "db": 0}]}
|
||
- 简单文本格式: 每行一个命令
|
||
|
||
增强功能:
|
||
- 命令注入防护(shell字符检测)
|
||
- 连接池管理
|
||
- 可配置超时
|
||
- 原子性执行(全部成功或记录失败)
|
||
|
||
Args:
|
||
ticket: 工单对象
|
||
instance: Redis 实例连接信息
|
||
|
||
Returns:
|
||
ExecutionReport: 执行报告
|
||
"""
|
||
import redis.asyncio as aioredis
|
||
|
||
report = ExecutionReport()
|
||
start_time = time.time()
|
||
|
||
# 获取超时配置
|
||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||
_default_timeout = _get_executor_timeout('redis')
|
||
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||
|
||
try:
|
||
# 解析命令
|
||
commands = self._parse_redis_commands(ticket.sql_content, ticket.target_database)
|
||
if not commands:
|
||
report.status = "failed"
|
||
report.errors.append("未解析到有效的 Redis 命令")
|
||
report.validation_passed = False
|
||
return report
|
||
|
||
# 语法预校验
|
||
self._validate_redis_commands(commands)
|
||
|
||
# 命令注入防护检查
|
||
for cmd_info in commands:
|
||
command = cmd_info.get("command", "")
|
||
shell_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
|
||
for char in shell_chars:
|
||
if char in command:
|
||
raise SyntaxValidationError(
|
||
f"Redis 命令包含非法字符 '{char}': {command}"
|
||
)
|
||
|
||
# 建立连接(使用连接池)
|
||
conn = aioredis.Redis(
|
||
host=instance.host,
|
||
port=instance.port,
|
||
password=instance.password or None,
|
||
decode_responses=True,
|
||
socket_timeout=timeout,
|
||
socket_connect_timeout=10,
|
||
health_check_interval=30,
|
||
)
|
||
|
||
# 连接健康检查
|
||
try:
|
||
await asyncio.wait_for(conn.ping(), timeout=10)
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"Redis 连接健康检查失败: {e}")
|
||
report.validation_passed = False
|
||
return report
|
||
|
||
try:
|
||
# 执行前快照
|
||
report.before_snapshot = await self._capture_redis_before_snapshot(
|
||
conn, commands
|
||
)
|
||
|
||
# 执行命令(带重试机制)
|
||
output_lines: List[str] = []
|
||
success_count = 0
|
||
max_retries = 1
|
||
|
||
for cmd_info in commands:
|
||
command = cmd_info["command"]
|
||
db = cmd_info.get("db", 0)
|
||
|
||
# 验证 DB 号范围
|
||
if not isinstance(db, int) or db < 0 or db > 15:
|
||
report.errors.append(f"非法的 DB 号: {db}(必须在 0-15 之间)")
|
||
continue
|
||
|
||
# 选择数据库
|
||
if db != 0:
|
||
await conn.select(db)
|
||
|
||
parts = command.strip().split()
|
||
if not parts:
|
||
continue
|
||
|
||
cmd_name = parts[0].upper()
|
||
args = parts[1:]
|
||
|
||
# 执行命令(带重试)
|
||
retry_count = 0
|
||
executed = False
|
||
last_error = None
|
||
|
||
while retry_count <= max_retries and not executed:
|
||
try:
|
||
result = await asyncio.wait_for(
|
||
conn.execute_command(cmd_name, *args),
|
||
timeout=timeout,
|
||
)
|
||
success_count += 1
|
||
output_lines.append(
|
||
f"[DB{db}] {command} => {self._format_redis_result(result)}"
|
||
)
|
||
executed = True
|
||
except asyncio.TimeoutError:
|
||
retry_count += 1
|
||
last_error = f"超时({timeout}秒)"
|
||
if retry_count > max_retries:
|
||
output_lines.append(
|
||
f"[DB{db}] {command} => 超时({timeout}秒,已重试{max_retries}次)"
|
||
)
|
||
report.errors.append(f"命令超时: {command}")
|
||
except Exception as e:
|
||
retry_count += 1
|
||
last_error = str(e)
|
||
if retry_count > max_retries:
|
||
output_lines.append(f"[DB{db}] {command} => 错误: {e}")
|
||
report.errors.append(f"命令执行失败 '{command}': {e}")
|
||
|
||
report.affected_rows = success_count
|
||
report.raw_output = "\n".join(output_lines)
|
||
|
||
# 执行后快照
|
||
report.after_snapshot = await self._capture_redis_after_snapshot(
|
||
conn, commands
|
||
)
|
||
|
||
# 验证
|
||
report.validation_passed = self._validate_redis_execution(
|
||
commands, report
|
||
)
|
||
|
||
if report.errors:
|
||
report.status = "partial" if success_count > 0 else "failed"
|
||
else:
|
||
report.status = "success"
|
||
|
||
finally:
|
||
await conn.close()
|
||
|
||
except SyntaxValidationError as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"语法校验失败: {e}")
|
||
report.validation_passed = False
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"连接异常: {e}")
|
||
report.validation_passed = False
|
||
|
||
report.duration_ms = int((time.time() - start_time) * 1000)
|
||
return report
|
||
|
||
# ===== MongoDB 执行 =====
|
||
|
||
async def _execute_mongo(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
|
||
"""执行 MongoDB 工单。
|
||
|
||
MongoDB 工单格式(JSON):
|
||
{
|
||
"collection": "users",
|
||
"operation": "updateMany",
|
||
"filter": {"status": "inactive"},
|
||
"update": {"$set": {"status": "active"}}
|
||
}
|
||
|
||
增强功能:
|
||
- 可配置超时
|
||
- 连接池管理
|
||
- 操作验证增强
|
||
- writeConcern 配置(数据持久化保证)
|
||
|
||
Args:
|
||
ticket: 工单对象
|
||
instance: MongoDB 实例连接信息
|
||
|
||
Returns:
|
||
ExecutionReport: 执行报告
|
||
"""
|
||
report = ExecutionReport()
|
||
start_time = time.time()
|
||
|
||
# 获取超时配置
|
||
custom_timeout = getattr(ticket, 'options', None) or {}
|
||
_default_timeout = _get_executor_timeout('mongodb')
|
||
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||
|
||
try:
|
||
# 解析工单内容
|
||
operation_spec = self._parse_mongo_operation(ticket.sql_content)
|
||
collection_name = operation_spec["collection"]
|
||
operation = operation_spec["operation"]
|
||
|
||
# 语法预校验
|
||
self._validate_mongo_operation(operation_spec)
|
||
|
||
# 额外的安全验证:禁止危险操作
|
||
forbidden_ops = {"drop", "dropIndex", "renameCollection"}
|
||
if operation in forbidden_ops:
|
||
raise SyntaxValidationError(f"MongoDB 操作 '{operation}' 被禁止")
|
||
|
||
# 导入 motor(异步 MongoDB 驱动)
|
||
try:
|
||
from motor.motor_asyncio import AsyncIOMotorClient
|
||
except ImportError:
|
||
report.status = "failed"
|
||
report.errors.append("缺少 motor 依赖,请安装: pip install motor")
|
||
report.validation_passed = False
|
||
return report
|
||
|
||
# 建立连接(增强配置)
|
||
database = ticket.target_database or instance.database or "admin"
|
||
conn_str = (
|
||
f"mongodb://{instance.username}:{instance.password}"
|
||
f"@{instance.host}:{instance.port}/{database}"
|
||
)
|
||
client = AsyncIOMotorClient(
|
||
conn_str,
|
||
serverSelectionTimeoutMS=5000,
|
||
connectTimeoutMS=10000,
|
||
socketTimeoutMS=timeout * 1000,
|
||
maxPoolSize=10,
|
||
minPoolSize=1,
|
||
maxIdleTimeMS=60000,
|
||
waitQueueTimeoutMS=5000,
|
||
)
|
||
|
||
# 连接健康检查
|
||
try:
|
||
await asyncio.wait_for(client.admin.command("ping"), timeout=5)
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"MongoDB 连接健康检查失败: {e}")
|
||
report.validation_passed = False
|
||
client.close()
|
||
return report
|
||
|
||
try:
|
||
db = client[database]
|
||
collection = db[collection_name]
|
||
|
||
# 执行前快照
|
||
report.before_snapshot = await self._capture_mongo_before_snapshot(
|
||
collection, operation_spec
|
||
)
|
||
|
||
# 执行操作(带超时和错误处理)
|
||
filter_doc = operation_spec.get("filter", {})
|
||
update_doc = operation_spec.get("update", {})
|
||
doc = operation_spec.get("document", {})
|
||
options = operation_spec.get("options", {})
|
||
# 强制 majority write concern 保证数据持久化
|
||
exec_options = {"w": "majority", "j": True}
|
||
if options:
|
||
exec_options.update(options)
|
||
|
||
result_obj: Any = None
|
||
|
||
if operation == "insertOne":
|
||
result_obj = await asyncio.wait_for(
|
||
collection.with_options(write_concern=exec_options.get("w")).insert_one(doc),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = 1 if result_obj.inserted_id else 0
|
||
report.raw_output = f"插入成功,ID: {result_obj.inserted_id}"
|
||
|
||
elif operation == "insertMany":
|
||
docs = operation_spec.get("documents", [])
|
||
if not docs:
|
||
raise SyntaxValidationError("insertMany 操作需要非空的 'documents' 数组")
|
||
if len(docs) > 10000:
|
||
raise SyntaxValidationError(f"insertMany 最多支持 10000 条文档,当前 {len(docs)}")
|
||
result_obj = await asyncio.wait_for(
|
||
collection.insert_many(docs),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = len(result_obj.inserted_ids)
|
||
report.raw_output = f"批量插入成功,共 {len(result_obj.inserted_ids)} 条"
|
||
|
||
elif operation == "updateOne":
|
||
result_obj = await asyncio.wait_for(
|
||
collection.update_one(filter_doc, update_doc, **options),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = result_obj.modified_count
|
||
report.raw_output = (
|
||
f"匹配 {result_obj.matched_count} 条,"
|
||
f"修改 {result_obj.modified_count} 条"
|
||
)
|
||
|
||
elif operation == "updateMany":
|
||
result_obj = await asyncio.wait_for(
|
||
collection.update_many(filter_doc, update_doc, **options),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = result_obj.modified_count
|
||
report.raw_output = (
|
||
f"匹配 {result_obj.matched_count} 条,"
|
||
f"修改 {result_obj.modified_count} 条"
|
||
)
|
||
|
||
elif operation == "deleteOne":
|
||
result_obj = await asyncio.wait_for(
|
||
collection.delete_one(filter_doc),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = result_obj.deleted_count
|
||
report.raw_output = f"删除 {result_obj.deleted_count} 条"
|
||
|
||
elif operation == "deleteMany":
|
||
# deleteMany 安全限制:必须先查询确认影响范围
|
||
pre_count = await collection.count_documents(filter_doc)
|
||
if pre_count > 10000:
|
||
raise SyntaxValidationError(
|
||
f"deleteMany 将影响 {pre_count} 条文档,超过安全限制 10000"
|
||
)
|
||
result_obj = await asyncio.wait_for(
|
||
collection.delete_many(filter_doc),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = result_obj.deleted_count
|
||
report.raw_output = f"批量删除 {result_obj.deleted_count} 条"
|
||
|
||
elif operation == "replaceOne":
|
||
result_obj = await asyncio.wait_for(
|
||
collection.replace_one(filter_doc, doc, **options),
|
||
timeout=timeout,
|
||
)
|
||
report.affected_rows = result_obj.modified_count
|
||
report.raw_output = (
|
||
f"匹配 {result_obj.matched_count} 条,"
|
||
f"替换 {result_obj.modified_count} 条"
|
||
)
|
||
|
||
# 执行后快照
|
||
report.after_snapshot = await self._capture_mongo_after_snapshot(
|
||
collection, operation_spec
|
||
)
|
||
|
||
# 验证
|
||
report.validation_passed = self._validate_mongo_execution(
|
||
operation_spec, report
|
||
)
|
||
|
||
if report.validation_passed:
|
||
report.status = "success"
|
||
logger.info("工单 %d MongoDB %s 执行成功", ticket.id, operation)
|
||
else:
|
||
report.status = "partial"
|
||
logger.warning("工单 %d MongoDB %s 验证未通过", ticket.id, operation)
|
||
|
||
except asyncio.TimeoutError:
|
||
report.status = "failed"
|
||
report.errors.append(f"执行超时({timeout}秒)")
|
||
report.validation_passed = False
|
||
logger.warning("工单 %d MongoDB 执行超时", ticket.id)
|
||
except SyntaxValidationError as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"验证失败: {e}")
|
||
report.validation_passed = False
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"执行异常: {e}")
|
||
report.validation_passed = False
|
||
logger.exception("工单 %d MongoDB 执行异常", ticket.id)
|
||
finally:
|
||
client.close()
|
||
|
||
except SyntaxValidationError as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"语法校验失败: {e}")
|
||
report.validation_passed = False
|
||
except Exception as e:
|
||
report.status = "failed"
|
||
report.errors.append(f"连接异常: {e}")
|
||
report.validation_passed = False
|
||
|
||
report.duration_ms = int((time.time() - start_time) * 1000)
|
||
return report
|
||
|
||
# ===== 数据备份 =====
|
||
|
||
async def _backup_mysql_data(
|
||
self, conn: Any, sql_content: str, db_type: str
|
||
) -> str:
|
||
"""执行前备份将被修改的数据。
|
||
|
||
对 UPDATE/DELETE 操作,将被影响的数据行导出为 JSON 格式备份。
|
||
|
||
Args:
|
||
conn: 数据库连接
|
||
sql_content: SQL 内容
|
||
db_type: 数据库类型(mysql/postgres)
|
||
|
||
Returns:
|
||
备份信息的字符串描述
|
||
"""
|
||
backup_summaries: List[str] = []
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||
table_name = self._extract_table_name(stmt, sql_type)
|
||
|
||
if sql_type == SQLType.UPDATE and table_name:
|
||
where_clause = self._extract_where_clause_text(stmt)
|
||
if where_clause:
|
||
try:
|
||
row_count = await self._get_affected_count(
|
||
conn, table_name, where_clause, db_type
|
||
)
|
||
if row_count > 0:
|
||
backup_summaries.append(
|
||
f"UPDATE `{table_name}` 将影响 {row_count} 行数据"
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
elif sql_type == SQLType.DELETE and table_name:
|
||
where_clause = self._extract_where_clause_text(stmt)
|
||
if where_clause:
|
||
try:
|
||
row_count = await self._get_affected_count(
|
||
conn, table_name, where_clause, db_type
|
||
)
|
||
if row_count > 0:
|
||
backup_summaries.append(
|
||
f"DELETE `{table_name}` 将删除 {row_count} 行数据"
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
return "; ".join(backup_summaries) if backup_summaries else "无需备份"
|
||
|
||
# ===== 执行前快照 =====
|
||
|
||
async def _capture_sql_before_snapshot(
|
||
self, conn: Any, sql_content: str, db_type: str
|
||
) -> str:
|
||
"""执行前捕获 SQL 数据库的数据快照。
|
||
|
||
对 UPDATE/DELETE 操作,查询将被影响的数据行。
|
||
对 INSERT 操作,记录表当前行数。
|
||
|
||
Args:
|
||
conn: 数据库连接
|
||
sql_content: SQL 内容
|
||
db_type: 数据库类型(mysql/postgres)
|
||
|
||
Returns:
|
||
快照信息的字符串表示
|
||
"""
|
||
snapshots: List[str] = []
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||
table_name = self._extract_table_name(stmt, sql_type)
|
||
|
||
if sql_type == SQLType.UPDATE:
|
||
where_clause = self._extract_where_clause_text(stmt)
|
||
if table_name:
|
||
try:
|
||
count = await self._get_affected_count(
|
||
conn, table_name, where_clause, db_type
|
||
)
|
||
snapshots.append(
|
||
f"UPDATE 表 `{table_name}`: 将有 {count} 行被更新"
|
||
f"{' (WHERE: ' + where_clause + ')' if where_clause else ' (全表更新)' }"
|
||
)
|
||
except Exception as e:
|
||
snapshots.append(f"UPDATE 表 `{table_name}`: 快照获取失败 - {e}")
|
||
|
||
elif sql_type == SQLType.DELETE:
|
||
where_clause = self._extract_where_clause_text(stmt)
|
||
if table_name:
|
||
try:
|
||
count = await self._get_affected_count(
|
||
conn, table_name, where_clause, db_type
|
||
)
|
||
snapshots.append(
|
||
f"DELETE 表 `{table_name}`: 将有 {count} 行被删除"
|
||
f"{' (WHERE: ' + where_clause + ')' if where_clause else ' (全表删除)' }"
|
||
)
|
||
except Exception as e:
|
||
snapshots.append(f"DELETE 表 `{table_name}`: 快照获取失败 - {e}")
|
||
|
||
elif sql_type == SQLType.INSERT:
|
||
if table_name:
|
||
try:
|
||
current_count = await self._get_table_row_count(
|
||
conn, table_name, db_type
|
||
)
|
||
snapshots.append(
|
||
f"INSERT 表 `{table_name}`: 当前 {current_count} 行"
|
||
)
|
||
except Exception as e:
|
||
snapshots.append(f"INSERT 表 `{table_name}`: 行数获取失败 - {e}")
|
||
|
||
return "\n".join(snapshots) if snapshots else "无快照信息"
|
||
|
||
async def _capture_sql_after_snapshot(
|
||
self, conn: Any, sql_content: str, db_type: str
|
||
) -> str:
|
||
"""执行后捕获 SQL 数据库的数据快照。
|
||
|
||
Args:
|
||
conn: 数据库连接
|
||
sql_content: SQL 内容
|
||
db_type: 数据库类型
|
||
|
||
Returns:
|
||
执行后的快照信息
|
||
"""
|
||
snapshots: List[str] = []
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||
table_name = self._extract_table_name(stmt, sql_type)
|
||
|
||
if table_name and sql_type in (SQLType.UPDATE, SQLType.DELETE, SQLType.INSERT):
|
||
try:
|
||
current_count = await self._get_table_row_count(
|
||
conn, table_name, db_type
|
||
)
|
||
snapshots.append(f"表 `{table_name}`: 当前 {current_count} 行")
|
||
except Exception as e:
|
||
snapshots.append(f"表 `{table_name}`: 行数获取失败 - {e}")
|
||
|
||
return "\n".join(snapshots) if snapshots else "无快照信息"
|
||
|
||
async def _capture_redis_before_snapshot(
|
||
self, conn: Any, commands: List[Dict[str, Any]]
|
||
) -> str:
|
||
"""执行前捕获 Redis 键的快照。
|
||
|
||
记录将被修改的键的当前值和 TTL。
|
||
|
||
Args:
|
||
conn: Redis 连接
|
||
commands: 命令列表
|
||
|
||
Returns:
|
||
键快照信息的字符串表示
|
||
"""
|
||
snapshots: List[str] = []
|
||
keys_to_capture: set = set()
|
||
|
||
# 提取可能被修改的键
|
||
for cmd_info in commands:
|
||
command = cmd_info["command"]
|
||
db = cmd_info.get("db", 0)
|
||
parts = command.strip().split()
|
||
if len(parts) >= 2:
|
||
cmd_name = parts[0].upper()
|
||
if cmd_name in REDIS_WRITE_COMMANDS:
|
||
key = parts[1]
|
||
keys_to_capture.add((key, db))
|
||
|
||
# 查询每个键的当前值
|
||
for key, db in keys_to_capture:
|
||
try:
|
||
if db != 0:
|
||
await conn.select(db)
|
||
key_type = await conn.type(key)
|
||
ttl = await conn.ttl(key)
|
||
|
||
if key_type == "none":
|
||
snapshots.append(f"[DB{db}] 键 '{key}': 不存在")
|
||
else:
|
||
# 获取值(限制长度)
|
||
value_preview = await self._get_redis_value_preview(conn, key, key_type)
|
||
ttl_str = f"TTL={ttl}s" if ttl > 0 else "无过期时间"
|
||
snapshots.append(
|
||
f"[DB{db}] 键 '{key}' ({key_type}, {ttl_str}): {value_preview}"
|
||
)
|
||
except Exception as e:
|
||
snapshots.append(f"[DB{db}] 键 '{key}': 快照获取失败 - {e}")
|
||
|
||
return "\n".join(snapshots) if snapshots else "无键被修改"
|
||
|
||
async def _capture_redis_after_snapshot(
|
||
self, conn: Any, commands: List[Dict[str, Any]]
|
||
) -> str:
|
||
"""执行后捕获 Redis 键的快照。"""
|
||
return await self._capture_redis_before_snapshot(conn, commands)
|
||
|
||
async def _capture_mongo_before_snapshot(
|
||
self, collection: Any, operation_spec: Dict[str, Any]
|
||
) -> str:
|
||
"""执行前捕获 MongoDB 文档快照。
|
||
|
||
记录将被修改的文档数量和示例。
|
||
|
||
Args:
|
||
collection: MongoDB 集合对象
|
||
operation_spec: 操作规格
|
||
|
||
Returns:
|
||
文档快照信息的字符串表示
|
||
"""
|
||
operation = operation_spec.get("operation", "")
|
||
filter_doc = operation_spec.get("filter", {})
|
||
|
||
try:
|
||
if operation in ("updateOne", "updateMany", "deleteOne", "deleteMany", "replaceOne"):
|
||
count = await collection.count_documents(filter_doc)
|
||
return f"操作 '{operation}': 匹配 {count} 条文档\n筛选条件: {json.dumps(filter_doc, ensure_ascii=False)}"
|
||
elif operation in ("insertOne", "insertMany"):
|
||
return f"操作 '{operation}': 将插入新文档"
|
||
return f"操作 '{operation}': 无前置快照"
|
||
except Exception as e:
|
||
return f"快照获取失败: {e}"
|
||
|
||
async def _capture_mongo_after_snapshot(
|
||
self, collection: Any, operation_spec: Dict[str, Any]
|
||
) -> str:
|
||
"""执行后捕获 MongoDB 文档快照。"""
|
||
operation = operation_spec.get("operation", "")
|
||
filter_doc = operation_spec.get("filter", {})
|
||
|
||
try:
|
||
if operation in ("updateOne", "updateMany", "replaceOne"):
|
||
count = await collection.count_documents(filter_doc)
|
||
return f"操作 '{operation}' 完成后: 匹配 {count} 条文档"
|
||
elif operation in ("deleteOne", "deleteMany"):
|
||
return f"操作 '{operation}' 已完成"
|
||
elif operation in ("insertOne", "insertMany"):
|
||
total = await collection.estimated_document_count()
|
||
return f"操作 '{operation}' 完成后: 集合约 {total} 条文档"
|
||
return f"操作 '{operation}': 无后置快照"
|
||
except Exception as e:
|
||
return f"快照获取失败: {e}"
|
||
|
||
# ===== 验证方法 =====
|
||
|
||
def _validate_sql_execution(self, sql_content: str, report: ExecutionReport) -> bool:
|
||
"""验证 SQL 执行结果。
|
||
|
||
验证项:
|
||
1. 影响行数 >= 0
|
||
2. UPDATE/DELETE 有 WHERE 条件时,影响行数应合理
|
||
3. 无错误信息
|
||
|
||
Args:
|
||
sql_content: SQL 内容
|
||
report: 执行报告
|
||
|
||
Returns:
|
||
验证是否通过
|
||
"""
|
||
passed = True
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||
|
||
# 检查 UPDATE/DELETE 是否缺少 WHERE
|
||
if sql_type in (SQLType.UPDATE, SQLType.DELETE):
|
||
has_where = self._extract_where_clause_text(stmt) is not None
|
||
if not has_where:
|
||
report.warnings.append(
|
||
f"{sql_type.value} 操作缺少 WHERE 条件,影响全表"
|
||
)
|
||
|
||
# 检查是否有执行错误
|
||
if report.errors:
|
||
passed = False
|
||
|
||
return passed
|
||
|
||
def _validate_redis_execution(
|
||
self, commands: List[Dict[str, Any]], report: ExecutionReport
|
||
) -> bool:
|
||
"""验证 Redis 执行结果。
|
||
|
||
验证项:
|
||
1. 所有命令都已执行
|
||
2. 写命令返回预期结果
|
||
|
||
Args:
|
||
commands: 命令列表
|
||
report: 执行报告
|
||
|
||
Returns:
|
||
验证是否通过
|
||
"""
|
||
passed = True
|
||
|
||
# 检查是否有错误
|
||
if report.errors:
|
||
passed = False
|
||
|
||
# 检查是否所有命令都执行成功
|
||
if report.affected_rows != len(commands):
|
||
report.warnings.append(
|
||
f"仅 {report.affected_rows}/{len(commands)} 条命令成功执行"
|
||
)
|
||
|
||
return passed
|
||
|
||
def _validate_mongo_execution(
|
||
self, operation_spec: Dict[str, Any], report: ExecutionReport
|
||
) -> bool:
|
||
"""验证 MongoDB 执行结果。
|
||
|
||
验证项:
|
||
1. 修改文档数符合预期
|
||
2. 无错误信息
|
||
|
||
Args:
|
||
operation_spec: 操作规格
|
||
report: 执行报告
|
||
|
||
Returns:
|
||
验证是否通过
|
||
"""
|
||
passed = True
|
||
operation = operation_spec.get("operation", "")
|
||
|
||
# 检查写操作是否有影响
|
||
if operation in ("updateOne", "updateMany", "replaceOne"):
|
||
if report.affected_rows == 0:
|
||
report.warnings.append(f"{operation} 未修改任何文档")
|
||
|
||
elif operation in ("deleteOne", "deleteMany"):
|
||
if report.affected_rows == 0:
|
||
report.warnings.append(f"{operation} 未删除任何文档")
|
||
|
||
elif operation in ("insertOne",):
|
||
if report.affected_rows != 1:
|
||
report.errors.append(f"insertOne 应插入 1 条文档,实际 {report.affected_rows}")
|
||
passed = False
|
||
|
||
# 检查错误
|
||
if report.errors:
|
||
passed = False
|
||
|
||
return passed
|
||
|
||
# ===== 语法预校验 =====
|
||
|
||
def _validate_mysql_syntax(self, sql_content: str) -> None:
|
||
"""预校验 MySQL 语法。
|
||
|
||
使用 sqlparse 解析验证 SQL 语句的基本合法性。
|
||
|
||
Args:
|
||
sql_content: SQL 内容
|
||
|
||
Raises:
|
||
SyntaxValidationError: 语法校验失败
|
||
"""
|
||
if not sql_content or not sql_content.strip():
|
||
raise SyntaxValidationError("SQL 内容不能为空")
|
||
|
||
statements = self._split_statements(sql_content)
|
||
for stmt in statements:
|
||
stmt = stmt.strip()
|
||
if not stmt:
|
||
continue
|
||
|
||
# 使用 sqlparse 解析
|
||
parsed = sqlparse.parse(stmt)
|
||
if not parsed or not parsed[0].tokens:
|
||
raise SyntaxValidationError(f"无法解析 SQL: {stmt[:100]}")
|
||
|
||
# 检查是否为合法的 SQL 语句(DML + DDL)
|
||
sql_type = self._analyzer._detect_sql_type(stmt)
|
||
allowed_types = (
|
||
SQLType.INSERT, SQLType.UPDATE, SQLType.DELETE, SQLType.SELECT,
|
||
SQLType.CREATE, SQLType.ALTER, SQLType.DROP, SQLType.TRUNCATE,
|
||
)
|
||
if sql_type not in allowed_types:
|
||
raise SyntaxValidationError(
|
||
f"不支持的 SQL 类型: {sql_type.value}, 仅支持 INSERT/UPDATE/DELETE/CREATE/ALTER/DROP/TRUNCATE"
|
||
)
|
||
|
||
def _validate_postgres_syntax(self, sql_content: str) -> None:
|
||
"""预校验 PostgreSQL 语法。
|
||
|
||
PostgreSQL 语法校验与 MySQL 类似,使用 sqlparse 进行基础解析。
|
||
|
||
Args:
|
||
sql_content: SQL 内容
|
||
|
||
Raises:
|
||
SyntaxValidationError: 语法校验失败
|
||
"""
|
||
# PostgreSQL 基础语法校验与 MySQL 相同
|
||
self._validate_mysql_syntax(sql_content)
|
||
|
||
def _validate_redis_commands(self, commands: List[Dict[str, Any]]) -> None:
|
||
"""预校验 Redis 命令格式。
|
||
|
||
验证命令名和参数数量是否合法。
|
||
|
||
Args:
|
||
commands: 命令列表
|
||
|
||
Raises:
|
||
SyntaxValidationError: 语法校验失败
|
||
"""
|
||
for cmd_info in commands:
|
||
command = cmd_info.get("command", "").strip()
|
||
if not command:
|
||
raise SyntaxValidationError("命令不能为空")
|
||
|
||
parts = command.split()
|
||
cmd_name = parts[0].upper()
|
||
|
||
# 检查命令名是否合法(基本检查)
|
||
if not re.match(r"^[A-Z]+$", cmd_name):
|
||
raise SyntaxValidationError(f"非法的命令名: {cmd_name}")
|
||
|
||
# 检查参数数量(基本检查)
|
||
if len(parts) < 2 and cmd_name not in (
|
||
"PING", "INFO", "DBSIZE", "FLUSHDB", "FLUSHALL", "SAVE",
|
||
"BGSAVE", "LASTSAVE", "MONITOR", "SYNC", "SHUTDOWN",
|
||
):
|
||
raise SyntaxValidationError(f"命令 {cmd_name} 缺少参数")
|
||
|
||
def _validate_mongo_operation(self, operation_spec: Dict[str, Any]) -> None:
|
||
"""预校验 MongoDB 操作。
|
||
|
||
验证 JSON 格式和操作类型是否合法。
|
||
|
||
Args:
|
||
operation_spec: 操作规格字典
|
||
|
||
Raises:
|
||
SyntaxValidationError: 语法校验失败
|
||
"""
|
||
if not isinstance(operation_spec, dict):
|
||
raise SyntaxValidationError("MongoDB 工单必须是 JSON 对象")
|
||
|
||
collection = operation_spec.get("collection")
|
||
if not collection or not isinstance(collection, str):
|
||
raise SyntaxValidationError("缺少或非法的 'collection' 字段")
|
||
|
||
operation = operation_spec.get("operation")
|
||
if operation not in MONGO_SUPPORTED_OPERATIONS:
|
||
raise SyntaxValidationError(
|
||
f"不支持的操作类型: {operation}, "
|
||
f"支持: {', '.join(MONGO_SUPPORTED_OPERATIONS)}"
|
||
)
|
||
|
||
# 检查各操作必需参数
|
||
if operation in ("insertOne",) and "document" not in operation_spec:
|
||
raise SyntaxValidationError("insertOne 操作需要 'document' 字段")
|
||
|
||
if operation in ("insertMany",) and "documents" not in operation_spec:
|
||
raise SyntaxValidationError("insertMany 操作需要 'documents' 字段")
|
||
|
||
if operation in ("updateOne", "updateMany", "replaceOne"):
|
||
if "filter" not in operation_spec:
|
||
raise SyntaxValidationError(f"{operation} 操作需要 'filter' 字段")
|
||
if "update" not in operation_spec and operation != "replaceOne":
|
||
raise SyntaxValidationError(f"{operation} 操作需要 'update' 字段")
|
||
if operation == "replaceOne" and "document" not in operation_spec:
|
||
raise SyntaxValidationError("replaceOne 操作需要 'document' 字段")
|
||
|
||
if operation in ("deleteOne", "deleteMany"):
|
||
if "filter" not in operation_spec:
|
||
raise SyntaxValidationError(f"{operation} 操作需要 'filter' 字段")
|
||
|
||
# ===== 工具方法 =====
|
||
|
||
@staticmethod
|
||
def _split_statements(sql_content: str) -> List[str]:
|
||
"""将 SQL 内容按分号分割为多条语句。
|
||
|
||
智能处理字符串内的分号。
|
||
|
||
Args:
|
||
sql_content: SQL 内容
|
||
|
||
Returns:
|
||
语句列表
|
||
"""
|
||
if not sql_content:
|
||
return []
|
||
|
||
# 使用 sqlparse 进行更智能的分割
|
||
parsed = sqlparse.split(sql_content)
|
||
return [p.strip() for p in parsed if p.strip()]
|
||
|
||
def _extract_table_name(self, sql: str, sql_type: SQLType) -> Optional[str]:
|
||
"""从 SQL 语句中提取表名。
|
||
|
||
Args:
|
||
sql: SQL 语句
|
||
sql_type: SQL 类型
|
||
|
||
Returns:
|
||
表名,解析失败返回 None
|
||
"""
|
||
sql = sql.strip()
|
||
upper = sql.upper()
|
||
|
||
if sql_type == SQLType.INSERT:
|
||
match = re.search(r"INSERT\s+(?:IGNORE\s+)?INTO\s+`?([^`\s(]+)`?", upper)
|
||
if match:
|
||
return match.group(1).strip()
|
||
|
||
elif sql_type == SQLType.UPDATE:
|
||
match = re.search(r"UPDATE\s+`?([^`\s,]+)`?", upper)
|
||
if match:
|
||
return match.group(1).strip()
|
||
|
||
elif sql_type == SQLType.DELETE:
|
||
match = re.search(r"DELETE\s+FROM\s+`?([^`\s(]+)`?", upper)
|
||
if not match:
|
||
match = re.search(r"DELETE\s+`?([^`\s(]+)`?\s+FROM", upper)
|
||
if match:
|
||
return match.group(1).strip()
|
||
|
||
return None
|
||
|
||
@staticmethod
|
||
def _extract_where_clause_text(sql: str) -> Optional[str]:
|
||
"""从 SQL 语句中提取 WHERE 子句文本。
|
||
|
||
Args:
|
||
sql: SQL 语句
|
||
|
||
Returns:
|
||
WHERE 条件文本,没有则返回 None
|
||
"""
|
||
parsed = sqlparse.parse(sql)
|
||
if not parsed:
|
||
return None
|
||
|
||
statement = parsed[0]
|
||
|
||
# 使用 sqlparse 的 Where 对象
|
||
from sqlparse.sql import Where
|
||
for token in statement.tokens:
|
||
if isinstance(token, Where):
|
||
where_text = str(token).replace("WHERE", "", 1).strip()
|
||
return where_text
|
||
|
||
# 回退:检查关键字
|
||
from sqlparse.tokens import Keyword
|
||
where_tokens = []
|
||
in_where = False
|
||
for token in statement.tokens:
|
||
if token.ttype is Keyword and token.normalized == "WHERE":
|
||
in_where = True
|
||
continue
|
||
if in_where:
|
||
if token.ttype is Keyword and token.normalized in (
|
||
"ORDER", "GROUP", "HAVING", "LIMIT",
|
||
):
|
||
break
|
||
where_tokens.append(str(token))
|
||
|
||
if where_tokens:
|
||
return "".join(where_tokens).strip()
|
||
return None
|
||
|
||
async def _get_affected_count(
|
||
self, conn: Any, table_name: str, where_clause: Optional[str], db_type: str
|
||
) -> int:
|
||
"""获取将被影响的数据行数。
|
||
|
||
Args:
|
||
conn: 数据库连接
|
||
table_name: 表名
|
||
where_clause: WHERE 条件
|
||
db_type: 数据库类型
|
||
|
||
Returns:
|
||
行数
|
||
"""
|
||
# 安全过滤表名(只允许字母数字下划线)
|
||
safe_table = re.sub(r"[^\w]", "", table_name)
|
||
if not safe_table:
|
||
raise ValueError("非法的表名")
|
||
|
||
if db_type == "postgres":
|
||
sql = f'SELECT COUNT(*) FROM "{safe_table}"'
|
||
else:
|
||
sql = f"SELECT COUNT(*) FROM `{safe_table}`"
|
||
|
||
if where_clause:
|
||
sql += f" WHERE {where_clause}"
|
||
sql += " LIMIT 10001"
|
||
|
||
# 根据连接类型执行查询
|
||
if db_type == "mysql":
|
||
async with conn.cursor() as cur:
|
||
await cur.execute(sql)
|
||
row = await cur.fetchone()
|
||
return row[0] if row else 0
|
||
else:
|
||
# PostgreSQL (asyncpg)
|
||
row = await conn.fetchval(sql)
|
||
return row or 0
|
||
|
||
async def _get_table_row_count(
|
||
self, conn: Any, table_name: str, db_type: str
|
||
) -> int:
|
||
"""获取表的当前行数。
|
||
|
||
Args:
|
||
conn: 数据库连接
|
||
table_name: 表名
|
||
db_type: 数据库类型
|
||
|
||
Returns:
|
||
行数
|
||
"""
|
||
safe_table = re.sub(r"[^\w]", "", table_name)
|
||
if not safe_table:
|
||
raise ValueError("非法的表名")
|
||
|
||
if db_type == "postgres":
|
||
sql = f'SELECT COUNT(*) FROM "{safe_table}"'
|
||
else:
|
||
sql = f"SELECT COUNT(*) FROM `{safe_table}`"
|
||
|
||
sql += " LIMIT 10001"
|
||
|
||
if db_type == "mysql":
|
||
async with conn.cursor() as cur:
|
||
await cur.execute(sql)
|
||
row = await cur.fetchone()
|
||
return row[0] if row else 0
|
||
else:
|
||
row = await conn.fetchval(sql)
|
||
return row or 0
|
||
|
||
@staticmethod
|
||
def _parse_redis_commands(content: str, target_database: str = "0") -> List[Dict[str, Any]]:
|
||
"""解析 Redis 工单内容为命令列表。
|
||
|
||
支持 JSON 格式和简单文本格式。
|
||
|
||
Args:
|
||
content: 工单内容
|
||
target_database: 目标 DB 编号(来自 ticket.target_database)
|
||
|
||
Returns:
|
||
命令列表,每项为 {"command": str, "db": int}
|
||
"""
|
||
content = content.strip()
|
||
if not content:
|
||
return []
|
||
|
||
# 解析默认 db 编号
|
||
try:
|
||
default_db = int(target_database)
|
||
except (ValueError, TypeError):
|
||
default_db = 0
|
||
|
||
# 尝试 JSON 解析
|
||
if content.startswith("{"):
|
||
try:
|
||
data = json.loads(content)
|
||
if isinstance(data, dict) and "commands" in data:
|
||
commands = []
|
||
for cmd in data["commands"]:
|
||
if isinstance(cmd, dict) and "command" in cmd:
|
||
commands.append({
|
||
"command": cmd["command"],
|
||
"db": cmd.get("db", default_db),
|
||
})
|
||
elif isinstance(cmd, str):
|
||
commands.append({"command": cmd, "db": default_db})
|
||
return commands
|
||
# 可能是单条命令的 JSON
|
||
if "command" in data:
|
||
return [{"command": data["command"], "db": data.get("db", default_db)}]
|
||
except json.JSONDecodeError:
|
||
pass # 不是 JSON,尝试文本格式
|
||
|
||
# 简单文本格式:每行一个命令
|
||
commands = []
|
||
for line in content.split("\n"):
|
||
line = line.strip()
|
||
if line and not line.startswith("#"):
|
||
commands.append({"command": line, "db": default_db})
|
||
|
||
return commands
|
||
|
||
@staticmethod
|
||
def _parse_mongo_operation(content: str) -> Dict[str, Any]:
|
||
"""解析 MongoDB 工单内容为操作规格。
|
||
|
||
Args:
|
||
content: 工单内容(JSON 字符串)
|
||
|
||
Returns:
|
||
操作规格字典
|
||
|
||
Raises:
|
||
SyntaxValidationError: JSON 解析失败
|
||
"""
|
||
content = content.strip()
|
||
if not content:
|
||
raise SyntaxValidationError("MongoDB 工单内容不能为空")
|
||
|
||
try:
|
||
data = json.loads(content)
|
||
if not isinstance(data, dict):
|
||
raise SyntaxValidationError("MongoDB 工单必须是 JSON 对象")
|
||
return data
|
||
except json.JSONDecodeError as e:
|
||
raise SyntaxValidationError(f"JSON 解析失败: {e}")
|
||
|
||
@staticmethod
|
||
async def _get_redis_value_preview(
|
||
conn: Any, key: str, key_type: str, max_len: int = 200
|
||
) -> str:
|
||
"""获取 Redis 键值的预览。
|
||
|
||
Args:
|
||
conn: Redis 连接
|
||
key: 键名
|
||
key_type: 键类型
|
||
max_len: 最大预览长度
|
||
|
||
Returns:
|
||
值预览字符串
|
||
"""
|
||
try:
|
||
if key_type == "string":
|
||
value = await conn.get(key)
|
||
preview = str(value)[:max_len]
|
||
return f"'{preview}'"
|
||
elif key_type == "hash":
|
||
value = await conn.hgetall(key)
|
||
preview = str(value)[:max_len]
|
||
return f"{preview}"
|
||
elif key_type == "list":
|
||
length = await conn.llen(key)
|
||
return f"<list, {length} items>"
|
||
elif key_type == "set":
|
||
length = await conn.scard(key)
|
||
return f"<set, {length} members>"
|
||
elif key_type == "zset":
|
||
length = await conn.zcard(key)
|
||
return f"<zset, {length} members>"
|
||
else:
|
||
return f"<{key_type}>"
|
||
except Exception:
|
||
return "<无法读取>"
|
||
|
||
@staticmethod
|
||
def _format_redis_result(result: Any) -> str:
|
||
"""格式化 Redis 命令执行结果。
|
||
|
||
Args:
|
||
result: 原始结果
|
||
|
||
Returns:
|
||
格式化后的字符串
|
||
"""
|
||
if result is None:
|
||
return "(nil)"
|
||
if isinstance(result, bytes):
|
||
return result.decode("utf-8", errors="replace")
|
||
if isinstance(result, bool):
|
||
return "1" if result else "0"
|
||
if isinstance(result, (list, set, tuple)):
|
||
items = []
|
||
for item in result:
|
||
if isinstance(item, bytes):
|
||
items.append(item.decode("utf-8", errors="replace"))
|
||
else:
|
||
items.append(str(item))
|
||
return "[" + ", ".join(items[:10]) + ("..." if len(items) > 10 else "") + "]"
|
||
if isinstance(result, dict):
|
||
preview = str(result)[:200]
|
||
return preview
|
||
return str(result)
|