"""多数据库工单执行引擎 — 支持 MySQL/PostgreSQL/Redis/MongoDB 的变更执行。 执行流程: 1. 接收审批通过的工单 2. 按 db_type 分发到对应执行器 3. 并发锁检查（防止同一工单并发执行） 4. 执行前数据备份/快照（用于回滚和验证） 5. 显式事务中执行变更 6. 执行后验证 7. 成功则提交，失败则自动回滚 8. 生成执行报告 验证机制: - SQL数据库: 检查影响行数是否符合预期、语法合法性、事务回滚 - Redis: 检查命令结果、键是否存在/已被修改 - MongoDB: 检查修改文档数、文档内容验证 - 通用: 执行超时控制、连接健康检查、并发执行锁 安全增强: - 并发执行锁: 防止同一工单被并发执行 - 事务管理: SQL数据库使用显式事务，失败自动回滚 - 数据备份: 执行前自动备份受影响数据 - 超时控制: 可配置的超时，防止长时间阻塞 """ import asyncio import hashlib import json import logging import re import time from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set import sqlparse from app.models.instance import Instance from app.models.ticket import Ticket from app.services.sql_analyzer import SQLAnalyzer, SQLType logger = logging.getLogger(__name__) # 超时配置（秒）—— 默认值，优先从 system_settings 读取 SQL_TIMEOUT = 60 REDIS_TIMEOUT = 30 MONGO_TIMEOUT = 60 def _get_executor_timeout(db_type: str) -> int: """从 query_service 配置缓存获取执行超时。""" from app.services.query_service import current_config cfg = current_config() key = {"mysql": "sql_timeout", "postgres": "sql_timeout", "redis": "redis_timeout", "mongodb": "mongo_timeout"}.get(db_type, "sql_timeout") return cfg.get(key, SQL_TIMEOUT) # 并发执行锁: 防止同一工单被并发执行 _ticket_execution_locks: Set[int] = set() _lock_lock = asyncio.Lock() # Redis 写命令集合（用于验证） REDIS_WRITE_COMMANDS = { "SET", "SETEX", "SETNX", "MSET", "HSET", "HMSET", "HINCRBY", "LPUSH", "RPUSH", "LPOP", "RPOP", "LREM", "LSET", "LTRIM", "SADD", "SREM", "SPOP", "SMOVE", "ZADD", "ZREM", "ZINCRBY", "ZREMRANGEBYSCORE", "ZREMRANGEBYRANK", "DEL", "HDEL", "EXPIRE", "EXPIREAT", "PERSIST", "RENAME", "RENAMENX", "INCR", "INCRBY", "DECR", "DECRBY", "APPEND", "SETBIT", "FLUSHDB", "FLUSHALL", } # MongoDB 支持的操作类型 MONGO_SUPPORTED_OPERATIONS = { "insertOne", "insertMany", "updateOne", "updateMany", "deleteOne", "deleteMany", "replaceOne", } @dataclass class ExecutionReport: """执行报告，记录工单执行的完整结果。""" status: str = "success" # success / failed / partial affected_rows: int = 0 # 影响行数 duration_ms: int = 0 # 执行耗时 before_snapshot: str = "" # 执行前快照 after_snapshot: str = "" # 执行后快照 validation_passed: bool = True # 验证是否通过 warnings: List[str] = field(default_factory=list) # 警告信息 errors: List[str] = field(default_factory=list) # 错误信息 raw_output: str = "" # 原始输出 def to_json(self) -> str: """将执行报告序列化为 JSON 字符串。""" return json.dumps( { "status": self.status, "affected_rows": self.affected_rows, "duration_ms": self.duration_ms, "before_snapshot": self.before_snapshot, "after_snapshot": self.after_snapshot, "validation_passed": self.validation_passed, "warnings": self.warnings, "errors": self.errors, "raw_output": self.raw_output, }, ensure_ascii=False, default=str, ) @classmethod def from_exception(cls, exc: Exception, stage: str = "执行") -> "ExecutionReport": """从异常创建失败的执行报告。""" return cls( status="failed", errors=[f"{stage}失败: {type(exc).__name__}: {exc}"], duration_ms=0, validation_passed=False, ) class TicketExecutorError(Exception): """工单执行引擎异常。""" pass class SyntaxValidationError(Exception): """语法预校验异常。""" pass class ExecutionLockError(Exception): """工单并发执行锁冲突异常。""" pass class TicketExecutor: """多数据库工单执行引擎。 支持 MySQL、PostgreSQL、Redis 和 MongoDB 四种类型的工单执行。 提供执行前快照、执行后验证、超时控制、并发锁、事务管理等安全机制。 Attributes: _analyzer: SQL 分析器，用于解析 SQL 类型和语法验证 """ def __init__(self) -> None: """初始化工单执行引擎。""" self._analyzer = SQLAnalyzer() # ===== 并发执行锁 ===== @staticmethod async def _acquire_execution_lock(ticket_id: int) -> None: """获取工单执行锁。 防止同一工单被并发执行多次。 Args: ticket_id: 工单ID Raises: ExecutionLockError: 工单正在执行中 """ async with _lock_lock: if ticket_id in _ticket_execution_locks: raise ExecutionLockError(f"工单 {ticket_id} 正在执行中，请勿重复提交") _ticket_execution_locks.add(ticket_id) logger.info("工单 %d 获取执行锁", ticket_id) @staticmethod async def _release_execution_lock(ticket_id: int) -> None: """释放工单执行锁。 Args: ticket_id: 工单ID """ async with _lock_lock: _ticket_execution_locks.discard(ticket_id) logger.info("工单 %d 释放执行锁", ticket_id) # ===== 公共接口 ===== async def execute(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行工单。 根据实例的数据库类型分发到对应的执行器。 使用并发锁防止同一工单被并发执行。 Args: ticket: 工单对象，包含要执行的 SQL/命令内容 instance: 数据源实例，包含连接信息 Returns: ExecutionReport: 执行报告 Raises: ExecutionLockError: 工单正在执行中 ValueError: 不支持的数据库类型 """ # 1. 获取并发执行锁 await self._acquire_execution_lock(ticket.id) try: if instance.db_type == "mysql": return await self._execute_mysql(ticket, instance) elif instance.db_type in ("postgres", "postgresql"): return await self._execute_postgres(ticket, instance) elif instance.db_type == "redis": return await self._execute_redis(ticket, instance) elif instance.db_type == "mongo": return await self._execute_mongo(ticket, instance) else: raise ValueError(f"不支持的执行类型: {instance.db_type}") finally: # 确保锁一定会被释放 await self._release_execution_lock(ticket.id) # ===== MySQL 执行 ===== async def _execute_mysql(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行 MySQL 工单。""" import aiomysql report = ExecutionReport() start_time = time.time() sql_content = ticket.sql_content.strip() exec_strategy = getattr(ticket, "exec_strategy", "stop_on_error") or "stop_on_error" # 获取超时配置（从 ticket 或选项中） custom_timeout = getattr(ticket, 'options', None) or {} _default_timeout = _get_executor_timeout('mysql') timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout try: # 1. 语法预校验 self._validate_mysql_syntax(sql_content) # 2. 建立连接（显式事务，autocommit=False） conn = await aiomysql.connect( host=instance.host, port=instance.port, user=instance.username, password=instance.password, db=ticket.target_database, autocommit=False, connect_timeout=10, ) try: # 3. 执行前快照和数据备份 report.before_snapshot = await self._capture_sql_before_snapshot( conn, sql_content, "mysql" ) # 执行前数据备份（用于回滚） backup_data = await self._backup_mysql_data(conn, sql_content, "mysql") if backup_data: report.before_snapshot += f"\n[数据备份] {backup_data}" # 4. 在事务中执行 SQL total_affected = 0 output_lines: List[str] = [] execution_failed = False failed_count = 0 statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue try: async with conn.cursor() as cur: await asyncio.wait_for(cur.execute(stmt), timeout=timeout) affected = cur.rowcount total_affected += max(0, affected) if affected is not None else 0 output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行") except Exception as stmt_error: failed_count += 1 execution_failed = True output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}") report.errors.append(f"语句执行失败: {stmt_error}") if exec_strategy == "stop_on_error": break # continue: 跳过失败继续执行 # transaction: 任何失败都回滚（默认行为，break 后走 rollback） report.affected_rows = total_affected report.raw_output = "\n".join(output_lines) # 5. 执行后快照 report.after_snapshot = await self._capture_sql_after_snapshot( conn, sql_content, "mysql" ) # 6. 验证执行结果 report.validation_passed = self._validate_sql_execution( sql_content, report ) # 事务决策：根据执行策略处理 if exec_strategy == "continue" and failed_count > 0 and failed_count < len(statements): # continue 模式：部分失败也算成功，提交已执行的语句 await conn.commit() report.status = "partial_success" report.errors.insert(0, f"部分失败: {failed_count}/{len(statements)} 条语句失败") logger.warning("工单 %d MySQL 部分执行成功: %d 条失败", ticket.id, failed_count) elif not execution_failed and report.validation_passed: await conn.commit() report.status = "success" logger.info("工单 %d MySQL 执行成功，事务已提交", ticket.id) else: await conn.rollback() report.status = "failed" if not report.errors: report.errors.append("执行结果验证未通过，事务已回滚") logger.warning("工单 %d MySQL 执行失败，事务已回滚", ticket.id) except asyncio.TimeoutError: report.status = "failed" report.errors.append(f"执行超时（{timeout}秒）") report.validation_passed = False try: await conn.rollback() logger.info("工单 %d 超时后事务已回滚", ticket.id) except Exception: pass except Exception as e: report.status = "failed" report.errors.append(f"执行异常: {e}") report.validation_passed = False try: await conn.rollback() logger.info("工单 %d 异常后事务已回滚", ticket.id) except Exception: pass finally: conn.close() except SyntaxValidationError as e: report.status = "failed" report.errors.append(f"语法校验失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"连接异常: {e}") report.validation_passed = False report.duration_ms = int((time.time() - start_time) * 1000) return report # ===== PostgreSQL 执行 ===== async def _execute_postgres(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行 PostgreSQL 工单。 执行流程与 MySQL 类似，使用 asyncpg 驱动，支持显式事务管理。 Args: ticket: 工单对象 instance: PostgreSQL 实例连接信息 Returns: ExecutionReport: 执行报告 """ import asyncpg report = ExecutionReport() start_time = time.time() sql_content = ticket.sql_content.strip() database = ticket.target_database or instance.database or "postgres" # 获取超时配置 custom_timeout = getattr(ticket, 'options', None) or {} _default_timeout = _get_executor_timeout('postgres') timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout try: # 1. 语法预校验 self._validate_postgres_syntax(sql_content) # 2. 建立连接（显式事务） dsn = ( f"postgresql://{instance.username}:{instance.password}" f"@{instance.host}:{instance.port}/{database}" ) conn = await asyncpg.connect( dsn, command_timeout=timeout, server_settings={ "application_name": "sql_platform_executor", "statement_timeout": str(timeout * 1000), # PostgreSQL 使用毫秒 }, ) try: # 开始显式事务 transaction = conn.transaction() await transaction.start() try: # 3. 执行前快照和数据备份 report.before_snapshot = await self._capture_sql_before_snapshot( conn, sql_content, "postgres" ) # 执行前数据备份 backup_data = await self._backup_mysql_data(conn, sql_content, "postgres") if backup_data: report.before_snapshot += f"\n[数据备份] {backup_data}" # 4. 在事务中执行 SQL total_affected = 0 output_lines: List[str] = [] execution_failed = False statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue try: status = await asyncio.wait_for( conn.execute(stmt), timeout=timeout ) # status like "UPDATE 3" or "INSERT 0 1" parts = status.split() affected = 0 if len(parts) >= 2 and parts[1].isdigit(): affected = int(parts[1]) total_affected += affected output_lines.append(f"[{stmt[:50]}...] 状态: {status}") except Exception as stmt_error: execution_failed = True output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}") report.errors.append(f"语句执行失败: {stmt_error}") break report.affected_rows = total_affected report.raw_output = "\n".join(output_lines) # 5. 执行后快照 report.after_snapshot = await self._capture_sql_after_snapshot( conn, sql_content, "postgres" ) # 6. 验证执行结果 report.validation_passed = self._validate_sql_execution( sql_content, report ) # 事务决策 if not execution_failed and report.validation_passed: await transaction.commit() report.status = "success" logger.info("工单 %d PostgreSQL 执行成功，事务已提交", ticket.id) else: await transaction.rollback() report.status = "failed" if not report.errors: report.errors.append("执行结果验证未通过，事务已回滚") logger.warning("工单 %d PostgreSQL 执行失败，事务已回滚", ticket.id) except asyncio.TimeoutError: await transaction.rollback() report.status = "failed" report.errors.append(f"执行超时（{timeout}秒），事务已回滚") report.validation_passed = False logger.warning("工单 %d PostgreSQL 超时，事务已回滚", ticket.id) except Exception as e: await transaction.rollback() report.status = "failed" report.errors.append(f"执行异常: {e}，事务已回滚") report.validation_passed = False logger.warning("工单 %d PostgreSQL 异常，事务已回滚: %s", ticket.id, e) finally: await conn.close() except SyntaxValidationError as e: report.status = "failed" report.errors.append(f"语法校验失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"连接异常: {e}") report.validation_passed = False report.duration_ms = int((time.time() - start_time) * 1000) return report # ===== Redis 执行 ===== async def _execute_redis(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行 Redis 工单。 Redis 工单格式支持: - JSON 格式: {"commands": [{"command": "SET key value", "db": 0}]} - 简单文本格式: 每行一个命令 增强功能: - 命令注入防护（shell字符检测） - 连接池管理 - 可配置超时 - 原子性执行（全部成功或记录失败） Args: ticket: 工单对象 instance: Redis 实例连接信息 Returns: ExecutionReport: 执行报告 """ import redis.asyncio as aioredis report = ExecutionReport() start_time = time.time() # 获取超时配置 custom_timeout = getattr(ticket, 'options', None) or {} _default_timeout = _get_executor_timeout('redis') timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout try: # 解析命令 commands = self._parse_redis_commands(ticket.sql_content, ticket.target_database) if not commands: report.status = "failed" report.errors.append("未解析到有效的 Redis 命令") report.validation_passed = False return report # 语法预校验 self._validate_redis_commands(commands) # 命令注入防护检查 for cmd_info in commands: command = cmd_info.get("command", "") shell_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"] for char in shell_chars: if char in command: raise SyntaxValidationError( f"Redis 命令包含非法字符 '{char}': {command}" ) # 建立连接（使用连接池） conn = aioredis.Redis( host=instance.host, port=instance.port, password=instance.password or None, decode_responses=True, socket_timeout=timeout, socket_connect_timeout=10, health_check_interval=30, ) # 连接健康检查 try: await asyncio.wait_for(conn.ping(), timeout=10) except Exception as e: report.status = "failed" report.errors.append(f"Redis 连接健康检查失败: {e}") report.validation_passed = False return report try: # 执行前快照 report.before_snapshot = await self._capture_redis_before_snapshot( conn, commands ) # 执行命令（带重试机制） output_lines: List[str] = [] success_count = 0 max_retries = 1 for cmd_info in commands: command = cmd_info["command"] db = cmd_info.get("db", 0) # 验证 DB 号范围 if not isinstance(db, int) or db < 0 or db > 15: report.errors.append(f"非法的 DB 号: {db}（必须在 0-15 之间）") continue # 选择数据库 if db != 0: await conn.select(db) parts = command.strip().split() if not parts: continue cmd_name = parts[0].upper() args = parts[1:] # 执行命令（带重试） retry_count = 0 executed = False last_error = None while retry_count <= max_retries and not executed: try: result = await asyncio.wait_for( conn.execute_command(cmd_name, *args), timeout=timeout, ) success_count += 1 output_lines.append( f"[DB{db}] {command} => {self._format_redis_result(result)}" ) executed = True except asyncio.TimeoutError: retry_count += 1 last_error = f"超时（{timeout}秒）" if retry_count > max_retries: output_lines.append( f"[DB{db}] {command} => 超时（{timeout}秒，已重试{max_retries}次）" ) report.errors.append(f"命令超时: {command}") except Exception as e: retry_count += 1 last_error = str(e) if retry_count > max_retries: output_lines.append(f"[DB{db}] {command} => 错误: {e}") report.errors.append(f"命令执行失败 '{command}': {e}") report.affected_rows = success_count report.raw_output = "\n".join(output_lines) # 执行后快照 report.after_snapshot = await self._capture_redis_after_snapshot( conn, commands ) # 验证 report.validation_passed = self._validate_redis_execution( commands, report ) if report.errors: report.status = "partial" if success_count > 0 else "failed" else: report.status = "success" finally: await conn.close() except SyntaxValidationError as e: report.status = "failed" report.errors.append(f"语法校验失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"连接异常: {e}") report.validation_passed = False report.duration_ms = int((time.time() - start_time) * 1000) return report # ===== MongoDB 执行 ===== async def _execute_mongo(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行 MongoDB 工单。 MongoDB 工单格式（JSON）: { "collection": "users", "operation": "updateMany", "filter": {"status": "inactive"}, "update": {"$set": {"status": "active"}} } 增强功能: - 可配置超时 - 连接池管理 - 操作验证增强 - writeConcern 配置（数据持久化保证） Args: ticket: 工单对象 instance: MongoDB 实例连接信息 Returns: ExecutionReport: 执行报告 """ report = ExecutionReport() start_time = time.time() # 获取超时配置 custom_timeout = getattr(ticket, 'options', None) or {} _default_timeout = _get_executor_timeout('mongodb') timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout try: # 解析工单内容 operation_spec = self._parse_mongo_operation(ticket.sql_content) collection_name = operation_spec["collection"] operation = operation_spec["operation"] # 语法预校验 self._validate_mongo_operation(operation_spec) # 额外的安全验证：禁止危险操作 forbidden_ops = {"drop", "dropIndex", "renameCollection"} if operation in forbidden_ops: raise SyntaxValidationError(f"MongoDB 操作 '{operation}' 被禁止") # 导入 motor（异步 MongoDB 驱动） try: from motor.motor_asyncio import AsyncIOMotorClient except ImportError: report.status = "failed" report.errors.append("缺少 motor 依赖，请安装: pip install motor") report.validation_passed = False return report # 建立连接（增强配置） database = ticket.target_database or instance.database or "admin" conn_str = ( f"mongodb://{instance.username}:{instance.password}" f"@{instance.host}:{instance.port}/{database}" ) client = AsyncIOMotorClient( conn_str, serverSelectionTimeoutMS=5000, connectTimeoutMS=10000, socketTimeoutMS=timeout * 1000, maxPoolSize=10, minPoolSize=1, maxIdleTimeMS=60000, waitQueueTimeoutMS=5000, ) # 连接健康检查 try: await asyncio.wait_for(client.admin.command("ping"), timeout=5) except Exception as e: report.status = "failed" report.errors.append(f"MongoDB 连接健康检查失败: {e}") report.validation_passed = False client.close() return report try: db = client[database] collection = db[collection_name] # 执行前快照 report.before_snapshot = await self._capture_mongo_before_snapshot( collection, operation_spec ) # 执行操作（带超时和错误处理） filter_doc = operation_spec.get("filter", {}) update_doc = operation_spec.get("update", {}) doc = operation_spec.get("document", {}) options = operation_spec.get("options", {}) # 强制 majority write concern 保证数据持久化 exec_options = {"w": "majority", "j": True} if options: exec_options.update(options) result_obj: Any = None if operation == "insertOne": result_obj = await asyncio.wait_for( collection.with_options(write_concern=exec_options.get("w")).insert_one(doc), timeout=timeout, ) report.affected_rows = 1 if result_obj.inserted_id else 0 report.raw_output = f"插入成功，ID: {result_obj.inserted_id}" elif operation == "insertMany": docs = operation_spec.get("documents", []) if not docs: raise SyntaxValidationError("insertMany 操作需要非空的 'documents' 数组") if len(docs) > 10000: raise SyntaxValidationError(f"insertMany 最多支持 10000 条文档，当前 {len(docs)}") result_obj = await asyncio.wait_for( collection.insert_many(docs), timeout=timeout, ) report.affected_rows = len(result_obj.inserted_ids) report.raw_output = f"批量插入成功，共 {len(result_obj.inserted_ids)} 条" elif operation == "updateOne": result_obj = await asyncio.wait_for( collection.update_one(filter_doc, update_doc, **options), timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( f"匹配 {result_obj.matched_count} 条，" f"修改 {result_obj.modified_count} 条" ) elif operation == "updateMany": result_obj = await asyncio.wait_for( collection.update_many(filter_doc, update_doc, **options), timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( f"匹配 {result_obj.matched_count} 条，" f"修改 {result_obj.modified_count} 条" ) elif operation == "deleteOne": result_obj = await asyncio.wait_for( collection.delete_one(filter_doc), timeout=timeout, ) report.affected_rows = result_obj.deleted_count report.raw_output = f"删除 {result_obj.deleted_count} 条" elif operation == "deleteMany": # deleteMany 安全限制：必须先查询确认影响范围 pre_count = await collection.count_documents(filter_doc) if pre_count > 10000: raise SyntaxValidationError( f"deleteMany 将影响 {pre_count} 条文档，超过安全限制 10000" ) result_obj = await asyncio.wait_for( collection.delete_many(filter_doc), timeout=timeout, ) report.affected_rows = result_obj.deleted_count report.raw_output = f"批量删除 {result_obj.deleted_count} 条" elif operation == "replaceOne": result_obj = await asyncio.wait_for( collection.replace_one(filter_doc, doc, **options), timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( f"匹配 {result_obj.matched_count} 条，" f"替换 {result_obj.modified_count} 条" ) # 执行后快照 report.after_snapshot = await self._capture_mongo_after_snapshot( collection, operation_spec ) # 验证 report.validation_passed = self._validate_mongo_execution( operation_spec, report ) if report.validation_passed: report.status = "success" logger.info("工单 %d MongoDB %s 执行成功", ticket.id, operation) else: report.status = "partial" logger.warning("工单 %d MongoDB %s 验证未通过", ticket.id, operation) except asyncio.TimeoutError: report.status = "failed" report.errors.append(f"执行超时（{timeout}秒）") report.validation_passed = False logger.warning("工单 %d MongoDB 执行超时", ticket.id) except SyntaxValidationError as e: report.status = "failed" report.errors.append(f"验证失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"执行异常: {e}") report.validation_passed = False logger.exception("工单 %d MongoDB 执行异常", ticket.id) finally: client.close() except SyntaxValidationError as e: report.status = "failed" report.errors.append(f"语法校验失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"连接异常: {e}") report.validation_passed = False report.duration_ms = int((time.time() - start_time) * 1000) return report # ===== 数据备份 ===== async def _backup_mysql_data( self, conn: Any, sql_content: str, db_type: str ) -> str: """执行前备份将被修改的数据。 对 UPDATE/DELETE 操作，将被影响的数据行导出为 JSON 格式备份。 Args: conn: 数据库连接 sql_content: SQL 内容 db_type: 数据库类型（mysql/postgres） Returns: 备份信息的字符串描述 """ backup_summaries: List[str] = [] statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue sql_type = self._analyzer._detect_sql_type(stmt) table_name = self._extract_table_name(stmt, sql_type) if sql_type == SQLType.UPDATE and table_name: where_clause = self._extract_where_clause_text(stmt) if where_clause: try: row_count = await self._get_affected_count( conn, table_name, where_clause, db_type ) if row_count > 0: backup_summaries.append( f"UPDATE `{table_name}` 将影响 {row_count} 行数据" ) except Exception: pass elif sql_type == SQLType.DELETE and table_name: where_clause = self._extract_where_clause_text(stmt) if where_clause: try: row_count = await self._get_affected_count( conn, table_name, where_clause, db_type ) if row_count > 0: backup_summaries.append( f"DELETE `{table_name}` 将删除 {row_count} 行数据" ) except Exception: pass return "; ".join(backup_summaries) if backup_summaries else "无需备份" # ===== 执行前快照 ===== async def _capture_sql_before_snapshot( self, conn: Any, sql_content: str, db_type: str ) -> str: """执行前捕获 SQL 数据库的数据快照。 对 UPDATE/DELETE 操作，查询将被影响的数据行。 对 INSERT 操作，记录表当前行数。 Args: conn: 数据库连接 sql_content: SQL 内容 db_type: 数据库类型（mysql/postgres） Returns: 快照信息的字符串表示 """ snapshots: List[str] = [] statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue sql_type = self._analyzer._detect_sql_type(stmt) table_name = self._extract_table_name(stmt, sql_type) if sql_type == SQLType.UPDATE: where_clause = self._extract_where_clause_text(stmt) if table_name: try: count = await self._get_affected_count( conn, table_name, where_clause, db_type ) snapshots.append( f"UPDATE 表 `{table_name}`: 将有 {count} 行被更新" f"{' (WHERE: ' + where_clause + ')' if where_clause else ' (全表更新)' }" ) except Exception as e: snapshots.append(f"UPDATE 表 `{table_name}`: 快照获取失败 - {e}") elif sql_type == SQLType.DELETE: where_clause = self._extract_where_clause_text(stmt) if table_name: try: count = await self._get_affected_count( conn, table_name, where_clause, db_type ) snapshots.append( f"DELETE 表 `{table_name}`: 将有 {count} 行被删除" f"{' (WHERE: ' + where_clause + ')' if where_clause else ' (全表删除)' }" ) except Exception as e: snapshots.append(f"DELETE 表 `{table_name}`: 快照获取失败 - {e}") elif sql_type == SQLType.INSERT: if table_name: try: current_count = await self._get_table_row_count( conn, table_name, db_type ) snapshots.append( f"INSERT 表 `{table_name}`: 当前 {current_count} 行" ) except Exception as e: snapshots.append(f"INSERT 表 `{table_name}`: 行数获取失败 - {e}") return "\n".join(snapshots) if snapshots else "无快照信息" async def _capture_sql_after_snapshot( self, conn: Any, sql_content: str, db_type: str ) -> str: """执行后捕获 SQL 数据库的数据快照。 Args: conn: 数据库连接 sql_content: SQL 内容 db_type: 数据库类型 Returns: 执行后的快照信息 """ snapshots: List[str] = [] statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue sql_type = self._analyzer._detect_sql_type(stmt) table_name = self._extract_table_name(stmt, sql_type) if table_name and sql_type in (SQLType.UPDATE, SQLType.DELETE, SQLType.INSERT): try: current_count = await self._get_table_row_count( conn, table_name, db_type ) snapshots.append(f"表 `{table_name}`: 当前 {current_count} 行") except Exception as e: snapshots.append(f"表 `{table_name}`: 行数获取失败 - {e}") return "\n".join(snapshots) if snapshots else "无快照信息" async def _capture_redis_before_snapshot( self, conn: Any, commands: List[Dict[str, Any]] ) -> str: """执行前捕获 Redis 键的快照。 记录将被修改的键的当前值和 TTL。 Args: conn: Redis 连接 commands: 命令列表 Returns: 键快照信息的字符串表示 """ snapshots: List[str] = [] keys_to_capture: set = set() # 提取可能被修改的键 for cmd_info in commands: command = cmd_info["command"] db = cmd_info.get("db", 0) parts = command.strip().split() if len(parts) >= 2: cmd_name = parts[0].upper() if cmd_name in REDIS_WRITE_COMMANDS: key = parts[1] keys_to_capture.add((key, db)) # 查询每个键的当前值 for key, db in keys_to_capture: try: if db != 0: await conn.select(db) key_type = await conn.type(key) ttl = await conn.ttl(key) if key_type == "none": snapshots.append(f"[DB{db}] 键 '{key}': 不存在") else: # 获取值（限制长度） value_preview = await self._get_redis_value_preview(conn, key, key_type) ttl_str = f"TTL={ttl}s" if ttl > 0 else "无过期时间" snapshots.append( f"[DB{db}] 键 '{key}' ({key_type}, {ttl_str}): {value_preview}" ) except Exception as e: snapshots.append(f"[DB{db}] 键 '{key}': 快照获取失败 - {e}") return "\n".join(snapshots) if snapshots else "无键被修改" async def _capture_redis_after_snapshot( self, conn: Any, commands: List[Dict[str, Any]] ) -> str: """执行后捕获 Redis 键的快照。""" return await self._capture_redis_before_snapshot(conn, commands) async def _capture_mongo_before_snapshot( self, collection: Any, operation_spec: Dict[str, Any] ) -> str: """执行前捕获 MongoDB 文档快照。 记录将被修改的文档数量和示例。 Args: collection: MongoDB 集合对象 operation_spec: 操作规格 Returns: 文档快照信息的字符串表示 """ operation = operation_spec.get("operation", "") filter_doc = operation_spec.get("filter", {}) try: if operation in ("updateOne", "updateMany", "deleteOne", "deleteMany", "replaceOne"): count = await collection.count_documents(filter_doc) return f"操作 '{operation}': 匹配 {count} 条文档\n筛选条件: {json.dumps(filter_doc, ensure_ascii=False)}" elif operation in ("insertOne", "insertMany"): return f"操作 '{operation}': 将插入新文档" return f"操作 '{operation}': 无前置快照" except Exception as e: return f"快照获取失败: {e}" async def _capture_mongo_after_snapshot( self, collection: Any, operation_spec: Dict[str, Any] ) -> str: """执行后捕获 MongoDB 文档快照。""" operation = operation_spec.get("operation", "") filter_doc = operation_spec.get("filter", {}) try: if operation in ("updateOne", "updateMany", "replaceOne"): count = await collection.count_documents(filter_doc) return f"操作 '{operation}' 完成后: 匹配 {count} 条文档" elif operation in ("deleteOne", "deleteMany"): return f"操作 '{operation}' 已完成" elif operation in ("insertOne", "insertMany"): total = await collection.estimated_document_count() return f"操作 '{operation}' 完成后: 集合约 {total} 条文档" return f"操作 '{operation}': 无后置快照" except Exception as e: return f"快照获取失败: {e}" # ===== 验证方法 ===== def _validate_sql_execution(self, sql_content: str, report: ExecutionReport) -> bool: """验证 SQL 执行结果。 验证项: 1. 影响行数 >= 0 2. UPDATE/DELETE 有 WHERE 条件时，影响行数应合理 3. 无错误信息 Args: sql_content: SQL 内容 report: 执行报告 Returns: 验证是否通过 """ passed = True statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue sql_type = self._analyzer._detect_sql_type(stmt) # 检查 UPDATE/DELETE 是否缺少 WHERE if sql_type in (SQLType.UPDATE, SQLType.DELETE): has_where = self._extract_where_clause_text(stmt) is not None if not has_where: report.warnings.append( f"{sql_type.value} 操作缺少 WHERE 条件，影响全表" ) # 检查是否有执行错误 if report.errors: passed = False return passed def _validate_redis_execution( self, commands: List[Dict[str, Any]], report: ExecutionReport ) -> bool: """验证 Redis 执行结果。 验证项: 1. 所有命令都已执行 2. 写命令返回预期结果 Args: commands: 命令列表 report: 执行报告 Returns: 验证是否通过 """ passed = True # 检查是否有错误 if report.errors: passed = False # 检查是否所有命令都执行成功 if report.affected_rows != len(commands): report.warnings.append( f"仅 {report.affected_rows}/{len(commands)} 条命令成功执行" ) return passed def _validate_mongo_execution( self, operation_spec: Dict[str, Any], report: ExecutionReport ) -> bool: """验证 MongoDB 执行结果。 验证项: 1. 修改文档数符合预期 2. 无错误信息 Args: operation_spec: 操作规格 report: 执行报告 Returns: 验证是否通过 """ passed = True operation = operation_spec.get("operation", "") # 检查写操作是否有影响 if operation in ("updateOne", "updateMany", "replaceOne"): if report.affected_rows == 0: report.warnings.append(f"{operation} 未修改任何文档") elif operation in ("deleteOne", "deleteMany"): if report.affected_rows == 0: report.warnings.append(f"{operation} 未删除任何文档") elif operation in ("insertOne",): if report.affected_rows != 1: report.errors.append(f"insertOne 应插入 1 条文档，实际 {report.affected_rows}") passed = False # 检查错误 if report.errors: passed = False return passed # ===== 语法预校验 ===== def _validate_mysql_syntax(self, sql_content: str) -> None: """预校验 MySQL 语法。 使用 sqlparse 解析验证 SQL 语句的基本合法性。 Args: sql_content: SQL 内容 Raises: SyntaxValidationError: 语法校验失败 """ if not sql_content or not sql_content.strip(): raise SyntaxValidationError("SQL 内容不能为空") statements = self._split_statements(sql_content) for stmt in statements: stmt = stmt.strip() if not stmt: continue # 使用 sqlparse 解析 parsed = sqlparse.parse(stmt) if not parsed or not parsed[0].tokens: raise SyntaxValidationError(f"无法解析 SQL: {stmt[:100]}") # 检查是否为合法的 SQL 语句（DML + DDL） sql_type = self._analyzer._detect_sql_type(stmt) allowed_types = ( SQLType.INSERT, SQLType.UPDATE, SQLType.DELETE, SQLType.SELECT, SQLType.CREATE, SQLType.ALTER, SQLType.DROP, SQLType.TRUNCATE, ) if sql_type not in allowed_types: raise SyntaxValidationError( f"不支持的 SQL 类型: {sql_type.value}, 仅支持 INSERT/UPDATE/DELETE/CREATE/ALTER/DROP/TRUNCATE" ) def _validate_postgres_syntax(self, sql_content: str) -> None: """预校验 PostgreSQL 语法。 PostgreSQL 语法校验与 MySQL 类似，使用 sqlparse 进行基础解析。 Args: sql_content: SQL 内容 Raises: SyntaxValidationError: 语法校验失败 """ # PostgreSQL 基础语法校验与 MySQL 相同 self._validate_mysql_syntax(sql_content) def _validate_redis_commands(self, commands: List[Dict[str, Any]]) -> None: """预校验 Redis 命令格式。 验证命令名和参数数量是否合法。 Args: commands: 命令列表 Raises: SyntaxValidationError: 语法校验失败 """ for cmd_info in commands: command = cmd_info.get("command", "").strip() if not command: raise SyntaxValidationError("命令不能为空") parts = command.split() cmd_name = parts[0].upper() # 检查命令名是否合法（基本检查） if not re.match(r"^[A-Z]+$", cmd_name): raise SyntaxValidationError(f"非法的命令名: {cmd_name}") # 检查参数数量（基本检查） if len(parts) < 2 and cmd_name not in ( "PING", "INFO", "DBSIZE", "FLUSHDB", "FLUSHALL", "SAVE", "BGSAVE", "LASTSAVE", "MONITOR", "SYNC", "SHUTDOWN", ): raise SyntaxValidationError(f"命令 {cmd_name} 缺少参数") def _validate_mongo_operation(self, operation_spec: Dict[str, Any]) -> None: """预校验 MongoDB 操作。 验证 JSON 格式和操作类型是否合法。 Args: operation_spec: 操作规格字典 Raises: SyntaxValidationError: 语法校验失败 """ if not isinstance(operation_spec, dict): raise SyntaxValidationError("MongoDB 工单必须是 JSON 对象") collection = operation_spec.get("collection") if not collection or not isinstance(collection, str): raise SyntaxValidationError("缺少或非法的 'collection' 字段") operation = operation_spec.get("operation") if operation not in MONGO_SUPPORTED_OPERATIONS: raise SyntaxValidationError( f"不支持的操作类型: {operation}, " f"支持: {', '.join(MONGO_SUPPORTED_OPERATIONS)}" ) # 检查各操作必需参数 if operation in ("insertOne",) and "document" not in operation_spec: raise SyntaxValidationError("insertOne 操作需要 'document' 字段") if operation in ("insertMany",) and "documents" not in operation_spec: raise SyntaxValidationError("insertMany 操作需要 'documents' 字段") if operation in ("updateOne", "updateMany", "replaceOne"): if "filter" not in operation_spec: raise SyntaxValidationError(f"{operation} 操作需要 'filter' 字段") if "update" not in operation_spec and operation != "replaceOne": raise SyntaxValidationError(f"{operation} 操作需要 'update' 字段") if operation == "replaceOne" and "document" not in operation_spec: raise SyntaxValidationError("replaceOne 操作需要 'document' 字段") if operation in ("deleteOne", "deleteMany"): if "filter" not in operation_spec: raise SyntaxValidationError(f"{operation} 操作需要 'filter' 字段") # ===== 工具方法 ===== @staticmethod def _split_statements(sql_content: str) -> List[str]: """将 SQL 内容按分号分割为多条语句。 智能处理字符串内的分号。 Args: sql_content: SQL 内容 Returns: 语句列表 """ if not sql_content: return [] # 使用 sqlparse 进行更智能的分割 parsed = sqlparse.split(sql_content) return [p.strip() for p in parsed if p.strip()] def _extract_table_name(self, sql: str, sql_type: SQLType) -> Optional[str]: """从 SQL 语句中提取表名。 Args: sql: SQL 语句 sql_type: SQL 类型 Returns: 表名，解析失败返回 None """ sql = sql.strip() upper = sql.upper() if sql_type == SQLType.INSERT: match = re.search(r"INSERT\s+(?:IGNORE\s+)?INTO\s+`?([^`\s(]+)`?", upper) if match: return match.group(1).strip() elif sql_type == SQLType.UPDATE: match = re.search(r"UPDATE\s+`?([^`\s,]+)`?", upper) if match: return match.group(1).strip() elif sql_type == SQLType.DELETE: match = re.search(r"DELETE\s+FROM\s+`?([^`\s(]+)`?", upper) if not match: match = re.search(r"DELETE\s+`?([^`\s(]+)`?\s+FROM", upper) if match: return match.group(1).strip() return None @staticmethod def _extract_where_clause_text(sql: str) -> Optional[str]: """从 SQL 语句中提取 WHERE 子句文本。 Args: sql: SQL 语句 Returns: WHERE 条件文本，没有则返回 None """ parsed = sqlparse.parse(sql) if not parsed: return None statement = parsed[0] # 使用 sqlparse 的 Where 对象 from sqlparse.sql import Where for token in statement.tokens: if isinstance(token, Where): where_text = str(token).replace("WHERE", "", 1).strip() return where_text # 回退：检查关键字 from sqlparse.tokens import Keyword where_tokens = [] in_where = False for token in statement.tokens: if token.ttype is Keyword and token.normalized == "WHERE": in_where = True continue if in_where: if token.ttype is Keyword and token.normalized in ( "ORDER", "GROUP", "HAVING", "LIMIT", ): break where_tokens.append(str(token)) if where_tokens: return "".join(where_tokens).strip() return None async def _get_affected_count( self, conn: Any, table_name: str, where_clause: Optional[str], db_type: str ) -> int: """获取将被影响的数据行数。 Args: conn: 数据库连接 table_name: 表名 where_clause: WHERE 条件 db_type: 数据库类型 Returns: 行数 """ # 安全过滤表名（只允许字母数字下划线） safe_table = re.sub(r"[^\w]", "", table_name) if not safe_table: raise ValueError("非法的表名") if db_type == "postgres": sql = f'SELECT COUNT(*) FROM "{safe_table}"' else: sql = f"SELECT COUNT(*) FROM `{safe_table}`" if where_clause: sql += f" WHERE {where_clause}" sql += " LIMIT 10001" # 根据连接类型执行查询 if db_type == "mysql": async with conn.cursor() as cur: await cur.execute(sql) row = await cur.fetchone() return row[0] if row else 0 else: # PostgreSQL (asyncpg) row = await conn.fetchval(sql) return row or 0 async def _get_table_row_count( self, conn: Any, table_name: str, db_type: str ) -> int: """获取表的当前行数。 Args: conn: 数据库连接 table_name: 表名 db_type: 数据库类型 Returns: 行数 """ safe_table = re.sub(r"[^\w]", "", table_name) if not safe_table: raise ValueError("非法的表名") if db_type == "postgres": sql = f'SELECT COUNT(*) FROM "{safe_table}"' else: sql = f"SELECT COUNT(*) FROM `{safe_table}`" sql += " LIMIT 10001" if db_type == "mysql": async with conn.cursor() as cur: await cur.execute(sql) row = await cur.fetchone() return row[0] if row else 0 else: row = await conn.fetchval(sql) return row or 0 @staticmethod def _parse_redis_commands(content: str, target_database: str = "0") -> List[Dict[str, Any]]: """解析 Redis 工单内容为命令列表。 支持 JSON 格式和简单文本格式。 Args: content: 工单内容 target_database: 目标 DB 编号（来自 ticket.target_database） Returns: 命令列表，每项为 {"command": str, "db": int} """ content = content.strip() if not content: return [] # 解析默认 db 编号 try: default_db = int(target_database) except (ValueError, TypeError): default_db = 0 # 尝试 JSON 解析 if content.startswith("{"): try: data = json.loads(content) if isinstance(data, dict) and "commands" in data: commands = [] for cmd in data["commands"]: if isinstance(cmd, dict) and "command" in cmd: commands.append({ "command": cmd["command"], "db": cmd.get("db", default_db), }) elif isinstance(cmd, str): commands.append({"command": cmd, "db": default_db}) return commands # 可能是单条命令的 JSON if "command" in data: return [{"command": data["command"], "db": data.get("db", default_db)}] except json.JSONDecodeError: pass # 不是 JSON，尝试文本格式 # 简单文本格式：每行一个命令 commands = [] for line in content.split("\n"): line = line.strip() if line and not line.startswith("#"): commands.append({"command": line, "db": default_db}) return commands @staticmethod def _parse_mongo_operation(content: str) -> Dict[str, Any]: """解析 MongoDB 工单内容为操作规格。 Args: content: 工单内容（JSON 字符串） Returns: 操作规格字典 Raises: SyntaxValidationError: JSON 解析失败 """ content = content.strip() if not content: raise SyntaxValidationError("MongoDB 工单内容不能为空") try: data = json.loads(content) if not isinstance(data, dict): raise SyntaxValidationError("MongoDB 工单必须是 JSON 对象") return data except json.JSONDecodeError as e: raise SyntaxValidationError(f"JSON 解析失败: {e}") @staticmethod async def _get_redis_value_preview( conn: Any, key: str, key_type: str, max_len: int = 200 ) -> str: """获取 Redis 键值的预览。 Args: conn: Redis 连接 key: 键名 key_type: 键类型 max_len: 最大预览长度 Returns: 值预览字符串 """ try: if key_type == "string": value = await conn.get(key) preview = str(value)[:max_len] return f"'{preview}'" elif key_type == "hash": value = await conn.hgetall(key) preview = str(value)[:max_len] return f"{preview}" elif key_type == "list": length = await conn.llen(key) return f"" elif key_type == "set": length = await conn.scard(key) return f"" elif key_type == "zset": length = await conn.zcard(key) return f"" else: return f"<{key_type}>" except Exception: return "<无法读取>" @staticmethod def _format_redis_result(result: Any) -> str: """格式化 Redis 命令执行结果。 Args: result: 原始结果 Returns: 格式化后的字符串 """ if result is None: return "(nil)" if isinstance(result, bytes): return result.decode("utf-8", errors="replace") if isinstance(result, bool): return "1" if result else "0" if isinstance(result, (list, set, tuple)): items = [] for item in result: if isinstance(item, bytes): items.append(item.decode("utf-8", errors="replace")) else: items.append(str(item)) return "[" + ", ".join(items[:10]) + ("..." if len(items) > 10 else "") + "]" if isinstance(result, dict): preview = str(result)[:200] return preview return str(result)