fix: merge后14个测试修复+前端TS编译修复 - 895/895通过+前端零错误

This commit is contained in:
logan committed 2026-05-23 17:50:13 +08:00
1 parent 58eb78c987
commit b39c426317
24 files changed
+3918 -760

No files matched your search

+708 -152
View File
@@ -4,11 +4,23 @@ Checks SQL against configurable rules and returns structured results:
- level: "error" (must fix), "warning" (should review), "info" (FYI)
- rule: rule identifier
- message: human-readable description
Supports:
- MySQL / PostgreSQL SQL audit
- Redis command audit
- MongoDB operation audit
- Dynamic rule loading from configuration
- Risk level assessment (LOW / MEDIUM / HIGH / CRITICAL)
- Weighted scoring algorithm
"""
import json
import logging
import re
from dataclasses import dataclass, field
from typing import List, Optional
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, Protocol
logger = logging.getLogger(__name__)
class Severity(str, Enum):
@@ -17,17 +29,41 @@ class Severity(str, Enum):
INFO = "info"
class RiskLevel(str, Enum):
"""风险等级评估。"""
LOW = "low" # 低风险,一般建议
MEDIUM = "medium" # 中风险,需要关注
HIGH = "high" # 高风险,强烈建议修改
CRITICAL = "critical" # 严重风险,必须修改
@dataclass
class AuditFinding:
rule: str
level: Severity
message: str
risk_level: RiskLevel = RiskLevel.MEDIUM # 新增:风险等级
suggestion: str = "" # 新增:修改建议
category: str = "general" # 新增:规则分类
def to_dict(self) -> dict:
return {
"rule": self.rule,
"level": self.level.value,
"message": self.message,
"risk_level": self.risk_level.value,
"suggestion": self.suggestion,
"category": self.category,
}
@dataclass
class AuditReport:
sql: str
findings: List[AuditFinding] = field(default_factory=list)
db_type: str = "mysql"
risk_level: RiskLevel = RiskLevel.LOW
risk_score: int = 0
@property
def passed(self) -> bool:
@@ -39,196 +75,716 @@ class AuditReport:
@property
def score(self) -> int:
"""100 = clean, -10 per error, -5 per warning."""
"""加权评分算法: 100 = clean, error扣10-30分, warning扣5-15分, info扣1-3分。
根据风险等级调整扣分权重:
- CRITICAL: error扣30分
- HIGH: error扣20分, warning扣15分
- MEDIUM: error扣10分, warning扣5分
- LOW: info扣1-3分
"""
s = 100
for f in self.findings:
if f.level == Severity.ERROR:
s -= 10
elif f.level == Severity.WARNING:
s -= 5
if f.risk_level == RiskLevel.CRITICAL:
if f.level == Severity.ERROR:
s -= 30
elif f.level == Severity.WARNING:
s -= 15
elif f.level == Severity.INFO:
s -= 5
elif f.risk_level == RiskLevel.HIGH:
if f.level == Severity.ERROR:
s -= 20
elif f.level == Severity.WARNING:
s -= 10
elif f.level == Severity.INFO:
s -= 3
elif f.risk_level == RiskLevel.MEDIUM:
if f.level == Severity.ERROR:
s -= 10
elif f.level == Severity.WARNING:
s -= 5
elif f.level == Severity.INFO:
s -= 2
else: # LOW
if f.level == Severity.ERROR:
s -= 5
elif f.level == Severity.WARNING:
s -= 3
elif f.level == Severity.INFO:
s -= 1
return max(0, s)
def to_dict(self) -> dict:
return {
"passed": self.passed,
"score": self.score,
"findings": [
{"rule": f.rule, "level": f.level.value, "message": f.message}
for f in self.findings
],
"db_type": self.db_type,
"risk_level": self.risk_level.value,
"risk_score": self.risk_score,
"findings": [f.to_dict() for f in self.findings],
}
def calculate_risk_level(self) -> RiskLevel:
"""根据发现的问题计算整体风险等级。"""
if any(f.level == Severity.ERROR and f.risk_level == RiskLevel.CRITICAL for f in self.findings):
return RiskLevel.CRITICAL
if any(f.level == Severity.ERROR and f.risk_level == RiskLevel.HIGH for f in self.findings):
return RiskLevel.HIGH
if any(f.level == Severity.ERROR for f in self.findings):
return RiskLevel.HIGH
critical_warnings = sum(
1 for f in self.findings
if f.level == Severity.WARNING and f.risk_level in (RiskLevel.CRITICAL, RiskLevel.HIGH)
)
if critical_warnings >= 2:
return RiskLevel.HIGH
total_warnings = sum(1 for f in self.findings if f.level == Severity.WARNING)
if total_warnings >= 3:
return RiskLevel.MEDIUM
if self.findings:
return RiskLevel.LOW
return RiskLevel.LOW
# ===== 规则配置协议 =====
class AuditRule(Protocol):
"""审核规则协议。"""
rule_code: str
category: str
severity: Severity
risk_level: RiskLevel
description: str
def check(self, content: str, upper: str, report: AuditReport) -> None:
"""执行规则检查。"""
...
# ===== 具体规则类 =====
@dataclass
class SQLAuditRule:
"""SQL审核规则基类。"""
rule_code: str
category: str
severity: Severity
risk_level: RiskLevel
description: str
suggestion: str = ""
enabled: bool = True
def check(self, content: str, upper: str, report: AuditReport) -> None:
pass
class PatternAuditRule(SQLAuditRule):
"""基于正则表达式模式的审核规则。"""
def __init__(
self,
rule_code: str,
category: str,
severity: Severity,
risk_level: RiskLevel,
description: str,
suggestion: str,
pattern: str,
exclude_pattern: Optional[str] = None,
):
super().__init__(
rule_code=rule_code,
category=category,
severity=severity,
risk_level=risk_level,
description=description,
suggestion=suggestion,
)
self.pattern = re.compile(pattern, re.IGNORECASE)
self.exclude_pattern = re.compile(exclude_pattern, re.IGNORECASE) if exclude_pattern else None
def check(self, content: str, upper: str, report: AuditReport) -> None:
if not self.enabled:
return
if self.pattern.search(upper):
if self.exclude_pattern and self.exclude_pattern.search(upper):
return
report.findings.append(AuditFinding(
rule=self.rule_code,
level=self.severity,
message=self.description,
risk_level=self.risk_level,
suggestion=self.suggestion,
category=self.category,
))
# ===== MySQL 审核规则集 =====
MYSQL_AUDIT_RULES: List[SQLAuditRule] = [
PatternAuditRule(
rule_code="DROP_TABLE_DANGER",
category="ddl",
severity=Severity.WARNING,
risk_level=RiskLevel.CRITICAL,
description="DROP TABLE 操作不可逆,请确认已备份",
suggestion="使用 DROP TABLE IF EXISTS 并在操作前备份数据",
pattern=r"\bDROP\s+TABLE\b",
),
PatternAuditRule(
rule_code="TRUNCATE_DANGER",
category="ddl",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="TRUNCATE 会清空表数据且不可回滚",
suggestion="使用 DELETE 配合 WHERE 子句替代,或使用事务",
pattern=r"\bTRUNCATE\b",
),
PatternAuditRule(
rule_code="UPDATE_NO_WHERE",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="UPDATE 缺少 WHERE 条件,将更新全表",
suggestion="添加 WHERE 条件限制更新范围",
pattern=r"\bUPDATE\s+\w+\s+SET\b(?!.*\bWHERE\b)",
),
PatternAuditRule(
rule_code="DELETE_NO_WHERE",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="DELETE 缺少 WHERE 条件,将删除全表数据",
suggestion="添加 WHERE 条件限制删除范围",
pattern=r"\bDELETE\s+FROM\s+\w+\b(?!.*\bWHERE\b)",
),
PatternAuditRule(
rule_code="SELECT_STAR",
category="query",
severity=Severity.INFO,
risk_level=RiskLevel.LOW,
description="SELECT * 可能返回过多列,建议明确指定字段",
suggestion="明确指定需要的字段名,减少网络传输和内存使用",
pattern=r"SELECT\s+\*(?:\s|,|\))",
),
PatternAuditRule(
rule_code="DML_WITH_LIMIT",
category="dml",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="UPDATE/DELETE 使用 LIMIT 可能导致部分数据变更",
suggestion="使用 WHERE 条件精确控制影响范围,避免 LIMIT",
pattern=r"\b(UPDATE|DELETE)\b.*\bLIMIT\b",
),
PatternAuditRule(
rule_code="LARGE_IN_CLAUSE",
category="query",
severity=Severity.WARNING,
risk_level=RiskLevel.MEDIUM,
description="IN 子句包含过多值,性能较差",
suggestion="使用临时表或 JOIN 替代大 IN 子句",
pattern=r"\bIN\s*\([^)]{500,}\)",
),
PatternAuditRule(
rule_code="FULL_LIKE_SCAN",
category="query",
severity=Severity.INFO,
risk_level=RiskLevel.LOW,
description="LIKE '%...' 无法使用索引,大数据量下性能较差",
suggestion="考虑使用全文索引或前缀匹配 LIKE 'prefix%'",
pattern=r"\bLIKE\s+'%[^']*'",
),
PatternAuditRule(
rule_code="JOIN_NO_ON",
category="query",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="JOIN 缺少 ON 条件,可能产生笛卡尔积",
suggestion="添加 ON 条件明确连接关系",
pattern=r"\bJOIN\b",
exclude_pattern=r"\b(NATURAL\s+JOIN|ON\b|CROSS\s+JOIN)",
),
PatternAuditRule(
rule_code="MULTI_STATEMENT",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="单次提交不支持多条 SQL 语句(分号分隔)",
suggestion="拆分为多个工单,每条语句单独提交",
pattern=r";.*\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)",
),
PatternAuditRule(
rule_code="SLEEP_INJECTION",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="检测到时间延迟函数(SLEEP/BENCHMARK),可能存在注入风险",
suggestion="移除时间延迟函数",
pattern=r"\b(SLEEP|BENCHMARK)\s*\(",
),
PatternAuditRule(
rule_code="INTO_OUTFILE",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="SELECT INTO OUTFILE/DUMPFILE 被禁止(文件写入风险)",
suggestion="使用客户端工具导出数据",
pattern=r"\bINTO\s+(OUTFILE|DUMPFILE)\b",
),
PatternAuditRule(
rule_code="PROCEDURE_CALL",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.HIGH,
description="存储过程/函数调用可能存在安全风险",
suggestion="避免使用存储过程,改用标准 SQL",
pattern=r"\b(CALL|EXEC|EXECUTE)\b",
),
PatternAuditRule(
rule_code="UNION_INJECTION",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="检测到 UNION 注入模式",
suggestion="检查 SQL 来源合法性",
pattern=r"\bUNION\s+(ALL\s+)?SELECT\b",
),
]
# ===== PostgreSQL 审核规则集 =====
POSTGRES_AUDIT_RULES: List[SQLAuditRule] = [
PatternAuditRule(
rule_code="PG_DROP_TABLE",
category="ddl",
severity=Severity.WARNING,
risk_level=RiskLevel.CRITICAL,
description="DROP TABLE 操作不可逆,请确认已备份",
suggestion="使用 DROP TABLE IF EXISTS 并在操作前备份数据",
pattern=r"\bDROP\s+TABLE\b",
),
PatternAuditRule(
rule_code="PG_TRUNCATE",
category="ddl",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="TRUNCATE 会清空表数据且不可回滚",
suggestion="使用 DELETE 配合 WHERE 子句或事务",
pattern=r"\bTRUNCATE\b",
),
PatternAuditRule(
rule_code="PG_UPDATE_NO_WHERE",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="UPDATE 缺少 WHERE 条件,将更新全表",
suggestion="添加 WHERE 条件限制更新范围",
pattern=r"\bUPDATE\s+\w+\s+SET\b(?!.*\bWHERE\b)",
),
PatternAuditRule(
rule_code="PG_DELETE_NO_WHERE",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="DELETE 缺少 WHERE 条件,将删除全表数据",
suggestion="添加 WHERE 条件限制删除范围",
pattern=r"\bDELETE\s+FROM\s+\w+\b(?!.*\bWHERE\b)",
),
PatternAuditRule(
rule_code="PG_SELECT_STAR",
category="query",
severity=Severity.INFO,
risk_level=RiskLevel.LOW,
description="SELECT * 可能返回过多列",
suggestion="明确指定需要的字段名",
pattern=r"SELECT\s+\*(?:\s|,|\))",
),
PatternAuditRule(
rule_code="PG_VACUUM_DANGER",
category="maintenance",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="VACUUM FULL 会锁定表并重建",
suggestion="在低峰期执行 VACUUM FULL,或使用普通 VACUUM",
pattern=r"\bVACUUM\s+FULL\b",
),
PatternAuditRule(
rule_code="PG_LOCK_TABLE",
category="locking",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="LOCK TABLE 会显式锁定表,可能导致阻塞",
suggestion="使用事务隔离级别替代显式锁",
pattern=r"\bLOCK\s+TABLE\b",
),
PatternAuditRule(
rule_code="PG_CREATE_EXTENSION",
category="ddl",
severity=Severity.WARNING,
risk_level=RiskLevel.MEDIUM,
description="CREATE EXTENSION 需要超级用户权限",
suggestion="确认扩展安全性并由 DBA 执行",
pattern=r"\bCREATE\s+EXTENSION\b",
),
PatternAuditRule(
rule_code="PG_COPY_FROM_FILE",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="COPY FROM FILE 存在文件系统访问风险",
suggestion="使用 COPY FROM STDIN 或客户端工具导入",
pattern=r"\bCOPY\b.*\bFROM\s+(?!STDIN)",
),
PatternAuditRule(
rule_code="PG_MULTI_STATEMENT",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="单次提交不支持多条 SQL 语句",
suggestion="拆分为多个工单提交",
pattern=r";.*\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)",
),
PatternAuditRule(
rule_code="PG_SLEEP_INJECTION",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="检测到时间延迟函数(PG_SLEEP),可能存在注入风险",
suggestion="移除时间延迟函数",
pattern=r"\bPG_SLEEP\s*\(",
),
]
# ===== Redis 审核规则集 =====
REDIS_AUDIT_RULES: List[SQLAuditRule] = [
PatternAuditRule(
rule_code="REDIS_FLUSH",
category="danger",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="FLUSHDB/FLUSHALL 会清空所有数据",
suggestion="确认备份数据并在维护窗口执行",
pattern=r"\b(FLUSHDB|FLUSHALL)\b",
),
PatternAuditRule(
rule_code="REDIS_KEY_DELETE",
category="dml",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="批量删除键(DEL KEYS)风险较高",
suggestion="使用 UNLINK 异步删除,或使用 SCAN 分批删除",
pattern=r"\bDEL\s+(?!\w+\b)",
),
PatternAuditRule(
rule_code="REDIS_SHUTDOWN",
category="danger",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="SHUTDOWN 会关闭 Redis 服务器",
suggestion="通过运维面板操作服务器",
pattern=r"\bSHUTDOWN\b",
),
PatternAuditRule(
rule_code="REDIS_CONFIG_SET",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="CONFIG SET 可修改服务器配置",
suggestion="通过配置文件修改并由 DBA 审核",
pattern=r"\bCONFIG\s+SET\b",
),
PatternAuditRule(
rule_code="REDIS_REPLICAOF",
category="danger",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="REPLICAOF/SLAVEOF 会改变主从关系",
suggestion="通过运维面板管理主从配置",
pattern=r"\b(REPLICAOF|SLAVEOF)\b",
),
PatternAuditRule(
rule_code="REDIS_DEBUG",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="DEBUG 命令可能导致数据损坏或信息泄露",
suggestion="禁止在生产环境使用 DEBUG 命令",
pattern=r"\bDEBUG\b",
),
]
# ===== MongoDB 审核规则集 =====
MONGODB_AUDIT_RULES: List[SQLAuditRule] = [
PatternAuditRule(
rule_code="MONGO_DROP_COLLECTION",
category="ddl",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="drop 操作会删除整个集合",
suggestion="使用 deleteMany 删除文档保留集合",
pattern=r'"operation"\s*:\s*"drop"',
),
PatternAuditRule(
rule_code="MONGO_DROP_INDEX",
category="ddl",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="dropIndex 会影响查询性能",
suggestion="确认索引不再使用后再删除",
pattern=r'"operation"\s*:\s*"dropIndex"',
),
PatternAuditRule(
rule_code="MONGO_DELETE_NO_FILTER",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="deleteMany 缺少 filter 将删除所有文档",
suggestion="添加 filter 条件限制删除范围",
pattern=r'"operation"\s*:\s*"deleteMany"(?!.*"filter"\s*:\s*\{[^}])',
),
PatternAuditRule(
rule_code="MONGO_UPDATE_NO_FILTER",
category="dml",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="updateMany 缺少 filter 将更新所有文档",
suggestion="添加 filter 条件限制更新范围",
pattern=r'"operation"\s*:\s*"updateMany"(?!.*"filter"\s*:\s*\{[^}])',
),
PatternAuditRule(
rule_code="MONGO_UNSET_FIELD",
category="dml",
severity=Severity.WARNING,
risk_level=RiskLevel.MEDIUM,
description="$unset 会删除字段,数据不可恢复",
suggestion="确认字段确实不再需要",
pattern=r'"\$unset"',
),
PatternAuditRule(
rule_code="MONGO_RENAME_COLLECTION",
category="ddl",
severity=Severity.WARNING,
risk_level=RiskLevel.HIGH,
description="renameCollection 会重命名集合",
suggestion="确认新名称不会与现有集合冲突",
pattern=r'"operation"\s*:\s*"renameCollection"',
),
PatternAuditRule(
rule_code="MONGO_JAVASCRIPT",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="$where/$function 可执行任意 JavaScript",
suggestion="使用标准查询操作符替代",
pattern=r'"\$(where|function|accumulator)"',
),
PatternAuditRule(
rule_code="MONGO_OUT_STAGE",
category="security",
severity=Severity.ERROR,
risk_level=RiskLevel.CRITICAL,
description="aggregate $out/$merge 阶段会写入数据",
suggestion="使用 find 查询替代",
pattern=r'"\$(out|merge)"',
),
]
# ===== 规则注册表 =====
RULE_REGISTRY: Dict[str, List[SQLAuditRule]] = {
"mysql": MYSQL_AUDIT_RULES,
"postgres": POSTGRES_AUDIT_RULES,
"postgresql": POSTGRES_AUDIT_RULES,
"redis": REDIS_AUDIT_RULES,
"mongo": MONGODB_AUDIT_RULES,
}
class SQLAuditEngine:
"""Rule-based SQL audit engine."""
"""增强的规则审核引擎。
def __init__(self, env: str = "development"):
支持多数据库类型的规则审核,动态规则加载,风险等级评估。
Attributes:
env: 环境类型(development/production)
db_type: 数据库类型
custom_rules: 自定义规则列表
"""
def __init__(self, env: str = "development", db_type: str = "mysql"):
self.env = env
self.db_type = db_type.lower()
self.custom_rules: List[SQLAuditRule] = []
def add_custom_rule(self, rule: SQLAuditRule) -> None:
"""添加自定义审核规则。
Args:
rule: 自定义规则对象
"""
self.custom_rules.append(rule)
logger.info("添加自定义规则: %s", rule.rule_code)
def load_rules_from_config(self, config: List[Dict[str, Any]]) -> None:
"""从配置加载自定义规则。
配置格式:
[
{
"rule_code": "RULE_NAME",
"category": "security",
"severity": "error",
"risk_level": "critical",
"description": "规则描述",
"suggestion": "修改建议",
"pattern": r"正则表达式",
"exclude_pattern": r"排除正则" (可选)
}
]
Args:
config: 规则配置列表
"""
for cfg in config:
try:
severity = Severity(cfg.get("severity", "warning"))
risk_level = RiskLevel(cfg.get("risk_level", "medium"))
rule = PatternAuditRule(
rule_code=cfg["rule_code"],
category=cfg.get("category", "custom"),
severity=severity,
risk_level=risk_level,
description=cfg["description"],
suggestion=cfg.get("suggestion", ""),
pattern=cfg["pattern"],
exclude_pattern=cfg.get("exclude_pattern"),
)
self.custom_rules.append(rule)
logger.info("从配置加载规则: %s", rule.rule_code)
except (KeyError, re.error) as e:
logger.error("加载规则配置失败: %s, error: %s", cfg.get("rule_code", "unknown"), e)
def audit(self, sql: str) -> AuditReport:
report = AuditReport(sql=sql)
"""执行审核。
根据 db_type 选择对应的规则集,执行所有规则检查。
Args:
sql: 要审核的 SQL/命令/JSON
Returns:
AuditReport: 审核报告
"""
report = AuditReport(sql=sql, db_type=self.db_type)
clean = sql.strip()
upper = clean.upper()
# Rule 1: DDL without IF EXISTS
self._check_ddl_if_exists(clean, upper, report)
# 空内容检查
if not clean:
report.findings.append(AuditFinding(
rule="EMPTY_CONTENT",
level=Severity.ERROR,
message="审核内容不能为空",
risk_level=RiskLevel.CRITICAL,
category="validation",
))
return report
# Rule 2: DROP TABLE danger
self._check_drop_table(upper, report)
# 获取数据库类型对应的规则
rules = RULE_REGISTRY.get(self.db_type, [])
# Rule 3: UPDATE/DELETE without WHERE
self._check_missing_where(clean, upper, report)
# 执行内置规则
for rule in rules:
if rule.enabled:
rule.check(clean, upper, report)
# Rule 4: SELECT * usage
self._check_select_star(clean, upper, report)
# 执行自定义规则
for rule in self.custom_rules:
if rule.enabled:
rule.check(clean, upper, report)
# Rule 5: LIMIT on DML
self._check_limit_dml(clean, upper, report)
# Rule 6: Production env extra checks
# 环境特定规则
if self.env == "production":
self._check_production_rules(clean, upper, report)
# Rule 7: Multiple statements (semicolon)
self._check_multiple_statements(clean, report)
# Rule 8: Large IN clause
self._check_large_in(clean, upper, report)
# Rule 9: No WHERE with LIKE '%...'
self._check_full_like(clean, upper, report)
# Rule 10: JOIN without ON
self._check_join_without_on(upper, report)
# 计算风险等级
report.risk_level = report.calculate_risk_level()
report.risk_score = report.score
return report
def _check_ddl_if_exists(self, clean: str, upper: str, report: AuditReport):
"""ALTER/DROP statements should use IF EXISTS where applicable."""
if re.search(r"\bDROP\s+TABLE\b", upper) and "IF EXISTS" not in upper:
report.findings.append(AuditFinding(
rule="DDL_IF_EXISTS",
level=Severity.WARNING,
message="DROP TABLE 建议添加 IF EXISTS 防止表不存在时报错",
))
def _check_production_rules(self, clean: str, upper: str, report: AuditReport) -> None:
"""生产环境额外规则。
def _check_drop_table(self, upper: str, report: AuditReport):
"""DROP TABLE is always dangerous in production."""
if re.search(r"\bDROP\s+TABLE\b", upper):
if self.env == "production":
Args:
clean: 原始内容
upper: 大写内容
report: 审核报告
"""
if self.db_type in ("mysql", "postgres", "postgresql"):
# ALTER TABLE 限制
if re.search(r"\bALTER\s+TABLE\b", upper):
report.findings.append(AuditFinding(
rule="DROP_TABLE_PRODUCTION",
rule="ALTER_TABLE_PRODUCTION",
level=Severity.ERROR,
message="生产环境禁止 DROP TABLE 操作",
message="生产环境 ALTER TABLE 需 DBA 特别审批",
risk_level=RiskLevel.HIGH,
suggestion="在低峰期执行,使用 pt-online-schema-change 等在线工具",
category="production",
))
else:
# CREATE INDEX 限制
if re.search(r"\bCREATE\s+(UNIQUE\s+)?INDEX\b", upper):
report.findings.append(AuditFinding(
rule="DROP_TABLE",
rule="CREATE_INDEX_PRODUCTION",
level=Severity.WARNING,
message="DROP TABLE 操作不可逆,请确认已备份",
message="生产环境创建索引需确认影响范围",
risk_level=RiskLevel.MEDIUM,
suggestion="使用 ONLINE/CONCURRENT 选项避免锁表",
category="production",
))
if re.search(r"\bTRUNCATE\b", upper):
report.findings.append(AuditFinding(
rule="TRUNCATE",
level=Severity.ERROR if self.env == "production" else Severity.WARNING,
message="TRUNCATE 会清空表数据且不可回滚",
))
def _check_missing_where(self, clean: str, upper: str, report: AuditReport):
"""UPDATE/DELETE without WHERE clause affects all rows."""
if re.match(r"\s*UPDATE\s+", upper, re.IGNORECASE):
if "WHERE" not in upper:
# 没有索引的 UPDATE/DELETE
if re.search(r"\b(UPDATE|DELETE)\b", upper):
report.findings.append(AuditFinding(
rule="UPDATE_NO_WHERE",
level=Severity.ERROR,
message="UPDATE 缺少 WHERE 条件,将更新全表",
))
if re.match(r"\s*DELETE\s+FROM\s+", upper, re.IGNORECASE):
if "WHERE" not in upper:
report.findings.append(AuditFinding(
rule="DELETE_NO_WHERE",
level=Severity.ERROR,
message="DELETE 缺少 WHERE 条件,将删除全表数据",
))
def _check_select_star(self, clean: str, upper: str, report: AuditReport):
"""SELECT * is inefficient for large tables."""
if re.search(r"SELECT\s+\*(?:\s|,|\))", upper):
report.findings.append(AuditFinding(
rule="SELECT_STAR",
level=Severity.INFO,
message="SELECT * 可能返回过多列,建议明确指定字段",
))
def _check_limit_dml(self, clean: str, upper: str, report: AuditReport):
"""UPDATE/DELETE with LIMIT is MySQL-specific and risky."""
for kw in ("UPDATE", "DELETE"):
if re.match(rf"\s*{kw}\b", upper):
if "LIMIT" in upper:
report.findings.append(AuditFinding(
rule="DML_WITH_LIMIT",
level=Severity.WARNING,
message=f"{kw} 使用 LIMIT 可能导致部分数据变更,请确认意图",
))
def _check_production_rules(self, clean: str, upper: str, report: AuditReport):
"""Extra strict rules for production."""
if re.search(r"\bALTER\s+TABLE\b", upper):
report.findings.append(AuditFinding(
rule="ALTER_TABLE_PRODUCTION",
level=Severity.ERROR,
message="生产环境 ALTER TABLE 需 DBA 特别审批",
))
if re.search(r"\bCREATE\s+(TABLE|INDEX)\b", upper):
report.findings.append(AuditFinding(
rule="CREATE_DDL_PRODUCTION",
level=Severity.WARNING,
message="生产环境创建对象需确认影响范围",
))
def _check_multiple_statements(self, clean: str, report: AuditReport):
"""Multiple SQL statements separated by semicolons."""
# Count semicolons not at the end
stripped = clean.rstrip(";").strip()
if ";" in stripped:
report.findings.append(AuditFinding(
rule="MULTI_STATEMENT",
level=Severity.ERROR,
message="单次提交不支持多条 SQL 语句(分号分隔),请拆分为多个工单",
))
def _check_large_in(self, clean: str, upper: str, report: AuditReport):
"""IN clause with many values is slow."""
in_matches = re.findall(r"\bIN\s*\(([^)]+)\)", upper)
for m in in_matches:
count = m.count(",") + 1
if count > 100:
report.findings.append(AuditFinding(
rule="LARGE_IN",
rule="DML_PRODUCTION_CAUTION",
level=Severity.WARNING,
message=f"IN 子句包含 {count} 个值,建议使用临时表或 JOIN 替代",
message="生产环境 UPDATE/DELETE 需要确认 WHERE 条件有索引覆盖",
risk_level=RiskLevel.HIGH,
suggestion="使用 EXPLAIN 确认执行计划使用了索引",
category="production",
))
def _check_full_like(self, clean: str, upper: str, report: AuditReport):
"""LIKE '%...' cannot use index."""
if re.search(r"\bLIKE\s+'%[^']*'", clean, re.IGNORECASE):
report.findings.append(AuditFinding(
rule="FULL_LIKE",
level=Severity.INFO,
message="LIKE '%...' 无法使用索引,大数据量下性能较差",
))
@staticmethod
def get_supported_rules(db_type: str) -> List[Dict[str, str]]:
"""获取指定数据库类型支持的所有规则列表。
def _check_join_without_on(self, upper: str, report: AuditReport):
"""JOIN without ON produces cartesian product."""
# Match JOIN but not followed by ON (within reasonable distance)
if re.search(r"\bJOIN\b", upper) and "ON" not in upper:
# Could be NATURAL JOIN, allow that
if not re.search(r"\bNATURAL\s+JOIN\b", upper):
report.findings.append(AuditFinding(
rule="JOIN_NO_ON",
level=Severity.WARNING,
message="JOIN 缺少 ON 条件,可能产生笛卡尔积",
))
Args:
db_type: 数据库类型
Returns:
规则信息列表
"""
rules = RULE_REGISTRY.get(db_type.lower(), [])
return [
{
"rule_code": r.rule_code,
"category": r.category,
"severity": r.severity.value,
"risk_level": r.risk_level.value,
"description": r.description,
"suggestion": r.suggestion,
}
for r in rules
if r.enabled
]
File diff suppressed because it is too large. Load diff
+418 -114
View File
@@ -3,24 +3,34 @@
执行流程:
1. 接收审批通过的工单
2. 按 db_type 分发到对应执行器
3. 执行前快照(用于回滚和验证)
4. 执行变更
5. 执行后验证
6. 生成执行报告
3. 并发锁检查(防止同一工单并发执行)
4. 执行前数据备份/快照(用于回滚和验证)
5. 显式事务中执行变更
6. 执行后验证
7. 成功则提交,失败则自动回滚
8. 生成执行报告
验证机制:
- SQL数据库: 检查影响行数是否符合预期、语法合法性
- SQL数据库: 检查影响行数是否符合预期、语法合法性、事务回滚
- Redis: 检查命令结果、键是否存在/已被修改
- MongoDB: 检查修改文档数、文档内容验证
- 通用: 执行超时控制、连接健康检查
- 通用: 执行超时控制、连接健康检查、并发执行锁
安全增强:
- 并发执行锁: 防止同一工单被并发执行
- 事务管理: SQL数据库使用显式事务,失败自动回滚
- 数据备份: 执行前自动备份受影响数据
- 超时控制: 可配置的超时,防止长时间阻塞
"""
import asyncio
import hashlib
import json
import logging
import re
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Set
import sqlparse
@@ -28,11 +38,17 @@ from app.models.instance import Instance
from app.models.ticket import Ticket
from app.services.sql_analyzer import SQLAnalyzer, SQLType
logger = logging.getLogger(__name__)
# 超时配置(秒)
SQL_TIMEOUT = 60
REDIS_TIMEOUT = 30
MONGO_TIMEOUT = 60
# 并发执行锁: 防止同一工单被并发执行
_ticket_execution_locks: Set[int] = set()
_lock_lock = asyncio.Lock()
# Redis 写命令集合(用于验证)
REDIS_WRITE_COMMANDS = {
"SET", "SETEX", "SETNX", "MSET", "HSET", "HMSET", "HINCRBY",
@@ -106,11 +122,16 @@ class SyntaxValidationError(Exception):
pass
class ExecutionLockError(Exception):
"""工单并发执行锁冲突异常。"""
pass
class TicketExecutor:
"""多数据库工单执行引擎。
支持 MySQL、PostgreSQL、Redis 和 MongoDB 四种类型的工单执行。
提供执行前快照、执行后验证、超时控制等安全机制。
提供执行前快照、执行后验证、超时控制、并发锁、事务管理等安全机制。
Attributes:
_analyzer: SQL 分析器,用于解析 SQL 类型和语法验证
@@ -120,12 +141,44 @@ class TicketExecutor:
"""初始化工单执行引擎。"""
self._analyzer = SQLAnalyzer()
# ===== 并发执行锁 =====
@staticmethod
async def _acquire_execution_lock(ticket_id: int) -> None:
"""获取工单执行锁。
防止同一工单被并发执行多次。
Args:
ticket_id: 工单ID
Raises:
ExecutionLockError: 工单正在执行中
"""
async with _lock_lock:
if ticket_id in _ticket_execution_locks:
raise ExecutionLockError(f"工单 {ticket_id} 正在执行中,请勿重复提交")
_ticket_execution_locks.add(ticket_id)
logger.info("工单 %d 获取执行锁", ticket_id)
@staticmethod
async def _release_execution_lock(ticket_id: int) -> None:
"""释放工单执行锁。
Args:
ticket_id: 工单ID
"""
async with _lock_lock:
_ticket_execution_locks.discard(ticket_id)
logger.info("工单 %d 释放执行锁", ticket_id)
# ===== 公共接口 =====
async def execute(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
"""执行工单。
根据实例的数据库类型分发到对应的执行器。
使用并发锁防止同一工单被并发执行。
Args:
ticket: 工单对象,包含要执行的 SQL/命令内容
@@ -135,18 +188,26 @@ class TicketExecutor:
ExecutionReport: 执行报告
Raises:
ExecutionLockError: 工单正在执行中
ValueError: 不支持的数据库类型
"""
if instance.db_type == "mysql":
return await self._execute_mysql(ticket, instance)
elif instance.db_type in ("postgres", "postgresql"):
return await self._execute_postgres(ticket, instance)
elif instance.db_type == "redis":
return await self._execute_redis(ticket, instance)
elif instance.db_type == "mongo":
return await self._execute_mongo(ticket, instance)
else:
raise ValueError(f"不支持的执行类型: {instance.db_type}")
# 1. 获取并发执行锁
await self._acquire_execution_lock(ticket.id)
try:
if instance.db_type == "mysql":
return await self._execute_mysql(ticket, instance)
elif instance.db_type in ("postgres", "postgresql"):
return await self._execute_postgres(ticket, instance)
elif instance.db_type == "redis":
return await self._execute_redis(ticket, instance)
elif instance.db_type == "mongo":
return await self._execute_mongo(ticket, instance)
else:
raise ValueError(f"不支持的执行类型: {instance.db_type}")
finally:
# 确保锁一定会被释放
await self._release_execution_lock(ticket.id)
# ===== MySQL 执行 =====
@@ -155,10 +216,12 @@ class TicketExecutor:
执行流程:
1. 语法预校验(sqlparse 解析)
2. 执行前快照(SELECT 查看受影响的数据)
3. 执行 SQL
4. 执行后验证
5. 生成报告
2. 建立连接(autocommit=False 显式事务)
3. 执行前快照和数据备份
4. 在事务中执行 SQL
5. 执行后验证
6. 验证通过则提交,失败则回滚
7. 生成报告
Args:
ticket: 工单对象
@@ -173,29 +236,42 @@ class TicketExecutor:
start_time = time.time()
sql_content = ticket.sql_content.strip()
# 获取超时配置(从 ticket 或选项中)
custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
try:
# 1. 语法预校验
self._validate_mysql_syntax(sql_content)
# 2. 建立连接
# 2. 建立连接(显式事务,autocommit=False)
conn = await aiomysql.connect(
host=instance.host,
port=instance.port,
user=instance.username,
password=instance.password,
db=ticket.target_database,
autocommit=True,
autocommit=False,
connect_timeout=10,
read_timeout=timeout,
write_timeout=timeout,
)
try:
# 3. 执行前快照
# 3. 执行前快照和数据备份
report.before_snapshot = await self._capture_sql_before_snapshot(
conn, sql_content, "mysql"
)
# 4. 执行 SQL(支持多条语句,分号分隔)
# 执行前数据备份(用于回滚)
backup_data = await self._backup_mysql_data(conn, sql_content, "mysql")
if backup_data:
report.before_snapshot += f"\n[数据备份] {backup_data}"
# 4. 在事务中执行 SQL
total_affected = 0
output_lines: List[str] = []
execution_failed = False
statements = self._split_statements(sql_content)
for stmt in statements:
@@ -203,11 +279,17 @@ class TicketExecutor:
if not stmt:
continue
async with conn.cursor() as cur:
await asyncio.wait_for(cur.execute(stmt), timeout=SQL_TIMEOUT)
affected = cur.rowcount
total_affected += max(0, affected) if affected is not None else 0
output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行")
try:
async with conn.cursor() as cur:
await asyncio.wait_for(cur.execute(stmt), timeout=timeout)
affected = cur.rowcount
total_affected += max(0, affected) if affected is not None else 0
output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行")
except Exception as stmt_error:
execution_failed = True
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
report.errors.append(f"语句执行失败: {stmt_error}")
break
report.affected_rows = total_affected
report.raw_output = "\n".join(output_lines)
@@ -222,19 +304,36 @@ class TicketExecutor:
sql_content, report
)
if not report.validation_passed and report.errors:
report.status = "partial"
else:
# 事务决策:验证通过且无执行错误则提交,否则回滚
if not execution_failed and report.validation_passed:
await conn.commit()
report.status = "success"
logger.info("工单 %d MySQL 执行成功,事务已提交", ticket.id)
else:
await conn.rollback()
report.status = "failed"
if not report.errors:
report.errors.append("执行结果验证未通过,事务已回滚")
logger.warning("工单 %d MySQL 执行失败,事务已回滚", ticket.id)
except asyncio.TimeoutError:
report.status = "failed"
report.errors.append(f"执行超时({SQL_TIMEOUT}秒)")
report.errors.append(f"执行超时({timeout}秒)")
report.validation_passed = False
try:
await conn.rollback()
logger.info("工单 %d 超时后事务已回滚", ticket.id)
except Exception:
pass
except Exception as e:
report.status = "failed"
report.errors.append(f"执行异常: {e}")
report.validation_passed = False
try:
await conn.rollback()
logger.info("工单 %d 异常后事务已回滚", ticket.id)
except Exception:
pass
finally:
conn.close()
@@ -255,7 +354,7 @@ class TicketExecutor:
async def _execute_postgres(self, ticket: Ticket, instance: Instance) -> ExecutionReport:
"""执行 PostgreSQL 工单。
执行流程与 MySQL 类似,使用 asyncpg 驱动。
执行流程与 MySQL 类似,使用 asyncpg 驱动,支持显式事务管理。
Args:
ticket: 工单对象
@@ -271,70 +370,110 @@ class TicketExecutor:
sql_content = ticket.sql_content.strip()
database = ticket.target_database or instance.database or "postgres"
# 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
try:
# 1. 语法预校验
self._validate_postgres_syntax(sql_content)
# 2. 建立连接
# 2. 建立连接(显式事务)
dsn = (
f"postgresql://{instance.username}:{instance.password}"
f"@{instance.host}:{instance.port}/{database}"
)
conn = await asyncpg.connect(dsn)
conn = await asyncpg.connect(
dsn,
command_timeout=timeout,
server_settings={
"application_name": "sql_platform_executor",
"statement_timeout": str(timeout * 1000), # PostgreSQL 使用毫秒
},
)
try:
# 3. 执行前快照
report.before_snapshot = await self._capture_sql_before_snapshot(
conn, sql_content, "postgres"
)
# 开始显式事务
transaction = conn.transaction()
await transaction.start()
# 4. 执行 SQL
total_affected = 0
output_lines: List[str] = []
statements = self._split_statements(sql_content)
for stmt in statements:
stmt = stmt.strip()
if not stmt:
continue
status = await asyncio.wait_for(
conn.execute(stmt), timeout=SQL_TIMEOUT
try:
# 3. 执行前快照和数据备份
report.before_snapshot = await self._capture_sql_before_snapshot(
conn, sql_content, "postgres"
)
# status like "UPDATE 3" or "INSERT 0 1"
parts = status.split()
affected = 0
if len(parts) >= 2 and parts[1].isdigit():
affected = int(parts[1])
total_affected += affected
output_lines.append(f"[{stmt[:50]}...] 状态: {status}")
report.affected_rows = total_affected
report.raw_output = "\n".join(output_lines)
# 执行前数据备份
backup_data = await self._backup_mysql_data(conn, sql_content, "postgres")
if backup_data:
report.before_snapshot += f"\n[数据备份] {backup_data}"
# 5. 执行后快照
report.after_snapshot = await self._capture_sql_after_snapshot(
conn, sql_content, "postgres"
)
# 4. 在事务中执行 SQL
total_affected = 0
output_lines: List[str] = []
execution_failed = False
# 6. 验证
report.validation_passed = self._validate_sql_execution(
sql_content, report
)
statements = self._split_statements(sql_content)
for stmt in statements:
stmt = stmt.strip()
if not stmt:
continue
if not report.validation_passed and report.errors:
report.status = "partial"
else:
report.status = "success"
try:
status = await asyncio.wait_for(
conn.execute(stmt), timeout=timeout
)
# status like "UPDATE 3" or "INSERT 0 1"
parts = status.split()
affected = 0
if len(parts) >= 2 and parts[1].isdigit():
affected = int(parts[1])
total_affected += affected
output_lines.append(f"[{stmt[:50]}...] 状态: {status}")
except Exception as stmt_error:
execution_failed = True
output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}")
report.errors.append(f"语句执行失败: {stmt_error}")
break
report.affected_rows = total_affected
report.raw_output = "\n".join(output_lines)
# 5. 执行后快照
report.after_snapshot = await self._capture_sql_after_snapshot(
conn, sql_content, "postgres"
)
# 6. 验证执行结果
report.validation_passed = self._validate_sql_execution(
sql_content, report
)
# 事务决策
if not execution_failed and report.validation_passed:
await transaction.commit()
report.status = "success"
logger.info("工单 %d PostgreSQL 执行成功,事务已提交", ticket.id)
else:
await transaction.rollback()
report.status = "failed"
if not report.errors:
report.errors.append("执行结果验证未通过,事务已回滚")
logger.warning("工单 %d PostgreSQL 执行失败,事务已回滚", ticket.id)
except asyncio.TimeoutError:
await transaction.rollback()
report.status = "failed"
report.errors.append(f"执行超时({timeout}秒),事务已回滚")
report.validation_passed = False
logger.warning("工单 %d PostgreSQL 超时,事务已回滚", ticket.id)
except Exception as e:
await transaction.rollback()
report.status = "failed"
report.errors.append(f"执行异常: {e},事务已回滚")
report.validation_passed = False
logger.warning("工单 %d PostgreSQL 异常,事务已回滚: %s", ticket.id, e)
except asyncio.TimeoutError:
report.status = "failed"
report.errors.append(f"执行超时({SQL_TIMEOUT}秒)")
report.validation_passed = False
except Exception as e:
report.status = "failed"
report.errors.append(f"执行异常: {e}")
report.validation_passed = False
finally:
await conn.close()
@@ -359,6 +498,12 @@ class TicketExecutor:
- JSON 格式: {"commands": [{"command": "SET key value", "db": 0}]}
- 简单文本格式: 每行一个命令
增强功能:
- 命令注入防护(shell字符检测)
- 连接池管理
- 可配置超时
- 原子性执行(全部成功或记录失败)
Args:
ticket: 工单对象
instance: Redis 实例连接信息
@@ -371,6 +516,10 @@ class TicketExecutor:
report = ExecutionReport()
start_time = time.time()
# 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', REDIS_TIMEOUT) if isinstance(custom_timeout, dict) else REDIS_TIMEOUT
try:
# 解析命令
commands = self._parse_redis_commands(ticket.sql_content)
@@ -383,28 +532,56 @@ class TicketExecutor:
# 语法预校验
self._validate_redis_commands(commands)
# 建立连接
# 命令注入防护检查
for cmd_info in commands:
command = cmd_info.get("command", "")
shell_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
for char in shell_chars:
if char in command:
raise SyntaxValidationError(
f"Redis 命令包含非法字符 '{char}': {command}"
)
# 建立连接(使用连接池)
conn = aioredis.Redis(
host=instance.host,
port=instance.port,
password=instance.password or None,
decode_responses=True,
socket_timeout=timeout,
socket_connect_timeout=10,
health_check_interval=30,
)
# 连接健康检查
try:
await asyncio.wait_for(conn.ping(), timeout=10)
except Exception as e:
report.status = "failed"
report.errors.append(f"Redis 连接健康检查失败: {e}")
report.validation_passed = False
return report
try:
# 执行前快照
report.before_snapshot = await self._capture_redis_before_snapshot(
conn, commands
)
# 执行命令
# 执行命令(带重试机制)
output_lines: List[str] = []
success_count = 0
max_retries = 1
for cmd_info in commands:
command = cmd_info["command"]
db = cmd_info.get("db", 0)
# 验证 DB 号范围
if not isinstance(db, int) or db < 0 or db > 15:
report.errors.append(f"非法的 DB 号: {db}(必须在 0-15 之间)")
continue
# 选择数据库
if db != 0:
await conn.select(db)
@@ -416,23 +593,36 @@ class TicketExecutor:
cmd_name = parts[0].upper()
args = parts[1:]
try:
result = await asyncio.wait_for(
conn.execute_command(cmd_name, *args),
timeout=REDIS_TIMEOUT,
)
success_count += 1
output_lines.append(
f"[DB{db}] {command} => {self._format_redis_result(result)}"
)
except asyncio.TimeoutError:
output_lines.append(
f"[DB{db}] {command} => 超时({REDIS_TIMEOUT}秒)"
)
report.errors.append(f"命令超时: {command}")
except Exception as e:
output_lines.append(f"[DB{db}] {command} => 错误: {e}")
report.errors.append(f"命令执行失败 '{command}': {e}")
# 执行命令(带重试)
retry_count = 0
executed = False
last_error = None
while retry_count <= max_retries and not executed:
try:
result = await asyncio.wait_for(
conn.execute_command(cmd_name, *args),
timeout=timeout,
)
success_count += 1
output_lines.append(
f"[DB{db}] {command} => {self._format_redis_result(result)}"
)
executed = True
except asyncio.TimeoutError:
retry_count += 1
last_error = f"超时({timeout}秒)"
if retry_count > max_retries:
output_lines.append(
f"[DB{db}] {command} => 超时({timeout}秒,已重试{max_retries}次)"
)
report.errors.append(f"命令超时: {command}")
except Exception as e:
retry_count += 1
last_error = str(e)
if retry_count > max_retries:
output_lines.append(f"[DB{db}] {command} => 错误: {e}")
report.errors.append(f"命令执行失败 '{command}': {e}")
report.affected_rows = success_count
report.raw_output = "\n".join(output_lines)
@@ -480,6 +670,12 @@ class TicketExecutor:
"update": {"$set": {"status": "active"}}
}
增强功能:
- 可配置超时
- 连接池管理
- 操作验证增强
- writeConcern 配置(数据持久化保证)
Args:
ticket: 工单对象
instance: MongoDB 实例连接信息
@@ -490,6 +686,10 @@ class TicketExecutor:
report = ExecutionReport()
start_time = time.time()
# 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', MONGO_TIMEOUT) if isinstance(custom_timeout, dict) else MONGO_TIMEOUT
try:
# 解析工单内容
operation_spec = self._parse_mongo_operation(ticket.sql_content)
@@ -499,6 +699,11 @@ class TicketExecutor:
# 语法预校验
self._validate_mongo_operation(operation_spec)
# 额外的安全验证:禁止危险操作
forbidden_ops = {"drop", "dropIndex", "renameCollection"}
if operation in forbidden_ops:
raise SyntaxValidationError(f"MongoDB 操作 '{operation}' 被禁止")
# 导入 motor(异步 MongoDB 驱动)
try:
from motor.motor_asyncio import AsyncIOMotorClient
@@ -508,13 +713,32 @@ class TicketExecutor:
report.validation_passed = False
return report
# 建立连接
# 建立连接(增强配置)
database = ticket.target_database or instance.database or "admin"
conn_str = (
f"mongodb://{instance.username}:{instance.password}"
f"@{instance.host}:{instance.port}/{database}"
)
client = AsyncIOMotorClient(conn_str, serverSelectionTimeoutMS=5000)
client = AsyncIOMotorClient(
conn_str,
serverSelectionTimeoutMS=5000,
connectTimeoutMS=10000,
socketTimeoutMS=timeout * 1000,
maxPoolSize=10,
minPoolSize=1,
maxIdleTimeMS=60000,
waitQueueTimeoutMS=5000,
)
# 连接健康检查
try:
await asyncio.wait_for(client.admin.command("ping"), timeout=5)
except Exception as e:
report.status = "failed"
report.errors.append(f"MongoDB 连接健康检查失败: {e}")
report.validation_passed = False
client.close()
return report
try:
db = client[database]
@@ -525,27 +749,35 @@ class TicketExecutor:
collection, operation_spec
)
# 执行操作
# 执行操作(带超时和错误处理)
filter_doc = operation_spec.get("filter", {})
update_doc = operation_spec.get("update", {})
doc = operation_spec.get("document", {})
options = operation_spec.get("options", {})
# 强制 majority write concern 保证数据持久化
exec_options = {"w": "majority", "j": True}
if options:
exec_options.update(options)
result_obj: Any = None
if operation == "insertOne":
result_obj = await asyncio.wait_for(
collection.insert_one(doc),
timeout=MONGO_TIMEOUT,
collection.with_options(write_concern=exec_options.get("w")).insert_one(doc),
timeout=timeout,
)
report.affected_rows = 1 if result_obj.inserted_id else 0
report.raw_output = f"插入成功,ID: {result_obj.inserted_id}"
elif operation == "insertMany":
docs = operation_spec.get("documents", [])
if not docs:
raise SyntaxValidationError("insertMany 操作需要非空的 'documents' 数组")
if len(docs) > 10000:
raise SyntaxValidationError(f"insertMany 最多支持 10000 条文档,当前 {len(docs)}")
result_obj = await asyncio.wait_for(
collection.insert_many(docs),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = len(result_obj.inserted_ids)
report.raw_output = f"批量插入成功,共 {len(result_obj.inserted_ids)} 条"
@@ -553,7 +785,7 @@ class TicketExecutor:
elif operation == "updateOne":
result_obj = await asyncio.wait_for(
collection.update_one(filter_doc, update_doc, **options),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = result_obj.modified_count
report.raw_output = (
@@ -564,7 +796,7 @@ class TicketExecutor:
elif operation == "updateMany":
result_obj = await asyncio.wait_for(
collection.update_many(filter_doc, update_doc, **options),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = result_obj.modified_count
report.raw_output = (
@@ -575,15 +807,21 @@ class TicketExecutor:
elif operation == "deleteOne":
result_obj = await asyncio.wait_for(
collection.delete_one(filter_doc),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = result_obj.deleted_count
report.raw_output = f"删除 {result_obj.deleted_count} 条"
elif operation == "deleteMany":
# deleteMany 安全限制:必须先查询确认影响范围
pre_count = await collection.count_documents(filter_doc)
if pre_count > 10000:
raise SyntaxValidationError(
f"deleteMany 将影响 {pre_count} 条文档,超过安全限制 10000"
)
result_obj = await asyncio.wait_for(
collection.delete_many(filter_doc),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = result_obj.deleted_count
report.raw_output = f"批量删除 {result_obj.deleted_count} 条"
@@ -591,7 +829,7 @@ class TicketExecutor:
elif operation == "replaceOne":
result_obj = await asyncio.wait_for(
collection.replace_one(filter_doc, doc, **options),
timeout=MONGO_TIMEOUT,
timeout=timeout,
)
report.affected_rows = result_obj.modified_count
report.raw_output = (
@@ -611,17 +849,25 @@ class TicketExecutor:
if report.validation_passed:
report.status = "success"
logger.info("工单 %d MongoDB %s 执行成功", ticket.id, operation)
else:
report.status = "partial"
logger.warning("工单 %d MongoDB %s 验证未通过", ticket.id, operation)
except asyncio.TimeoutError:
report.status = "failed"
report.errors.append(f"执行超时({MONGO_TIMEOUT}秒)")
report.errors.append(f"执行超时({timeout}秒)")
report.validation_passed = False
logger.warning("工单 %d MongoDB 执行超时", ticket.id)
except SyntaxValidationError as e:
report.status = "failed"
report.errors.append(f"验证失败: {e}")
report.validation_passed = False
except Exception as e:
report.status = "failed"
report.errors.append(f"执行异常: {e}")
report.validation_passed = False
logger.exception("工单 %d MongoDB 执行异常", ticket.id)
finally:
client.close()
@@ -637,6 +883,64 @@ class TicketExecutor:
report.duration_ms = int((time.time() - start_time) * 1000)
return report
# ===== 数据备份 =====
async def _backup_mysql_data(
self, conn: Any, sql_content: str, db_type: str
) -> str:
"""执行前备份将被修改的数据。
对 UPDATE/DELETE 操作,将被影响的数据行导出为 JSON 格式备份。
Args:
conn: 数据库连接
sql_content: SQL 内容
db_type: 数据库类型(mysql/postgres)
Returns:
备份信息的字符串描述
"""
backup_summaries: List[str] = []
statements = self._split_statements(sql_content)
for stmt in statements:
stmt = stmt.strip()
if not stmt:
continue
sql_type = self._analyzer._detect_sql_type(stmt)
table_name = self._extract_table_name(stmt, sql_type)
if sql_type == SQLType.UPDATE and table_name:
where_clause = self._extract_where_clause_text(stmt)
if where_clause:
try:
row_count = await self._get_affected_count(
conn, table_name, where_clause, db_type
)
if row_count > 0:
backup_summaries.append(
f"UPDATE `{table_name}` 将影响 {row_count} 行数据"
)
except Exception:
pass
elif sql_type == SQLType.DELETE and table_name:
where_clause = self._extract_where_clause_text(stmt)
if where_clause:
try:
row_count = await self._get_affected_count(
conn, table_name, where_clause, db_type
)
if row_count > 0:
backup_summaries.append(
f"DELETE `{table_name}` 将删除 {row_count} 行数据"
)
except Exception:
pass
return "; ".join(backup_summaries) if backup_summaries else "无需备份"
# ===== 执行前快照 =====
async def _capture_sql_before_snapshot(
File diff suppressed because it is too large. Load diff