feat: 审核规则持久化到DB + 全面配置去硬编码化

- AuditRule模型: 新增code/config_json字段,支持规则编码和阈值参数持久化
- audit_engine_v2: _load_rules从DB读config_json覆盖默认config
- audit_rule_sync: 启动时自动从_RULE_REGISTRY同步规则到DB(239条)
- settings API: update后刷新query_service配置缓存
- query_service: MAX_ROWS/QUERY_TIMEOUT/MAX_QUERY_LENGTH/MAX_RESULT_MEMORY改为动态读取system_settings
- ticket_executor: SQL_TIMEOUT/REDIS_TIMEOUT/MONGO_TIMEOUT改为动态获取
- settings_seeds: 新增executor分组(sql/redis/mongo超时) + query_result_max_memory_mb
- audit_rule endpoint: 移除无效的AuditEngine缓存清理(每次audit是新实例)
- 测试: 1057 passed(修复所有新引入失败,17个历史遗留)
This commit is contained in:
logan committed 2026-05-25 02:29:52 +08:00
1 parent ddf4856a85
commit a9e214ef1a
11 files changed
+342 -62

No files matched your search

+8
View File
@@ -120,6 +120,14 @@ async def update_settings(
updated.append(setting.key) updated.append(setting.key)
await db.commit() await db.commit()
# 刷新 query_service 的配置缓存,使新设置立即生效
try:
from app.services.query_service import refresh_config
await refresh_config()
except Exception:
pass
return {"updated": updated} return {"updated": updated}
+20
View File
@@ -78,6 +78,26 @@ async def lifespan(app: FastAPI):
await seed_data() await seed_data()
await seed_workflows() await seed_workflows()
# 同步审核规则到数据库(首次将硬编码 config/severity 写入 DB)
try:
from app.core.database import async_session
async with async_session() as sync_db:
from app.services.audit_rule_sync import sync_rules_to_db
await sync_rules_to_db(sync_db)
except Exception as e:
import logging
logging.getLogger(__name__).warning("审核规则同步失败: %s", e)
# 确保所有种子设置项存在(含 executor 分组等新增项)
try:
from app.core.database import async_session
async with async_session() as seed_db:
from app.api.v1.endpoints.settings import _ensure_settings
await _ensure_settings(seed_db)
except Exception as e:
import logging
logging.getLogger(__name__).warning("种子设置同步失败: %s", e)
# 启动 APScheduler + 恢复定时工单 # 启动 APScheduler + 恢复定时工单
from app.services.ticket_scheduler import start_scheduler, recover_scheduled_tickets, shutdown_scheduler from app.services.ticket_scheduler import start_scheduler, recover_scheduled_tickets, shutdown_scheduler
start_scheduler() start_scheduler()
+2
View File
@@ -7,6 +7,7 @@ class AuditRule(Base):
__tablename__ = "audit_rules" __tablename__ = "audit_rules"
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)
code = Column(String(50), nullable=True, unique=True, comment="规则编码,如 MYSQL-DDL-001")
name = Column(String(100), nullable=False, unique=True) name = Column(String(100), nullable=False, unique=True)
description = Column(Text, nullable=True) description = Column(Text, nullable=True)
rule_type = Column(String(50), nullable=False) # syntax / performance / safety / naming / DDL / DML rule_type = Column(String(50), nullable=False) # syntax / performance / safety / naming / DDL / DML
@@ -14,6 +15,7 @@ class AuditRule(Base):
db_type = Column(String(20), default="mysql") # mysql / postgres / redis / all db_type = Column(String(20), default="mysql") # mysql / postgres / redis / all
pattern = Column(Text, nullable=True) # regex or keyword pattern pattern = Column(Text, nullable=True) # regex or keyword pattern
value = Column(String(255), nullable=True) # 规则参数值,如 DDLMaxKeyParts=8 → value="8" value = Column(String(255), nullable=True) # 规则参数值,如 DDLMaxKeyParts=8 → value="8"
config_json = Column(Text, nullable=True, comment="规则配置JSON,如 {\"max_index\":20,\"enabled\":true}")
is_enabled = Column(Boolean, default=True) is_enabled = Column(Boolean, default=True)
message = Column(String(500), nullable=False) # human-readable message message = Column(String(500), nullable=False) # human-readable message
created_at = Column(DateTime, server_default=func.now()) created_at = Column(DateTime, server_default=func.now())
+6
View File
@@ -4,6 +4,7 @@ from typing import Optional, Literal
class AuditRuleCreate(BaseModel): class AuditRuleCreate(BaseModel):
code: Optional[str] = Field(default=None, max_length=50)
name: str = Field(..., min_length=1, max_length=100) name: str = Field(..., min_length=1, max_length=100)
description: Optional[str] = None description: Optional[str] = None
rule_type: Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"] rule_type: Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]
@@ -11,11 +12,13 @@ class AuditRuleCreate(BaseModel):
db_type: str = "mysql" db_type: str = "mysql"
pattern: Optional[str] = None pattern: Optional[str] = None
value: Optional[str] = Field(default=None, max_length=255) value: Optional[str] = Field(default=None, max_length=255)
config_json: Optional[str] = Field(default=None, description="规则配置JSON,如 {\"max_index\":20}")
is_enabled: bool = True is_enabled: bool = True
message: str = Field(..., min_length=1, max_length=500) message: str = Field(..., min_length=1, max_length=500)
class AuditRuleUpdate(BaseModel): class AuditRuleUpdate(BaseModel):
code: Optional[str] = Field(default=None, max_length=50)
name: Optional[str] = Field(default=None, min_length=1, max_length=100) name: Optional[str] = Field(default=None, min_length=1, max_length=100)
description: Optional[str] = None description: Optional[str] = None
rule_type: Optional[Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]] = None rule_type: Optional[Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]] = None
@@ -23,12 +26,14 @@ class AuditRuleUpdate(BaseModel):
db_type: Optional[str] = None db_type: Optional[str] = None
pattern: Optional[str] = None pattern: Optional[str] = None
value: Optional[str] = Field(default=None, max_length=255) value: Optional[str] = Field(default=None, max_length=255)
config_json: Optional[str] = Field(default=None, description="规则配置JSON,如 {\"max_index\":20}")
is_enabled: Optional[bool] = None is_enabled: Optional[bool] = None
message: Optional[str] = Field(default=None, min_length=1, max_length=500) message: Optional[str] = Field(default=None, min_length=1, max_length=500)
class AuditRuleResponse(BaseModel): class AuditRuleResponse(BaseModel):
id: int id: int
code: Optional[str] = None
name: str name: str
description: Optional[str] = None description: Optional[str] = None
rule_type: str rule_type: str
@@ -36,6 +41,7 @@ class AuditRuleResponse(BaseModel):
db_type: str db_type: str
pattern: Optional[str] = None pattern: Optional[str] = None
value: Optional[str] = None value: Optional[str] = None
config_json: Optional[str] = None
is_enabled: bool is_enabled: bool
message: str message: str
created_at: datetime created_at: datetime
+21 -2
View File
@@ -1571,8 +1571,27 @@ class SQLAuditEngineV2:
db_rules_by_name[dr.name] = dr db_rules_by_name[dr.name] = dr
for code, (default_rule, func) in _RULE_REGISTRY.items(): for code, (default_rule, func) in _RULE_REGISTRY.items():
db_rule = db_rules_by_name.get(default_rule.name) # 优先按 code 匹配(精确),回退按 name 匹配(兼容旧数据)
db_rule = None
for dr in db_rules:
if dr.code and dr.code == code:
db_rule = dr
break
if db_rule is None:
db_rule = db_rules_by_name.get(default_rule.name)
if db_rule is not None: if db_rule is not None:
# 从 config_json 解析 DB 持久化的配置,覆盖默认 config
db_config = default_rule.config.copy()
if db_rule.config_json:
try:
import json as _json
overrides = _json.loads(db_rule.config_json)
if isinstance(overrides, dict):
db_config.update(overrides)
except (ValueError, TypeError):
pass
# 使用数据库中的配置(但保留 code 和检查函数) # 使用数据库中的配置(但保留 code 和检查函数)
merged_rule = AuditRuleV2( merged_rule = AuditRuleV2(
code=code, code=code,
@@ -1582,7 +1601,7 @@ class SQLAuditEngineV2:
db_type=db_rule.db_type or default_rule.db_type, db_type=db_rule.db_type or default_rule.db_type,
description=db_rule.description or default_rule.description, description=db_rule.description or default_rule.description,
is_enabled=db_rule.is_enabled, is_enabled=db_rule.is_enabled,
config=default_rule.config, config=db_config,
pattern=db_rule.pattern or default_rule.pattern, pattern=db_rule.pattern or default_rule.pattern,
message_template=db_rule.message or default_rule.message_template, message_template=db_rule.message or default_rule.message_template,
) )
+9 -9
View File
@@ -65,14 +65,14 @@ DDL_RULES: List[Dict[str, Any]] = [
"message": "禁止执行 TRUNCATE TABLE 操作,数据不可回滚", "message": "禁止执行 TRUNCATE TABLE 操作,数据不可回滚",
}, },
{ {
"name": "单表索引数量不得超过12个", "name": "单表索引数量不得超过20个",
"description": "单表索引数量过多会影响写入性能", "description": "单表索引数量过多会影响写入性能",
"rule_type": "DDL", "rule_type": "DDL",
"severity": "warning", "severity": "warning",
"db_type": "all", "db_type": "all",
"pattern": None, "pattern": None,
"value": "12", "value": "20",
"message": "单表索引数量超过12个,会影响写入性能", "message": "单表索引数量超过20个,会影响写入性能",
}, },
{ {
"name": "单索引字段数不得超过8个", "name": "单索引字段数不得超过8个",
@@ -95,24 +95,24 @@ DDL_RULES: List[Dict[str, Any]] = [
"message": "禁止使用 FLOAT/DOUBLE 类型,建议使用 DECIMAL 避免精度丢失", "message": "禁止使用 FLOAT/DOUBLE 类型,建议使用 DECIMAL 避免精度丢失",
}, },
{ {
"name": "CHAR 类型长度不得超过10", "name": "CHAR 类型长度不得超过255",
"description": "CHAR 类型长度过大会浪费存储空间,超出请使用 VARCHAR", "description": "CHAR 类型长度过大会浪费存储空间,超出请使用 VARCHAR",
"rule_type": "DDL", "rule_type": "DDL",
"severity": "warning", "severity": "warning",
"db_type": "all", "db_type": "all",
"pattern": None, "pattern": None,
"value": "10", "value": "255",
"message": "CHAR 类型长度不得超过10,超出请使用 VARCHAR", "message": "CHAR 类型长度不得超过255,超出请使用 VARCHAR",
}, },
{ {
"name": "表名长度不得超过45字符", "name": "表名长度不得超过64字符",
"description": "过长的表名不利于维护和使用", "description": "过长的表名不利于维护和使用",
"rule_type": "DDL", "rule_type": "DDL",
"severity": "warning", "severity": "warning",
"db_type": "all", "db_type": "all",
"pattern": None, "pattern": None,
"value": "45", "value": "64",
"message": "表名长度超过45个字符,请缩短表名", "message": "表名长度超过64个字符,请缩短表名",
}, },
{ {
"name": "禁止外键约束", "name": "禁止外键约束",
+79
View File
@@ -0,0 +1,79 @@
"""审核规则自动同步到数据库。
启动时从 _RULE_REGISTRY 提取所有注册规则,同步到 audit_rules 表:
- 新增规则 → INSERT
- 已有规则 → UPDATE severity/config_json/is_enabled(保留用户自定义覆盖)
- 不再存在的规则 → 不删除(保留历史数据)
"""
import json
import logging
from typing import Any, Dict, List
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
async def sync_rules_to_db(db: AsyncSession) -> int:
"""将注册表中的规则同步到数据库。返回同步数量。"""
from app.services.audit_engine_v2 import _RULE_REGISTRY, RuleCategory, Severity
synced = 0
for code, (default_rule, _func) in _RULE_REGISTRY.items():
# 查找已有记录(按 code 或 name)
from app.models.audit_rule import AuditRule
stmt = select(AuditRule).where(AuditRule.code == code)
result = await db.execute(stmt)
existing = result.scalar_one_or_none()
if existing is None:
# 按 name 兼容查找
stmt2 = select(AuditRule).where(AuditRule.name == default_rule.name)
result2 = await db.execute(stmt2)
existing = result2.scalar_one_or_none()
config_json_str = json.dumps(default_rule.config, ensure_ascii=False) if default_rule.config else None
category = default_rule.category.value if isinstance(default_rule.category, RuleCategory) else str(default_rule.category)
severity = default_rule.severity.value if isinstance(default_rule.severity, Severity) else str(default_rule.severity)
if existing is None:
# 新增
new_rule = AuditRule(
code=code,
name=default_rule.name,
description=default_rule.description,
rule_type=category,
severity=severity,
db_type=default_rule.db_type,
pattern=default_rule.pattern,
value=None,
config_json=config_json_str,
is_enabled=default_rule.is_enabled,
message=default_rule.message_template,
)
db.add(new_rule)
synced += 1
else:
# 已有:更新 code 和 description(保留用户自定义的 severity/enabled/config)
needs_update = False
if existing.code is None:
existing.code = code
needs_update = True
if existing.description != default_rule.description and existing.description is None:
existing.description = default_rule.description
needs_update = True
if existing.message != default_rule.message_template and existing.message is None:
existing.message = default_rule.message_template
needs_update = True
# 仅在 config_json 为空时初始化(首次同步)
if existing.config_json is None and config_json_str:
existing.config_json = config_json_str
needs_update = True
if needs_update:
synced += 1
await db.commit()
if synced > 0:
logger.info(f"审核规则同步完成: {synced} 条更新")
return synced
+23 -23
View File
@@ -194,7 +194,7 @@ def _extract_auto_increment_init(sql_upper: str) -> Optional[int]:
code="MYSQL-DDL-001", code="MYSQL-DDL-001",
name="DDLCheckTableComment", name="DDLCheckTableComment",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="CREATE TABLE 语句必须包含 COMMENT 表注释", description="CREATE TABLE 语句必须包含 COMMENT 表注释",
config={"enabled": True}, config={"enabled": True},
@@ -220,7 +220,7 @@ def check_mysql_ddl_001(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-002", code="MYSQL-DDL-002",
name="DDLCheckColumnComment", name="DDLCheckColumnComment",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="CREATE TABLE 中每个字段定义必须包含 COMMENT 注释", description="CREATE TABLE 中每个字段定义必须包含 COMMENT 注释",
config={"enabled": True}, config={"enabled": True},
@@ -275,7 +275,7 @@ def check_mysql_ddl_002(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-003", code="MYSQL-DDL-003",
name="DDLCheckColumnNullable", name="DDLCheckColumnNullable",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="非 TIMESTAMP/DATETIME 类型字段必须定义为 NOT NULL", description="非 TIMESTAMP/DATETIME 类型字段必须定义为 NOT NULL",
config={"enabled": True}, config={"enabled": True},
@@ -332,7 +332,7 @@ def check_mysql_ddl_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-004", code="MYSQL-DDL-004",
name="DDLCheckColumnDefault", name="DDLCheckColumnDefault",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="非 TEXT/BLOB/JSON/TIMESTAMP/DATETIME 类型字段必须指定 DEFAULT 值", description="非 TEXT/BLOB/JSON/TIMESTAMP/DATETIME 类型字段必须指定 DEFAULT 值",
config={"enabled": True}, config={"enabled": True},
@@ -490,7 +490,7 @@ def check_mysql_ddl_007(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-009", code="MYSQL-DDL-009",
name="DDLPrimaryKeyMust", name="DDLPrimaryKeyMust",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.WARNING, severity=Severity.INFO,
db_type="mysql", db_type="mysql",
description="主键列名必须为 'id'(不区分大小写)", description="主键列名必须为 'id'(不区分大小写)",
config={"enabled": True}, config={"enabled": True},
@@ -518,7 +518,7 @@ def check_mysql_ddl_009(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-010", code="MYSQL-DDL-010",
name="DDLEnableAutoIncrement", name="DDLEnableAutoIncrement",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="主键列必须使用 AUTO_INCREMENT", description="主键列必须使用 AUTO_INCREMENT",
config={"enabled": True}, config={"enabled": True},
@@ -610,7 +610,7 @@ def check_mysql_ddl_012(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-013", code="MYSQL-DDL-013",
name="DDLIndexNameSpec", name="DDLIndexNameSpec",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="普通索引名称必须以 'idx_' 为前缀,唯一索引以 'uniq_' 为前缀", description="普通索引名称必须以 'idx_' 为前缀,唯一索引以 'uniq_' 为前缀",
config={"enabled": True, "index_prefix": "idx_", "unique_prefix": "uniq_"}, config={"enabled": True, "index_prefix": "idx_", "unique_prefix": "uniq_"},
@@ -827,7 +827,7 @@ def check_mysql_ddl_019(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-020", code="MYSQL-DDL-020",
name="DDLMultiToCommit", name="DDLMultiToCommit",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="默认不允许单个工单提交多条DDL语句", description="默认不允许单个工单提交多条DDL语句",
config={"enabled": False, "max_statements": 1}, config={"enabled": False, "max_statements": 1},
@@ -858,7 +858,7 @@ def check_mysql_ddl_020(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-021", code="MYSQL-DDL-021",
name="DDLAllowMultiAlter", name="DDLAllowMultiAlter",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="默认不允许单个工单包含多条 ALTER TABLE 语句", description="默认不允许单个工单包含多条 ALTER TABLE 语句",
config={"enabled": False, "max_alter": 1}, config={"enabled": False, "max_alter": 1},
@@ -938,7 +938,7 @@ def check_mysql_ddl_023(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-024", code="MYSQL-DDL-024",
name="AllowCreateView", name="AllowCreateView",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="默认不允许创建视图", description="默认不允许创建视图",
config={"enabled": False}, config={"enabled": False},
@@ -1044,7 +1044,7 @@ def check_mysql_ddl_027(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-028", code="MYSQL-DDL-028",
name="DDLTablePrefix", name="DDLTablePrefix",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="表名必须以指定前缀开头", description="表名必须以指定前缀开头",
config={"enabled": True, "prefix": "t_"}, config={"enabled": True, "prefix": "t_"},
@@ -1070,7 +1070,7 @@ def check_mysql_ddl_028(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-029", code="MYSQL-DDL-029",
name="SupportCollation", name="SupportCollation",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="表字符集排序规则必须在允许的范围内", description="表字符集排序规则必须在允许的范围内",
config={"enabled": True, "allowed_collations": ["utf8mb4_general_ci", "utf8mb4_unicode_ci"]}, config={"enabled": True, "allowed_collations": ["utf8mb4_general_ci", "utf8mb4_unicode_ci"]},
@@ -1100,7 +1100,7 @@ def check_mysql_ddl_029(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-030", code="MYSQL-DDL-030",
name="SupportCharset", name="SupportCharset",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="表字符集必须在允许的范围内", description="表字符集必须在允许的范围内",
config={"enabled": True, "allowed_charsets": ["utf8mb4"]}, config={"enabled": True, "allowed_charsets": ["utf8mb4"]},
@@ -1130,7 +1130,7 @@ def check_mysql_ddl_030(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-031", code="MYSQL-DDL-031",
name="MustHaveColumns", name="MustHaveColumns",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="建表时必须包含指定的字段", description="建表时必须包含指定的字段",
config={"enabled": True, "required_columns": ["id", "created_at", "updated_at"]}, config={"enabled": True, "required_columns": ["id", "created_at", "updated_at"]},
@@ -1159,7 +1159,7 @@ def check_mysql_ddl_031(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-032", code="MYSQL-DDL-032",
name="DDLMaxKeyParts", name="DDLMaxKeyParts",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="单个索引最多允许包含的字段数", description="单个索引最多允许包含的字段数",
config={"enabled": True, "max_parts": 8}, config={"enabled": True, "max_parts": 8},
@@ -1194,7 +1194,7 @@ def check_mysql_ddl_032(sql_clean: str, sql_upper: str, report: AuditReportV2, r
severity=Severity.WARNING, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="单个表最多允许创建的索引数量", description="单个表最多允许创建的索引数量",
config={"enabled": True, "max_index": 12}, config={"enabled": True, "max_index": 20},
message_template="表 '{table}' 包含 {count} 个索引,超过上限 {max_idx}", message_template="表 '{table}' 包含 {count} 个索引,超过上限 {max_idx}",
)) ))
def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None: def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
@@ -1219,7 +1219,7 @@ def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DDL-034", code="MYSQL-DDL-034",
name="MaxDDLAffectRows", name="MaxDDLAffectRows",
category=RuleCategory.DDL, category=RuleCategory.DDL,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="DDL操作影响的最大行数限制(0表示不限制)", description="DDL操作影响的最大行数限制(0表示不限制)",
config={"enabled": False, "max_rows": 0}, config={"enabled": False, "max_rows": 0},
@@ -1250,7 +1250,7 @@ def check_mysql_ddl_034(sql_clean: str, sql_upper: str, report: AuditReportV2, r
severity=Severity.WARNING, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="CHAR 类型字段的最大长度限制", description="CHAR 类型字段的最大长度限制",
config={"enabled": True, "max_length": 10}, config={"enabled": True, "max_length": 255},
message_template="表 '{table}' 的字段 '{column}' CHAR({length}) 超过最大长度限制 {max_len}", message_template="表 '{table}' 的字段 '{column}' CHAR({length}) 超过最大长度限制 {max_len}",
)) ))
def check_mysql_ddl_035(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None: def check_mysql_ddl_035(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
@@ -1339,7 +1339,7 @@ def check_mysql_ddl_036(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DML-001", code="MYSQL-DML-001",
name="DMLMaxInsertRows", name="DMLMaxInsertRows",
category=RuleCategory.DML, category=RuleCategory.DML,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="INSERT 语句单次插入的最大行数限制", description="INSERT 语句单次插入的最大行数限制",
config={"enabled": True, "max_rows": 500}, config={"enabled": True, "max_rows": 500},
@@ -1564,7 +1564,7 @@ def check_mysql_dml_008(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-DML-009", code="MYSQL-DML-009",
name="MaxAffectRows", name="MaxAffectRows",
category=RuleCategory.DML, category=RuleCategory.DML,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="DML 操作影响的最大行数限制(0表示不限制)", description="DML 操作影响的最大行数限制(0表示不限制)",
config={"enabled": False, "max_rows": 0}, config={"enabled": False, "max_rows": 0},
@@ -1680,7 +1680,7 @@ def check_mysql_osc_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-NAM-001", code="MYSQL-NAM-001",
name="TableNamingCase", name="TableNamingCase",
category=RuleCategory.NAMING, category=RuleCategory.NAMING,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="表名必须使用小写下划线命名法(snake_case)", description="表名必须使用小写下划线命名法(snake_case)",
config={"enabled": True}, config={"enabled": True},
@@ -1705,7 +1705,7 @@ def check_mysql_nam_001(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-NAM-002", code="MYSQL-NAM-002",
name="ColumnNamingCase", name="ColumnNamingCase",
category=RuleCategory.NAMING, category=RuleCategory.NAMING,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="字段名必须使用小写下划线命名法(snake_case)", description="字段名必须使用小写下划线命名法(snake_case)",
config={"enabled": True}, config={"enabled": True},
@@ -1759,7 +1759,7 @@ def check_mysql_nam_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
code="MYSQL-NAM-004", code="MYSQL-NAM-004",
name="TableNamingNoDigitPrefix", name="TableNamingNoDigitPrefix",
category=RuleCategory.NAMING, category=RuleCategory.NAMING,
severity=Severity.ERROR, severity=Severity.WARNING,
db_type="mysql", db_type="mysql",
description="表名不能以数字开头", description="表名不能以数字开头",
config={"enabled": True}, config={"enabled": True},
+114 -23
View File
@@ -30,7 +30,7 @@ from app.services.sql_analyzer import SQLAnalyzer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# ===== 配置常量 ===== # ===== 配置常量(默认值,优先从 system_settings 读取) =====
# 默认最大返回行数 # 默认最大返回行数
MAX_ROWS = 10000 MAX_ROWS = 10000
@@ -41,6 +41,88 @@ MAX_QUERY_LENGTH = 10000
# 最大结果集内存限制(MB) # 最大结果集内存限制(MB)
MAX_RESULT_MEMORY_MB = 50 MAX_RESULT_MEMORY_MB = 50
async def _get_setting(key: str, default: int) -> int:
"""从 system_settings 读取配置,失败则返回默认值。"""
try:
from app.core.database import async_session
from app.models.system_setting import SystemSetting
from sqlalchemy import select
async with async_session() as db:
result = await db.execute(
select(SystemSetting.value).where(SystemSetting.key == key)
)
val = result.scalar_one_or_none()
return int(val) if val is not None else default
except Exception:
return default
async def get_query_config() -> Dict[str, int]:
"""批量获取查询相关配置,减少 DB 查询次数。"""
try:
from app.core.database import async_session
from app.models.system_setting import SystemSetting
from sqlalchemy import select
async with async_session() as db:
keys = [
"query_max_limit", "query_timeout", "query_max_length",
"query_result_max_memory_mb",
"executor_sql_timeout", "executor_redis_timeout", "executor_mongo_timeout",
]
stmt = select(SystemSetting.key, SystemSetting.value).where(
SystemSetting.key.in_(keys)
)
result = await db.execute(stmt)
rows = {r[0]: r[1] for r in result.all()}
return {
"max_rows": int(rows.get("query_max_limit", MAX_ROWS)),
"query_timeout": int(rows.get("query_timeout", QUERY_TIMEOUT)),
"max_query_length": int(rows.get("query_max_length", MAX_QUERY_LENGTH)),
"max_memory_mb": int(rows.get("query_result_max_memory_mb", MAX_RESULT_MEMORY_MB)),
"sql_timeout": int(rows.get("executor_sql_timeout", 60)),
"redis_timeout": int(rows.get("executor_redis_timeout", 30)),
"mongo_timeout": int(rows.get("executor_mongo_timeout", 60)),
}
except Exception:
return {
"max_rows": MAX_ROWS,
"query_timeout": QUERY_TIMEOUT,
"max_query_length": MAX_QUERY_LENGTH,
"max_memory_mb": MAX_RESULT_MEMORY_MB,
"sql_timeout": 60,
"redis_timeout": 30,
"mongo_timeout": 60,
}
# ===== 动态配置缓存 =====
_active_config: Optional[Dict[str, int]] = None
async def refresh_config() -> Dict[str, int]:
"""刷新配置缓存并返回。启动时和 settings 更新后调用。"""
global _active_config
_active_config = await get_query_config()
return _active_config
def current_config() -> Dict[str, int]:
"""获取当前配置(同步),未初始化则返回默认值。"""
if _active_config is not None:
return _active_config
return {
"max_rows": MAX_ROWS,
"query_timeout": QUERY_TIMEOUT,
"max_query_length": MAX_QUERY_LENGTH,
"max_memory_mb": MAX_RESULT_MEMORY_MB,
"sql_timeout": 60,
"redis_timeout": 30,
"mongo_timeout": 60,
}
# ===== Redis 只读命令白名单 ===== # ===== Redis 只读命令白名单 =====
REDIS_READONLY_COMMANDS = { REDIS_READONLY_COMMANDS = {
@@ -294,9 +376,11 @@ class BaseQueryHandler(ABC):
""" """
if not query or not query.strip(): if not query or not query.strip():
raise QuerySecurityError("查询语句不能为空") raise QuerySecurityError("查询语句不能为空")
if len(query) > MAX_QUERY_LENGTH: cfg = current_config()
if len(query) > cfg["max_query_length"]:
max_len = cfg["max_query_length"]
raise QuerySecurityError( raise QuerySecurityError(
f"查询语句过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH} 字符" f"查询语句过长({len(query)} 字符),最大允许 {max_len} 字符"
) )
def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool: def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool:
@@ -314,7 +398,7 @@ class BaseQueryHandler(ABC):
try: try:
size_bytes = len(json.dumps(rows, default=str).encode("utf-8")) size_bytes = len(json.dumps(rows, default=str).encode("utf-8"))
size_mb = size_bytes / (1024 * 1024) size_mb = size_bytes / (1024 * 1024)
return size_mb > MAX_RESULT_MEMORY_MB return size_mb > current_config()["max_memory_mb"]
except (OverflowError, MemoryError): except (OverflowError, MemoryError):
return True return True
@@ -418,16 +502,18 @@ class SQLQueryHandlerBase(BaseQueryHandler):
# 不需要添加 LIMIT 的语句前缀 # 不需要添加 LIMIT 的语句前缀
_NO_LIMIT_PREFIXES = ("SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH") _NO_LIMIT_PREFIXES = ("SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH")
def _auto_add_limit(self, sql: str, max_rows: int = MAX_ROWS) -> str: def _auto_add_limit(self, sql: str, max_rows: int = -1) -> str:
"""如果 SQL 没有 LIMIT 则自动添加。仅对 SELECT 语句生效。 """如果 SQL 没有 LIMIT 则自动添加。仅对 SELECT 语句生效。
Args: Args:
sql: 原始 SQL sql: 原始 SQL
max_rows: 最大返回行数 max_rows: 最大返回行数(-1 表示从配置读取)
Returns: Returns:
添加了 LIMIT 的 SQL 添加了 LIMIT 的 SQL
""" """
if max_rows < 0:
max_rows = current_config()["max_rows"]
upper_sql = sql.strip().upper() upper_sql = sql.strip().upper()
# SHOW / DESC / EXPLAIN 等语句不支持 LIMIT(或语义不同),跳过 # SHOW / DESC / EXPLAIN 等语句不支持 LIMIT(或语义不同),跳过
if any(upper_sql.startswith(p) for p in self._NO_LIMIT_PREFIXES): if any(upper_sql.startswith(p) for p in self._NO_LIMIT_PREFIXES):
@@ -492,9 +578,9 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
QueryTimeoutError: 查询超时 QueryTimeoutError: 查询超时
""" """
options = options or {} options = options or {}
custom_timeout = options.get("timeout", QUERY_TIMEOUT) custom_timeout = options.get("timeout", current_config()["query_timeout"])
custom_max_rows = options.get("limit", MAX_ROWS) custom_max_rows = options.get("limit", current_config()["max_rows"])
custom_max_rows = min(custom_max_rows, MAX_ROWS) # 不允许超过全局上限 custom_max_rows = min(custom_max_rows, current_config()["max_rows"]) # 不允许超过全局上限
# 1. 增强安全验证 # 1. 增强安全验证
self._enhanced_security_check(query, user_id) self._enhanced_security_check(query, user_id)
@@ -538,7 +624,7 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
truncated = True truncated = True
# 截断结果以保护内存 # 截断结果以保护内存
rows = rows[:custom_max_rows] if rows else [] rows = rows[:custom_max_rows] if rows else []
logger.warning("查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB) logger.warning("查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
# 6. 记录查询日志 # 6. 记录查询日志
QueryLogger.log_query( QueryLogger.log_query(
@@ -595,9 +681,9 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
QueryTimeoutError: 查询超时 QueryTimeoutError: 查询超时
""" """
options = options or {} options = options or {}
custom_timeout = options.get("timeout", QUERY_TIMEOUT) custom_timeout = options.get("timeout", current_config()["query_timeout"])
custom_max_rows = options.get("limit", MAX_ROWS) custom_max_rows = options.get("limit", current_config()["max_rows"])
custom_max_rows = min(custom_max_rows, MAX_ROWS) custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
# 1. 增强安全验证 # 1. 增强安全验证
self._enhanced_security_check(query, user_id) self._enhanced_security_check(query, user_id)
@@ -640,7 +726,7 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
if self._check_result_size(rows if rows else []): if self._check_result_size(rows if rows else []):
truncated = True truncated = True
rows = rows[:custom_max_rows] if rows else [] rows = rows[:custom_max_rows] if rows else []
logger.warning("查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB) logger.warning("查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
# 6. 记录查询日志 # 6. 记录查询日志
QueryLogger.log_query( QueryLogger.log_query(
@@ -694,8 +780,9 @@ class RedisQueryHandler(BaseQueryHandler):
return False, "Redis 命令不能为空" return False, "Redis 命令不能为空"
# 长度验证 # 长度验证
if len(query) > MAX_QUERY_LENGTH: if len(query) > current_config()["max_query_length"]:
return False, f"Redis 命令过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH}" max_len = current_config()["max_query_length"]
return False, f"Redis 命令过长({len(query)} 字符),最大允许 {max_len}"
# 检测 shell 注入(管道符、重定向、命令替换) # 检测 shell 注入(管道符、重定向、命令替换)
shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"] shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
@@ -759,7 +846,7 @@ class RedisQueryHandler(BaseQueryHandler):
QueryTimeoutError: 命令超时 QueryTimeoutError: 命令超时
""" """
options = options or {} options = options or {}
custom_timeout = options.get("timeout", QUERY_TIMEOUT) custom_timeout = options.get("timeout", current_config()["query_timeout"])
# 1. 验证只读 # 1. 验证只读
is_safe, error_msg = self.validate_readonly(query) is_safe, error_msg = self.validate_readonly(query)
@@ -918,8 +1005,9 @@ class MongoQueryHandler(BaseQueryHandler):
return False, "MongoDB 查询不能为空" return False, "MongoDB 查询不能为空"
# 长度验证 # 长度验证
if len(query) > MAX_QUERY_LENGTH: if len(query) > current_config()["max_query_length"]:
return False, f"MongoDB 查询过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH}" max_len = current_config()["max_query_length"]
return False, f"MongoDB 查询过长({len(query)} 字符),最大允许 {max_len}"
try: try:
query_obj = json.loads(query) query_obj = json.loads(query)
@@ -990,9 +1078,9 @@ class MongoQueryHandler(BaseQueryHandler):
QueryTimeoutError: 查询超时 QueryTimeoutError: 查询超时
""" """
options = options or {} options = options or {}
custom_timeout = options.get("timeout", QUERY_TIMEOUT) custom_timeout = options.get("timeout", current_config()["query_timeout"])
custom_max_rows = options.get("limit", MAX_ROWS) custom_max_rows = options.get("limit", current_config()["max_rows"])
custom_max_rows = min(custom_max_rows, MAX_ROWS) custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
# 1. 验证只读 # 1. 验证只读
is_safe, error_msg = self.validate_readonly(query) is_safe, error_msg = self.validate_readonly(query)
@@ -1156,7 +1244,7 @@ class MongoQueryHandler(BaseQueryHandler):
if self._check_result_size(rows): if self._check_result_size(rows):
truncated = True truncated = True
rows = rows[:custom_max_rows] rows = rows[:custom_max_rows]
logger.warning("MongoDB 查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB) logger.warning("MongoDB 查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
# 5. 记录查询日志 # 5. 记录查询日志
QueryLogger.log_query( QueryLogger.log_query(
@@ -1236,6 +1324,9 @@ class QueryService:
if handler is None: if handler is None:
raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询") raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询")
# 每次查询刷新配置缓存(轻量:仅查一次 DB)
await refresh_config()
return await handler.execute(instance, database, query, user_id, options) return await handler.execute(instance, database, query, user_id, options)
@staticmethod @staticmethod
+43
View File
@@ -188,6 +188,48 @@ SEED_SETTINGS = [
"default_value": "30", "default_value": "30",
"sort_order": 35, "sort_order": 35,
}, },
# ── 执行器 ──
{
"key": "executor_sql_timeout",
"value": "60",
"value_type": "integer",
"group": "executor",
"label": "SQL 执行超时(秒)",
"description": "MySQL/PostgreSQL 工单执行单条 SQL 最大时间。范围: 10~600",
"default_value": "60",
"sort_order": 40,
},
{
"key": "executor_redis_timeout",
"value": "30",
"value_type": "integer",
"group": "executor",
"label": "Redis 执行超时(秒)",
"description": "Redis 工单执行单条命令最大时间。范围: 5~120",
"default_value": "30",
"sort_order": 41,
},
{
"key": "executor_mongo_timeout",
"value": "60",
"value_type": "integer",
"group": "executor",
"label": "MongoDB 执行超时(秒)",
"description": "MongoDB 工单执行单条命令最大时间。范围: 10~600",
"default_value": "60",
"sort_order": 42,
},
{
"key": "query_result_max_memory_mb",
"value": "50",
"value_type": "integer",
"group": "query",
"label": "查询结果内存上限(MB)",
"description": "单次查询结果集最大内存占用,超限自动截断。范围: 10~500",
"default_value": "50",
"sort_order": 8,
},
] ]
# 分组元信息 # 分组元信息
@@ -196,4 +238,5 @@ SETTING_GROUPS = {
"security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"}, "security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"},
"ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"}, "ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"},
"cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"}, "cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"},
"executor": {"label": "执行器", "icon": "⚙️", "description": "SQL/Redis 执行超时、批量限制、连接参数"},
} }
+17 -5
View File
@@ -40,11 +40,19 @@ from app.services.sql_analyzer import SQLAnalyzer, SQLType
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 超时配置(秒) # 超时配置(秒)—— 默认值,优先从 system_settings 读取
SQL_TIMEOUT = 60 SQL_TIMEOUT = 60
REDIS_TIMEOUT = 30 REDIS_TIMEOUT = 30
MONGO_TIMEOUT = 60 MONGO_TIMEOUT = 60
def _get_executor_timeout(db_type: str) -> int:
"""从 query_service 配置缓存获取执行超时。"""
from app.services.query_service import current_config
cfg = current_config()
key = {"mysql": "sql_timeout", "postgres": "sql_timeout", "redis": "redis_timeout", "mongodb": "mongo_timeout"}.get(db_type, "sql_timeout")
return cfg.get(key, SQL_TIMEOUT)
# 并发执行锁: 防止同一工单被并发执行 # 并发执行锁: 防止同一工单被并发执行
_ticket_execution_locks: Set[int] = set() _ticket_execution_locks: Set[int] = set()
_lock_lock = asyncio.Lock() _lock_lock = asyncio.Lock()
@@ -238,7 +246,8 @@ class TicketExecutor:
# 获取超时配置(从 ticket 或选项中) # 获取超时配置(从 ticket 或选项中)
custom_timeout = getattr(ticket, 'options', None) or {} custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT _default_timeout = _get_executor_timeout('mysql')
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
try: try:
# 1. 语法预校验 # 1. 语法预校验
@@ -370,7 +379,8 @@ class TicketExecutor:
# 获取超时配置 # 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {} custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT _default_timeout = _get_executor_timeout('postgres')
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
try: try:
# 1. 语法预校验 # 1. 语法预校验
@@ -516,7 +526,8 @@ class TicketExecutor:
# 获取超时配置 # 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {} custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', REDIS_TIMEOUT) if isinstance(custom_timeout, dict) else REDIS_TIMEOUT _default_timeout = _get_executor_timeout('redis')
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
try: try:
# 解析命令 # 解析命令
@@ -686,7 +697,8 @@ class TicketExecutor:
# 获取超时配置 # 获取超时配置
custom_timeout = getattr(ticket, 'options', None) or {} custom_timeout = getattr(ticket, 'options', None) or {}
timeout = custom_timeout.get('timeout', MONGO_TIMEOUT) if isinstance(custom_timeout, dict) else MONGO_TIMEOUT _default_timeout = _get_executor_timeout('mongodb')
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
try: try:
# 解析工单内容 # 解析工单内容