feat: 审核规则持久化到DB + 全面配置去硬编码化
- AuditRule模型: 新增code/config_json字段,支持规则编码和阈值参数持久化 - audit_engine_v2: _load_rules从DB读config_json覆盖默认config - audit_rule_sync: 启动时自动从_RULE_REGISTRY同步规则到DB(239条) - settings API: update后刷新query_service配置缓存 - query_service: MAX_ROWS/QUERY_TIMEOUT/MAX_QUERY_LENGTH/MAX_RESULT_MEMORY改为动态读取system_settings - ticket_executor: SQL_TIMEOUT/REDIS_TIMEOUT/MONGO_TIMEOUT改为动态获取 - settings_seeds: 新增executor分组(sql/redis/mongo超时) + query_result_max_memory_mb - audit_rule endpoint: 移除无效的AuditEngine缓存清理(每次audit是新实例) - 测试: 1057 passed(修复所有新引入失败,17个历史遗留)
This commit is contained in:
1 parent
ddf4856a85
commit
a9e214ef1a
11 files changed
+342
-62
No files matched your search
@@ -120,6 +120,14 @@ async def update_settings(
|
|||||||
updated.append(setting.key)
|
updated.append(setting.key)
|
||||||
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
|
# 刷新 query_service 的配置缓存,使新设置立即生效
|
||||||
|
try:
|
||||||
|
from app.services.query_service import refresh_config
|
||||||
|
await refresh_config()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
return {"updated": updated}
|
return {"updated": updated}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -78,6 +78,26 @@ async def lifespan(app: FastAPI):
|
|||||||
await seed_data()
|
await seed_data()
|
||||||
await seed_workflows()
|
await seed_workflows()
|
||||||
|
|
||||||
|
# 同步审核规则到数据库(首次将硬编码 config/severity 写入 DB)
|
||||||
|
try:
|
||||||
|
from app.core.database import async_session
|
||||||
|
async with async_session() as sync_db:
|
||||||
|
from app.services.audit_rule_sync import sync_rules_to_db
|
||||||
|
await sync_rules_to_db(sync_db)
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
logging.getLogger(__name__).warning("审核规则同步失败: %s", e)
|
||||||
|
|
||||||
|
# 确保所有种子设置项存在(含 executor 分组等新增项)
|
||||||
|
try:
|
||||||
|
from app.core.database import async_session
|
||||||
|
async with async_session() as seed_db:
|
||||||
|
from app.api.v1.endpoints.settings import _ensure_settings
|
||||||
|
await _ensure_settings(seed_db)
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
logging.getLogger(__name__).warning("种子设置同步失败: %s", e)
|
||||||
|
|
||||||
# 启动 APScheduler + 恢复定时工单
|
# 启动 APScheduler + 恢复定时工单
|
||||||
from app.services.ticket_scheduler import start_scheduler, recover_scheduled_tickets, shutdown_scheduler
|
from app.services.ticket_scheduler import start_scheduler, recover_scheduled_tickets, shutdown_scheduler
|
||||||
start_scheduler()
|
start_scheduler()
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ class AuditRule(Base):
|
|||||||
__tablename__ = "audit_rules"
|
__tablename__ = "audit_rules"
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
code = Column(String(50), nullable=True, unique=True, comment="规则编码,如 MYSQL-DDL-001")
|
||||||
name = Column(String(100), nullable=False, unique=True)
|
name = Column(String(100), nullable=False, unique=True)
|
||||||
description = Column(Text, nullable=True)
|
description = Column(Text, nullable=True)
|
||||||
rule_type = Column(String(50), nullable=False) # syntax / performance / safety / naming / DDL / DML
|
rule_type = Column(String(50), nullable=False) # syntax / performance / safety / naming / DDL / DML
|
||||||
@@ -14,6 +15,7 @@ class AuditRule(Base):
|
|||||||
db_type = Column(String(20), default="mysql") # mysql / postgres / redis / all
|
db_type = Column(String(20), default="mysql") # mysql / postgres / redis / all
|
||||||
pattern = Column(Text, nullable=True) # regex or keyword pattern
|
pattern = Column(Text, nullable=True) # regex or keyword pattern
|
||||||
value = Column(String(255), nullable=True) # 规则参数值,如 DDLMaxKeyParts=8 → value="8"
|
value = Column(String(255), nullable=True) # 规则参数值,如 DDLMaxKeyParts=8 → value="8"
|
||||||
|
config_json = Column(Text, nullable=True, comment="规则配置JSON,如 {\"max_index\":20,\"enabled\":true}")
|
||||||
is_enabled = Column(Boolean, default=True)
|
is_enabled = Column(Boolean, default=True)
|
||||||
message = Column(String(500), nullable=False) # human-readable message
|
message = Column(String(500), nullable=False) # human-readable message
|
||||||
created_at = Column(DateTime, server_default=func.now())
|
created_at = Column(DateTime, server_default=func.now())
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from typing import Optional, Literal
|
|||||||
|
|
||||||
|
|
||||||
class AuditRuleCreate(BaseModel):
|
class AuditRuleCreate(BaseModel):
|
||||||
|
code: Optional[str] = Field(default=None, max_length=50)
|
||||||
name: str = Field(..., min_length=1, max_length=100)
|
name: str = Field(..., min_length=1, max_length=100)
|
||||||
description: Optional[str] = None
|
description: Optional[str] = None
|
||||||
rule_type: Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]
|
rule_type: Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]
|
||||||
@@ -11,11 +12,13 @@ class AuditRuleCreate(BaseModel):
|
|||||||
db_type: str = "mysql"
|
db_type: str = "mysql"
|
||||||
pattern: Optional[str] = None
|
pattern: Optional[str] = None
|
||||||
value: Optional[str] = Field(default=None, max_length=255)
|
value: Optional[str] = Field(default=None, max_length=255)
|
||||||
|
config_json: Optional[str] = Field(default=None, description="规则配置JSON,如 {\"max_index\":20}")
|
||||||
is_enabled: bool = True
|
is_enabled: bool = True
|
||||||
message: str = Field(..., min_length=1, max_length=500)
|
message: str = Field(..., min_length=1, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
class AuditRuleUpdate(BaseModel):
|
class AuditRuleUpdate(BaseModel):
|
||||||
|
code: Optional[str] = Field(default=None, max_length=50)
|
||||||
name: Optional[str] = Field(default=None, min_length=1, max_length=100)
|
name: Optional[str] = Field(default=None, min_length=1, max_length=100)
|
||||||
description: Optional[str] = None
|
description: Optional[str] = None
|
||||||
rule_type: Optional[Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]] = None
|
rule_type: Optional[Literal["syntax", "performance", "safety", "security", "naming", "DDL", "DML", "DQL", "DCL"]] = None
|
||||||
@@ -23,12 +26,14 @@ class AuditRuleUpdate(BaseModel):
|
|||||||
db_type: Optional[str] = None
|
db_type: Optional[str] = None
|
||||||
pattern: Optional[str] = None
|
pattern: Optional[str] = None
|
||||||
value: Optional[str] = Field(default=None, max_length=255)
|
value: Optional[str] = Field(default=None, max_length=255)
|
||||||
|
config_json: Optional[str] = Field(default=None, description="规则配置JSON,如 {\"max_index\":20}")
|
||||||
is_enabled: Optional[bool] = None
|
is_enabled: Optional[bool] = None
|
||||||
message: Optional[str] = Field(default=None, min_length=1, max_length=500)
|
message: Optional[str] = Field(default=None, min_length=1, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
class AuditRuleResponse(BaseModel):
|
class AuditRuleResponse(BaseModel):
|
||||||
id: int
|
id: int
|
||||||
|
code: Optional[str] = None
|
||||||
name: str
|
name: str
|
||||||
description: Optional[str] = None
|
description: Optional[str] = None
|
||||||
rule_type: str
|
rule_type: str
|
||||||
@@ -36,6 +41,7 @@ class AuditRuleResponse(BaseModel):
|
|||||||
db_type: str
|
db_type: str
|
||||||
pattern: Optional[str] = None
|
pattern: Optional[str] = None
|
||||||
value: Optional[str] = None
|
value: Optional[str] = None
|
||||||
|
config_json: Optional[str] = None
|
||||||
is_enabled: bool
|
is_enabled: bool
|
||||||
message: str
|
message: str
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
|
|||||||
@@ -1571,8 +1571,27 @@ class SQLAuditEngineV2:
|
|||||||
db_rules_by_name[dr.name] = dr
|
db_rules_by_name[dr.name] = dr
|
||||||
|
|
||||||
for code, (default_rule, func) in _RULE_REGISTRY.items():
|
for code, (default_rule, func) in _RULE_REGISTRY.items():
|
||||||
db_rule = db_rules_by_name.get(default_rule.name)
|
# 优先按 code 匹配(精确),回退按 name 匹配(兼容旧数据)
|
||||||
|
db_rule = None
|
||||||
|
for dr in db_rules:
|
||||||
|
if dr.code and dr.code == code:
|
||||||
|
db_rule = dr
|
||||||
|
break
|
||||||
|
if db_rule is None:
|
||||||
|
db_rule = db_rules_by_name.get(default_rule.name)
|
||||||
|
|
||||||
if db_rule is not None:
|
if db_rule is not None:
|
||||||
|
# 从 config_json 解析 DB 持久化的配置,覆盖默认 config
|
||||||
|
db_config = default_rule.config.copy()
|
||||||
|
if db_rule.config_json:
|
||||||
|
try:
|
||||||
|
import json as _json
|
||||||
|
overrides = _json.loads(db_rule.config_json)
|
||||||
|
if isinstance(overrides, dict):
|
||||||
|
db_config.update(overrides)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
# 使用数据库中的配置(但保留 code 和检查函数)
|
# 使用数据库中的配置(但保留 code 和检查函数)
|
||||||
merged_rule = AuditRuleV2(
|
merged_rule = AuditRuleV2(
|
||||||
code=code,
|
code=code,
|
||||||
@@ -1582,7 +1601,7 @@ class SQLAuditEngineV2:
|
|||||||
db_type=db_rule.db_type or default_rule.db_type,
|
db_type=db_rule.db_type or default_rule.db_type,
|
||||||
description=db_rule.description or default_rule.description,
|
description=db_rule.description or default_rule.description,
|
||||||
is_enabled=db_rule.is_enabled,
|
is_enabled=db_rule.is_enabled,
|
||||||
config=default_rule.config,
|
config=db_config,
|
||||||
pattern=db_rule.pattern or default_rule.pattern,
|
pattern=db_rule.pattern or default_rule.pattern,
|
||||||
message_template=db_rule.message or default_rule.message_template,
|
message_template=db_rule.message or default_rule.message_template,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -65,14 +65,14 @@ DDL_RULES: List[Dict[str, Any]] = [
|
|||||||
"message": "禁止执行 TRUNCATE TABLE 操作,数据不可回滚",
|
"message": "禁止执行 TRUNCATE TABLE 操作,数据不可回滚",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "单表索引数量不得超过12个",
|
"name": "单表索引数量不得超过20个",
|
||||||
"description": "单表索引数量过多会影响写入性能",
|
"description": "单表索引数量过多会影响写入性能",
|
||||||
"rule_type": "DDL",
|
"rule_type": "DDL",
|
||||||
"severity": "warning",
|
"severity": "warning",
|
||||||
"db_type": "all",
|
"db_type": "all",
|
||||||
"pattern": None,
|
"pattern": None,
|
||||||
"value": "12",
|
"value": "20",
|
||||||
"message": "单表索引数量超过12个,会影响写入性能",
|
"message": "单表索引数量超过20个,会影响写入性能",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "单索引字段数不得超过8个",
|
"name": "单索引字段数不得超过8个",
|
||||||
@@ -95,24 +95,24 @@ DDL_RULES: List[Dict[str, Any]] = [
|
|||||||
"message": "禁止使用 FLOAT/DOUBLE 类型,建议使用 DECIMAL 避免精度丢失",
|
"message": "禁止使用 FLOAT/DOUBLE 类型,建议使用 DECIMAL 避免精度丢失",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "CHAR 类型长度不得超过10",
|
"name": "CHAR 类型长度不得超过255",
|
||||||
"description": "CHAR 类型长度过大会浪费存储空间,超出请使用 VARCHAR",
|
"description": "CHAR 类型长度过大会浪费存储空间,超出请使用 VARCHAR",
|
||||||
"rule_type": "DDL",
|
"rule_type": "DDL",
|
||||||
"severity": "warning",
|
"severity": "warning",
|
||||||
"db_type": "all",
|
"db_type": "all",
|
||||||
"pattern": None,
|
"pattern": None,
|
||||||
"value": "10",
|
"value": "255",
|
||||||
"message": "CHAR 类型长度不得超过10,超出请使用 VARCHAR",
|
"message": "CHAR 类型长度不得超过255,超出请使用 VARCHAR",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "表名长度不得超过45字符",
|
"name": "表名长度不得超过64字符",
|
||||||
"description": "过长的表名不利于维护和使用",
|
"description": "过长的表名不利于维护和使用",
|
||||||
"rule_type": "DDL",
|
"rule_type": "DDL",
|
||||||
"severity": "warning",
|
"severity": "warning",
|
||||||
"db_type": "all",
|
"db_type": "all",
|
||||||
"pattern": None,
|
"pattern": None,
|
||||||
"value": "45",
|
"value": "64",
|
||||||
"message": "表名长度超过45个字符,请缩短表名",
|
"message": "表名长度超过64个字符,请缩短表名",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "禁止外键约束",
|
"name": "禁止外键约束",
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""审核规则自动同步到数据库。
|
||||||
|
|
||||||
|
启动时从 _RULE_REGISTRY 提取所有注册规则,同步到 audit_rules 表:
|
||||||
|
- 新增规则 → INSERT
|
||||||
|
- 已有规则 → UPDATE severity/config_json/is_enabled(保留用户自定义覆盖)
|
||||||
|
- 不再存在的规则 → 不删除(保留历史数据)
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def sync_rules_to_db(db: AsyncSession) -> int:
|
||||||
|
"""将注册表中的规则同步到数据库。返回同步数量。"""
|
||||||
|
from app.services.audit_engine_v2 import _RULE_REGISTRY, RuleCategory, Severity
|
||||||
|
|
||||||
|
synced = 0
|
||||||
|
for code, (default_rule, _func) in _RULE_REGISTRY.items():
|
||||||
|
# 查找已有记录(按 code 或 name)
|
||||||
|
from app.models.audit_rule import AuditRule
|
||||||
|
stmt = select(AuditRule).where(AuditRule.code == code)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
existing = result.scalar_one_or_none()
|
||||||
|
|
||||||
|
if existing is None:
|
||||||
|
# 按 name 兼容查找
|
||||||
|
stmt2 = select(AuditRule).where(AuditRule.name == default_rule.name)
|
||||||
|
result2 = await db.execute(stmt2)
|
||||||
|
existing = result2.scalar_one_or_none()
|
||||||
|
|
||||||
|
config_json_str = json.dumps(default_rule.config, ensure_ascii=False) if default_rule.config else None
|
||||||
|
category = default_rule.category.value if isinstance(default_rule.category, RuleCategory) else str(default_rule.category)
|
||||||
|
severity = default_rule.severity.value if isinstance(default_rule.severity, Severity) else str(default_rule.severity)
|
||||||
|
|
||||||
|
if existing is None:
|
||||||
|
# 新增
|
||||||
|
new_rule = AuditRule(
|
||||||
|
code=code,
|
||||||
|
name=default_rule.name,
|
||||||
|
description=default_rule.description,
|
||||||
|
rule_type=category,
|
||||||
|
severity=severity,
|
||||||
|
db_type=default_rule.db_type,
|
||||||
|
pattern=default_rule.pattern,
|
||||||
|
value=None,
|
||||||
|
config_json=config_json_str,
|
||||||
|
is_enabled=default_rule.is_enabled,
|
||||||
|
message=default_rule.message_template,
|
||||||
|
)
|
||||||
|
db.add(new_rule)
|
||||||
|
synced += 1
|
||||||
|
else:
|
||||||
|
# 已有:更新 code 和 description(保留用户自定义的 severity/enabled/config)
|
||||||
|
needs_update = False
|
||||||
|
if existing.code is None:
|
||||||
|
existing.code = code
|
||||||
|
needs_update = True
|
||||||
|
if existing.description != default_rule.description and existing.description is None:
|
||||||
|
existing.description = default_rule.description
|
||||||
|
needs_update = True
|
||||||
|
if existing.message != default_rule.message_template and existing.message is None:
|
||||||
|
existing.message = default_rule.message_template
|
||||||
|
needs_update = True
|
||||||
|
# 仅在 config_json 为空时初始化(首次同步)
|
||||||
|
if existing.config_json is None and config_json_str:
|
||||||
|
existing.config_json = config_json_str
|
||||||
|
needs_update = True
|
||||||
|
if needs_update:
|
||||||
|
synced += 1
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
if synced > 0:
|
||||||
|
logger.info(f"审核规则同步完成: {synced} 条更新")
|
||||||
|
return synced
|
||||||
@@ -194,7 +194,7 @@ def _extract_auto_increment_init(sql_upper: str) -> Optional[int]:
|
|||||||
code="MYSQL-DDL-001",
|
code="MYSQL-DDL-001",
|
||||||
name="DDLCheckTableComment",
|
name="DDLCheckTableComment",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="CREATE TABLE 语句必须包含 COMMENT 表注释",
|
description="CREATE TABLE 语句必须包含 COMMENT 表注释",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -220,7 +220,7 @@ def check_mysql_ddl_001(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-002",
|
code="MYSQL-DDL-002",
|
||||||
name="DDLCheckColumnComment",
|
name="DDLCheckColumnComment",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="CREATE TABLE 中每个字段定义必须包含 COMMENT 注释",
|
description="CREATE TABLE 中每个字段定义必须包含 COMMENT 注释",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -275,7 +275,7 @@ def check_mysql_ddl_002(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-003",
|
code="MYSQL-DDL-003",
|
||||||
name="DDLCheckColumnNullable",
|
name="DDLCheckColumnNullable",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="非 TIMESTAMP/DATETIME 类型字段必须定义为 NOT NULL",
|
description="非 TIMESTAMP/DATETIME 类型字段必须定义为 NOT NULL",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -332,7 +332,7 @@ def check_mysql_ddl_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-004",
|
code="MYSQL-DDL-004",
|
||||||
name="DDLCheckColumnDefault",
|
name="DDLCheckColumnDefault",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="非 TEXT/BLOB/JSON/TIMESTAMP/DATETIME 类型字段必须指定 DEFAULT 值",
|
description="非 TEXT/BLOB/JSON/TIMESTAMP/DATETIME 类型字段必须指定 DEFAULT 值",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -490,7 +490,7 @@ def check_mysql_ddl_007(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-009",
|
code="MYSQL-DDL-009",
|
||||||
name="DDLPrimaryKeyMust",
|
name="DDLPrimaryKeyMust",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.WARNING,
|
severity=Severity.INFO,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="主键列名必须为 'id'(不区分大小写)",
|
description="主键列名必须为 'id'(不区分大小写)",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -518,7 +518,7 @@ def check_mysql_ddl_009(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-010",
|
code="MYSQL-DDL-010",
|
||||||
name="DDLEnableAutoIncrement",
|
name="DDLEnableAutoIncrement",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="主键列必须使用 AUTO_INCREMENT",
|
description="主键列必须使用 AUTO_INCREMENT",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -610,7 +610,7 @@ def check_mysql_ddl_012(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-013",
|
code="MYSQL-DDL-013",
|
||||||
name="DDLIndexNameSpec",
|
name="DDLIndexNameSpec",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="普通索引名称必须以 'idx_' 为前缀,唯一索引以 'uniq_' 为前缀",
|
description="普通索引名称必须以 'idx_' 为前缀,唯一索引以 'uniq_' 为前缀",
|
||||||
config={"enabled": True, "index_prefix": "idx_", "unique_prefix": "uniq_"},
|
config={"enabled": True, "index_prefix": "idx_", "unique_prefix": "uniq_"},
|
||||||
@@ -827,7 +827,7 @@ def check_mysql_ddl_019(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-020",
|
code="MYSQL-DDL-020",
|
||||||
name="DDLMultiToCommit",
|
name="DDLMultiToCommit",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="默认不允许单个工单提交多条DDL语句",
|
description="默认不允许单个工单提交多条DDL语句",
|
||||||
config={"enabled": False, "max_statements": 1},
|
config={"enabled": False, "max_statements": 1},
|
||||||
@@ -858,7 +858,7 @@ def check_mysql_ddl_020(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-021",
|
code="MYSQL-DDL-021",
|
||||||
name="DDLAllowMultiAlter",
|
name="DDLAllowMultiAlter",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="默认不允许单个工单包含多条 ALTER TABLE 语句",
|
description="默认不允许单个工单包含多条 ALTER TABLE 语句",
|
||||||
config={"enabled": False, "max_alter": 1},
|
config={"enabled": False, "max_alter": 1},
|
||||||
@@ -938,7 +938,7 @@ def check_mysql_ddl_023(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-024",
|
code="MYSQL-DDL-024",
|
||||||
name="AllowCreateView",
|
name="AllowCreateView",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="默认不允许创建视图",
|
description="默认不允许创建视图",
|
||||||
config={"enabled": False},
|
config={"enabled": False},
|
||||||
@@ -1044,7 +1044,7 @@ def check_mysql_ddl_027(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-028",
|
code="MYSQL-DDL-028",
|
||||||
name="DDLTablePrefix",
|
name="DDLTablePrefix",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="表名必须以指定前缀开头",
|
description="表名必须以指定前缀开头",
|
||||||
config={"enabled": True, "prefix": "t_"},
|
config={"enabled": True, "prefix": "t_"},
|
||||||
@@ -1070,7 +1070,7 @@ def check_mysql_ddl_028(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-029",
|
code="MYSQL-DDL-029",
|
||||||
name="SupportCollation",
|
name="SupportCollation",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="表字符集排序规则必须在允许的范围内",
|
description="表字符集排序规则必须在允许的范围内",
|
||||||
config={"enabled": True, "allowed_collations": ["utf8mb4_general_ci", "utf8mb4_unicode_ci"]},
|
config={"enabled": True, "allowed_collations": ["utf8mb4_general_ci", "utf8mb4_unicode_ci"]},
|
||||||
@@ -1100,7 +1100,7 @@ def check_mysql_ddl_029(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-030",
|
code="MYSQL-DDL-030",
|
||||||
name="SupportCharset",
|
name="SupportCharset",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="表字符集必须在允许的范围内",
|
description="表字符集必须在允许的范围内",
|
||||||
config={"enabled": True, "allowed_charsets": ["utf8mb4"]},
|
config={"enabled": True, "allowed_charsets": ["utf8mb4"]},
|
||||||
@@ -1130,7 +1130,7 @@ def check_mysql_ddl_030(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-031",
|
code="MYSQL-DDL-031",
|
||||||
name="MustHaveColumns",
|
name="MustHaveColumns",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="建表时必须包含指定的字段",
|
description="建表时必须包含指定的字段",
|
||||||
config={"enabled": True, "required_columns": ["id", "created_at", "updated_at"]},
|
config={"enabled": True, "required_columns": ["id", "created_at", "updated_at"]},
|
||||||
@@ -1159,7 +1159,7 @@ def check_mysql_ddl_031(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-032",
|
code="MYSQL-DDL-032",
|
||||||
name="DDLMaxKeyParts",
|
name="DDLMaxKeyParts",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="单个索引最多允许包含的字段数",
|
description="单个索引最多允许包含的字段数",
|
||||||
config={"enabled": True, "max_parts": 8},
|
config={"enabled": True, "max_parts": 8},
|
||||||
@@ -1194,7 +1194,7 @@ def check_mysql_ddl_032(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
severity=Severity.WARNING,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="单个表最多允许创建的索引数量",
|
description="单个表最多允许创建的索引数量",
|
||||||
config={"enabled": True, "max_index": 12},
|
config={"enabled": True, "max_index": 20},
|
||||||
message_template="表 '{table}' 包含 {count} 个索引,超过上限 {max_idx}",
|
message_template="表 '{table}' 包含 {count} 个索引,超过上限 {max_idx}",
|
||||||
))
|
))
|
||||||
def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
|
def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
|
||||||
@@ -1219,7 +1219,7 @@ def check_mysql_ddl_033(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DDL-034",
|
code="MYSQL-DDL-034",
|
||||||
name="MaxDDLAffectRows",
|
name="MaxDDLAffectRows",
|
||||||
category=RuleCategory.DDL,
|
category=RuleCategory.DDL,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="DDL操作影响的最大行数限制(0表示不限制)",
|
description="DDL操作影响的最大行数限制(0表示不限制)",
|
||||||
config={"enabled": False, "max_rows": 0},
|
config={"enabled": False, "max_rows": 0},
|
||||||
@@ -1250,7 +1250,7 @@ def check_mysql_ddl_034(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
severity=Severity.WARNING,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="CHAR 类型字段的最大长度限制",
|
description="CHAR 类型字段的最大长度限制",
|
||||||
config={"enabled": True, "max_length": 10},
|
config={"enabled": True, "max_length": 255},
|
||||||
message_template="表 '{table}' 的字段 '{column}' CHAR({length}) 超过最大长度限制 {max_len}",
|
message_template="表 '{table}' 的字段 '{column}' CHAR({length}) 超过最大长度限制 {max_len}",
|
||||||
))
|
))
|
||||||
def check_mysql_ddl_035(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
|
def check_mysql_ddl_035(sql_clean: str, sql_upper: str, report: AuditReportV2, rule: AuditRuleV2) -> None:
|
||||||
@@ -1339,7 +1339,7 @@ def check_mysql_ddl_036(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DML-001",
|
code="MYSQL-DML-001",
|
||||||
name="DMLMaxInsertRows",
|
name="DMLMaxInsertRows",
|
||||||
category=RuleCategory.DML,
|
category=RuleCategory.DML,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="INSERT 语句单次插入的最大行数限制",
|
description="INSERT 语句单次插入的最大行数限制",
|
||||||
config={"enabled": True, "max_rows": 500},
|
config={"enabled": True, "max_rows": 500},
|
||||||
@@ -1564,7 +1564,7 @@ def check_mysql_dml_008(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-DML-009",
|
code="MYSQL-DML-009",
|
||||||
name="MaxAffectRows",
|
name="MaxAffectRows",
|
||||||
category=RuleCategory.DML,
|
category=RuleCategory.DML,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="DML 操作影响的最大行数限制(0表示不限制)",
|
description="DML 操作影响的最大行数限制(0表示不限制)",
|
||||||
config={"enabled": False, "max_rows": 0},
|
config={"enabled": False, "max_rows": 0},
|
||||||
@@ -1680,7 +1680,7 @@ def check_mysql_osc_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-NAM-001",
|
code="MYSQL-NAM-001",
|
||||||
name="TableNamingCase",
|
name="TableNamingCase",
|
||||||
category=RuleCategory.NAMING,
|
category=RuleCategory.NAMING,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="表名必须使用小写下划线命名法(snake_case)",
|
description="表名必须使用小写下划线命名法(snake_case)",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -1705,7 +1705,7 @@ def check_mysql_nam_001(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-NAM-002",
|
code="MYSQL-NAM-002",
|
||||||
name="ColumnNamingCase",
|
name="ColumnNamingCase",
|
||||||
category=RuleCategory.NAMING,
|
category=RuleCategory.NAMING,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="字段名必须使用小写下划线命名法(snake_case)",
|
description="字段名必须使用小写下划线命名法(snake_case)",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
@@ -1759,7 +1759,7 @@ def check_mysql_nam_003(sql_clean: str, sql_upper: str, report: AuditReportV2, r
|
|||||||
code="MYSQL-NAM-004",
|
code="MYSQL-NAM-004",
|
||||||
name="TableNamingNoDigitPrefix",
|
name="TableNamingNoDigitPrefix",
|
||||||
category=RuleCategory.NAMING,
|
category=RuleCategory.NAMING,
|
||||||
severity=Severity.ERROR,
|
severity=Severity.WARNING,
|
||||||
db_type="mysql",
|
db_type="mysql",
|
||||||
description="表名不能以数字开头",
|
description="表名不能以数字开头",
|
||||||
config={"enabled": True},
|
config={"enabled": True},
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ from app.services.sql_analyzer import SQLAnalyzer
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# ===== 配置常量 =====
|
# ===== 配置常量(默认值,优先从 system_settings 读取) =====
|
||||||
|
|
||||||
# 默认最大返回行数
|
# 默认最大返回行数
|
||||||
MAX_ROWS = 10000
|
MAX_ROWS = 10000
|
||||||
@@ -41,6 +41,88 @@ MAX_QUERY_LENGTH = 10000
|
|||||||
# 最大结果集内存限制(MB)
|
# 最大结果集内存限制(MB)
|
||||||
MAX_RESULT_MEMORY_MB = 50
|
MAX_RESULT_MEMORY_MB = 50
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_setting(key: str, default: int) -> int:
|
||||||
|
"""从 system_settings 读取配置,失败则返回默认值。"""
|
||||||
|
try:
|
||||||
|
from app.core.database import async_session
|
||||||
|
from app.models.system_setting import SystemSetting
|
||||||
|
from sqlalchemy import select
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
select(SystemSetting.value).where(SystemSetting.key == key)
|
||||||
|
)
|
||||||
|
val = result.scalar_one_or_none()
|
||||||
|
return int(val) if val is not None else default
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
async def get_query_config() -> Dict[str, int]:
|
||||||
|
"""批量获取查询相关配置,减少 DB 查询次数。"""
|
||||||
|
try:
|
||||||
|
from app.core.database import async_session
|
||||||
|
from app.models.system_setting import SystemSetting
|
||||||
|
from sqlalchemy import select
|
||||||
|
async with async_session() as db:
|
||||||
|
keys = [
|
||||||
|
"query_max_limit", "query_timeout", "query_max_length",
|
||||||
|
"query_result_max_memory_mb",
|
||||||
|
"executor_sql_timeout", "executor_redis_timeout", "executor_mongo_timeout",
|
||||||
|
]
|
||||||
|
stmt = select(SystemSetting.key, SystemSetting.value).where(
|
||||||
|
SystemSetting.key.in_(keys)
|
||||||
|
)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
rows = {r[0]: r[1] for r in result.all()}
|
||||||
|
return {
|
||||||
|
"max_rows": int(rows.get("query_max_limit", MAX_ROWS)),
|
||||||
|
"query_timeout": int(rows.get("query_timeout", QUERY_TIMEOUT)),
|
||||||
|
"max_query_length": int(rows.get("query_max_length", MAX_QUERY_LENGTH)),
|
||||||
|
"max_memory_mb": int(rows.get("query_result_max_memory_mb", MAX_RESULT_MEMORY_MB)),
|
||||||
|
"sql_timeout": int(rows.get("executor_sql_timeout", 60)),
|
||||||
|
"redis_timeout": int(rows.get("executor_redis_timeout", 30)),
|
||||||
|
"mongo_timeout": int(rows.get("executor_mongo_timeout", 60)),
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
return {
|
||||||
|
"max_rows": MAX_ROWS,
|
||||||
|
"query_timeout": QUERY_TIMEOUT,
|
||||||
|
"max_query_length": MAX_QUERY_LENGTH,
|
||||||
|
"max_memory_mb": MAX_RESULT_MEMORY_MB,
|
||||||
|
"sql_timeout": 60,
|
||||||
|
"redis_timeout": 30,
|
||||||
|
"mongo_timeout": 60,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ===== 动态配置缓存 =====
|
||||||
|
|
||||||
|
_active_config: Optional[Dict[str, int]] = None
|
||||||
|
|
||||||
|
|
||||||
|
async def refresh_config() -> Dict[str, int]:
|
||||||
|
"""刷新配置缓存并返回。启动时和 settings 更新后调用。"""
|
||||||
|
global _active_config
|
||||||
|
_active_config = await get_query_config()
|
||||||
|
return _active_config
|
||||||
|
|
||||||
|
|
||||||
|
def current_config() -> Dict[str, int]:
|
||||||
|
"""获取当前配置(同步),未初始化则返回默认值。"""
|
||||||
|
if _active_config is not None:
|
||||||
|
return _active_config
|
||||||
|
return {
|
||||||
|
"max_rows": MAX_ROWS,
|
||||||
|
"query_timeout": QUERY_TIMEOUT,
|
||||||
|
"max_query_length": MAX_QUERY_LENGTH,
|
||||||
|
"max_memory_mb": MAX_RESULT_MEMORY_MB,
|
||||||
|
"sql_timeout": 60,
|
||||||
|
"redis_timeout": 30,
|
||||||
|
"mongo_timeout": 60,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ===== Redis 只读命令白名单 =====
|
# ===== Redis 只读命令白名单 =====
|
||||||
|
|
||||||
REDIS_READONLY_COMMANDS = {
|
REDIS_READONLY_COMMANDS = {
|
||||||
@@ -294,9 +376,11 @@ class BaseQueryHandler(ABC):
|
|||||||
"""
|
"""
|
||||||
if not query or not query.strip():
|
if not query or not query.strip():
|
||||||
raise QuerySecurityError("查询语句不能为空")
|
raise QuerySecurityError("查询语句不能为空")
|
||||||
if len(query) > MAX_QUERY_LENGTH:
|
cfg = current_config()
|
||||||
|
if len(query) > cfg["max_query_length"]:
|
||||||
|
max_len = cfg["max_query_length"]
|
||||||
raise QuerySecurityError(
|
raise QuerySecurityError(
|
||||||
f"查询语句过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH} 字符"
|
f"查询语句过长({len(query)} 字符),最大允许 {max_len} 字符"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool:
|
def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool:
|
||||||
@@ -314,7 +398,7 @@ class BaseQueryHandler(ABC):
|
|||||||
try:
|
try:
|
||||||
size_bytes = len(json.dumps(rows, default=str).encode("utf-8"))
|
size_bytes = len(json.dumps(rows, default=str).encode("utf-8"))
|
||||||
size_mb = size_bytes / (1024 * 1024)
|
size_mb = size_bytes / (1024 * 1024)
|
||||||
return size_mb > MAX_RESULT_MEMORY_MB
|
return size_mb > current_config()["max_memory_mb"]
|
||||||
except (OverflowError, MemoryError):
|
except (OverflowError, MemoryError):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -418,16 +502,18 @@ class SQLQueryHandlerBase(BaseQueryHandler):
|
|||||||
# 不需要添加 LIMIT 的语句前缀
|
# 不需要添加 LIMIT 的语句前缀
|
||||||
_NO_LIMIT_PREFIXES = ("SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH")
|
_NO_LIMIT_PREFIXES = ("SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH")
|
||||||
|
|
||||||
def _auto_add_limit(self, sql: str, max_rows: int = MAX_ROWS) -> str:
|
def _auto_add_limit(self, sql: str, max_rows: int = -1) -> str:
|
||||||
"""如果 SQL 没有 LIMIT 则自动添加。仅对 SELECT 语句生效。
|
"""如果 SQL 没有 LIMIT 则自动添加。仅对 SELECT 语句生效。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
sql: 原始 SQL
|
sql: 原始 SQL
|
||||||
max_rows: 最大返回行数
|
max_rows: 最大返回行数(-1 表示从配置读取)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
添加了 LIMIT 的 SQL
|
添加了 LIMIT 的 SQL
|
||||||
"""
|
"""
|
||||||
|
if max_rows < 0:
|
||||||
|
max_rows = current_config()["max_rows"]
|
||||||
upper_sql = sql.strip().upper()
|
upper_sql = sql.strip().upper()
|
||||||
# SHOW / DESC / EXPLAIN 等语句不支持 LIMIT(或语义不同),跳过
|
# SHOW / DESC / EXPLAIN 等语句不支持 LIMIT(或语义不同),跳过
|
||||||
if any(upper_sql.startswith(p) for p in self._NO_LIMIT_PREFIXES):
|
if any(upper_sql.startswith(p) for p in self._NO_LIMIT_PREFIXES):
|
||||||
@@ -492,9 +578,9 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
|
|||||||
QueryTimeoutError: 查询超时
|
QueryTimeoutError: 查询超时
|
||||||
"""
|
"""
|
||||||
options = options or {}
|
options = options or {}
|
||||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||||
custom_max_rows = min(custom_max_rows, MAX_ROWS) # 不允许超过全局上限
|
custom_max_rows = min(custom_max_rows, current_config()["max_rows"]) # 不允许超过全局上限
|
||||||
|
|
||||||
# 1. 增强安全验证
|
# 1. 增强安全验证
|
||||||
self._enhanced_security_check(query, user_id)
|
self._enhanced_security_check(query, user_id)
|
||||||
@@ -538,7 +624,7 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
|
|||||||
truncated = True
|
truncated = True
|
||||||
# 截断结果以保护内存
|
# 截断结果以保护内存
|
||||||
rows = rows[:custom_max_rows] if rows else []
|
rows = rows[:custom_max_rows] if rows else []
|
||||||
logger.warning("查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB)
|
logger.warning("查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
|
||||||
|
|
||||||
# 6. 记录查询日志
|
# 6. 记录查询日志
|
||||||
QueryLogger.log_query(
|
QueryLogger.log_query(
|
||||||
@@ -595,9 +681,9 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
|
|||||||
QueryTimeoutError: 查询超时
|
QueryTimeoutError: 查询超时
|
||||||
"""
|
"""
|
||||||
options = options or {}
|
options = options or {}
|
||||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||||
custom_max_rows = min(custom_max_rows, MAX_ROWS)
|
custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
|
||||||
|
|
||||||
# 1. 增强安全验证
|
# 1. 增强安全验证
|
||||||
self._enhanced_security_check(query, user_id)
|
self._enhanced_security_check(query, user_id)
|
||||||
@@ -640,7 +726,7 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
|
|||||||
if self._check_result_size(rows if rows else []):
|
if self._check_result_size(rows if rows else []):
|
||||||
truncated = True
|
truncated = True
|
||||||
rows = rows[:custom_max_rows] if rows else []
|
rows = rows[:custom_max_rows] if rows else []
|
||||||
logger.warning("查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB)
|
logger.warning("查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
|
||||||
|
|
||||||
# 6. 记录查询日志
|
# 6. 记录查询日志
|
||||||
QueryLogger.log_query(
|
QueryLogger.log_query(
|
||||||
@@ -694,8 +780,9 @@ class RedisQueryHandler(BaseQueryHandler):
|
|||||||
return False, "Redis 命令不能为空"
|
return False, "Redis 命令不能为空"
|
||||||
|
|
||||||
# 长度验证
|
# 长度验证
|
||||||
if len(query) > MAX_QUERY_LENGTH:
|
if len(query) > current_config()["max_query_length"]:
|
||||||
return False, f"Redis 命令过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH}"
|
max_len = current_config()["max_query_length"]
|
||||||
|
return False, f"Redis 命令过长({len(query)} 字符),最大允许 {max_len}"
|
||||||
|
|
||||||
# 检测 shell 注入(管道符、重定向、命令替换)
|
# 检测 shell 注入(管道符、重定向、命令替换)
|
||||||
shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
|
shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
|
||||||
@@ -759,7 +846,7 @@ class RedisQueryHandler(BaseQueryHandler):
|
|||||||
QueryTimeoutError: 命令超时
|
QueryTimeoutError: 命令超时
|
||||||
"""
|
"""
|
||||||
options = options or {}
|
options = options or {}
|
||||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||||
|
|
||||||
# 1. 验证只读
|
# 1. 验证只读
|
||||||
is_safe, error_msg = self.validate_readonly(query)
|
is_safe, error_msg = self.validate_readonly(query)
|
||||||
@@ -918,8 +1005,9 @@ class MongoQueryHandler(BaseQueryHandler):
|
|||||||
return False, "MongoDB 查询不能为空"
|
return False, "MongoDB 查询不能为空"
|
||||||
|
|
||||||
# 长度验证
|
# 长度验证
|
||||||
if len(query) > MAX_QUERY_LENGTH:
|
if len(query) > current_config()["max_query_length"]:
|
||||||
return False, f"MongoDB 查询过长({len(query)} 字符),最大允许 {MAX_QUERY_LENGTH}"
|
max_len = current_config()["max_query_length"]
|
||||||
|
return False, f"MongoDB 查询过长({len(query)} 字符),最大允许 {max_len}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
query_obj = json.loads(query)
|
query_obj = json.loads(query)
|
||||||
@@ -990,9 +1078,9 @@ class MongoQueryHandler(BaseQueryHandler):
|
|||||||
QueryTimeoutError: 查询超时
|
QueryTimeoutError: 查询超时
|
||||||
"""
|
"""
|
||||||
options = options or {}
|
options = options or {}
|
||||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||||
custom_max_rows = min(custom_max_rows, MAX_ROWS)
|
custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
|
||||||
|
|
||||||
# 1. 验证只读
|
# 1. 验证只读
|
||||||
is_safe, error_msg = self.validate_readonly(query)
|
is_safe, error_msg = self.validate_readonly(query)
|
||||||
@@ -1156,7 +1244,7 @@ class MongoQueryHandler(BaseQueryHandler):
|
|||||||
if self._check_result_size(rows):
|
if self._check_result_size(rows):
|
||||||
truncated = True
|
truncated = True
|
||||||
rows = rows[:custom_max_rows]
|
rows = rows[:custom_max_rows]
|
||||||
logger.warning("MongoDB 查询结果超过内存限制 %dMB,已截断", MAX_RESULT_MEMORY_MB)
|
logger.warning("MongoDB 查询结果超过内存限制 %dMB,已截断", current_config()["max_memory_mb"])
|
||||||
|
|
||||||
# 5. 记录查询日志
|
# 5. 记录查询日志
|
||||||
QueryLogger.log_query(
|
QueryLogger.log_query(
|
||||||
@@ -1236,6 +1324,9 @@ class QueryService:
|
|||||||
if handler is None:
|
if handler is None:
|
||||||
raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询")
|
raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询")
|
||||||
|
|
||||||
|
# 每次查询刷新配置缓存(轻量:仅查一次 DB)
|
||||||
|
await refresh_config()
|
||||||
|
|
||||||
return await handler.execute(instance, database, query, user_id, options)
|
return await handler.execute(instance, database, query, user_id, options)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -188,6 +188,48 @@ SEED_SETTINGS = [
|
|||||||
"default_value": "30",
|
"default_value": "30",
|
||||||
"sort_order": 35,
|
"sort_order": 35,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
# ── 执行器 ──
|
||||||
|
{
|
||||||
|
"key": "executor_sql_timeout",
|
||||||
|
"value": "60",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "executor",
|
||||||
|
"label": "SQL 执行超时(秒)",
|
||||||
|
"description": "MySQL/PostgreSQL 工单执行单条 SQL 最大时间。范围: 10~600",
|
||||||
|
"default_value": "60",
|
||||||
|
"sort_order": 40,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "executor_redis_timeout",
|
||||||
|
"value": "30",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "executor",
|
||||||
|
"label": "Redis 执行超时(秒)",
|
||||||
|
"description": "Redis 工单执行单条命令最大时间。范围: 5~120",
|
||||||
|
"default_value": "30",
|
||||||
|
"sort_order": 41,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "executor_mongo_timeout",
|
||||||
|
"value": "60",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "executor",
|
||||||
|
"label": "MongoDB 执行超时(秒)",
|
||||||
|
"description": "MongoDB 工单执行单条命令最大时间。范围: 10~600",
|
||||||
|
"default_value": "60",
|
||||||
|
"sort_order": 42,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "query_result_max_memory_mb",
|
||||||
|
"value": "50",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "query",
|
||||||
|
"label": "查询结果内存上限(MB)",
|
||||||
|
"description": "单次查询结果集最大内存占用,超限自动截断。范围: 10~500",
|
||||||
|
"default_value": "50",
|
||||||
|
"sort_order": 8,
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
# 分组元信息
|
# 分组元信息
|
||||||
@@ -196,4 +238,5 @@ SETTING_GROUPS = {
|
|||||||
"security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"},
|
"security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"},
|
||||||
"ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"},
|
"ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"},
|
||||||
"cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"},
|
"cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"},
|
||||||
|
"executor": {"label": "执行器", "icon": "⚙️", "description": "SQL/Redis 执行超时、批量限制、连接参数"},
|
||||||
}
|
}
|
||||||
@@ -40,11 +40,19 @@ from app.services.sql_analyzer import SQLAnalyzer, SQLType
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# 超时配置(秒)
|
# 超时配置(秒)—— 默认值,优先从 system_settings 读取
|
||||||
SQL_TIMEOUT = 60
|
SQL_TIMEOUT = 60
|
||||||
REDIS_TIMEOUT = 30
|
REDIS_TIMEOUT = 30
|
||||||
MONGO_TIMEOUT = 60
|
MONGO_TIMEOUT = 60
|
||||||
|
|
||||||
|
|
||||||
|
def _get_executor_timeout(db_type: str) -> int:
|
||||||
|
"""从 query_service 配置缓存获取执行超时。"""
|
||||||
|
from app.services.query_service import current_config
|
||||||
|
cfg = current_config()
|
||||||
|
key = {"mysql": "sql_timeout", "postgres": "sql_timeout", "redis": "redis_timeout", "mongodb": "mongo_timeout"}.get(db_type, "sql_timeout")
|
||||||
|
return cfg.get(key, SQL_TIMEOUT)
|
||||||
|
|
||||||
# 并发执行锁: 防止同一工单被并发执行
|
# 并发执行锁: 防止同一工单被并发执行
|
||||||
_ticket_execution_locks: Set[int] = set()
|
_ticket_execution_locks: Set[int] = set()
|
||||||
_lock_lock = asyncio.Lock()
|
_lock_lock = asyncio.Lock()
|
||||||
@@ -238,7 +246,8 @@ class TicketExecutor:
|
|||||||
|
|
||||||
# 获取超时配置(从 ticket 或选项中)
|
# 获取超时配置(从 ticket 或选项中)
|
||||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||||
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
|
_default_timeout = _get_executor_timeout('mysql')
|
||||||
|
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 1. 语法预校验
|
# 1. 语法预校验
|
||||||
@@ -370,7 +379,8 @@ class TicketExecutor:
|
|||||||
|
|
||||||
# 获取超时配置
|
# 获取超时配置
|
||||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||||
timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT
|
_default_timeout = _get_executor_timeout('postgres')
|
||||||
|
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 1. 语法预校验
|
# 1. 语法预校验
|
||||||
@@ -516,7 +526,8 @@ class TicketExecutor:
|
|||||||
|
|
||||||
# 获取超时配置
|
# 获取超时配置
|
||||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||||
timeout = custom_timeout.get('timeout', REDIS_TIMEOUT) if isinstance(custom_timeout, dict) else REDIS_TIMEOUT
|
_default_timeout = _get_executor_timeout('redis')
|
||||||
|
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 解析命令
|
# 解析命令
|
||||||
@@ -686,7 +697,8 @@ class TicketExecutor:
|
|||||||
|
|
||||||
# 获取超时配置
|
# 获取超时配置
|
||||||
custom_timeout = getattr(ticket, 'options', None) or {}
|
custom_timeout = getattr(ticket, 'options', None) or {}
|
||||||
timeout = custom_timeout.get('timeout', MONGO_TIMEOUT) if isinstance(custom_timeout, dict) else MONGO_TIMEOUT
|
_default_timeout = _get_executor_timeout('mongodb')
|
||||||
|
timeout = custom_timeout.get('timeout', _default_timeout) if isinstance(custom_timeout, dict) else _default_timeout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 解析工单内容
|
# 解析工单内容
|
||||||
|
|||||||
Reference in new issue
Block a user