feat: 审核规则持久化到DB + 全面配置去硬编码化
- AuditRule模型: 新增code/config_json字段，支持规则编码和阈值参数持久化 - audit_engine_v2: _load_rules从DB读config_json覆盖默认config - audit_rule_sync: 启动时自动从_RULE_REGISTRY同步规则到DB(239条) - settings API: update后刷新query_service配置缓存 - query_service: MAX_ROWS/QUERY_TIMEOUT/MAX_QUERY_LENGTH/MAX_RESULT_MEMORY改为动态读取system_settings - ticket_executor: SQL_TIMEOUT/REDIS_TIMEOUT/MONGO_TIMEOUT改为动态获取 - settings_seeds: 新增executor分组(sql/redis/mongo超时) + query_result_max_memory_mb - audit_rule endpoint: 移除无效的AuditEngine缓存清理（每次audit是新实例） - 测试: 1057 passed（修复所有新引入失败，17个历史遗留）
This commit is contained in:
logan committed
1 parent
ddf4856a85
commit
a9e214ef1a
11 files changed
+342
-62
No files matched your search
@@ -30,7 +30,7 @@ from app.services.sql_analyzer import SQLAnalyzer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ===== 配置常量 =====
|
||||
# ===== 配置常量（默认值，优先从 system_settings 读取） =====
|
||||
|
||||
# 默认最大返回行数
|
||||
MAX_ROWS = 10000
|
||||
@@ -41,6 +41,88 @@ MAX_QUERY_LENGTH = 10000
|
||||
# 最大结果集内存限制（MB）
|
||||
MAX_RESULT_MEMORY_MB = 50
|
||||
|
||||
|
||||
async def _get_setting(key: str, default: int) -> int:
|
||||
"""从 system_settings 读取配置，失败则返回默认值。"""
|
||||
try:
|
||||
from app.core.database import async_session
|
||||
from app.models.system_setting import SystemSetting
|
||||
from sqlalchemy import select
|
||||
async with async_session() as db:
|
||||
result = await db.execute(
|
||||
select(SystemSetting.value).where(SystemSetting.key == key)
|
||||
)
|
||||
val = result.scalar_one_or_none()
|
||||
return int(val) if val is not None else default
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
async def get_query_config() -> Dict[str, int]:
|
||||
"""批量获取查询相关配置，减少 DB 查询次数。"""
|
||||
try:
|
||||
from app.core.database import async_session
|
||||
from app.models.system_setting import SystemSetting
|
||||
from sqlalchemy import select
|
||||
async with async_session() as db:
|
||||
keys = [
|
||||
"query_max_limit", "query_timeout", "query_max_length",
|
||||
"query_result_max_memory_mb",
|
||||
"executor_sql_timeout", "executor_redis_timeout", "executor_mongo_timeout",
|
||||
]
|
||||
stmt = select(SystemSetting.key, SystemSetting.value).where(
|
||||
SystemSetting.key.in_(keys)
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
rows = {r[0]: r[1] for r in result.all()}
|
||||
return {
|
||||
"max_rows": int(rows.get("query_max_limit", MAX_ROWS)),
|
||||
"query_timeout": int(rows.get("query_timeout", QUERY_TIMEOUT)),
|
||||
"max_query_length": int(rows.get("query_max_length", MAX_QUERY_LENGTH)),
|
||||
"max_memory_mb": int(rows.get("query_result_max_memory_mb", MAX_RESULT_MEMORY_MB)),
|
||||
"sql_timeout": int(rows.get("executor_sql_timeout", 60)),
|
||||
"redis_timeout": int(rows.get("executor_redis_timeout", 30)),
|
||||
"mongo_timeout": int(rows.get("executor_mongo_timeout", 60)),
|
||||
}
|
||||
except Exception:
|
||||
return {
|
||||
"max_rows": MAX_ROWS,
|
||||
"query_timeout": QUERY_TIMEOUT,
|
||||
"max_query_length": MAX_QUERY_LENGTH,
|
||||
"max_memory_mb": MAX_RESULT_MEMORY_MB,
|
||||
"sql_timeout": 60,
|
||||
"redis_timeout": 30,
|
||||
"mongo_timeout": 60,
|
||||
}
|
||||
|
||||
|
||||
# ===== 动态配置缓存 =====
|
||||
|
||||
_active_config: Optional[Dict[str, int]] = None
|
||||
|
||||
|
||||
async def refresh_config() -> Dict[str, int]:
|
||||
"""刷新配置缓存并返回。启动时和 settings 更新后调用。"""
|
||||
global _active_config
|
||||
_active_config = await get_query_config()
|
||||
return _active_config
|
||||
|
||||
|
||||
def current_config() -> Dict[str, int]:
|
||||
"""获取当前配置（同步），未初始化则返回默认值。"""
|
||||
if _active_config is not None:
|
||||
return _active_config
|
||||
return {
|
||||
"max_rows": MAX_ROWS,
|
||||
"query_timeout": QUERY_TIMEOUT,
|
||||
"max_query_length": MAX_QUERY_LENGTH,
|
||||
"max_memory_mb": MAX_RESULT_MEMORY_MB,
|
||||
"sql_timeout": 60,
|
||||
"redis_timeout": 30,
|
||||
"mongo_timeout": 60,
|
||||
}
|
||||
|
||||
|
||||
# ===== Redis 只读命令白名单 =====
|
||||
|
||||
REDIS_READONLY_COMMANDS = {
|
||||
@@ -294,9 +376,11 @@ class BaseQueryHandler(ABC):
|
||||
"""
|
||||
if not query or not query.strip():
|
||||
raise QuerySecurityError("查询语句不能为空")
|
||||
if len(query) > MAX_QUERY_LENGTH:
|
||||
cfg = current_config()
|
||||
if len(query) > cfg["max_query_length"]:
|
||||
max_len = cfg["max_query_length"]
|
||||
raise QuerySecurityError(
|
||||
f"查询语句过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH} 字符"
|
||||
f"查询语句过长（{len(query)} 字符），最大允许 {max_len} 字符"
|
||||
)
|
||||
|
||||
def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool:
|
||||
@@ -314,7 +398,7 @@ class BaseQueryHandler(ABC):
|
||||
try:
|
||||
size_bytes = len(json.dumps(rows, default=str).encode("utf-8"))
|
||||
size_mb = size_bytes / (1024 * 1024)
|
||||
return size_mb > MAX_RESULT_MEMORY_MB
|
||||
return size_mb > current_config()["max_memory_mb"]
|
||||
except (OverflowError, MemoryError):
|
||||
return True
|
||||
|
||||
@@ -418,16 +502,18 @@ class SQLQueryHandlerBase(BaseQueryHandler):
|
||||
# 不需要添加 LIMIT 的语句前缀
|
||||
_NO_LIMIT_PREFIXES = ("SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH")
|
||||
|
||||
def _auto_add_limit(self, sql: str, max_rows: int = MAX_ROWS) -> str:
|
||||
def _auto_add_limit(self, sql: str, max_rows: int = -1) -> str:
|
||||
"""如果 SQL 没有 LIMIT 则自动添加。仅对 SELECT 语句生效。
|
||||
|
||||
Args:
|
||||
sql: 原始 SQL
|
||||
max_rows: 最大返回行数
|
||||
max_rows: 最大返回行数（-1 表示从配置读取）
|
||||
|
||||
Returns:
|
||||
添加了 LIMIT 的 SQL
|
||||
"""
|
||||
if max_rows < 0:
|
||||
max_rows = current_config()["max_rows"]
|
||||
upper_sql = sql.strip().upper()
|
||||
# SHOW / DESC / EXPLAIN 等语句不支持 LIMIT（或语义不同），跳过
|
||||
if any(upper_sql.startswith(p) for p in self._NO_LIMIT_PREFIXES):
|
||||
@@ -492,9 +578,9 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
|
||||
QueryTimeoutError: 查询超时
|
||||
"""
|
||||
options = options or {}
|
||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
||||
custom_max_rows = min(custom_max_rows, MAX_ROWS) # 不允许超过全局上限
|
||||
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||
custom_max_rows = min(custom_max_rows, current_config()["max_rows"]) # 不允许超过全局上限
|
||||
|
||||
# 1. 增强安全验证
|
||||
self._enhanced_security_check(query, user_id)
|
||||
@@ -538,7 +624,7 @@ class MySQLQueryHandler(SQLQueryHandlerBase):
|
||||
truncated = True
|
||||
# 截断结果以保护内存
|
||||
rows = rows[:custom_max_rows] if rows else []
|
||||
logger.warning("查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB)
|
||||
logger.warning("查询结果超过内存限制 %dMB，已截断", current_config()["max_memory_mb"])
|
||||
|
||||
# 6. 记录查询日志
|
||||
QueryLogger.log_query(
|
||||
@@ -595,9 +681,9 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
|
||||
QueryTimeoutError: 查询超时
|
||||
"""
|
||||
options = options or {}
|
||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
||||
custom_max_rows = min(custom_max_rows, MAX_ROWS)
|
||||
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||
custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
|
||||
|
||||
# 1. 增强安全验证
|
||||
self._enhanced_security_check(query, user_id)
|
||||
@@ -640,7 +726,7 @@ class PostgresQueryHandler(SQLQueryHandlerBase):
|
||||
if self._check_result_size(rows if rows else []):
|
||||
truncated = True
|
||||
rows = rows[:custom_max_rows] if rows else []
|
||||
logger.warning("查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB)
|
||||
logger.warning("查询结果超过内存限制 %dMB，已截断", current_config()["max_memory_mb"])
|
||||
|
||||
# 6. 记录查询日志
|
||||
QueryLogger.log_query(
|
||||
@@ -694,8 +780,9 @@ class RedisQueryHandler(BaseQueryHandler):
|
||||
return False, "Redis 命令不能为空"
|
||||
|
||||
# 长度验证
|
||||
if len(query) > MAX_QUERY_LENGTH:
|
||||
return False, f"Redis 命令过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH}"
|
||||
if len(query) > current_config()["max_query_length"]:
|
||||
max_len = current_config()["max_query_length"]
|
||||
return False, f"Redis 命令过长（{len(query)} 字符），最大允许 {max_len}"
|
||||
|
||||
# 检测 shell 注入（管道符、重定向、命令替换）
|
||||
shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"]
|
||||
@@ -759,7 +846,7 @@ class RedisQueryHandler(BaseQueryHandler):
|
||||
QueryTimeoutError: 命令超时
|
||||
"""
|
||||
options = options or {}
|
||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
||||
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||
|
||||
# 1. 验证只读
|
||||
is_safe, error_msg = self.validate_readonly(query)
|
||||
@@ -918,8 +1005,9 @@ class MongoQueryHandler(BaseQueryHandler):
|
||||
return False, "MongoDB 查询不能为空"
|
||||
|
||||
# 长度验证
|
||||
if len(query) > MAX_QUERY_LENGTH:
|
||||
return False, f"MongoDB 查询过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH}"
|
||||
if len(query) > current_config()["max_query_length"]:
|
||||
max_len = current_config()["max_query_length"]
|
||||
return False, f"MongoDB 查询过长（{len(query)} 字符），最大允许 {max_len}"
|
||||
|
||||
try:
|
||||
query_obj = json.loads(query)
|
||||
@@ -990,9 +1078,9 @@ class MongoQueryHandler(BaseQueryHandler):
|
||||
QueryTimeoutError: 查询超时
|
||||
"""
|
||||
options = options or {}
|
||||
custom_timeout = options.get("timeout", QUERY_TIMEOUT)
|
||||
custom_max_rows = options.get("limit", MAX_ROWS)
|
||||
custom_max_rows = min(custom_max_rows, MAX_ROWS)
|
||||
custom_timeout = options.get("timeout", current_config()["query_timeout"])
|
||||
custom_max_rows = options.get("limit", current_config()["max_rows"])
|
||||
custom_max_rows = min(custom_max_rows, current_config()["max_rows"])
|
||||
|
||||
# 1. 验证只读
|
||||
is_safe, error_msg = self.validate_readonly(query)
|
||||
@@ -1156,7 +1244,7 @@ class MongoQueryHandler(BaseQueryHandler):
|
||||
if self._check_result_size(rows):
|
||||
truncated = True
|
||||
rows = rows[:custom_max_rows]
|
||||
logger.warning("MongoDB 查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB)
|
||||
logger.warning("MongoDB 查询结果超过内存限制 %dMB，已截断", current_config()["max_memory_mb"])
|
||||
|
||||
# 5. 记录查询日志
|
||||
QueryLogger.log_query(
|
||||
@@ -1236,6 +1324,9 @@ class QueryService:
|
||||
if handler is None:
|
||||
raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询")
|
||||
|
||||
# 每次查询刷新配置缓存（轻量：仅查一次 DB）
|
||||
await refresh_config()
|
||||
|
||||
return await handler.execute(instance, database, query, user_id, options)
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in new issue
Block a user