From b39c426317df6f908374e5050bdf7f109557640c Mon Sep 17 00:00:00 2001 From: logan Date: Sat, 23 May 2026 17:50:13 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20merge=E5=90=8E14=E4=B8=AA=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E4=BF=AE=E5=A4=8D+=E5=89=8D=E7=AB=AFTS=E7=BC=96?= =?UTF-8?q?=E8=AF=91=E4=BF=AE=E5=A4=8D=20-=20895/895=E9=80=9A=E8=BF=87+?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E9=9B=B6=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/endpoints/ticket.py | 544 ++++++--- backend/app/models/workflow.py | 331 +++++- backend/app/services/audit_engine.py | 860 +++++++++++--- backend/app/services/query_service.py | 722 ++++++++++-- backend/app/services/ticket_executor.py | 532 +++++++-- backend/app/services/workflow_engine.py | 1290 ++++++++++++++++++--- backend/tests/test_audit_engine.py | 17 +- backend/tests/test_tickets.py | 2 +- frontend/src/api/auth.ts | 6 + frontend/src/components/SqlEditor.tsx | 2 - frontend/src/hooks/useAuth.tsx | 53 +- frontend/src/pages/AuditLogsPage.tsx | 4 +- frontend/src/pages/AuditRulesPage.tsx | 6 +- frontend/src/pages/Dashboard.tsx | 4 +- frontend/src/pages/DataDictionaryPage.tsx | 3 +- frontend/src/pages/InstanceGroupPage.tsx | 4 +- frontend/src/pages/QueriesPage.tsx | 6 +- frontend/src/pages/SlowQueryPage.tsx | 2 +- frontend/src/pages/TicketCreatePage.tsx | 35 +- frontend/src/pages/TicketDetailPage.tsx | 6 +- frontend/src/pages/TicketsPage.tsx | 5 +- frontend/src/pages/UsersPage.tsx | 4 +- frontend/src/pages/WorkflowPage.tsx | 2 +- merge-analysis-report.md | 238 ++++ 24 files changed, 3918 insertions(+), 760 deletions(-) create mode 100644 merge-analysis-report.md diff --git a/backend/app/api/v1/endpoints/ticket.py b/backend/app/api/v1/endpoints/ticket.py index a382860..74de4ca 100644 --- a/backend/app/api/v1/endpoints/ticket.py +++ b/backend/app/api/v1/endpoints/ticket.py @@ -1,11 +1,21 @@ -from typing import Optional, List, Dict, Any, TYPE_CHECKING +"""工单管理 API — 支持多数据库的变更工单生命周期管理。 + +核心功能: +- 工单创建、审核、执行、取消、回滚 +- 多级审批流支持 +- 工单状态机管理（严格的状态转换） +- 自动 SQL 审核（创建时触发） +- 执行前二次确认 +- 执行后通知触发 +- 完整操作审计 +""" + from datetime import datetime, timezone +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, List, Optional -if TYPE_CHECKING: - from app.services.ticket_executor import ExecutionReport - -from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from pydantic import BaseModel, Field, field_validator from sqlalchemy import select, func, or_ from sqlalchemy.ext.asyncio import AsyncSession @@ -14,25 +24,103 @@ from app.core.security import get_current_user, require_role from app.models.user import User from app.models.ticket import Ticket from app.models.instance import Instance +from app.models.workflow import WorkflowDefinition, WorkflowStep, TicketApproval +from app.models.audit_log import AuditLog from app.schemas.ticket import TicketCreate, TicketReview, TicketResponse from app.core.crypto import decrypt_password from app.core.permissions import check_instance_access, check_instance_permission from app.services.audit_engine import SQLAuditEngine from app.services.audit_engine_v2 import SQLAuditEngineV2 from app.services.audit_log_service import AuditLogService -from app.models.workflow import WorkflowDefinition, WorkflowStep, TicketApproval +from app.services.query_service import MAX_QUERY_LENGTH import json +import logging +logger = logging.getLogger(__name__) -class AuditRequest(BaseModel): - sql_content: str - instance_id: int - database: str = "" +if TYPE_CHECKING: + from app.services.ticket_executor import ExecutionReport router = APIRouter(prefix="/tickets", tags=["工单管理"]) +# ===== 工单状态机 ===== + +class TicketStatus(str, Enum): + """工单状态枚举。""" + PENDING = "pending" # 待审批 + REVIEWING = "reviewing" # 审批中（多级审批） + APPROVED = "approved" # 已通过 + REJECTED = "rejected" # 已拒绝 + EXECUTING = "executing" # 执行中 + SUCCESS = "success" # 执行成功 + FAILED = "failed" # 执行失败 + ROLLBACK = "rollback" # 已回滚 + + +# 状态转换图: 当前状态 -> 允许的目标状态 +TICKET_STATE_TRANSITIONS: Dict[str, List[str]] = { + TicketStatus.PENDING.value: [TicketStatus.REVIEWING.value, TicketStatus.APPROVED.value, TicketStatus.REJECTED.value], + TicketStatus.REVIEWING.value: [TicketStatus.REVIEWING.value, TicketStatus.APPROVED.value, TicketStatus.REJECTED.value], + TicketStatus.APPROVED.value: [TicketStatus.EXECUTING.value, TicketStatus.REJECTED.value], + TicketStatus.EXECUTING.value: [TicketStatus.SUCCESS.value, TicketStatus.FAILED.value], + TicketStatus.SUCCESS.value: [TicketStatus.ROLLBACK.value], + TicketStatus.FAILED.value: [TicketStatus.ROLLBACK.value, TicketStatus.APPROVED.value], + TicketStatus.REJECTED.value: [], # 终态，不可再转换 + TicketStatus.ROLLBACK.value: [], # 终态，不可再转换 +} + + +def _validate_status_transition(current_status: str, new_status: str) -> bool: + """验证工单状态转换是否合法。 + + Args: + current_status: 当前状态 + new_status: 目标状态 + + Returns: + 转换是否合法 + """ + allowed = TICKET_STATE_TRANSITIONS.get(current_status, []) + return new_status in allowed + + +# ===== 请求模型 ===== + + +class AuditRequest(BaseModel): + """SQL 审核请求。""" + sql_content: str = Field(..., min_length=1, max_length=MAX_QUERY_LENGTH) + instance_id: int + database: str = "" + + @field_validator('sql_content') + @classmethod + def validate_sql_length(cls, v): + if len(v) > MAX_QUERY_LENGTH: + raise ValueError(f"SQL 内容不能超过 {MAX_QUERY_LENGTH} 字符") + return v + + +class TicketExecuteConfirm(BaseModel): + """工单执行二次确认请求。""" + confirmed: bool = Field(..., description="是否确认执行") + comment: str = Field(default="", max_length=500, description="执行备注") + + +# ===== 辅助函数 ===== + + async def _enrich_ticket(ticket: Ticket, db: AsyncSession) -> Dict[str, Any]: + """丰富工单数据，添加申请人/审批人名称等信息。 + + Args: + ticket: 工单对象 + db: 数据库会话 + + Returns: + 丰富的工单字典 + """ data = { "id": ticket.id, "title": ticket.title, @@ -68,27 +156,121 @@ async def _enrich_ticket(ticket: Ticket, db: AsyncSession) -> Dict[str, Any]: return data +async def _get_instance(db: AsyncSession, instance_id: int) -> Instance: + """获取数据源实例，不存在则抛出 404。 + + Args: + db: 数据库会话 + instance_id: 实例ID + + Returns: + Instance 对象 + + Raises: + HTTPException: 实例不存在 + """ + result = await db.execute(select(Instance).where(Instance.id == instance_id)) + instance = result.scalar_one_or_none() + if not instance: + raise HTTPException(status_code=404, detail="数据源不存在") + return instance + + +async def _get_ticket(db: AsyncSession, ticket_id: int) -> Ticket: + """获取工单，不存在则抛出 404。 + + Args: + db: 数据库会话 + ticket_id: 工单ID + + Returns: + Ticket 对象 + + Raises: + HTTPException: 工单不存在 + """ + result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) + ticket = result.scalar_one_or_none() + if not ticket: + raise HTTPException(status_code=404, detail="工单不存在") + return ticket + + +def _validate_ticket_content(data: TicketCreate) -> None: + """验证工单内容安全性。 + + Args: + data: 工单创建数据 + + Raises: + HTTPException: 验证失败 + """ + # SQL 内容长度检查 + if len(data.sql_content) > MAX_QUERY_LENGTH: + raise HTTPException( + status_code=400, + detail=f"SQL 内容过长（{len(data.sql_content)} 字符），最大允许 {MAX_QUERY_LENGTH} 字符", + ) + + # 检测 null 字节 + if "\x00" in data.sql_content: + raise HTTPException(status_code=400, detail="SQL 内容包含非法字符（null 字节）") + + # 检测零宽字符（可能的注入攻击） + zero_width_chars = ["\u200b", "\u200c", "\u200d", "\ufeff"] + for char in zero_width_chars: + if char in data.sql_content: + raise HTTPException( + status_code=400, + detail="SQL 内容包含非法的零宽 Unicode 字符", + ) + + # 回滚 SQL 长度检查 + if data.rollback_sql and len(data.rollback_sql) > MAX_QUERY_LENGTH: + raise HTTPException( + status_code=400, + detail=f"回滚 SQL 过长（{len(data.rollback_sql)} 字符），最大允许 {MAX_QUERY_LENGTH} 字符", + ) + + +# ===== API 端点 ===== + + @router.get("") async def list_tickets( - page: int = Query(1, ge=1), - page_size: int = Query(10, ge=1, le=100), + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(10, ge=1, le=100, description="每页数量"), status: Optional[str] = None, db_type: Optional[str] = None, keyword: Optional[str] = None, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): + """获取工单列表。 + + - admin/dba 可查看所有工单 + - developer 只能查看自己的工单 + """ stmt = select(Ticket) - # developer 只能看到自己的工单 + # 权限过滤: developer 只能看到自己的工单 if current_user.role == "developer": stmt = stmt.where(Ticket.applicant_id == current_user.id) + # 状态过滤 if status: + # 验证状态值合法性 + valid_statuses = [s.value for s in TicketStatus] + if status not in valid_statuses: + raise HTTPException(status_code=400, detail=f"无效的状态值: {status}") stmt = stmt.where(Ticket.status == status) + if db_type: stmt = stmt.where(Ticket.db_type == db_type) + if keyword: + # 限制关键词长度防止 DoS + keyword = keyword[:100] stmt = stmt.where( or_( Ticket.title.contains(keyword), @@ -96,10 +278,17 @@ async def list_tickets( ) ) + # 总数查询 total_stmt = select(func.count()).select_from(stmt.subquery()) - total = (await db.execute(total_stmt)).scalar() + total_result = await db.execute(total_stmt) + total = total_result.scalar() or 0 - stmt = stmt.order_by(Ticket.create_time.desc()).offset((page - 1) * page_size).limit(page_size) + # 分页查询 + stmt = ( + stmt.order_by(Ticket.create_time.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + ) result = await db.execute(stmt) tickets = result.scalars().all() @@ -107,7 +296,12 @@ async def list_tickets( for t in tickets: items.append(await _enrich_ticket(t, db)) - return {"total": total, "page": page, "page_size": page_size, "items": items} + return { + "total": total, + "page": page, + "page_size": page_size, + "items": items, + } @router.post("", status_code=201) @@ -116,26 +310,39 @@ async def create_ticket( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - # Verify instance exists - inst_result = await db.execute(select(Instance).where(Instance.id == data.instance_id)) - instance = inst_result.scalar_one_or_none() - if not instance: - raise HTTPException(status_code=400, detail="数据源不存在") + """创建工单。 - # 权限组检查：用户是否有权对该数据源提交工单 + 流程: + 1. 验证数据源存在和权限 + 2. 验证工单内容安全性 + 3. 自动 SQL 审核 + 4. 创建工单记录 + 5. 如有多级审批流，创建审批步骤 + """ + # 1. 验证数据源 + instance = await _get_instance(db, data.instance_id) + + # 2. 权限检查 try: await check_instance_access(current_user, instance, db) except HTTPException: raise HTTPException(status_code=403, detail="您没有权限对该数据源提交变更工单") - # P1: 操作级权限检查 await check_instance_permission(current_user, instance, "ticket:create", db) - # Auto-audit SQL before creating ticket (使用 V2 引擎) - v2_engine = SQLAuditEngineV2(db_session=db, env="production" if data.db_type == "mysql" else "development") - v2_report = await v2_engine.audit(data.sql_content, db_type=data.db_type) + # 3. 验证工单内容安全性 + _validate_ticket_content(data) - # Check for default workflow to determine initial status + # 4. 自动审核 SQL + env = "production" if instance.env == "production" else "development" + v2_engine = SQLAuditEngineV2(db_session=db, env=env) + try: + v2_report = await v2_engine.audit(data.sql_content, db_type=data.db_type) + except Exception as e: + logger.error("SQL 审核失败: %s", e) + v2_report = None + + # 5. 检查是否有默认审批流 default_wf_result = await db.execute( select(WorkflowDefinition).where( WorkflowDefinition.is_default == True, @@ -144,8 +351,9 @@ async def create_ticket( ) default_workflow = default_wf_result.scalar_one_or_none() - initial_status = "reviewing" if default_workflow else "pending" + initial_status = TicketStatus.REVIEWING.value if default_workflow else TicketStatus.PENDING.value + # 6. 创建工单 ticket = Ticket( title=data.title, db_type=data.db_type, @@ -157,12 +365,12 @@ async def create_ticket( priority=data.priority, applicant_id=current_user.id, status=initial_status, - audit_result=json.dumps(v2_report.to_dict(), ensure_ascii=False) if v2_report.findings else None, + audit_result=json.dumps(v2_report.to_dict(), ensure_ascii=False) if v2_report and v2_report.findings else None, ) db.add(ticket) await db.flush() # flush to get ticket.id - # If default workflow exists, create TicketApproval entries for each step + # 7. 创建审批步骤（多级审批流） if default_workflow: wf_steps_result = await db.execute( select(WorkflowStep) @@ -181,17 +389,22 @@ async def create_ticket( db.add(approval) await db.flush() - # Log ticket creation + # 8. 记录审计日志 + audit_detail = "" + if v2_report: + audit_detail = f"SQL审核评分: {v2_report.score}, 通过: {v2_report.passed}, 规则数: {len(v2_report.findings)}" await AuditLogService.log( db, user_id=current_user.id, action="CREATE_TICKET", target_type="ticket", target_id=ticket.id, - detail=f"SQL审核评分: {v2_report.score}, 通过: {v2_report.passed}, 规则数: {len(v2_report.findings)}", + detail=audit_detail or "工单创建", ) await db.commit() await db.refresh(ticket) + result = await _enrich_ticket(ticket, db) - result["audit_report"] = v2_report.to_dict() + if v2_report: + result["audit_report"] = v2_report.to_dict() return result @@ -200,6 +413,7 @@ async def ticket_statistics( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): + """获取工单统计信息。""" stmt = select(Ticket.status, func.count(Ticket.id)).group_by(Ticket.status) result = await db.execute(stmt) stats = {row[0]: row[1] for row in result.all()} @@ -214,6 +428,7 @@ async def ticket_statistics( "executing": stats.get("executing", 0), "success": stats.get("success", 0), "failed": stats.get("failed", 0), + "rollback": stats.get("rollback", 0), } @@ -223,26 +438,35 @@ async def audit_sql( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - # Verify instance exists - inst_result = await db.execute(select(Instance).where(Instance.id == data.instance_id)) - instance = inst_result.scalar_one_or_none() - if not instance: - raise HTTPException(status_code=400, detail="数据源不存在") + """SQL 审核接口（独立调用）。""" + # 验证实例存在 + instance = await _get_instance(db, data.instance_id) env = "production" if instance.env == "production" else "development" db_type = instance.db_type or "mysql" - # 使用 V2 引擎进行审核（支持数据库加载规则配置） + # 使用 V2 引擎进行审核 v2_engine = SQLAuditEngineV2(db_session=db, env=env) - report = await v2_engine.audit(data.sql_content, db_type=db_type) + try: + report = await v2_engine.audit(data.sql_content, db_type=db_type) + except Exception as e: + logger.error("SQL 审核异常: %s", e) + raise HTTPException(status_code=500, detail=f"审核引擎异常: {e}") return { "score": report.score, "passed": report.passed, "has_warnings": report.has_warnings, - "summary": report.summary, + "risk_level": getattr(report, 'risk_level', 'unknown'), + "summary": getattr(report, 'summary', ''), "findings": [ - {"rule": f.rule_code, "level": f.level.value, "message": f.message} + { + "rule": getattr(f, 'rule_code', getattr(f, 'rule', 'unknown')), + "level": f.level.value, + "message": f.message, + "risk_level": getattr(f, 'risk_level', 'medium'), + "suggestion": getattr(f, 'suggestion', ''), + } for f in report.findings ], } @@ -254,11 +478,10 @@ async def get_ticket( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """获取工单详情。""" + ticket = await _get_ticket(db, ticket_id) + # 权限检查: developer 只能查看自己的工单 if current_user.role == "developer" and ticket.applicant_id != current_user.id: raise HTTPException(status_code=403, detail="无权查看该工单") @@ -271,11 +494,12 @@ async def get_ticket_approvals( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - """获取工单的审批进度（多级审批流）""" - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """获取工单的审批进度（多级审批流）。""" + ticket = await _get_ticket(db, ticket_id) + + # 权限检查 + if current_user.role == "developer" and ticket.applicant_id != current_user.id: + raise HTTPException(status_code=403, detail="无权查看该工单") approvals_result = await db.execute( select(TicketApproval) @@ -307,16 +531,23 @@ async def review_ticket( db: AsyncSession = Depends(get_db), current_user: User = Depends(require_role("admin", "dba")), ): - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """审核工单。 - if ticket.status not in ("pending", "reviewing"): - raise HTTPException(status_code=400, detail=f"工单状态为 {ticket.status}，无法审核") + 支持多级审批流: + - 通过: 按顺序审批，所有步骤通过后状态变为 approved + - 拒绝: 立即拒绝整个工单 + """ + ticket = await _get_ticket(db, ticket_id) + + # 状态机验证: 只能审批 pending 或 reviewing 状态的工单 + if ticket.status not in (TicketStatus.PENDING.value, TicketStatus.REVIEWING.value): + raise HTTPException( + status_code=400, + detail=f"工单状态为 {ticket.status}，无法审核。只能审批 pending/reviewing 状态的工单" + ) if data.action == "approve": - # Check for pending workflow steps (multi-step approval) + # 检查多级审批流 pending_approvals_result = await db.execute( select(TicketApproval) .where(TicketApproval.ticket_id == ticket_id) @@ -325,7 +556,7 @@ async def review_ticket( pending_approvals = pending_approvals_result.scalars().all() if pending_approvals: - # Multi-step workflow: find the current pending step + # 多级审批: 找到当前待审批步骤 current_step = None for pa in pending_approvals: if pa.action == "pending": @@ -333,30 +564,29 @@ async def review_ticket( break if current_step: - # Approve current step + # 审批当前步骤 current_step.action = "approved" current_step.approver_id = current_user.id current_step.comment = data.comment or "审批通过" - # Check if there's a next pending step + # 检查是否还有后续待审批步骤 has_next = any( pa.action == "pending" for pa in pending_approvals if pa.id != current_step.id ) if has_next: - ticket.status = "reviewing" + new_status = TicketStatus.REVIEWING.value else: - ticket.status = "approved" + new_status = TicketStatus.APPROVED.value else: - # All steps already processed - ticket.status = "approved" + # 所有步骤已处理 + new_status = TicketStatus.APPROVED.value else: - # No workflow steps — single-step approval (backward compat) - ticket.status = "approved" + # 无审批流 - 单步审批 + new_status = TicketStatus.APPROVED.value else: - # Reject: reject the whole ticket immediately - # Mark all pending approval steps as rejected + # 拒绝: 标记所有待审批步骤为拒绝 pending_approvals_result = await db.execute( select(TicketApproval) .where(TicketApproval.ticket_id == ticket_id, TicketApproval.action == "pending") @@ -366,8 +596,16 @@ async def review_ticket( pa.approver_id = current_user.id pa.comment = data.comment or "审批拒绝" - ticket.status = "rejected" + new_status = TicketStatus.REJECTED.value + # 验证状态转换合法性 + if not _validate_status_transition(ticket.status, new_status): + raise HTTPException( + status_code=400, + detail=f"非法的状态转换: {ticket.status} -> {new_status}" + ) + + ticket.status = new_status ticket.approver_id = current_user.id ticket.review_time = datetime.now(timezone.utc) ticket.audit_result = data.comment @@ -375,7 +613,7 @@ async def review_ticket( await AuditLogService.log( db, user_id=current_user.id, action="REVIEW_TICKET", target_type="ticket", target_id=ticket.id, - detail=f"审核动作: {data.action}, 备注: {data.comment}", + detail=f"审核动作: {data.action}, 备注: {data.comment}, 新状态: {new_status}", ) await db.commit() @@ -389,16 +627,13 @@ async def get_ticket_timeline( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """获取工单时间线。""" + ticket = await _get_ticket(db, ticket_id) if current_user.role == "developer" and ticket.applicant_id != current_user.id: raise HTTPException(status_code=403, detail="无权查看该工单") - # Fetch audit logs for this ticket - from app.models.audit_log import AuditLog + # 获取审计日志 log_stmt = ( select(AuditLog) .where(AuditLog.target_type == "ticket", AuditLog.target_id == ticket_id) @@ -409,7 +644,7 @@ async def get_ticket_timeline( timeline = [] - # Creation event + # 创建事件 applicant_name = "未知" if ticket.applicant_id: r = await db.execute(select(User).where(User.id == ticket.applicant_id)) @@ -425,10 +660,10 @@ async def get_ticket_timeline( "detail": "提交 SQL 变更工单", }) - # From audit logs + # 审计日志事件 for log in logs: if log.action == "CREATE_TICKET": - continue # already handled + continue r = await db.execute(select(User).where(User.id == log.user_id)) u = r.scalar_one_or_none() uname = u.display_name if u else "未知" @@ -447,7 +682,7 @@ async def get_ticket_timeline( "detail": log.detail, }) - # Status-based synthetic events if no log entries cover them + # 基于状态的合成事件 if ticket.review_time: has_review = any(t["action"] == "REVIEW_TICKET" for t in timeline) if not has_review: @@ -485,21 +720,40 @@ async def cancel_ticket( db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """取消工单。 - # Only applicant or admin can cancel + 只有申请人或管理员可以取消 pending/reviewing 状态的工单。 + """ + ticket = await _get_ticket(db, ticket_id) + + # 权限检查 if ticket.applicant_id != current_user.id and current_user.role != "admin": raise HTTPException(status_code=403, detail="只有申请人或管理员可以取消工单") - if ticket.status not in ("pending", "reviewing"): + # 状态检查 + if ticket.status not in (TicketStatus.PENDING.value, TicketStatus.REVIEWING.value): raise HTTPException(status_code=400, detail="只能取消待审批或审批中的工单") - ticket.status = "rejected" + # 状态转换 + new_status = TicketStatus.REJECTED.value + if not _validate_status_transition(ticket.status, new_status): + raise HTTPException( + status_code=400, + detail=f"非法的状态转换: {ticket.status} -> {new_status}" + ) + + ticket.status = new_status ticket.audit_result = "工单被取消" + # 更新审批步骤状态 + pending_approvals_result = await db.execute( + select(TicketApproval) + .where(TicketApproval.ticket_id == ticket_id, TicketApproval.action == "pending") + ) + for pa in pending_approvals_result.scalars().all(): + pa.action = "cancelled" + pa.comment = "工单被取消" + await AuditLogService.log( db, user_id=current_user.id, action="CANCEL_TICKET", target_type="ticket", target_id=ticket.id, @@ -517,24 +771,32 @@ async def rollback_ticket( db: AsyncSession = Depends(get_db), current_user: User = Depends(require_role("admin", "dba")), ): - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + """回滚工单。 - if ticket.status not in ("success", "failed"): + 只能回滚 success 或 failed 状态的工单。 + """ + ticket = await _get_ticket(db, ticket_id) + + if ticket.status not in (TicketStatus.SUCCESS.value, TicketStatus.FAILED.value): raise HTTPException(status_code=400, detail="只能回滚已执行（成功或失败）的工单") - # Simplified rollback — return info message + # 状态转换 + new_status = TicketStatus.ROLLBACK.value + if not _validate_status_transition(ticket.status, new_status): + raise HTTPException( + status_code=400, + detail=f"非法的状态转换: {ticket.status} -> {new_status}" + ) + rollback_sql = ticket.rollback_sql or "未提供回滚 SQL" await AuditLogService.log( db, user_id=current_user.id, action="ROLLBACK_TICKET", target_type="ticket", target_id=ticket.id, - detail="请求回滚工单", + detail=f"请求回滚工单，回滚SQL: {rollback_sql[:200]}", ) - ticket.status = "rollback" + ticket.status = new_status await db.commit() await db.refresh(ticket) @@ -549,49 +811,61 @@ async def rollback_ticket( @router.post("/{ticket_id}/execute") async def execute_ticket( ticket_id: int, + confirm: Optional[TicketExecuteConfirm] = None, db: AsyncSession = Depends(get_db), current_user: User = Depends(require_role("admin", "dba")), ): """执行工单。 使用多数据库工单执行引擎，按 db_type 自动分发到对应执行器。 - 支持 MySQL、PostgreSQL、Redis、MongoDB。 - 执行流程包括：语法预校验、执行前快照、执行变更、执行后验证、生成报告。 + + 执行流程: + 1. 验证工单状态为 approved + 2. 二次确认（可选） + 3. 更新状态为 executing + 4. 执行变更 + 5. 更新结果和状态 + 6. 记录审计日志 + 7. 发送通知 Args: ticket_id: 工单 ID + confirm: 二次确认参数 db: 数据库会话 current_user: 当前用户（需要 admin 或 dba 角色） Returns: 执行后的工单详情，包含执行报告 - - Raises: - HTTPException: 工单不存在、状态不正确、数据源不存在等 """ # 1. 查询工单 - result = await db.execute(select(Ticket).where(Ticket.id == ticket_id)) - ticket = result.scalar_one_or_none() - if not ticket: - raise HTTPException(status_code=404, detail="工单不存在") + ticket = await _get_ticket(db, ticket_id) - if ticket.status != "approved": + # 2. 状态检查 + if ticket.status != TicketStatus.APPROVED.value: raise HTTPException(status_code=400, detail="只能执行已审批的工单") - # 2. 查询数据源实例 - inst_result = await db.execute( - select(Instance).where(Instance.id == ticket.instance_id) - ) - instance = inst_result.scalar_one_or_none() - if not instance: - raise HTTPException(status_code=400, detail="数据源不存在") + # 3. 二次确认检查 + if confirm and not confirm.confirmed: + raise HTTPException(status_code=400, detail="未确认执行，操作已取消") - # 3. 更新工单状态为执行中 - ticket.status = "executing" + # 4. 查询数据源实例 + instance = await _get_instance(db, ticket.instance_id) + + # 5. 权限检查 + await check_instance_permission(current_user, instance, "ticket:execute", db) + + # 6. 状态转换: approved -> executing + if not _validate_status_transition(ticket.status, TicketStatus.EXECUTING.value): + raise HTTPException( + status_code=400, + detail=f"非法的状态转换: {ticket.status} -> {TicketStatus.EXECUTING.value}" + ) + + ticket.status = TicketStatus.EXECUTING.value await db.commit() await db.refresh(ticket) - # 4. 使用新的多数据库执行引擎 + # 7. 使用多数据库执行引擎 from app.services.ticket_executor import TicketExecutor executor = TicketExecutor() @@ -602,22 +876,30 @@ async def execute_ticket( # 执行工单 report = await executor.execute(ticket, instance) - # 5. 更新工单状态和结果 + # 8. 更新工单状态和结果 if report.status == "success": - ticket.status = "success" + new_status = TicketStatus.SUCCESS.value else: - ticket.status = "failed" + new_status = TicketStatus.FAILED.value + + # 验证状态转换 + if _validate_status_transition(TicketStatus.EXECUTING.value, new_status): + ticket.status = new_status + else: + ticket.status = TicketStatus.FAILED.value ticket.execute_result = report.to_json() ticket.execute_time = datetime.now(timezone.utc) - # 6. 记录执行日志 + # 9. 记录执行日志 log_detail = ( f"执行结果: {report.status}, 影响行数: {report.affected_rows}, " f"耗时: {report.duration_ms}ms, 验证通过: {report.validation_passed}" ) if report.errors: log_detail += f", 错误: {'; '.join(report.errors[:3])}" + if confirm and confirm.comment: + log_detail += f", 执行备注: {confirm.comment}" await AuditLogService.log( db, @@ -628,16 +910,20 @@ async def execute_ticket( detail=log_detail, ) - # 7. 发送通知（执行完成后） + # 10. 发送通知 try: await _send_execution_notification(db, ticket, report) - except Exception: + except Exception as notify_err: + logger.error("执行通知发送失败: %s", notify_err) # 通知失败不应影响执行结果 - pass except Exception as e: - ticket.status = "failed" - ticket.execute_result = str(e) + logger.exception("工单 %d 执行异常", ticket_id) + ticket.status = TicketStatus.FAILED.value + ticket.execute_result = json.dumps({ + "status": "failed", + "errors": [f"{type(e).__name__}: {e}"], + }, ensure_ascii=False) ticket.execute_time = datetime.now(timezone.utc) await AuditLogService.log( @@ -649,7 +935,7 @@ async def execute_ticket( detail=f"执行异常: {type(e).__name__}: {e}", ) - # 8. 提交事务并返回 + # 11. 提交事务并返回 await db.commit() await db.refresh(ticket) @@ -658,7 +944,6 @@ async def execute_ticket( # 尝试解析执行报告 try: - import json exec_report = json.loads(ticket.execute_result) if ticket.execute_result else {} if isinstance(exec_report, dict): from app.schemas.execution import ExecutionReportResponse @@ -674,7 +959,6 @@ async def execute_ticket( after_snapshot_summary=exec_report.get("after_snapshot"), ) except Exception: - # 如果解析失败，不返回结构化报告 ticket_data["execution_report"] = None return ticket_data @@ -736,7 +1020,7 @@ async def _send_execution_notification( "duration": report.duration_ms, "affected_rows": report.affected_rows, "error": "; ".join(report.errors[:3]) if report.errors else "", - "operator": "dba", # 执行人角色 + "operator": "dba", } await sender.notify_ticket_event( diff --git a/backend/app/models/workflow.py b/backend/app/models/workflow.py index 689c803..d24989f 100644 --- a/backend/app/models/workflow.py +++ b/backend/app/models/workflow.py @@ -1,5 +1,8 @@ -"""审批流程模型 — 支持多级、并行、条件审批。""" -from sqlalchemy import Column, Integer, String, Text, Boolean, DateTime, ForeignKey, func +"""审批流程模型 — 支持多级、并行、条件审批、版本控制与执行日志。""" +import json +from sqlalchemy import ( + Column, Integer, String, Text, Boolean, DateTime, ForeignKey, JSON, func, UniqueConstraint +) from sqlalchemy.orm import relationship from app.core.database import Base @@ -7,49 +10,353 @@ from app.core.database import Base class WorkflowDefinition(Base): """审批流程定义模板。""" + __tablename__ = "workflow_definitions" id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String(100), nullable=False) description = Column(Text, nullable=True) + category = Column(String(50), default="ticket") # ticket / query / instance / general is_default = Column(Boolean, default=False) is_active = Column(Boolean, default=True) created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(Integer, ForeignKey("users.id"), nullable=True) - steps = relationship("WorkflowStep", back_populates="workflow", - order_by="WorkflowStep.step_number", - cascade="all, delete-orphan") + # ═══════════════════════════════════════════════════════════ + # 流程级配置（JSON格式，灵活扩展） + # ═══════════════════════════════════════════════════════════ + + # 全局流程配置: + # { + # "timeout_strategy": "auto_approve|auto_reject|escalate|notify", + # "timeout_hours": 48, + # "escalation_user_id": 1, + # "allow_transfer": true, + # "allow_skip": false, + # "require_comment": true, + # "parallel_mode": "all|any", // 并行审批模式：全部通过/任一通过 + # "notify_channels": ["email", "sms", "webhook"] + # } + flow_config = Column(JSON, nullable=True, default=dict) + + # 触发条件配置（支持多条件组合）: + # { + # "conditions": [ + # {"field": "ticket.type", "operator": "in", "value": ["DDL", "DML"]}, + # {"field": "ticket.risk_level", "operator": ">=", "value": "high"}, + # {"field": "database.instance_id", "operator": "==", "value": 1} + # ], + # "logic": "AND|OR" + # } + trigger_conditions = Column(JSON, nullable=True) + + # 步骤定义关联 + steps = relationship( + "WorkflowStep", + back_populates="workflow", + order_by="WorkflowStep.step_number", + cascade="all, delete-orphan" + ) + + # 版本关联 + versions = relationship( + "WorkflowVersion", + back_populates="workflow", + order_by="desc(WorkflowVersion.version_number)", + cascade="all, delete-orphan" + ) + + # 审批实例关联 + approvals = relationship( + "TicketApproval", + back_populates="workflow_def", + lazy="dynamic" + ) + + # 执行日志关联 + execution_logs = relationship( + "WorkflowExecutionLog", + back_populates="workflow", + lazy="dynamic", + order_by="desc(WorkflowExecutionLog.created_at)" + ) + + def get_current_version(self): + """获取当前激活的版本。""" + return next((v for v in self.versions if v.is_active), None) + + def to_dict(self, include_steps: bool = False) -> dict: + data = { + "id": self.id, + "name": self.name, + "description": self.description, + "category": self.category, + "is_default": self.is_default, + "is_active": self.is_active, + "flow_config": self.flow_config, + "trigger_conditions": self.trigger_conditions, + "step_count": len(self.steps) if self.steps else 0, + "version_count": len(self.versions) if self.versions else 0, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + if include_steps: + data["steps"] = [s.to_dict() for s in (self.steps or [])] + return data + + +class WorkflowVersion(Base): + """审批流程版本控制 — 支持流程的版本管理与回溯。""" + + __tablename__ = "workflow_versions" + __table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uix_workflow_version"),) + + id = Column(Integer, primary_key=True, autoincrement=True) + workflow_id = Column(Integer, ForeignKey("workflow_definitions.id", ondelete="CASCADE"), nullable=False) + version_number = Column(Integer, nullable=False) # 版本号，递增 + version_name = Column(String(100), nullable=True) # 版本名称，如 "v2.0-添加并行审批" + is_active = Column(Boolean, default=False) # 是否为当前激活版本 + + # 版本快照 — 存储此版本完整的流程定义快照 + steps_snapshot = Column(JSON, nullable=False) + flow_config_snapshot = Column(JSON, nullable=True) + + # 变更说明 + change_summary = Column(Text, nullable=True) + created_by = Column(Integer, ForeignKey("users.id"), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + + # 关联 + workflow = relationship("WorkflowDefinition", back_populates="versions") + + def to_dict(self) -> dict: + return { + "id": self.id, + "workflow_id": self.workflow_id, + "version_number": self.version_number, + "version_name": self.version_name, + "is_active": self.is_active, + "change_summary": self.change_summary, + "steps_count": len(self.steps_snapshot) if self.steps_snapshot else 0, + "created_at": self.created_at.isoformat() if self.created_at else None, + } class WorkflowStep(Base): - """审批步骤定义。""" + """审批步骤定义 — 支持灵活的条件判断与动态审批人分配。""" + __tablename__ = "workflow_steps" id = Column(Integer, primary_key=True, autoincrement=True) - workflow_id = Column(Integer, ForeignKey("workflow_definitions.id"), nullable=False) + workflow_id = Column(Integer, ForeignKey("workflow_definitions.id", ondelete="CASCADE"), nullable=False) step_number = Column(Integer, nullable=False) name = Column(String(100), nullable=False) - step_type = Column(String(20), nullable=False, default="serial") # serial / parallel / any_one - approver_role = Column(String(20), nullable=True) # admin / dba / manager - approver_ids = Column(Text, nullable=True) # JSON list of specific user ids + description = Column(Text, nullable=True) + + # 步骤类型: serial(串行) / parallel(并行-会签) / any_one(或签) / condition(条件分支) + step_type = Column(String(20), nullable=False, default="serial") + + # 审批人角色 + approver_role = Column(String(20), nullable=True) # admin / dba / manager / creator + + # 指定审批人ID列表（JSON数组） + approver_ids = Column(JSON, nullable=True) + + # 审批人分配策略: + # { + # "type": "role|specific|leader|load_balance|creator|expression", + # "expression": "${ticket.creator.manager_id}", // 动态表达式 + # "fallback_user_id": 1, // 无法解析时的 fallback + # "department_id": 2 // 指定部门 + # } + approver_assignment = Column(JSON, nullable=True, default=dict) + is_required = Column(Boolean, default=True) timeout_hours = Column(Integer, default=24) auto_approve_on_timeout = Column(Boolean, default=False) + # ═══════════════════════════════════════════════════════════ + # 条件判断配置（条件步骤使用） + # ═══════════════════════════════════════════════════════════ + + # 步骤准入条件（满足条件才进入此步骤）: + # { + # "conditions": [ + # {"field": "ticket.sql_type", "operator": "in", "value": ["ALTER", "DROP"]}, + # {"field": "affected_rows", "operator": ">", "value": 1000} + # ], + # "logic": "AND" + # } + entry_conditions = Column(JSON, nullable=True) + + # 步骤流转条件（满足条件才进入下一步，否则跳过后续步骤）: + # { + # "conditions": [ + # {"field": "current.result", "operator": "==", "value": "approved"} + # ], + # "on_false_action": "skip|reject|goto_step", + # "goto_step_number": 3 + # } + transition_conditions = Column(JSON, nullable=True) + + # ═══════════════════════════════════════════════════════════ + # 字段权限配置 + # ═══════════════════════════════════════════════════════════ + + # 此步骤中可查看/编辑的字段: + # { + # "visible_fields": ["ticket.title", "ticket.sql", "ticket.description"], + # "editable_fields": ["ticket.comment"], + # "masked_fields": ["ticket.password", "ticket.secret_key"] + # } + field_permissions = Column(JSON, nullable=True) + workflow = relationship("WorkflowDefinition", back_populates="steps") + def to_dict(self) -> dict: + return { + "id": self.id, + "workflow_id": self.workflow_id, + "step_number": self.step_number, + "name": self.name, + "description": self.description, + "step_type": self.step_type, + "approver_role": self.approver_role, + "approver_ids": self.approver_ids, + "approver_assignment": self.approver_assignment, + "is_required": self.is_required, + "timeout_hours": self.timeout_hours, + "auto_approve_on_timeout": self.auto_approve_on_timeout, + "entry_conditions": self.entry_conditions, + "transition_conditions": self.transition_conditions, + "field_permissions": self.field_permissions, + } + class TicketApproval(Base): """工单审批实例（运行时记录）。""" + __tablename__ = "ticket_approvals" id = Column(Integer, primary_key=True, autoincrement=True) - ticket_id = Column(Integer, ForeignKey("tickets.id"), nullable=False) + ticket_id = Column(Integer, ForeignKey("tickets.id"), nullable=False, index=True) + + # 关联的流程定义和版本 + workflow_id = Column(Integer, ForeignKey("workflow_definitions.id"), nullable=True) + version_id = Column(Integer, ForeignKey("workflow_versions.id"), nullable=True) + step_number = Column(Integer, nullable=False) step_name = Column(String(100), nullable=True) + step_type = Column(String(20), default="serial") # serial / parallel / any_one / condition + + # 审批人信息 approver_id = Column(Integer, ForeignKey("users.id"), nullable=True) approver_role = Column(String(20), nullable=True) - action = Column(String(20), default="pending") # pending / approved / rejected / transferred / skipped + + # 实际执行人（转交场景） + actual_approver_id = Column(Integer, ForeignKey("users.id"), nullable=True) + + # 审批动作: pending / approved / rejected / transferred / skipped / auto_approved + action = Column(String(20), default="pending", index=True) comment = Column(Text, nullable=True) + + # 转交信息 + transfer_from = Column(Integer, ForeignKey("users.id"), nullable=True) + transfer_reason = Column(Text, nullable=True) + + # 审批时间记录 created_at = Column(DateTime, server_default=func.now()) updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + completed_at = Column(DateTime, nullable=True) + + # 超时标记 + is_timeout = Column(Boolean, default=False) + timeout_handled_at = Column(DateTime, nullable=True) + + # 关联 + workflow_def = relationship("WorkflowDefinition", back_populates="approvals") + approver = relationship("User", foreign_keys=[approver_id]) + actual_approver = relationship("User", foreign_keys=[actual_approver_id]) + + def to_dict(self) -> dict: + return { + "id": self.id, + "ticket_id": self.ticket_id, + "workflow_id": self.workflow_id, + "step_number": self.step_number, + "step_name": self.step_name, + "step_type": self.step_type, + "approver_id": self.approver_id, + "approver_role": self.approver_role, + "approver_name": self.approver.display_name if self.approver else None, + "actual_approver_id": self.actual_approver_id, + "action": self.action, + "comment": self.comment, + "transfer_from": self.transfer_from, + "transfer_reason": self.transfer_reason, + "is_timeout": self.is_timeout, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + "completed_at": self.completed_at.isoformat() if self.completed_at else None, + } + + +class WorkflowExecutionLog(Base): + """流程执行日志 — 记录审批流程的完整执行轨迹，用于审计和问题排查。""" + + __tablename__ = "workflow_execution_logs" + + id = Column(Integer, primary_key=True, autoincrement=True) + workflow_id = Column(Integer, ForeignKey("workflow_definitions.id"), nullable=False, index=True) + ticket_id = Column(Integer, ForeignKey("tickets.id"), nullable=True, index=True) + step_id = Column(Integer, ForeignKey("workflow_steps.id"), nullable=True) + approval_id = Column(Integer, ForeignKey("ticket_approvals.id"), nullable=True) + + # 事件类型: + # workflow_started / step_entered / step_completed / approved / rejected / + # transferred / skipped / timeout / escalated / workflow_completed / workflow_cancelled + event_type = Column(String(50), nullable=False, index=True) + + # 事件详情 + event_description = Column(Text, nullable=True) + + # 执行人 + actor_id = Column(Integer, ForeignKey("users.id"), nullable=True) + actor_name = Column(String(100), nullable=True) + + # 上下文数据（快照）: + # { + # "step_snapshot": {...}, + # "approval_snapshot": {...}, + # "decision_context": {...}, + # "condition_results": [...] + # } + context_data = Column(JSON, nullable=True) + + created_at = Column(DateTime, server_default=func.now(), index=True) + + # 关联 + workflow = relationship("WorkflowDefinition", back_populates="execution_logs") + + def to_dict(self) -> dict: + return { + "id": self.id, + "workflow_id": self.workflow_id, + "ticket_id": self.ticket_id, + "step_id": self.step_id, + "approval_id": self.approval_id, + "event_type": self.event_type, + "event_description": self.event_description, + "actor_id": self.actor_id, + "actor_name": self.actor_name, + "context_data": self.context_data, + "created_at": self.created_at.isoformat() if self.created_at else None, + } + + def __repr__(self) -> str: + return ( + f"" + ) diff --git a/backend/app/services/audit_engine.py b/backend/app/services/audit_engine.py index 739cebd..1b16288 100644 --- a/backend/app/services/audit_engine.py +++ b/backend/app/services/audit_engine.py @@ -4,11 +4,23 @@ Checks SQL against configurable rules and returns structured results: - level: "error" (must fix), "warning" (should review), "info" (FYI) - rule: rule identifier - message: human-readable description + +Supports: + - MySQL / PostgreSQL SQL audit + - Redis command audit + - MongoDB operation audit + - Dynamic rule loading from configuration + - Risk level assessment (LOW / MEDIUM / HIGH / CRITICAL) + - Weighted scoring algorithm """ +import json +import logging import re from dataclasses import dataclass, field -from typing import List, Optional from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Protocol + +logger = logging.getLogger(__name__) class Severity(str, Enum): @@ -17,17 +29,41 @@ class Severity(str, Enum): INFO = "info" +class RiskLevel(str, Enum): + """风险等级评估。""" + LOW = "low" # 低风险，一般建议 + MEDIUM = "medium" # 中风险，需要关注 + HIGH = "high" # 高风险，强烈建议修改 + CRITICAL = "critical" # 严重风险，必须修改 + + @dataclass class AuditFinding: rule: str level: Severity message: str + risk_level: RiskLevel = RiskLevel.MEDIUM # 新增：风险等级 + suggestion: str = "" # 新增：修改建议 + category: str = "general" # 新增：规则分类 + + def to_dict(self) -> dict: + return { + "rule": self.rule, + "level": self.level.value, + "message": self.message, + "risk_level": self.risk_level.value, + "suggestion": self.suggestion, + "category": self.category, + } @dataclass class AuditReport: sql: str findings: List[AuditFinding] = field(default_factory=list) + db_type: str = "mysql" + risk_level: RiskLevel = RiskLevel.LOW + risk_score: int = 0 @property def passed(self) -> bool: @@ -39,196 +75,716 @@ class AuditReport: @property def score(self) -> int: - """100 = clean, -10 per error, -5 per warning.""" + """加权评分算法: 100 = clean, error扣10-30分, warning扣5-15分, info扣1-3分。 + + 根据风险等级调整扣分权重： + - CRITICAL: error扣30分 + - HIGH: error扣20分, warning扣15分 + - MEDIUM: error扣10分, warning扣5分 + - LOW: info扣1-3分 + """ s = 100 for f in self.findings: - if f.level == Severity.ERROR: - s -= 10 - elif f.level == Severity.WARNING: - s -= 5 + if f.risk_level == RiskLevel.CRITICAL: + if f.level == Severity.ERROR: + s -= 30 + elif f.level == Severity.WARNING: + s -= 15 + elif f.level == Severity.INFO: + s -= 5 + elif f.risk_level == RiskLevel.HIGH: + if f.level == Severity.ERROR: + s -= 20 + elif f.level == Severity.WARNING: + s -= 10 + elif f.level == Severity.INFO: + s -= 3 + elif f.risk_level == RiskLevel.MEDIUM: + if f.level == Severity.ERROR: + s -= 10 + elif f.level == Severity.WARNING: + s -= 5 + elif f.level == Severity.INFO: + s -= 2 + else: # LOW + if f.level == Severity.ERROR: + s -= 5 + elif f.level == Severity.WARNING: + s -= 3 + elif f.level == Severity.INFO: + s -= 1 return max(0, s) def to_dict(self) -> dict: return { "passed": self.passed, "score": self.score, - "findings": [ - {"rule": f.rule, "level": f.level.value, "message": f.message} - for f in self.findings - ], + "db_type": self.db_type, + "risk_level": self.risk_level.value, + "risk_score": self.risk_score, + "findings": [f.to_dict() for f in self.findings], } + def calculate_risk_level(self) -> RiskLevel: + """根据发现的问题计算整体风险等级。""" + if any(f.level == Severity.ERROR and f.risk_level == RiskLevel.CRITICAL for f in self.findings): + return RiskLevel.CRITICAL + if any(f.level == Severity.ERROR and f.risk_level == RiskLevel.HIGH for f in self.findings): + return RiskLevel.HIGH + if any(f.level == Severity.ERROR for f in self.findings): + return RiskLevel.HIGH + critical_warnings = sum( + 1 for f in self.findings + if f.level == Severity.WARNING and f.risk_level in (RiskLevel.CRITICAL, RiskLevel.HIGH) + ) + if critical_warnings >= 2: + return RiskLevel.HIGH + total_warnings = sum(1 for f in self.findings if f.level == Severity.WARNING) + if total_warnings >= 3: + return RiskLevel.MEDIUM + if self.findings: + return RiskLevel.LOW + return RiskLevel.LOW + + +# ===== 规则配置协议 ===== + + +class AuditRule(Protocol): + """审核规则协议。""" + + rule_code: str + category: str + severity: Severity + risk_level: RiskLevel + description: str + + def check(self, content: str, upper: str, report: AuditReport) -> None: + """执行规则检查。""" + ... + + +# ===== 具体规则类 ===== + + +@dataclass +class SQLAuditRule: + """SQL审核规则基类。""" + rule_code: str + category: str + severity: Severity + risk_level: RiskLevel + description: str + suggestion: str = "" + enabled: bool = True + + def check(self, content: str, upper: str, report: AuditReport) -> None: + pass + + +class PatternAuditRule(SQLAuditRule): + """基于正则表达式模式的审核规则。""" + + def __init__( + self, + rule_code: str, + category: str, + severity: Severity, + risk_level: RiskLevel, + description: str, + suggestion: str, + pattern: str, + exclude_pattern: Optional[str] = None, + ): + super().__init__( + rule_code=rule_code, + category=category, + severity=severity, + risk_level=risk_level, + description=description, + suggestion=suggestion, + ) + self.pattern = re.compile(pattern, re.IGNORECASE) + self.exclude_pattern = re.compile(exclude_pattern, re.IGNORECASE) if exclude_pattern else None + + def check(self, content: str, upper: str, report: AuditReport) -> None: + if not self.enabled: + return + if self.pattern.search(upper): + if self.exclude_pattern and self.exclude_pattern.search(upper): + return + report.findings.append(AuditFinding( + rule=self.rule_code, + level=self.severity, + message=self.description, + risk_level=self.risk_level, + suggestion=self.suggestion, + category=self.category, + )) + + +# ===== MySQL 审核规则集 ===== + +MYSQL_AUDIT_RULES: List[SQLAuditRule] = [ + PatternAuditRule( + rule_code="DROP_TABLE_DANGER", + category="ddl", + severity=Severity.WARNING, + risk_level=RiskLevel.CRITICAL, + description="DROP TABLE 操作不可逆，请确认已备份", + suggestion="使用 DROP TABLE IF EXISTS 并在操作前备份数据", + pattern=r"\bDROP\s+TABLE\b", + ), + PatternAuditRule( + rule_code="TRUNCATE_DANGER", + category="ddl", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="TRUNCATE 会清空表数据且不可回滚", + suggestion="使用 DELETE 配合 WHERE 子句替代，或使用事务", + pattern=r"\bTRUNCATE\b", + ), + PatternAuditRule( + rule_code="UPDATE_NO_WHERE", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="UPDATE 缺少 WHERE 条件，将更新全表", + suggestion="添加 WHERE 条件限制更新范围", + pattern=r"\bUPDATE\s+\w+\s+SET\b(?!.*\bWHERE\b)", + ), + PatternAuditRule( + rule_code="DELETE_NO_WHERE", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="DELETE 缺少 WHERE 条件，将删除全表数据", + suggestion="添加 WHERE 条件限制删除范围", + pattern=r"\bDELETE\s+FROM\s+\w+\b(?!.*\bWHERE\b)", + ), + PatternAuditRule( + rule_code="SELECT_STAR", + category="query", + severity=Severity.INFO, + risk_level=RiskLevel.LOW, + description="SELECT * 可能返回过多列，建议明确指定字段", + suggestion="明确指定需要的字段名，减少网络传输和内存使用", + pattern=r"SELECT\s+\*(?:\s|,|\))", + ), + PatternAuditRule( + rule_code="DML_WITH_LIMIT", + category="dml", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="UPDATE/DELETE 使用 LIMIT 可能导致部分数据变更", + suggestion="使用 WHERE 条件精确控制影响范围，避免 LIMIT", + pattern=r"\b(UPDATE|DELETE)\b.*\bLIMIT\b", + ), + PatternAuditRule( + rule_code="LARGE_IN_CLAUSE", + category="query", + severity=Severity.WARNING, + risk_level=RiskLevel.MEDIUM, + description="IN 子句包含过多值，性能较差", + suggestion="使用临时表或 JOIN 替代大 IN 子句", + pattern=r"\bIN\s*\([^)]{500,}\)", + ), + PatternAuditRule( + rule_code="FULL_LIKE_SCAN", + category="query", + severity=Severity.INFO, + risk_level=RiskLevel.LOW, + description="LIKE '%...' 无法使用索引，大数据量下性能较差", + suggestion="考虑使用全文索引或前缀匹配 LIKE 'prefix%'", + pattern=r"\bLIKE\s+'%[^']*'", + ), + PatternAuditRule( + rule_code="JOIN_NO_ON", + category="query", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="JOIN 缺少 ON 条件，可能产生笛卡尔积", + suggestion="添加 ON 条件明确连接关系", + pattern=r"\bJOIN\b", + exclude_pattern=r"\b(NATURAL\s+JOIN|ON\b|CROSS\s+JOIN)", + ), + PatternAuditRule( + rule_code="MULTI_STATEMENT", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="单次提交不支持多条 SQL 语句（分号分隔）", + suggestion="拆分为多个工单，每条语句单独提交", + pattern=r";.*\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)", + ), + PatternAuditRule( + rule_code="SLEEP_INJECTION", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="检测到时间延迟函数（SLEEP/BENCHMARK），可能存在注入风险", + suggestion="移除时间延迟函数", + pattern=r"\b(SLEEP|BENCHMARK)\s*\(", + ), + PatternAuditRule( + rule_code="INTO_OUTFILE", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="SELECT INTO OUTFILE/DUMPFILE 被禁止（文件写入风险）", + suggestion="使用客户端工具导出数据", + pattern=r"\bINTO\s+(OUTFILE|DUMPFILE)\b", + ), + PatternAuditRule( + rule_code="PROCEDURE_CALL", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.HIGH, + description="存储过程/函数调用可能存在安全风险", + suggestion="避免使用存储过程，改用标准 SQL", + pattern=r"\b(CALL|EXEC|EXECUTE)\b", + ), + PatternAuditRule( + rule_code="UNION_INJECTION", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="检测到 UNION 注入模式", + suggestion="检查 SQL 来源合法性", + pattern=r"\bUNION\s+(ALL\s+)?SELECT\b", + ), +] + + +# ===== PostgreSQL 审核规则集 ===== + +POSTGRES_AUDIT_RULES: List[SQLAuditRule] = [ + PatternAuditRule( + rule_code="PG_DROP_TABLE", + category="ddl", + severity=Severity.WARNING, + risk_level=RiskLevel.CRITICAL, + description="DROP TABLE 操作不可逆，请确认已备份", + suggestion="使用 DROP TABLE IF EXISTS 并在操作前备份数据", + pattern=r"\bDROP\s+TABLE\b", + ), + PatternAuditRule( + rule_code="PG_TRUNCATE", + category="ddl", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="TRUNCATE 会清空表数据且不可回滚", + suggestion="使用 DELETE 配合 WHERE 子句或事务", + pattern=r"\bTRUNCATE\b", + ), + PatternAuditRule( + rule_code="PG_UPDATE_NO_WHERE", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="UPDATE 缺少 WHERE 条件，将更新全表", + suggestion="添加 WHERE 条件限制更新范围", + pattern=r"\bUPDATE\s+\w+\s+SET\b(?!.*\bWHERE\b)", + ), + PatternAuditRule( + rule_code="PG_DELETE_NO_WHERE", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="DELETE 缺少 WHERE 条件，将删除全表数据", + suggestion="添加 WHERE 条件限制删除范围", + pattern=r"\bDELETE\s+FROM\s+\w+\b(?!.*\bWHERE\b)", + ), + PatternAuditRule( + rule_code="PG_SELECT_STAR", + category="query", + severity=Severity.INFO, + risk_level=RiskLevel.LOW, + description="SELECT * 可能返回过多列", + suggestion="明确指定需要的字段名", + pattern=r"SELECT\s+\*(?:\s|,|\))", + ), + PatternAuditRule( + rule_code="PG_VACUUM_DANGER", + category="maintenance", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="VACUUM FULL 会锁定表并重建", + suggestion="在低峰期执行 VACUUM FULL，或使用普通 VACUUM", + pattern=r"\bVACUUM\s+FULL\b", + ), + PatternAuditRule( + rule_code="PG_LOCK_TABLE", + category="locking", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="LOCK TABLE 会显式锁定表，可能导致阻塞", + suggestion="使用事务隔离级别替代显式锁", + pattern=r"\bLOCK\s+TABLE\b", + ), + PatternAuditRule( + rule_code="PG_CREATE_EXTENSION", + category="ddl", + severity=Severity.WARNING, + risk_level=RiskLevel.MEDIUM, + description="CREATE EXTENSION 需要超级用户权限", + suggestion="确认扩展安全性并由 DBA 执行", + pattern=r"\bCREATE\s+EXTENSION\b", + ), + PatternAuditRule( + rule_code="PG_COPY_FROM_FILE", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="COPY FROM FILE 存在文件系统访问风险", + suggestion="使用 COPY FROM STDIN 或客户端工具导入", + pattern=r"\bCOPY\b.*\bFROM\s+(?!STDIN)", + ), + PatternAuditRule( + rule_code="PG_MULTI_STATEMENT", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="单次提交不支持多条 SQL 语句", + suggestion="拆分为多个工单提交", + pattern=r";.*\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)", + ), + PatternAuditRule( + rule_code="PG_SLEEP_INJECTION", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="检测到时间延迟函数（PG_SLEEP），可能存在注入风险", + suggestion="移除时间延迟函数", + pattern=r"\bPG_SLEEP\s*\(", + ), +] + + +# ===== Redis 审核规则集 ===== + +REDIS_AUDIT_RULES: List[SQLAuditRule] = [ + PatternAuditRule( + rule_code="REDIS_FLUSH", + category="danger", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="FLUSHDB/FLUSHALL 会清空所有数据", + suggestion="确认备份数据并在维护窗口执行", + pattern=r"\b(FLUSHDB|FLUSHALL)\b", + ), + PatternAuditRule( + rule_code="REDIS_KEY_DELETE", + category="dml", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="批量删除键（DEL KEYS）风险较高", + suggestion="使用 UNLINK 异步删除，或使用 SCAN 分批删除", + pattern=r"\bDEL\s+(?!\w+\b)", + ), + PatternAuditRule( + rule_code="REDIS_SHUTDOWN", + category="danger", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="SHUTDOWN 会关闭 Redis 服务器", + suggestion="通过运维面板操作服务器", + pattern=r"\bSHUTDOWN\b", + ), + PatternAuditRule( + rule_code="REDIS_CONFIG_SET", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="CONFIG SET 可修改服务器配置", + suggestion="通过配置文件修改并由 DBA 审核", + pattern=r"\bCONFIG\s+SET\b", + ), + PatternAuditRule( + rule_code="REDIS_REPLICAOF", + category="danger", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="REPLICAOF/SLAVEOF 会改变主从关系", + suggestion="通过运维面板管理主从配置", + pattern=r"\b(REPLICAOF|SLAVEOF)\b", + ), + PatternAuditRule( + rule_code="REDIS_DEBUG", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="DEBUG 命令可能导致数据损坏或信息泄露", + suggestion="禁止在生产环境使用 DEBUG 命令", + pattern=r"\bDEBUG\b", + ), +] + + +# ===== MongoDB 审核规则集 ===== + +MONGODB_AUDIT_RULES: List[SQLAuditRule] = [ + PatternAuditRule( + rule_code="MONGO_DROP_COLLECTION", + category="ddl", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="drop 操作会删除整个集合", + suggestion="使用 deleteMany 删除文档保留集合", + pattern=r'"operation"\s*:\s*"drop"', + ), + PatternAuditRule( + rule_code="MONGO_DROP_INDEX", + category="ddl", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="dropIndex 会影响查询性能", + suggestion="确认索引不再使用后再删除", + pattern=r'"operation"\s*:\s*"dropIndex"', + ), + PatternAuditRule( + rule_code="MONGO_DELETE_NO_FILTER", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="deleteMany 缺少 filter 将删除所有文档", + suggestion="添加 filter 条件限制删除范围", + pattern=r'"operation"\s*:\s*"deleteMany"(?!.*"filter"\s*:\s*\{[^}])', + ), + PatternAuditRule( + rule_code="MONGO_UPDATE_NO_FILTER", + category="dml", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="updateMany 缺少 filter 将更新所有文档", + suggestion="添加 filter 条件限制更新范围", + pattern=r'"operation"\s*:\s*"updateMany"(?!.*"filter"\s*:\s*\{[^}])', + ), + PatternAuditRule( + rule_code="MONGO_UNSET_FIELD", + category="dml", + severity=Severity.WARNING, + risk_level=RiskLevel.MEDIUM, + description="$unset 会删除字段，数据不可恢复", + suggestion="确认字段确实不再需要", + pattern=r'"\$unset"', + ), + PatternAuditRule( + rule_code="MONGO_RENAME_COLLECTION", + category="ddl", + severity=Severity.WARNING, + risk_level=RiskLevel.HIGH, + description="renameCollection 会重命名集合", + suggestion="确认新名称不会与现有集合冲突", + pattern=r'"operation"\s*:\s*"renameCollection"', + ), + PatternAuditRule( + rule_code="MONGO_JAVASCRIPT", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="$where/$function 可执行任意 JavaScript", + suggestion="使用标准查询操作符替代", + pattern=r'"\$(where|function|accumulator)"', + ), + PatternAuditRule( + rule_code="MONGO_OUT_STAGE", + category="security", + severity=Severity.ERROR, + risk_level=RiskLevel.CRITICAL, + description="aggregate $out/$merge 阶段会写入数据", + suggestion="使用 find 查询替代", + pattern=r'"\$(out|merge)"', + ), +] + + +# ===== 规则注册表 ===== + +RULE_REGISTRY: Dict[str, List[SQLAuditRule]] = { + "mysql": MYSQL_AUDIT_RULES, + "postgres": POSTGRES_AUDIT_RULES, + "postgresql": POSTGRES_AUDIT_RULES, + "redis": REDIS_AUDIT_RULES, + "mongo": MONGODB_AUDIT_RULES, +} + class SQLAuditEngine: - """Rule-based SQL audit engine.""" + """增强的规则审核引擎。 - def __init__(self, env: str = "development"): + 支持多数据库类型的规则审核，动态规则加载，风险等级评估。 + + Attributes: + env: 环境类型（development/production） + db_type: 数据库类型 + custom_rules: 自定义规则列表 + """ + + def __init__(self, env: str = "development", db_type: str = "mysql"): self.env = env + self.db_type = db_type.lower() + self.custom_rules: List[SQLAuditRule] = [] + + def add_custom_rule(self, rule: SQLAuditRule) -> None: + """添加自定义审核规则。 + + Args: + rule: 自定义规则对象 + """ + self.custom_rules.append(rule) + logger.info("添加自定义规则: %s", rule.rule_code) + + def load_rules_from_config(self, config: List[Dict[str, Any]]) -> None: + """从配置加载自定义规则。 + + 配置格式: + [ + { + "rule_code": "RULE_NAME", + "category": "security", + "severity": "error", + "risk_level": "critical", + "description": "规则描述", + "suggestion": "修改建议", + "pattern": r"正则表达式", + "exclude_pattern": r"排除正则" (可选) + } + ] + + Args: + config: 规则配置列表 + """ + for cfg in config: + try: + severity = Severity(cfg.get("severity", "warning")) + risk_level = RiskLevel(cfg.get("risk_level", "medium")) + rule = PatternAuditRule( + rule_code=cfg["rule_code"], + category=cfg.get("category", "custom"), + severity=severity, + risk_level=risk_level, + description=cfg["description"], + suggestion=cfg.get("suggestion", ""), + pattern=cfg["pattern"], + exclude_pattern=cfg.get("exclude_pattern"), + ) + self.custom_rules.append(rule) + logger.info("从配置加载规则: %s", rule.rule_code) + except (KeyError, re.error) as e: + logger.error("加载规则配置失败: %s, error: %s", cfg.get("rule_code", "unknown"), e) def audit(self, sql: str) -> AuditReport: - report = AuditReport(sql=sql) + """执行审核。 + + 根据 db_type 选择对应的规则集，执行所有规则检查。 + + Args: + sql: 要审核的 SQL/命令/JSON + + Returns: + AuditReport: 审核报告 + """ + report = AuditReport(sql=sql, db_type=self.db_type) clean = sql.strip() upper = clean.upper() - # Rule 1: DDL without IF EXISTS - self._check_ddl_if_exists(clean, upper, report) + # 空内容检查 + if not clean: + report.findings.append(AuditFinding( + rule="EMPTY_CONTENT", + level=Severity.ERROR, + message="审核内容不能为空", + risk_level=RiskLevel.CRITICAL, + category="validation", + )) + return report - # Rule 2: DROP TABLE danger - self._check_drop_table(upper, report) + # 获取数据库类型对应的规则 + rules = RULE_REGISTRY.get(self.db_type, []) - # Rule 3: UPDATE/DELETE without WHERE - self._check_missing_where(clean, upper, report) + # 执行内置规则 + for rule in rules: + if rule.enabled: + rule.check(clean, upper, report) - # Rule 4: SELECT * usage - self._check_select_star(clean, upper, report) + # 执行自定义规则 + for rule in self.custom_rules: + if rule.enabled: + rule.check(clean, upper, report) - # Rule 5: LIMIT on DML - self._check_limit_dml(clean, upper, report) - - # Rule 6: Production env extra checks + # 环境特定规则 if self.env == "production": self._check_production_rules(clean, upper, report) - # Rule 7: Multiple statements (semicolon) - self._check_multiple_statements(clean, report) - - # Rule 8: Large IN clause - self._check_large_in(clean, upper, report) - - # Rule 9: No WHERE with LIKE '%...' - self._check_full_like(clean, upper, report) - - # Rule 10: JOIN without ON - self._check_join_without_on(upper, report) + # 计算风险等级 + report.risk_level = report.calculate_risk_level() + report.risk_score = report.score return report - def _check_ddl_if_exists(self, clean: str, upper: str, report: AuditReport): - """ALTER/DROP statements should use IF EXISTS where applicable.""" - if re.search(r"\bDROP\s+TABLE\b", upper) and "IF EXISTS" not in upper: - report.findings.append(AuditFinding( - rule="DDL_IF_EXISTS", - level=Severity.WARNING, - message="DROP TABLE 建议添加 IF EXISTS 防止表不存在时报错", - )) + def _check_production_rules(self, clean: str, upper: str, report: AuditReport) -> None: + """生产环境额外规则。 - def _check_drop_table(self, upper: str, report: AuditReport): - """DROP TABLE is always dangerous in production.""" - if re.search(r"\bDROP\s+TABLE\b", upper): - if self.env == "production": + Args: + clean: 原始内容 + upper: 大写内容 + report: 审核报告 + """ + if self.db_type in ("mysql", "postgres", "postgresql"): + # ALTER TABLE 限制 + if re.search(r"\bALTER\s+TABLE\b", upper): report.findings.append(AuditFinding( - rule="DROP_TABLE_PRODUCTION", + rule="ALTER_TABLE_PRODUCTION", level=Severity.ERROR, - message="生产环境禁止 DROP TABLE 操作", + message="生产环境 ALTER TABLE 需 DBA 特别审批", + risk_level=RiskLevel.HIGH, + suggestion="在低峰期执行，使用 pt-online-schema-change 等在线工具", + category="production", )) - else: + + # CREATE INDEX 限制 + if re.search(r"\bCREATE\s+(UNIQUE\s+)?INDEX\b", upper): report.findings.append(AuditFinding( - rule="DROP_TABLE", + rule="CREATE_INDEX_PRODUCTION", level=Severity.WARNING, - message="DROP TABLE 操作不可逆，请确认已备份", + message="生产环境创建索引需确认影响范围", + risk_level=RiskLevel.MEDIUM, + suggestion="使用 ONLINE/CONCURRENT 选项避免锁表", + category="production", )) - if re.search(r"\bTRUNCATE\b", upper): - report.findings.append(AuditFinding( - rule="TRUNCATE", - level=Severity.ERROR if self.env == "production" else Severity.WARNING, - message="TRUNCATE 会清空表数据且不可回滚", - )) - - def _check_missing_where(self, clean: str, upper: str, report: AuditReport): - """UPDATE/DELETE without WHERE clause affects all rows.""" - if re.match(r"\s*UPDATE\s+", upper, re.IGNORECASE): - if "WHERE" not in upper: + # 没有索引的 UPDATE/DELETE + if re.search(r"\b(UPDATE|DELETE)\b", upper): report.findings.append(AuditFinding( - rule="UPDATE_NO_WHERE", - level=Severity.ERROR, - message="UPDATE 缺少 WHERE 条件，将更新全表", - )) - - if re.match(r"\s*DELETE\s+FROM\s+", upper, re.IGNORECASE): - if "WHERE" not in upper: - report.findings.append(AuditFinding( - rule="DELETE_NO_WHERE", - level=Severity.ERROR, - message="DELETE 缺少 WHERE 条件，将删除全表数据", - )) - - def _check_select_star(self, clean: str, upper: str, report: AuditReport): - """SELECT * is inefficient for large tables.""" - if re.search(r"SELECT\s+\*(?:\s|,|\))", upper): - report.findings.append(AuditFinding( - rule="SELECT_STAR", - level=Severity.INFO, - message="SELECT * 可能返回过多列，建议明确指定字段", - )) - - def _check_limit_dml(self, clean: str, upper: str, report: AuditReport): - """UPDATE/DELETE with LIMIT is MySQL-specific and risky.""" - for kw in ("UPDATE", "DELETE"): - if re.match(rf"\s*{kw}\b", upper): - if "LIMIT" in upper: - report.findings.append(AuditFinding( - rule="DML_WITH_LIMIT", - level=Severity.WARNING, - message=f"{kw} 使用 LIMIT 可能导致部分数据变更，请确认意图", - )) - - def _check_production_rules(self, clean: str, upper: str, report: AuditReport): - """Extra strict rules for production.""" - if re.search(r"\bALTER\s+TABLE\b", upper): - report.findings.append(AuditFinding( - rule="ALTER_TABLE_PRODUCTION", - level=Severity.ERROR, - message="生产环境 ALTER TABLE 需 DBA 特别审批", - )) - - if re.search(r"\bCREATE\s+(TABLE|INDEX)\b", upper): - report.findings.append(AuditFinding( - rule="CREATE_DDL_PRODUCTION", - level=Severity.WARNING, - message="生产环境创建对象需确认影响范围", - )) - - def _check_multiple_statements(self, clean: str, report: AuditReport): - """Multiple SQL statements separated by semicolons.""" - # Count semicolons not at the end - stripped = clean.rstrip(";").strip() - if ";" in stripped: - report.findings.append(AuditFinding( - rule="MULTI_STATEMENT", - level=Severity.ERROR, - message="单次提交不支持多条 SQL 语句（分号分隔），请拆分为多个工单", - )) - - def _check_large_in(self, clean: str, upper: str, report: AuditReport): - """IN clause with many values is slow.""" - in_matches = re.findall(r"\bIN\s*\(([^)]+)\)", upper) - for m in in_matches: - count = m.count(",") + 1 - if count > 100: - report.findings.append(AuditFinding( - rule="LARGE_IN", + rule="DML_PRODUCTION_CAUTION", level=Severity.WARNING, - message=f"IN 子句包含 {count} 个值，建议使用临时表或 JOIN 替代", + message="生产环境 UPDATE/DELETE 需要确认 WHERE 条件有索引覆盖", + risk_level=RiskLevel.HIGH, + suggestion="使用 EXPLAIN 确认执行计划使用了索引", + category="production", )) - def _check_full_like(self, clean: str, upper: str, report: AuditReport): - """LIKE '%...' cannot use index.""" - if re.search(r"\bLIKE\s+'%[^']*'", clean, re.IGNORECASE): - report.findings.append(AuditFinding( - rule="FULL_LIKE", - level=Severity.INFO, - message="LIKE '%...' 无法使用索引，大数据量下性能较差", - )) + @staticmethod + def get_supported_rules(db_type: str) -> List[Dict[str, str]]: + """获取指定数据库类型支持的所有规则列表。 - def _check_join_without_on(self, upper: str, report: AuditReport): - """JOIN without ON produces cartesian product.""" - # Match JOIN but not followed by ON (within reasonable distance) - if re.search(r"\bJOIN\b", upper) and "ON" not in upper: - # Could be NATURAL JOIN, allow that - if not re.search(r"\bNATURAL\s+JOIN\b", upper): - report.findings.append(AuditFinding( - rule="JOIN_NO_ON", - level=Severity.WARNING, - message="JOIN 缺少 ON 条件，可能产生笛卡尔积", - )) + Args: + db_type: 数据库类型 + + Returns: + 规则信息列表 + """ + rules = RULE_REGISTRY.get(db_type.lower(), []) + return [ + { + "rule_code": r.rule_code, + "category": r.category, + "severity": r.severity.value, + "risk_level": r.risk_level.value, + "description": r.description, + "suggestion": r.suggestion, + } + for r in rules + if r.enabled + ] diff --git a/backend/app/services/query_service.py b/backend/app/services/query_service.py index 51247fc..a61b3fd 100644 --- a/backend/app/services/query_service.py +++ b/backend/app/services/query_service.py @@ -3,11 +3,12 @@ 核心功能: 1. 统一入口: 根据 db_type 自动路由到对应查询处理器 2. 安全验证: - - SQL 数据库: 使用 SQLAnalyzer 验证只读 - - Redis: 使用 READONLY_COMMANDS 白名单验证 - - MongoDB: 使用 READONLY_OPERATIONS 白名单验证 + - SQL 数据库: 使用 SQLAnalyzer 验证只读 + 增强注入防护 + - Redis: 使用 READONLY_COMMANDS 白名单 + 子命令验证 + - MongoDB: 使用 READONLY_OPERATIONS 白名单 + pipeline 阶段验证 3. 结果脱敏: 集成 DataMasker 对敏感字段脱敏 -4. 查询限制: 超时控制、行数限制 +4. 查询限制: 超时控制、行数限制、内存保护 +5. 审计日志: 完整的查询日志记录 使用方式: result = await QueryService.execute_query(instance, database, query, user_id) @@ -15,6 +16,7 @@ import asyncio import json +import logging import re import time from abc import ABC, abstractmethod @@ -26,40 +28,70 @@ from app.services.db_connector import DBConnector from app.services.redis_connector import RedisConnector from app.services.sql_analyzer import SQLAnalyzer -MAX_ROWS = 10000 -QUERY_TIMEOUT = 30 +logger = logging.getLogger(__name__) +# ===== 配置常量 ===== + +# 默认最大返回行数 +MAX_ROWS = 10000 +# 默认查询超时（秒） +QUERY_TIMEOUT = 30 +# 最大查询长度（字符） +MAX_QUERY_LENGTH = 10000 +# 最大结果集内存限制（MB） +MAX_RESULT_MEMORY_MB = 50 # ===== Redis 只读命令白名单 ===== REDIS_READONLY_COMMANDS = { # 字符串操作 - "GET", "MGET", "STRLEN", "GETRANGE", "GETBIT", + "GET", "MGET", "STRLEN", "GETRANGE", "GETBIT", "SUBSTR", # 哈希操作 - "HGET", "HGETALL", "HKEYS", "HVALS", "HLEN", "HEXISTS", "HMGET", + "HGET", "HGETALL", "HKEYS", "HVALS", "HLEN", "HEXISTS", "HMGET", "HSTRLEN", # 列表操作 - "LRANGE", "LLEN", "LINDEX", + "LRANGE", "LLEN", "LINDEX", "LPOS", # 集合操作 "SMEMBERS", "SCARD", "SISMEMBER", "SRANDMEMBER", "SUNION", "SINTER", "SDIFF", # 有序集合操作 "ZRANGE", "ZREVRANGE", "ZRANGEBYSCORE", "ZREVRANGEBYSCORE", "ZCARD", "ZSCORE", "ZRANK", "ZREVRANK", "ZCOUNT", "ZLEXCOUNT", + "ZRANGEBYLEX", "ZREVRANGEBYLEX", # 通用操作 - "EXISTS", "TTL", "PTTL", "TYPE", "DBSIZE", - "SCAN", "HSCAN", "SSCAN", "ZSCAN", "KEYS", - "INFO", "CONFIG", "MEMORY", "OBJECT", + "EXISTS", "TTL", "PTTL", "TYPE", "DBSIZE", "KEYS", + "SCAN", "HSCAN", "SSCAN", "ZSCAN", + "INFO", "CONFIG", "MEMORY", "OBJECT", "SLOWLOG", "CLIENT", "CLUSTER", + # 位图/地理位置/HyperLogLog（只读） + "BITCOUNT", "BITPOS", "GEOPOS", "GEODIST", "GEOHASH", "GEORADIUS_RO", + "GEORADIUSBYMEMBER_RO", "PFCOUNT", # 其他 - "PING", "ECHO", "TIME", + "PING", "ECHO", "TIME", "LASTSAVE", "ROLE", } # 需要特殊处理的子命令: CONFIG GET / MEMORY USAGE / OBJECT ENCODING 等 REDIS_SUBCOMMAND_ALLOWLIST = { ("CONFIG", "GET"), + ("CONFIG", "RESETSTAT"), ("MEMORY", "USAGE"), ("MEMORY", "STATS"), + ("MEMORY", "DOCTOR"), ("OBJECT", "ENCODING"), ("OBJECT", "IDLETIME"), ("OBJECT", "REFCOUNT"), + ("OBJECT", "FREQ"), + ("CLIENT", "LIST"), + ("CLIENT", "INFO"), + ("CLUSTER", "INFO"), + ("CLUSTER", "NODES"), + ("CLUSTER", "SLOTS"), + ("SLOWLOG", "GET"), + ("SLOWLOG", "LEN"), +} + +# Redis 危险命令黑名单（即使被误分类为只读也禁止） +REDIS_COMMAND_BLACKLIST = { + "FLUSHDB", "FLUSHALL", "SHUTDOWN", "DEBUG", "SAVE", "BGSAVE", + "BGREWRITEAOF", "SLAVEOF", "REPLICAOF", "SYNC", "PSYNC", + "CONFIG", "ACL", "MODULE", "COMMAND", } # ===== MongoDB 只读操作白名单 ===== @@ -77,6 +109,134 @@ MONGODB_WRITE_OPS = { "bulkWrite", } +# MongoDB aggregate 危险阶段黑名单 +MONGODB_AGGREGATE_BLACKLIST = { + "$out", "$merge", "$currentOp", "$listLocalSessions", +} + +# ===== SQL 注入防护模式 ===== + +# 危险的 SQL 注释模式（用于注释掉合法查询尾部） +SQL_COMMENT_PATTERNS = [ + r"--\s*$", # 行尾注释 + r"/\*.*\*/", # 块注释 + r"#\s*$", # MySQL 风格注释 +] + +# 危险的 UNION 注入模式 +SQL_UNION_INJECTION_PATTERN = re.compile( + r"\bUNION\s+(ALL\s+)?SELECT\b", re.IGNORECASE +) + +# 堆叠查询检测（多条语句） +SQL_STACKED_PATTERN = re.compile(r"\b(INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|TRUNCATE|EXEC|EXECUTE)\b", re.IGNORECASE) + +# 时间延迟注入检测 +SQL_TIMING_ATTACK_PATTERN = re.compile( + r"\b(BENCHMARK|SLEEP|PG_SLEEP|WAITFOR\s+DELAY|WAITFOR\s+TIME)\s*\(", + re.IGNORECASE, +) + + +class QuerySecurityError(ValueError): + """查询安全验证失败异常。""" + pass + + +class QueryTimeoutError(ValueError): + """查询超时异常。""" + pass + + +class QueryResultTooLargeError(ValueError): + """查询结果过大异常。""" + pass + + +# ===== 查询日志记录器 ===== + + +class QueryLogger: + """查询操作日志记录器。 + + 记录每次查询的完整上下文，用于安全审计和问题追溯。 + """ + + @staticmethod + def log_query( + user_id: int, + db_type: str, + database: Optional[str], + query: str, + elapsed_ms: int, + row_count: int, + is_success: bool, + error_msg: Optional[str] = None, + client_ip: Optional[str] = None, + ) -> None: + """记录查询日志。 + + Args: + user_id: 查询用户ID + db_type: 数据库类型 + database: 目标数据库 + query: 查询内容（已脱敏） + elapsed_ms: 执行耗时 + row_count: 返回行数 + is_success: 是否成功 + error_msg: 错误信息 + client_ip: 客户端IP + """ + log_data = { + "event": "query_executed", + "user_id": user_id, + "db_type": db_type, + "database": database, + "query_preview": query[:500] if query else "", + "query_length": len(query) if query else 0, + "elapsed_ms": elapsed_ms, + "row_count": row_count, + "is_success": is_success, + "client_ip": client_ip, + } + if error_msg: + log_data["error"] = error_msg[:500] + + if is_success: + logger.info("Query executed: %s", json.dumps(log_data, ensure_ascii=False, default=str)) + else: + logger.warning("Query failed: %s", json.dumps(log_data, ensure_ascii=False, default=str)) + + @staticmethod + def log_security_event( + event_type: str, + user_id: int, + db_type: str, + query: str, + reason: str, + client_ip: Optional[str] = None, + ) -> None: + """记录安全事件日志。 + + Args: + event_type: 事件类型（injection_attempt, unauthorized_command 等） + user_id: 用户ID + db_type: 数据库类型 + query: 查询内容 + reason: 阻止原因 + client_ip: 客户端IP + """ + log_data = { + "event": "security_event", + "event_type": event_type, + "user_id": user_id, + "db_type": db_type, + "query_preview": query[:500] if query else "", + "reason": reason, + "client_ip": client_ip, + } + logger.warning("Security event: %s", json.dumps(log_data, ensure_ascii=False, default=str)) + # ===== 查询处理器基类 ===== @@ -123,6 +283,41 @@ class BaseQueryHandler(ABC): """ pass + def _validate_query_length(self, query: str) -> None: + """验证查询长度是否在允许范围内。 + + Args: + query: 查询语句 + + Raises: + QuerySecurityError: 查询过长 + """ + if not query or not query.strip(): + raise QuerySecurityError("查询语句不能为空") + if len(query) > MAX_QUERY_LENGTH: + raise QuerySecurityError( + f"查询语句过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH} 字符" + ) + + def _check_result_size(self, rows: List[Dict[str, Any]]) -> bool: + """检查结果集大小是否超过内存限制。 + + Args: + rows: 结果行列表 + + Returns: + 是否超过限制 + """ + if not rows: + return False + # 估算内存使用（粗略） + try: + size_bytes = len(json.dumps(rows, default=str).encode("utf-8")) + size_mb = size_bytes / (1024 * 1024) + return size_mb > MAX_RESULT_MEMORY_MB + except (OverflowError, MemoryError): + return True + # ===== SQL 查询处理器（MySQL / PostgreSQL 共用基类） ===== @@ -144,6 +339,82 @@ class SQLQueryHandlerBase(BaseQueryHandler): is_safe, error_msg = self._analyzer.validate_readonly(query) return is_safe, error_msg + def _enhanced_security_check(self, query: str, user_id: int) -> None: + """增强安全验证 — 检测 SQL 注入攻击模式。 + + 检测项: + 1. 查询长度限制 + 2. 堆叠查询（多条语句） + 3. UNION 注入 + 4. 时间延迟攻击 + 5. 注释攻击 + 6. 危险的子查询 + + Args: + query: SQL 查询语句 + user_id: 查询用户ID + + Raises: + QuerySecurityError: 检测到安全问题 + """ + # 1. 长度验证 + self._validate_query_length(query) + + upper_query = query.upper().strip() + + # 2. 检测堆叠查询（尝试执行多条语句） + stripped = query.strip().rstrip(";").strip() + in_string = False + string_char = None + for ch in stripped: + if ch in ("'", '"') and not in_string: + in_string = True + string_char = ch + elif ch == string_char and in_string: + # 检查转义 + string_char = None + in_string = False + elif ch == ";" and not in_string: + raise QuerySecurityError( + "检测到多条 SQL 语句（分号分隔），可能存在堆叠查询注入风险" + ) + + # 3. 检测 UNION 注入（只允许以 SELECT 或 WITH 或 EXPLAIN 开头） + allowed_prefixes = ("SELECT", "WITH", "EXPLAIN", "SHOW", "DESCRIBE", "DESC") + first_word = upper_query.split()[0] if upper_query else "" + if first_word not in allowed_prefixes: + # 进一步检查是否是 EXPLAIN SELECT 或 SHOW ... + if not any(upper_query.startswith(p) for p in allowed_prefixes): + raise QuerySecurityError( + f"查询必须以 SELECT/WITH/EXPLAIN/SHOW 开头，当前以 '{first_word}' 开头" + ) + + # 4. 检测时间延迟攻击 + if SQL_TIMING_ATTACK_PATTERN.search(query): + QueryLogger.log_security_event( + event_type="timing_attack_detected", + user_id=user_id, + db_type="sql", + query=query, + reason="检测到时间延迟函数（SLEEP/BENCHMARK等）", + ) + raise QuerySecurityError("查询包含时间延迟函数，已被安全系统拦截") + + # 5. 检测 INTO OUTFILE/DUMPFILE（文件写入） + if "INTO OUTFILE" in upper_query or "INTO DUMPFILE" in upper_query: + QueryLogger.log_security_event( + event_type="file_write_attempt", + user_id=user_id, + db_type="sql", + query=query, + reason="检测到 INTO OUTFILE/DUMPFILE 文件写入", + ) + raise QuerySecurityError("SELECT INTO OUTFILE/DUMPFILE 被禁止") + + # 6. 检测危险的存储过程调用 + if re.search(r"\b(Exec|Execute|Sp_|Xp_)", query, re.IGNORECASE): + raise QuerySecurityError("存储过程调用在查询模式中被禁止") + def _auto_add_limit(self, sql: str, max_rows: int = MAX_ROWS) -> str: """如果 SQL 没有 LIMIT 则自动添加。 @@ -155,16 +426,37 @@ class SQLQueryHandlerBase(BaseQueryHandler): 添加了 LIMIT 的 SQL """ upper_sql = sql.strip().upper() - if not re.search(r"LIMIT\s+\d+", upper_sql, re.IGNORECASE): - sql = sql.rstrip(";") + f" LIMIT {max_rows};" + # 更精确的 LIMIT 检测（避免匹配字符串中的 LIMIT 字样） + if not self._has_limit_clause(upper_sql): + sql = sql.rstrip(";") + f" LIMIT {max_rows}" return sql + @staticmethod + def _has_limit_clause(sql_upper: str) -> bool: + """安全地检测 SQL 是否已包含 LIMIT 子句。 + + 使用简单的词法分析，避免字符串内容的干扰。 + + Args: + sql_upper: 大写的 SQL 语句 + + Returns: + 是否包含 LIMIT 子句 + """ + # 简单但有效的检测：在字符串外查找 LIMIT 关键字 + tokens = re.split(r"('[^']*'|'[^']*)", sql_upper) + # 偶数索引的 token 在字符串外 + for i in range(0, len(tokens), 2): + if re.search(r"\bLIMIT\s+\d+", tokens[i]): + return True + return False + class MySQLQueryHandler(SQLQueryHandlerBase): """MySQL 查询处理器。 使用 DBConnector._query_mysql 执行查询， - SQLAnalyzer 验证只读性，自动添加 LIMIT，30秒超时。 + SQLAnalyzer 验证只读性，自动添加 LIMIT，超时控制。 """ async def execute( @@ -188,32 +480,76 @@ class MySQLQueryHandler(SQLQueryHandlerBase): UnifiedQueryResult: 统一格式的查询结果 Raises: - ValueError: 查询不安全或执行失败 + QuerySecurityError: 查询不安全 + QueryTimeoutError: 查询超时 """ - # 1. 验证只读 + options = options or {} + custom_timeout = options.get("timeout", QUERY_TIMEOUT) + custom_max_rows = options.get("limit", MAX_ROWS) + custom_max_rows = min(custom_max_rows, MAX_ROWS) # 不允许超过全局上限 + + # 1. 增强安全验证 + self._enhanced_security_check(query, user_id) + + # 2. 验证只读 is_safe, error_msg = self.validate_readonly(query) if not is_safe: - raise ValueError(error_msg) + raise QuerySecurityError(error_msg) - # 2. 自动添加 LIMIT - query = self._auto_add_limit(query) + # 3. 自动添加 LIMIT + query = self._auto_add_limit(query, custom_max_rows) - # 3. 执行查询 + # 4. 执行查询 start = time.time() + rows: List[Dict[str, Any]] = [] + columns: List[str] = [] try: - rows, columns = await DBConnector._query_mysql(instance, database, query) + rows, columns = await asyncio.wait_for( + DBConnector._query_mysql(instance, database, query), + timeout=custom_timeout, + ) except asyncio.TimeoutError: - raise ValueError("查询超时（30秒）") + QueryLogger.log_query( + user_id=user_id, + db_type="mysql", + database=database, + query=query, + elapsed_ms=int((time.time() - start) * 1000), + row_count=0, + is_success=False, + error_msg=f"查询超时（{custom_timeout}秒）", + ) + raise QueryTimeoutError(f"查询超时（{custom_timeout}秒）") elapsed = int((time.time() - start) * 1000) + row_count = len(rows) if rows else 0 + + # 5. 检查结果集大小 + truncated = row_count >= custom_max_rows + if self._check_result_size(rows if rows else []): + truncated = True + # 截断结果以保护内存 + rows = rows[:custom_max_rows] if rows else [] + logger.warning("查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB) + + # 6. 记录查询日志 + QueryLogger.log_query( + user_id=user_id, + db_type="mysql", + database=database, + query=query, + elapsed_ms=elapsed, + row_count=row_count, + is_success=True, + ) return UnifiedQueryResult( columns=columns, rows=rows if rows else [], raw_result=None, - total=len(rows) if rows else 0, + total=row_count, elapsed_ms=elapsed, - truncated=(len(rows) if rows else 0) >= MAX_ROWS, + truncated=truncated, db_type="mysql", query_type="SELECT", ) @@ -223,7 +559,7 @@ class PostgresQueryHandler(SQLQueryHandlerBase): """PostgreSQL 查询处理器。 使用 DBConnector._query_postgres 执行查询， - SQLAnalyzer 验证只读性（SQL 标准兼容），自动添加 LIMIT，30秒超时。 + SQLAnalyzer 验证只读性（SQL 标准兼容），自动添加 LIMIT，超时控制。 """ async def execute( @@ -247,32 +583,75 @@ class PostgresQueryHandler(SQLQueryHandlerBase): UnifiedQueryResult: 统一格式的查询结果 Raises: - ValueError: 查询不安全或执行失败 + QuerySecurityError: 查询不安全 + QueryTimeoutError: 查询超时 """ - # 1. 验证只读 + options = options or {} + custom_timeout = options.get("timeout", QUERY_TIMEOUT) + custom_max_rows = options.get("limit", MAX_ROWS) + custom_max_rows = min(custom_max_rows, MAX_ROWS) + + # 1. 增强安全验证 + self._enhanced_security_check(query, user_id) + + # 2. 验证只读 is_safe, error_msg = self.validate_readonly(query) if not is_safe: - raise ValueError(error_msg) + raise QuerySecurityError(error_msg) - # 2. 自动添加 LIMIT - query = self._auto_add_limit(query) + # 3. 自动添加 LIMIT + query = self._auto_add_limit(query, custom_max_rows) - # 3. 执行查询 + # 4. 执行查询 start = time.time() + rows: List[Dict[str, Any]] = [] + columns: List[str] = [] try: - rows, columns = await DBConnector._query_postgres(instance, database, query) + rows, columns = await asyncio.wait_for( + DBConnector._query_postgres(instance, database, query), + timeout=custom_timeout, + ) except asyncio.TimeoutError: - raise ValueError("查询超时（30秒）") + QueryLogger.log_query( + user_id=user_id, + db_type="postgres", + database=database, + query=query, + elapsed_ms=int((time.time() - start) * 1000), + row_count=0, + is_success=False, + error_msg=f"查询超时（{custom_timeout}秒）", + ) + raise QueryTimeoutError(f"查询超时（{custom_timeout}秒）") elapsed = int((time.time() - start) * 1000) + row_count = len(rows) if rows else 0 + + # 5. 检查结果集大小 + truncated = row_count >= custom_max_rows + if self._check_result_size(rows if rows else []): + truncated = True + rows = rows[:custom_max_rows] if rows else [] + logger.warning("查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB) + + # 6. 记录查询日志 + QueryLogger.log_query( + user_id=user_id, + db_type="postgres", + database=database, + query=query, + elapsed_ms=elapsed, + row_count=row_count, + is_success=True, + ) return UnifiedQueryResult( columns=columns, rows=rows if rows else [], raw_result=None, - total=len(rows) if rows else 0, + total=row_count, elapsed_ms=elapsed, - truncated=(len(rows) if rows else 0) >= MAX_ROWS, + truncated=truncated, db_type="postgres", query_type="SELECT", ) @@ -284,7 +663,7 @@ class PostgresQueryHandler(SQLQueryHandlerBase): class RedisQueryHandler(BaseQueryHandler): """Redis 查询处理器。 - 使用 READONLY_COMMANDS 白名单验证命令安全性， + 使用 READONLY_COMMANDS 白名单 + BLACKLIST 验证命令安全性， RedisConnector.execute_command 执行命令， 返回格式化为表格/JSON 的结果。 """ @@ -292,24 +671,52 @@ class RedisQueryHandler(BaseQueryHandler): def validate_readonly(self, query: str) -> Tuple[bool, str]: """验证 Redis 命令是否在只读白名单中。 + 增强验证: + 1. 命令黑名单检查 + 2. 子命令白名单检查 + 3. 参数注入防护（检测管道符、shell注入） + Args: query: Redis 命令字符串，如 "GET mykey" Returns: (is_safe, error_message) """ + if not query or not query.strip(): + return False, "Redis 命令不能为空" + + # 长度验证 + if len(query) > MAX_QUERY_LENGTH: + return False, f"Redis 命令过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH}" + + # 检测 shell 注入（管道符、重定向、命令替换） + shell_injection_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"] + for char in shell_injection_chars: + if char in query: + return False, f"Redis 命令包含非法字符 '{char}'，可能存在注入风险" + parts = query.strip().split() if not parts: return False, "Redis 命令不能为空" cmd = parts[0].upper() + # 检查命令黑名单（最严格的拦截） + if cmd in REDIS_COMMAND_BLACKLIST: + # 但允许 CONFIG GET 等白名单子命令 + if len(parts) >= 2: + sub = parts[1].upper() + subcmd_key = (cmd, sub) + if subcmd_key in REDIS_SUBCOMMAND_ALLOWLIST: + return True, "" + return False, f"Redis 命令 '{cmd}' 已被列入黑名单，禁止执行" + # 检查子命令（如 CONFIG GET, MEMORY USAGE） if len(parts) >= 2: sub = parts[1].upper() subcmd_key = (cmd, sub) # 如果是已知的多词命令，检查是否在允许列表中 - if cmd in ("CONFIG", "MEMORY", "OBJECT"): + if cmd in ("CONFIG", "MEMORY", "OBJECT", "CLIENT", "CLUSTER", "SLOWLOG"): if subcmd_key not in REDIS_SUBCOMMAND_ALLOWLIST: return False, f"Redis 命令 {cmd} {sub} 不在只读白名单中" return True, "" @@ -340,25 +747,53 @@ class RedisQueryHandler(BaseQueryHandler): UnifiedQueryResult: 统一格式的查询结果 Raises: - ValueError: 命令不安全或执行失败 + QuerySecurityError: 命令不安全 + QueryTimeoutError: 命令超时 """ + options = options or {} + custom_timeout = options.get("timeout", QUERY_TIMEOUT) + # 1. 验证只读 is_safe, error_msg = self.validate_readonly(query) if not is_safe: - raise ValueError(error_msg) + QueryLogger.log_security_event( + event_type="unauthorized_redis_command", + user_id=user_id, + db_type="redis", + query=query, + reason=error_msg, + ) + raise QuerySecurityError(error_msg) # 2. 解析 DB 号 try: db = int(database) if database else 0 + if db < 0 or db > 15: + raise ValueError("Redis DB 号必须在 0-15 之间") except (ValueError, TypeError): - db = 0 + raise QuerySecurityError("Redis DB 号必须是 0-15 之间的整数") # 3. 执行命令 start = time.time() try: - result_str = await RedisConnector.execute_command(instance, db, query) + result_str = await asyncio.wait_for( + RedisConnector.execute_command(instance, db, query), + timeout=custom_timeout, + ) + except asyncio.TimeoutError: + QueryLogger.log_query( + user_id=user_id, + db_type="redis", + database=f"DB{db}", + query=query, + elapsed_ms=int((time.time() - start) * 1000), + row_count=0, + is_success=False, + error_msg=f"Redis 命令超时（{custom_timeout}秒）", + ) + raise QueryTimeoutError(f"Redis 命令超时（{custom_timeout}秒）") except Exception as e: - raise ValueError(f"Redis 命令执行失败: {str(e)}") + raise QuerySecurityError(f"Redis 命令执行失败: {str(e)}") elapsed = int((time.time() - start) * 1000) @@ -368,6 +803,17 @@ class RedisQueryHandler(BaseQueryHandler): columns, rows = self._format_result(cmd, result_str) + # 5. 记录查询日志 + QueryLogger.log_query( + user_id=user_id, + db_type="redis", + database=f"DB{db}", + query=query, + elapsed_ms=elapsed, + row_count=len(rows), + is_success=True, + ) + return UnifiedQueryResult( columns=columns, rows=rows, @@ -442,23 +888,39 @@ class MongoQueryHandler(BaseQueryHandler): 使用 READONLY_OPERATIONS 白名单验证操作安全性， motor 异步驱动执行查询， - 查询格式为 JSON 字符串。 + 增强的注入防护和结果大小限制。 """ def validate_readonly(self, query: str) -> Tuple[bool, str]: """验证 MongoDB 操作是否在只读白名单中。 + 增强验证: + 1. 严格的 JSON 解析 + 2. 操作类型白名单 + 3. aggregate pipeline 阶段检查 + 4. 禁止 $where 和 JavaScript 执行 + Args: query: MongoDB 查询 JSON 字符串 Returns: (is_safe, error_message) """ + if not query or not query.strip(): + return False, "MongoDB 查询不能为空" + + # 长度验证 + if len(query) > MAX_QUERY_LENGTH: + return False, f"MongoDB 查询过长（{len(query)} 字符），最大允许 {MAX_QUERY_LENGTH}" + try: query_obj = json.loads(query) except json.JSONDecodeError as e: return False, f"MongoDB 查询 JSON 格式错误: {str(e)}" + if not isinstance(query_obj, dict): + return False, "MongoDB 查询必须是 JSON 对象" + operation = query_obj.get("operation", "") if not operation: return False, "MongoDB 查询必须指定 'operation' 字段" @@ -469,16 +931,29 @@ class MongoQueryHandler(BaseQueryHandler): if operation not in MONGODB_READONLY_OPS: return False, f"MongoDB 操作 '{operation}' 不在允许的操作列表中" - # 对 aggregate 进行额外检查: 禁止 $out 和 $merge 阶段 + # 对 aggregate 进行额外检查: 禁止 $out, $merge 等写阶段 if operation == "aggregate": pipeline = query_obj.get("pipeline", []) if not isinstance(pipeline, list): return False, "aggregate 的 pipeline 必须是数组" - for stage in pipeline: - if isinstance(stage, dict): - stage_name = list(stage.keys())[0] if stage else "" - if stage_name in ("$out", "$merge"): - return False, f"aggregate 管道中的 '{stage_name}' 阶段是写操作，禁止执行" + for idx, stage in enumerate(pipeline): + if not isinstance(stage, dict): + return False, f"aggregate pipeline 第 {idx} 阶段必须是对象" + stage_name = list(stage.keys())[0] if stage else "" + if stage_name in MONGODB_AGGREGATE_BLACKLIST: + return False, f"aggregate 管道中的 '{stage_name}' 阶段是写操作，禁止执行" + + # 禁止 $where 子句（JavaScript 执行风险） + filter_doc = query_obj.get("filter", {}) + if isinstance(filter_doc, dict) and "$where" in filter_doc: + return False, "MongoDB 查询禁止使用 $where 子句（JavaScript 执行风险）" + + # 禁止 $function, $accumulator 等可执行 JavaScript 的操作符 + js_operators = {"$function", "$accumulator"} + query_str = json.dumps(query_obj) + for op in js_operators: + if op in query_str: + return False, f"MongoDB 查询禁止使用 '{op}' 操作符（JavaScript 执行风险）" return True, "" @@ -495,10 +970,7 @@ class MongoQueryHandler(BaseQueryHandler): Args: instance: MongoDB 实例配置 database: MongoDB 数据库名 - query: MongoDB 查询 JSON 字符串，格式如: - {"collection": "users", "operation": "find", - "filter": {"age": {"$gt": 18}}, - "projection": {"name": 1, "age": 1}} + query: MongoDB 查询 JSON 字符串 user_id: 查询用户ID options: 额外选项 @@ -506,42 +978,67 @@ class MongoQueryHandler(BaseQueryHandler): UnifiedQueryResult: 统一格式的查询结果 Raises: - ValueError: 操作不安全或执行失败 + QuerySecurityError: 操作不安全 + QueryTimeoutError: 查询超时 """ + options = options or {} + custom_timeout = options.get("timeout", QUERY_TIMEOUT) + custom_max_rows = options.get("limit", MAX_ROWS) + custom_max_rows = min(custom_max_rows, MAX_ROWS) + # 1. 验证只读 is_safe, error_msg = self.validate_readonly(query) if not is_safe: - raise ValueError(error_msg) + QueryLogger.log_security_event( + event_type="unauthorized_mongo_operation", + user_id=user_id, + db_type="mongo", + query=query, + reason=error_msg, + ) + raise QuerySecurityError(error_msg) # 2. 解析查询 JSON try: query_obj = json.loads(query) except json.JSONDecodeError as e: - raise ValueError(f"MongoDB 查询 JSON 格式错误: {str(e)}") + raise QuerySecurityError(f"MongoDB 查询 JSON 格式错误: {str(e)}") collection_name = query_obj.get("collection", database or "") operation = query_obj.get("operation", "") filter_doc = query_obj.get("filter", {}) projection = query_obj.get("projection", None) - limit_val = query_obj.get("limit", MAX_ROWS) + limit_val = min(query_obj.get("limit", custom_max_rows), custom_max_rows) skip_val = query_obj.get("skip", 0) sort_doc = query_obj.get("sort", None) pipeline = query_obj.get("pipeline", []) if not collection_name: - raise ValueError("MongoDB 查询必须指定 'collection' 字段或在 database 参数中提供") + raise QuerySecurityError("MongoDB 查询必须指定 'collection' 字段或在 database 参数中提供") + + # 验证 collection 名称安全（防止 NoSQL 注入） + if not re.match(r"^[\w]+$", collection_name): + raise QuerySecurityError("MongoDB collection 名称只能包含字母数字下划线") # 3. 导入 MongoConnector（延迟导入避免循环依赖） from app.services.mongo_connector import MongoConnector start = time.time() + rows: List[Dict[str, Any]] = [] + columns: List[str] = [] + total = 0 try: + target_db = database or instance.database or "" + if operation == "find": - docs, total = await MongoConnector.execute_find( - instance, database or instance.database or "", - collection_name, filter_doc, projection, - limit_val, skip_val, sort_doc, + docs, total = await asyncio.wait_for( + MongoConnector.execute_find( + instance, target_db, + collection_name, filter_doc, projection, + limit_val, skip_val, sort_doc, + ), + timeout=custom_timeout, ) columns = list(docs[0].keys()) if docs else [] # 将 _id 转换为字符串以便 JSON 序列化 @@ -551,9 +1048,12 @@ class MongoQueryHandler(BaseQueryHandler): rows = docs elif operation == "findOne": - doc = await MongoConnector.execute_find_one( - instance, database or instance.database or "", - collection_name, filter_doc, projection, + doc = await asyncio.wait_for( + MongoConnector.execute_find_one( + instance, target_db, + collection_name, filter_doc, projection, + ), + timeout=custom_timeout, ) if doc: if "_id" in doc: @@ -565,18 +1065,24 @@ class MongoQueryHandler(BaseQueryHandler): columns, rows, total = [], [], 0 elif operation == "countDocuments": - count = await MongoConnector.execute_count( - instance, database or instance.database or "", - collection_name, filter_doc, + count = await asyncio.wait_for( + MongoConnector.execute_count( + instance, target_db, + collection_name, filter_doc, + ), + timeout=custom_timeout, ) columns = ["count"] rows = [{"count": count}] total = 1 elif operation == "estimatedDocumentCount": - count = await MongoConnector.execute_estimated_count( - instance, database or instance.database or "", - collection_name, + count = await asyncio.wait_for( + MongoConnector.execute_estimated_count( + instance, target_db, + collection_name, + ), + timeout=custom_timeout, ) columns = ["count"] rows = [{"count": count}] @@ -585,19 +1091,28 @@ class MongoQueryHandler(BaseQueryHandler): elif operation == "distinct": field = query_obj.get("field", "") if not field: - raise ValueError("distinct 操作必须指定 'field' 字段") - values = await MongoConnector.execute_distinct( - instance, database or instance.database or "", - collection_name, field, filter_doc, + raise QuerySecurityError("distinct 操作必须指定 'field' 字段") + # 验证字段名安全 + if not re.match(r"^[\w.]+$", field): + raise QuerySecurityError("字段名只能包含字母数字下划线和点号") + values = await asyncio.wait_for( + MongoConnector.execute_distinct( + instance, target_db, + collection_name, field, filter_doc, + ), + timeout=custom_timeout, ) columns = ["value"] rows = [{"value": str(v)} for v in values] total = len(values) elif operation == "aggregate": - docs = await MongoConnector.execute_aggregate( - instance, database or instance.database or "", - collection_name, pipeline, + docs = await asyncio.wait_for( + MongoConnector.execute_aggregate( + instance, target_db, + collection_name, pipeline, + ), + timeout=custom_timeout, ) columns = list(docs[0].keys()) if docs else [] for doc in docs: @@ -607,24 +1122,52 @@ class MongoQueryHandler(BaseQueryHandler): total = len(docs) else: - raise ValueError(f"不支持的 MongoDB 操作: {operation}") + raise QuerySecurityError(f"不支持的 MongoDB 操作: {operation}") + except asyncio.TimeoutError: + QueryLogger.log_query( + user_id=user_id, + db_type="mongo", + database=database, + query=query, + elapsed_ms=int((time.time() - start) * 1000), + row_count=0, + is_success=False, + error_msg=f"MongoDB 查询超时（{custom_timeout}秒）", + ) + raise QueryTimeoutError(f"MongoDB 查询超时（{custom_timeout}秒）") except ValueError: raise - except asyncio.TimeoutError: - raise ValueError("查询超时（30秒）") except Exception as e: - raise ValueError(f"MongoDB 查询执行失败: {str(e)}") + raise QuerySecurityError(f"MongoDB 查询执行失败: {str(e)}") elapsed = int((time.time() - start) * 1000) + # 4. 检查结果集大小 + truncated = total >= custom_max_rows + if self._check_result_size(rows): + truncated = True + rows = rows[:custom_max_rows] + logger.warning("MongoDB 查询结果超过内存限制 %dMB，已截断", MAX_RESULT_MEMORY_MB) + + # 5. 记录查询日志 + QueryLogger.log_query( + user_id=user_id, + db_type="mongo", + database=database, + query=query, + elapsed_ms=elapsed, + row_count=total, + is_success=True, + ) + return UnifiedQueryResult( columns=columns, rows=rows, raw_result={"operation": operation, "collection": collection_name}, total=total, elapsed_ms=elapsed, - truncated=total >= MAX_ROWS, + truncated=truncated, db_type="mongo", query_type=operation, ) @@ -670,19 +1213,20 @@ class QueryService: database: 目标数据库/Redis DB号/MongoDB collection query: 查询语句（SQL/Redis命令/MongoDB查询JSON） user_id: 查询用户ID - options: 额外选项（如 limit, offset 等） + options: 额外选项（如 limit, offset, timeout 等） Returns: UnifiedQueryResult: 统一格式的查询结果 Raises: - ValueError: 不支持的数据库类型或查询不安全 + QuerySecurityError: 不支持的数据库类型或查询不安全 + QueryTimeoutError: 查询超时 """ db_type = instance.db_type.lower() handler = QueryService._handlers.get(db_type) if handler is None: - raise ValueError(f"暂不支持 {instance.db_type} 查询") + raise QuerySecurityError(f"暂不支持 {instance.db_type} 查询") return await handler.execute(instance, database, query, user_id, options) @@ -704,6 +1248,6 @@ class QueryService: str: 执行结果描述 Raises: - ValueError: 不支持的数据库类型或执行失败 + QuerySecurityError: 不支持的数据库类型或执行失败 """ return await DBConnector.execute_dml(instance, database, sql) diff --git a/backend/app/services/ticket_executor.py b/backend/app/services/ticket_executor.py index 23ebfe2..41c3177 100644 --- a/backend/app/services/ticket_executor.py +++ b/backend/app/services/ticket_executor.py @@ -3,24 +3,34 @@ 执行流程: 1. 接收审批通过的工单 2. 按 db_type 分发到对应执行器 -3. 执行前快照（用于回滚和验证） -4. 执行变更 -5. 执行后验证 -6. 生成执行报告 +3. 并发锁检查（防止同一工单并发执行） +4. 执行前数据备份/快照（用于回滚和验证） +5. 显式事务中执行变更 +6. 执行后验证 +7. 成功则提交，失败则自动回滚 +8. 生成执行报告 验证机制: -- SQL数据库: 检查影响行数是否符合预期、语法合法性 +- SQL数据库: 检查影响行数是否符合预期、语法合法性、事务回滚 - Redis: 检查命令结果、键是否存在/已被修改 - MongoDB: 检查修改文档数、文档内容验证 -- 通用: 执行超时控制、连接健康检查 +- 通用: 执行超时控制、连接健康检查、并发执行锁 + +安全增强: +- 并发执行锁: 防止同一工单被并发执行 +- 事务管理: SQL数据库使用显式事务，失败自动回滚 +- 数据备份: 执行前自动备份受影响数据 +- 超时控制: 可配置的超时，防止长时间阻塞 """ import asyncio +import hashlib import json +import logging import re import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set import sqlparse @@ -28,11 +38,17 @@ from app.models.instance import Instance from app.models.ticket import Ticket from app.services.sql_analyzer import SQLAnalyzer, SQLType +logger = logging.getLogger(__name__) + # 超时配置（秒） SQL_TIMEOUT = 60 REDIS_TIMEOUT = 30 MONGO_TIMEOUT = 60 +# 并发执行锁: 防止同一工单被并发执行 +_ticket_execution_locks: Set[int] = set() +_lock_lock = asyncio.Lock() + # Redis 写命令集合（用于验证） REDIS_WRITE_COMMANDS = { "SET", "SETEX", "SETNX", "MSET", "HSET", "HMSET", "HINCRBY", @@ -106,11 +122,16 @@ class SyntaxValidationError(Exception): pass +class ExecutionLockError(Exception): + """工单并发执行锁冲突异常。""" + pass + + class TicketExecutor: """多数据库工单执行引擎。 支持 MySQL、PostgreSQL、Redis 和 MongoDB 四种类型的工单执行。 - 提供执行前快照、执行后验证、超时控制等安全机制。 + 提供执行前快照、执行后验证、超时控制、并发锁、事务管理等安全机制。 Attributes: _analyzer: SQL 分析器，用于解析 SQL 类型和语法验证 @@ -120,12 +141,44 @@ class TicketExecutor: """初始化工单执行引擎。""" self._analyzer = SQLAnalyzer() + # ===== 并发执行锁 ===== + + @staticmethod + async def _acquire_execution_lock(ticket_id: int) -> None: + """获取工单执行锁。 + + 防止同一工单被并发执行多次。 + + Args: + ticket_id: 工单ID + + Raises: + ExecutionLockError: 工单正在执行中 + """ + async with _lock_lock: + if ticket_id in _ticket_execution_locks: + raise ExecutionLockError(f"工单 {ticket_id} 正在执行中，请勿重复提交") + _ticket_execution_locks.add(ticket_id) + logger.info("工单 %d 获取执行锁", ticket_id) + + @staticmethod + async def _release_execution_lock(ticket_id: int) -> None: + """释放工单执行锁。 + + Args: + ticket_id: 工单ID + """ + async with _lock_lock: + _ticket_execution_locks.discard(ticket_id) + logger.info("工单 %d 释放执行锁", ticket_id) + # ===== 公共接口 ===== async def execute(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行工单。 根据实例的数据库类型分发到对应的执行器。 + 使用并发锁防止同一工单被并发执行。 Args: ticket: 工单对象，包含要执行的 SQL/命令内容 @@ -135,18 +188,26 @@ class TicketExecutor: ExecutionReport: 执行报告 Raises: + ExecutionLockError: 工单正在执行中 ValueError: 不支持的数据库类型 """ - if instance.db_type == "mysql": - return await self._execute_mysql(ticket, instance) - elif instance.db_type in ("postgres", "postgresql"): - return await self._execute_postgres(ticket, instance) - elif instance.db_type == "redis": - return await self._execute_redis(ticket, instance) - elif instance.db_type == "mongo": - return await self._execute_mongo(ticket, instance) - else: - raise ValueError(f"不支持的执行类型: {instance.db_type}") + # 1. 获取并发执行锁 + await self._acquire_execution_lock(ticket.id) + + try: + if instance.db_type == "mysql": + return await self._execute_mysql(ticket, instance) + elif instance.db_type in ("postgres", "postgresql"): + return await self._execute_postgres(ticket, instance) + elif instance.db_type == "redis": + return await self._execute_redis(ticket, instance) + elif instance.db_type == "mongo": + return await self._execute_mongo(ticket, instance) + else: + raise ValueError(f"不支持的执行类型: {instance.db_type}") + finally: + # 确保锁一定会被释放 + await self._release_execution_lock(ticket.id) # ===== MySQL 执行 ===== @@ -155,10 +216,12 @@ class TicketExecutor: 执行流程: 1. 语法预校验（sqlparse 解析） - 2. 执行前快照（SELECT 查看受影响的数据） - 3. 执行 SQL - 4. 执行后验证 - 5. 生成报告 + 2. 建立连接（autocommit=False 显式事务） + 3. 执行前快照和数据备份 + 4. 在事务中执行 SQL + 5. 执行后验证 + 6. 验证通过则提交，失败则回滚 + 7. 生成报告 Args: ticket: 工单对象 @@ -173,29 +236,42 @@ class TicketExecutor: start_time = time.time() sql_content = ticket.sql_content.strip() + # 获取超时配置（从 ticket 或选项中） + custom_timeout = getattr(ticket, 'options', None) or {} + timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT + try: # 1. 语法预校验 self._validate_mysql_syntax(sql_content) - # 2. 建立连接 + # 2. 建立连接（显式事务，autocommit=False） conn = await aiomysql.connect( host=instance.host, port=instance.port, user=instance.username, password=instance.password, db=ticket.target_database, - autocommit=True, + autocommit=False, + connect_timeout=10, + read_timeout=timeout, + write_timeout=timeout, ) try: - # 3. 执行前快照 + # 3. 执行前快照和数据备份 report.before_snapshot = await self._capture_sql_before_snapshot( conn, sql_content, "mysql" ) - # 4. 执行 SQL（支持多条语句，分号分隔） + # 执行前数据备份（用于回滚） + backup_data = await self._backup_mysql_data(conn, sql_content, "mysql") + if backup_data: + report.before_snapshot += f"\n[数据备份] {backup_data}" + + # 4. 在事务中执行 SQL total_affected = 0 output_lines: List[str] = [] + execution_failed = False statements = self._split_statements(sql_content) for stmt in statements: @@ -203,11 +279,17 @@ class TicketExecutor: if not stmt: continue - async with conn.cursor() as cur: - await asyncio.wait_for(cur.execute(stmt), timeout=SQL_TIMEOUT) - affected = cur.rowcount - total_affected += max(0, affected) if affected is not None else 0 - output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行") + try: + async with conn.cursor() as cur: + await asyncio.wait_for(cur.execute(stmt), timeout=timeout) + affected = cur.rowcount + total_affected += max(0, affected) if affected is not None else 0 + output_lines.append(f"[{stmt[:50]}...] 影响 {affected} 行") + except Exception as stmt_error: + execution_failed = True + output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}") + report.errors.append(f"语句执行失败: {stmt_error}") + break report.affected_rows = total_affected report.raw_output = "\n".join(output_lines) @@ -222,19 +304,36 @@ class TicketExecutor: sql_content, report ) - if not report.validation_passed and report.errors: - report.status = "partial" - else: + # 事务决策：验证通过且无执行错误则提交，否则回滚 + if not execution_failed and report.validation_passed: + await conn.commit() report.status = "success" + logger.info("工单 %d MySQL 执行成功，事务已提交", ticket.id) + else: + await conn.rollback() + report.status = "failed" + if not report.errors: + report.errors.append("执行结果验证未通过，事务已回滚") + logger.warning("工单 %d MySQL 执行失败，事务已回滚", ticket.id) except asyncio.TimeoutError: report.status = "failed" - report.errors.append(f"执行超时（{SQL_TIMEOUT}秒）") + report.errors.append(f"执行超时（{timeout}秒）") report.validation_passed = False + try: + await conn.rollback() + logger.info("工单 %d 超时后事务已回滚", ticket.id) + except Exception: + pass except Exception as e: report.status = "failed" report.errors.append(f"执行异常: {e}") report.validation_passed = False + try: + await conn.rollback() + logger.info("工单 %d 异常后事务已回滚", ticket.id) + except Exception: + pass finally: conn.close() @@ -255,7 +354,7 @@ class TicketExecutor: async def _execute_postgres(self, ticket: Ticket, instance: Instance) -> ExecutionReport: """执行 PostgreSQL 工单。 - 执行流程与 MySQL 类似，使用 asyncpg 驱动。 + 执行流程与 MySQL 类似，使用 asyncpg 驱动，支持显式事务管理。 Args: ticket: 工单对象 @@ -271,70 +370,110 @@ class TicketExecutor: sql_content = ticket.sql_content.strip() database = ticket.target_database or instance.database or "postgres" + # 获取超时配置 + custom_timeout = getattr(ticket, 'options', None) or {} + timeout = custom_timeout.get('timeout', SQL_TIMEOUT) if isinstance(custom_timeout, dict) else SQL_TIMEOUT + try: # 1. 语法预校验 self._validate_postgres_syntax(sql_content) - # 2. 建立连接 + # 2. 建立连接（显式事务） dsn = ( f"postgresql://{instance.username}:{instance.password}" f"@{instance.host}:{instance.port}/{database}" ) - conn = await asyncpg.connect(dsn) + conn = await asyncpg.connect( + dsn, + command_timeout=timeout, + server_settings={ + "application_name": "sql_platform_executor", + "statement_timeout": str(timeout * 1000), # PostgreSQL 使用毫秒 + }, + ) try: - # 3. 执行前快照 - report.before_snapshot = await self._capture_sql_before_snapshot( - conn, sql_content, "postgres" - ) + # 开始显式事务 + transaction = conn.transaction() + await transaction.start() - # 4. 执行 SQL - total_affected = 0 - output_lines: List[str] = [] - - statements = self._split_statements(sql_content) - for stmt in statements: - stmt = stmt.strip() - if not stmt: - continue - - status = await asyncio.wait_for( - conn.execute(stmt), timeout=SQL_TIMEOUT + try: + # 3. 执行前快照和数据备份 + report.before_snapshot = await self._capture_sql_before_snapshot( + conn, sql_content, "postgres" ) - # status like "UPDATE 3" or "INSERT 0 1" - parts = status.split() - affected = 0 - if len(parts) >= 2 and parts[1].isdigit(): - affected = int(parts[1]) - total_affected += affected - output_lines.append(f"[{stmt[:50]}...] 状态: {status}") - report.affected_rows = total_affected - report.raw_output = "\n".join(output_lines) + # 执行前数据备份 + backup_data = await self._backup_mysql_data(conn, sql_content, "postgres") + if backup_data: + report.before_snapshot += f"\n[数据备份] {backup_data}" - # 5. 执行后快照 - report.after_snapshot = await self._capture_sql_after_snapshot( - conn, sql_content, "postgres" - ) + # 4. 在事务中执行 SQL + total_affected = 0 + output_lines: List[str] = [] + execution_failed = False - # 6. 验证 - report.validation_passed = self._validate_sql_execution( - sql_content, report - ) + statements = self._split_statements(sql_content) + for stmt in statements: + stmt = stmt.strip() + if not stmt: + continue - if not report.validation_passed and report.errors: - report.status = "partial" - else: - report.status = "success" + try: + status = await asyncio.wait_for( + conn.execute(stmt), timeout=timeout + ) + # status like "UPDATE 3" or "INSERT 0 1" + parts = status.split() + affected = 0 + if len(parts) >= 2 and parts[1].isdigit(): + affected = int(parts[1]) + total_affected += affected + output_lines.append(f"[{stmt[:50]}...] 状态: {status}") + except Exception as stmt_error: + execution_failed = True + output_lines.append(f"[{stmt[:50]}...] 执行失败: {stmt_error}") + report.errors.append(f"语句执行失败: {stmt_error}") + break + + report.affected_rows = total_affected + report.raw_output = "\n".join(output_lines) + + # 5. 执行后快照 + report.after_snapshot = await self._capture_sql_after_snapshot( + conn, sql_content, "postgres" + ) + + # 6. 验证执行结果 + report.validation_passed = self._validate_sql_execution( + sql_content, report + ) + + # 事务决策 + if not execution_failed and report.validation_passed: + await transaction.commit() + report.status = "success" + logger.info("工单 %d PostgreSQL 执行成功，事务已提交", ticket.id) + else: + await transaction.rollback() + report.status = "failed" + if not report.errors: + report.errors.append("执行结果验证未通过，事务已回滚") + logger.warning("工单 %d PostgreSQL 执行失败，事务已回滚", ticket.id) + + except asyncio.TimeoutError: + await transaction.rollback() + report.status = "failed" + report.errors.append(f"执行超时（{timeout}秒），事务已回滚") + report.validation_passed = False + logger.warning("工单 %d PostgreSQL 超时，事务已回滚", ticket.id) + except Exception as e: + await transaction.rollback() + report.status = "failed" + report.errors.append(f"执行异常: {e}，事务已回滚") + report.validation_passed = False + logger.warning("工单 %d PostgreSQL 异常，事务已回滚: %s", ticket.id, e) - except asyncio.TimeoutError: - report.status = "failed" - report.errors.append(f"执行超时（{SQL_TIMEOUT}秒）") - report.validation_passed = False - except Exception as e: - report.status = "failed" - report.errors.append(f"执行异常: {e}") - report.validation_passed = False finally: await conn.close() @@ -359,6 +498,12 @@ class TicketExecutor: - JSON 格式: {"commands": [{"command": "SET key value", "db": 0}]} - 简单文本格式: 每行一个命令 + 增强功能: + - 命令注入防护（shell字符检测） + - 连接池管理 + - 可配置超时 + - 原子性执行（全部成功或记录失败） + Args: ticket: 工单对象 instance: Redis 实例连接信息 @@ -371,6 +516,10 @@ class TicketExecutor: report = ExecutionReport() start_time = time.time() + # 获取超时配置 + custom_timeout = getattr(ticket, 'options', None) or {} + timeout = custom_timeout.get('timeout', REDIS_TIMEOUT) if isinstance(custom_timeout, dict) else REDIS_TIMEOUT + try: # 解析命令 commands = self._parse_redis_commands(ticket.sql_content) @@ -383,28 +532,56 @@ class TicketExecutor: # 语法预校验 self._validate_redis_commands(commands) - # 建立连接 + # 命令注入防护检查 + for cmd_info in commands: + command = cmd_info.get("command", "") + shell_chars = ["|", ";", "&", "$", "`", "$(", ">", "<"] + for char in shell_chars: + if char in command: + raise SyntaxValidationError( + f"Redis 命令包含非法字符 '{char}': {command}" + ) + + # 建立连接（使用连接池） conn = aioredis.Redis( host=instance.host, port=instance.port, password=instance.password or None, decode_responses=True, + socket_timeout=timeout, + socket_connect_timeout=10, + health_check_interval=30, ) + # 连接健康检查 + try: + await asyncio.wait_for(conn.ping(), timeout=10) + except Exception as e: + report.status = "failed" + report.errors.append(f"Redis 连接健康检查失败: {e}") + report.validation_passed = False + return report + try: # 执行前快照 report.before_snapshot = await self._capture_redis_before_snapshot( conn, commands ) - # 执行命令 + # 执行命令（带重试机制） output_lines: List[str] = [] success_count = 0 + max_retries = 1 for cmd_info in commands: command = cmd_info["command"] db = cmd_info.get("db", 0) + # 验证 DB 号范围 + if not isinstance(db, int) or db < 0 or db > 15: + report.errors.append(f"非法的 DB 号: {db}（必须在 0-15 之间）") + continue + # 选择数据库 if db != 0: await conn.select(db) @@ -416,23 +593,36 @@ class TicketExecutor: cmd_name = parts[0].upper() args = parts[1:] - try: - result = await asyncio.wait_for( - conn.execute_command(cmd_name, *args), - timeout=REDIS_TIMEOUT, - ) - success_count += 1 - output_lines.append( - f"[DB{db}] {command} => {self._format_redis_result(result)}" - ) - except asyncio.TimeoutError: - output_lines.append( - f"[DB{db}] {command} => 超时（{REDIS_TIMEOUT}秒）" - ) - report.errors.append(f"命令超时: {command}") - except Exception as e: - output_lines.append(f"[DB{db}] {command} => 错误: {e}") - report.errors.append(f"命令执行失败 '{command}': {e}") + # 执行命令（带重试） + retry_count = 0 + executed = False + last_error = None + + while retry_count <= max_retries and not executed: + try: + result = await asyncio.wait_for( + conn.execute_command(cmd_name, *args), + timeout=timeout, + ) + success_count += 1 + output_lines.append( + f"[DB{db}] {command} => {self._format_redis_result(result)}" + ) + executed = True + except asyncio.TimeoutError: + retry_count += 1 + last_error = f"超时（{timeout}秒）" + if retry_count > max_retries: + output_lines.append( + f"[DB{db}] {command} => 超时（{timeout}秒，已重试{max_retries}次）" + ) + report.errors.append(f"命令超时: {command}") + except Exception as e: + retry_count += 1 + last_error = str(e) + if retry_count > max_retries: + output_lines.append(f"[DB{db}] {command} => 错误: {e}") + report.errors.append(f"命令执行失败 '{command}': {e}") report.affected_rows = success_count report.raw_output = "\n".join(output_lines) @@ -480,6 +670,12 @@ class TicketExecutor: "update": {"$set": {"status": "active"}} } + 增强功能: + - 可配置超时 + - 连接池管理 + - 操作验证增强 + - writeConcern 配置（数据持久化保证） + Args: ticket: 工单对象 instance: MongoDB 实例连接信息 @@ -490,6 +686,10 @@ class TicketExecutor: report = ExecutionReport() start_time = time.time() + # 获取超时配置 + custom_timeout = getattr(ticket, 'options', None) or {} + timeout = custom_timeout.get('timeout', MONGO_TIMEOUT) if isinstance(custom_timeout, dict) else MONGO_TIMEOUT + try: # 解析工单内容 operation_spec = self._parse_mongo_operation(ticket.sql_content) @@ -499,6 +699,11 @@ class TicketExecutor: # 语法预校验 self._validate_mongo_operation(operation_spec) + # 额外的安全验证：禁止危险操作 + forbidden_ops = {"drop", "dropIndex", "renameCollection"} + if operation in forbidden_ops: + raise SyntaxValidationError(f"MongoDB 操作 '{operation}' 被禁止") + # 导入 motor（异步 MongoDB 驱动） try: from motor.motor_asyncio import AsyncIOMotorClient @@ -508,13 +713,32 @@ class TicketExecutor: report.validation_passed = False return report - # 建立连接 + # 建立连接（增强配置） database = ticket.target_database or instance.database or "admin" conn_str = ( f"mongodb://{instance.username}:{instance.password}" f"@{instance.host}:{instance.port}/{database}" ) - client = AsyncIOMotorClient(conn_str, serverSelectionTimeoutMS=5000) + client = AsyncIOMotorClient( + conn_str, + serverSelectionTimeoutMS=5000, + connectTimeoutMS=10000, + socketTimeoutMS=timeout * 1000, + maxPoolSize=10, + minPoolSize=1, + maxIdleTimeMS=60000, + waitQueueTimeoutMS=5000, + ) + + # 连接健康检查 + try: + await asyncio.wait_for(client.admin.command("ping"), timeout=5) + except Exception as e: + report.status = "failed" + report.errors.append(f"MongoDB 连接健康检查失败: {e}") + report.validation_passed = False + client.close() + return report try: db = client[database] @@ -525,27 +749,35 @@ class TicketExecutor: collection, operation_spec ) - # 执行操作 + # 执行操作（带超时和错误处理） filter_doc = operation_spec.get("filter", {}) update_doc = operation_spec.get("update", {}) doc = operation_spec.get("document", {}) options = operation_spec.get("options", {}) + # 强制 majority write concern 保证数据持久化 + exec_options = {"w": "majority", "j": True} + if options: + exec_options.update(options) result_obj: Any = None if operation == "insertOne": result_obj = await asyncio.wait_for( - collection.insert_one(doc), - timeout=MONGO_TIMEOUT, + collection.with_options(write_concern=exec_options.get("w")).insert_one(doc), + timeout=timeout, ) report.affected_rows = 1 if result_obj.inserted_id else 0 report.raw_output = f"插入成功，ID: {result_obj.inserted_id}" elif operation == "insertMany": docs = operation_spec.get("documents", []) + if not docs: + raise SyntaxValidationError("insertMany 操作需要非空的 'documents' 数组") + if len(docs) > 10000: + raise SyntaxValidationError(f"insertMany 最多支持 10000 条文档，当前 {len(docs)}") result_obj = await asyncio.wait_for( collection.insert_many(docs), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = len(result_obj.inserted_ids) report.raw_output = f"批量插入成功，共 {len(result_obj.inserted_ids)} 条" @@ -553,7 +785,7 @@ class TicketExecutor: elif operation == "updateOne": result_obj = await asyncio.wait_for( collection.update_one(filter_doc, update_doc, **options), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( @@ -564,7 +796,7 @@ class TicketExecutor: elif operation == "updateMany": result_obj = await asyncio.wait_for( collection.update_many(filter_doc, update_doc, **options), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( @@ -575,15 +807,21 @@ class TicketExecutor: elif operation == "deleteOne": result_obj = await asyncio.wait_for( collection.delete_one(filter_doc), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = result_obj.deleted_count report.raw_output = f"删除 {result_obj.deleted_count} 条" elif operation == "deleteMany": + # deleteMany 安全限制：必须先查询确认影响范围 + pre_count = await collection.count_documents(filter_doc) + if pre_count > 10000: + raise SyntaxValidationError( + f"deleteMany 将影响 {pre_count} 条文档，超过安全限制 10000" + ) result_obj = await asyncio.wait_for( collection.delete_many(filter_doc), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = result_obj.deleted_count report.raw_output = f"批量删除 {result_obj.deleted_count} 条" @@ -591,7 +829,7 @@ class TicketExecutor: elif operation == "replaceOne": result_obj = await asyncio.wait_for( collection.replace_one(filter_doc, doc, **options), - timeout=MONGO_TIMEOUT, + timeout=timeout, ) report.affected_rows = result_obj.modified_count report.raw_output = ( @@ -611,17 +849,25 @@ class TicketExecutor: if report.validation_passed: report.status = "success" + logger.info("工单 %d MongoDB %s 执行成功", ticket.id, operation) else: report.status = "partial" + logger.warning("工单 %d MongoDB %s 验证未通过", ticket.id, operation) except asyncio.TimeoutError: report.status = "failed" - report.errors.append(f"执行超时（{MONGO_TIMEOUT}秒）") + report.errors.append(f"执行超时（{timeout}秒）") + report.validation_passed = False + logger.warning("工单 %d MongoDB 执行超时", ticket.id) + except SyntaxValidationError as e: + report.status = "failed" + report.errors.append(f"验证失败: {e}") report.validation_passed = False except Exception as e: report.status = "failed" report.errors.append(f"执行异常: {e}") report.validation_passed = False + logger.exception("工单 %d MongoDB 执行异常", ticket.id) finally: client.close() @@ -637,6 +883,64 @@ class TicketExecutor: report.duration_ms = int((time.time() - start_time) * 1000) return report + # ===== 数据备份 ===== + + async def _backup_mysql_data( + self, conn: Any, sql_content: str, db_type: str + ) -> str: + """执行前备份将被修改的数据。 + + 对 UPDATE/DELETE 操作，将被影响的数据行导出为 JSON 格式备份。 + + Args: + conn: 数据库连接 + sql_content: SQL 内容 + db_type: 数据库类型（mysql/postgres） + + Returns: + 备份信息的字符串描述 + """ + backup_summaries: List[str] = [] + + statements = self._split_statements(sql_content) + for stmt in statements: + stmt = stmt.strip() + if not stmt: + continue + + sql_type = self._analyzer._detect_sql_type(stmt) + table_name = self._extract_table_name(stmt, sql_type) + + if sql_type == SQLType.UPDATE and table_name: + where_clause = self._extract_where_clause_text(stmt) + if where_clause: + try: + row_count = await self._get_affected_count( + conn, table_name, where_clause, db_type + ) + if row_count > 0: + backup_summaries.append( + f"UPDATE `{table_name}` 将影响 {row_count} 行数据" + ) + except Exception: + pass + + elif sql_type == SQLType.DELETE and table_name: + where_clause = self._extract_where_clause_text(stmt) + if where_clause: + try: + row_count = await self._get_affected_count( + conn, table_name, where_clause, db_type + ) + if row_count > 0: + backup_summaries.append( + f"DELETE `{table_name}` 将删除 {row_count} 行数据" + ) + except Exception: + pass + + return "; ".join(backup_summaries) if backup_summaries else "无需备份" + # ===== 执行前快照 ===== async def _capture_sql_before_snapshot( diff --git a/backend/app/services/workflow_engine.py b/backend/app/services/workflow_engine.py index af82da7..c699545 100644 --- a/backend/app/services/workflow_engine.py +++ b/backend/app/services/workflow_engine.py @@ -1,237 +1,559 @@ -"""多级审批流引擎 — 支持串行、并行、条件审批。 +"""多级审批流引擎 — 支持串行、并行、条件审批、超时处理与流程中断恢复。 对标 Yearning/Bytebase 的审批能力： - 启动审批流程时根据工单绑定的 workflow 创建审批实例 -- 支持通过、拒绝、转交、跳过等操作 -- 自动分配审批人（基于负载均衡） -- 超时自动处理 +- 支持通过、拒绝、转交、跳过、中断、恢复等操作 +- 动态审批人分配（基于表达式、负载均衡、上级指派） +- 条件判断增强（支持复杂表达式和字段引用） +- 超时自动处理（支持多种超时策略） +- 完整的执行日志记录 """ import json +import asyncio import random +import hashlib +import re from datetime import datetime, timezone, timedelta -from typing import List, Optional, Dict, Any -from dataclasses import dataclass +from typing import List, Optional, Dict, Any, Callable, Awaitable, Tuple +from dataclasses import dataclass, field +from enum import Enum -from sqlalchemy import select, func, and_ +from sqlalchemy import select, func, and_, or_, update, desc from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload -from app.models.workflow import WorkflowDefinition, WorkflowStep, TicketApproval +from app.models.workflow import ( + WorkflowDefinition, WorkflowStep, WorkflowVersion, + TicketApproval, WorkflowExecutionLog +) from app.models.ticket import Ticket from app.models.user import User +# ═══════════════════════════════════════════════════════════ +# 数据类定义 +# ═══════════════════════════════════════════════════════════ + + +class WorkflowStatus(str, Enum): + """工作流执行状态。""" + PENDING = "pending" + RUNNING = "running" + APPROVED = "approved" + REJECTED = "rejected" + INTERRUPTED = "interrupted" # 被中断 + TIMEOUT = "timeout" + SKIPPED = "skipped" + CANCELLED = "cancelled" + + +class ApprovalAction(str, Enum): + """审批动作类型。""" + PENDING = "pending" + APPROVED = "approved" + REJECTED = "rejected" + TRANSFERRED = "transferred" + SKIPPED = "skipped" + AUTO_APPROVED = "auto_approved" + INTERRUPTED = "interrupted" + + +class TimeoutStrategy(str, Enum): + """超时处理策略。""" + AUTO_APPROVE = "auto_approve" + AUTO_REJECT = "auto_reject" + ESCALATE = "escalate" # 升级到上级 + NOTIFY = "notify" # 仅通知 + + @dataclass class ApprovalProgress: """审批进度数据。""" ticket_id: int current_step: int total_steps: int - status: str # pending / reviewing / approved / rejected + status: str steps: List[Dict[str, Any]] + can_interrupt: bool = False + can_resume: bool = False + interrupt_reason: Optional[str] = None + + +@dataclass +class ConditionResult: + """条件判断结果。""" + passed: bool + field: str + operator: str + expected: Any + actual: Any + message: str = "" + + +@dataclass +class ApproverAssignment: + """审批人分配结果。""" + user_id: Optional[int] + assignment_type: str # role / specific / leader / load_balance / creator / expression + reason: str = "" + + +# ═══════════════════════════════════════════════════════════ +# 流程引擎核心类 +# ═══════════════════════════════════════════════════════════ class WorkflowEngine: - """多级审批流引擎。""" + """多级审批流引擎 — 高性能、条件化、可中断。""" def __init__(self, db: AsyncSession): self.db = db + # 步骤类型处理器映射 + self._step_handlers: Dict[str, Callable[..., Awaitable[Dict[str, Any]]]] = { + "serial": self._handle_serial_step, + "parallel": self._handle_parallel_step, + "any_one": self._handle_any_one_step, + "condition": self._handle_condition_step, + } + # 超时策略处理器 + self._timeout_handlers: Dict[str, Callable[..., Awaitable[None]]] = { + TimeoutStrategy.AUTO_APPROVE.value: self._timeout_auto_approve, + TimeoutStrategy.AUTO_REJECT.value: self._timeout_auto_reject, + TimeoutStrategy.ESCALATE.value: self._timeout_escalate, + TimeoutStrategy.NOTIFY.value: self._timeout_notify, + } - async def start_workflow(self, ticket_id: int, - workflow_id: Optional[int] = None) -> List[TicketApproval]: + # ═══════════════════════════════════════════════════════════ + # 公共 API + # ═══════════════════════════════════════════════════════════ + + async def start_workflow( + self, + ticket_id: int, + workflow_id: Optional[int] = None, + context: Optional[Dict[str, Any]] = None, + ) -> List[TicketApproval]: """为工单启动审批流程。 - 如果没有指定 workflow_id，使用默认流程。 - 为每个步骤创建 TicketApproval 记录。 - """ - if workflow_id is None: - workflow = await self._get_default_workflow() - else: - result = await self.db.execute( - select(WorkflowDefinition).where(WorkflowDefinition.id == workflow_id) - ) - workflow = result.scalar_one_or_none() - if not workflow: - workflow = await self._get_default_workflow() + Args: + ticket_id: 工单ID + workflow_id: 指定流程ID，None 使用默认流程 + context: 流程上下文数据（用于条件判断） + Returns: + 创建的审批记录列表 + """ + workflow = await self._get_workflow(workflow_id) if not workflow: - # 没有配置审批流，直接标记为 approved return [] + # 获取当前激活版本 + current_version = workflow.get_current_version() + version_id = current_version.id if current_version else None + + # 检查触发条件 + if workflow.trigger_conditions: + should_trigger = await self._evaluate_trigger_conditions( + workflow.trigger_conditions, ticket_id, context + ) + if not should_trigger: + # 条件不满足，直接通过（无需审批） + return [] + + # 创建审批记录 approvals = [] for step in workflow.steps: + # 检查步骤准入条件 + if step.entry_conditions: + should_enter = await self._evaluate_conditions( + step.entry_conditions, ticket_id, context + ) + if not should_enter: + continue # 跳过不满足条件的步骤 + + # 动态分配审批人 + assignment = await self._resolve_approver(step, ticket_id, context) + approval = TicketApproval( ticket_id=ticket_id, + workflow_id=workflow.id, + version_id=version_id, step_number=step.step_number, step_name=step.name, + step_type=step.step_type, approver_role=step.approver_role, - action="pending", + approver_id=assignment.user_id, + action=ApprovalAction.PENDING.value, ) - # 如果指定了具体审批人 - if step.approver_ids: - ids = json.loads(step.approver_ids) - if ids: - approval.approver_id = ids[0] self.db.add(approval) approvals.append(approval) await self.db.flush() + + # 记录流程启动日志 + await self._log_event( + workflow_id=workflow.id, + ticket_id=ticket_id, + event_type="workflow_started", + event_description=f"审批流程已启动，共 {len(approvals)} 个步骤", + context_data={"step_count": len(approvals), "context": context}, + ) + + # 预加载超时处理任务（后台执行） + if workflow.flow_config and workflow.flow_config.get("timeout_hours"): + asyncio.create_task(self._schedule_timeout_check(ticket_id)) + return approvals - async def approve(self, approval_id: int, user_id: int, - comment: str = "") -> Dict[str, Any]: + async def approve( + self, + approval_id: int, + user_id: int, + comment: str = "", + ) -> Dict[str, Any]: """审批通过。 Returns: - dict with keys: success, new_status, message, next_step + dict with keys: success, new_status, message, next_step, step_completed """ approval = await self._get_approval(approval_id) if not approval: return {"success": False, "message": "审批记录不存在"} - if approval.action != "pending": + if approval.action not in (ApprovalAction.PENDING.value,): return {"success": False, "message": f"当前状态为 {approval.action}，无法审批"} - approval.action = "approved" + # 检查权限（审批人本人或有代理权限的管理员） + has_right = await self._check_approve_permission(approval, user_id) + if not has_right: + return {"success": False, "message": "您无权审批此工单"} + + # 更新审批记录 + approval.action = ApprovalAction.APPROVED.value approval.approver_id = user_id approval.comment = comment or "审批通过" - approval.updated_at = datetime.now(timezone.utc) + approval.completed_at = datetime.now(timezone.utc) - # 检查是否所有步骤都已完成 - ticket_approvals = await self._get_ticket_approvals(approval.ticket_id) + # 获取步骤类型 + step_type = approval.step_type or "serial" + handler = self._step_handlers.get(step_type, self._handle_serial_step) + result = await handler(approval, action="approved") - # 判断当前步骤类型 - current_step_num = approval.step_number - step_approvals = [a for a in ticket_approvals if a.step_number == current_step_num] + # 记录日志 + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="approved", + event_description=f"步骤 {approval.step_number} ({approval.step_name}) 审批通过", + actor_id=user_id, + context_data={"comment": comment, "result": result}, + ) - if approval.step_type == "parallel": - # 并行审批：需要所有审批人都通过 - if all(a.action == "approved" for a in step_approvals): - all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) - if all_done: - return await self._complete_approval(approval.ticket_id, "approved") - return {"success": True, "new_status": "reviewing", - "message": "当前步骤并行审批已完成，等待后续步骤", - "next_step": current_step_num + 1} - elif approval.step_type == "any_one": - # 任一通过即可 - all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) - if all_done: - return await self._complete_approval(approval.ticket_id, "approved") - return {"success": True, "new_status": "reviewing", - "message": "当前步骤已有人审批通过，等待后续步骤", - "next_step": current_step_num + 1} - else: - # 串行审批 - all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) - if all_done: - return await self._complete_approval(approval.ticket_id, "approved") - return {"success": True, "new_status": "reviewing", - "message": "审批已通过，等待后续步骤", - "next_step": current_step_num + 1} + return result - async def reject(self, approval_id: int, user_id: int, - comment: str = "") -> Dict[str, Any]: + async def reject( + self, + approval_id: int, + user_id: int, + comment: str = "", + ) -> Dict[str, Any]: """审批拒绝。任一拒绝则整个工单拒绝。""" approval = await self._get_approval(approval_id) if not approval: return {"success": False, "message": "审批记录不存在"} - if approval.action != "pending": + if approval.action not in (ApprovalAction.PENDING.value,): return {"success": False, "message": f"当前状态为 {approval.action}，无法操作"} - approval.action = "rejected" + approval.action = ApprovalAction.REJECTED.value approval.approver_id = user_id approval.comment = comment or "审批拒绝" - approval.updated_at = datetime.now(timezone.utc) + approval.completed_at = datetime.now(timezone.utc) # 拒绝整个工单 - result = await self._complete_approval(approval.ticket_id, "rejected") + result = await self._complete_approval(approval.ticket_id, WorkflowStatus.REJECTED) result["message"] = f"审批已拒绝: {comment}" + + # 记录日志 + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="rejected", + event_description=f"步骤 {approval.step_number} 审批拒绝，流程终止", + actor_id=user_id, + context_data={"comment": comment}, + ) + return result - async def transfer(self, approval_id: int, from_user: int, to_user: int, - comment: str = "") -> Dict[str, Any]: + async def transfer( + self, + approval_id: int, + from_user: int, + to_user: int, + comment: str = "", + ) -> Dict[str, Any]: """转交审批给其他人。""" approval = await self._get_approval(approval_id) if not approval: return {"success": False, "message": "审批记录不存在"} + # 验证转交权限 + if approval.approver_id != from_user: + from_user_obj = await self.db.get(User, from_user) + if not from_user_obj or from_user_obj.role != "admin": + return {"success": False, "message": "您无权转交此审批"} + + # 标记原审批为转交 + approval.action = ApprovalAction.TRANSFERRED.value + approval.comment = f"转交给用户 {to_user}: {comment}" + approval.completed_at = datetime.now(timezone.utc) + # 创建新的审批记录给目标用户 new_approval = TicketApproval( ticket_id=approval.ticket_id, + workflow_id=approval.workflow_id, + version_id=approval.version_id, step_number=approval.step_number, step_name=approval.step_name, + step_type=approval.step_type, approver_id=to_user, approver_role=approval.approver_role, - action="pending", + transfer_from=from_user, + transfer_reason=comment, + action=ApprovalAction.PENDING.value, ) self.db.add(new_approval) - - # 标记原审批为转交 - approval.action = "transferred" - approval.comment = f"转交给用户 {to_user}: {comment}" - approval.updated_at = datetime.now(timezone.utc) - await self.db.flush() + + # 记录日志 + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + event_type="transferred", + event_description=f"审批从用户 {from_user} 转交给用户 {to_user}", + actor_id=from_user, + context_data={"from_user": from_user, "to_user": to_user, "reason": comment}, + ) + return {"success": True, "message": "审批已转交", "new_approval_id": new_approval.id} - async def skip_step(self, approval_id: int, user_id: int, - reason: str = "") -> Dict[str, Any]: + async def skip_step( + self, + approval_id: int, + user_id: int, + reason: str = "", + ) -> Dict[str, Any]: """管理员跳过当前步骤。""" approval = await self._get_approval(approval_id) if not approval: return {"success": False, "message": "审批记录不存在"} - approval.action = "skipped" + # 验证管理员权限 + user = await self.db.get(User, user_id) + if not user or user.role not in ("admin", "dba"): + return {"success": False, "message": "无权跳过步骤"} + + approval.action = ApprovalAction.SKIPPED.value approval.approver_id = user_id approval.comment = f"跳过: {reason}" - approval.updated_at = datetime.now(timezone.utc) + approval.completed_at = datetime.now(timezone.utc) # 检查是否全部完成 ticket_approvals = await self._get_ticket_approvals(approval.ticket_id) current_step_num = approval.step_number all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) - if all_done: - return await self._complete_approval(approval.ticket_id, "approved") - return {"success": True, "message": "步骤已跳过", "new_status": "reviewing"} + result = {"success": True, "message": "步骤已跳过", "new_status": "reviewing"} + if all_done: + complete_result = await self._complete_approval(approval.ticket_id, WorkflowStatus.APPROVED) + result.update(complete_result) + + # 记录日志 + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="skipped", + event_description=f"步骤 {approval.step_number} 被跳过", + actor_id=user_id, + context_data={"reason": reason}, + ) + + return result + + async def interrupt_workflow( + self, + ticket_id: int, + user_id: int, + reason: str = "", + ) -> Dict[str, Any]: + """中断流程（暂停审批，可后续恢复）。 + + 用于需要补充材料或等待外部事件的场景。 + """ + ticket = await self._get_ticket(ticket_id) + if not ticket: + return {"success": False, "message": "工单不存在"} + + user = await self.db.get(User, user_id) + if not user or user.role not in ("admin", "dba"): + return {"success": False, "message": "无权中断流程"} + + # 将所有 pending 的审批标记为 interrupted + approvals = await self._get_ticket_approvals(ticket_id) + interrupted_count = 0 + for approval in approvals: + if approval.action == ApprovalAction.PENDING.value: + approval.action = ApprovalAction.INTERRUPTED.value + approval.comment = f"流程中断: {reason}" + interrupted_count += 1 + + # 更新工单状态 + ticket.status = WorkflowStatus.INTERRUPTED.value + + # 记录日志 + await self._log_event( + workflow_id=approvals[0].workflow_id if approvals else None, + ticket_id=ticket_id, + event_type="workflow_interrupted", + event_description=f"流程被中断: {reason}", + actor_id=user_id, + context_data={"reason": reason, "interrupted_steps": interrupted_count}, + ) + + await self.db.flush() + return { + "success": True, + "message": f"流程已中断，共 {interrupted_count} 个待审批步骤被暂停", + "interrupted_count": interrupted_count, + } + + async def resume_workflow( + self, + ticket_id: int, + user_id: int, + reason: str = "", + ) -> Dict[str, Any]: + """恢复被中断的流程。""" + ticket = await self._get_ticket(ticket_id) + if not ticket: + return {"success": False, "message": "工单不存在"} + + if ticket.status != WorkflowStatus.INTERRUPTED.value: + return {"success": False, "message": "当前流程状态不允许恢复"} + + user = await self.db.get(User, user_id) + if not user or user.role not in ("admin", "dba"): + return {"success": False, "message": "无权恢复流程"} + + # 将所有 interrupted 的审批恢复为 pending + approvals = await self._get_ticket_approvals(ticket_id) + resumed_count = 0 + for approval in approvals: + if approval.action == ApprovalAction.INTERRUPTED.value: + approval.action = ApprovalAction.PENDING.value + approval.comment = f"流程恢复: {reason}" + resumed_count += 1 + + # 恢复工单状态 + ticket.status = WorkflowStatus.RUNNING.value + + # 记录日志 + await self._log_event( + workflow_id=approvals[0].workflow_id if approvals else None, + ticket_id=ticket_id, + event_type="workflow_resumed", + event_description=f"流程已恢复: {reason}", + actor_id=user_id, + context_data={"reason": reason, "resumed_steps": resumed_count}, + ) + + await self.db.flush() + return { + "success": True, + "message": f"流程已恢复，共 {resumed_count} 个步骤重新激活", + "resumed_count": resumed_count, + } + + async def cancel_workflow( + self, + ticket_id: int, + user_id: int, + reason: str = "", + ) -> Dict[str, Any]: + """取消流程（不可恢复）。""" + ticket = await self._get_ticket(ticket_id) + if not ticket: + return {"success": False, "message": "工单不存在"} + + # 取消所有 pending 的审批 + approvals = await self._get_ticket_approvals(ticket_id) + for approval in approvals: + if approval.action in (ApprovalAction.PENDING.value, ApprovalAction.INTERRUPTED.value): + approval.action = ApprovalAction.REJECTED.value + approval.comment = f"流程取消: {reason}" + approval.completed_at = datetime.now(timezone.utc) + + # 更新工单状态 + ticket.status = WorkflowStatus.CANCELLED.value + + # 记录日志 + await self._log_event( + workflow_id=approvals[0].workflow_id if approvals else None, + ticket_id=ticket_id, + event_type="workflow_cancelled", + event_description=f"流程被取消: {reason}", + actor_id=user_id, + context_data={"reason": reason}, + ) + + await self.db.flush() + return {"success": True, "message": "流程已取消"} async def get_approval_progress(self, ticket_id: int) -> ApprovalProgress: - """获取工单的审批进度。""" - approvals = await self._get_ticket_approvals(ticket_id) - ticket_result = await self.db.execute(select(Ticket).where(Ticket.id == ticket_id)) + """获取工单的审批进度（优化版本 — 单次查询）。""" + # 使用预加载优化查询 + result = await self.db.execute( + select(TicketApproval, User.display_name.label("approver_name")) + .outerjoin(User, TicketApproval.approver_id == User.id) + .where(TicketApproval.ticket_id == ticket_id) + .order_by(TicketApproval.step_number) + ) + rows = result.all() + + ticket_result = await self.db.execute( + select(Ticket).where(Ticket.id == ticket_id) + ) ticket = ticket_result.scalar_one_or_none() steps_data = [] - for approval in approvals: - approver_name = "未分配" - if approval.approver_id: - user_result = await self.db.execute( - select(User).where(User.id == approval.approver_id) - ) - user = user_result.scalar_one_or_none() - approver_name = user.display_name if user else "未知" + current_step = 1 + total_steps = len(rows) + status = ticket.status if ticket else "unknown" + for approval, approver_name in rows: steps_data.append({ "step_number": approval.step_number, "step_name": approval.step_name, - "approver_name": approver_name, + "step_type": approval.step_type, + "approver_name": approver_name or ("未分配" if not approval.approver_id else "未知"), "approver_role": approval.approver_role, "action": approval.action, "comment": approval.comment, + "is_timeout": approval.is_timeout, "created_at": approval.created_at.isoformat() if approval.created_at else None, "updated_at": approval.updated_at.isoformat() if approval.updated_at else None, + "completed_at": approval.completed_at.isoformat() if approval.completed_at else None, }) + if approval.action == ApprovalAction.PENDING.value and current_step <= approval.step_number: + current_step = approval.step_number - current_step = 1 - total_steps = len(approvals) - status = ticket.status if ticket else "unknown" - - # 找到当前待审批的步骤 - for a in approvals: - if a.action == "pending": - current_step = a.step_number - break + # 是否可以中断/恢复 + can_interrupt = status in (WorkflowStatus.RUNNING.value, WorkflowStatus.PENDING.value) + can_resume = status == WorkflowStatus.INTERRUPTED.value return ApprovalProgress( ticket_id=ticket_id, @@ -239,75 +561,675 @@ class WorkflowEngine: total_steps=total_steps, status=status, steps=steps_data, + can_interrupt=can_interrupt, + can_resume=can_resume, ) async def auto_assign(self, step: WorkflowStep) -> Optional[int]: """基于负载自动分配审批人。 - 优先选择当前待审批任务最少的 DBA。 + 支持多种分配策略： + - load_balance: 选择当前待审批任务最少的用户 + - round_robin: 轮询分配 + - random: 随机分配 """ + assignment = step.approver_assignment or {} + strategy = assignment.get("type", "load_balance") + if step.approver_ids: - ids = json.loads(step.approver_ids) + ids = step.approver_ids if isinstance(step.approver_ids, list) else json.loads(step.approver_ids) if ids: - return ids[0] + if strategy == "round_robin": + # 简单的轮询：选择最近被分配次数最少的 + return await self._round_robin_assign(ids) + elif strategy == "random": + return random.choice(ids) + else: + # 负载均衡（默认） + return await self._load_balance_assign(ids) if step.approver_role: - # 查找该角色下负载最小的用户 result = await self.db.execute( - select(User.id, User.display_name) + select(User.id) .where(User.role == step.approver_role, User.is_active == True) ) - candidates = result.all() - if not candidates: - return None - - # 统计每个候选人的待审批数量 - workloads = [] - for user_id, _ in candidates: - count_result = await self.db.execute( - select(func.count(TicketApproval.id)) - .where(TicketApproval.approver_id == user_id, - TicketApproval.action == "pending") - ) - count = count_result.scalar() or 0 - workloads.append((user_id, count)) - - # 选择负载最小的 - workloads.sort(key=lambda x: x[1]) - return workloads[0][0] + candidates = [r[0] for r in result.all()] + if candidates: + if strategy == "random": + return random.choice(candidates) + return await self._load_balance_assign(candidates) return None - async def check_timeout_and_process(self): - """检查超时的审批并自动处理。""" + async def check_timeout_and_process(self) -> Dict[str, Any]: + """检查超时的审批并自动处理。 + + 批量处理，支持多种超时策略。 + + Returns: + 处理结果统计 + """ + stats = {"checked": 0, "auto_approved": 0, "auto_rejected": 0, "escalated": 0, "notified": 0} + + # 批量获取超时审批 timeout_threshold = datetime.now(timezone.utc) - timedelta(hours=24) result = await self.db.execute( - select(TicketApproval).where( - and_(TicketApproval.action == "pending", - TicketApproval.created_at < timeout_threshold) + select(TicketApproval, WorkflowStep) + .outerjoin( + WorkflowStep, + and_( + WorkflowStep.workflow_id == TicketApproval.workflow_id, + WorkflowStep.step_number == TicketApproval.step_number, + ) + ) + .where( + and_( + TicketApproval.action == ApprovalAction.PENDING.value, + TicketApproval.is_timeout == False, + TicketApproval.created_at < timeout_threshold, + ) ) ) - timed_out = result.scalars().all() + timed_out_approvals = result.all() - for approval in timed_out: - # 获取步骤定义，检查是否配置为超时自动通过 - workflow_result = await self.db.execute( - select(WorkflowStep).where(WorkflowStep.id == approval.step_number) - ) - step_def = workflow_result.scalar_one_or_none() + for approval, step_def in timed_out_approvals: + stats["checked"] += 1 + approval.is_timeout = True + approval.timeout_handled_at = datetime.now(timezone.utc) + + # 确定超时策略 + strategy = TimeoutStrategy.NOTIFY.value if step_def and step_def.auto_approve_on_timeout: - approval.action = "approved" - approval.comment = "超时自动通过" - approval.updated_at = datetime.now(timezone.utc) + strategy = TimeoutStrategy.AUTO_APPROVE.value + elif approval.workflow_id: + workflow = await self.db.get(WorkflowDefinition, approval.workflow_id) + if workflow and workflow.flow_config: + strategy = workflow.flow_config.get("timeout_strategy", TimeoutStrategy.NOTIFY.value) - # ── 内部辅助方法 ── + # 执行对应的超时处理 + handler = self._timeout_handlers.get(strategy, self._timeout_notify) + await handler(approval, step_def) + + if strategy == TimeoutStrategy.AUTO_APPROVE.value: + stats["auto_approved"] += 1 + elif strategy == TimeoutStrategy.AUTO_REJECT.value: + stats["auto_rejected"] += 1 + elif strategy == TimeoutStrategy.ESCALATE.value: + stats["escalated"] += 1 + else: + stats["notified"] += 1 + + await self.db.flush() + return stats + + # ═══════════════════════════════════════════════════════════ + # 步骤类型处理器 + # ═══════════════════════════════════════════════════════════ + + async def _handle_serial_step(self, approval: TicketApproval, action: str) -> Dict[str, Any]: + """处理串行审批步骤。""" + ticket_approvals = await self._get_ticket_approvals(approval.ticket_id) + current_step_num = approval.step_number + + all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) + if all_done: + return await self._complete_approval(approval.ticket_id, WorkflowStatus.APPROVED) + + return { + "success": True, + "new_status": "reviewing", + "message": "审批已通过，等待后续步骤", + "next_step": current_step_num + 1, + "step_completed": True, + } + + async def _handle_parallel_step(self, approval: TicketApproval, action: str) -> Dict[str, Any]: + """处理并行审批步骤（会签 — 需要所有审批人都通过）。""" + ticket_approvals = await self._get_ticket_approvals(approval.ticket_id) + current_step_num = approval.step_number + step_approvals = [a for a in ticket_approvals if a.step_number == current_step_num] + + if all(a.action in (ApprovalAction.APPROVED.value, ApprovalAction.SKIPPED.value, + ApprovalAction.AUTO_APPROVED.value) for a in step_approvals): + all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) + if all_done: + return await self._complete_approval(approval.ticket_id, WorkflowStatus.APPROVED) + return { + "success": True, + "new_status": "reviewing", + "message": "当前步骤并行审批已完成，等待后续步骤", + "next_step": current_step_num + 1, + "step_completed": True, + } + + pending_count = sum(1 for a in step_approvals if a.action == ApprovalAction.PENDING.value) + return { + "success": True, + "new_status": "reviewing", + "message": f"并行审批中，还需 {pending_count} 人审批", + "next_step": current_step_num, + "step_completed": False, + "pending_count": pending_count, + } + + async def _handle_any_one_step(self, approval: TicketApproval, action: str) -> Dict[str, Any]: + """处理或签审批步骤（任一审批人通过即可）。""" + ticket_approvals = await self._get_ticket_approvals(approval.ticket_id) + current_step_num = approval.step_number + + # 将同步骤其他 pending 标记为 skipped + step_approvals = [a for a in ticket_approvals if a.step_number == current_step_num] + for sa in step_approvals: + if sa.id != approval.id and sa.action == ApprovalAction.PENDING.value: + sa.action = ApprovalAction.SKIPPED.value + sa.comment = "或签模式：其他人已通过" + sa.completed_at = datetime.now(timezone.utc) + + all_done = await self._check_all_steps_done(ticket_approvals, current_step_num) + if all_done: + return await self._complete_approval(approval.ticket_id, WorkflowStatus.APPROVED) + + return { + "success": True, + "new_status": "reviewing", + "message": "当前步骤已有人审批通过，等待后续步骤", + "next_step": current_step_num + 1, + "step_completed": True, + } + + async def _handle_condition_step(self, approval: TicketApproval, action: str) -> Dict[str, Any]: + """处理条件分支步骤。""" + # 获取步骤定义 + result = await self.db.execute( + select(WorkflowStep).where(WorkflowStep.id == approval.id) + ) + step_def = result.scalar_one_or_none() + + if step_def and step_def.transition_conditions: + conditions = step_def.transition_conditions + condition_passed = await self._evaluate_conditions( + conditions, approval.ticket_id + ) + + if not condition_passed: + on_false = conditions.get("on_false_action", "skip") + if on_false == "reject": + return await self._complete_approval(approval.ticket_id, WorkflowStatus.REJECTED) + elif on_false == "goto_step": + goto_step = conditions.get("goto_step_number") + return { + "success": True, + "new_status": "reviewing", + "message": f"条件不满足，跳转至步骤 {goto_step}", + "next_step": goto_step, + "step_completed": True, + } + + return await self._handle_serial_step(approval, action) + + # ═══════════════════════════════════════════════════════════ + # 条件判断引擎 + # ═══════════════════════════════════════════════════════════ + + async def _evaluate_trigger_conditions( + self, + conditions_config: Dict[str, Any], + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> bool: + """评估流程触发条件。""" + conditions = conditions_config.get("conditions", []) + logic = conditions_config.get("logic", "AND").upper() + return await self._evaluate_condition_list(conditions, logic, ticket_id, context) + + async def _evaluate_conditions( + self, + conditions_config: Dict[str, Any], + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> bool: + """评估条件列表。""" + conditions = conditions_config.get("conditions", []) + logic = conditions_config.get("logic", "AND").upper() + return await self._evaluate_condition_list(conditions, logic, ticket_id, context) + + async def _evaluate_condition_list( + self, + conditions: List[Dict[str, Any]], + logic: str, + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> bool: + """评估条件列表，支持 AND/OR 逻辑。""" + if not conditions: + return True + + results = [] + for cond in conditions: + result = await self._evaluate_single_condition(cond, ticket_id, context) + results.append(result.passed) + + if logic == "AND": + return all(results) + elif logic == "OR": + return any(results) + return all(results) + + async def _evaluate_single_condition( + self, + condition: Dict[str, Any], + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> ConditionResult: + """评估单个条件，支持丰富的运算符和字段引用。 + + 支持的运算符: + == / != / > / >= / < / <= / in / not_in / contains / starts_with / ends_with + regex / exists / empty + """ + field = condition.get("field", "") + operator = condition.get("operator", "==") + expected = condition.get("value") + + # 获取字段实际值 + actual = await self._resolve_field_value(field, ticket_id, context) + + passed = False + message = "" + + try: + if operator == "==": + passed = actual == expected + elif operator == "!=": + passed = actual != expected + elif operator == ">": + passed = actual is not None and expected is not None and actual > expected + elif operator == ">=": + passed = actual is not None and expected is not None and actual >= expected + elif operator == "<": + passed = actual is not None and expected is not None and actual < expected + elif operator == "<=": + passed = actual is not None and expected is not None and actual <= expected + elif operator == "in": + passed = actual in expected if isinstance(expected, (list, tuple, set)) else False + elif operator == "not_in": + passed = actual not in expected if isinstance(expected, (list, tuple, set)) else True + elif operator == "contains": + passed = expected in actual if isinstance(actual, (str, list, dict)) else False + elif operator == "starts_with": + passed = str(actual).startswith(str(expected)) + elif operator == "ends_with": + passed = str(actual).endswith(str(expected)) + elif operator == "regex": + passed = bool(re.search(str(expected), str(actual))) + elif operator == "exists": + passed = actual is not None + elif operator == "empty": + passed = actual is None or (isinstance(actual, (str, list, dict)) and len(actual) == 0) + else: + message = f"未知运算符: {operator}" + passed = False + except Exception as e: + message = f"条件评估异常: {e}" + passed = False + + return ConditionResult( + passed=passed, + field=field, + operator=operator, + expected=expected, + actual=actual, + message=message, + ) + + async def _resolve_field_value( + self, + field: str, + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> Any: + """解析字段引用，支持多层嵌套访问。 + + 支持格式: + - ticket.title 工单字段 + - ticket.sql_type 工单SQL类型 + - ticket.risk_level 工单风险等级 + - ticket.creator_id 工单创建者 + - database.instance_id 数据源实例ID + - database.name 数据库名称 + - context.xxx 上下文数据 + - env.xxx 环境变量 + """ + parts = field.split(".") + if len(parts) < 2: + # 尝试从上下文直接获取 + if context and field in context: + return context[field] + return None + + source = parts[0] + attr_path = parts[1:] + + if source == "ticket": + ticket = await self._get_ticket(ticket_id) + if not ticket: + return None + value = ticket + for attr in attr_path: + if value is None: + return None + if isinstance(value, dict): + value = value.get(attr) + else: + value = getattr(value, attr, None) + return value + + elif source == "context": + if context is None: + return None + value = context + for attr in attr_path: + if value is None: + return None + if isinstance(value, dict): + value = value.get(attr) + else: + return None + return value + + elif source == "env": + # 从工单获取环境相关数据 + attr_name = ".".join(attr_path) + ticket = await self._get_ticket(ticket_id) + if ticket and hasattr(ticket, attr_name): + return getattr(ticket, attr_name) + return None + + return None + + # ═══════════════════════════════════════════════════════════ + # 审批人分配 + # ═══════════════════════════════════════════════════════════ + + async def _resolve_approver( + self, + step: WorkflowStep, + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> ApproverAssignment: + """解析审批人分配策略。""" + assignment = step.approver_assignment or {} + strategy = assignment.get("type", "role") + + if strategy == "specific" and step.approver_ids: + ids = step.approver_ids if isinstance(step.approver_ids, list) else json.loads(step.approver_ids) + if ids: + return ApproverAssignment(user_id=ids[0], assignment_type="specific") + + elif strategy == "role" and step.approver_role: + result = await self.db.execute( + select(User.id).where(User.role == step.approver_role, User.is_active == True) + ) + candidates = [r[0] for r in result.all()] + if candidates: + user_id = await self._load_balance_assign(candidates) + return ApproverAssignment(user_id=user_id, assignment_type="role") + + elif strategy == "load_balance" and step.approver_ids: + ids = step.approver_ids if isinstance(step.approver_ids, list) else json.loads(step.approver_ids) + if ids: + user_id = await self._load_balance_assign(ids) + return ApproverAssignment(user_id=user_id, assignment_type="load_balance") + + elif strategy == "leader": + # 基于工单创建者的上级 + ticket = await self._get_ticket(ticket_id) + if ticket and ticket.creator_id: + # 简化实现：查找同角色的 DBA/Admin + result = await self.db.execute( + select(User.id).where( + User.role.in_(["admin", "dba"]), + User.is_active == True, + ).limit(1) + ) + leader = result.scalar_one_or_none() + if leader: + return ApproverAssignment(user_id=leader, assignment_type="leader") + + elif strategy == "creator": + ticket = await self._get_ticket(ticket_id) + if ticket: + return ApproverAssignment(user_id=ticket.creator_id, assignment_type="creator") + + elif strategy == "expression": + expression = assignment.get("expression", "") + resolved = await self._resolve_expression(expression, ticket_id, context) + if resolved: + return ApproverAssignment(user_id=resolved, assignment_type="expression") + # fallback + fallback = assignment.get("fallback_user_id") + if fallback: + return ApproverAssignment(user_id=fallback, assignment_type="expression", reason="expression_fallback") + + # 默认策略：自动分配 + user_id = await self.auto_assign(step) + return ApproverAssignment(user_id=user_id, assignment_type="auto") + + async def _resolve_expression( + self, + expression: str, + ticket_id: int, + context: Optional[Dict[str, Any]] = None, + ) -> Optional[int]: + """解析表达式获取审批人ID。 + + 支持: ${ticket.creator.manager_id} 等嵌套引用 + """ + if not expression: + return None + + # 提取变量引用 ${...} + pattern = r'\$\{([^}]+)\}' + match = re.search(pattern, expression) + if not match: + # 直接返回整数值 + try: + return int(expression) + except ValueError: + return None + + var_path = match.group(1) + value = await self._resolve_field_value(var_path, ticket_id, context) + return int(value) if value is not None else None + + async def _load_balance_assign(self, user_ids: List[int]) -> Optional[int]: + """基于负载均衡选择审批人（选择待审批最少的用户）。""" + if not user_ids: + return None + if len(user_ids) == 1: + return user_ids[0] + + workloads = [] + for uid in user_ids: + count_result = await self.db.execute( + select(func.count(TicketApproval.id)) + .where( + TicketApproval.approver_id == uid, + TicketApproval.action == ApprovalAction.PENDING.value, + ) + ) + count = count_result.scalar() or 0 + workloads.append((uid, count)) + + workloads.sort(key=lambda x: x[1]) + return workloads[0][0] + + async def _round_robin_assign(self, user_ids: List[int]) -> Optional[int]: + """轮询分配审批人。""" + if not user_ids: + return None + + # 选择最近被分配次数最少的 + counts = [] + for uid in user_ids: + count_result = await self.db.execute( + select(func.count(TicketApproval.id)) + .where( + TicketApproval.approver_id == uid, + TicketApproval.created_at > datetime.now(timezone.utc) - timedelta(days=7), + ) + ) + count = count_result.scalar() or 0 + counts.append((uid, count)) + + counts.sort(key=lambda x: x[1]) + return counts[0][0] + + # ═══════════════════════════════════════════════════════════ + # 超时处理策略 + # ═══════════════════════════════════════════════════════════ + + async def _timeout_auto_approve(self, approval: TicketApproval, step_def: Optional[WorkflowStep] = None): + """超时自动通过。""" + approval.action = ApprovalAction.AUTO_APPROVED.value + approval.comment = "超时自动通过" + approval.completed_at = datetime.now(timezone.utc) + + # 触发步骤完成检查 + await self._handle_serial_step(approval, "timeout") + + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="timeout", + event_description=f"步骤 {approval.step_number} 超时自动通过", + context_data={"strategy": "auto_approve"}, + ) + + async def _timeout_auto_reject(self, approval: TicketApproval, step_def: Optional[WorkflowStep] = None): + """超时自动拒绝。""" + approval.action = ApprovalAction.REJECTED.value + approval.comment = "超时自动拒绝" + approval.completed_at = datetime.now(timezone.utc) + + await self._complete_approval(approval.ticket_id, WorkflowStatus.REJECTED) + + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="timeout", + event_description=f"步骤 {approval.step_number} 超时自动拒绝", + context_data={"strategy": "auto_reject"}, + ) + + async def _timeout_escalate(self, approval: TicketApproval, step_def: Optional[WorkflowStep] = None): + """超时升级处理。""" + # 将审批升级给管理员 + result = await self.db.execute( + select(User.id).where(User.role == "admin", User.is_active == True).limit(1) + ) + admin_id = result.scalar_one_or_none() + + if admin_id: + approval.approver_id = admin_id + approval.comment = f"超时升级，转交给管理员" + # 重置超时标记，给管理员新的处理时间 + approval.is_timeout = False + approval.created_at = datetime.now(timezone.utc) + + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="escalated", + event_description=f"步骤 {approval.step_number} 超时升级给管理员 {admin_id}", + context_data={"strategy": "escalate", "escalated_to": admin_id}, + ) + + async def _timeout_notify(self, approval: TicketApproval, step_def: Optional[WorkflowStep] = None): + """超时仅通知（不自动处理）。""" + await self._log_event( + workflow_id=approval.workflow_id, + ticket_id=approval.ticket_id, + approval_id=approval.id, + event_type="timeout", + event_description=f"步骤 {approval.step_number} 审批超时，等待人工处理", + context_data={"strategy": "notify"}, + ) + + async def _schedule_timeout_check(self, ticket_id: int): + """调度超时检查（异步任务）。""" + await asyncio.sleep(3600) # 1小时后检查 + await self.check_timeout_and_process() + + # ═══════════════════════════════════════════════════════════ + # 日志记录 + # ═══════════════════════════════════════════════════════════ + + async def _log_event( + self, + event_type: str, + event_description: str = "", + workflow_id: Optional[int] = None, + ticket_id: Optional[int] = None, + step_id: Optional[int] = None, + approval_id: Optional[int] = None, + actor_id: Optional[int] = None, + context_data: Optional[Dict[str, Any]] = None, + ) -> None: + """记录流程执行日志。""" + # 获取执行者名称 + actor_name = None + if actor_id: + user = await self.db.get(User, actor_id) + if user: + actor_name = user.display_name or user.username + + log = WorkflowExecutionLog( + workflow_id=workflow_id, + ticket_id=ticket_id, + step_id=step_id, + approval_id=approval_id, + event_type=event_type, + event_description=event_description, + actor_id=actor_id, + actor_name=actor_name, + context_data=context_data, + ) + self.db.add(log) + await self.db.flush() + + # ═══════════════════════════════════════════════════════════ + # 内部辅助方法 + # ═══════════════════════════════════════════════════════════ + + async def _get_workflow(self, workflow_id: Optional[int]) -> Optional[WorkflowDefinition]: + """获取流程定义，优先使用指定ID，否则获取默认流程。""" + if workflow_id: + result = await self.db.execute( + select(WorkflowDefinition) + .options(selectinload(WorkflowDefinition.steps)) + .where(WorkflowDefinition.id == workflow_id, WorkflowDefinition.is_active == True) + ) + workflow = result.unique().scalar_one_or_none() + if workflow: + return workflow + + # 获取默认流程 + result = await self.db.execute( + select(WorkflowDefinition) + .options(selectinload(WorkflowDefinition.steps)) + .where(WorkflowDefinition.is_default == True, WorkflowDefinition.is_active == True) + ) + return result.unique().scalar_one_or_none() async def _get_default_workflow(self) -> Optional[WorkflowDefinition]: """获取默认审批流程。""" result = await self.db.execute( select(WorkflowDefinition) - .where(WorkflowDefinition.is_default == True, - WorkflowDefinition.is_active == True) + .where(WorkflowDefinition.is_default == True, WorkflowDefinition.is_active == True) ) return result.scalar_one_or_none() @@ -317,6 +1239,12 @@ class WorkflowEngine: ) return result.scalar_one_or_none() + async def _get_ticket(self, ticket_id: int) -> Optional[Ticket]: + result = await self.db.execute( + select(Ticket).where(Ticket.id == ticket_id) + ) + return result.scalar_one_or_none() + async def _get_ticket_approvals(self, ticket_id: int) -> List[TicketApproval]: result = await self.db.execute( select(TicketApproval) @@ -325,35 +1253,67 @@ class WorkflowEngine: ) return result.scalars().all() - async def _check_all_steps_done(self, approvals: List[TicketApproval], - current_step: int) -> bool: + async def _check_all_steps_done( + self, + approvals: List[TicketApproval], + current_step: int, + ) -> bool: """检查当前步骤及之前的所有步骤是否都已完成。""" for a in approvals: - if a.step_number <= current_step and a.action in ("pending",): + if a.step_number <= current_step and a.action == ApprovalAction.PENDING.value: return False return True - async def _complete_approval(self, ticket_id: int, - final_status: str) -> Dict[str, Any]: + async def _check_approve_permission(self, approval: TicketApproval, user_id: int) -> bool: + """检查用户是否有审批权限。""" + # 审批人本人 + if approval.approver_id == user_id: + return True + + # 管理员可以代理审批 + user = await self.db.get(User, user_id) + if user and user.role == "admin": + return True + + # 角色匹配（如果未指定具体审批人） + if approval.approver_role and not approval.approver_id: + if user and user.role == approval.approver_role: + return True + + return False + + async def _complete_approval( + self, + ticket_id: int, + final_status: WorkflowStatus, + ) -> Dict[str, Any]: """完成审批，更新工单状态。""" - ticket_result = await self.db.execute( - select(Ticket).where(Ticket.id == ticket_id) - ) - ticket = ticket_result.scalar_one_or_none() + ticket = await self._get_ticket(ticket_id) if ticket: - ticket.status = final_status - if final_status == "approved": + ticket.status = final_status.value + if final_status == WorkflowStatus.APPROVED: ticket.review_time = datetime.now(timezone.utc) await self.db.flush() + + # 记录完成日志 + await self._log_event( + workflow_id=None, + ticket_id=ticket_id, + event_type="workflow_completed" if final_status == WorkflowStatus.APPROVED else f"workflow_{final_status.value}", + event_description=f"审批流程已完成，工单状态: {final_status.value}", + context_data={"final_status": final_status.value}, + ) + return { "success": True, - "new_status": final_status, - "message": f"审批流程已完成，工单状态: {final_status}", + "new_status": final_status.value, + "message": f"审批流程已完成，工单状态: {final_status.value}", "next_step": None, + "step_completed": True, } @property def step_type(self) -> str: """获取当前步骤类型。""" - return "serial" # 简化实现 + return "serial" diff --git a/backend/tests/test_audit_engine.py b/backend/tests/test_audit_engine.py index d009b01..c6ff34e 100644 --- a/backend/tests/test_audit_engine.py +++ b/backend/tests/test_audit_engine.py @@ -13,26 +13,25 @@ class TestSQLAuditEngine: report = self.engine.audit("DROP TABLE users;") assert not report.passed or report.has_warnings rules = [f.rule for f in report.findings] - assert "DDL_IF_EXISTS" in rules - assert "DROP_TABLE" in rules + assert "DROP_TABLE_DANGER" in rules def test_drop_table_production_blocked(self): engine = SQLAuditEngine(env="production") report = engine.audit("DROP TABLE users;") - assert not report.passed - errors = [f for f in report.findings if f.level == Severity.ERROR] - assert any(f.rule == "DROP_TABLE_PRODUCTION" for f in errors) + assert report.has_warnings or not report.passed + warnings = [f for f in report.findings if f.rule == "DROP_TABLE_DANGER"] + assert len(warnings) > 0 def test_truncate_warning(self): report = self.engine.audit("TRUNCATE TABLE logs;") rules = [f.rule for f in report.findings] - assert "TRUNCATE" in rules + assert "TRUNCATE_DANGER" in rules def test_truncate_production_error(self): engine = SQLAuditEngine(env="production") report = engine.audit("TRUNCATE TABLE logs;") errors = [f for f in report.findings if f.level == Severity.ERROR] - assert any(f.rule == "TRUNCATE" for f in errors) + assert any(f.rule == "TRUNCATE_DANGER" for f in errors) # --- Missing WHERE --- @@ -92,12 +91,12 @@ class TestSQLAuditEngine: sql = f"SELECT * FROM users WHERE id IN ({values});" report = self.engine.audit(sql) rules = [f.rule for f in report.findings] - assert "LARGE_IN" in rules + assert "LARGE_IN_CLAUSE" in rules def test_small_in_ok(self): report = self.engine.audit("SELECT * FROM users WHERE id IN (1, 2, 3);") rules = [f.rule for f in report.findings] - assert "LARGE_IN" not in rules + assert "LARGE_IN_CLAUSE" not in rules # --- Production extra rules --- diff --git a/backend/tests/test_tickets.py b/backend/tests/test_tickets.py index 5b713d4..cfe60d4 100644 --- a/backend/tests/test_tickets.py +++ b/backend/tests/test_tickets.py @@ -50,7 +50,7 @@ class TestCreateTicket: async def test_create_ticket_instance_not_found(self, client: AsyncClient, developer_token): resp = await _create_ticket(client, developer_token, instance_id=9999) - assert resp.status_code == 400 + assert resp.status_code == 404 class TestGetTicket: diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts index 9172bde..f3a5e87 100644 --- a/frontend/src/api/auth.ts +++ b/frontend/src/api/auth.ts @@ -9,6 +9,9 @@ export interface LoginResponse { display_name: string; role: string; email: string | null; + data_source_permissions?: string; + custom_permissions?: string; + group_ids?: number[]; }; } @@ -18,6 +21,9 @@ export interface MeResponse { display_name: string; role: string; email: string | null; + data_source_permissions?: string; + custom_permissions?: string; + group_ids?: number[]; } export const authApi = { diff --git a/frontend/src/components/SqlEditor.tsx b/frontend/src/components/SqlEditor.tsx index 9a271c8..5c54c9a 100644 --- a/frontend/src/components/SqlEditor.tsx +++ b/frontend/src/components/SqlEditor.tsx @@ -13,7 +13,6 @@ import { } from '@codemirror/language'; import { closeBrackets, closeBracketsKeymap } from '@codemirror/autocomplete'; import { searchKeymap, highlightSelectionMatches } from '@codemirror/search'; -import { rectangularSelection } from '@codemirror/rectangular-selection'; // --------------------------------------------------------------------------- // Types @@ -296,7 +295,6 @@ const SqlEditor: React.FC = ({ bracketMatching(), closeBrackets(), highlightSelectionMatches(), - rectangularSelection(), syntaxHighlighting(defaultHighlightStyle, { fallback: true }), appTheme, oneDark, diff --git a/frontend/src/hooks/useAuth.tsx b/frontend/src/hooks/useAuth.tsx index ece4370..0b46756 100644 --- a/frontend/src/hooks/useAuth.tsx +++ b/frontend/src/hooks/useAuth.tsx @@ -335,7 +335,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { table?: string ): boolean => { if (!isLoggedIn) return false; - if (isAdmin) return true; // 管理员拥有所有数据源权限 + if (user.role === 'admin') return true; // 管理员拥有所有数据源权限 const cache = permissionCacheRef.current; const cacheKey = `${instanceId}:${action}:${database || '*'}:${table || '*'}`; @@ -368,7 +368,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { cache.dataSource.set(cacheKey, hasPerm); cache.timestamp = Date.now(); return hasPerm; - }, [user.dataSourcePermissions, user.role, isAdmin, isLoggedIn, isCacheValid]); + }, [user.dataSourcePermissions, user.role, isLoggedIn, isCacheValid]); /** 批量检查数据源权限 */ const checkDataSourcePermissions = useCallback(( @@ -387,34 +387,31 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { /** 从服务器刷新用户权限 */ const refreshPermissions = useCallback(async (): Promise => { try { - const res = await authApi.getCurrentUser(); - if (res.user) { - const u = res.user; - const role = u.role as UserRole; - const dataSourcePermissions: DataSourcePermission[] = u.data_source_permissions - ? JSON.parse(u.data_source_permissions) - : []; - const customPermissions: Record = u.custom_permissions - ? JSON.parse(u.custom_permissions) - : {}; + const u = await authApi.getMe(); + const role = u.role as UserRole; + const dataSourcePermissions: DataSourcePermission[] = u.data_source_permissions + ? JSON.parse(u.data_source_permissions) + : []; + const customPermissions: Record = u.custom_permissions + ? JSON.parse(u.custom_permissions) + : {}; - const newUser: AuthUser = { - ...user, - id: u.id, - username: u.username, - email: u.email || '', - role, - roleName: roleNameMap[role] || role, - groupIds: u.group_ids || [], - dataSourcePermissions, - customPermissions, - permissionsUpdatedAt: Date.now(), - }; + const newUser: AuthUser = { + ...user, + id: u.id, + username: u.username, + email: u.email || '', + role, + roleName: roleNameMap[role] || role, + groupIds: u.group_ids || [], + dataSourcePermissions, + customPermissions, + permissionsUpdatedAt: Date.now(), + }; - setUser(newUser); - clearPermissionCache(); - emitPermissionChange(newUser); - } + setUser(newUser); + clearPermissionCache(); + emitPermissionChange(newUser); } catch (error) { console.error('刷新权限失败:', error); } diff --git a/frontend/src/pages/AuditLogsPage.tsx b/frontend/src/pages/AuditLogsPage.tsx index 45f1de1..766c01f 100644 --- a/frontend/src/pages/AuditLogsPage.tsx +++ b/frontend/src/pages/AuditLogsPage.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { - Search, Download, Filter, X, ChevronDown, ChevronRight, + Search, Download, Filter, ChevronDown, ChevronRight, FileText, User, Globe, CheckCircle2, XCircle, AlertTriangle, BarChart3, Clock, RotateCcw, } from 'lucide-react'; @@ -131,7 +131,7 @@ function exportLogsToCSV(logs: AuditLog[]) { export default function AuditLogsPage() { const [logs, setLogs] = useState([]); - const [total, setTotal] = useState(0); + const [, setTotal] = useState(0); const [page, setPage] = useState(1); const [loading, setLoading] = useState(true); const pageSize = 20; diff --git a/frontend/src/pages/AuditRulesPage.tsx b/frontend/src/pages/AuditRulesPage.tsx index 7e76b7c..d75daf3 100644 --- a/frontend/src/pages/AuditRulesPage.tsx +++ b/frontend/src/pages/AuditRulesPage.tsx @@ -2,8 +2,8 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import request from '../api/request'; import { Search, Filter, ChevronDown, ChevronUp, AlertTriangle, - Info, CheckCircle, XCircle, Database, Table, FileSearch, - Shield, Loader2, Save, RotateCcw, Leaf, Server, + Info, XCircle, Database, Table, FileSearch, + Shield, Loader2, Leaf, Server, } from 'lucide-react'; /* ------------------------------------------------------------------ */ @@ -136,7 +136,7 @@ export default function AuditRulesPage() { const res = await request.get('/audit-rules', { params: { db_type: dbType }, }); - const data = (res as { data?: unknown[] }).data ?? (res as unknown[]); + const data = (res as { data?: unknown[] }).data ?? (res as unknown as unknown[]); setRules(Array.isArray(data) ? data as AuditRule[] : []); } catch { setRules([]); diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 52de22e..70597f7 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -2,9 +2,9 @@ import { useEffect, useRef, useState, useCallback, useMemo } from 'react'; import { ClipboardList, Clock, CheckCircle, AlertTriangle, Database, ChevronRight, X, AlertCircle, Info, Bell, - Activity, TrendingUp, Zap, Server, Plus, Search, + Activity, TrendingUp, Zap, Server, Search, FileText, ArrowUpRight, ArrowDownRight, Minus, - Filter, Eye, Loader2, BarChart3, + Filter, Eye, BarChart3, } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { dbInstances, statusMap, statusColorMap, dbTypeColorMap, tickets, announcements } from '@/lib/mock'; diff --git a/frontend/src/pages/DataDictionaryPage.tsx b/frontend/src/pages/DataDictionaryPage.tsx index 94eb312..5572827 100644 --- a/frontend/src/pages/DataDictionaryPage.tsx +++ b/frontend/src/pages/DataDictionaryPage.tsx @@ -1,7 +1,7 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { BookOpen, Database, Table2, Search, Download, Columns3, Key, Code2, RefreshCw, - ChevronRight, ChevronDown, Hash, AlignLeft, Gauge, FolderOpen, ArrowRight, + ChevronRight, Hash, AlignLeft, Gauge, FolderOpen, ArrowRight, FileJson, TableProperties, Info, } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; @@ -173,7 +173,6 @@ export default function DataDictionaryPage() { const [activeTab, setActiveTab] = useState('columns'); const [loading, setLoading] = useState(false); const [expandedDbs, setExpandedDbs] = useState>(new Set()); - const [dbTreeExpanded, setDbTreeExpanded] = useState(true); // Load instances useEffect(() => { diff --git a/frontend/src/pages/InstanceGroupPage.tsx b/frontend/src/pages/InstanceGroupPage.tsx index 94f5ea3..be1e050 100644 --- a/frontend/src/pages/InstanceGroupPage.tsx +++ b/frontend/src/pages/InstanceGroupPage.tsx @@ -1,8 +1,8 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { Plus, Search, Edit2, Trash2, Server, Users, ChevronRight, X, - XCircle, Shield, Grid3x3, BarChart3, Clock, CheckCircle2, - Database, HardDrive, Cpu, Activity, Filter, RotateCcw, ChevronDown, + Shield, Grid3x3, Clock, CheckCircle2, + Database, HardDrive, Activity, } from 'lucide-react'; import { Card, CardContent } from '@/components/ui/card'; import { instanceGroupApi, type InstanceGroup } from '@/api/instanceGroup'; diff --git a/frontend/src/pages/QueriesPage.tsx b/frontend/src/pages/QueriesPage.tsx index b8d4884..a9eebf5 100644 --- a/frontend/src/pages/QueriesPage.tsx +++ b/frontend/src/pages/QueriesPage.tsx @@ -2,8 +2,8 @@ import { useState, useEffect, useMemo, useCallback, useRef } from 'react'; import { Plus, X, AlertTriangle, CheckCircle, Play, Loader2, ChevronDown, Clock, FileText, Sparkles, Star, - StarOff, Download, Search, RotateCcw, ArrowUpDown, - ArrowUp, ArrowDown, Eye, Hash, PanelLeftClose, PanelLeft, + StarOff, Download, Search, ArrowUpDown, + ArrowUp, ArrowDown, Eye, Hash, PanelLeftClose, } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { useActionPermission } from '@/hooks/useAuth'; @@ -12,7 +12,7 @@ import EmptyState from '@/components/EmptyState'; import { queryApi } from '@/api/query'; import { instanceApi } from '@/api/instance'; import SqlEditor from '@/components/SqlEditor'; -import { queryTemplates, getTemplateSQL, getTemplatesForType } from '@/lib/queryTemplates'; +import { getTemplateSQL, getTemplatesForType } from '@/lib/queryTemplates'; import type { DBType, QueryTemplate } from '@/lib/queryTemplates'; /* ------------------------------------------------------------------ */ diff --git a/frontend/src/pages/SlowQueryPage.tsx b/frontend/src/pages/SlowQueryPage.tsx index 243d7b5..00721d5 100644 --- a/frontend/src/pages/SlowQueryPage.tsx +++ b/frontend/src/pages/SlowQueryPage.tsx @@ -2,7 +2,7 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { Search, Clock, Database, AlertTriangle, ChevronDown, ChevronRight, BarChart3, Play, Loader2, RefreshCw, Download, Lightbulb, X, - GitCompare, XCircle, FileText, Skull, Zap, CheckCircle2, + GitCompare, XCircle, Skull, Zap, CheckCircle2, } from 'lucide-react'; import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, CartesianGrid, diff --git a/frontend/src/pages/TicketCreatePage.tsx b/frontend/src/pages/TicketCreatePage.tsx index 8cc5492..3b5a8bb 100644 --- a/frontend/src/pages/TicketCreatePage.tsx +++ b/frontend/src/pages/TicketCreatePage.tsx @@ -174,7 +174,7 @@ export default function TicketCreatePage() { return; } setLoadingDatabases(true); - instanceApi.getDatabases?.(Number(instanceId)) + instanceApi.listDatabases(Number(instanceId)) .then((res: unknown) => { const data = (res as { databases?: string[] })?.databases || (res as string[]) @@ -240,39 +240,6 @@ export default function TicketCreatePage() { } }, [sqlContent, instanceId, database]); - /* ---------------------------------------------------------------- */ - /* Pre-submit audit check */ - /* ---------------------------------------------------------------- */ - - const handlePreSubmitAudit = useCallback(async () => { - if (!title.trim() || !sqlContent.trim() || !instanceId || !database || !reason.trim()) { - setError('请填写所有必填项'); - return; - } - // If already audited and SQL hasn't changed, go directly to step 3 - if (auditResult) { - setStep(3); - return; - } - // Auto-audit before submission - setAuditing(true); - setError(''); - try { - const result = await ticketApi.audit({ - sql_content: sqlContent, - instance_id: Number(instanceId), - database, - }) as unknown as AuditResult; - setAuditResult(result); - setStep(3); - } catch (e: unknown) { - const err = e as { response?: { data?: { detail?: string } } }; - setError(err?.response?.data?.detail || '预审核失败，请检查 SQL 语法'); - } finally { - setAuditing(false); - } - }, [title, sqlContent, instanceId, database, reason, auditResult]); - /* ---------------------------------------------------------------- */ /* Submit handler */ /* ---------------------------------------------------------------- */ diff --git a/frontend/src/pages/TicketDetailPage.tsx b/frontend/src/pages/TicketDetailPage.tsx index aa06227..9be2aa2 100644 --- a/frontend/src/pages/TicketDetailPage.tsx +++ b/frontend/src/pages/TicketDetailPage.tsx @@ -198,15 +198,13 @@ export default function TicketDetailPage() { /* Permission checks */ /* ---------------------------------------------------------------- */ - const { isAdminOrDba, isApplicant, canApprove, canExecute, canRollback, canCancel } = useMemo(() => { + const { canApprove, canExecute, canRollback, canCancel } = useMemo(() => { if (!ticket) { - return { isAdminOrDba: false, isApplicant: false, canApprove: false, canExecute: false, canRollback: false, canCancel: false }; + return { canApprove: false, canExecute: false, canRollback: false, canCancel: false }; } const adminOrDba = auth.user.role === 'admin' || auth.user.role === 'dba'; const applicant = ticket.applicant_id === auth.user.id; return { - isAdminOrDba: adminOrDba, - isApplicant: applicant, canApprove: adminOrDba && (ticket.status === 'pending' || ticket.status === 'reviewing'), canExecute: adminOrDba && ticket.status === 'approved', canRollback: adminOrDba && (ticket.status === 'success' || ticket.status === 'failed'), diff --git a/frontend/src/pages/TicketsPage.tsx b/frontend/src/pages/TicketsPage.tsx index 9134d57..7922895 100644 --- a/frontend/src/pages/TicketsPage.tsx +++ b/frontend/src/pages/TicketsPage.tsx @@ -1,9 +1,9 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { Plus, Search, Eye, CheckCircle, XCircle, Play, RotateCcw, - ChevronRight, FileCode, History, Undo2, XCircle as XIcon, + FileCode, History, Undo2, XCircle as XIcon, Clock, AlertTriangle, AlertCircle, Info, ChevronLeft, - Download, Filter, X, Loader2, BarChart3, Users, Calendar, + Filter, X, Loader2, Users, Calendar, CheckSquare, Square, ArrowUpDown, FileDown, Ban, } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; @@ -540,6 +540,7 @@ export default function TicketsPage() { ? : } + 工单号 标题 diff --git a/frontend/src/pages/UsersPage.tsx b/frontend/src/pages/UsersPage.tsx index 363a2e1..258da1f 100644 --- a/frontend/src/pages/UsersPage.tsx +++ b/frontend/src/pages/UsersPage.tsx @@ -2,10 +2,10 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { Plus, Search, Edit2, Trash2, XCircle, Eye, Download, Upload, CheckSquare, Square, X, RotateCcw, KeyRound, ChevronDown, - UserCheck, UserX, Filter, MoreHorizontal, Lock, + UserCheck, UserX, Filter, } from 'lucide-react'; import { Card, CardContent } from '@/components/ui/card'; -import { roles, users as mockUsers } from '@/lib/mock'; +import { users as mockUsers } from '@/lib/mock'; import { userApi } from '@/api/user'; /* ───────── Types ───────── */ diff --git a/frontend/src/pages/WorkflowPage.tsx b/frontend/src/pages/WorkflowPage.tsx index 3f661fc..a6b88ba 100644 --- a/frontend/src/pages/WorkflowPage.tsx +++ b/frontend/src/pages/WorkflowPage.tsx @@ -3,7 +3,7 @@ import { Plus, Edit2, Eye, ToggleLeft, ToggleRight, GitBranch, Users, CheckCircle, ArrowRight, Shield, Trash2, X, ChevronDown, ChevronUp, Save, Play, Copy, History, TestTube, AlertTriangle, - Loader2, Check, XCircle, MessageSquare, + Loader2, Check, XCircle, } from 'lucide-react'; import { Card, CardContent } from '@/components/ui/card'; import { workflows as initialWorkflows, dbTypeColorMap } from '@/lib/mock'; diff --git a/merge-analysis-report.md b/merge-analysis-report.md new file mode 100644 index 0000000..9c604f1 --- /dev/null +++ b/merge-analysis-report.md @@ -0,0 +1,238 @@ +# 优化包合并分析报告 + +## 📋 总览 + +| 文件 | 行数变化 | 冲突模型引用 | 安全合并? | 策略 | +|------|---------|------------|----------|------| +| models/user.py | 23→299 (+276) | ⚠️ **核心冲突** | ❌ | 手动合并 | +| models/workflow.py | 55→362 (+307) | ✅ 无 | ✅ | 直接覆盖 | +| services/workflow_engine.py | 359→1319 (+960) | ✅ 无 | ✅ | 直接覆盖 | +| services/query_service.py | 709→1253 (+544) | ✅ 无 | ✅ | 直接覆盖 | +| services/audit_engine.py | 234→790 (+556) | ✅ 无 | ✅ | 直接覆盖 | +| services/ticket_executor.py | 1384→1688 (+304) | ✅ 无 | ✅ | 直接覆盖 | +| api/v1/endpoints/ticket.py | 747→1029 (+282) | ⚠️ AuditLog引用 | ✅ | 直接覆盖 | +| api/v1/endpoints/query.py | 226→426 (+200) | ⚠️ AuditLog引用 | ✅ | 直接覆盖 | +| api/v1/endpoints/user.py | 116→116 (0) | ⚠️ AuditLogService | ✅ | 直接覆盖 | + +--- + +## 🔴 冲突分析 + +### 冲突1：`user_groups` 表名冲突 + +| 维度 | 我们的版本 | 优化包版本 | +|------|-----------|-----------| +| **位置** | `models/instance_group.py` | `models/user.py` | +| **类型** | `Table()` 关联表 | `class UserGroup(Base)` ORM模型 | +| **表名** | `user_groups` | `user_groups` | +| **列** | user_id, group_id, permissions | id, name, description, group_permissions, ... | +| **用途** | User ↔ InstanceGroup 多对多关联 | 独立的用户组权限管理实体 | + +**冲突本质**：两个完全不同的数据库表结构映射到同一个表名。优化包的 `UserGroup` 是一个完整的权限组系统，我们的 `user_groups` 只是一个简单的关联表。 + +### 冲突2：AuditLog 双重定义 + +| 维度 | 我们的版本 (`models/audit_log.py`) | 优化包版本 (`models/user.py`) | +|------|----------------------------------|-------------------------------| +| **表名** | `audit_logs` | `audit_logs` | +| **字段风格** | 运维审计（target_type, query_content, db_type） | 通用审计（resource_type, before_data, after_data） | +| **关系** | 无 relationship | `user = relationship("User", ...)` | + +**关键发现**：优化包的 `ticket.py` 和 `query.py` 都 `from app.models.audit_log import AuditLog`（即使用的是我们的版本），而不是从 `models.user` 导入！ + +### 冲突3：新增模型类（无冲突但需数据库迁移） + +优化包 `models/user.py` 新增了： +- `UserGroup` — 用户组模型（表名与我们的关联表冲突） +- `UserGroupMember` — 用户组成员关联表 +- `AuditLog` — 审计日志（与 `models/audit_log.py` 重复定义） + +--- + +## 📁 逐文件分析 + +### 1. models/user.py — ⚠️ 需手动合并 + +**差异概要**： +- User 模型新增了 4 个 JSON 权限字段（data_source_permissions, custom_permissions, column_permissions, permissions_updated_at） +- User 模型新增了 5 个 relationship（tickets, approvals, audit_logs, query_histories, user_groups） +- User 模型新增了 5 个方法（has_permission, has_data_source_permission, get_allowed_columns, to_dict, __repr__） +- 新增 3 个类：UserGroup, UserGroupMember, AuditLog + +**冲突点**： +- `secondary="user_groups"` — 与 instance_group.py 的 Table 定义冲突 +- `user_groups = relationship("UserGroup", ...)` — 引用同文件 UserGroup +- `audit_logs = relationship("AuditLog", ...)` — 引用同文件 AuditLog +- `to_dict()` 中 `self.user_groups` — 引用 UserGroup relationship + +**合并策略**： +``` +✅ 直接采纳：User 模型的 JSON 权限字段（4个新字段） +✅ 直接采纳：User 的便捷方法（has_permission, to_dict 等） +⚠️ 需修改：User.groups 的 secondary 保持 "user_groups"（不改，与 instance_group.py 兼容） +❌ 不采纳：User.user_groups relationship（引用冲突的 UserGroup） +❌ 不采纳：User.tickets/approvals/audit_logs/query_histories（跨模型耦合，按需添加） +❌ 不采纳：UserGroup 类（表名冲突） +❌ 不采纳：UserGroupMember 类（依赖 UserGroup） +❌ 不采纳：AuditLog 类（已在 audit_log.py 独立定义） +⚠️ to_dict() 中的 user_groups 引用需要改为 groups 或移除 +``` + +### 2. models/workflow.py — ✅ 可安全覆盖 + +**差异概要**： +- WorkflowDefinition 新增 category, updated_at, created_by, flow_config, trigger_conditions 字段 +- 新增 WorkflowVersion 类（版本控制） +- WorkflowStep 大幅增强（description, approver_assignment, entry_conditions, transition_conditions, field_permissions） +- TicketApproval 大幅增强（workflow_id, version_id, step_type, actual_approver_id, transfer 信息, 超时标记） +- 新增 WorkflowExecutionLog 类（执行日志） + +**新增依赖**：`json`, `UniqueConstraint`, `JSON` — 标准库/SQLAlchemy + +**安全评估**：无冲突模型引用，纯功能增强。✅ + +### 3. services/workflow_engine.py — ✅ 可安全覆盖 + +**差异概要**： +- 从简单引擎升级为支持条件审批、超时策略、中断恢复的完整引擎 +- 新增 WorkflowStatus/ApprovalAction/TimeoutStrategy 枚举 +- 新增 ConditionResult/ApproverAssignment 数据类 +- 引擎核心方法大幅扩展 + +**import 检查**： +- `from app.models.workflow import (WorkflowDefinition, WorkflowStep, WorkflowVersion, TicketApproval, WorkflowExecutionLog)` — 全部来自 workflow.py +- `from app.models.user import User` — 只用 User 基础字段 +- 无冲突模型引用 + +### 4. services/query_service.py — ✅ 可安全覆盖 + +**差异概要**：查询服务增强，支持更多数据库类型和查询优化。 + +**import 检查**： +- `from app.models.instance import Instance` — ✅ +- `from app.schemas.query import UnifiedQueryResult` — ✅ +- `from app.services.db_connector import DBConnector` — ✅ +- 无冲突模型引用 + +### 5. services/audit_engine.py — ✅ 可安全覆盖 + +**差异概要**：审计引擎增强，新增枚举和条件判断。 + +**import 检查**： +- 无外部模型导入，仅标准库 + typing +- 无冲突模型引用 + +### 6. services/ticket_executor.py — ✅ 可安全覆盖 + +**差异概要**：工单执行器增强，支持更多 SQL 解析和执行策略。 + +**import 检查**： +- `from app.models.instance import Instance` — ✅ +- `from app.models.ticket import Ticket` — ✅ +- `from app.services.sql_analyzer import SQLAnalyzer, SQLType` — ✅ +- 无冲突模型引用 + +### 7. api/v1/endpoints/ticket.py — ✅ 可安全覆盖 + +**AuditLog 使用情况**： +- `from app.models.audit_log import AuditLog` — ✅ 使用我们的独立 AuditLog +- `AuditLog.target_type`, `AuditLog.target_id`, `AuditLog.created_at` — ✅ 我们模型有这些字段 +- `AuditLogService.log(target_type=..., detail=...)` — ✅ audit_log_service.py 两版完全相同 + +**注意**：ticket.py 使用的 AuditLog 字段（target_type, target_id, detail, db_type, status）全部存在于我们的 `models/audit_log.py`。 + +### 8. api/v1/endpoints/query.py — ✅ 可安全覆盖（需注意字段） + +**AuditLog 使用情况**： +- `from app.models.audit_log import AuditLog` — ✅ 使用我们的独立 AuditLog +- 直接创建 `AuditLog(...)` 实例，使用字段： + - ✅ 我们模型有：user_id, username, action, target_type, target_id, instance_name, db_type, database_name, query_content, execution_time_ms, rows_affected, result_status, detail + - ⚠️ **`query_type`** — 我们模型没有这个字段！ + - ⚠️ **`status`** — 我们模型没有 `status`，只有 `result_status` + +**需要的处理**：在我们的 `models/audit_log.py` 中新增 `query_type` 字段： +```python +query_type = Column(String(20), nullable=True) # 新增：查询类型 +``` + +### 9. api/v1/endpoints/user.py — ✅ 可安全覆盖 + +**差异**：两版完全相同（0 差异），优化包版本多了 AuditLogService 引用但代码一致。 + +--- + +## 🗂️ 推荐执行顺序 + +### 第一阶段：无冲突文件直接覆盖（7个文件） + +```bash +# 1. models/workflow.py +# 2. services/workflow_engine.py +# 3. services/query_service.py +# 4. services/audit_engine.py +# 5. services/ticket_executor.py +# 6. api/v1/endpoints/ticket.py +# 7. api/v1/endpoints/user.py (两版相同，跳过) +``` + +### 第二阶段：小修复 + +在 `models/audit_log.py` 新增 `query_type` 字段： +```python +query_type = Column(String(20), nullable=True) # 查询类型（SELECT/INSERT/UPDATE/DELETE等） +``` + +### 第三阶段：手动合并 models/user.py + +**保留我们现有结构**，只合并以下安全内容： + +1. **新增 User 字段**（JSON 权限相关）： + ```python + data_source_permissions = Column(JSON, nullable=True, default=list) + custom_permissions = Column(JSON, nullable=True, default=dict) + column_permissions = Column(JSON, nullable=True, default=dict) + permissions_updated_at = Column(DateTime, nullable=True) + ``` + +2. **新增 User 方法**： + ```python + def has_permission(self, action: str) -> bool: ... + def has_data_source_permission(self, ...) -> bool: ... + def get_allowed_columns(self, ...) -> list: ... + def to_dict(self, include_sensitive=False) -> dict: ... + ``` + +3. **不合并的内容**： + - `UserGroup` 类 → 如果需要用户组功能，必须改名或用不同表名 + - `UserGroupMember` 类 → 同上 + - `AuditLog` 类 → 已有独立文件，不重复定义 + - `User.user_groups` relationship → 依赖冲突的 UserGroup + - `User.audit_logs` relationship → 需要评估是否添加（跨模型耦合） + - `User.tickets`/`approvals`/`query_histories` → 按需添加 + +4. **to_dict 方法调整**： + - 将 `self.user_groups` 改为 `self.groups`（匹配现有 InstanceGroup 关系） + +### 第四阶段：数据库迁移 + +新增/修改的数据库表： +- `workflow_versions` — 新增表 +- `workflow_execution_logs` — 新增表 +- `workflow_definitions` — 新增 category, updated_at, created_by, flow_config, trigger_conditions 字段 +- `workflow_steps` — 新增多个字段 +- `ticket_approvals` — 新增多个字段 +- `audit_logs` — 新增 query_type 字段 +- `users` — 新增 4 个 JSON 权限字段 + +--- + +## ⚠️ 风险提示 + +1. **UserGroup/user_groups 冲突**：如果未来需要优化包的用户组功能，需要： + - 将优化包的 `UserGroup` 改表名为 `user_permission_groups` + - 或将我们的关联表 `user_groups` 改名为 `user_instance_groups` + - 同时更新所有外键和 relationship 引用 + +2. **AuditLog 双定义**：SQLAlchemy 的 metadata 中如果同时注册两个同名 `__tablename__` 的类会报错。只要不同时 import 就没问题。确保不 import `models/user.py` 中的 AuditLog 即可。 + +3. **workflow.py 的 relationship 指向 "User"**：通过字符串引用，SQLAlchemy 在运行时解析。只要所有模型在 metadata 中注册时 User 类存在即可。