From 00db0e5bcab7acd82969bd22736e1e88082cdc6d Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 25 May 2026 00:19:38 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=B8=85=E7=90=86=20=E2=80=94=20=E7=B3=BB=E7=BB=9F=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE+=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1+=E6=89=8B?= =?UTF-8?q?=E5=8A=A8=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 系统设置新增 cleanup 分组(6项配置): - 启用自动清理(开关) - 清理执行时间(默认凌晨3点) - 工单保留天数(默认90天) - 审计日志保留天数(默认180天) - 查询历史保留天数(默认30天) - 定时任务日志保留天数(默认30天) 2. log_cleanup.py: 自动清理服务 - 只清理已完成/已拒绝的工单 - 按保留天数过滤 - 0天=不清理 3. APScheduler cron job: 每天03:00自动清理 4. API endpoints: - POST /settings/cleanup/run — 手动触发清理 - GET /settings/cleanup/preview — 预览待清理数量 5. 前端 PlatformConfigPage: - GROUP_META 新增 cleanup 分组 - CleanupActions 子组件: 预览+手动清理按钮 6. 清空所有历史工单数据(tickets/ticket_approvals/audit_logs/query_histories) --- backend/app/api/v1/endpoints/settings.py | 74 ++++++++++++ backend/app/services/log_cleanup.py | 131 ++++++++++++++++++++++ backend/app/services/settings_seeds.py | 63 +++++++++++ backend/app/services/ticket_scheduler.py | 17 +++ frontend/src/pages/PlatformConfigPage.tsx | 67 +++++++++++ 5 files changed, 352 insertions(+) create mode 100644 backend/app/services/log_cleanup.py diff --git a/backend/app/api/v1/endpoints/settings.py b/backend/app/api/v1/endpoints/settings.py index b13dcd9..614d9da 100644 --- a/backend/app/api/v1/endpoints/settings.py +++ b/backend/app/api/v1/endpoints/settings.py @@ -160,3 +160,77 @@ async def get_public_query_settings( ) settings = result.scalars().all() return {s.key: s.get_typed_value() for s in settings} + + +@router.post("/cleanup/run") +async def run_manual_cleanup( + current_user: User = Depends(require_role("admin")), +): + """管理员手动触发日志清理。""" + from app.services.log_cleanup import run_cleanup + stats = await run_cleanup() + return {"message": "清理完成", "stats": stats} + + +@router.get("/cleanup/preview") +async def preview_cleanup( + db: AsyncSession = Depends(get_db), + current_user: User = Depends(require_role("admin")), +): + """预览将被清理的记录数量（不实际删除）。""" + from datetime import datetime, timedelta, timezone as tz + from sqlalchemy import func as sa_func + + await _ensure_settings(db) + + async def _get_int(key: str) -> int: + result = await db.execute(select(SystemSetting).where(SystemSetting.key == key)) + s = result.scalar_one_or_none() + return int(s.value) if s else 0 + + preview = {} + + # 工单 + ticket_days = await _get_int('cleanup_ticket_days') + if ticket_days > 0: + cutoff = datetime.now(tz.utc) - timedelta(days=ticket_days) + from app.models.ticket import Ticket + result = await db.execute( + select(sa_func.count(Ticket.id)).where( + Ticket.status.in_(['success', 'failed', 'rejected']), + Ticket.create_time < cutoff, + ) + ) + preview['tickets'] = result.scalar() + + # 审计日志 + audit_days = await _get_int('cleanup_audit_log_days') + if audit_days > 0: + cutoff = datetime.now(tz.utc) - timedelta(days=audit_days) + from app.models.audit_log import AuditLog + result = await db.execute( + select(sa_func.count(AuditLog.id)).where(AuditLog.created_at < cutoff) + ) + preview['audit_logs'] = result.scalar() + + # 查询历史 + query_days = await _get_int('cleanup_query_history_days') + if query_days > 0: + cutoff = datetime.now(tz.utc) - timedelta(days=query_days) + from app.models.query_history import QueryHistory + result = await db.execute( + select(sa_func.count(QueryHistory.id)).where(QueryHistory.created_at < cutoff) + ) + preview['query_histories'] = result.scalar() + + # 定时任务日志 + task_log_days = await _get_int('cleanup_scheduled_task_log_days') + if task_log_days > 0: + cutoff = datetime.now(tz.utc) - timedelta(days=task_log_days) + from app.models.scheduled_task import ScheduledTaskLog + result = await db.execute( + select(sa_func.count(ScheduledTaskLog.id)).where(ScheduledTaskLog.started_at < cutoff) + ) + preview['scheduled_task_logs'] = result.scalar() + + return preview diff --git a/backend/app/services/log_cleanup.py b/backend/app/services/log_cleanup.py new file mode 100644 index 0000000..7636b2f --- /dev/null +++ b/backend/app/services/log_cleanup.py @@ -0,0 +1,131 @@ +"""日志自动清理服务 — 基于 APScheduler cron job。 + +每天定时清理过期记录: +- 数据变更工单 (tickets + ticket_approvals) +- 审计日志 (audit_logs) +- 查询历史 (query_histories) +- 定时任务日志 (scheduled_task_logs) + +清理规则: +- 只清理已完成/已拒绝的工单（status in ['success', 'failed', 'rejected']） +- 每项保留天数由 system_settings 中对应 key 控制 +- 0 天 = 不清理 +- 清理结果记录到 audit_logs +""" +import logging +from datetime import datetime, timedelta, timezone +from typing import Optional + +from sqlalchemy import select, delete, func + +from app.core.database import async_session +from app.models.system_setting import SystemSetting +from app.models.ticket import Ticket +from app.models.audit_log import AuditLog +from app.models.query_history import QueryHistory +from app.models.scheduled_task import ScheduledTaskLog + +logger = logging.getLogger(__name__) + + +async def _get_setting(db, key: str) -> Optional[str]: + """获取系统设置值。""" + result = await db.execute(select(SystemSetting).where(SystemSetting.key == key)) + s = result.scalar_one_or_none() + return s.value if s else None + + +async def _cleanup_tickets(db, days: int) -> int: + """清理过期工单及其审批记录。""" + if days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + + # 先找要删除的工单 ID + result = await db.execute( + select(Ticket.id).where( + Ticket.status.in_(['success', 'failed', 'rejected']), + Ticket.create_time < cutoff, + ) + ) + ticket_ids = [row[0] for row in result.all()] + if not ticket_ids: + return 0 + + # 删除审批记录 + from app.models.ticket import TicketApproval + for tid in ticket_ids: + await db.execute(delete(TicketApproval).where(TicketApproval.ticket_id == tid)) + + # 删除工单 + await db.execute(delete(Ticket).where(Ticket.id.in_(ticket_ids))) + await db.commit() + return len(ticket_ids) + + +async def _cleanup_audit_logs(db, days: int) -> int: + """清理过期审计日志。""" + if days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + result = await db.execute(delete(AuditLog).where(AuditLog.created_at < cutoff)) + await db.commit() + return result.rowcount + + +async def _cleanup_query_histories(db, days: int) -> int: + """清理过期查询历史。""" + if days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + result = await db.execute(delete(QueryHistory).where(QueryHistory.created_at < cutoff)) + await db.commit() + return result.rowcount + + +async def _cleanup_scheduled_task_logs(db, days: int) -> int: + """清理过期定时任务日志。""" + if days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + result = await db.execute(delete(ScheduledTaskLog).where(ScheduledTaskLog.started_at < cutoff)) + await db.commit() + return result.rowcount + + +async def run_cleanup() -> dict: + """执行一次完整清理，返回各项统计。""" + stats = { + 'tickets': 0, + 'audit_logs': 0, + 'query_histories': 0, + 'scheduled_task_logs': 0, + } + + try: + async with async_session() as db: + enabled = await _get_setting(db, 'cleanup_enabled') + if enabled != 'true': + logger.info('自动清理已禁用，跳过') + return stats + + ticket_days = int(await _get_setting(db, 'cleanup_ticket_days') or '0') + audit_days = int(await _get_setting(db, 'cleanup_audit_log_days') or '0') + query_days = int(await _get_setting(db, 'cleanup_query_history_days') or '0') + task_log_days = int(await _get_setting(db, 'cleanup_scheduled_task_log_days') or '0') + + stats['tickets'] = await _cleanup_tickets(db, ticket_days) + stats['audit_logs'] = await _cleanup_audit_logs(db, audit_days) + stats['query_histories'] = await _cleanup_query_histories(db, query_days) + stats['scheduled_task_logs'] = await _cleanup_scheduled_task_logs(db, task_log_days) + + logger.info(f'自动清理完成: {stats}') + except Exception as e: + logger.error(f'自动清理失败: {e}') + + return stats + + +async def run_cleanup_sync(): + """sync wrapper for APScheduler job.""" + return await run_cleanup() diff --git a/backend/app/services/settings_seeds.py b/backend/app/services/settings_seeds.py index 9d661ca..228ba85 100644 --- a/backend/app/services/settings_seeds.py +++ b/backend/app/services/settings_seeds.py @@ -126,6 +126,68 @@ SEED_SETTINGS = [ "default_value": "true", "sort_order": 21, }, + + # ── 日志清理 ── + { + "key": "cleanup_enabled", + "value": "true", + "value_type": "boolean", + "group": "cleanup", + "label": "启用自动清理", + "description": "开启后每天凌晨自动清理过期记录", + "default_value": "true", + "sort_order": 30, + }, + { + "key": "cleanup_run_hour", + "value": "3", + "value_type": "integer", + "group": "cleanup", + "label": "清理执行时间（时）", + "description": "每天几点执行自动清理（0-23），默认凌晨 3 点", + "default_value": "3", + "sort_order": 31, + }, + { + "key": "cleanup_ticket_days", + "value": "90", + "value_type": "integer", + "group": "cleanup", + "label": "数据变更工单保留天数", + "description": "已完成/已拒绝的工单超过保留天数将被自动清理（0 表示不清理）", + "default_value": "90", + "sort_order": 32, + }, + { + "key": "cleanup_audit_log_days", + "value": "180", + "value_type": "integer", + "group": "cleanup", + "label": "审计日志保留天数", + "description": "审计日志超过保留天数将被自动清理（0 表示不清理）", + "default_value": "180", + "sort_order": 33, + }, + { + "key": "cleanup_query_history_days", + "value": "30", + "value_type": "integer", + "group": "cleanup", + "label": "查询历史保留天数", + "description": "查询历史超过保留天数将被自动清理（0 表示不清理）", + "default_value": "30", + "sort_order": 34, + }, + { + "key": "cleanup_scheduled_task_log_days", + "value": "30", + "value_type": "integer", + "group": "cleanup", + "label": "定时任务日志保留天数", + "description": "定时任务执行日志超过保留天数将被自动清理（0 表示不清理）", + "default_value": "30", + "sort_order": 35, + }, ] # 分组元信息 @@ -133,4 +195,5 @@ SETTING_GROUPS = { "query": {"label": "查询控制", "icon": "🔍", "description": "控制查询行为、LIMIT、超时和脱敏策略"}, "security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"}, "ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"}, + "cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"}, } diff --git a/backend/app/services/ticket_scheduler.py b/backend/app/services/ticket_scheduler.py index 7413b42..45bbe0c 100644 --- a/backend/app/services/ticket_scheduler.py +++ b/backend/app/services/ticket_scheduler.py @@ -48,6 +48,23 @@ def start_scheduler() -> None: scheduler.start() logger.info("APScheduler 已启动 (SQLite 持久化)") + # 注册日志自动清理 cron job（每天凌晨执行） + try: + scheduler.remove_job('log_cleanup') + except Exception: + pass + from app.services.log_cleanup import run_cleanup_sync + scheduler.add_job( + run_cleanup_sync, + 'cron', + hour=3, + minute=0, + id='log_cleanup', + replace_existing=True, + name='日志自动清理', + ) + logger.info("日志清理 cron job 已注册 (每天 03:00)") + def shutdown_scheduler() -> None: """关闭调度器。""" diff --git a/frontend/src/pages/PlatformConfigPage.tsx b/frontend/src/pages/PlatformConfigPage.tsx index 3f08834..c9722a3 100644 --- a/frontend/src/pages/PlatformConfigPage.tsx +++ b/frontend/src/pages/PlatformConfigPage.tsx @@ -15,11 +15,75 @@ interface SettingItem { type SettingsMap = Record; +/* ── 清理操作子组件 ── */ +function CleanupActions() { + const [preview, setPreview] = useState | null>(null); + const [loading, setLoading] = useState(false); + const [cleaning, setCleaning] = useState(false); + const [result, setResult] = useState(''); + + const handlePreview = async () => { + setLoading(true); + setResult(''); + try { + const res = await api.get('/settings/cleanup/preview') as Record; + setPreview(res); + } catch { + setResult('预览失败'); + } finally { + setLoading(false); + } + }; + + const handleCleanup = async () => { + setCleaning(true); + setResult(''); + try { + const res = await api.post('/settings/cleanup/run') as { message: string; stats: Record }; + const total = Object.values(res.stats).reduce((a, b) => a + b, 0); + setResult(`清理完成，共删除 ${total} 条记录`); + setPreview(null); + } catch { + setResult('清理失败'); + } finally { + setCleaning(false); + } + }; + + return ( +
+
+ + {preview && ( + + )} +
+ {preview && ( +
+ {Object.entries(preview).map(([k, v]) => ( +
{k}: {v} 条
+ ))} +
+ )} + {result && ( +
{result}
+ )} +
+ ); +} + /* ── 分组元信息 ── */ const GROUP_META: Record = { query: { label: '查询控制', icon: '🔍', desc: '查询权限、LIMIT 上限、超时设置' }, security: { label: '安全策略', icon: '🔒', desc: '登录安全、密码策略、审计日志' }, ticket: { label: '工单流程', icon: '📋', desc: '审批流程、回滚控制' }, + cleanup: { label: '日志清理', icon: '🧹', desc: '数据变更工单、审计日志、查询历史的自动清理策略' }, }; export default function PlatformConfigPage() { @@ -179,6 +243,9 @@ export default function PlatformConfigPage() { ); })} + + {/* 日志清理分组专属操作区 */} + {group === 'cleanup' && } ); })}