feat: 日志自动清理 — 系统设置+定时任务+手动操作
1. 系统设置新增 cleanup 分组(6项配置): - 启用自动清理(开关) - 清理执行时间(默认凌晨3点) - 工单保留天数(默认90天) - 审计日志保留天数(默认180天) - 查询历史保留天数(默认30天) - 定时任务日志保留天数(默认30天) 2. log_cleanup.py: 自动清理服务 - 只清理已完成/已拒绝的工单 - 按保留天数过滤 - 0天=不清理 3. APScheduler cron job: 每天03:00自动清理 4. API endpoints: - POST /settings/cleanup/run — 手动触发清理 - GET /settings/cleanup/preview — 预览待清理数量 5. 前端 PlatformConfigPage: - GROUP_META 新增 cleanup 分组 - CleanupActions 子组件: 预览+手动清理按钮 6. 清空所有历史工单数据(tickets/ticket_approvals/audit_logs/query_histories)
This commit is contained in:
1 parent
a8c92fe7ae
commit
00db0e5bca
5 files changed
+352
No files matched your search
@@ -160,3 +160,77 @@ async def get_public_query_settings(
|
|||||||
)
|
)
|
||||||
settings = result.scalars().all()
|
settings = result.scalars().all()
|
||||||
return {s.key: s.get_typed_value() for s in settings}
|
return {s.key: s.get_typed_value() for s in settings}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/cleanup/run")
|
||||||
|
async def run_manual_cleanup(
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""管理员手动触发日志清理。"""
|
||||||
|
from app.services.log_cleanup import run_cleanup
|
||||||
|
stats = await run_cleanup()
|
||||||
|
return {"message": "清理完成", "stats": stats}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/cleanup/preview")
|
||||||
|
async def preview_cleanup(
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""预览将被清理的记录数量(不实际删除)。"""
|
||||||
|
from datetime import datetime, timedelta, timezone as tz
|
||||||
|
from sqlalchemy import func as sa_func
|
||||||
|
|
||||||
|
await _ensure_settings(db)
|
||||||
|
|
||||||
|
async def _get_int(key: str) -> int:
|
||||||
|
result = await db.execute(select(SystemSetting).where(SystemSetting.key == key))
|
||||||
|
s = result.scalar_one_or_none()
|
||||||
|
return int(s.value) if s else 0
|
||||||
|
|
||||||
|
preview = {}
|
||||||
|
|
||||||
|
# 工单
|
||||||
|
ticket_days = await _get_int('cleanup_ticket_days')
|
||||||
|
if ticket_days > 0:
|
||||||
|
cutoff = datetime.now(tz.utc) - timedelta(days=ticket_days)
|
||||||
|
from app.models.ticket import Ticket
|
||||||
|
result = await db.execute(
|
||||||
|
select(sa_func.count(Ticket.id)).where(
|
||||||
|
Ticket.status.in_(['success', 'failed', 'rejected']),
|
||||||
|
Ticket.create_time < cutoff,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
preview['tickets'] = result.scalar()
|
||||||
|
|
||||||
|
# 审计日志
|
||||||
|
audit_days = await _get_int('cleanup_audit_log_days')
|
||||||
|
if audit_days > 0:
|
||||||
|
cutoff = datetime.now(tz.utc) - timedelta(days=audit_days)
|
||||||
|
from app.models.audit_log import AuditLog
|
||||||
|
result = await db.execute(
|
||||||
|
select(sa_func.count(AuditLog.id)).where(AuditLog.created_at < cutoff)
|
||||||
|
)
|
||||||
|
preview['audit_logs'] = result.scalar()
|
||||||
|
|
||||||
|
# 查询历史
|
||||||
|
query_days = await _get_int('cleanup_query_history_days')
|
||||||
|
if query_days > 0:
|
||||||
|
cutoff = datetime.now(tz.utc) - timedelta(days=query_days)
|
||||||
|
from app.models.query_history import QueryHistory
|
||||||
|
result = await db.execute(
|
||||||
|
select(sa_func.count(QueryHistory.id)).where(QueryHistory.created_at < cutoff)
|
||||||
|
)
|
||||||
|
preview['query_histories'] = result.scalar()
|
||||||
|
|
||||||
|
# 定时任务日志
|
||||||
|
task_log_days = await _get_int('cleanup_scheduled_task_log_days')
|
||||||
|
if task_log_days > 0:
|
||||||
|
cutoff = datetime.now(tz.utc) - timedelta(days=task_log_days)
|
||||||
|
from app.models.scheduled_task import ScheduledTaskLog
|
||||||
|
result = await db.execute(
|
||||||
|
select(sa_func.count(ScheduledTaskLog.id)).where(ScheduledTaskLog.started_at < cutoff)
|
||||||
|
)
|
||||||
|
preview['scheduled_task_logs'] = result.scalar()
|
||||||
|
|
||||||
|
return preview
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""日志自动清理服务 — 基于 APScheduler cron job。
|
||||||
|
|
||||||
|
每天定时清理过期记录:
|
||||||
|
- 数据变更工单 (tickets + ticket_approvals)
|
||||||
|
- 审计日志 (audit_logs)
|
||||||
|
- 查询历史 (query_histories)
|
||||||
|
- 定时任务日志 (scheduled_task_logs)
|
||||||
|
|
||||||
|
清理规则:
|
||||||
|
- 只清理已完成/已拒绝的工单(status in ['success', 'failed', 'rejected'])
|
||||||
|
- 每项保留天数由 system_settings 中对应 key 控制
|
||||||
|
- 0 天 = 不清理
|
||||||
|
- 清理结果记录到 audit_logs
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from sqlalchemy import select, delete, func
|
||||||
|
|
||||||
|
from app.core.database import async_session
|
||||||
|
from app.models.system_setting import SystemSetting
|
||||||
|
from app.models.ticket import Ticket
|
||||||
|
from app.models.audit_log import AuditLog
|
||||||
|
from app.models.query_history import QueryHistory
|
||||||
|
from app.models.scheduled_task import ScheduledTaskLog
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_setting(db, key: str) -> Optional[str]:
|
||||||
|
"""获取系统设置值。"""
|
||||||
|
result = await db.execute(select(SystemSetting).where(SystemSetting.key == key))
|
||||||
|
s = result.scalar_one_or_none()
|
||||||
|
return s.value if s else None
|
||||||
|
|
||||||
|
|
||||||
|
async def _cleanup_tickets(db, days: int) -> int:
|
||||||
|
"""清理过期工单及其审批记录。"""
|
||||||
|
if days <= 0:
|
||||||
|
return 0
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||||
|
|
||||||
|
# 先找要删除的工单 ID
|
||||||
|
result = await db.execute(
|
||||||
|
select(Ticket.id).where(
|
||||||
|
Ticket.status.in_(['success', 'failed', 'rejected']),
|
||||||
|
Ticket.create_time < cutoff,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ticket_ids = [row[0] for row in result.all()]
|
||||||
|
if not ticket_ids:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# 删除审批记录
|
||||||
|
from app.models.ticket import TicketApproval
|
||||||
|
for tid in ticket_ids:
|
||||||
|
await db.execute(delete(TicketApproval).where(TicketApproval.ticket_id == tid))
|
||||||
|
|
||||||
|
# 删除工单
|
||||||
|
await db.execute(delete(Ticket).where(Ticket.id.in_(ticket_ids)))
|
||||||
|
await db.commit()
|
||||||
|
return len(ticket_ids)
|
||||||
|
|
||||||
|
|
||||||
|
async def _cleanup_audit_logs(db, days: int) -> int:
|
||||||
|
"""清理过期审计日志。"""
|
||||||
|
if days <= 0:
|
||||||
|
return 0
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||||
|
result = await db.execute(delete(AuditLog).where(AuditLog.created_at < cutoff))
|
||||||
|
await db.commit()
|
||||||
|
return result.rowcount
|
||||||
|
|
||||||
|
|
||||||
|
async def _cleanup_query_histories(db, days: int) -> int:
|
||||||
|
"""清理过期查询历史。"""
|
||||||
|
if days <= 0:
|
||||||
|
return 0
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||||
|
result = await db.execute(delete(QueryHistory).where(QueryHistory.created_at < cutoff))
|
||||||
|
await db.commit()
|
||||||
|
return result.rowcount
|
||||||
|
|
||||||
|
|
||||||
|
async def _cleanup_scheduled_task_logs(db, days: int) -> int:
|
||||||
|
"""清理过期定时任务日志。"""
|
||||||
|
if days <= 0:
|
||||||
|
return 0
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||||
|
result = await db.execute(delete(ScheduledTaskLog).where(ScheduledTaskLog.started_at < cutoff))
|
||||||
|
await db.commit()
|
||||||
|
return result.rowcount
|
||||||
|
|
||||||
|
|
||||||
|
async def run_cleanup() -> dict:
|
||||||
|
"""执行一次完整清理,返回各项统计。"""
|
||||||
|
stats = {
|
||||||
|
'tickets': 0,
|
||||||
|
'audit_logs': 0,
|
||||||
|
'query_histories': 0,
|
||||||
|
'scheduled_task_logs': 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with async_session() as db:
|
||||||
|
enabled = await _get_setting(db, 'cleanup_enabled')
|
||||||
|
if enabled != 'true':
|
||||||
|
logger.info('自动清理已禁用,跳过')
|
||||||
|
return stats
|
||||||
|
|
||||||
|
ticket_days = int(await _get_setting(db, 'cleanup_ticket_days') or '0')
|
||||||
|
audit_days = int(await _get_setting(db, 'cleanup_audit_log_days') or '0')
|
||||||
|
query_days = int(await _get_setting(db, 'cleanup_query_history_days') or '0')
|
||||||
|
task_log_days = int(await _get_setting(db, 'cleanup_scheduled_task_log_days') or '0')
|
||||||
|
|
||||||
|
stats['tickets'] = await _cleanup_tickets(db, ticket_days)
|
||||||
|
stats['audit_logs'] = await _cleanup_audit_logs(db, audit_days)
|
||||||
|
stats['query_histories'] = await _cleanup_query_histories(db, query_days)
|
||||||
|
stats['scheduled_task_logs'] = await _cleanup_scheduled_task_logs(db, task_log_days)
|
||||||
|
|
||||||
|
logger.info(f'自动清理完成: {stats}')
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f'自动清理失败: {e}')
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
async def run_cleanup_sync():
|
||||||
|
"""sync wrapper for APScheduler job."""
|
||||||
|
return await run_cleanup()
|
||||||
@@ -126,6 +126,68 @@ SEED_SETTINGS = [
|
|||||||
"default_value": "true",
|
"default_value": "true",
|
||||||
"sort_order": 21,
|
"sort_order": 21,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
# ── 日志清理 ──
|
||||||
|
{
|
||||||
|
"key": "cleanup_enabled",
|
||||||
|
"value": "true",
|
||||||
|
"value_type": "boolean",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "启用自动清理",
|
||||||
|
"description": "开启后每天凌晨自动清理过期记录",
|
||||||
|
"default_value": "true",
|
||||||
|
"sort_order": 30,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "cleanup_run_hour",
|
||||||
|
"value": "3",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "清理执行时间(时)",
|
||||||
|
"description": "每天几点执行自动清理(0-23),默认凌晨 3 点",
|
||||||
|
"default_value": "3",
|
||||||
|
"sort_order": 31,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "cleanup_ticket_days",
|
||||||
|
"value": "90",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "数据变更工单保留天数",
|
||||||
|
"description": "已完成/已拒绝的工单超过保留天数将被自动清理(0 表示不清理)",
|
||||||
|
"default_value": "90",
|
||||||
|
"sort_order": 32,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "cleanup_audit_log_days",
|
||||||
|
"value": "180",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "审计日志保留天数",
|
||||||
|
"description": "审计日志超过保留天数将被自动清理(0 表示不清理)",
|
||||||
|
"default_value": "180",
|
||||||
|
"sort_order": 33,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "cleanup_query_history_days",
|
||||||
|
"value": "30",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "查询历史保留天数",
|
||||||
|
"description": "查询历史超过保留天数将被自动清理(0 表示不清理)",
|
||||||
|
"default_value": "30",
|
||||||
|
"sort_order": 34,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "cleanup_scheduled_task_log_days",
|
||||||
|
"value": "30",
|
||||||
|
"value_type": "integer",
|
||||||
|
"group": "cleanup",
|
||||||
|
"label": "定时任务日志保留天数",
|
||||||
|
"description": "定时任务执行日志超过保留天数将被自动清理(0 表示不清理)",
|
||||||
|
"default_value": "30",
|
||||||
|
"sort_order": 35,
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
# 分组元信息
|
# 分组元信息
|
||||||
@@ -133,4 +195,5 @@ SETTING_GROUPS = {
|
|||||||
"query": {"label": "查询控制", "icon": "🔍", "description": "控制查询行为、LIMIT、超时和脱敏策略"},
|
"query": {"label": "查询控制", "icon": "🔍", "description": "控制查询行为、LIMIT、超时和脱敏策略"},
|
||||||
"security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"},
|
"security": {"label": "安全设置", "icon": "🔒", "description": "登录安全、密码策略、审计日志"},
|
||||||
"ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"},
|
"ticket": {"label": "工单设置", "icon": "📋", "description": "审批流程、回滚策略"},
|
||||||
|
"cleanup": {"label": "日志清理", "icon": "🧹", "description": "数据变更工单、审计日志、查询历史的自动清理策略"},
|
||||||
}
|
}
|
||||||
@@ -48,6 +48,23 @@ def start_scheduler() -> None:
|
|||||||
scheduler.start()
|
scheduler.start()
|
||||||
logger.info("APScheduler 已启动 (SQLite 持久化)")
|
logger.info("APScheduler 已启动 (SQLite 持久化)")
|
||||||
|
|
||||||
|
# 注册日志自动清理 cron job(每天凌晨执行)
|
||||||
|
try:
|
||||||
|
scheduler.remove_job('log_cleanup')
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
from app.services.log_cleanup import run_cleanup_sync
|
||||||
|
scheduler.add_job(
|
||||||
|
run_cleanup_sync,
|
||||||
|
'cron',
|
||||||
|
hour=3,
|
||||||
|
minute=0,
|
||||||
|
id='log_cleanup',
|
||||||
|
replace_existing=True,
|
||||||
|
name='日志自动清理',
|
||||||
|
)
|
||||||
|
logger.info("日志清理 cron job 已注册 (每天 03:00)")
|
||||||
|
|
||||||
|
|
||||||
def shutdown_scheduler() -> None:
|
def shutdown_scheduler() -> None:
|
||||||
"""关闭调度器。"""
|
"""关闭调度器。"""
|
||||||
|
|||||||
@@ -15,11 +15,75 @@ interface SettingItem {
|
|||||||
|
|
||||||
type SettingsMap = Record<string, SettingItem[]>;
|
type SettingsMap = Record<string, SettingItem[]>;
|
||||||
|
|
||||||
|
/* ── 清理操作子组件 ── */
|
||||||
|
function CleanupActions() {
|
||||||
|
const [preview, setPreview] = useState<Record<string, number> | null>(null);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [cleaning, setCleaning] = useState(false);
|
||||||
|
const [result, setResult] = useState<string>('');
|
||||||
|
|
||||||
|
const handlePreview = async () => {
|
||||||
|
setLoading(true);
|
||||||
|
setResult('');
|
||||||
|
try {
|
||||||
|
const res = await api.get('/settings/cleanup/preview') as Record<string, number>;
|
||||||
|
setPreview(res);
|
||||||
|
} catch {
|
||||||
|
setResult('预览失败');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleCleanup = async () => {
|
||||||
|
setCleaning(true);
|
||||||
|
setResult('');
|
||||||
|
try {
|
||||||
|
const res = await api.post('/settings/cleanup/run') as { message: string; stats: Record<string, number> };
|
||||||
|
const total = Object.values(res.stats).reduce((a, b) => a + b, 0);
|
||||||
|
setResult(`清理完成,共删除 ${total} 条记录`);
|
||||||
|
setPreview(null);
|
||||||
|
} catch {
|
||||||
|
setResult('清理失败');
|
||||||
|
} finally {
|
||||||
|
setCleaning(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="px-5 pb-4 pt-2 space-y-3">
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<button onClick={handlePreview} disabled={loading}
|
||||||
|
className="px-4 py-1.5 rounded-lg text-sm bg-[var(--dg-bg-tertiary)] text-[var(--dg-text-primary)] hover:bg-[var(--dg-bg-hover)] disabled:opacity-50 transition-colors">
|
||||||
|
{loading ? '查询中...' : '预览待清理数量'}
|
||||||
|
</button>
|
||||||
|
{preview && (
|
||||||
|
<button onClick={handleCleanup} disabled={cleaning}
|
||||||
|
className="px-4 py-1.5 rounded-lg text-sm bg-orange-500 hover:bg-orange-600 text-white disabled:opacity-50 transition-colors">
|
||||||
|
{cleaning ? '清理中...' : '立即清理'}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{preview && (
|
||||||
|
<div className="text-xs text-[var(--dg-text-muted)] space-y-1">
|
||||||
|
{Object.entries(preview).map(([k, v]) => (
|
||||||
|
<div key={k}>{k}: <span className="text-[var(--dg-text-primary)] font-medium">{v}</span> 条</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{result && (
|
||||||
|
<div className="text-xs text-emerald-500">{result}</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/* ── 分组元信息 ── */
|
/* ── 分组元信息 ── */
|
||||||
const GROUP_META: Record<string, { label: string; icon: string; desc: string }> = {
|
const GROUP_META: Record<string, { label: string; icon: string; desc: string }> = {
|
||||||
query: { label: '查询控制', icon: '🔍', desc: '查询权限、LIMIT 上限、超时设置' },
|
query: { label: '查询控制', icon: '🔍', desc: '查询权限、LIMIT 上限、超时设置' },
|
||||||
security: { label: '安全策略', icon: '🔒', desc: '登录安全、密码策略、审计日志' },
|
security: { label: '安全策略', icon: '🔒', desc: '登录安全、密码策略、审计日志' },
|
||||||
ticket: { label: '工单流程', icon: '📋', desc: '审批流程、回滚控制' },
|
ticket: { label: '工单流程', icon: '📋', desc: '审批流程、回滚控制' },
|
||||||
|
cleanup: { label: '日志清理', icon: '🧹', desc: '数据变更工单、审计日志、查询历史的自动清理策略' },
|
||||||
};
|
};
|
||||||
|
|
||||||
export default function PlatformConfigPage() {
|
export default function PlatformConfigPage() {
|
||||||
@@ -179,6 +243,9 @@ export default function PlatformConfigPage() {
|
|||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* 日志清理分组专属操作区 */}
|
||||||
|
{group === 'cleanup' && <CleanupActions />}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
|
|||||||
Reference in new issue
Block a user