logan
4c9427e546
- P0: 实例访问控制全线接入权限组 (829测试) - P1: permissions字段激活 + 操作级权限检查 (843测试) - 前端: 删除数据权限页面, 权限组合并为唯一入口 - 新增: CodeMirror编辑器 + 查询模板 + 慢查询分析 - 新增: MongoDB管理页面 + Dashboard增强 - 修复: Pydantic V2迁移 + 前端TS错误修复
270 lines
10 KiB
Python
270 lines
10 KiB
Python
"""慢日志收集器 — 从数据库收集和分析慢查询日志。
|
|
|
|
支持:
|
|
- MySQL: performance_schema.events_statements_summary_by_digest
|
|
- PostgreSQL: pg_stat_statements
|
|
|
|
生成慢查询报告，包含 TOP N 慢查询、执行时间趋势等。
|
|
"""
|
|
import hashlib
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from app.models.instance import Instance
|
|
from app.services.db_connector import DBConnector
|
|
from app.core.crypto import decrypt_password
|
|
|
|
|
|
@dataclass
|
|
class SlowLogEntry:
|
|
"""慢日志条目。"""
|
|
sql_digest: str # SQL 指纹
|
|
sql_text: str # 归一化 SQL
|
|
execution_count: int # 执行次数
|
|
avg_time_ms: float # 平均执行时间(ms)
|
|
max_time_ms: float # 最大执行时间(ms)
|
|
min_time_ms: float # 最小执行时间(ms)
|
|
rows_examined: int # 扫描行数
|
|
rows_sent: int # 返回行数
|
|
first_seen: Optional[str] = None
|
|
last_seen: Optional[str] = None
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"sql_digest": self.sql_digest,
|
|
"sql_text": self.sql_text[:200] if self.sql_text else "",
|
|
"execution_count": self.execution_count,
|
|
"avg_time_ms": round(self.avg_time_ms, 2),
|
|
"max_time_ms": round(self.max_time_ms, 2),
|
|
"min_time_ms": round(self.min_time_ms, 2),
|
|
"rows_examined": self.rows_examined,
|
|
"rows_sent": self.rows_sent,
|
|
"first_seen": self.first_seen,
|
|
"last_seen": self.last_seen,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SlowLogReport:
|
|
"""慢日志分析报告。"""
|
|
total_slow_queries: int
|
|
avg_execution_time_ms: float
|
|
max_execution_time_ms: float
|
|
top_queries: List[SlowLogEntry]
|
|
time_distribution: Dict[str, int] # 按时间段分布
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"total_slow_queries": self.total_slow_queries,
|
|
"avg_execution_time_ms": round(self.avg_execution_time_ms, 2),
|
|
"max_execution_time_ms": round(self.max_execution_time_ms, 2),
|
|
"top_queries": [q.to_dict() for q in self.top_queries],
|
|
"time_distribution": self.time_distribution,
|
|
}
|
|
|
|
|
|
class SlowLogCollector:
|
|
"""慢日志收集器。"""
|
|
|
|
# 内存中的慢日志存储: {instance_id: [SlowLogEntry, ...]}
|
|
_logs: Dict[int, List[SlowLogEntry]] = {}
|
|
|
|
async def collect_mysql_slow_logs(self, instance: Instance,
|
|
database: str,
|
|
limit: int = 50) -> List[SlowLogEntry]:
|
|
"""从 MySQL performance_schema 收集慢日志。
|
|
|
|
需要 MySQL 开启 performance_schema。
|
|
"""
|
|
sql = """
|
|
SELECT
|
|
DIGEST_TEXT as sql_text,
|
|
COUNT_STAR as execution_count,
|
|
ROUND(AVG_TIMER_WAIT / 1000000000, 3) as avg_time_ms,
|
|
ROUND(MAX_TIMER_WAIT / 1000000000, 3) as max_time_ms,
|
|
ROUND(MIN_TIMER_WAIT / 1000000000, 3) as min_time_ms,
|
|
SUM_ROWS_EXAMINED as rows_examined,
|
|
SUM_ROWS_SENT as rows_sent,
|
|
FIRST_SEEN,
|
|
LAST_SEEN
|
|
FROM performance_schema.events_statements_summary_by_digest
|
|
WHERE SCHEMA_NAME = %s
|
|
AND AVG_TIMER_WAIT > 100000000000 -- > 100ms
|
|
ORDER BY AVG_TIMER_WAIT DESC
|
|
LIMIT %s
|
|
"""
|
|
|
|
try:
|
|
instance.password = decrypt_password(instance.password)
|
|
rows, _ = await DBConnector._query_mysql(instance, "performance_schema", sql)
|
|
except Exception as e:
|
|
# performance_schema 可能不可用，使用替代查询
|
|
return await self._collect_mysql_slow_logs_fallback(instance, database, limit)
|
|
|
|
entries = []
|
|
for row in rows:
|
|
sql_text = row.get("sql_text", "")
|
|
digest = hashlib.md5(sql_text.encode()).hexdigest()[:16]
|
|
entries.append(SlowLogEntry(
|
|
sql_digest=digest,
|
|
sql_text=sql_text[:500],
|
|
execution_count=int(row.get("execution_count", 0) or 0),
|
|
avg_time_ms=float(row.get("avg_time_ms", 0) or 0),
|
|
max_time_ms=float(row.get("max_time_ms", 0) or 0),
|
|
min_time_ms=float(row.get("min_time_ms", 0) or 0),
|
|
rows_examined=int(row.get("rows_examined", 0) or 0),
|
|
rows_sent=int(row.get("rows_sent", 0) or 0),
|
|
first_seen=str(row.get("FIRST_SEEN", "")) if row.get("FIRST_SEEN") else None,
|
|
last_seen=str(row.get("LAST_SEEN", "")) if row.get("LAST_SEEN") else None,
|
|
))
|
|
|
|
return entries
|
|
|
|
async def _collect_mysql_slow_logs_fallback(self, instance: Instance,
|
|
database: str,
|
|
limit: int) -> List[SlowLogEntry]:
|
|
"""备用方案：使用 SHOW PROCESSLIST 获取当前慢查询。"""
|
|
sql = "SHOW FULL PROCESSLIST"
|
|
|
|
try:
|
|
instance.password = decrypt_password(instance.password)
|
|
rows, _ = await DBConnector._query_mysql(instance, database, sql)
|
|
except Exception:
|
|
return []
|
|
|
|
entries = []
|
|
for row in rows:
|
|
time_val = row.get("Time", 0)
|
|
info = row.get("Info", "")
|
|
if time_val and int(time_val) > 1 and info and len(info) > 10:
|
|
digest = hashlib.md5(info.encode()).hexdigest()[:16]
|
|
entries.append(SlowLogEntry(
|
|
sql_digest=digest,
|
|
sql_text=info[:500],
|
|
execution_count=1,
|
|
avg_time_ms=float(time_val) * 1000,
|
|
max_time_ms=float(time_val) * 1000,
|
|
min_time_ms=float(time_val) * 1000,
|
|
rows_examined=0,
|
|
rows_sent=0,
|
|
))
|
|
|
|
return entries[:limit]
|
|
|
|
async def collect_postgres_slow_logs(self, instance: Instance,
|
|
database: str,
|
|
limit: int = 50) -> List[SlowLogEntry]:
|
|
"""从 PostgreSQL pg_stat_statements 收集慢日志。
|
|
|
|
需要安装 pg_stat_statements 扩展。
|
|
"""
|
|
sql = """
|
|
SELECT
|
|
query as sql_text,
|
|
calls as execution_count,
|
|
ROUND((total_exec_time / calls)::numeric, 3) as avg_time_ms,
|
|
ROUND(max_exec_time::numeric, 3) as max_time_ms,
|
|
ROUND(min_exec_time::numeric, 3) as min_time_ms,
|
|
rows as rows_examined,
|
|
rows as rows_sent
|
|
FROM pg_stat_statements
|
|
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = $1)
|
|
AND total_exec_time / calls > 100 -- > 100ms
|
|
ORDER BY total_exec_time / calls DESC
|
|
LIMIT $2
|
|
"""
|
|
|
|
try:
|
|
instance.password = decrypt_password(instance.password)
|
|
# 需要先创建扩展
|
|
await DBConnector._query_postgres(instance, database,
|
|
"CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
|
|
rows, _ = await DBConnector._query_postgres(instance, database, sql)
|
|
except Exception:
|
|
# pg_stat_statements 可能不可用
|
|
return []
|
|
|
|
entries = []
|
|
for row in rows:
|
|
sql_text = row.get("sql_text", "")
|
|
digest = hashlib.md5(sql_text.encode()).hexdigest()[:16]
|
|
entries.append(SlowLogEntry(
|
|
sql_digest=digest,
|
|
sql_text=sql_text[:500],
|
|
execution_count=int(row.get("execution_count", 0) or 0),
|
|
avg_time_ms=float(row.get("avg_time_ms", 0) or 0),
|
|
max_time_ms=float(row.get("max_time_ms", 0) or 0),
|
|
min_time_ms=float(row.get("min_time_ms", 0) or 0),
|
|
rows_examined=int(row.get("rows_examined", 0) or 0),
|
|
rows_sent=int(row.get("rows_sent", 0) or 0),
|
|
))
|
|
|
|
return entries
|
|
|
|
async def save_slow_logs(self, instance_id: int,
|
|
logs: List[SlowLogEntry]) -> None:
|
|
"""保存慢日志到内存存储。"""
|
|
if instance_id not in self._logs:
|
|
self._logs[instance_id] = []
|
|
|
|
# 追加新日志，限制每个实例最多保存 1000 条
|
|
self._logs[instance_id].extend(logs)
|
|
if len(self._logs[instance_id]) > 1000:
|
|
self._logs[instance_id] = self._logs[instance_id][-1000:]
|
|
|
|
def get_slow_logs(self, instance_id: int) -> List[SlowLogEntry]:
|
|
"""获取保存的慢日志。"""
|
|
return self._logs.get(instance_id, [])
|
|
|
|
def get_report(self, instance_id: int) -> SlowLogReport:
|
|
"""生成慢日志报告。"""
|
|
logs = self._logs.get(instance_id, [])
|
|
|
|
if not logs:
|
|
return SlowLogReport(
|
|
total_slow_queries=0,
|
|
avg_execution_time_ms=0,
|
|
max_execution_time_ms=0,
|
|
top_queries=[],
|
|
time_distribution={},
|
|
)
|
|
|
|
total_avg = sum(q.avg_time_ms for q in logs) / len(logs)
|
|
max_time = max(q.max_time_ms for q in logs)
|
|
|
|
# 按执行时间分布
|
|
distribution = {
|
|
"0-100ms": 0,
|
|
"100-500ms": 0,
|
|
"500ms-1s": 0,
|
|
"1s-5s": 0,
|
|
">5s": 0,
|
|
}
|
|
for q in logs:
|
|
if q.avg_time_ms < 100:
|
|
distribution["0-100ms"] += 1
|
|
elif q.avg_time_ms < 500:
|
|
distribution["100-500ms"] += 1
|
|
elif q.avg_time_ms < 1000:
|
|
distribution["500ms-1s"] += 1
|
|
elif q.avg_time_ms < 5000:
|
|
distribution["1s-5s"] += 1
|
|
else:
|
|
distribution[">5s"] += 1
|
|
|
|
# TOP 10
|
|
top = sorted(logs, key=lambda x: x.avg_time_ms, reverse=True)[:10]
|
|
|
|
return SlowLogReport(
|
|
total_slow_queries=len(logs),
|
|
avg_execution_time_ms=round(total_avg, 2),
|
|
max_execution_time_ms=round(max_time, 2),
|
|
top_queries=top,
|
|
time_distribution=distribution,
|
|
)
|
|
|
|
def clear_logs(self, instance_id: int) -> None:
|
|
"""清除指定实例的慢日志。"""
|
|
self._logs.pop(instance_id, None)
|