"""慢日志收集器 — 从数据库收集和分析慢查询日志。 支持: - MySQL: performance_schema.events_statements_summary_by_digest - PostgreSQL: pg_stat_statements 生成慢查询报告，包含 TOP N 慢查询、执行时间趋势等。 """ import hashlib from dataclasses import dataclass, field from datetime import datetime from typing import List, Dict, Any, Optional from app.models.instance import Instance from app.services.db_connector import DBConnector from app.core.crypto import decrypt_password @dataclass class SlowLogEntry: """慢日志条目。""" sql_digest: str # SQL 指纹 sql_text: str # 归一化 SQL execution_count: int # 执行次数 avg_time_ms: float # 平均执行时间(ms) max_time_ms: float # 最大执行时间(ms) min_time_ms: float # 最小执行时间(ms) rows_examined: int # 扫描行数 rows_sent: int # 返回行数 first_seen: Optional[str] = None last_seen: Optional[str] = None def to_dict(self) -> dict: return { "sql_digest": self.sql_digest, "sql_text": self.sql_text[:200] if self.sql_text else "", "execution_count": self.execution_count, "avg_time_ms": round(self.avg_time_ms, 2), "max_time_ms": round(self.max_time_ms, 2), "min_time_ms": round(self.min_time_ms, 2), "rows_examined": self.rows_examined, "rows_sent": self.rows_sent, "first_seen": self.first_seen, "last_seen": self.last_seen, } @dataclass class SlowLogReport: """慢日志分析报告。""" total_slow_queries: int avg_execution_time_ms: float max_execution_time_ms: float top_queries: List[SlowLogEntry] time_distribution: Dict[str, int] # 按时间段分布 def to_dict(self) -> dict: return { "total_slow_queries": self.total_slow_queries, "avg_execution_time_ms": round(self.avg_execution_time_ms, 2), "max_execution_time_ms": round(self.max_execution_time_ms, 2), "top_queries": [q.to_dict() for q in self.top_queries], "time_distribution": self.time_distribution, } class SlowLogCollector: """慢日志收集器。""" # 内存中的慢日志存储: {instance_id: [SlowLogEntry, ...]} _logs: Dict[int, List[SlowLogEntry]] = {} async def collect_mysql_slow_logs(self, instance: Instance, database: str, limit: int = 50) -> List[SlowLogEntry]: """从 MySQL performance_schema 收集慢日志。 需要 MySQL 开启 performance_schema。 """ sql = """ SELECT DIGEST_TEXT as sql_text, COUNT_STAR as execution_count, ROUND(AVG_TIMER_WAIT / 1000000000, 3) as avg_time_ms, ROUND(MAX_TIMER_WAIT / 1000000000, 3) as max_time_ms, ROUND(MIN_TIMER_WAIT / 1000000000, 3) as min_time_ms, SUM_ROWS_EXAMINED as rows_examined, SUM_ROWS_SENT as rows_sent, FIRST_SEEN, LAST_SEEN FROM performance_schema.events_statements_summary_by_digest WHERE SCHEMA_NAME = %s AND AVG_TIMER_WAIT > 100000000000 -- > 100ms ORDER BY AVG_TIMER_WAIT DESC LIMIT %s """ try: instance.password = decrypt_password(instance.password) rows, _ = await DBConnector._query_mysql(instance, "performance_schema", sql) except Exception as e: # performance_schema 可能不可用，使用替代查询 return await self._collect_mysql_slow_logs_fallback(instance, database, limit) entries = [] for row in rows: sql_text = row.get("sql_text", "") digest = hashlib.md5(sql_text.encode()).hexdigest()[:16] entries.append(SlowLogEntry( sql_digest=digest, sql_text=sql_text[:500], execution_count=int(row.get("execution_count", 0) or 0), avg_time_ms=float(row.get("avg_time_ms", 0) or 0), max_time_ms=float(row.get("max_time_ms", 0) or 0), min_time_ms=float(row.get("min_time_ms", 0) or 0), rows_examined=int(row.get("rows_examined", 0) or 0), rows_sent=int(row.get("rows_sent", 0) or 0), first_seen=str(row.get("FIRST_SEEN", "")) if row.get("FIRST_SEEN") else None, last_seen=str(row.get("LAST_SEEN", "")) if row.get("LAST_SEEN") else None, )) return entries async def _collect_mysql_slow_logs_fallback(self, instance: Instance, database: str, limit: int) -> List[SlowLogEntry]: """备用方案：使用 SHOW PROCESSLIST 获取当前慢查询。""" sql = "SHOW FULL PROCESSLIST" try: instance.password = decrypt_password(instance.password) rows, _ = await DBConnector._query_mysql(instance, database, sql) except Exception: return [] entries = [] for row in rows: time_val = row.get("Time", 0) info = row.get("Info", "") if time_val and int(time_val) > 1 and info and len(info) > 10: digest = hashlib.md5(info.encode()).hexdigest()[:16] entries.append(SlowLogEntry( sql_digest=digest, sql_text=info[:500], execution_count=1, avg_time_ms=float(time_val) * 1000, max_time_ms=float(time_val) * 1000, min_time_ms=float(time_val) * 1000, rows_examined=0, rows_sent=0, )) return entries[:limit] async def collect_postgres_slow_logs(self, instance: Instance, database: str, limit: int = 50) -> List[SlowLogEntry]: """从 PostgreSQL pg_stat_statements 收集慢日志。 需要安装 pg_stat_statements 扩展。 """ sql = """ SELECT query as sql_text, calls as execution_count, ROUND((total_exec_time / calls)::numeric, 3) as avg_time_ms, ROUND(max_exec_time::numeric, 3) as max_time_ms, ROUND(min_exec_time::numeric, 3) as min_time_ms, rows as rows_examined, rows as rows_sent FROM pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = $1) AND total_exec_time / calls > 100 -- > 100ms ORDER BY total_exec_time / calls DESC LIMIT $2 """ try: instance.password = decrypt_password(instance.password) # 需要先创建扩展 await DBConnector._query_postgres(instance, database, "CREATE EXTENSION IF NOT EXISTS pg_stat_statements") rows, _ = await DBConnector._query_postgres(instance, database, sql) except Exception: # pg_stat_statements 可能不可用 return [] entries = [] for row in rows: sql_text = row.get("sql_text", "") digest = hashlib.md5(sql_text.encode()).hexdigest()[:16] entries.append(SlowLogEntry( sql_digest=digest, sql_text=sql_text[:500], execution_count=int(row.get("execution_count", 0) or 0), avg_time_ms=float(row.get("avg_time_ms", 0) or 0), max_time_ms=float(row.get("max_time_ms", 0) or 0), min_time_ms=float(row.get("min_time_ms", 0) or 0), rows_examined=int(row.get("rows_examined", 0) or 0), rows_sent=int(row.get("rows_sent", 0) or 0), )) return entries async def save_slow_logs(self, instance_id: int, logs: List[SlowLogEntry]) -> None: """保存慢日志到内存存储。""" if instance_id not in self._logs: self._logs[instance_id] = [] # 追加新日志，限制每个实例最多保存 1000 条 self._logs[instance_id].extend(logs) if len(self._logs[instance_id]) > 1000: self._logs[instance_id] = self._logs[instance_id][-1000:] def get_slow_logs(self, instance_id: int) -> List[SlowLogEntry]: """获取保存的慢日志。""" return self._logs.get(instance_id, []) def get_report(self, instance_id: int) -> SlowLogReport: """生成慢日志报告。""" logs = self._logs.get(instance_id, []) if not logs: return SlowLogReport( total_slow_queries=0, avg_execution_time_ms=0, max_execution_time_ms=0, top_queries=[], time_distribution={}, ) total_avg = sum(q.avg_time_ms for q in logs) / len(logs) max_time = max(q.max_time_ms for q in logs) # 按执行时间分布 distribution = { "0-100ms": 0, "100-500ms": 0, "500ms-1s": 0, "1s-5s": 0, ">5s": 0, } for q in logs: if q.avg_time_ms < 100: distribution["0-100ms"] += 1 elif q.avg_time_ms < 500: distribution["100-500ms"] += 1 elif q.avg_time_ms < 1000: distribution["500ms-1s"] += 1 elif q.avg_time_ms < 5000: distribution["1s-5s"] += 1 else: distribution[">5s"] += 1 # TOP 10 top = sorted(logs, key=lambda x: x.avg_time_ms, reverse=True)[:10] return SlowLogReport( total_slow_queries=len(logs), avg_execution_time_ms=round(total_avg, 2), max_execution_time_ms=round(max_time, 2), top_queries=top, time_distribution=distribution, ) def clear_logs(self, instance_id: int) -> None: """清除指定实例的慢日志。""" self._logs.pop(instance_id, None)