opsxsql/backend/app/services/slow_log_collector.py

"""慢日志收集器 — 从数据库收集和分析慢查询日志。

支持:
- MySQL: performance_schema.events_statements_summary_by_digest
- PostgreSQL: pg_stat_statements

生成慢查询报告，包含 TOP N 慢查询、执行时间趋势等。
"""
import hashlib
from dataclasses import dataclass, field
from datetime import datetime
from typing import List, Dict, Any, Optional

from app.models.instance import Instance
from app.services.db_connector import DBConnector
from app.core.crypto import decrypt_password


@dataclass
class SlowLogEntry:
    """慢日志条目。"""
    sql_digest: str           # SQL 指纹
    sql_text: str             # 归一化 SQL
    execution_count: int      # 执行次数
    avg_time_ms: float        # 平均执行时间(ms)
    max_time_ms: float        # 最大执行时间(ms)
    min_time_ms: float        # 最小执行时间(ms)
    rows_examined: int        # 扫描行数
    rows_sent: int            # 返回行数
    first_seen: Optional[str] = None
    last_seen: Optional[str] = None

    def to_dict(self) -> dict:
        return {
            "sql_digest": self.sql_digest,
            "sql_text": self.sql_text[:200] if self.sql_text else "",
            "execution_count": self.execution_count,
            "avg_time_ms": round(self.avg_time_ms, 2),
            "max_time_ms": round(self.max_time_ms, 2),
            "min_time_ms": round(self.min_time_ms, 2),
            "rows_examined": self.rows_examined,
            "rows_sent": self.rows_sent,
            "first_seen": self.first_seen,
            "last_seen": self.last_seen,
        }


@dataclass
class SlowLogReport:
    """慢日志分析报告。"""
    total_slow_queries: int
    avg_execution_time_ms: float
    max_execution_time_ms: float
    top_queries: List[SlowLogEntry]
    time_distribution: Dict[str, int]  # 按时间段分布

    def to_dict(self) -> dict:
        return {
            "total_slow_queries": self.total_slow_queries,
            "avg_execution_time_ms": round(self.avg_execution_time_ms, 2),
            "max_execution_time_ms": round(self.max_execution_time_ms, 2),
            "top_queries": [q.to_dict() for q in self.top_queries],
            "time_distribution": self.time_distribution,
        }


class SlowLogCollector:
    """慢日志收集器。"""

    # 内存中的慢日志存储: {instance_id: [SlowLogEntry, ...]}
    _logs: Dict[int, List[SlowLogEntry]] = {}

    async def collect_mysql_slow_logs(self, instance: Instance,
                                       database: str,
                                       limit: int = 50) -> List[SlowLogEntry]:
        """从 MySQL performance_schema 收集慢日志。

        需要 MySQL 开启 performance_schema。
        """
        sql = """
        SELECT
            DIGEST_TEXT as sql_text,
            COUNT_STAR as execution_count,
            ROUND(AVG_TIMER_WAIT / 1000000000, 3) as avg_time_ms,
            ROUND(MAX_TIMER_WAIT / 1000000000, 3) as max_time_ms,
            ROUND(MIN_TIMER_WAIT / 1000000000, 3) as min_time_ms,
            SUM_ROWS_EXAMINED as rows_examined,
            SUM_ROWS_SENT as rows_sent,
            FIRST_SEEN,
            LAST_SEEN
        FROM performance_schema.events_statements_summary_by_digest
        WHERE SCHEMA_NAME = %s
          AND AVG_TIMER_WAIT > 100000000000  -- > 100ms
        ORDER BY AVG_TIMER_WAIT DESC
        LIMIT %s
        """

        try:
            instance.password = decrypt_password(instance.password)
            rows, _ = await DBConnector._query_mysql(instance, "performance_schema", sql)
        except Exception as e:
            # performance_schema 可能不可用，使用替代查询
            return await self._collect_mysql_slow_logs_fallback(instance, database, limit)

        entries = []
        for row in rows:
            sql_text = row.get("sql_text", "")
            digest = hashlib.md5(sql_text.encode()).hexdigest()[:16]
            entries.append(SlowLogEntry(
                sql_digest=digest,
                sql_text=sql_text[:500],
                execution_count=int(row.get("execution_count", 0) or 0),
                avg_time_ms=float(row.get("avg_time_ms", 0) or 0),
                max_time_ms=float(row.get("max_time_ms", 0) or 0),
                min_time_ms=float(row.get("min_time_ms", 0) or 0),
                rows_examined=int(row.get("rows_examined", 0) or 0),
                rows_sent=int(row.get("rows_sent", 0) or 0),
                first_seen=str(row.get("FIRST_SEEN", "")) if row.get("FIRST_SEEN") else None,
                last_seen=str(row.get("LAST_SEEN", "")) if row.get("LAST_SEEN") else None,
            ))

        return entries

    async def _collect_mysql_slow_logs_fallback(self, instance: Instance,
                                                  database: str,
                                                  limit: int) -> List[SlowLogEntry]:
        """备用方案：使用 SHOW PROCESSLIST 获取当前慢查询。"""
        sql = "SHOW FULL PROCESSLIST"

        try:
            instance.password = decrypt_password(instance.password)
            rows, _ = await DBConnector._query_mysql(instance, database, sql)
        except Exception:
            return []

        entries = []
        for row in rows:
            time_val = row.get("Time", 0)
            info = row.get("Info", "")
            if time_val and int(time_val) > 1 and info and len(info) > 10:
                digest = hashlib.md5(info.encode()).hexdigest()[:16]
                entries.append(SlowLogEntry(
                    sql_digest=digest,
                    sql_text=info[:500],
                    execution_count=1,
                    avg_time_ms=float(time_val) * 1000,
                    max_time_ms=float(time_val) * 1000,
                    min_time_ms=float(time_val) * 1000,
                    rows_examined=0,
                    rows_sent=0,
                ))

        return entries[:limit]

    async def collect_postgres_slow_logs(self, instance: Instance,
                                          database: str,
                                          limit: int = 50) -> List[SlowLogEntry]:
        """从 PostgreSQL pg_stat_statements 收集慢日志。

        需要安装 pg_stat_statements 扩展。
        """
        sql = """
        SELECT
            query as sql_text,
            calls as execution_count,
            ROUND((total_exec_time / calls)::numeric, 3) as avg_time_ms,
            ROUND(max_exec_time::numeric, 3) as max_time_ms,
            ROUND(min_exec_time::numeric, 3) as min_time_ms,
            rows as rows_examined,
            rows as rows_sent
        FROM pg_stat_statements
        WHERE dbid = (SELECT oid FROM pg_database WHERE datname = $1)
          AND total_exec_time / calls > 100  -- > 100ms
        ORDER BY total_exec_time / calls DESC
        LIMIT $2
        """

        try:
            instance.password = decrypt_password(instance.password)
            # 需要先创建扩展
            await DBConnector._query_postgres(instance, database,
                "CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
            rows, _ = await DBConnector._query_postgres(instance, database, sql)
        except Exception:
            # pg_stat_statements 可能不可用
            return []

        entries = []
        for row in rows:
            sql_text = row.get("sql_text", "")
            digest = hashlib.md5(sql_text.encode()).hexdigest()[:16]
            entries.append(SlowLogEntry(
                sql_digest=digest,
                sql_text=sql_text[:500],
                execution_count=int(row.get("execution_count", 0) or 0),
                avg_time_ms=float(row.get("avg_time_ms", 0) or 0),
                max_time_ms=float(row.get("max_time_ms", 0) or 0),
                min_time_ms=float(row.get("min_time_ms", 0) or 0),
                rows_examined=int(row.get("rows_examined", 0) or 0),
                rows_sent=int(row.get("rows_sent", 0) or 0),
            ))

        return entries

    async def save_slow_logs(self, instance_id: int,
                              logs: List[SlowLogEntry]) -> None:
        """保存慢日志到内存存储。"""
        if instance_id not in self._logs:
            self._logs[instance_id] = []

        # 追加新日志，限制每个实例最多保存 1000 条
        self._logs[instance_id].extend(logs)
        if len(self._logs[instance_id]) > 1000:
            self._logs[instance_id] = self._logs[instance_id][-1000:]

    def get_slow_logs(self, instance_id: int) -> List[SlowLogEntry]:
        """获取保存的慢日志。"""
        return self._logs.get(instance_id, [])

    def get_report(self, instance_id: int) -> SlowLogReport:
        """生成慢日志报告。"""
        logs = self._logs.get(instance_id, [])

        if not logs:
            return SlowLogReport(
                total_slow_queries=0,
                avg_execution_time_ms=0,
                max_execution_time_ms=0,
                top_queries=[],
                time_distribution={},
            )

        total_avg = sum(q.avg_time_ms for q in logs) / len(logs)
        max_time = max(q.max_time_ms for q in logs)

        # 按执行时间分布
        distribution = {
            "0-100ms": 0,
            "100-500ms": 0,
            "500ms-1s": 0,
            "1s-5s": 0,
            ">5s": 0,
        }
        for q in logs:
            if q.avg_time_ms < 100:
                distribution["0-100ms"] += 1
            elif q.avg_time_ms < 500:
                distribution["100-500ms"] += 1
            elif q.avg_time_ms < 1000:
                distribution["500ms-1s"] += 1
            elif q.avg_time_ms < 5000:
                distribution["1s-5s"] += 1
            else:
                distribution[">5s"] += 1

        # TOP 10
        top = sorted(logs, key=lambda x: x.avg_time_ms, reverse=True)[:10]

        return SlowLogReport(
            total_slow_queries=len(logs),
            avg_execution_time_ms=round(total_avg, 2),
            max_execution_time_ms=round(max_time, 2),
            top_queries=top,
            time_distribution=distribution,
        )

    def clear_logs(self, instance_id: int) -> None:
        """清除指定实例的慢日志。"""
        self._logs.pop(instance_id, None)