Skip to content
Python
16.3 KB
Raw Blame
"""
HY Memory - System Metrics Collector

进程级单例,收集系统负载指标并按分钟粒度落盘到 SQLite。
支持按时间段查询聚合数据。

Usage:
    from hy_memory.metrics import MetricsCollector

    metrics = MetricsCollector.get()
    metrics.sys1_start()
    # ... 处理 ...
    metrics.sys1_end({"sys1_waiting_ms": 100, ...})

    # 查询最近 5 分钟
    snapshot = await metrics.get_snapshot(minutes=5)
"""

import asyncio
import logging
import threading
import time
from collections import deque
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional

logger = logging.getLogger(__name__)

# 滑动窗口大小(秒)
_WINDOW_SECONDS = 60

# flush 间隔(秒)
_FLUSH_INTERVAL = 60

# cleanup 间隔(秒)
_CLEANUP_INTERVAL = 3600

# 数据保留天数
_RETENTION_DAYS = 7


class MetricsCollector:
    """
    进程级 metrics 收集器。

    线程安全(所有写入通过 _mu 保护)。
    按分钟粒度将增量数据持久化到 SQLite,支持时间段聚合查询。
    """

    _instance: Optional["MetricsCollector"] = None
    _instance_lock = threading.Lock()

    @classmethod
    def get(cls) -> "MetricsCollector":
        """获取单例实例"""
        if cls._instance is None:
            with cls._instance_lock:
                if cls._instance is None:
                    cls._instance = cls()
        return cls._instance

    @classmethod
    def reset(cls) -> None:
        """重置单例(仅测试用)"""
        cls._instance = None

    def __init__(self):
        self._start_time = time.time()
        self._mu = threading.Lock()
        self._cache = None
        self._background_started = False
        self._flush_task: Optional[asyncio.Task] = None
        self._cleanup_task: Optional[asyncio.Task] = None

        # ── 实时计数(不落盘) ──
        self._active_sys1 = 0
        self._active_sys2 = 0
        self._queued_sys2 = 0

        # ── 当前分钟桶(待 flush) ──
        self._current_minute = self._get_minute_key(datetime.now())
        self._bucket = self._empty_bucket()
        self._pending_flush: List[tuple] = []  # [(minute_key, bucket_data), ...]

        # ── 滑动窗口(最近 60s 完成数) ──
        self._sys1_completions: deque = deque()
        self._sys2_completions: deque = deque()

    # ================================================================
    # 初始化
    # ================================================================

    def bind_cache(self, cache) -> None:
        """绑定 cache 实例用于落盘"""
        self._cache = cache

    async def start_background_tasks(self) -> None:
        """启动后台 flush + cleanup 协程(调用一次即可)"""
        if self._background_started:
            return
        self._background_started = True
        self._flush_task = asyncio.create_task(
            self._flush_loop(), name="hy-memory-metrics-flush",
        )
        self._cleanup_task = asyncio.create_task(
            self._cleanup_loop(), name="hy-memory-metrics-cleanup",
        )
        logger.info("[metrics] background tasks started (flush=60s, cleanup=3600s)")

    async def stop_background_tasks(self) -> None:
        """Cancel and join background tasks before their owning loop closes."""
        tasks = [
            task for task in (self._flush_task, self._cleanup_task)
            if task is not None
        ]
        self._flush_task = None
        self._cleanup_task = None
        self._background_started = False
        for task in tasks:
            task.cancel()
        if tasks:
            await asyncio.gather(*tasks, return_exceptions=True)
        logger.info("[metrics] background tasks stopped")

    # ================================================================
    # 采集 API
    # ================================================================

    def sys1_start(self) -> None:
        """S1 请求开始"""
        with self._mu:
            self._active_sys1 += 1
            self._bucket["sys1_started"] += 1

    def sys1_end(self, timing: Dict[str, float], success: bool = True) -> None:
        """S1 请求结束"""
        now = time.time()
        with self._mu:
            self._active_sys1 -= 1
            if success:
                self._bucket["sys1_completed"] += 1
            else:
                self._bucket["sys1_failed"] += 1

            # 累加 timing
            for key in ("sys1_waiting_ms", "sys1_l1_process_ms", "sys1_workflow_ms", "sys1_ops_avg_ms"):
                val = timing.get(key, 0)
                self._bucket["sys1_timing_sums"][key] = (
                    self._bucket["sys1_timing_sums"].get(key, 0) + val
                )

            # 滑动窗口
            self._sys1_completions.append(now)

            # 可能需要翻转 bucket
            self._maybe_rotate_bucket()

    def sys2_enqueue(self) -> None:
        """S2 任务入队"""
        with self._mu:
            self._queued_sys2 += 1

    def sys2_start(self) -> None:
        """S2 任务开始处理(从队列取出)"""
        with self._mu:
            self._active_sys2 += 1
            self._queued_sys2 = max(0, self._queued_sys2 - 1)
            self._bucket["sys2_started"] += 1

    def sys2_end(self, timing: Dict[str, float], success: bool = True) -> None:
        """S2 任务结束"""
        now = time.time()
        with self._mu:
            self._active_sys2 -= 1
            if success:
                self._bucket["sys2_completed"] += 1
            else:
                self._bucket["sys2_failed"] += 1

            # 累加 timing
            for key in ("sys2_waiting_ms", "sys2_preprocess_ms", "sys2_agent_generate_ms",
                        "sys2_agent_tools_avg_ms", "sys2_sweeper_ms"):
                val = timing.get(key, 0)
                self._bucket["sys2_timing_sums"][key] = (
                    self._bucket["sys2_timing_sums"].get(key, 0) + val
                )

            # 滑动窗口
            self._sys2_completions.append(now)

            self._maybe_rotate_bucket()

    def record_vdb_op(self, elapsed_ms: float) -> None:
        """记录一次 VDB 操作"""
        with self._mu:
            self._bucket["vdb_ops"] += 1
            self._bucket["vdb_ops_sum_ms"] += elapsed_ms

    def record_graph_op(self, elapsed_ms: float) -> None:
        """记录一次 Graph 操作"""
        with self._mu:
            self._bucket["graph_ops"] += 1
            self._bucket["graph_ops_sum_ms"] += elapsed_ms

    # ================================================================
    # 查询 API
    # ================================================================

    async def get_snapshot(self, minutes: int = 5) -> Dict[str, Any]:
        """
        获取最近 N 分钟的聚合指标。

        从 SQLite 读取历史分钟数据 + 合并当前分钟内存数据。
        """
        now = datetime.now()
        start_ts = (now - timedelta(minutes=minutes)).strftime("%Y-%m-%dT%H:%M")
        end_ts = now.strftime("%Y-%m-%dT%H:%M")

        # 从 SQLite 读取历史
        history_buckets: List[Dict] = []
        if self._cache and hasattr(self._cache, "load_metrics_range"):
            try:
                history_buckets = await self._cache.load_metrics_range(start_ts, end_ts)
            except Exception as e:
                logger.warning(f"[metrics] load_metrics_range failed: {e}")

        # 合并当前内存 bucket + pending_flush
        with self._mu:
            current_bucket = dict(self._bucket)
            current_bucket["sys1_timing_sums"] = dict(self._bucket["sys1_timing_sums"])
            current_bucket["sys2_timing_sums"] = dict(self._bucket["sys2_timing_sums"])

            # pending_flush 中的旧桶也需要包含
            pending_buckets = [data for _, data in self._pending_flush]

            # 实时数据
            active_sys1 = self._active_sys1
            active_sys2 = self._active_sys2
            queued_sys2 = self._queued_sys2

            # 滑动窗口:清理过期
            cutoff = time.time() - _WINDOW_SECONDS
            while self._sys1_completions and self._sys1_completions[0] < cutoff:
                self._sys1_completions.popleft()
            while self._sys2_completions and self._sys2_completions[0] < cutoff:
                self._sys2_completions.popleft()
            sys1_last_60s = len(self._sys1_completions)
            sys2_last_60s = len(self._sys2_completions)

        # 聚合所有 buckets
        all_buckets = history_buckets + pending_buckets + [current_bucket]
        agg = self._aggregate_buckets(all_buckets)

        # 构建返回值
        sys1_count = agg["sys1_completed"] + agg["sys1_failed"]
        sys2_count = agg["sys2_completed"] + agg["sys2_failed"]

        # 平均 latency
        avg_latency = {}
        if sys1_count > 0:
            s1_sums = agg["sys1_timing_sums"]
            avg_latency["sys1_waiting"] = round(s1_sums.get("sys1_waiting_ms", 0) / sys1_count, 1)
            avg_latency["sys1_l1_process"] = round(s1_sums.get("sys1_l1_process_ms", 0) / sys1_count, 1)
            avg_latency["sys1_workflow"] = round(s1_sums.get("sys1_workflow_ms", 0) / sys1_count, 1)
            avg_latency["sys1_ops_avg"] = round(s1_sums.get("sys1_ops_avg_ms", 0) / sys1_count, 1)
            avg_latency["sys1_total"] = round(
                avg_latency["sys1_waiting"] + avg_latency["sys1_l1_process"] + avg_latency["sys1_workflow"], 1
            )

        if sys2_count > 0:
            s2_sums = agg["sys2_timing_sums"]
            avg_latency["sys2_waiting"] = round(s2_sums.get("sys2_waiting_ms", 0) / sys2_count, 1)
            avg_latency["sys2_preprocess"] = round(s2_sums.get("sys2_preprocess_ms", 0) / sys2_count, 1)
            avg_latency["sys2_agent_generate"] = round(s2_sums.get("sys2_agent_generate_ms", 0) / sys2_count, 1)
            avg_latency["sys2_agent_tools_avg"] = round(s2_sums.get("sys2_agent_tools_avg_ms", 0) / sys2_count, 1)
            avg_latency["sys2_sweeper"] = round(s2_sums.get("sys2_sweeper_ms", 0) / sys2_count, 1)
            avg_latency["sys2_total"] = round(
                avg_latency["sys2_waiting"] + avg_latency["sys2_preprocess"] +
                avg_latency["sys2_agent_generate"] + avg_latency["sys2_sweeper"], 1
            )

        # Storage ops
        storage_ops = {}
        if agg["vdb_ops"] > 0:
            storage_ops["vdb_ops_total"] = agg["vdb_ops"]
            storage_ops["vdb_ops_avg_ms"] = round(agg["vdb_ops_sum_ms"] / agg["vdb_ops"], 1)
        if agg["graph_ops"] > 0:
            storage_ops["graph_ops_total"] = agg["graph_ops"]
            storage_ops["graph_ops_avg_ms"] = round(agg["graph_ops_sum_ms"] / agg["graph_ops"], 1)

        return {
            "uptime_seconds": round(time.time() - self._start_time, 1),
            "window_minutes": minutes,
            "requests": {
                "total": agg["sys1_started"],
                "active_sys1": active_sys1,
                "active_sys2": active_sys2,
                "queued_sys2": queued_sys2,
                "completed": agg["sys1_completed"],
                "failed": agg["sys1_failed"],
            },
            "avg_latency_ms": avg_latency,
            "throughput": {
                "sys1_completed_last_60s": sys1_last_60s,
                "sys2_completed_last_60s": sys2_last_60s,
            },
            "storage_ops": storage_ops,
            "sys2_requests": {
                "total": agg["sys2_started"],
                "completed": agg["sys2_completed"],
                "failed": agg["sys2_failed"],
            },
            "since": start_ts,
        }

    # ================================================================
    # 落盘
    # ================================================================

    async def flush(self) -> None:
        """将当前分钟桶 + pending 队列写入 SQLite"""
        if self._cache is None:
            return

        with self._mu:
            # 收集所有待写入的 buckets
            to_flush: List[tuple] = list(self._pending_flush)
            self._pending_flush.clear()

            # 当前桶也 flush(如果非空)
            if not self._is_bucket_empty(self._bucket):
                to_flush.append((self._current_minute, self._bucket))
                self._bucket = self._empty_bucket()

            self._current_minute = self._get_minute_key(datetime.now())

        for minute_key, data in to_flush:
            try:
                await self._cache.store_metrics_minute(minute_key, data)
                logger.debug(f"[metrics] flushed minute={minute_key}")
            except Exception as e:
                logger.warning(f"[metrics] flush failed for {minute_key}: {e}")

    # ================================================================
    # 内部方法
    # ================================================================

    @staticmethod
    def _get_minute_key(dt: datetime) -> str:
        """返回分钟精度的 key,如 '2026-05-23T13:07'"""
        return dt.strftime("%Y-%m-%dT%H:%M")

    @staticmethod
    def _empty_bucket() -> Dict[str, Any]:
        return {
            "sys1_started": 0,
            "sys1_completed": 0,
            "sys1_failed": 0,
            "sys2_started": 0,
            "sys2_completed": 0,
            "sys2_failed": 0,
            "sys1_timing_sums": {},
            "sys2_timing_sums": {},
            "vdb_ops": 0,
            "vdb_ops_sum_ms": 0.0,
            "graph_ops": 0,
            "graph_ops_sum_ms": 0.0,
        }

    @staticmethod
    def _is_bucket_empty(bucket: Dict) -> bool:
        return (
            bucket["sys1_started"] == 0
            and bucket["sys2_started"] == 0
            and bucket["vdb_ops"] == 0
            and bucket["graph_ops"] == 0
        )

    def _maybe_rotate_bucket(self) -> None:
        """如果当前分钟已过,翻转 bucket(在 _mu 锁内调用)"""
        now_minute = self._get_minute_key(datetime.now())
        if now_minute != self._current_minute:
            # 旧 bucket 移入待 flush 队列
            if not self._is_bucket_empty(self._bucket):
                self._pending_flush.append((self._current_minute, self._bucket))
            self._current_minute = now_minute
            self._bucket = self._empty_bucket()

    @staticmethod
    def _aggregate_buckets(buckets: List[Dict]) -> Dict[str, Any]:
        """聚合多个 bucket 的数据"""
        agg = MetricsCollector._empty_bucket()
        for b in buckets:
            agg["sys1_started"] += b.get("sys1_started", 0)
            agg["sys1_completed"] += b.get("sys1_completed", 0)
            agg["sys1_failed"] += b.get("sys1_failed", 0)
            agg["sys2_started"] += b.get("sys2_started", 0)
            agg["sys2_completed"] += b.get("sys2_completed", 0)
            agg["sys2_failed"] += b.get("sys2_failed", 0)
            agg["vdb_ops"] += b.get("vdb_ops", 0)
            agg["vdb_ops_sum_ms"] += b.get("vdb_ops_sum_ms", 0)
            agg["graph_ops"] += b.get("graph_ops", 0)
            agg["graph_ops_sum_ms"] += b.get("graph_ops_sum_ms", 0)
            # timing sums
            for key, val in b.get("sys1_timing_sums", {}).items():
                agg["sys1_timing_sums"][key] = agg["sys1_timing_sums"].get(key, 0) + val
            for key, val in b.get("sys2_timing_sums", {}).items():
                agg["sys2_timing_sums"][key] = agg["sys2_timing_sums"].get(key, 0) + val
        return agg

    # ================================================================
    # 后台协程
    # ================================================================

    async def _flush_loop(self) -> None:
        """每分钟 flush 当前 bucket 到 SQLite"""
        while True:
            await asyncio.sleep(_FLUSH_INTERVAL)
            try:
                await self.flush()
            except Exception as e:
                logger.warning(f"[metrics] flush_loop error: {e}")

    async def _cleanup_loop(self) -> None:
        """每小时清理 7 天前的 metrics 数据"""
        while True:
            await asyncio.sleep(_CLEANUP_INTERVAL)
            if self._cache is None:
                continue
            try:
                cutoff = (datetime.now() - timedelta(days=_RETENTION_DAYS)).strftime("%Y-%m-%dT%H:%M")
                await self._cache.cleanup_old_metrics(cutoff)
                logger.debug(f"[metrics] cleanup done, removed before {cutoff}")
            except Exception as e:
                logger.warning(f"[metrics] cleanup error: {e}")