Skip to content
Python
5.5 KB
Raw Blame
"""Process-wide, event-loop-local concurrency budgets for memory workflows.

One top-level write fans out into many LLM, embedding, retrieval, graph, and
episode operations.  These gates bound that fan-out without changing the
synchronous API contract: a write still waits for all work that belongs to it.
"""

from __future__ import annotations

import asyncio
from contextlib import asynccontextmanager
from contextvars import ContextVar
from dataclasses import dataclass
import logging
import os
import threading
import time
import weakref


logger = logging.getLogger(__name__)

_workload_kind: ContextVar[str] = ContextVar(
    "hy_memory_workload_kind", default="foreground",
)
_states: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict]" = (
    weakref.WeakKeyDictionary()
)
_states_lock = threading.Lock()


@dataclass
class _GateStats:
    active: int = 0
    max_active: int = 0
    wait_count: int = 0
    waited_count: int = 0
    total_wait_ms: float = 0.0
    max_wait_ms: float = 0.0


def _limit(name: str, default: int) -> int:
    try:
        return max(1, int(os.getenv(name, str(default))))
    except (TypeError, ValueError):
        return default


def _gate(limit: int) -> dict:
    return {
        "semaphore": asyncio.Semaphore(limit),
        "limit": limit,
        "stats": _GateStats(),
    }


def _state() -> dict:
    loop = asyncio.get_running_loop()
    with _states_lock:
        state = _states.get(loop)
        if state is None:
            state = {
                "llm": _gate(_limit("MEMORY_LLM_GLOBAL_CONCURRENCY", 32)),
                "derived_llm": _gate(
                    _limit("MEMORY_DERIVED_LLM_CONCURRENCY", 20)
                ),
                "episode": _gate(
                    _limit("MEMORY_DYNAMIC_EPISODE_CONCURRENCY", 20)
                ),
                "graph": _gate(
                    _limit("MEMORY_GRAPH_PROJECTION_CONCURRENCY", 20)
                ),
                # Kuzu has one process-wide executor worker.  A projection-level
                # gate of one prevents dozens of projections from each enqueueing
                # hundreds of tiny Cypher calls ahead of one another.
                "graph_kuzu": _gate(
                    _limit("MEMORY_KUZU_PROJECTION_CONCURRENCY", 1)
                ),
                "embedding": _gate(
                    _limit("MEMORY_EMBEDDING_GLOBAL_CONCURRENCY", 20)
                ),
                # Reconcile can create 2 searches per extracted fact.  Keep local
                # stores especially narrow because every query scans the local
                # collection and competes for CPU/memory bandwidth.
                "reconcile_retrieval": _gate(
                    _limit("MEMORY_RECONCILE_RETRIEVAL_CONCURRENCY", 16)
                ),
                "reconcile_retrieval_local": _gate(
                    _limit("MEMORY_RECONCILE_LOCAL_RETRIEVAL_CONCURRENCY", 2)
                ),
            }
            _states[loop] = state
        return state


@asynccontextmanager
async def _slot(name: str, trace_name: str):
    gate = _state()[name]
    stats = gate["stats"]
    stats.wait_count += 1
    started = time.perf_counter()
    await gate["semaphore"].acquire()
    wait_ms = (time.perf_counter() - started) * 1000
    stats.total_wait_ms += wait_ms
    stats.max_wait_ms = max(stats.max_wait_ms, wait_ms)
    if wait_ms >= 1.0:
        stats.waited_count += 1
    stats.active += 1
    stats.max_active = max(stats.max_active, stats.active)
    warn_ms = _limit("MEMORY_WORKLOAD_GATE_LOG_MS", 1000)
    if wait_ms >= warn_ms:
        logger.info(
            "TRACE_PERF %s wait_ms=%.1f active=%d limit=%d",
            trace_name, wait_ms, stats.active, gate["limit"],
        )
    try:
        yield
    finally:
        stats.active -= 1
        gate["semaphore"].release()


def workload_snapshot() -> dict:
    result = {}
    for name, gate in _state().items():
        stats = gate["stats"]
        result[name] = {
            "limit": gate["limit"],
            "active": stats.active,
            "max_active": stats.max_active,
            "wait_count": stats.wait_count,
            "waited_count": stats.waited_count,
            "total_wait_ms": round(stats.total_wait_ms, 3),
            "max_wait_ms": round(stats.max_wait_ms, 3),
        }
    return result


@asynccontextmanager
async def workload_scope(kind: str):
    token = _workload_kind.set(str(kind or "foreground"))
    try:
        yield
    finally:
        _workload_kind.reset(token)


@asynccontextmanager
async def llm_slot():
    if _workload_kind.get() == "derived":
        async with _slot("derived_llm", "DERIVED_LLM_GATE_WAIT"):
            async with _slot("llm", "LLM_GATE_WAIT"):
                yield
    else:
        async with _slot("llm", "LLM_GATE_WAIT"):
            yield


@asynccontextmanager
async def episode_slot():
    async with _slot("episode", "EPISODE_GATE_WAIT"):
        yield


@asynccontextmanager
async def graph_slot(*, kuzu: bool = False):
    name = "graph_kuzu" if kuzu else "graph"
    trace = "KUZU_GRAPH_GATE_WAIT" if kuzu else "GRAPH_GATE_WAIT"
    async with _slot(name, trace):
        yield


@asynccontextmanager
async def embedding_slot():
    async with _slot("embedding", "EMBED_GATE_WAIT"):
        yield


@asynccontextmanager
async def reconcile_retrieval_slot(*, local: bool = False):
    name = "reconcile_retrieval_local" if local else "reconcile_retrieval"
    trace = (
        "RECONCILE_LOCAL_RETRIEVAL_GATE_WAIT"
        if local else "RECONCILE_RETRIEVAL_GATE_WAIT"
    )
    async with _slot(name, trace):
        yield