Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(memory): cap local embedder CPU thread oversubscription (#198) (#1559)
Krishna Chaitanyab84afbfb8339
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a1a555e6c..dfba370d7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -20,6 +20,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`<headroom_proactive_expansion>` XML tags, giving downstream consumers(LLMs, loggers, attribution parsers) a machine-readable provenanceboundary and preventing misattribution in multi-agent threads.+- **memory/embedder:** cap CPU thread oversubscription in the local+ torch/sentence-transformers embedder. Concurrent encodes previously each+ fanned out to ~`os.cpu_count()` BLAS/OpenMP threads, so under load the memory+ path starved the asyncio event loop and spiked `/livez` latency to several+ seconds. CPU encodes now run on a dedicated, size-limited executor whose+ workers each pin their thread pool, bounding total embedding threads to+ `HEADROOM_EMBED_CONCURRENCY` × `HEADROOM_EMBED_NUM_THREADS` (defaults+ `min(4, cpu)` × 1). The ONNX embedder already capped its threads; this brings+ the torch path to parity+ ([#198](https://github.com/headroomlabs-ai/headroom/issues/198)).### Changeddiff --git a/headroom/memory/adapters/embedders.py b/headroom/memory/adapters/embedders.pyindex fce7bea3f..1120b4fed 100644--- a/headroom/memory/adapters/embedders.py+++ b/headroom/memory/adapters/embedders.py@@ -42,6 +42,102 @@ logging.getLogger("httpx").setLevel(logging.WARNING)logger = logging.getLogger(__name__)+# =============================================================================+# Local (torch / sentence-transformers) CPU thread cap — issue #198+# =============================================================================+# A long-lived proxy serves many requests concurrently. Each torch ``encode()``+# fans out to BLAS (MKL / OpenBLAS / Accelerate) + OpenMP worker threads, which+# default to roughly ``os.cpu_count()``. Under concurrency this oversubscribes+# the CPU — N in-flight encodes x ~cpu_count threads each thrash the scheduler+# and starve the asyncio event loop, so liveness probes (``/livez``) spike to+# multiple seconds even though the loop itself is idle (issue #198).+#+# Capping intra-op parallelism makes a single encode modestly slower but lets+# concurrent encodes scale linearly without thread-pool thrash — the standard+# trade-off for serving torch models inside an async server. The ONNX embedder+# already caps its threads (see ``onnx_runtime.create_cpu_session_options``);+# this brings the torch path to parity.+#+# torch's OpenMP thread count is per-thread, and encodes run on executor worker+# threads, so a one-shot cap would miss most workers. Instead, CPU encodes run+# on a dedicated, size-limited executor whose ``initializer`` pins each worker's+# thread pool once. Total embedding threads are then bounded by+# ``workers (HEADROOM_EMBED_CONCURRENCY) x threads-per-encode+# (HEADROOM_EMBED_NUM_THREADS)``. Applies to the CPU device only (GPU/MPS do+# their compute off-CPU).+_EMBED_THREADS_ENV = "HEADROOM_EMBED_NUM_THREADS"+_DEFAULT_EMBED_THREADS = 1+_EMBED_CONCURRENCY_ENV = "HEADROOM_EMBED_CONCURRENCY"+_DEFAULT_EMBED_CONCURRENCY = 4+_BLAS_THREAD_ENV_VARS = (+ "OMP_NUM_THREADS",+ "OPENBLAS_NUM_THREADS",+ "MKL_NUM_THREADS",+ "NUMEXPR_NUM_THREADS",+ "VECLIB_MAXIMUM_THREADS",+)+++def _resolve_positive_int_env(env_var: str, default: int) -> int:+ """Read a positive integer from ``env_var``, falling back to ``default``.++ A non-positive or unparseable value logs a warning and returns a safe value+ (>= 1) rather than disabling the limit.+ """+ raw = os.environ.get(env_var)+ if raw is None:+ return default+ try:+ value = int(raw)+ except (TypeError, ValueError):+ logger.warning("Invalid %s=%r; falling back to %d.", env_var, raw, default)+ return default+ if value < 1:+ logger.warning("%s=%d is below 1; using 1.", env_var, value)+ return 1+ return value+++def _resolve_embed_thread_cap() -> int:+ """Resolve the per-encode CPU thread cap (``HEADROOM_EMBED_NUM_THREADS``)."""+ return _resolve_positive_int_env(_EMBED_THREADS_ENV, _DEFAULT_EMBED_THREADS)+++def _resolve_embed_concurrency() -> int:+ """Resolve the max concurrent CPU encodes (``HEADROOM_EMBED_CONCURRENCY``).++ Defaults to ``min(4, os.cpu_count())`` so embedding cannot occupy every core+ and starve the event loop, while still allowing useful parallelism.+ """+ cpu = os.cpu_count() or 1+ raw = os.environ.get(_EMBED_CONCURRENCY_ENV)+ if raw is None:+ return max(1, min(_DEFAULT_EMBED_CONCURRENCY, cpu))+ return _resolve_positive_int_env(+ _EMBED_CONCURRENCY_ENV, max(1, min(_DEFAULT_EMBED_CONCURRENCY, cpu))+ )+++def _init_cpu_embed_worker() -> None:+ """Pin a CPU embed worker's thread pool (runs once per worker; issue #198).++ Sets BLAS/OpenMP env defaults (``setdefault`` never overrides an operator's+ explicit setting) and bounds torch's intra-op pool for this worker thread.+ Best-effort: failures never block embedding.+ """+ n = _resolve_embed_thread_cap()+ for var in _BLAS_THREAD_ENV_VARS:+ os.environ.setdefault(var, str(n))+ try:+ import torch++ torch.set_num_threads(n)+ except ImportError:+ pass+ except Exception as exc: # pragma: no cover - defensive, never block embedding+ logger.debug("Could not cap torch intra-op thread pool: %s", exc)++def _normalize_embedding(embedding: np.ndarray) -> np.ndarray:"""Normalize embedding to unit vector for cosine similarity.@@ -172,6 +268,19 @@ class LocalEmbedder:else:self._device = self._detect_device()+ # CPU: run encodes on a dedicated, size-limited executor whose workers+ # each pin their torch/BLAS/OpenMP thread pool (issue #198). Without this,+ # N concurrent encodes on the shared default executor each fan out to+ # ~os.cpu_count() BLAS threads and starve the asyncio event loop, spiking+ # /livez latency. Total embed threads are bounded by workers x per-encode+ # threads.+ if self._device == "cpu" and self._executor is None:+ self._executor = ThreadPoolExecutor(+ max_workers=_resolve_embed_concurrency(),+ thread_name_prefix="cpu-embed",+ initializer=_init_cpu_embed_worker,+ )+# torch-MPS is not thread-safe: concurrent encode() calls from the default# multi-worker executor abort with "commit an already committed command# buffer" (verified). Funnel every encode through one worker thread when ondiff --git a/tests/test_memory/test_embedder_mps_serialization.py b/tests/test_memory/test_embedder_mps_serialization.pyindex 31229af09..54abfe200 100644--- a/tests/test_memory/test_embedder_mps_serialization.py+++ b/tests/test_memory/test_embedder_mps_serialization.py@@ -20,13 +20,16 @@ from headroom.memory.adapters.embedders import LocalEmbedder # noqa: E402_HAS_MPS = bool(getattr(torch.backends, "mps", None)) and torch.backends.mps.is_available()-async def test_cpu_uses_shared_executor() -> None:- """On CPU the dedicated executor stays None (unchanged default-pool behavior)."""+async def test_cpu_uses_dedicated_thread_capped_executor() -> None:+ """On CPU a dedicated, size-limited executor is used so encodes run with a+ bounded thread pool instead of oversubscribing BLAS/OMP threads (issue #198)."""emb = LocalEmbedder(device="cpu")await emb.embed("hello world")assert emb._device == "cpu"- assert emb._executor is None # default shared executor, not serialized+ assert emb._executor is not None # dedicated capped pool, not the shared default+ assert emb._executor._max_workers >= 1 # type: ignore[attr-defined]await emb.close()+ assert emb._executor is None # close() tears it down@pytest.mark.skipif(not _HAS_MPS, reason="requires Apple-Silicon MPS")diff --git a/tests/test_memory/test_embedder_thread_cap.py b/tests/test_memory/test_embedder_thread_cap.pynew file mode 100644index 000000000..14e8761aa--- /dev/null+++ b/tests/test_memory/test_embedder_thread_cap.py@@ -0,0 +1,115 @@+"""Regression tests for the LocalEmbedder CPU thread cap (issue #198).++Under concurrent load the torch/sentence-transformers embedder oversubscribes+BLAS/OpenMP threads (≈ ``os.cpu_count()`` per ``encode()``), starving the+asyncio event loop and spiking ``/livez`` latency. ``LocalEmbedder`` now runs+CPU encodes on a dedicated, size-limited executor whose workers each pin their+torch/BLAS/OpenMP thread pool, bounding total embedding threads to+``HEADROOM_EMBED_CONCURRENCY x HEADROOM_EMBED_NUM_THREADS``. The ONNX embedder+already caps its threads; this brings the torch path to parity.+"""++from __future__ import annotations++import os++import pytest++from headroom.memory.adapters import embedders+from headroom.memory.adapters.embedders import (+ _BLAS_THREAD_ENV_VARS,+ _init_cpu_embed_worker,+ _resolve_embed_concurrency,+ _resolve_embed_thread_cap,+)++# ---------------------------------------------------------------------------+# Env resolution (no torch required)+# ---------------------------------------------------------------------------+++def test_thread_cap_default_when_unset(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_EMBED_NUM_THREADS", raising=False)+ assert _resolve_embed_thread_cap() == 1+++def test_thread_cap_reads_positive_int(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "3")+ assert _resolve_embed_thread_cap() == 3+++def test_thread_cap_invalid_falls_back(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "not-a-number")+ assert _resolve_embed_thread_cap() == 1+++def test_thread_cap_non_positive_is_clamped(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "0")+ assert _resolve_embed_thread_cap() == 1+++def test_concurrency_default_is_bounded(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_EMBED_CONCURRENCY", raising=False)+ value = _resolve_embed_concurrency()+ assert 1 <= value <= 4+ assert value <= (os.cpu_count() or 1)+++def test_concurrency_reads_positive_int(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_EMBED_CONCURRENCY", "7")+ assert _resolve_embed_concurrency() == 7+++# ---------------------------------------------------------------------------+# Worker initializer env application (no torch required)+# ---------------------------------------------------------------------------+++def test_worker_init_sets_blas_env_defaults(monkeypatch: pytest.MonkeyPatch) -> None:+ for var in _BLAS_THREAD_ENV_VARS:+ monkeypatch.delenv(var, raising=False)+ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "2")++ _init_cpu_embed_worker()++ for var in _BLAS_THREAD_ENV_VARS:+ assert os.environ[var] == "2", var+++def test_worker_init_does_not_override_operator_env(monkeypatch: pytest.MonkeyPatch) -> None:+ """An explicit operator setting must win over our default (setdefault)."""+ monkeypatch.setenv("OMP_NUM_THREADS", "8")+ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "1")++ _init_cpu_embed_worker()++ assert os.environ["OMP_NUM_THREADS"] == "8"+++# ---------------------------------------------------------------------------+# Behavioral: real CPU load path bounds every encode worker's thread pool+# ---------------------------------------------------------------------------+++async def test_cpu_embed_workers_are_thread_capped(monkeypatch: pytest.MonkeyPatch) -> None:+ """CPU encodes run on a dedicated, size-limited executor and every worker+ pins its torch intra-op thread pool to the configured cap."""+ torch = pytest.importorskip("torch")+ pytest.importorskip("sentence_transformers")++ monkeypatch.setenv("HEADROOM_EMBED_NUM_THREADS", "1")+ monkeypatch.setenv("HEADROOM_EMBED_CONCURRENCY", "2")++ emb = embedders.LocalEmbedder(device="cpu")+ await emb.embed("hello world")++ assert emb._device == "cpu"+ assert emb._executor is not None+ assert emb._executor._max_workers == 2 # type: ignore[attr-defined]++ # Probe the actual encode workers: each was pinned to 1 intra-op thread.+ futures = [emb._executor.submit(torch.get_num_threads) for _ in range(4)]+ assert [f.result() for f in futures] == [1, 1, 1, 1]++ await emb.close()+ assert emb._executor is None # close() tears the executor down