Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(image): reuse image models instead of rebuilding them per request (#2513) (#2536)

Abhay Singh2a63ec70b656
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.py
index f9ba35a78..5dc7baa97 100644
--- a/headroom/image/compressor.py
+++ b/headroom/image/compressor.py
@@ -138,8 +138,17 @@ class ImageCompressor:
self.use_siglip = use_siglip
self.device = device
- # Lazy-loaded router
+ # Lazy-loaded routers. The ONNX router loads native ort.InferenceSession
+ # models that Python's GC does not eagerly reclaim, so building one per
+ # compress() call grew RSS unboundedly under image traffic (#2513).
+ # Cache it on the instance and reuse it.
self._router: TrainedRouter | None = None
+ self._onnx_router: Any = None
+
+ # Set on a process-wide shared instance (see the isolation worker and
+ # _get_image_compressor) so a per-request close() does not unload models
+ # the next request would just reload.
+ self._is_singleton = False
# Last compression result (for metrics)
self.last_result: CompressionResult | None = None
@@ -163,13 +172,36 @@ class ImageCompressor:
)
return self._router
+ def _get_onnx_router(self) -> Any:
+ """Lazy-load and cache the ONNX technique router.
+
+ Building an ``OnnxTechniqueRouter`` loads native ``ort.InferenceSession``
+ models; creating one per ``compress()`` call leaked native memory and
+ grew RSS unboundedly under image traffic (#2513). Cache one per instance.
+ """
+ if self._onnx_router is None:
+ from .onnx_router import OnnxTechniqueRouter
+
+ self._onnx_router = OnnxTechniqueRouter(use_siglip=self.use_siglip)
+ return self._onnx_router
+
def close(self, unload_models: bool = True) -> None:
- """Release any router-held model state."""
+ """Release any router-held model state.
+
+ A process-wide shared instance (``_is_singleton``) keeps its models
+ loaded across requests, so a per-request ``close()`` must be a no-op
+ there; otherwise every image request would reload the ONNX/torch models
+ it just cached, reintroducing the #2513 leak.
+ """
+ if self._is_singleton:
+ return
if self._router is not None:
# Only loaded routers hold heavyweight image models; plain has_images()
# checks remain cheap and have nothing to release.
self._router.release_models(unload_registry=unload_models)
self._router = None
+ # Drop the cached ONNX router so its native sessions can be reclaimed.
+ self._onnx_router = None
def has_images(self, messages: list[dict[str, Any]]) -> bool:
"""Check if messages contain images."""
@@ -675,9 +707,7 @@ class ImageCompressor:
confidence = 0.0
else:
try:
- from .onnx_router import OnnxTechniqueRouter
-
- onnx_router = OnnxTechniqueRouter(use_siglip=self.use_siglip)
+ onnx_router = self._get_onnx_router()
decision = onnx_router.classify(image_data, query)
technique = decision.technique
confidence = decision.confidence
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 7dfc7d91b..562cb3385 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -929,24 +929,36 @@ async def request_with_transient_retry(
# Image compression availability (do not retain a global compressor instance)
_image_compressor_available: bool | None = None
+_image_compressor_instance: Any = None
def _get_image_compressor():
- """Create a short-lived image compressor on demand."""
- global _image_compressor_available
+ """Return the process-wide image compressor, or None if unavailable.
+
+ The compressor caches heavyweight models; creating a new one per request
+ (and a new ONNX router per image) accumulated native memory and grew RSS
+ unboundedly (#2513). Reuse a single shared instance. It is marked a
+ singleton so a caller's per-request ``close()`` is a no-op and the models
+ stay loaded. The main-process handlers only call ``has_images()`` on it (the
+ heavy compression runs in the isolation worker), but sharing still avoids a
+ fresh object per request.
+ """
+ global _image_compressor_available, _image_compressor_instance
if _image_compressor_available is False:
return None
+ if _image_compressor_instance is not None:
+ return _image_compressor_instance
try:
from headroom.image import ImageCompressor
- # Callers own closing the compressor; this helper only memoizes whether
- # the optional image stack is importable.
- compressor = ImageCompressor()
+ instance = ImageCompressor()
+ instance._is_singleton = True
if _image_compressor_available is None:
logger.info("Image compression enabled (model: chopratejas/technique-router)")
_image_compressor_available = True
- return compressor
+ _image_compressor_instance = instance
+ return instance
except ImportError as e:
if _image_compressor_available is not False:
logger.warning(f"Image compression not available: {e}")
diff --git a/headroom/proxy/image_isolation.py b/headroom/proxy/image_isolation.py
index b0176ef7a..2e2addf54 100644
--- a/headroom/proxy/image_isolation.py
+++ b/headroom/proxy/image_isolation.py
@@ -16,27 +16,42 @@ logger = logging.getLogger("headroom.proxy")
_IMAGE_POOL_LOCK = threading.Lock()
_IMAGE_POOL: ProcessPoolExecutor | None = None
+# Per-worker-process cached compressor. The image pool is a persistent
+# single-worker ProcessPoolExecutor, so building a fresh ImageCompressor (and,
+# inside it, a fresh ONNX router loading native ort.InferenceSession models) on
+# every call accumulated native memory in the worker and grew RSS to 1+ GB over
+# a day (#2513). Load the models once per worker and reuse them.
+_WORKER_COMPRESSOR: Any = None
+
+
+def _get_worker_compressor() -> Any:
+ global _WORKER_COMPRESSOR
+ if _WORKER_COMPRESSOR is None:
+ from headroom.image import ImageCompressor
+
+ instance = ImageCompressor()
+ # Shared across calls in this worker: don't let a per-call close() unload
+ # the models the next call reuses.
+ instance._is_singleton = True
+ _WORKER_COMPRESSOR = instance
+ return _WORKER_COMPRESSOR
+
def _compress_messages_worker(
messages: list[dict[str, Any]],
provider: str,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
- from headroom.image import ImageCompressor
-
- compressor = ImageCompressor()
- try:
- compressed = compressor.compress(messages, provider=provider)
- if compressor.last_result is None:
- return compressed, None
- return compressed, {
- "technique": compressor.last_result.technique.value,
- "original_tokens": compressor.last_result.original_tokens,
- "compressed_tokens": compressor.last_result.compressed_tokens,
- "confidence": compressor.last_result.confidence,
- "savings_percent": compressor.last_result.savings_percent,
- }
- finally:
- compressor.close()
+ compressor = _get_worker_compressor()
+ compressed = compressor.compress(messages, provider=provider)
+ if compressor.last_result is None:
+ return compressed, None
+ return compressed, {
+ "technique": compressor.last_result.technique.value,
+ "original_tokens": compressor.last_result.original_tokens,
+ "compressed_tokens": compressor.last_result.compressed_tokens,
+ "confidence": compressor.last_result.confidence,
+ "savings_percent": compressor.last_result.savings_percent,
+ }
def _success_worker(
diff --git a/tests/test_image_compressor_singleton_reuse.py b/tests/test_image_compressor_singleton_reuse.py
new file mode 100644
index 000000000..4890780d8
--- /dev/null
+++ b/tests/test_image_compressor_singleton_reuse.py
@@ -0,0 +1,82 @@
+"""Image models must be loaded once and reused, not rebuilt per request (#2513).
+
+Every image request built a new ImageCompressor and a new OnnxTechniqueRouter,
+each loading native ort.InferenceSession models that grew worker RSS to 1+ GB
+over a day. These tests pin the caching / singleton behavior with the heavy
+model construction mocked out.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from headroom.image.compressor import ImageCompressor
+
+
+def test_onnx_router_is_built_once_and_cached() -> None:
+ compressor = ImageCompressor()
+ fake_router = MagicMock(name="OnnxTechniqueRouter")
+
+ with patch("headroom.image.onnx_router.OnnxTechniqueRouter", return_value=fake_router) as ctor:
+ first = compressor._get_onnx_router()
+ second = compressor._get_onnx_router()
+
+ assert first is second is fake_router
+ assert ctor.call_count == 1
+
+
+def test_close_is_a_noop_on_a_singleton_instance() -> None:
+ compressor = ImageCompressor()
+ compressor._is_singleton = True
+ router = MagicMock()
+ compressor._router = router
+ compressor._onnx_router = MagicMock()
+
+ compressor.close()
+
+ # Models stay loaded so the next request reuses them.
+ assert compressor._router is router
+ assert compressor._onnx_router is not None
+ router.release_models.assert_not_called()
+
+
+def test_close_releases_models_on_a_non_singleton_instance() -> None:
+ compressor = ImageCompressor()
+ router = MagicMock()
+ compressor._router = router
+ compressor._onnx_router = MagicMock()
+
+ compressor.close()
+
+ router.release_models.assert_called_once()
+ assert compressor._router is None
+ assert compressor._onnx_router is None
+
+
+def test_get_image_compressor_returns_a_shared_singleton() -> None:
+ import headroom.proxy.helpers as helpers
+
+ helpers._image_compressor_available = None
+ helpers._image_compressor_instance = None
+ try:
+ a = helpers._get_image_compressor()
+ b = helpers._get_image_compressor()
+ assert a is not None
+ assert a is b
+ assert a._is_singleton is True
+ finally:
+ helpers._image_compressor_available = None
+ helpers._image_compressor_instance = None
+
+
+def test_worker_compressor_is_reused_across_calls() -> None:
+ import headroom.proxy.image_isolation as iso
+
+ iso._WORKER_COMPRESSOR = None
+ try:
+ a = iso._get_worker_compressor()
+ b = iso._get_worker_compressor()
+ assert a is b
+ assert a._is_singleton is True
+ finally:
+ iso._WORKER_COMPRESSOR = None
diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.py
index 9292f9930..bb86076aa 100644
--- a/tests/test_proxy_handler_helpers.py
+++ b/tests/test_proxy_handler_helpers.py
@@ -721,10 +721,13 @@ def test_anthropic_image_compression_helper_only_rewrites_latest_eligible_turn()
) == [compressed]
-def test_proxy_helper_creates_fresh_image_compressors(monkeypatch) -> None:
+def test_proxy_helper_reuses_a_singleton_image_compressor(monkeypatch) -> None:
+ # #2513: the compressor caches heavyweight models, so it must be a
+ # process-wide singleton rather than a fresh instance per request.
from headroom.proxy import helpers
monkeypatch.setattr(helpers, "_image_compressor_available", None)
+ monkeypatch.setattr(helpers, "_image_compressor_instance", None)
_FreshCompressor.instances = 0
with patch("headroom.image.ImageCompressor", _FreshCompressor):
@@ -732,9 +735,9 @@ def test_proxy_helper_creates_fresh_image_compressors(monkeypatch) -> None:
second = helpers._get_image_compressor()
assert isinstance(first, _FreshCompressor)
- assert isinstance(second, _FreshCompressor)
- assert first is not second
- assert _FreshCompressor.instances == 2
+ assert first is second
+ assert first._is_singleton is True
+ assert _FreshCompressor.instances == 1
def test_proxy_helper_caches_image_stack_import_failure(monkeypatch) -> None:
@@ -751,6 +754,7 @@ def test_proxy_helper_caches_image_stack_import_failure(monkeypatch) -> None:
return real_import(name, *args, **kwargs)
monkeypatch.setattr(helpers, "_image_compressor_available", None)
+ monkeypatch.setattr(helpers, "_image_compressor_instance", None)
monkeypatch.setattr(builtins, "__import__", fake_import)
assert helpers._get_image_compressor() is None