Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(image): reuse image models instead of rebuilding them per request (#2513) (#2536)
Abhay Singh2a63ec70b656
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.pyindex f9ba35a78..5dc7baa97 100644--- a/headroom/image/compressor.py+++ b/headroom/image/compressor.py@@ -138,8 +138,17 @@ class ImageCompressor:self.use_siglip = use_siglipself.device = device- # Lazy-loaded router+ # Lazy-loaded routers. The ONNX router loads native ort.InferenceSession+ # models that Python's GC does not eagerly reclaim, so building one per+ # compress() call grew RSS unboundedly under image traffic (#2513).+ # Cache it on the instance and reuse it.self._router: TrainedRouter | None = None+ self._onnx_router: Any = None++ # Set on a process-wide shared instance (see the isolation worker and+ # _get_image_compressor) so a per-request close() does not unload models+ # the next request would just reload.+ self._is_singleton = False# Last compression result (for metrics)self.last_result: CompressionResult | None = None@@ -163,13 +172,36 @@ class ImageCompressor:)return self._router+ def _get_onnx_router(self) -> Any:+ """Lazy-load and cache the ONNX technique router.++ Building an ``OnnxTechniqueRouter`` loads native ``ort.InferenceSession``+ models; creating one per ``compress()`` call leaked native memory and+ grew RSS unboundedly under image traffic (#2513). Cache one per instance.+ """+ if self._onnx_router is None:+ from .onnx_router import OnnxTechniqueRouter++ self._onnx_router = OnnxTechniqueRouter(use_siglip=self.use_siglip)+ return self._onnx_router+def close(self, unload_models: bool = True) -> None:- """Release any router-held model state."""+ """Release any router-held model state.++ A process-wide shared instance (``_is_singleton``) keeps its models+ loaded across requests, so a per-request ``close()`` must be a no-op+ there; otherwise every image request would reload the ONNX/torch models+ it just cached, reintroducing the #2513 leak.+ """+ if self._is_singleton:+ returnif self._router is not None:# Only loaded routers hold heavyweight image models; plain has_images()# checks remain cheap and have nothing to release.self._router.release_models(unload_registry=unload_models)self._router = None+ # Drop the cached ONNX router so its native sessions can be reclaimed.+ self._onnx_router = Nonedef has_images(self, messages: list[dict[str, Any]]) -> bool:"""Check if messages contain images."""@@ -675,9 +707,7 @@ class ImageCompressor:confidence = 0.0else:try:- from .onnx_router import OnnxTechniqueRouter-- onnx_router = OnnxTechniqueRouter(use_siglip=self.use_siglip)+ onnx_router = self._get_onnx_router()decision = onnx_router.classify(image_data, query)technique = decision.techniqueconfidence = decision.confidencediff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 7dfc7d91b..562cb3385 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -929,24 +929,36 @@ async def request_with_transient_retry(# Image compression availability (do not retain a global compressor instance)_image_compressor_available: bool | None = None+_image_compressor_instance: Any = Nonedef _get_image_compressor():- """Create a short-lived image compressor on demand."""- global _image_compressor_available+ """Return the process-wide image compressor, or None if unavailable.++ The compressor caches heavyweight models; creating a new one per request+ (and a new ONNX router per image) accumulated native memory and grew RSS+ unboundedly (#2513). Reuse a single shared instance. It is marked a+ singleton so a caller's per-request ``close()`` is a no-op and the models+ stay loaded. The main-process handlers only call ``has_images()`` on it (the+ heavy compression runs in the isolation worker), but sharing still avoids a+ fresh object per request.+ """+ global _image_compressor_available, _image_compressor_instanceif _image_compressor_available is False:return None+ if _image_compressor_instance is not None:+ return _image_compressor_instancetry:from headroom.image import ImageCompressor- # Callers own closing the compressor; this helper only memoizes whether- # the optional image stack is importable.- compressor = ImageCompressor()+ instance = ImageCompressor()+ instance._is_singleton = Trueif _image_compressor_available is None:logger.info("Image compression enabled (model: chopratejas/technique-router)")_image_compressor_available = True- return compressor+ _image_compressor_instance = instance+ return instanceexcept ImportError as e:if _image_compressor_available is not False:logger.warning(f"Image compression not available: {e}")diff --git a/headroom/proxy/image_isolation.py b/headroom/proxy/image_isolation.pyindex b0176ef7a..2e2addf54 100644--- a/headroom/proxy/image_isolation.py+++ b/headroom/proxy/image_isolation.py@@ -16,27 +16,42 @@ logger = logging.getLogger("headroom.proxy")_IMAGE_POOL_LOCK = threading.Lock()_IMAGE_POOL: ProcessPoolExecutor | None = None+# Per-worker-process cached compressor. The image pool is a persistent+# single-worker ProcessPoolExecutor, so building a fresh ImageCompressor (and,+# inside it, a fresh ONNX router loading native ort.InferenceSession models) on+# every call accumulated native memory in the worker and grew RSS to 1+ GB over+# a day (#2513). Load the models once per worker and reuse them.+_WORKER_COMPRESSOR: Any = None+++def _get_worker_compressor() -> Any:+ global _WORKER_COMPRESSOR+ if _WORKER_COMPRESSOR is None:+ from headroom.image import ImageCompressor++ instance = ImageCompressor()+ # Shared across calls in this worker: don't let a per-call close() unload+ # the models the next call reuses.+ instance._is_singleton = True+ _WORKER_COMPRESSOR = instance+ return _WORKER_COMPRESSOR+def _compress_messages_worker(messages: list[dict[str, Any]],provider: str,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:- from headroom.image import ImageCompressor-- compressor = ImageCompressor()- try:- compressed = compressor.compress(messages, provider=provider)- if compressor.last_result is None:- return compressed, None- return compressed, {- "technique": compressor.last_result.technique.value,- "original_tokens": compressor.last_result.original_tokens,- "compressed_tokens": compressor.last_result.compressed_tokens,- "confidence": compressor.last_result.confidence,- "savings_percent": compressor.last_result.savings_percent,- }- finally:- compressor.close()+ compressor = _get_worker_compressor()+ compressed = compressor.compress(messages, provider=provider)+ if compressor.last_result is None:+ return compressed, None+ return compressed, {+ "technique": compressor.last_result.technique.value,+ "original_tokens": compressor.last_result.original_tokens,+ "compressed_tokens": compressor.last_result.compressed_tokens,+ "confidence": compressor.last_result.confidence,+ "savings_percent": compressor.last_result.savings_percent,+ }def _success_worker(diff --git a/tests/test_image_compressor_singleton_reuse.py b/tests/test_image_compressor_singleton_reuse.pynew file mode 100644index 000000000..4890780d8--- /dev/null+++ b/tests/test_image_compressor_singleton_reuse.py@@ -0,0 +1,82 @@+"""Image models must be loaded once and reused, not rebuilt per request (#2513).++Every image request built a new ImageCompressor and a new OnnxTechniqueRouter,+each loading native ort.InferenceSession models that grew worker RSS to 1+ GB+over a day. These tests pin the caching / singleton behavior with the heavy+model construction mocked out.+"""++from __future__ import annotations++from unittest.mock import MagicMock, patch++from headroom.image.compressor import ImageCompressor+++def test_onnx_router_is_built_once_and_cached() -> None:+ compressor = ImageCompressor()+ fake_router = MagicMock(name="OnnxTechniqueRouter")++ with patch("headroom.image.onnx_router.OnnxTechniqueRouter", return_value=fake_router) as ctor:+ first = compressor._get_onnx_router()+ second = compressor._get_onnx_router()++ assert first is second is fake_router+ assert ctor.call_count == 1+++def test_close_is_a_noop_on_a_singleton_instance() -> None:+ compressor = ImageCompressor()+ compressor._is_singleton = True+ router = MagicMock()+ compressor._router = router+ compressor._onnx_router = MagicMock()++ compressor.close()++ # Models stay loaded so the next request reuses them.+ assert compressor._router is router+ assert compressor._onnx_router is not None+ router.release_models.assert_not_called()+++def test_close_releases_models_on_a_non_singleton_instance() -> None:+ compressor = ImageCompressor()+ router = MagicMock()+ compressor._router = router+ compressor._onnx_router = MagicMock()++ compressor.close()++ router.release_models.assert_called_once()+ assert compressor._router is None+ assert compressor._onnx_router is None+++def test_get_image_compressor_returns_a_shared_singleton() -> None:+ import headroom.proxy.helpers as helpers++ helpers._image_compressor_available = None+ helpers._image_compressor_instance = None+ try:+ a = helpers._get_image_compressor()+ b = helpers._get_image_compressor()+ assert a is not None+ assert a is b+ assert a._is_singleton is True+ finally:+ helpers._image_compressor_available = None+ helpers._image_compressor_instance = None+++def test_worker_compressor_is_reused_across_calls() -> None:+ import headroom.proxy.image_isolation as iso++ iso._WORKER_COMPRESSOR = None+ try:+ a = iso._get_worker_compressor()+ b = iso._get_worker_compressor()+ assert a is b+ assert a._is_singleton is True+ finally:+ iso._WORKER_COMPRESSOR = Nonediff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex 9292f9930..bb86076aa 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -721,10 +721,13 @@ def test_anthropic_image_compression_helper_only_rewrites_latest_eligible_turn()) == [compressed]-def test_proxy_helper_creates_fresh_image_compressors(monkeypatch) -> None:+def test_proxy_helper_reuses_a_singleton_image_compressor(monkeypatch) -> None:+ # #2513: the compressor caches heavyweight models, so it must be a+ # process-wide singleton rather than a fresh instance per request.from headroom.proxy import helpersmonkeypatch.setattr(helpers, "_image_compressor_available", None)+ monkeypatch.setattr(helpers, "_image_compressor_instance", None)_FreshCompressor.instances = 0with patch("headroom.image.ImageCompressor", _FreshCompressor):@@ -732,9 +735,9 @@ def test_proxy_helper_creates_fresh_image_compressors(monkeypatch) -> None:second = helpers._get_image_compressor()assert isinstance(first, _FreshCompressor)- assert isinstance(second, _FreshCompressor)- assert first is not second- assert _FreshCompressor.instances == 2+ assert first is second+ assert first._is_singleton is True+ assert _FreshCompressor.instances == 1def test_proxy_helper_caches_image_stack_import_failure(monkeypatch) -> None:@@ -751,6 +754,7 @@ def test_proxy_helper_caches_image_stack_import_failure(monkeypatch) -> None:return real_import(name, *args, **kwargs)monkeypatch.setattr(helpers, "_image_compressor_available", None)+ monkeypatch.setattr(helpers, "_image_compressor_instance", None)monkeypatch.setattr(builtins, "__import__", fake_import)assert helpers._get_image_compressor() is None