Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(image): OCR and SigLIP each image once, not once per turn (#3941)
gglucassa8561fb4be1b
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.pyindex 448d52be4..e14c1dcc3 100644--- a/headroom/image/compressor.py+++ b/headroom/image/compressor.py@@ -25,7 +25,9 @@ import base64import ioimport loggingimport re+import timefrom dataclasses import dataclass+from functools import partialfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:@@ -34,10 +36,23 @@ if TYPE_CHECKING:# Import the enum from the dependency-free module so importing the compressor# does not eagerly import trained_router (and thus torch/transformers) — that# eager import crashed on Python 3.13+ (#2513).-from .image_types import Technique+from .image_types import ImageMemo, Techniquelogger = logging.getLogger(__name__)+# Retained OCR text per compressor. A dense full-screen screenshot OCRs to+# ~10-20 KB, so even at 100 KB each this keeps a 300-screenshot history, while a+# worker that lives across conversations cannot grow past it.+_OCR_MEMO_BYTES = 32 * 1024 * 1024+++class OcrEngineError(RuntimeError):+ """OCR engine failed to start or run: a fault of the engine, not the image.++ Unlike a ``None`` (no confident text), it must not be remembered for the+ image, so a recovered engine retries it.+ """+# OCR backend resolution — see issue #372.#@@ -147,6 +162,7 @@ class ImageCompressor:# Cache it on the instance and reuse it.self._router: TrainedRouter | None = Noneself._onnx_router: Any = None+ self._ocr_memo: ImageMemo[str | None] = ImageMemo(max_bytes=_OCR_MEMO_BYTES)# Set on a process-wide shared instance (see the isolation worker and# _get_image_compressor) so a per-request close() does not unload models@@ -415,7 +431,8 @@ class ImageCompressor:``None`` when nothing was detected.Returns extracted text if OCR is confident, ``None`` otherwise- (caller falls back to image-as-image).+ (caller falls back to image-as-image). Raises ``OcrEngineError`` when+ the engine fails to initialize or run."""ocr_cls, api_version = _resolve_rapidocr()if ocr_cls is None:@@ -434,7 +451,7 @@ class ImageCompressor:api_version,exc,)- return None+ raise OcrEngineError(str(exc)) from exctry:raw = self._ocr_engine(image_data)@@ -444,7 +461,7 @@ class ImageCompressor:api_version,exc,)- return None+ raise OcrEngineError(str(exc)) from excif api_version == "v1":# 1.x returns (list_of_tuples, elapsed). list may be empty@@ -525,8 +542,15 @@ class ImageCompressor:messages: list[dict[str, Any]],technique: Technique,provider: str,+ deadline: float | None = None,) -> list[dict[str, Any]]:- """Apply compression technique to messages."""+ """Apply compression technique to messages.++ ``deadline`` (a ``time.time()`` value) stops new OCR once passed: an image+ not OCR'd yet takes the no-text fallback this turn and is OCR'd on a later+ one, so a long history of new screenshots finishes inside the caller's+ timeout instead of being killed and redone from the start every turn.+ """if technique.value == "preserve":return messages@@ -572,7 +596,19 @@ class ImageCompressor:# --- TRANSCODE: OCR the image and replace with text ---if technique.value == "transcode" and image_bytes_for_ocr:- extracted = self._ocr_extract(image_bytes_for_ocr)+ extracted = None+ if (+ deadline is None+ or time.time() < deadline+ or image_bytes_for_ocr in self._ocr_memo+ ):+ try:+ extracted = self._ocr_memo.get(+ image_bytes_for_ocr,+ partial(self._ocr_extract, image_bytes_for_ocr),+ )+ except OcrEngineError:+ pass # not memoized: retried next turnif extracted:# Replace image with extracted textnew_content.append(@@ -648,6 +684,7 @@ class ImageCompressor:self,messages: list[dict[str, Any]],provider: str = "openai",+ deadline: float | None = None,) -> list[dict[str, Any]]:"""Compress images in messages.@@ -659,6 +696,8 @@ class ImageCompressor:Args:messages: LLM messages (OpenAI/Anthropic/Google format)provider: Target provider ('openai', 'anthropic', 'google')+ deadline: ``time.time()`` after which no new OCR starts (see+ ``_apply_compression``); None for no limit.Returns:Messages with compressed images@@ -730,7 +769,7 @@ class ImageCompressor:original_tokens = self._estimate_tokens(image_data, "high") + tile_saved# Step 3: Apply compression technique- compressed_messages = self._apply_compression(messages, technique, provider)+ compressed_messages = self._apply_compression(messages, technique, provider, deadline)# Count actual tokens AFTER compression by measuring the result.# If the image was replaced with text (OCR), count text tokens.diff --git a/headroom/image/image_types.py b/headroom/image/image_types.pyindex 572fcf7f3..483c33b8e 100644--- a/headroom/image/image_types.py+++ b/headroom/image/image_types.py@@ -10,8 +10,59 @@ initializing inside the proxy process (#2513).from __future__ import annotations+import hashlib+import sys+from collections import OrderedDict+from collections.abc import Callablefrom dataclasses import dataclassfrom enum import Enum+from typing import Generic, TypeVar++_T = TypeVar("_T")+++class ImageMemo(Generic[_T]):+ """Bounded LRU of per-image results, keyed on the image bytes' sha256.++ Every turn resends the whole conversation, so the same screenshots reach+ the image stack on every request; OCR and the SigLIP encoder are pure+ functions of the bytes, so each image pays for them once. Only what+ ``compute`` returns is remembered: a ``compute`` that raises caches+ nothing, so a failure that says nothing about the image is retried.++ Two bounds, whichever binds first: ``max_entries``, and ``max_bytes`` of+ retained values as ``sys.getsizeof`` measures them (exact for ``str``,+ shallow for other objects, so bound those by entries). A value larger+ than ``max_bytes`` on its own is returned uncached.++ A transcode turn scans the history oldest-first, and a history larger than+ the memo makes LRU thrash: every miss evicts the image the scan reaches+ next, so nothing is reused. The defaults sit far above one conversation's+ screenshots and only guard a worker that lives across conversations.+ """++ def __init__(self, max_entries: int = 4096, max_bytes: int = sys.maxsize) -> None:+ self._max_entries = max_entries+ self._max_bytes = max_bytes+ self._bytes = 0+ self._entries: OrderedDict[bytes, tuple[_T, int]] = OrderedDict()++ def __contains__(self, image_data: bytes) -> bool:+ return hashlib.sha256(image_data).digest() in self._entries++ def get(self, image_data: bytes, compute: Callable[[], _T]) -> _T:+ key = hashlib.sha256(image_data).digest()+ entry = self._entries.pop(key, None)+ if entry is None:+ value = compute()+ entry = (value, sys.getsizeof(value))+ if entry[1] > self._max_bytes:+ return value+ self._bytes += entry[1]+ self._entries[key] = entry # (re)inserted last: most recently used+ while len(self._entries) > self._max_entries or self._bytes > self._max_bytes:+ self._bytes -= self._entries.popitem(last=False)[1][1]+ return entry[0]class Technique(Enum):diff --git a/headroom/image/onnx_router.py b/headroom/image/onnx_router.pyindex 3471bd368..a6bc582b1 100644--- a/headroom/image/onnx_router.py+++ b/headroom/image/onnx_router.py@@ -15,12 +15,13 @@ import ioimport loggingimport mathimport os+from functools import partialfrom pathlib import Pathfrom typing import Anyimport numpy as np-from headroom.image.image_types import ImageSignals, RouteDecision, Technique+from headroom.image.image_types import ImageMemo, ImageSignals, RouteDecision, Techniquefrom headroom.onnx_runtime import create_cpu_session_options, hf_hub_download_local_firstlogger = logging.getLogger(__name__)@@ -85,6 +86,7 @@ class OnnxTechniqueRouter:self._siglip_session: Any = Noneself._text_embeddings: dict[str, np.ndarray] = {}self._siglip_processor: Any = None+ self._signals_memo: ImageMemo[ImageSignals] = ImageMemo()def _load_classifier(self) -> None:"""Lazy-load the technique router ONNX model."""@@ -178,45 +180,51 @@ class OnnxTechniqueRouter:return technique, confidencedef analyze_image(self, image_data: bytes) -> ImageSignals | None:- """Analyze image properties using SigLIP ONNX encoder."""+ """Analyze image properties using SigLIP ONNX encoder.++ Memoized per image; a failed analysis is not, so it is retried.+ """if not self.use_siglip:return Noneself._load_siglip()try:- from PIL import Image-- img = Image.open(io.BytesIO(image_data)).convert("RGB")- img = img.resize((224, 224), Image.Resampling.LANCZOS)-- # Convert to numpy: [1, 3, 224, 224], normalized to [-1, 1]- arr = np.array(img, dtype=np.float32) / 255.0- arr = (arr - 0.5) / 0.5 # Normalize to [-1, 1]- arr = arr.transpose(2, 0, 1) # HWC → CHW- pixel_values = arr[np.newaxis, ...] # Add batch dim-- embeds = self._siglip_session.run(None, {"pixel_values": pixel_values})[0]- embeds = embeds / np.linalg.norm(embeds, axis=-1, keepdims=True)-- def sigmoid(x: float) -> float:- return 1 / (1 + math.exp(-x * 5))-- scores = {}- for signal_name, text_emb in self._text_embeddings.items():- sim = (embeds @ text_emb.T).squeeze()- scores[signal_name] = sigmoid(float(sim.max()))-- return ImageSignals(- has_text=scores.get("has_text", 0.5),- is_document=scores.get("is_document", 0.5),- is_complex=scores.get("is_complex", 0.5),- has_small_details=scores.get("has_small_details", 0.5),- )+ return self._signals_memo.get(image_data, partial(self._encode_image, image_data))except Exception as e:logger.warning(f"SigLIP image analysis failed: {e}")return None+ def _encode_image(self, image_data: bytes) -> ImageSignals:+ from PIL import Image++ img = Image.open(io.BytesIO(image_data)).convert("RGB")+ img = img.resize((224, 224), Image.Resampling.LANCZOS)++ # Convert to numpy: [1, 3, 224, 224], normalized to [-1, 1]+ arr = np.array(img, dtype=np.float32) / 255.0+ arr = (arr - 0.5) / 0.5 # Normalize to [-1, 1]+ arr = arr.transpose(2, 0, 1) # HWC → CHW+ pixel_values = arr[np.newaxis, ...] # Add batch dim++ embeds = self._siglip_session.run(None, {"pixel_values": pixel_values})[0]+ embeds = embeds / np.linalg.norm(embeds, axis=-1, keepdims=True)++ def sigmoid(x: float) -> float:+ return 1 / (1 + math.exp(-x * 5))++ scores = {}+ for signal_name, text_emb in self._text_embeddings.items():+ sim = (embeds @ text_emb.T).squeeze()+ scores[signal_name] = sigmoid(float(sim.max()))++ return ImageSignals(+ has_text=scores.get("has_text", 0.5),+ is_document=scores.get("is_document", 0.5),+ is_complex=scores.get("is_complex", 0.5),+ has_small_details=scores.get("has_small_details", 0.5),+ )+def classify(self, image_data: bytes, query: str) -> RouteDecision:"""Combined query + image classification."""technique, query_confidence = self.classify_query(query)diff --git a/headroom/proxy/image_isolation.py b/headroom/proxy/image_isolation.pyindex fb291057e..1e92b229c 100644--- a/headroom/proxy/image_isolation.py+++ b/headroom/proxy/image_isolation.py@@ -11,6 +11,10 @@ from concurrent.futures import ProcessPoolExecutor, TimeoutErrorfrom concurrent.futures.process import BrokenProcessPoolfrom typing import Any+# Share of the call's timeout spent starting new OCR. One OCR already running+# when it runs out still finishes, so the rest of the budget is its margin.+_OCR_BUDGET_FRACTION = 2 / 3+logger = logging.getLogger("headroom.proxy")_IMAGE_POOL_LOCK = threading.Lock()@@ -40,9 +44,10 @@ def _get_worker_compressor() -> Any:def _compress_messages_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:compressor = _get_worker_compressor()- compressed = compressor.compress(messages, provider=provider)+ compressed = compressor.compress(messages, provider=provider, deadline=deadline)if compressor.last_result is None:return compressed, Nonereturn compressed, {@@ -57,6 +62,7 @@ def _compress_messages_worker(def _success_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:compressed = [dict(message) for message in messages]compressed[-1] = {**compressed[-1], "content": f"compressed:{provider}"}@@ -72,6 +78,7 @@ def _success_worker(def _raise_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:raise RuntimeError(f"boom:{provider}")@@ -79,6 +86,7 @@ def _raise_worker(def _sleep_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:time.sleep(0.2)return messages, None@@ -87,6 +95,7 @@ def _sleep_worker(def _hang_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:time.sleep(60)return messages, None@@ -95,12 +104,13 @@ def _hang_worker(def _abort_worker(messages: list[dict[str, Any]],provider: str,+ deadline: float | None = None,) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:os.abort()_IMAGE_WORKER: Callable[- [list[dict[str, Any]], str],+ [list[dict[str, Any]], str, float | None],tuple[list[dict[str, Any]], dict[str, Any] | None],] = _compress_messages_worker@@ -149,8 +159,9 @@ async def run_image_compression_isolated(loop = asyncio.get_running_loop()pool: ProcessPoolExecutor | None = Nonetry:+ deadline = time.time() + timeout * _OCR_BUDGET_FRACTIONpool = _image_pool()- future = loop.run_in_executor(pool, _IMAGE_WORKER, messages, provider)+ future = loop.run_in_executor(pool, _IMAGE_WORKER, messages, provider, deadline)return await asyncio.wait_for(future, timeout=timeout)except BrokenProcessPool:logger.warning("Image compression worker crashed; forwarding original image payload")diff --git a/tests/test_image_memo.py b/tests/test_image_memo.pynew file mode 100644index 000000000..95dd8e2cc--- /dev/null+++ b/tests/test_image_memo.py@@ -0,0 +1,198 @@+"""OCR and SigLIP run once per image, not once per turn.++Every turn resends the whole conversation, so each screenshot in history+reached the image stack again on every request: one proxy OCR'd the same+screenshot 114 times in two hours.+"""++from __future__ import annotations++import base64+import sys+import types+from typing import Any++import pytest++from headroom.image import compressor as compressor_module+from headroom.image.compressor import ImageCompressor+from headroom.image.image_types import ImageMemo, ImageSignals, Technique+from headroom.image.onnx_router import OnnxTechniqueRouter+++def _image(data: bytes) -> dict[str, Any]:+ return {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": base64.b64encode(data).decode(),+ },+ }+++def test_transcode_ocrs_each_image_once_across_turns(monkeypatch: pytest.MonkeyPatch) -> None:+ calls: list[bytes] = []++ class _RapidOCR:+ def __call__(self, image_data: bytes) -> Any:+ calls.append(image_data)+ return [(None, f"text of {image_data.decode()}", 0.99)], 0.0++ fake = types.ModuleType("rapidocr_onnxruntime")+ fake.RapidOCR = _RapidOCR # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake)+ monkeypatch.setattr(compressor_module, "_RESOLVED_OCR", None)++ compressor = ImageCompressor(use_siglip=False)+ turn1 = [{"role": "user", "content": [_image(b"a")]}]+ turn2 = [+ *turn1,+ {"role": "assistant", "content": "ok"},+ {"role": "user", "content": [_image(b"b")]},+ ]++ first = compressor._apply_compression(turn1, Technique.TRANSCODE, "anthropic")+ second = compressor._apply_compression(turn2, Technique.TRANSCODE, "anthropic")++ assert calls == [b"a", b"b"]+ assert second[0] == first[0]+ assert first[0]["content"] == [{"type": "text", "text": "[OCR from image]\ntext of a"}]+ assert second[2]["content"] == [{"type": "text", "text": "[OCR from image]\ntext of b"}]+++def test_past_deadline_ocrs_nothing_new_but_reuses_the_memo(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A cold history that would outrun the isolation timeout stops starting+ # OCR at the deadline, so the call returns, the worker and its memo+ # survive, and the next turn picks up where this one stopped.+ calls: list[bytes] = []+ compressor = ImageCompressor(use_siglip=False)+ monkeypatch.setattr(+ compressor, "_ocr_extract", lambda data: calls.append(data) or f"text of {data.decode()}"+ )+ seen = [{"role": "user", "content": [_image(b"a")]}]+ compressor._apply_compression(seen, Technique.TRANSCODE, "anthropic")+ history = [{"role": "user", "content": [_image(b"a"), _image(b"b")]}]+ calls.clear()++ late = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic", deadline=0.0)+ on_time = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")++ assert late[0]["content"][0] == {"type": "text", "text": "[OCR from image]\ntext of a"}+ assert late[0]["content"][1]["type"] == "image"+ assert on_time[0]["content"][1] == {"type": "text", "text": "[OCR from image]\ntext of b"}+ assert calls == [b"b"]+++def test_long_history_reuses_ocr_across_turns(monkeypatch: pytest.MonkeyPatch) -> None:+ # Each transcode turn scans the history oldest-first. With fewer slots+ # than images, every miss evicts the very image the scan reaches next, so+ # nothing was ever reused; 300 screenshots must fit.+ calls: list[bytes] = []+ compressor = ImageCompressor(use_siglip=False)+ monkeypatch.setattr(compressor, "_ocr_extract", lambda data: calls.append(data) or "text")+ images = [_image(str(i).encode()) for i in range(300)]+ history = [{"role": "user", "content": images}]++ for _ in range(2):+ compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")++ assert len(calls) == 300+++def test_ocr_engine_failure_is_retried_but_no_text_is_remembered(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # "No confident text" is a property of the image; an engine that raised+ # says nothing about it, so remembering that would skip the image for the+ # worker's lifetime.+ calls: list[bytes] = []+ engine_up = False++ class _RapidOCR:+ def __call__(self, image_data: bytes) -> Any:+ calls.append(image_data)+ if not engine_up:+ raise RuntimeError("onnxruntime session lost")+ return ([], 0.0) if image_data == b"blank" else ([(None, "hi", 0.99)], 0.0)++ fake = types.ModuleType("rapidocr_onnxruntime")+ fake.RapidOCR = _RapidOCR # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake)+ monkeypatch.setattr(compressor_module, "_RESOLVED_OCR", None)++ compressor = ImageCompressor(use_siglip=False)+ history = [{"role": "user", "content": [_image(b"blank"), _image(b"text")]}]++ down = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")+ engine_up = True+ up = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")+ again = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")++ assert [part["type"] for part in down[0]["content"]] == ["image", "image"]+ assert up[0]["content"][1] == {"type": "text", "text": "[OCR from image]\nhi"}+ assert again == up+ assert calls == [b"blank", b"text", b"blank", b"text"]+++def test_classify_analyzes_each_image_once(monkeypatch: pytest.MonkeyPatch) -> None:+ router = OnnxTechniqueRouter()+ analyzed: list[bytes] = []++ def encode(image_data: bytes) -> ImageSignals:+ analyzed.append(image_data)+ if len(analyzed) == 1:+ raise RuntimeError("siglip session lost")+ return ImageSignals(has_text=0.9, is_document=0.1, is_complex=0.1, has_small_details=0.1)++ monkeypatch.setattr(router, "classify_query", lambda query: (Technique.PRESERVE, 0.9))+ monkeypatch.setattr(router, "_load_siglip", lambda: None)+ monkeypatch.setattr(router, "_encode_image", encode)++ # The query changes every turn; the image analysis does not need to. The+ # first analysis of "a" fails, so it is retried rather than remembered.+ turns = ((b"a", "what"), (b"a", "and now?"), (b"a", "and?"), (b"b", "what"))+ for image_data, query in turns:+ router.classify(image_data, query)++ assert analyzed == [b"a", b"a", b"b"]+++def test_image_memo_evicts_least_recently_used() -> None:+ memo = ImageMemo[str](max_entries=2)+ computed: list[bytes] = []++ def get(data: bytes) -> str:+ return memo.get(data, lambda: computed.append(data) or data.decode())++ for data in (b"a", b"b", b"a", b"c", b"a", b"b"):+ assert get(data) == data.decode()++ # "a" stayed hot, so "c" evicted "b", and "b" evicted "c".+ assert computed == [b"a", b"b", b"c", b"b"]+++def test_image_memo_holds_at_most_max_bytes_of_values() -> None:+ # Entry count alone let a few thousand text-heavy screenshots pin hundreds+ # of MB of OCR text in a worker that outlives the conversation.+ text = "x" * 1000+ memo = ImageMemo[str](max_bytes=3 * sys.getsizeof(text))+ computed: list[bytes] = []++ def get(data: bytes, value: str = text) -> str:+ return memo.get(data, lambda: computed.append(data) or value)++ for data in (b"a", b"b", b"c", b"d", b"b"):+ get(data)+ assert computed == [b"a", b"b", b"c", b"d"] # "d" evicted "a"; "b" still held+ get(b"a")+ assert computed[-1] == b"a"++ # A value bigger than the whole budget is returned but evicts nothing.+ huge = "y" * 10_000+ assert get(b"huge", huge) == huge+ get(b"huge", huge)+ assert computed[-2:] == [b"huge", b"huge"]+ assert b"b" in memo and b"a" in memo