Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(image): OCR and SigLIP each image once, not once per turn (#3941)

gglucassa8561fb4be1b
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.py
index 448d52be4..e14c1dcc3 100644
--- a/headroom/image/compressor.py
+++ b/headroom/image/compressor.py
@@ -25,7 +25,9 @@ import base64
import io
import logging
import re
+import time
from dataclasses import dataclass
+from functools import partial
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -34,10 +36,23 @@ if TYPE_CHECKING:
# Import the enum from the dependency-free module so importing the compressor
# does not eagerly import trained_router (and thus torch/transformers) — that
# eager import crashed on Python 3.13+ (#2513).
-from .image_types import Technique
+from .image_types import ImageMemo, Technique
logger = logging.getLogger(__name__)
+# Retained OCR text per compressor. A dense full-screen screenshot OCRs to
+# ~10-20 KB, so even at 100 KB each this keeps a 300-screenshot history, while a
+# worker that lives across conversations cannot grow past it.
+_OCR_MEMO_BYTES = 32 * 1024 * 1024
+
+
+class OcrEngineError(RuntimeError):
+ """OCR engine failed to start or run: a fault of the engine, not the image.
+
+ Unlike a ``None`` (no confident text), it must not be remembered for the
+ image, so a recovered engine retries it.
+ """
+
# OCR backend resolution — see issue #372.
#
@@ -147,6 +162,7 @@ class ImageCompressor:
# Cache it on the instance and reuse it.
self._router: TrainedRouter | None = None
self._onnx_router: Any = None
+ self._ocr_memo: ImageMemo[str | None] = ImageMemo(max_bytes=_OCR_MEMO_BYTES)
# Set on a process-wide shared instance (see the isolation worker and
# _get_image_compressor) so a per-request close() does not unload models
@@ -415,7 +431,8 @@ class ImageCompressor:
``None`` when nothing was detected.
Returns extracted text if OCR is confident, ``None`` otherwise
- (caller falls back to image-as-image).
+ (caller falls back to image-as-image). Raises ``OcrEngineError`` when
+ the engine fails to initialize or run.
"""
ocr_cls, api_version = _resolve_rapidocr()
if ocr_cls is None:
@@ -434,7 +451,7 @@ class ImageCompressor:
api_version,
exc,
)
- return None
+ raise OcrEngineError(str(exc)) from exc
try:
raw = self._ocr_engine(image_data)
@@ -444,7 +461,7 @@ class ImageCompressor:
api_version,
exc,
)
- return None
+ raise OcrEngineError(str(exc)) from exc
if api_version == "v1":
# 1.x returns (list_of_tuples, elapsed). list may be empty
@@ -525,8 +542,15 @@ class ImageCompressor:
messages: list[dict[str, Any]],
technique: Technique,
provider: str,
+ deadline: float | None = None,
) -> list[dict[str, Any]]:
- """Apply compression technique to messages."""
+ """Apply compression technique to messages.
+
+ ``deadline`` (a ``time.time()`` value) stops new OCR once passed: an image
+ not OCR'd yet takes the no-text fallback this turn and is OCR'd on a later
+ one, so a long history of new screenshots finishes inside the caller's
+ timeout instead of being killed and redone from the start every turn.
+ """
if technique.value == "preserve":
return messages
@@ -572,7 +596,19 @@ class ImageCompressor:
# --- TRANSCODE: OCR the image and replace with text ---
if technique.value == "transcode" and image_bytes_for_ocr:
- extracted = self._ocr_extract(image_bytes_for_ocr)
+ extracted = None
+ if (
+ deadline is None
+ or time.time() < deadline
+ or image_bytes_for_ocr in self._ocr_memo
+ ):
+ try:
+ extracted = self._ocr_memo.get(
+ image_bytes_for_ocr,
+ partial(self._ocr_extract, image_bytes_for_ocr),
+ )
+ except OcrEngineError:
+ pass # not memoized: retried next turn
if extracted:
# Replace image with extracted text
new_content.append(
@@ -648,6 +684,7 @@ class ImageCompressor:
self,
messages: list[dict[str, Any]],
provider: str = "openai",
+ deadline: float | None = None,
) -> list[dict[str, Any]]:
"""Compress images in messages.
@@ -659,6 +696,8 @@ class ImageCompressor:
Args:
messages: LLM messages (OpenAI/Anthropic/Google format)
provider: Target provider ('openai', 'anthropic', 'google')
+ deadline: ``time.time()`` after which no new OCR starts (see
+ ``_apply_compression``); None for no limit.
Returns:
Messages with compressed images
@@ -730,7 +769,7 @@ class ImageCompressor:
original_tokens = self._estimate_tokens(image_data, "high") + tile_saved
# Step 3: Apply compression technique
- compressed_messages = self._apply_compression(messages, technique, provider)
+ compressed_messages = self._apply_compression(messages, technique, provider, deadline)
# Count actual tokens AFTER compression by measuring the result.
# If the image was replaced with text (OCR), count text tokens.
diff --git a/headroom/image/image_types.py b/headroom/image/image_types.py
index 572fcf7f3..483c33b8e 100644
--- a/headroom/image/image_types.py
+++ b/headroom/image/image_types.py
@@ -10,8 +10,59 @@ initializing inside the proxy process (#2513).
from __future__ import annotations
+import hashlib
+import sys
+from collections import OrderedDict
+from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
+from typing import Generic, TypeVar
+
+_T = TypeVar("_T")
+
+
+class ImageMemo(Generic[_T]):
+ """Bounded LRU of per-image results, keyed on the image bytes' sha256.
+
+ Every turn resends the whole conversation, so the same screenshots reach
+ the image stack on every request; OCR and the SigLIP encoder are pure
+ functions of the bytes, so each image pays for them once. Only what
+ ``compute`` returns is remembered: a ``compute`` that raises caches
+ nothing, so a failure that says nothing about the image is retried.
+
+ Two bounds, whichever binds first: ``max_entries``, and ``max_bytes`` of
+ retained values as ``sys.getsizeof`` measures them (exact for ``str``,
+ shallow for other objects, so bound those by entries). A value larger
+ than ``max_bytes`` on its own is returned uncached.
+
+ A transcode turn scans the history oldest-first, and a history larger than
+ the memo makes LRU thrash: every miss evicts the image the scan reaches
+ next, so nothing is reused. The defaults sit far above one conversation's
+ screenshots and only guard a worker that lives across conversations.
+ """
+
+ def __init__(self, max_entries: int = 4096, max_bytes: int = sys.maxsize) -> None:
+ self._max_entries = max_entries
+ self._max_bytes = max_bytes
+ self._bytes = 0
+ self._entries: OrderedDict[bytes, tuple[_T, int]] = OrderedDict()
+
+ def __contains__(self, image_data: bytes) -> bool:
+ return hashlib.sha256(image_data).digest() in self._entries
+
+ def get(self, image_data: bytes, compute: Callable[[], _T]) -> _T:
+ key = hashlib.sha256(image_data).digest()
+ entry = self._entries.pop(key, None)
+ if entry is None:
+ value = compute()
+ entry = (value, sys.getsizeof(value))
+ if entry[1] > self._max_bytes:
+ return value
+ self._bytes += entry[1]
+ self._entries[key] = entry # (re)inserted last: most recently used
+ while len(self._entries) > self._max_entries or self._bytes > self._max_bytes:
+ self._bytes -= self._entries.popitem(last=False)[1][1]
+ return entry[0]
class Technique(Enum):
diff --git a/headroom/image/onnx_router.py b/headroom/image/onnx_router.py
index 3471bd368..a6bc582b1 100644
--- a/headroom/image/onnx_router.py
+++ b/headroom/image/onnx_router.py
@@ -15,12 +15,13 @@ import io
import logging
import math
import os
+from functools import partial
from pathlib import Path
from typing import Any
import numpy as np
-from headroom.image.image_types import ImageSignals, RouteDecision, Technique
+from headroom.image.image_types import ImageMemo, ImageSignals, RouteDecision, Technique
from headroom.onnx_runtime import create_cpu_session_options, hf_hub_download_local_first
logger = logging.getLogger(__name__)
@@ -85,6 +86,7 @@ class OnnxTechniqueRouter:
self._siglip_session: Any = None
self._text_embeddings: dict[str, np.ndarray] = {}
self._siglip_processor: Any = None
+ self._signals_memo: ImageMemo[ImageSignals] = ImageMemo()
def _load_classifier(self) -> None:
"""Lazy-load the technique router ONNX model."""
@@ -178,45 +180,51 @@ class OnnxTechniqueRouter:
return technique, confidence
def analyze_image(self, image_data: bytes) -> ImageSignals | None:
- """Analyze image properties using SigLIP ONNX encoder."""
+ """Analyze image properties using SigLIP ONNX encoder.
+
+ Memoized per image; a failed analysis is not, so it is retried.
+ """
if not self.use_siglip:
return None
self._load_siglip()
try:
- from PIL import Image
-
- img = Image.open(io.BytesIO(image_data)).convert("RGB")
- img = img.resize((224, 224), Image.Resampling.LANCZOS)
-
- # Convert to numpy: [1, 3, 224, 224], normalized to [-1, 1]
- arr = np.array(img, dtype=np.float32) / 255.0
- arr = (arr - 0.5) / 0.5 # Normalize to [-1, 1]
- arr = arr.transpose(2, 0, 1) # HWC → CHW
- pixel_values = arr[np.newaxis, ...] # Add batch dim
-
- embeds = self._siglip_session.run(None, {"pixel_values": pixel_values})[0]
- embeds = embeds / np.linalg.norm(embeds, axis=-1, keepdims=True)
-
- def sigmoid(x: float) -> float:
- return 1 / (1 + math.exp(-x * 5))
-
- scores = {}
- for signal_name, text_emb in self._text_embeddings.items():
- sim = (embeds @ text_emb.T).squeeze()
- scores[signal_name] = sigmoid(float(sim.max()))
-
- return ImageSignals(
- has_text=scores.get("has_text", 0.5),
- is_document=scores.get("is_document", 0.5),
- is_complex=scores.get("is_complex", 0.5),
- has_small_details=scores.get("has_small_details", 0.5),
- )
+ return self._signals_memo.get(image_data, partial(self._encode_image, image_data))
except Exception as e:
logger.warning(f"SigLIP image analysis failed: {e}")
return None
+ def _encode_image(self, image_data: bytes) -> ImageSignals:
+ from PIL import Image
+
+ img = Image.open(io.BytesIO(image_data)).convert("RGB")
+ img = img.resize((224, 224), Image.Resampling.LANCZOS)
+
+ # Convert to numpy: [1, 3, 224, 224], normalized to [-1, 1]
+ arr = np.array(img, dtype=np.float32) / 255.0
+ arr = (arr - 0.5) / 0.5 # Normalize to [-1, 1]
+ arr = arr.transpose(2, 0, 1) # HWC → CHW
+ pixel_values = arr[np.newaxis, ...] # Add batch dim
+
+ embeds = self._siglip_session.run(None, {"pixel_values": pixel_values})[0]
+ embeds = embeds / np.linalg.norm(embeds, axis=-1, keepdims=True)
+
+ def sigmoid(x: float) -> float:
+ return 1 / (1 + math.exp(-x * 5))
+
+ scores = {}
+ for signal_name, text_emb in self._text_embeddings.items():
+ sim = (embeds @ text_emb.T).squeeze()
+ scores[signal_name] = sigmoid(float(sim.max()))
+
+ return ImageSignals(
+ has_text=scores.get("has_text", 0.5),
+ is_document=scores.get("is_document", 0.5),
+ is_complex=scores.get("is_complex", 0.5),
+ has_small_details=scores.get("has_small_details", 0.5),
+ )
+
def classify(self, image_data: bytes, query: str) -> RouteDecision:
"""Combined query + image classification."""
technique, query_confidence = self.classify_query(query)
diff --git a/headroom/proxy/image_isolation.py b/headroom/proxy/image_isolation.py
index fb291057e..1e92b229c 100644
--- a/headroom/proxy/image_isolation.py
+++ b/headroom/proxy/image_isolation.py
@@ -11,6 +11,10 @@ from concurrent.futures import ProcessPoolExecutor, TimeoutError
from concurrent.futures.process import BrokenProcessPool
from typing import Any
+# Share of the call's timeout spent starting new OCR. One OCR already running
+# when it runs out still finishes, so the rest of the budget is its margin.
+_OCR_BUDGET_FRACTION = 2 / 3
+
logger = logging.getLogger("headroom.proxy")
_IMAGE_POOL_LOCK = threading.Lock()
@@ -40,9 +44,10 @@ def _get_worker_compressor() -> Any:
def _compress_messages_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
compressor = _get_worker_compressor()
- compressed = compressor.compress(messages, provider=provider)
+ compressed = compressor.compress(messages, provider=provider, deadline=deadline)
if compressor.last_result is None:
return compressed, None
return compressed, {
@@ -57,6 +62,7 @@ def _compress_messages_worker(
def _success_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
compressed = [dict(message) for message in messages]
compressed[-1] = {**compressed[-1], "content": f"compressed:{provider}"}
@@ -72,6 +78,7 @@ def _success_worker(
def _raise_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
raise RuntimeError(f"boom:{provider}")
@@ -79,6 +86,7 @@ def _raise_worker(
def _sleep_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
time.sleep(0.2)
return messages, None
@@ -87,6 +95,7 @@ def _sleep_worker(
def _hang_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
time.sleep(60)
return messages, None
@@ -95,12 +104,13 @@ def _hang_worker(
def _abort_worker(
messages: list[dict[str, Any]],
provider: str,
+ deadline: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
os.abort()
_IMAGE_WORKER: Callable[
- [list[dict[str, Any]], str],
+ [list[dict[str, Any]], str, float | None],
tuple[list[dict[str, Any]], dict[str, Any] | None],
] = _compress_messages_worker
@@ -149,8 +159,9 @@ async def run_image_compression_isolated(
loop = asyncio.get_running_loop()
pool: ProcessPoolExecutor | None = None
try:
+ deadline = time.time() + timeout * _OCR_BUDGET_FRACTION
pool = _image_pool()
- future = loop.run_in_executor(pool, _IMAGE_WORKER, messages, provider)
+ future = loop.run_in_executor(pool, _IMAGE_WORKER, messages, provider, deadline)
return await asyncio.wait_for(future, timeout=timeout)
except BrokenProcessPool:
logger.warning("Image compression worker crashed; forwarding original image payload")
diff --git a/tests/test_image_memo.py b/tests/test_image_memo.py
new file mode 100644
index 000000000..95dd8e2cc
--- /dev/null
+++ b/tests/test_image_memo.py
@@ -0,0 +1,198 @@
+"""OCR and SigLIP run once per image, not once per turn.
+
+Every turn resends the whole conversation, so each screenshot in history
+reached the image stack again on every request: one proxy OCR'd the same
+screenshot 114 times in two hours.
+"""
+
+from __future__ import annotations
+
+import base64
+import sys
+import types
+from typing import Any
+
+import pytest
+
+from headroom.image import compressor as compressor_module
+from headroom.image.compressor import ImageCompressor
+from headroom.image.image_types import ImageMemo, ImageSignals, Technique
+from headroom.image.onnx_router import OnnxTechniqueRouter
+
+
+def _image(data: bytes) -> dict[str, Any]:
+ return {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": base64.b64encode(data).decode(),
+ },
+ }
+
+
+def test_transcode_ocrs_each_image_once_across_turns(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[bytes] = []
+
+ class _RapidOCR:
+ def __call__(self, image_data: bytes) -> Any:
+ calls.append(image_data)
+ return [(None, f"text of {image_data.decode()}", 0.99)], 0.0
+
+ fake = types.ModuleType("rapidocr_onnxruntime")
+ fake.RapidOCR = _RapidOCR # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake)
+ monkeypatch.setattr(compressor_module, "_RESOLVED_OCR", None)
+
+ compressor = ImageCompressor(use_siglip=False)
+ turn1 = [{"role": "user", "content": [_image(b"a")]}]
+ turn2 = [
+ *turn1,
+ {"role": "assistant", "content": "ok"},
+ {"role": "user", "content": [_image(b"b")]},
+ ]
+
+ first = compressor._apply_compression(turn1, Technique.TRANSCODE, "anthropic")
+ second = compressor._apply_compression(turn2, Technique.TRANSCODE, "anthropic")
+
+ assert calls == [b"a", b"b"]
+ assert second[0] == first[0]
+ assert first[0]["content"] == [{"type": "text", "text": "[OCR from image]\ntext of a"}]
+ assert second[2]["content"] == [{"type": "text", "text": "[OCR from image]\ntext of b"}]
+
+
+def test_past_deadline_ocrs_nothing_new_but_reuses_the_memo(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A cold history that would outrun the isolation timeout stops starting
+ # OCR at the deadline, so the call returns, the worker and its memo
+ # survive, and the next turn picks up where this one stopped.
+ calls: list[bytes] = []
+ compressor = ImageCompressor(use_siglip=False)
+ monkeypatch.setattr(
+ compressor, "_ocr_extract", lambda data: calls.append(data) or f"text of {data.decode()}"
+ )
+ seen = [{"role": "user", "content": [_image(b"a")]}]
+ compressor._apply_compression(seen, Technique.TRANSCODE, "anthropic")
+ history = [{"role": "user", "content": [_image(b"a"), _image(b"b")]}]
+ calls.clear()
+
+ late = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic", deadline=0.0)
+ on_time = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")
+
+ assert late[0]["content"][0] == {"type": "text", "text": "[OCR from image]\ntext of a"}
+ assert late[0]["content"][1]["type"] == "image"
+ assert on_time[0]["content"][1] == {"type": "text", "text": "[OCR from image]\ntext of b"}
+ assert calls == [b"b"]
+
+
+def test_long_history_reuses_ocr_across_turns(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Each transcode turn scans the history oldest-first. With fewer slots
+ # than images, every miss evicts the very image the scan reaches next, so
+ # nothing was ever reused; 300 screenshots must fit.
+ calls: list[bytes] = []
+ compressor = ImageCompressor(use_siglip=False)
+ monkeypatch.setattr(compressor, "_ocr_extract", lambda data: calls.append(data) or "text")
+ images = [_image(str(i).encode()) for i in range(300)]
+ history = [{"role": "user", "content": images}]
+
+ for _ in range(2):
+ compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")
+
+ assert len(calls) == 300
+
+
+def test_ocr_engine_failure_is_retried_but_no_text_is_remembered(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # "No confident text" is a property of the image; an engine that raised
+ # says nothing about it, so remembering that would skip the image for the
+ # worker's lifetime.
+ calls: list[bytes] = []
+ engine_up = False
+
+ class _RapidOCR:
+ def __call__(self, image_data: bytes) -> Any:
+ calls.append(image_data)
+ if not engine_up:
+ raise RuntimeError("onnxruntime session lost")
+ return ([], 0.0) if image_data == b"blank" else ([(None, "hi", 0.99)], 0.0)
+
+ fake = types.ModuleType("rapidocr_onnxruntime")
+ fake.RapidOCR = _RapidOCR # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake)
+ monkeypatch.setattr(compressor_module, "_RESOLVED_OCR", None)
+
+ compressor = ImageCompressor(use_siglip=False)
+ history = [{"role": "user", "content": [_image(b"blank"), _image(b"text")]}]
+
+ down = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")
+ engine_up = True
+ up = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")
+ again = compressor._apply_compression(history, Technique.TRANSCODE, "anthropic")
+
+ assert [part["type"] for part in down[0]["content"]] == ["image", "image"]
+ assert up[0]["content"][1] == {"type": "text", "text": "[OCR from image]\nhi"}
+ assert again == up
+ assert calls == [b"blank", b"text", b"blank", b"text"]
+
+
+def test_classify_analyzes_each_image_once(monkeypatch: pytest.MonkeyPatch) -> None:
+ router = OnnxTechniqueRouter()
+ analyzed: list[bytes] = []
+
+ def encode(image_data: bytes) -> ImageSignals:
+ analyzed.append(image_data)
+ if len(analyzed) == 1:
+ raise RuntimeError("siglip session lost")
+ return ImageSignals(has_text=0.9, is_document=0.1, is_complex=0.1, has_small_details=0.1)
+
+ monkeypatch.setattr(router, "classify_query", lambda query: (Technique.PRESERVE, 0.9))
+ monkeypatch.setattr(router, "_load_siglip", lambda: None)
+ monkeypatch.setattr(router, "_encode_image", encode)
+
+ # The query changes every turn; the image analysis does not need to. The
+ # first analysis of "a" fails, so it is retried rather than remembered.
+ turns = ((b"a", "what"), (b"a", "and now?"), (b"a", "and?"), (b"b", "what"))
+ for image_data, query in turns:
+ router.classify(image_data, query)
+
+ assert analyzed == [b"a", b"a", b"b"]
+
+
+def test_image_memo_evicts_least_recently_used() -> None:
+ memo = ImageMemo[str](max_entries=2)
+ computed: list[bytes] = []
+
+ def get(data: bytes) -> str:
+ return memo.get(data, lambda: computed.append(data) or data.decode())
+
+ for data in (b"a", b"b", b"a", b"c", b"a", b"b"):
+ assert get(data) == data.decode()
+
+ # "a" stayed hot, so "c" evicted "b", and "b" evicted "c".
+ assert computed == [b"a", b"b", b"c", b"b"]
+
+
+def test_image_memo_holds_at_most_max_bytes_of_values() -> None:
+ # Entry count alone let a few thousand text-heavy screenshots pin hundreds
+ # of MB of OCR text in a worker that outlives the conversation.
+ text = "x" * 1000
+ memo = ImageMemo[str](max_bytes=3 * sys.getsizeof(text))
+ computed: list[bytes] = []
+
+ def get(data: bytes, value: str = text) -> str:
+ return memo.get(data, lambda: computed.append(data) or value)
+
+ for data in (b"a", b"b", b"c", b"d", b"b"):
+ get(data)
+ assert computed == [b"a", b"b", b"c", b"d"] # "d" evicted "a"; "b" still held
+ get(b"a")
+ assert computed[-1] == b"a"
+
+ # A value bigger than the whole budget is returned but evicts nothing.
+ huge = "y" * 10_000
+ assert get(b"huge", huge) == huge
+ get(b"huge", huge)
+ assert computed[-2:] == [b"huge", b"huge"]
+ assert b"b" in memo and b"a" in memo