Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Re-enable image compression: tile optimizer + ONNX router, no PyTorch

chopratejas7984bd23893a
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.py
index 8e06c4c92..71db4e00e 100644
--- a/headroom/image/compressor.py
+++ b/headroom/image/compressor.py
@@ -359,6 +359,11 @@ class ImageCompressor:
) -> list[dict[str, Any]]:
"""Compress images in messages.
+ Pipeline:
+ 1. Tile-boundary alignment (pure math, zero quality loss)
+ 2. ML-based technique routing (ONNX, query + image analysis)
+ 3. Apply compression technique
+
Args:
messages: LLM messages (OpenAI/Anthropic/Google format)
provider: Target provider ('openai', 'anthropic', 'google')
@@ -369,26 +374,57 @@ class ImageCompressor:
if not self.has_images(messages):
return messages
- # Extract query and image
+ # Step 1: Tile-boundary optimization (always safe, pure math)
+ try:
+ from .tile_optimizer import optimize_images_in_messages
+
+ messages, tile_results = optimize_images_in_messages(messages, provider)
+ tile_saved = sum(r.tokens_saved for r in tile_results)
+ if tile_saved > 0:
+ logger.info(
+ f"Image tile optimization: saved {tile_saved} tokens "
+ f"({len(tile_results)} image(s))"
+ )
+ except Exception as e:
+ logger.debug(f"Tile optimization skipped: {e}")
+ tile_saved = 0
+
+ # Step 2: ML-based technique routing
query = self._extract_query(messages)
image_data = self._extract_image_data(messages)
if not query or not image_data:
- logger.debug("Could not extract query or image, skipping compression")
+ # Still got tile savings even without ML routing
+ if tile_saved > 0:
+ self.last_result = CompressionResult(
+ technique=Technique.PRESERVE,
+ original_tokens=tile_saved,
+ compressed_tokens=0,
+ confidence=1.0,
+ )
return messages
- # Route to technique
+ # Try ONNX router first (lightweight), fall back to PyTorch router
try:
- router = self._get_router()
- decision = router.classify(image_data, query)
+ from .onnx_router import OnnxTechniqueRouter
+
+ onnx_router = OnnxTechniqueRouter(use_siglip=self.use_siglip)
+ decision = onnx_router.classify(image_data, query)
technique = decision.technique
confidence = decision.confidence
- except Exception as e:
- logger.warning(f"Router failed, preserving image: {e}")
- technique = Technique.PRESERVE
- confidence = 0.0
-
- # Calculate tokens - compare by value since technique is from trained_router
+ except Exception as onnx_err:
+ logger.debug(f"ONNX router not available ({onnx_err}), trying PyTorch...")
+ try:
+ pt_router = self._get_router()
+ decision = pt_router.classify(image_data, query)
+ technique = decision.technique
+ confidence = decision.confidence
+ except Exception as e:
+ logger.warning(f"Router failed, preserving image: {e}")
+ technique = Technique.PRESERVE
+ confidence = 0.0
+
+ # Calculate tokens
original_tokens = self._estimate_tokens(image_data, "high")
if technique.value == "full_low":
@@ -402,10 +438,10 @@ class ImageCompressor:
else:
compressed_tokens = original_tokens
- # Store result
+ # Store result (include tile savings)
self.last_result = CompressionResult(
technique=technique,
- original_tokens=original_tokens,
+ original_tokens=original_tokens + tile_saved,
compressed_tokens=compressed_tokens,
confidence=confidence,
)
@@ -416,7 +452,7 @@ class ImageCompressor:
f"{self.last_result.savings_percent:.0f}% saved)"
)
- # Apply compression
+ # Step 3: Apply compression technique
return self._apply_compression(messages, technique, provider)
diff --git a/headroom/image/onnx_router.py b/headroom/image/onnx_router.py
new file mode 100644
index 000000000..dc009d051
--- /dev/null
+++ b/headroom/image/onnx_router.py
@@ -0,0 +1,203 @@
+"""ONNX-based image technique router — no PyTorch dependency.
+
+Uses ONNX INT8 models for both query classification (technique-router)
+and image analysis (SigLIP), matching the accuracy of the PyTorch
+versions at 15x smaller size and no GPU requirement.
+
+Models auto-downloaded from HuggingFace on first use:
+- chopratejas/technique-router-onnx (~32 MB)
+- chopratejas/siglip-image-encoder-onnx (~95 MB)
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import math
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from headroom.image.trained_router import ImageSignals, RouteDecision, Technique
+
+logger = logging.getLogger(__name__)
+
+_TECHNIQUE_ROUTER_REPO = "chopratejas/technique-router-onnx"
+_SIGLIP_ENCODER_REPO = "chopratejas/siglip-image-encoder-onnx"
+
+
+# ImageSignals, RouteDecision, Technique imported from trained_router
+
+
+class OnnxTechniqueRouter:
+ """ONNX-based technique router — no PyTorch dependency.
+
+ Uses:
+ 1. MiniLM ONNX INT8 classifier for query intent (~32 MB, ~5ms)
+ 2. SigLIP ONNX INT8 image encoder for image analysis (~95 MB, ~30ms)
+ 3. Pre-computed text embeddings for image property scoring (~25 KB)
+
+ Total: ~127 MB, runs on CPU with onnxruntime only.
+ """
+
+ def __init__(self, use_siglip: bool = True):
+ self.use_siglip = use_siglip
+ self._classifier_session: Any = None
+ self._tokenizer: Any = None
+ self._id2label: dict[int, str] = {}
+ self._siglip_session: Any = None
+ self._text_embeddings: dict[str, np.ndarray] = {}
+ self._siglip_processor: Any = None
+
+ def _load_classifier(self) -> None:
+ """Lazy-load the technique router ONNX model."""
+ if self._classifier_session is not None:
+ return
+
+ import onnxruntime as ort
+ from huggingface_hub import hf_hub_download
+ from tokenizers import Tokenizer
+
+ logger.info("Loading technique-router ONNX INT8...")
+
+ model_path = hf_hub_download(_TECHNIQUE_ROUTER_REPO, "model_quantized.onnx")
+ self._classifier_session = ort.InferenceSession(
+ model_path, providers=["CPUExecutionProvider"]
+ )
+
+ tokenizer_path = hf_hub_download(_TECHNIQUE_ROUTER_REPO, "tokenizer.json")
+ self._tokenizer = Tokenizer.from_file(tokenizer_path)
+ self._tokenizer.enable_truncation(max_length=64)
+ self._tokenizer.enable_padding(length=64)
+
+ # Load label mapping
+ import json
+
+ config_path = hf_hub_download(_TECHNIQUE_ROUTER_REPO, "config.json")
+ with open(config_path) as f:
+ config = json.load(f)
+ self._id2label = {int(k): v for k, v in config.get("id2label", {}).items()}
+
+ logger.info(
+ f"Technique router loaded: {len(self._id2label)} classes, "
+ f"ONNX INT8 ({Path(model_path).stat().st_size // 1024 // 1024} MB)"
+ )
+
+ def _load_siglip(self) -> None:
+ """Lazy-load the SigLIP ONNX image encoder."""
+ if self._siglip_session is not None:
+ return
+
+ import onnxruntime as ort
+ from huggingface_hub import hf_hub_download
+
+ logger.info("Loading SigLIP ONNX INT8 image encoder...")
+
+ model_path = hf_hub_download(_SIGLIP_ENCODER_REPO, "image_encoder_int8.onnx")
+ self._siglip_session = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
+
+ embeddings_path = hf_hub_download(_SIGLIP_ENCODER_REPO, "text_embeddings.npz")
+ loaded = np.load(embeddings_path)
+ self._text_embeddings = {k: loaded[k] for k in loaded.files}
+
+ logger.info(
+ f"SigLIP image encoder loaded: ONNX INT8 "
+ f"({Path(model_path).stat().st_size // 1024 // 1024} MB)"
+ )
+
+ def classify_query(self, query: str) -> tuple[Technique, float]:
+ """Classify query intent using ONNX technique router."""
+ self._load_classifier()
+
+ encoded = self._tokenizer.encode(query)
+ input_ids = np.array([encoded.ids], dtype=np.int64)
+ attention_mask = np.array([encoded.attention_mask], dtype=np.int64)
+ token_type_ids = np.zeros_like(input_ids, dtype=np.int64)
+
+ logits = self._classifier_session.run(
+ None,
+ {
+ "input_ids": input_ids,
+ "attention_mask": attention_mask,
+ "token_type_ids": token_type_ids,
+ },
+ )[0][0]
+
+ probs = np.exp(logits) / np.exp(logits).sum()
+ pred_id = int(np.argmax(probs))
+ confidence = float(probs[pred_id])
+
+ technique_name = self._id2label.get(pred_id, "preserve")
+ technique = Technique(technique_name)
+
+ return technique, confidence
+
+ def analyze_image(self, image_data: bytes) -> ImageSignals | None:
+ """Analyze image properties using SigLIP ONNX encoder."""
+ if not self.use_siglip:
+ return None
+
+ self._load_siglip()
+
+ try:
+ from PIL import Image
+
+ img = Image.open(io.BytesIO(image_data)).convert("RGB")
+ img = img.resize((224, 224), Image.Resampling.LANCZOS)
+
+ # Convert to numpy: [1, 3, 224, 224], normalized to [-1, 1]
+ arr = np.array(img, dtype=np.float32) / 255.0
+ arr = (arr - 0.5) / 0.5 # Normalize to [-1, 1]
+ arr = arr.transpose(2, 0, 1) # HWC → CHW
+ pixel_values = arr[np.newaxis, ...] # Add batch dim
+
+ embeds = self._siglip_session.run(None, {"pixel_values": pixel_values})[0]
+ embeds = embeds / np.linalg.norm(embeds, axis=-1, keepdims=True)
+
+ def sigmoid(x: float) -> float:
+ return 1 / (1 + math.exp(-x * 5))
+
+ scores = {}
+ for signal_name, text_emb in self._text_embeddings.items():
+ sim = (embeds @ text_emb.T).squeeze()
+ scores[signal_name] = sigmoid(float(sim.max()))
+
+ return ImageSignals(
+ has_text=scores.get("has_text", 0.5),
+ is_document=scores.get("is_document", 0.5),
+ is_complex=scores.get("is_complex", 0.5),
+ has_small_details=scores.get("has_small_details", 0.5),
+ )
+ except Exception as e:
+ logger.warning(f"SigLIP image analysis failed: {e}")
+ return None
+
+ def classify(self, image_data: bytes, query: str) -> RouteDecision:
+ """Combined query + image classification."""
+ technique, query_confidence = self.classify_query(query)
+ image_signals = self.analyze_image(image_data)
+
+ confidence = query_confidence
+ reason = f"Query → '{technique.value}' ({query_confidence:.0%})"
+
+ # Apply image-based adjustments
+ if image_signals:
+ if technique == Technique.TRANSCODE:
+ if image_signals.has_text < 0.4 and image_signals.is_document < 0.4:
+ confidence *= 0.8
+ reason += " (low text in image)"
+ elif technique == Technique.FULL_LOW:
+ if image_signals.has_small_details > 0.7:
+ reason += " (note: fine details detected)"
+ elif technique == Technique.PRESERVE:
+ if image_signals.has_small_details > 0.5 or image_signals.is_complex > 0.5:
+ confidence = min(1.0, confidence * 1.1)
+ reason += " (confirmed: complex/detailed)"
+
+ return RouteDecision(
+ technique=technique,
+ confidence=confidence,
+ reason=reason,
+ image_signals=image_signals,
+ )
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index a2eb3be34..bcb867aff 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -142,20 +142,20 @@ class OpenAIHandlerMixin:
if _bypass:
logger.info(f"[{request_id}] Bypass: skipping compression (header)")
- # TODO: Re-enable image compression once token counting is accurate.
- # See anthropic.py handler for details on why this is disabled.
- #
- # if self.config.image_optimize and messages and not _bypass:
- # compressor = _get_image_compressor()
- # if compressor and compressor.has_images(messages):
- # messages = compressor.compress(messages, provider="openai")
- # if compressor.last_result:
- # logger.info(
- # f"Image compression: {compressor.last_result.technique.value} "
- # f"({compressor.last_result.savings_percent:.0f}% saved, "
- # f"{compressor.last_result.original_tokens} -> "
- # f"{compressor.last_result.compressed_tokens} tokens)"
- # )
+ # Image compression: tile alignment + ML-based technique routing
+ if self.config.image_optimize and messages and not _bypass:
+ from headroom.proxy.helpers import _get_image_compressor
+
+ compressor = _get_image_compressor()
+ if compressor and compressor.has_images(messages):
+ messages = compressor.compress(messages, provider="openai")
+ if compressor.last_result:
+ logger.info(
+ f"[{request_id}] Image: {compressor.last_result.technique.value} "
+ f"({compressor.last_result.savings_percent:.0f}% saved, "
+ f"{compressor.last_result.original_tokens} → "
+ f"{compressor.last_result.compressed_tokens} tokens)"
+ )
headers = dict(request.headers.items())
headers.pop("host", None)
diff --git a/tests/test_image_compression.py b/tests/test_image_compression.py
new file mode 100644
index 000000000..240b8c0d9
--- /dev/null
+++ b/tests/test_image_compression.py
@@ -0,0 +1,307 @@
+"""Tests for image token compression pipeline.
+
+Tests tile-boundary optimization, ONNX technique routing,
+and the full compression pipeline across providers.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+
+import pytest
+
+# Tile optimizer is pure math — always available
+from headroom.image.tile_optimizer import (
+ estimate_anthropic_tokens,
+ estimate_openai_tokens,
+ find_optimal_anthropic_dimensions,
+ find_optimal_openai_dimensions,
+ optimize_images_in_messages,
+)
+
+# ---------------------------------------------------------------------------
+# Token estimation tests
+# ---------------------------------------------------------------------------
+
+
+class TestTokenEstimation:
+ def test_openai_low_detail(self):
+ assert estimate_openai_tokens(1920, 1080, "low") == 85
+
+ def test_openai_high_detail_single_tile(self):
+ assert estimate_openai_tokens(512, 512) == 85 + 170 # 1 tile
+
+ def test_openai_high_detail_multiple_tiles(self):
+ # 768x768 → ceil(768/512) * ceil(768/512) = 2*2 = 4 tiles
+ tokens = estimate_openai_tokens(768, 768)
+ assert tokens == 85 + 170 * 4 # 765
+
+ def test_openai_scales_large_images(self):
+ # 4000x3000 → scaled to fit 2048 then shortest to 768
+ # Tokens should be finite and reasonable
+ tokens = estimate_openai_tokens(4000, 3000)
+ assert 200 < tokens < 2000
+
+ def test_anthropic_formula(self):
+ # (1024 * 768) / 750 = 1048
+ tokens = estimate_anthropic_tokens(1024, 768)
+ assert tokens == (1024 * 768) // 750
+
+ def test_anthropic_caps_at_1568(self):
+ # 3000x2000 → scaled to 1568 max edge
+ tokens = estimate_anthropic_tokens(3000, 2000)
+ # After scaling: 1568 * 1045 → tokens = (1568*1045)//750
+ assert tokens < 2200 # Capped
+
+ def test_anthropic_caps_at_1_15mp(self):
+ # 1568x1568 = 2.46MP > 1.15MP → further scaled
+ tokens = estimate_anthropic_tokens(1568, 1568)
+ assert tokens <= 1534 # 1.15M / 750
+
+
+# ---------------------------------------------------------------------------
+# Tile optimization tests
+# ---------------------------------------------------------------------------
+
+
+class TestTileOptimization:
+ def test_full_hd_saves_tokens(self):
+ """1920x1080 → should reduce tile count."""
+ opt_w, opt_h = find_optimal_openai_dimensions(1920, 1080)
+ before = estimate_openai_tokens(1920, 1080)
+ after = estimate_openai_tokens(opt_w, opt_h)
+ assert after < before
+ assert before - after >= 340 # Significant savings
+
+ def test_already_optimal_no_change(self):
+ """512x512 is already on tile boundary."""
+ opt_w, opt_h = find_optimal_openai_dimensions(512, 512)
+ assert (opt_w, opt_h) == (512, 512)
+
+ def test_just_over_boundary(self):
+ """770x770 → should snap to 512x512."""
+ opt_w, opt_h = find_optimal_openai_dimensions(770, 770)
+ before = estimate_openai_tokens(770, 770)
+ after = estimate_openai_tokens(opt_w, opt_h)
+ assert after < before
+ assert after == 255 # 1 tile
+
+ def test_anthropic_caps_oversized(self):
+ """3000x2000 → capped to 1568 max edge."""
+ opt_w, opt_h = find_optimal_anthropic_dimensions(3000, 2000)
+ assert max(opt_w, opt_h) <= 1568
+
+ def test_anthropic_no_change_if_small(self):
+ """800x600 → no change needed."""
+ opt_w, opt_h = find_optimal_anthropic_dimensions(800, 600)
+ assert (opt_w, opt_h) == (800, 600)
+
+
+# ---------------------------------------------------------------------------
+# Message-level optimization tests
+# ---------------------------------------------------------------------------
+
+
+def _make_openai_image_message(width: int, height: int) -> list[dict]:
+ """Create an OpenAI-format message with a test image."""
+ from PIL import Image
+
+ img = Image.new("RGB", (width, height), "white")
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ b64 = base64.b64encode(buf.getvalue()).decode()
+
+ return [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What is this?"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{b64}"},
+ },
+ ],
+ }
+ ]
+
+
+def _make_anthropic_image_message(width: int, height: int) -> list[dict]:
+ """Create an Anthropic-format message with a test image."""
+ from PIL import Image
+
+ img = Image.new("RGB", (width, height), "white")
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ b64 = base64.b64encode(buf.getvalue()).decode()
+
+ return [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What is this?"},
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": b64,
+ },
+ },
+ ],
+ }
+ ]
+
+
+class TestMessageOptimization:
+ def test_openai_message_optimized(self):
+ """OpenAI message with large image gets tile-optimized."""
+ msgs = _make_openai_image_message(1920, 1080)
+ optimized, results = optimize_images_in_messages(msgs, "openai")
+ assert len(results) == 1
+ assert results[0].tokens_saved > 0
+ assert results[0].resized
+
+ def test_anthropic_oversized_no_token_change(self):
+ """Anthropic oversized image: provider would resize anyway, so no token savings.
+
+ Anthropic's formula is (w*h)/750 after their internal resize. Pre-resizing
+ to their limits doesn't change the token count — it only saves upload bandwidth.
+ The optimizer correctly returns no results (no token savings to report).
+ """
+ msgs = _make_anthropic_image_message(3000, 2000)
+ optimized, results = optimize_images_in_messages(msgs, "anthropic")
+ # No token savings — Anthropic would resize internally anyway
+ assert len(results) == 0
+
+ def test_no_image_no_change(self):
+ """Message without images passes through unchanged."""
+ msgs = [{"role": "user", "content": "Hello"}]
+ optimized, results = optimize_images_in_messages(msgs, "openai")
+ assert len(results) == 0
+ assert optimized == msgs
+
+ def test_text_content_preserved(self):
+ """Text content alongside image is preserved."""
+ msgs = _make_openai_image_message(1920, 1080)
+ optimized, results = optimize_images_in_messages(msgs, "openai")
+ text_blocks = [
+ b for b in optimized[0]["content"] if isinstance(b, dict) and b.get("type") == "text"
+ ]
+ assert len(text_blocks) == 1
+ assert text_blocks[0]["text"] == "What is this?"
+
+ def test_small_image_not_resized(self):
+ """Image already at optimal size is not changed."""
+ msgs = _make_openai_image_message(512, 512)
+ optimized, results = optimize_images_in_messages(msgs, "openai")
+ assert len(results) == 0 # No optimization needed
+
+
+# ---------------------------------------------------------------------------
+# ONNX Router tests (if available)
+# ---------------------------------------------------------------------------
+
+
+class TestOnnxRouter:
+ @pytest.fixture(autouse=True)
+ def _check_onnx(self):
+ try:
+ import onnxruntime # noqa: F401
+ from tokenizers import Tokenizer # noqa: F401
+ except ImportError:
+ pytest.skip("onnxruntime or tokenizers not installed")
+
+ def test_query_classification(self):
+ """ONNX router classifies queries into techniques."""
+ from headroom.image.onnx_router import OnnxTechniqueRouter, Technique
+
+ router = OnnxTechniqueRouter(use_siglip=False)
+
+ tech, conf = router.classify_query("What does the error message say?")
+ assert tech == Technique.TRANSCODE
+ assert conf > 0.5
+
+ tech, conf = router.classify_query("What's in the top left corner?")
+ assert tech == Technique.CROP
+ assert conf > 0.5
+
+ def test_preserve_for_detail_queries(self):
+ """Queries needing detail should route to PRESERVE or FULL_LOW."""
+ from headroom.image.onnx_router import OnnxTechniqueRouter, Technique
+
+ router = OnnxTechniqueRouter(use_siglip=False)
+
+ tech, _ = router.classify_query("Count every item in this image carefully")
+ assert tech in (Technique.PRESERVE, Technique.FULL_LOW)
+
+ def test_full_classify_with_image(self):
+ """Full classification with query + image analysis."""
+ from headroom.image.onnx_router import OnnxTechniqueRouter
+
+ router = OnnxTechniqueRouter(use_siglip=True)
+
+ # Create a simple test image
+ from PIL import Image
+
+ img = Image.new("RGB", (224, 224), "white")
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+
+ decision = router.classify(buf.getvalue(), "Read the text")
+ assert decision.technique is not None
+ assert decision.confidence > 0
+ assert decision.image_signals is not None
+
+
+# ---------------------------------------------------------------------------
+# Full pipeline test
+# ---------------------------------------------------------------------------
+
+
+class TestFullPipeline:
+ def test_compressor_with_openai_image(self):
+ """Full compressor pipeline on OpenAI format."""
+ from headroom.image import ImageCompressor
+
+ compressor = ImageCompressor(use_siglip=False)
+ msgs = _make_openai_image_message(1920, 1080)
+
+ result = compressor.compress(msgs, provider="openai")
+ # Should have processed the image (tile opt at minimum)
+ assert result is not None
+ assert len(result) == 1
+
+ def test_compressor_no_images(self):
+ """Compressor is no-op when no images present."""
+ from headroom.image import ImageCompressor
+
+ compressor = ImageCompressor(use_siglip=False)
+ msgs = [{"role": "user", "content": "Hello, no images here"}]
+
+ result = compressor.compress(msgs, provider="openai")
+ assert result == msgs
+
+ def test_has_images_openai(self):
+ """Detects images in OpenAI format."""
+ from headroom.image import ImageCompressor
+
+ compressor = ImageCompressor()
+ msgs = _make_openai_image_message(100, 100)
+ assert compressor.has_images(msgs)
+
+ def test_has_images_anthropic(self):
+ """Detects images in Anthropic format."""
+ from headroom.image import ImageCompressor
+
+ compressor = ImageCompressor()
+ msgs = _make_anthropic_image_message(100, 100)
+ assert compressor.has_images(msgs)
+
+ def test_no_images_detected(self):
+ """No false positives on text-only messages."""
+ from headroom.image import ImageCompressor
+
+ compressor = ImageCompressor()
+ msgs = [{"role": "user", "content": "Just text"}]
+ assert not compressor.has_images(msgs)