Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(compression/router): reuse apply()'s detection in compress(), not re-detect (#3343)
Abhay Singhedcdf660a7d3
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 70002c29f..71f7f6072 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -2161,11 +2161,17 @@ class ContentRouter(Transform):logger.debug("TOIN recording failed (non-fatal): %s", e)def _timed_compress(- self, content: str, context: str, bias: float+ self,+ content: str,+ context: str,+ bias: float,+ precomputed_detection: DetectionResult | None = None,) -> tuple[RouterCompressionResult, float]:"""Compress with wall-clock timing. Used by parallel executor."""t0 = time.perf_counter()- result = self.compress(content, context=context, bias=bias)+ result = self.compress(+ content, context=context, bias=bias, precomputed_detection=precomputed_detection+ )return result, (time.perf_counter() - t0) * 1000def compress(@@ -2174,6 +2180,7 @@ class ContentRouter(Transform):context: str = "",question: str | None = None,bias: float = 1.0,+ precomputed_detection: DetectionResult | None = None,) -> RouterCompressionResult:"""Compress content using optimal strategy based on content detection.@@ -2183,6 +2190,12 @@ class ContentRouter(Transform):question: Optional question for QA-aware compression. When provided,tokens relevant to answering this question are preserved.bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).+ precomputed_detection: A ``_detect_content(content)`` result the+ caller already computed for the same content. When supplied, it+ is reused instead of re-running the native detection chain — the+ router's hottest per-message cost. ``apply`` computes detection+ once per message (for its code-protection checks) and passes it+ here so a cache-miss message is not detected twice.Returns:RouterCompressionResult with compressed content and routing metadata.@@ -2230,7 +2243,11 @@ class ContentRouter(Transform):strategy = CompressionStrategy.KOMPRESSelse:mixed = is_mixed_content(content)- detection = _detect_content(content)+ detection = (+ precomputed_detection+ if precomputed_detection is not None+ else _detect_content(content)+ )strategy = self._determine_strategy(content, mixed=mixed, detection=detection)if debug_enabled:_log_router_debug(@@ -5079,8 +5096,11 @@ class ContentRouter(Transform):if idle_f is not None and math.isfinite(idle_f) and idle_f >= 0.0:netcost_p_alive_override = max(0.0, 1.0 - idle_f / netcost_ttl)- # Tasks: list of (slot_index, content, context, bias, content_key)- _PendingTask = tuple[int, str, str, float, int, bool]+ # Tasks: list of (slot_index, content, context, bias, content_key,+ # enforce_reversibility, precomputed_detection). The detection is the one+ # already computed in the routing loop below, threaded through so a+ # cache-miss message is not re-detected inside compress().+ _PendingTask = tuple[int, str, str, float, int, bool, "DetectionResult | None"]pending_tasks: list[_PendingTask] = []# #856 P2b (flag-gated, default off): net-cost frozen-floor unlock.@@ -5415,11 +5435,23 @@ class ContentRouter(Transform):route_counts["cache_hit"] += 1continue- # Cache miss — defer to parallel compression pass+ # Cache miss — defer to parallel compression pass. Carry the+ # detection already computed above so compress() need not re-run the+ # native detection chain on this same content. Only the full+ # detection is reused; under force_kompress the loop used the+ # lightweight regex detector and compress() skips detection entirely.route_counts.setdefault("cache_miss", 0)route_counts["cache_miss"] += 1pending_tasks.append(- (i, content, context, msg_bias, content_key, enforce_reversibility)+ (+ i,+ content,+ context,+ msg_bias,+ content_key,+ enforce_reversibility,+ None if force_kompress else detection,+ ))# --- Pass 2: Parallel compression of all cache-miss messages ---@@ -5432,7 +5464,7 @@ class ContentRouter(Transform):if max_workers <= 1 or len(pending_tasks) == 1:# Single task or parallelism disabled — compress inlinetask_results = []- for _, task_content, task_ctx, task_bias, _, _ in pending_tasks:+ for _, task_content, task_ctx, task_bias, _, _, task_detection in pending_tasks:t0 = time.perf_counter()deadline_s = _compression_deadline_seconds() if len(pending_tasks) == 1 else 0.0if deadline_s:@@ -5443,10 +5475,14 @@ class ContentRouter(Transform):_content: str = task_content,_context: str = task_ctx,_bias: float = task_bias,+ _detection: DetectionResult | None = task_detection,) -> None:try:_box["result"] = self.compress(- _content, context=_context, bias=_bias+ _content,+ context=_context,+ bias=_bias,+ precomputed_detection=_detection,)except BaseException as exc: # noqa: BLE001_box["error"] = exc@@ -5473,16 +5509,27 @@ class ContentRouter(Transform):else:r = box["result"]else:- r = self.compress(task_content, context=task_ctx, bias=task_bias)+ r = self.compress(+ task_content,+ context=task_ctx,+ bias=task_bias,+ precomputed_detection=task_detection,+ )compress_ms = (time.perf_counter() - t0) * 1000task_results.append((r, compress_ms))else:# Parallel compression via thread poolwith ThreadPoolExecutor(max_workers=max_workers) as executor:futures = []- for _, task_content, task_ctx, task_bias, _, _ in pending_tasks:+ for _, task_content, task_ctx, task_bias, _, _, task_detection in pending_tasks:futures.append(- executor.submit(self._timed_compress, task_content, task_ctx, task_bias)+ executor.submit(+ self._timed_compress,+ task_content,+ task_ctx,+ task_bias,+ task_detection,+ ))task_results = [f.result() for f in futures]@@ -5490,7 +5537,7 @@ class ContentRouter(Transform):compressor_timing["parallel_compress_total"] = parallel_ms# --- Pass 3: Merge results back (sequential, updates caches) ---- for (slot_idx, task_content, _, _, content_key, enforce_rev), (+ for (slot_idx, task_content, _, _, content_key, enforce_rev, _), (result,compress_ms,) in zip(pending_tasks, task_results):diff --git a/tests/test_content_router_detection_dedup.py b/tests/test_content_router_detection_dedup.pyindex 72ff51f05..f1b291043 100644--- a/tests/test_content_router_detection_dedup.py+++ b/tests/test_content_router_detection_dedup.py@@ -6,6 +6,12 @@ identical content: once for (default-off) debug logging, then again insideper-message cost, so ``compress`` now runs it once and threads the result into``_determine_strategy``. These tests pin the call count at one and prove thethreaded result routes identically to the recomputed one.++The same content was *also* detected a second time one level up: ``apply``'s+routing loop runs ``_detect_content`` on every message for its code-protection+checks, then ``compress`` (called in the cache-miss pass) detected the identical+bytes again. ``apply`` now threads its detection into ``compress`` via+``precomputed_detection`` so a cache-miss message is detected once, not twice."""from __future__ import annotations@@ -94,3 +100,67 @@ def test_determine_strategy_threaded_matches_recomputed(threaded = router._determine_strategy("payload", mixed=mixed, detection=detection)assert threaded is recomputed+++def test_apply_runs_detection_once_per_cache_miss_message(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # apply()'s routing loop detects each message once (for code protection),+ # then the cache-miss pass calls compress() on the same bytes. Before the+ # thread-through that was 2 native detections per cache-miss message; now+ # apply hands its detection to compress, so it must be exactly 1.+ from headroom.providers import OpenAIProvider+ from headroom.tokenizer import Tokenizer++ router = ContentRouter(+ ContentRouterConfig(min_section_tokens=10, prefer_code_aware_for_code=False)+ )+ calls = _count_detects(monkeypatch, DetectionResult(ContentType.PLAIN_TEXT, 1.0, {}))+ monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)+ # Isolate detection from the downstream compressor.+ monkeypatch.setattr(+ router,+ "_compress_pure",+ lambda *a, **k: RouterCompressionResult(+ compressed="x", original="x", strategy_used=CompressionStrategy.TEXT+ ),+ )+ tokenizer = Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")++ big = "a log line with plenty of tokens to clear the min threshold\n" * 200+ messages = [+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "get_logs", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "content": big},+ ]++ router.apply(messages, tokenizer)++ assert calls["n"] == 1+++def test_compress_precomputed_detection_matches_recomputed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Passing a precomputed detection into compress() must produce the same+ # result as letting it detect internally — the reuse changes cost, not output.+ router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))+ detection = DetectionResult(ContentType.PLAIN_TEXT, 1.0, {})+ monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)+ monkeypatch.setattr(content_router_module, "_detect_content", lambda content: detection)++ payload = "a representative plain-text blob that is comfortably non-empty " * 20+ recomputed = router.compress(payload)+ threaded = router.compress(payload, precomputed_detection=detection)++ assert threaded.strategy_used == recomputed.strategy_used+ assert threaded.compressed == recomputed.compresseddiff --git a/tests/test_content_router_single_item_deadline.py b/tests/test_content_router_single_item_deadline.pyindex 916deb270..82071ab48 100644--- a/tests/test_content_router_single_item_deadline.py+++ b/tests/test_content_router_single_item_deadline.py@@ -58,7 +58,7 @@ def _messages() -> list[dict[str, str]]:def test_single_cache_miss_fails_open_at_deadline(monkeypatch, caplog):router = _router()- def slow_compress(content, *, context="", bias=1.0):+ def slow_compress(content, *, context="", bias=1.0, precomputed_detection=None):time.sleep(0.2)return _compression_result(content, "compressed output")@@ -83,7 +83,9 @@ def test_single_cache_miss_preserves_under_deadline_output(monkeypatch):monkeypatch.setattr(router,"compress",- lambda content, *, context="", bias=1.0: _compression_result(content, "compressed output"),+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(+ content, "compressed output"+ ),)monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "1000")@@ -102,7 +104,9 @@ def test_single_cache_miss_preserves_disabled_deadline(monkeypatch):monkeypatch.setattr(router,"compress",- lambda content, *, context="", bias=1.0: _compression_result(content, "compressed output"),+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(+ content, "compressed output"+ ),)monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "0")@@ -160,7 +164,7 @@ def test_single_cache_miss_deadline_starts_before_kompress_load(monkeypatch, capmonkeypatch.setattr(router,"compress",- lambda content, *, context="", bias=1.0: _compression_result(+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(content,compressor.compress(content).compressed,),diff --git a/tests/test_dataset_recall_runner.py b/tests/test_dataset_recall_runner.pyindex 3b205b762..6ef8675e6 100644--- a/tests/test_dataset_recall_runner.py+++ b/tests/test_dataset_recall_runner.py@@ -72,7 +72,7 @@ def test_dataset_recall_records_compression_errors(monkeypatch) -> None:# as failed, the error is recorded, and the detail row carries it.from headroom.transforms.content_router import ContentRouter- def _boom(self, content, context="", question=None, bias=1.0):+ def _boom(self, content, context="", question=None, bias=1.0, precomputed_detection=None):raise RuntimeError("router exploded")monkeypatch.setattr(ContentRouter, "compress", _boom)diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 242b49220..ac298a3d6 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -441,7 +441,7 @@ def test_force_kompress_apply_uses_lightweight_detection(monkeypatch.setattr(router,"compress",- lambda content, context="", bias=1.0: RouterCompressionResult(+ lambda content, context="", bias=1.0, precomputed_detection=None: RouterCompressionResult(# CCR marker -> the original was stored and is retrievable, so the# #1307 reversibility gate accepts this lossy KOMPRESS tool result.compressed="compressed <<ccr:tool>>",@@ -923,7 +923,7 @@ def _make_router_with_mock_compress(monkeypatch: pytest.MonkeyPatch) -> ContentR``[compressed]`` payload at ratio 0.5 (passes the < min_ratio check)."""router = ContentRouter(ContentRouterConfig())- def fake_compress(content, context: str = "", bias: float = 1.0):+ def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):return SimpleNamespace(compressed=content[: len(content) // 2] + "[compressed]",compression_ratio=0.5,@@ -1237,7 +1237,7 @@ def _churn_router(monkeypatch: pytest.MonkeyPatch, ratio: float) -> ContentRoutecfg = ContentRouterConfig(min_ratio_relaxed=0.85, min_ratio_aggressive=0.65)router = ContentRouter(cfg)- def fake_compress(content, context: str = "", bias: float = 1.0):+ def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):return SimpleNamespace(# CCR marker keeps the compression recoverable so the #1307# tool-reversibility guard preserves it instead of restoring original.@@ -1376,7 +1376,7 @@ def test_model_not_ready_passthrough_is_not_frozen(monkeypatch.setattr(router,"compress",- lambda c, context="", bias=1.0: SimpleNamespace(+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT),)@@ -1402,7 +1402,7 @@ def test_model_not_ready_passthrough_is_not_frozen(monkeypatch.setattr(router,"compress",- lambda c, context="", bias=1.0: SimpleNamespace(+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(compressed="[C]"+ c[:20]+ " <<ccr:t>>", # recoverable marker so #1307 tool-reversibility guard keeps the compression@@ -1515,7 +1515,7 @@ def test_block_model_not_ready_passthrough_is_not_frozen(monkeypatch.setattr(router,"compress",- lambda c, context="", bias=1.0: SimpleNamespace(+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT),)@@ -1530,7 +1530,7 @@ def test_block_model_not_ready_passthrough_is_not_frozen(monkeypatch.setattr(router,"compress",- lambda c, context="", bias=1.0: SimpleNamespace(+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(compressed="[C]"+ c[:20]+ " <<ccr:t>>", # recoverable marker so #1307 tool-reversibility guard keeps the compression