Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(compression/router): reuse apply()'s detection in compress(), not re-detect (#3343)

Abhay Singhedcdf660a7d3
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 70002c29f..71f7f6072 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -2161,11 +2161,17 @@ class ContentRouter(Transform):
logger.debug("TOIN recording failed (non-fatal): %s", e)
def _timed_compress(
- self, content: str, context: str, bias: float
+ self,
+ content: str,
+ context: str,
+ bias: float,
+ precomputed_detection: DetectionResult | None = None,
) -> tuple[RouterCompressionResult, float]:
"""Compress with wall-clock timing. Used by parallel executor."""
t0 = time.perf_counter()
- result = self.compress(content, context=context, bias=bias)
+ result = self.compress(
+ content, context=context, bias=bias, precomputed_detection=precomputed_detection
+ )
return result, (time.perf_counter() - t0) * 1000
def compress(
@@ -2174,6 +2180,7 @@ class ContentRouter(Transform):
context: str = "",
question: str | None = None,
bias: float = 1.0,
+ precomputed_detection: DetectionResult | None = None,
) -> RouterCompressionResult:
"""Compress content using optimal strategy based on content detection.
@@ -2183,6 +2190,12 @@ class ContentRouter(Transform):
question: Optional question for QA-aware compression. When provided,
tokens relevant to answering this question are preserved.
bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).
+ precomputed_detection: A ``_detect_content(content)`` result the
+ caller already computed for the same content. When supplied, it
+ is reused instead of re-running the native detection chain — the
+ router's hottest per-message cost. ``apply`` computes detection
+ once per message (for its code-protection checks) and passes it
+ here so a cache-miss message is not detected twice.
Returns:
RouterCompressionResult with compressed content and routing metadata.
@@ -2230,7 +2243,11 @@ class ContentRouter(Transform):
strategy = CompressionStrategy.KOMPRESS
else:
mixed = is_mixed_content(content)
- detection = _detect_content(content)
+ detection = (
+ precomputed_detection
+ if precomputed_detection is not None
+ else _detect_content(content)
+ )
strategy = self._determine_strategy(content, mixed=mixed, detection=detection)
if debug_enabled:
_log_router_debug(
@@ -5079,8 +5096,11 @@ class ContentRouter(Transform):
if idle_f is not None and math.isfinite(idle_f) and idle_f >= 0.0:
netcost_p_alive_override = max(0.0, 1.0 - idle_f / netcost_ttl)
- # Tasks: list of (slot_index, content, context, bias, content_key)
- _PendingTask = tuple[int, str, str, float, int, bool]
+ # Tasks: list of (slot_index, content, context, bias, content_key,
+ # enforce_reversibility, precomputed_detection). The detection is the one
+ # already computed in the routing loop below, threaded through so a
+ # cache-miss message is not re-detected inside compress().
+ _PendingTask = tuple[int, str, str, float, int, bool, "DetectionResult | None"]
pending_tasks: list[_PendingTask] = []
# #856 P2b (flag-gated, default off): net-cost frozen-floor unlock.
@@ -5415,11 +5435,23 @@ class ContentRouter(Transform):
route_counts["cache_hit"] += 1
continue
- # Cache miss — defer to parallel compression pass
+ # Cache miss — defer to parallel compression pass. Carry the
+ # detection already computed above so compress() need not re-run the
+ # native detection chain on this same content. Only the full
+ # detection is reused; under force_kompress the loop used the
+ # lightweight regex detector and compress() skips detection entirely.
route_counts.setdefault("cache_miss", 0)
route_counts["cache_miss"] += 1
pending_tasks.append(
- (i, content, context, msg_bias, content_key, enforce_reversibility)
+ (
+ i,
+ content,
+ context,
+ msg_bias,
+ content_key,
+ enforce_reversibility,
+ None if force_kompress else detection,
+ )
)
# --- Pass 2: Parallel compression of all cache-miss messages ---
@@ -5432,7 +5464,7 @@ class ContentRouter(Transform):
if max_workers <= 1 or len(pending_tasks) == 1:
# Single task or parallelism disabled — compress inline
task_results = []
- for _, task_content, task_ctx, task_bias, _, _ in pending_tasks:
+ for _, task_content, task_ctx, task_bias, _, _, task_detection in pending_tasks:
t0 = time.perf_counter()
deadline_s = _compression_deadline_seconds() if len(pending_tasks) == 1 else 0.0
if deadline_s:
@@ -5443,10 +5475,14 @@ class ContentRouter(Transform):
_content: str = task_content,
_context: str = task_ctx,
_bias: float = task_bias,
+ _detection: DetectionResult | None = task_detection,
) -> None:
try:
_box["result"] = self.compress(
- _content, context=_context, bias=_bias
+ _content,
+ context=_context,
+ bias=_bias,
+ precomputed_detection=_detection,
)
except BaseException as exc: # noqa: BLE001
_box["error"] = exc
@@ -5473,16 +5509,27 @@ class ContentRouter(Transform):
else:
r = box["result"]
else:
- r = self.compress(task_content, context=task_ctx, bias=task_bias)
+ r = self.compress(
+ task_content,
+ context=task_ctx,
+ bias=task_bias,
+ precomputed_detection=task_detection,
+ )
compress_ms = (time.perf_counter() - t0) * 1000
task_results.append((r, compress_ms))
else:
# Parallel compression via thread pool
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = []
- for _, task_content, task_ctx, task_bias, _, _ in pending_tasks:
+ for _, task_content, task_ctx, task_bias, _, _, task_detection in pending_tasks:
futures.append(
- executor.submit(self._timed_compress, task_content, task_ctx, task_bias)
+ executor.submit(
+ self._timed_compress,
+ task_content,
+ task_ctx,
+ task_bias,
+ task_detection,
+ )
)
task_results = [f.result() for f in futures]
@@ -5490,7 +5537,7 @@ class ContentRouter(Transform):
compressor_timing["parallel_compress_total"] = parallel_ms
# --- Pass 3: Merge results back (sequential, updates caches) ---
- for (slot_idx, task_content, _, _, content_key, enforce_rev), (
+ for (slot_idx, task_content, _, _, content_key, enforce_rev, _), (
result,
compress_ms,
) in zip(pending_tasks, task_results):
diff --git a/tests/test_content_router_detection_dedup.py b/tests/test_content_router_detection_dedup.py
index 72ff51f05..f1b291043 100644
--- a/tests/test_content_router_detection_dedup.py
+++ b/tests/test_content_router_detection_dedup.py
@@ -6,6 +6,12 @@ identical content: once for (default-off) debug logging, then again inside
per-message cost, so ``compress`` now runs it once and threads the result into
``_determine_strategy``. These tests pin the call count at one and prove the
threaded result routes identically to the recomputed one.
+
+The same content was *also* detected a second time one level up: ``apply``'s
+routing loop runs ``_detect_content`` on every message for its code-protection
+checks, then ``compress`` (called in the cache-miss pass) detected the identical
+bytes again. ``apply`` now threads its detection into ``compress`` via
+``precomputed_detection`` so a cache-miss message is detected once, not twice.
"""
from __future__ import annotations
@@ -94,3 +100,67 @@ def test_determine_strategy_threaded_matches_recomputed(
threaded = router._determine_strategy("payload", mixed=mixed, detection=detection)
assert threaded is recomputed
+
+
+def test_apply_runs_detection_once_per_cache_miss_message(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # apply()'s routing loop detects each message once (for code protection),
+ # then the cache-miss pass calls compress() on the same bytes. Before the
+ # thread-through that was 2 native detections per cache-miss message; now
+ # apply hands its detection to compress, so it must be exactly 1.
+ from headroom.providers import OpenAIProvider
+ from headroom.tokenizer import Tokenizer
+
+ router = ContentRouter(
+ ContentRouterConfig(min_section_tokens=10, prefer_code_aware_for_code=False)
+ )
+ calls = _count_detects(monkeypatch, DetectionResult(ContentType.PLAIN_TEXT, 1.0, {}))
+ monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
+ # Isolate detection from the downstream compressor.
+ monkeypatch.setattr(
+ router,
+ "_compress_pure",
+ lambda *a, **k: RouterCompressionResult(
+ compressed="x", original="x", strategy_used=CompressionStrategy.TEXT
+ ),
+ )
+ tokenizer = Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")
+
+ big = "a log line with plenty of tokens to clear the min threshold\n" * 200
+ messages = [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "get_logs", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": big},
+ ]
+
+ router.apply(messages, tokenizer)
+
+ assert calls["n"] == 1
+
+
+def test_compress_precomputed_detection_matches_recomputed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Passing a precomputed detection into compress() must produce the same
+ # result as letting it detect internally — the reuse changes cost, not output.
+ router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))
+ detection = DetectionResult(ContentType.PLAIN_TEXT, 1.0, {})
+ monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
+ monkeypatch.setattr(content_router_module, "_detect_content", lambda content: detection)
+
+ payload = "a representative plain-text blob that is comfortably non-empty " * 20
+ recomputed = router.compress(payload)
+ threaded = router.compress(payload, precomputed_detection=detection)
+
+ assert threaded.strategy_used == recomputed.strategy_used
+ assert threaded.compressed == recomputed.compressed
diff --git a/tests/test_content_router_single_item_deadline.py b/tests/test_content_router_single_item_deadline.py
index 916deb270..82071ab48 100644
--- a/tests/test_content_router_single_item_deadline.py
+++ b/tests/test_content_router_single_item_deadline.py
@@ -58,7 +58,7 @@ def _messages() -> list[dict[str, str]]:
def test_single_cache_miss_fails_open_at_deadline(monkeypatch, caplog):
router = _router()
- def slow_compress(content, *, context="", bias=1.0):
+ def slow_compress(content, *, context="", bias=1.0, precomputed_detection=None):
time.sleep(0.2)
return _compression_result(content, "compressed output")
@@ -83,7 +83,9 @@ def test_single_cache_miss_preserves_under_deadline_output(monkeypatch):
monkeypatch.setattr(
router,
"compress",
- lambda content, *, context="", bias=1.0: _compression_result(content, "compressed output"),
+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(
+ content, "compressed output"
+ ),
)
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "1000")
@@ -102,7 +104,9 @@ def test_single_cache_miss_preserves_disabled_deadline(monkeypatch):
monkeypatch.setattr(
router,
"compress",
- lambda content, *, context="", bias=1.0: _compression_result(content, "compressed output"),
+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(
+ content, "compressed output"
+ ),
)
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "0")
@@ -160,7 +164,7 @@ def test_single_cache_miss_deadline_starts_before_kompress_load(monkeypatch, cap
monkeypatch.setattr(
router,
"compress",
- lambda content, *, context="", bias=1.0: _compression_result(
+ lambda content, *, context="", bias=1.0, precomputed_detection=None: _compression_result(
content,
compressor.compress(content).compressed,
),
diff --git a/tests/test_dataset_recall_runner.py b/tests/test_dataset_recall_runner.py
index 3b205b762..6ef8675e6 100644
--- a/tests/test_dataset_recall_runner.py
+++ b/tests/test_dataset_recall_runner.py
@@ -72,7 +72,7 @@ def test_dataset_recall_records_compression_errors(monkeypatch) -> None:
# as failed, the error is recorded, and the detail row carries it.
from headroom.transforms.content_router import ContentRouter
- def _boom(self, content, context="", question=None, bias=1.0):
+ def _boom(self, content, context="", question=None, bias=1.0, precomputed_detection=None):
raise RuntimeError("router exploded")
monkeypatch.setattr(ContentRouter, "compress", _boom)
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 242b49220..ac298a3d6 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -441,7 +441,7 @@ def test_force_kompress_apply_uses_lightweight_detection(
monkeypatch.setattr(
router,
"compress",
- lambda content, context="", bias=1.0: RouterCompressionResult(
+ lambda content, context="", bias=1.0, precomputed_detection=None: RouterCompressionResult(
# CCR marker -> the original was stored and is retrievable, so the
# #1307 reversibility gate accepts this lossy KOMPRESS tool result.
compressed="compressed <<ccr:tool>>",
@@ -923,7 +923,7 @@ def _make_router_with_mock_compress(monkeypatch: pytest.MonkeyPatch) -> ContentR
``[compressed]`` payload at ratio 0.5 (passes the < min_ratio check)."""
router = ContentRouter(ContentRouterConfig())
- def fake_compress(content, context: str = "", bias: float = 1.0):
+ def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
return SimpleNamespace(
compressed=content[: len(content) // 2] + "[compressed]",
compression_ratio=0.5,
@@ -1237,7 +1237,7 @@ def _churn_router(monkeypatch: pytest.MonkeyPatch, ratio: float) -> ContentRoute
cfg = ContentRouterConfig(min_ratio_relaxed=0.85, min_ratio_aggressive=0.65)
router = ContentRouter(cfg)
- def fake_compress(content, context: str = "", bias: float = 1.0):
+ def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
return SimpleNamespace(
# CCR marker keeps the compression recoverable so the #1307
# tool-reversibility guard preserves it instead of restoring original.
@@ -1376,7 +1376,7 @@ def test_model_not_ready_passthrough_is_not_frozen(
monkeypatch.setattr(
router,
"compress",
- lambda c, context="", bias=1.0: SimpleNamespace(
+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT
),
)
@@ -1402,7 +1402,7 @@ def test_model_not_ready_passthrough_is_not_frozen(
monkeypatch.setattr(
router,
"compress",
- lambda c, context="", bias=1.0: SimpleNamespace(
+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
compressed="[C]"
+ c[:20]
+ " <<ccr:t>>", # recoverable marker so #1307 tool-reversibility guard keeps the compression
@@ -1515,7 +1515,7 @@ def test_block_model_not_ready_passthrough_is_not_frozen(
monkeypatch.setattr(
router,
"compress",
- lambda c, context="", bias=1.0: SimpleNamespace(
+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT
),
)
@@ -1530,7 +1530,7 @@ def test_block_model_not_ready_passthrough_is_not_frozen(
monkeypatch.setattr(
router,
"compress",
- lambda c, context="", bias=1.0: SimpleNamespace(
+ lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
compressed="[C]"
+ c[:20]
+ " <<ccr:t>>", # recoverable marker so #1307 tool-reversibility guard keeps the compression