Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(router): compact JSON evades compression via whitespace token counting (#1857)
Dima Solodukha4e30dde2aca8
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 6bb34dc83..3ca85b0c0 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -58,6 +58,7 @@ from ..config import ()from ..parser import CCR_RETRIEVAL_MARKER_REfrom ..tokenizer import Tokenizer+from ..tokenizers.estimator import EstimatingTokenCounterfrom . import mixed_content as _mixed_contentfrom .base import Transformfrom .content_detector import ContentType, DetectionResult, _try_detect_log, _try_detect_search@@ -78,6 +79,26 @@ _detect_panic_warned = False_detect_native_unhealthy = False # circuit breaker: native detect hung once (#575)+# Shared calibrated fallback estimator (tiktoken cl100k_base ~90% accuracy,+# content-type aware incl. JSON). Kept as one module-level instance so the+# size heuristic lives in a single reusable place, not a hardcoded constant.+_TOKEN_ESTIMATOR = EstimatingTokenCounter()+++def _estimate_tokens(text: str) -> int:+ """Size-proportional token estimate for section ratio decisions.++ Whitespace splitting undercounts pathologically on compact+ machine-generated JSON (no spaces means the whole payload is ~1+ "word"), which made section compression ratios compute as ~1.0 and+ the min_ratio acceptance gate silently reject real compression.+ Delegates to the shared EstimatingTokenCounter — JSON/code aware and+ calibrated against tiktoken — so the estimate tracks real tokens across+ formats instead of a fixed chars/N constant.+ """+ return max(1, _TOKEN_ESTIMATOR.count_text(text))++def _compression_deadline_seconds() -> float:try:return max(@@ -346,7 +367,7 @@ def _section_debug(section: ContentSection, index: int) -> dict[str, Any]:"is_code_fence": getattr(section, "is_code_fence", False),"chars": len(section.content),"bytes": len(section.content.encode("utf-8", errors="replace")),- "tokens_estimate": len(section.content.split()),+ "tokens_estimate": _estimate_tokens(section.content),"json_shape": _json_shape(section.content),"content": section.content,}@@ -1485,7 +1506,7 @@ class ContentRouter(Transform):{"chars": len(content),"bytes": len(content.encode("utf-8", errors="replace")),- "tokens_estimate": len(content.split()),+ "tokens_estimate": _estimate_tokens(content),"json_shape": _json_shape(content),"mixed_indicators": _mixed_indicators(content),"context_chars": len(context),@@ -1704,7 +1725,7 @@ class ContentRouter(Transform):strategy = self._strategy_from_detection_type(section.content_type)# Compress section- original_tokens = len(section.content.split())+ original_tokens = _estimate_tokens(section.content)compressed_content, compressed_tokens, _section_chain = self._apply_strategy_to_content(section.content,strategy,@@ -1757,7 +1778,7 @@ class ContentRouter(Transform):Returns:RouterCompressionResult."""- original_tokens = len(content.split())+ original_tokens = _estimate_tokens(content)compressed, compressed_tokens, strategy_chain = self._apply_strategy_to_content(content, strategy, context, question=question, bias=bias@@ -1891,7 +1912,7 @@ class ContentRouter(Transform):final compressor without parsing decision_reason strings."""# Track original tokens for TOIN recording- original_tokens = len(content.split())+ original_tokens = _estimate_tokens(content)compressed: str | None = Nonecompressed_tokens: int | None = Nonerequested_strategy = strategy@@ -1920,7 +1941,7 @@ class ContentRouter(Transform):# never a marker-free lossy drop that could not be recovered.if self.config.lossless:if _ll_label is not None:- return _ll_content, len(_ll_content.split()), [_ll_label]+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label]return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value]# ── LOSSY / CCR mode: layer relevance-split + lossy ON TOP of the fold ─@@ -1944,7 +1965,7 @@ class ContentRouter(Transform):kind = "log" if strategy is CompressionStrategy.LOG else "search"split = self._relevance_split_compress(content, kind, context)if split is not None:- return split, len(split.split()), [kind, "relevance_split"]+ return split, _estimate_tokens(split), [kind, "relevance_split"]# No relevance split adopted → return the STAGE 0 lossless fold as the# floor. Lossless-then-lossy: before returning, run the aggressive lossy@@ -1961,7 +1982,7 @@ class ContentRouter(Transform):and not self._looks_like_diff(content))if _lossy_after_fold:- _fold_tokens = len(_ll_content.split())+ _fold_tokens = _estimate_tokens(_ll_content)try:_komp, _komp_tokens = self._try_ml_compressor(_ll_content, context, question)except Exception as exc: # noqa: BLE001@@ -1978,7 +1999,7 @@ class ContentRouter(Transform):_komp_tokens,[_ll_label, CompressionStrategy.KOMPRESS.value],)- return _ll_content, len(_ll_content.split()), [_ll_label]+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label]# CCR/lossy mode, nothing foldable (code/json/text/mixed) and no relevance# split → fall through to the lossy compressors below (kompress /@@ -2041,7 +2062,7 @@ class ContentRouter(Transform):result = crusher.crush(content, query=context, bias=bias)compressed, compressed_tokens = (result.compressed,- len(result.compressed.split()),+ _estimate_tokens(result.compressed),)decision_reason = "smart_crusher"# Fallback to Kompress (and possibly Log) is@@ -2057,7 +2078,7 @@ class ContentRouter(Transform):result = compressor.compress(content, context=context, bias=bias)compressed, compressed_tokens = (result.compressed,- len(result.compressed.split()),+ _estimate_tokens(result.compressed),)decision_reason = "search_compressor"@@ -2073,7 +2094,7 @@ class ContentRouter(Transform):# ratios meaningless against `original_tokens`.compressed, compressed_tokens = (result.compressed,- len(result.compressed.split()),+ _estimate_tokens(result.compressed),)decision_reason = "log_compressor"@@ -2085,7 +2106,7 @@ class ContentRouter(Transform):result = compressor.compress(content, context=context, bias=bias)compressed, compressed_tokens = (result.compressed,- len(result.compressed.split()),+ _estimate_tokens(result.compressed),)decision_reason = "tabular_compressor"@@ -2096,7 +2117,7 @@ class ContentRouter(Transform):result = compressor.compress(content, context=context)compressed, compressed_tokens = (result.compressed,- len(result.compressed.split()),+ _estimate_tokens(result.compressed),)decision_reason = "diff_compressor"@@ -2108,7 +2129,7 @@ class ContentRouter(Transform):result = extractor.extract(content)compressed = result.extracted# Estimate tokens from extracted text (simple word count)- compressed_tokens = len(compressed.split()) if compressed else 0+ compressed_tokens = _estimate_tokens(compressed) if compressed else 0decision_reason = "html_extractor"elif strategy == CompressionStrategy.KOMPRESS:@@ -2180,7 +2201,7 @@ class ContentRouter(Transform):except Exception as exc: # noqa: BLE001logger.debug("Log fallback failed for SMART_CRUSHER: %s", exc)else:- log_compressed_tokens = len(log_result.compressed.split())+ log_compressed_tokens = _estimate_tokens(log_result.compressed)if log_compressed_tokens < compressed_tokens:compressed = log_result.compressedcompressed_tokens = log_compressed_tokens@@ -2328,7 +2349,7 @@ class ContentRouter(Transform):# If the entire content is custom tags with nothing to compressif protected and not cleaned.strip():- return content, len(content.split())+ return content, _estimate_tokens(content)# Use the cleaned (tag-free) text for compressiontext_to_compress = cleaned if protected else content@@ -2373,7 +2394,7 @@ class ContentRouter(Transform):out = text_to_compressif protected:out = restore_tags(out, protected)- return out, len(out.split())+ return out, _estimate_tokens(out)# Primary: Kompress. On a cold cache the model is fetched once in the# background (ensure_background_load) instead of blocking this request@@ -2414,14 +2435,14 @@ class ContentRouter(Transform):logger.warning("Kompress failed: %s", e)if compressed is None:- return content, len(content.split())+ return content, _estimate_tokens(content)# Restore protected tag blocks into the compressed textif protected:compressed = restore_tags(compressed, protected)- compressed_tokens = len(compressed.split())+ compressed_tokens = _estimate_tokens(compressed)- return compressed, compressed_tokens or len(compressed.split())+ return compressed, compressed_tokens or _estimate_tokens(compressed)def _experimental_compress_read(self, content: Any, context: str = "") -> str | None:"""EXPERIMENT (HEADROOM_EXPERIMENTAL_READ_KEEP_RATIO): lightly Kompress adiff --git a/tests/test_content_router_compact_json.py b/tests/test_content_router_compact_json.pynew file mode 100644index 000000000..f1b4cbe80--- /dev/null+++ b/tests/test_content_router_compact_json.py@@ -0,0 +1,71 @@+"""Compact machine-generated JSON must not evade compression (token-estimate bug).++Whitespace-split token counting made a compact JSON payload (no spaces —+the default output of json.dumps with separators, JSON.stringify, boto3)+count as ~1 token, so compression ratios computed as ~1.0 and the+min_ratio gate rejected SmartCrusher's real output.+"""++from __future__ import annotations++import json+import random++import pytest++from headroom.transforms.content_router import (+ ContentRouter,+ ContentRouterConfig,+ _estimate_tokens,+)+++def test_estimate_tokens_monotone_on_compact_json() -> None:+ small = json.dumps([{"a": 1}] * 5, separators=(",", ":"))+ large = json.dumps([{"a": 1}] * 500, separators=(",", ":"))+ assert _estimate_tokens(large) > _estimate_tokens(small) > 1+++def test_compact_json_tool_result_compresses() -> None:+ tokenizer = pytest.importorskip("headroom.tokenizers.estimator")+ from headroom.tokenizer import Tokenizer++ tok = Tokenizer(tokenizer.EstimatingTokenCounter())+ random.seed(42)+ rows = [+ {+ "serviceArn": f"arn:aws:ecs:us-east-1:123456789012:service/x/svc-{s:03d}",+ "serviceName": f"svc-{s:03d}",+ "status": "ACTIVE",+ "desiredCount": random.randint(1, 6),+ "runningCount": random.randint(0, 6),+ }+ for s in range(150)+ ]+ payload = json.dumps(rows, separators=(",", ":"))+ assert " " not in payload[:200] # genuinely compact+ messages = [+ {"role": "user", "content": "Investigate."},+ {+ "role": "assistant",+ "content": [+ {"type": "text", "text": "Checking."},+ {"type": "tool_use", "id": "toolu_1", "name": "list_services", "input": {}},+ ],+ },+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": payload}],+ },+ {"role": "user", "content": "Summarize in one sentence."},+ ]+ router = ContentRouter(ContentRouterConfig(skip_user_messages=False))+ before = tok.count_messages(messages)+ result = router.apply(+ [json.loads(json.dumps(m)) for m in messages],+ tok,+ context="Summarize",+ frozen_message_count=0,+ )+ after = tok.count_messages(result.messages)+ assert after < before * 0.9, (before, after, result.transforms_applied[:5])diff --git a/tests/test_lossless_then_lossy.py b/tests/test_lossless_then_lossy.pyindex 19c7276ce..6dadeefc0 100644--- a/tests/test_lossless_then_lossy.py+++ b/tests/test_lossless_then_lossy.py@@ -15,6 +15,7 @@ from headroom.transforms.content_router import (CompressionStrategy,ContentRouter,ContentRouterConfig,+ _estimate_tokens,)from headroom.transforms.lossless_compaction import search_unheading@@ -148,7 +149,10 @@ def test_lossy_gate_boundary_default(monkeypatch):block = _grep_block()r, _ = _router(lossless_then_lossy=True)assert abs(r._lossy_min_extra_savings - 0.05) < 1e-9 # default: require >=5% extra- fold_tok = len(r._lossless_first(block, CompressionStrategy.SEARCH)[0].split())+ # Measure the fold with the SAME estimator the router's gate uses+ # (_estimate_tokens), not len(split()) — the two diverge, and the gate at+ # content_router.py compares _komp_tokens against _estimate_tokens(fold).+ fold_tok = _estimate_tokens(r._lossless_first(block, CompressionStrategy.SEARCH)[0])# Kompress that keeps 94% of fold tokens -> saves 6% >= the 5% floor -> chained.keep = int(fold_tok * 0.94)r._try_ml_compressor = lambda c, ctx, q=None: (" ".join(["w"] * keep), keep) # type: ignore@@ -161,7 +165,8 @@ def test_lossy_gate_env_override(monkeypatch):r, _ = _router(lossless_then_lossy=True)assert abs(r._lossy_min_extra_savings - 0.20) < 1e-9 # env override winsblock = _grep_block()- fold_tok = len(r._lossless_first(block, CompressionStrategy.SEARCH)[0].split())+ # Same estimator as the router gate (_estimate_tokens), not len(split()).+ fold_tok = _estimate_tokens(r._lossless_first(block, CompressionStrategy.SEARCH)[0])keep = int(fold_tok * 0.90) # saves 10%: passes the 5% default but FAILS the 20% overrider._try_ml_compressor = lambda c, ctx, q=None: (" ".join(["w"] * keep), keep) # type: ignore_, was, tr, rc = _run(r, block)diff --git a/tests/test_transforms/test_text_crusher_routing.py b/tests/test_transforms/test_text_crusher_routing.pyindex e3da83e7c..6da9891ec 100644--- a/tests/test_transforms/test_text_crusher_routing.py+++ b/tests/test_transforms/test_text_crusher_routing.py@@ -4,7 +4,7 @@ LogCompressor (which yields ~0 on prose) or ModernBERT (slow)."""from __future__ import annotations-from headroom.transforms.content_router import ContentRouter+from headroom.transforms.content_router import ContentRouter, _estimate_tokensdef _prose() -> str:@@ -28,7 +28,10 @@ def test_gate_routes_to_text_crusher_when_enabled(monkeypatch):out, ntok = router._try_ml_compressor(prose, "authentication tokens")assert router._kompress_gate_fires == 1- assert ntok < len(prose.split()) # TextCrusher actually compressed (LogCompressor ~0)+ # TextCrusher actually compressed (LogCompressor ~0). Compare like-for-like+ # against the same token estimator the compressor reports in, not a raw+ # word count, so the assertion tracks real token reduction.+ assert ntok < _estimate_tokens(prose)assert set(out.split()) <= set(prose.split()) # extractive: no invented wordsdiff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 02498bbac..fe5bf58ec 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -15,6 +15,7 @@ from headroom.transforms.content_router import (RoutingDecision,_create_content_signature,_detect_content,+ _estimate_tokens,_extract_json_block,_strip_detection_envelope,is_mixed_content,@@ -483,7 +484,7 @@ def test_content_router_mixed_pure_apply_and_toin(monkeypatch: pytest.MonkeyPatc)pure = router._compress_pure("some plain text", CompressionStrategy.TEXT, "ctx")assert pure.routing_log[0].content_type is ContentType.PLAIN_TEXT- assert pure.total_original_tokens == 3+ assert pure.total_original_tokens == _estimate_tokens("some plain text")assert pure.total_compressed_tokens == 1calls: list[dict] = []@@ -551,7 +552,7 @@ def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop()assert compressed == diff- assert compressed_tokens == len(diff.split())+ assert compressed_tokens == _estimate_tokens(diff)assert strategy_chain == ["diff"]@@ -579,7 +580,7 @@ def test_log_strategy_does_not_fallback_to_kompress_when_log_is_noop()assert compressed == log- assert compressed_tokens == len(log.split())+ assert compressed_tokens == _estimate_tokens(log)assert strategy_chain == ["log"]