Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(router): compact JSON evades compression via whitespace token counting (#1857)

Dima Solodukha4e30dde2aca8
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 6bb34dc83..3ca85b0c0 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -58,6 +58,7 @@ from ..config import (
)
from ..parser import CCR_RETRIEVAL_MARKER_RE
from ..tokenizer import Tokenizer
+from ..tokenizers.estimator import EstimatingTokenCounter
from . import mixed_content as _mixed_content
from .base import Transform
from .content_detector import ContentType, DetectionResult, _try_detect_log, _try_detect_search
@@ -78,6 +79,26 @@ _detect_panic_warned = False
_detect_native_unhealthy = False # circuit breaker: native detect hung once (#575)
+# Shared calibrated fallback estimator (tiktoken cl100k_base ~90% accuracy,
+# content-type aware incl. JSON). Kept as one module-level instance so the
+# size heuristic lives in a single reusable place, not a hardcoded constant.
+_TOKEN_ESTIMATOR = EstimatingTokenCounter()
+
+
+def _estimate_tokens(text: str) -> int:
+ """Size-proportional token estimate for section ratio decisions.
+
+ Whitespace splitting undercounts pathologically on compact
+ machine-generated JSON (no spaces means the whole payload is ~1
+ "word"), which made section compression ratios compute as ~1.0 and
+ the min_ratio acceptance gate silently reject real compression.
+ Delegates to the shared EstimatingTokenCounter — JSON/code aware and
+ calibrated against tiktoken — so the estimate tracks real tokens across
+ formats instead of a fixed chars/N constant.
+ """
+ return max(1, _TOKEN_ESTIMATOR.count_text(text))
+
+
def _compression_deadline_seconds() -> float:
try:
return max(
@@ -346,7 +367,7 @@ def _section_debug(section: ContentSection, index: int) -> dict[str, Any]:
"is_code_fence": getattr(section, "is_code_fence", False),
"chars": len(section.content),
"bytes": len(section.content.encode("utf-8", errors="replace")),
- "tokens_estimate": len(section.content.split()),
+ "tokens_estimate": _estimate_tokens(section.content),
"json_shape": _json_shape(section.content),
"content": section.content,
}
@@ -1485,7 +1506,7 @@ class ContentRouter(Transform):
{
"chars": len(content),
"bytes": len(content.encode("utf-8", errors="replace")),
- "tokens_estimate": len(content.split()),
+ "tokens_estimate": _estimate_tokens(content),
"json_shape": _json_shape(content),
"mixed_indicators": _mixed_indicators(content),
"context_chars": len(context),
@@ -1704,7 +1725,7 @@ class ContentRouter(Transform):
strategy = self._strategy_from_detection_type(section.content_type)
# Compress section
- original_tokens = len(section.content.split())
+ original_tokens = _estimate_tokens(section.content)
compressed_content, compressed_tokens, _section_chain = self._apply_strategy_to_content(
section.content,
strategy,
@@ -1757,7 +1778,7 @@ class ContentRouter(Transform):
Returns:
RouterCompressionResult.
"""
- original_tokens = len(content.split())
+ original_tokens = _estimate_tokens(content)
compressed, compressed_tokens, strategy_chain = self._apply_strategy_to_content(
content, strategy, context, question=question, bias=bias
@@ -1891,7 +1912,7 @@ class ContentRouter(Transform):
final compressor without parsing decision_reason strings.
"""
# Track original tokens for TOIN recording
- original_tokens = len(content.split())
+ original_tokens = _estimate_tokens(content)
compressed: str | None = None
compressed_tokens: int | None = None
requested_strategy = strategy
@@ -1920,7 +1941,7 @@ class ContentRouter(Transform):
# never a marker-free lossy drop that could not be recovered.
if self.config.lossless:
if _ll_label is not None:
- return _ll_content, len(_ll_content.split()), [_ll_label]
+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label]
return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value]
# ── LOSSY / CCR mode: layer relevance-split + lossy ON TOP of the fold ─
@@ -1944,7 +1965,7 @@ class ContentRouter(Transform):
kind = "log" if strategy is CompressionStrategy.LOG else "search"
split = self._relevance_split_compress(content, kind, context)
if split is not None:
- return split, len(split.split()), [kind, "relevance_split"]
+ return split, _estimate_tokens(split), [kind, "relevance_split"]
# No relevance split adopted → return the STAGE 0 lossless fold as the
# floor. Lossless-then-lossy: before returning, run the aggressive lossy
@@ -1961,7 +1982,7 @@ class ContentRouter(Transform):
and not self._looks_like_diff(content)
)
if _lossy_after_fold:
- _fold_tokens = len(_ll_content.split())
+ _fold_tokens = _estimate_tokens(_ll_content)
try:
_komp, _komp_tokens = self._try_ml_compressor(_ll_content, context, question)
except Exception as exc: # noqa: BLE001
@@ -1978,7 +1999,7 @@ class ContentRouter(Transform):
_komp_tokens,
[_ll_label, CompressionStrategy.KOMPRESS.value],
)
- return _ll_content, len(_ll_content.split()), [_ll_label]
+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label]
# CCR/lossy mode, nothing foldable (code/json/text/mixed) and no relevance
# split → fall through to the lossy compressors below (kompress /
@@ -2041,7 +2062,7 @@ class ContentRouter(Transform):
result = crusher.crush(content, query=context, bias=bias)
compressed, compressed_tokens = (
result.compressed,
- len(result.compressed.split()),
+ _estimate_tokens(result.compressed),
)
decision_reason = "smart_crusher"
# Fallback to Kompress (and possibly Log) is
@@ -2057,7 +2078,7 @@ class ContentRouter(Transform):
result = compressor.compress(content, context=context, bias=bias)
compressed, compressed_tokens = (
result.compressed,
- len(result.compressed.split()),
+ _estimate_tokens(result.compressed),
)
decision_reason = "search_compressor"
@@ -2073,7 +2094,7 @@ class ContentRouter(Transform):
# ratios meaningless against `original_tokens`.
compressed, compressed_tokens = (
result.compressed,
- len(result.compressed.split()),
+ _estimate_tokens(result.compressed),
)
decision_reason = "log_compressor"
@@ -2085,7 +2106,7 @@ class ContentRouter(Transform):
result = compressor.compress(content, context=context, bias=bias)
compressed, compressed_tokens = (
result.compressed,
- len(result.compressed.split()),
+ _estimate_tokens(result.compressed),
)
decision_reason = "tabular_compressor"
@@ -2096,7 +2117,7 @@ class ContentRouter(Transform):
result = compressor.compress(content, context=context)
compressed, compressed_tokens = (
result.compressed,
- len(result.compressed.split()),
+ _estimate_tokens(result.compressed),
)
decision_reason = "diff_compressor"
@@ -2108,7 +2129,7 @@ class ContentRouter(Transform):
result = extractor.extract(content)
compressed = result.extracted
# Estimate tokens from extracted text (simple word count)
- compressed_tokens = len(compressed.split()) if compressed else 0
+ compressed_tokens = _estimate_tokens(compressed) if compressed else 0
decision_reason = "html_extractor"
elif strategy == CompressionStrategy.KOMPRESS:
@@ -2180,7 +2201,7 @@ class ContentRouter(Transform):
except Exception as exc: # noqa: BLE001
logger.debug("Log fallback failed for SMART_CRUSHER: %s", exc)
else:
- log_compressed_tokens = len(log_result.compressed.split())
+ log_compressed_tokens = _estimate_tokens(log_result.compressed)
if log_compressed_tokens < compressed_tokens:
compressed = log_result.compressed
compressed_tokens = log_compressed_tokens
@@ -2328,7 +2349,7 @@ class ContentRouter(Transform):
# If the entire content is custom tags with nothing to compress
if protected and not cleaned.strip():
- return content, len(content.split())
+ return content, _estimate_tokens(content)
# Use the cleaned (tag-free) text for compression
text_to_compress = cleaned if protected else content
@@ -2373,7 +2394,7 @@ class ContentRouter(Transform):
out = text_to_compress
if protected:
out = restore_tags(out, protected)
- return out, len(out.split())
+ return out, _estimate_tokens(out)
# Primary: Kompress. On a cold cache the model is fetched once in the
# background (ensure_background_load) instead of blocking this request
@@ -2414,14 +2435,14 @@ class ContentRouter(Transform):
logger.warning("Kompress failed: %s", e)
if compressed is None:
- return content, len(content.split())
+ return content, _estimate_tokens(content)
# Restore protected tag blocks into the compressed text
if protected:
compressed = restore_tags(compressed, protected)
- compressed_tokens = len(compressed.split())
+ compressed_tokens = _estimate_tokens(compressed)
- return compressed, compressed_tokens or len(compressed.split())
+ return compressed, compressed_tokens or _estimate_tokens(compressed)
def _experimental_compress_read(self, content: Any, context: str = "") -> str | None:
"""EXPERIMENT (HEADROOM_EXPERIMENTAL_READ_KEEP_RATIO): lightly Kompress a
diff --git a/tests/test_content_router_compact_json.py b/tests/test_content_router_compact_json.py
new file mode 100644
index 000000000..f1b4cbe80
--- /dev/null
+++ b/tests/test_content_router_compact_json.py
@@ -0,0 +1,71 @@
+"""Compact machine-generated JSON must not evade compression (token-estimate bug).
+
+Whitespace-split token counting made a compact JSON payload (no spaces —
+the default output of json.dumps with separators, JSON.stringify, boto3)
+count as ~1 token, so compression ratios computed as ~1.0 and the
+min_ratio gate rejected SmartCrusher's real output.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+
+import pytest
+
+from headroom.transforms.content_router import (
+ ContentRouter,
+ ContentRouterConfig,
+ _estimate_tokens,
+)
+
+
+def test_estimate_tokens_monotone_on_compact_json() -> None:
+ small = json.dumps([{"a": 1}] * 5, separators=(",", ":"))
+ large = json.dumps([{"a": 1}] * 500, separators=(",", ":"))
+ assert _estimate_tokens(large) > _estimate_tokens(small) > 1
+
+
+def test_compact_json_tool_result_compresses() -> None:
+ tokenizer = pytest.importorskip("headroom.tokenizers.estimator")
+ from headroom.tokenizer import Tokenizer
+
+ tok = Tokenizer(tokenizer.EstimatingTokenCounter())
+ random.seed(42)
+ rows = [
+ {
+ "serviceArn": f"arn:aws:ecs:us-east-1:123456789012:service/x/svc-{s:03d}",
+ "serviceName": f"svc-{s:03d}",
+ "status": "ACTIVE",
+ "desiredCount": random.randint(1, 6),
+ "runningCount": random.randint(0, 6),
+ }
+ for s in range(150)
+ ]
+ payload = json.dumps(rows, separators=(",", ":"))
+ assert " " not in payload[:200] # genuinely compact
+ messages = [
+ {"role": "user", "content": "Investigate."},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "text", "text": "Checking."},
+ {"type": "tool_use", "id": "toolu_1", "name": "list_services", "input": {}},
+ ],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": payload}],
+ },
+ {"role": "user", "content": "Summarize in one sentence."},
+ ]
+ router = ContentRouter(ContentRouterConfig(skip_user_messages=False))
+ before = tok.count_messages(messages)
+ result = router.apply(
+ [json.loads(json.dumps(m)) for m in messages],
+ tok,
+ context="Summarize",
+ frozen_message_count=0,
+ )
+ after = tok.count_messages(result.messages)
+ assert after < before * 0.9, (before, after, result.transforms_applied[:5])
diff --git a/tests/test_lossless_then_lossy.py b/tests/test_lossless_then_lossy.py
index 19c7276ce..6dadeefc0 100644
--- a/tests/test_lossless_then_lossy.py
+++ b/tests/test_lossless_then_lossy.py
@@ -15,6 +15,7 @@ from headroom.transforms.content_router import (
CompressionStrategy,
ContentRouter,
ContentRouterConfig,
+ _estimate_tokens,
)
from headroom.transforms.lossless_compaction import search_unheading
@@ -148,7 +149,10 @@ def test_lossy_gate_boundary_default(monkeypatch):
block = _grep_block()
r, _ = _router(lossless_then_lossy=True)
assert abs(r._lossy_min_extra_savings - 0.05) < 1e-9 # default: require >=5% extra
- fold_tok = len(r._lossless_first(block, CompressionStrategy.SEARCH)[0].split())
+ # Measure the fold with the SAME estimator the router's gate uses
+ # (_estimate_tokens), not len(split()) — the two diverge, and the gate at
+ # content_router.py compares _komp_tokens against _estimate_tokens(fold).
+ fold_tok = _estimate_tokens(r._lossless_first(block, CompressionStrategy.SEARCH)[0])
# Kompress that keeps 94% of fold tokens -> saves 6% >= the 5% floor -> chained.
keep = int(fold_tok * 0.94)
r._try_ml_compressor = lambda c, ctx, q=None: (" ".join(["w"] * keep), keep) # type: ignore
@@ -161,7 +165,8 @@ def test_lossy_gate_env_override(monkeypatch):
r, _ = _router(lossless_then_lossy=True)
assert abs(r._lossy_min_extra_savings - 0.20) < 1e-9 # env override wins
block = _grep_block()
- fold_tok = len(r._lossless_first(block, CompressionStrategy.SEARCH)[0].split())
+ # Same estimator as the router gate (_estimate_tokens), not len(split()).
+ fold_tok = _estimate_tokens(r._lossless_first(block, CompressionStrategy.SEARCH)[0])
keep = int(fold_tok * 0.90) # saves 10%: passes the 5% default but FAILS the 20% override
r._try_ml_compressor = lambda c, ctx, q=None: (" ".join(["w"] * keep), keep) # type: ignore
_, was, tr, rc = _run(r, block)
diff --git a/tests/test_transforms/test_text_crusher_routing.py b/tests/test_transforms/test_text_crusher_routing.py
index e3da83e7c..6da9891ec 100644
--- a/tests/test_transforms/test_text_crusher_routing.py
+++ b/tests/test_transforms/test_text_crusher_routing.py
@@ -4,7 +4,7 @@ LogCompressor (which yields ~0 on prose) or ModernBERT (slow)."""
from __future__ import annotations
-from headroom.transforms.content_router import ContentRouter
+from headroom.transforms.content_router import ContentRouter, _estimate_tokens
def _prose() -> str:
@@ -28,7 +28,10 @@ def test_gate_routes_to_text_crusher_when_enabled(monkeypatch):
out, ntok = router._try_ml_compressor(prose, "authentication tokens")
assert router._kompress_gate_fires == 1
- assert ntok < len(prose.split()) # TextCrusher actually compressed (LogCompressor ~0)
+ # TextCrusher actually compressed (LogCompressor ~0). Compare like-for-like
+ # against the same token estimator the compressor reports in, not a raw
+ # word count, so the assertion tracks real token reduction.
+ assert ntok < _estimate_tokens(prose)
assert set(out.split()) <= set(prose.split()) # extractive: no invented words
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 02498bbac..fe5bf58ec 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -15,6 +15,7 @@ from headroom.transforms.content_router import (
RoutingDecision,
_create_content_signature,
_detect_content,
+ _estimate_tokens,
_extract_json_block,
_strip_detection_envelope,
is_mixed_content,
@@ -483,7 +484,7 @@ def test_content_router_mixed_pure_apply_and_toin(monkeypatch: pytest.MonkeyPatc
)
pure = router._compress_pure("some plain text", CompressionStrategy.TEXT, "ctx")
assert pure.routing_log[0].content_type is ContentType.PLAIN_TEXT
- assert pure.total_original_tokens == 3
+ assert pure.total_original_tokens == _estimate_tokens("some plain text")
assert pure.total_compressed_tokens == 1
calls: list[dict] = []
@@ -551,7 +552,7 @@ def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(
)
assert compressed == diff
- assert compressed_tokens == len(diff.split())
+ assert compressed_tokens == _estimate_tokens(diff)
assert strategy_chain == ["diff"]
@@ -579,7 +580,7 @@ def test_log_strategy_does_not_fallback_to_kompress_when_log_is_noop(
)
assert compressed == log
- assert compressed_tokens == len(log.split())
+ assert compressed_tokens == _estimate_tokens(log)
assert strategy_chain == ["log"]