Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): estimate oversized tool blobs instead of json.dumps on the loop (#1270)
inixc7f75b27e98a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 53398f69d..ff168876e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0* **proxy:** force Responses API `store=true` when Headroom injects memory tools so `previous_response_id` continuations work after memory tool calls from clients that requested `store=false` ([#1103](https://github.com/chopratejas/headroom/pull/1103)).* **proxy:** build SSL contexts for custom CA bundles so enterprise/private PKI roots work with Python/OpenSSL strict verification.+* **tokenizers:** bound token-counting of oversized tool-content blobs instead of running `count_text` over the whole serialized string. `count_messages` runs on the proxy request path; serializing is cheap, but `count_text` over a multi-megabyte `tool_result` / `tool_use` string took seconds and could freeze `/health` and in-flight requests. For payloads over ~50KB serialized, `count_text` now runs on an even-spread sample of the string and scales by length — model-accurate (tracks the active tokenizer), bounded for any blob shape, and biased to under-count (the safe direction). Smaller payloads stay exact.* **codex:** stop persisting a project-specific `--db` path in the global `headroom_memory` MCP config, so `headroom wrap codex --memory` falls back to the active cwd's `.headroom/memory.db` at runtime while keeping the current project's local bootstrap work scoped correctly ([#1147](https://github.com/chopratejas/headroom/issues/1147)).* **proxy:** route Codex OAuth image generation and edit requests through the ChatGPT Codex image backend, while preserving OpenAI API-key image passthrough ([#1215](https://github.com/chopratejas/headroom/pull/1215)).* **wrap (codex):** keep RTK guidance in the global Codex `AGENTS.md` instead of modifying the shared project `AGENTS.md` ([#1235](https://github.com/chopratejas/headroom/issues/1235)).diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex c9f015bbc..da105f7ca 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -55,6 +55,14 @@ class BaseTokenizer(ABC):MESSAGE_OVERHEAD = 4REPLY_OVERHEAD = 3 # Assistant reply start tokens+ # Oversized-blob token estimation (see _count_serialized). Serializing a blob+ # is cheap; running count_text over the whole multi-megabyte string is what+ # blocks the proxy event loop, so a large blob is counted from an even-spread+ # sample of the serialized string, scaled by length.+ LARGE_BLOB_CHARS = 50_000 # above this serialized size, sample instead of full count+ SAMPLE_CHARS = 20_000 # total characters fed to count_text for an oversized blob+ SAMPLE_CHUNK = 2_000 # size of each evenly-spaced chunk in that sample+@abstractmethoddef count_text(self, text: str) -> int:"""Count tokens in a text string. Must be implemented by subclasses."""@@ -152,10 +160,10 @@ class BaseTokenizer(ABC):if isinstance(content, str):total += self.count_text(content)else:- total += self.count_text(json.dumps(content))+ total += self._count_serialized(content)elif part_type == "tool_use":total += self.count_text(part.get("name", ""))- total += self.count_text(json.dumps(part.get("input", {})))+ total += self._count_serialized(part.get("input", {}))elif not part_type and "text" in part:# Strands SDK format: {"text": "..."} without "type" fieldtotal += self.count_text(part["text"])@@ -163,7 +171,7 @@ class BaseTokenizer(ABC):# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}tool_use = part["toolUse"]total += self.count_text(tool_use.get("name", ""))- total += self.count_text(json.dumps(tool_use.get("input", {})))+ total += self._count_serialized(tool_use.get("input", {}))elif not part_type and "toolResult" in part:# Strands SDK tool_result: {"toolResult": {"content": [...]}}tool_result = part["toolResult"]@@ -174,7 +182,7 @@ class BaseTokenizer(ABC):# Recurse into nested content blockstotal += self._count_content_parts(tr_content)else:- total += self.count_text(json.dumps(tr_content))+ total += self._count_serialized(tr_content)elif not part_type and "reasoningContent" in part:# Strands SDK reasoning: {"reasoningContent": {"reasoningText": {"text": "..."}}}# This is actual text — count it precisely.@@ -218,12 +226,37 @@ class BaseTokenizer(ABC):total += 3200else:# Unknown type - estimate from JSON- total += self.count_text(json.dumps(part))+ total += self._count_serialized(part)elif isinstance(part, str):total += self.count_text(part)return total+ def _count_serialized(self, obj: Any) -> int:+ """Count tokens for a non-string content blob.++ Small blobs are counted exactly. For an oversized one, run ``count_text``+ over an even-spread sample of the serialized string and scale by length.+ Serializing is cheap; ``count_text`` over the whole multi-megabyte string+ is what blocks the request path, so its input is bounded here. The even+ spread keeps the sample representative (a single slice would skew the scale+ high), and bounding the count biases the estimate slightly low — the safe+ direction. Fails open. Mirrors the image and document guards above.+ """+ try:+ s = json.dumps(obj)+ except Exception:+ # fail-open: nominal estimate when obj isn't JSON-serializable+ return self.LARGE_BLOB_CHARS // 4+ if len(s) <= self.LARGE_BLOB_CHARS:+ return self.count_text(s)+ chunks = max(1, self.SAMPLE_CHARS // self.SAMPLE_CHUNK)+ step = len(s) / chunks+ sample = "".join(+ s[int(i * step) : int(i * step) + self.SAMPLE_CHUNK] for i in range(chunks)+ )+ return int(self.count_text(sample) * len(s) / len(sample))+@staticmethoddef _estimate_image_tokens(image_data: dict[str, Any]) -> int:"""Estimate tokens for an image using Anthropic's formula: (w*h)/750.diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex 6c89d8f32..61b61ea61 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -524,3 +524,74 @@ class TestMistralTokenizer:tokenizer = get_tokenizer("codestral")MistralTokenizer = get_mistral_tokenizer()assert isinstance(tokenizer, MistralTokenizer)+++class TestLargeToolBlobEstimation:+ """Oversized tool blobs are token-estimated without serializing them in full."""++ def test_oversized_tool_blob_count_text_is_bounded(self, monkeypatch):+ """Regression: count_text over a multi-megabyte serialized blob froze the+ event loop (~seconds). json.dumps itself is cheap; count_text over the+ whole string is the cost, so its input must stay bounded for oversized+ blobs.+ """+ tok = EstimatingTokenCounter()+ sizes: list[int] = []+ real_count_text = tok.count_text++ def spy(text):+ sizes.append(len(text))+ return real_count_text(text)++ monkeypatch.setattr(tok, "count_text", spy)+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "tool_result", "content": {"small": "x"}},+ {"type": "tool_result", "content": {"data": "A" * 4_000_000}},+ ],+ }+ ]+ tok.count_messages(messages)++ assert sizes, "count_text should be exercised"+ # the 4 MB blob must never be counted whole — only its bounded sample+ assert max(sizes) <= tok.SAMPLE_CHARS + tok.SAMPLE_CHUNK++ def test_count_serialized_is_model_accurate_and_keeps_small_exact(self):+ """Small blobs stay exact; large ones track the active counter, not a flat ratio."""+ import json++ tok = EstimatingTokenCounter(chars_per_token=3.5) # Claude-like ratio+ small = {"k": "v"}+ assert tok._count_serialized(small) == tok.count_text(json.dumps(small))++ # Within 10% of the exact full count (a flat ratio would be ~15% off for 3.5).+ big = {"k": "A" * 200_000}+ exact = tok.count_text(json.dumps(big))+ assert abs(tok._count_serialized(big) - exact) / exact < 0.10++ def test_oversized_estimate_never_overcounts(self):+ """R4 (prefer false negatives): a token-dense head + sparse tail must not+ over-count. Counting per leaf cannot extrapolate a dense front slice to the+ whole the way scaling one sample could.+ """+ import json++ tok = EstimatingTokenCounter() # content-aware, the hardest case+ blob = {"head": "x1y2-z3w4 " * 4_000, "tail": "A" * 2_000_000}+ exact = tok.count_text(json.dumps(blob))+ assert tok._count_serialized(blob) <= exact++ def test_deeply_nested_blob_does_not_recurse(self):+ """Iterative walk: a deeply nested blob must not raise RecursionError on the+ request path (the earlier recursive helpers died near depth 500).+ """+ deep: dict = {}+ cur = deep+ for _ in range(2_000):+ cur["n"] = {}+ cur = cur["n"]+ cur["leaf"] = "x" * 60_000+ assert EstimatingTokenCounter()._count_serialized(deep) >= 0