Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): don't cache token-count inputs larger than max_chars (#4469)
Parideboyaebb729ef3f0
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex 52ae322f2..c18eb7a91 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -78,12 +78,15 @@ class TokenCountCache:must not run the eviction check (a dict hit would clear the wholecache and force a full re-encode of every cached message on thenext miss — the storm this exists to prevent).+ Oversized texts are never admitted, so warm entries survive."""if text in self._counts:self._counts[text] = countreturnif len(text) < self._min_chars:return+ if len(text) > self._max_chars:+ returnif len(self._counts) >= self._max_entries or self._chars >= self._max_chars:self._counts.clear()self._chars = 0diff --git a/tests/test_token_count_cache.py b/tests/test_token_count_cache.pyindex cfb6a1df3..199836764 100644--- a/tests/test_token_count_cache.py+++ b/tests/test_token_count_cache.py@@ -157,3 +157,22 @@ def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:assert cache._counts == {"aaaaaaaaaa-1": 10}cache.put("x", 5) # re-put: still no admission, no evictionassert cache._counts == {"aaaaaaaaaa-1": 10}+++def test_oversized_text_is_not_cached() -> None:+ cache = TokenCountCache(min_chars=1, max_chars=10)+ cache.put("x" * 20, 5)+ assert cache.get("x" * 20) is None+++def test_warm_entry_survives_oversized_put() -> None:+ cache = TokenCountCache(min_chars=1, max_chars=10)+ cache.put("warm", 7)+ cache.put("x" * 20, 5)+ assert cache.get("warm") == 7+++def test_text_of_exactly_max_chars_is_admitted() -> None:+ cache = TokenCountCache(min_chars=1, max_chars=10)+ cache.put("x" * 10, 5)+ assert cache.get("x" * 10) == 5