Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): don't cache token-count inputs larger than max_chars (#4469)

Parideboyaebb729ef3f0
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.py
index 52ae322f2..c18eb7a91 100644
--- a/headroom/tokenizers/base.py
+++ b/headroom/tokenizers/base.py
@@ -78,12 +78,15 @@ class TokenCountCache:
must not run the eviction check (a dict hit would clear the whole
cache and force a full re-encode of every cached message on the
next miss — the storm this exists to prevent).
+ Oversized texts are never admitted, so warm entries survive.
"""
if text in self._counts:
self._counts[text] = count
return
if len(text) < self._min_chars:
return
+ if len(text) > self._max_chars:
+ return
if len(self._counts) >= self._max_entries or self._chars >= self._max_chars:
self._counts.clear()
self._chars = 0
diff --git a/tests/test_token_count_cache.py b/tests/test_token_count_cache.py
index cfb6a1df3..199836764 100644
--- a/tests/test_token_count_cache.py
+++ b/tests/test_token_count_cache.py
@@ -157,3 +157,22 @@ def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:
assert cache._counts == {"aaaaaaaaaa-1": 10}
cache.put("x", 5) # re-put: still no admission, no eviction
assert cache._counts == {"aaaaaaaaaa-1": 10}
+
+
+def test_oversized_text_is_not_cached() -> None:
+ cache = TokenCountCache(min_chars=1, max_chars=10)
+ cache.put("x" * 20, 5)
+ assert cache.get("x" * 20) is None
+
+
+def test_warm_entry_survives_oversized_put() -> None:
+ cache = TokenCountCache(min_chars=1, max_chars=10)
+ cache.put("warm", 7)
+ cache.put("x" * 20, 5)
+ assert cache.get("warm") == 7
+
+
+def test_text_of_exactly_max_chars_is_admitted() -> None:
+ cache = TokenCountCache(min_chars=1, max_chars=10)
+ cache.put("x" * 10, 5)
+ assert cache.get("x" * 10) == 5