Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(tokenizer): memoize OpenAI token counts like the Anthropic counter (#3168)

Abhay Singh0a2c80d52076
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index 5243ebfca..4466d07ec 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -17,7 +17,11 @@ from typing import Any, cast
from headroom import paths as _paths
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
-from headroom.tokenizers.base import coerce_countable_text, count_content_blocks
+from headroom.tokenizers.base import (
+ TokenCountCache,
+ coerce_countable_text,
+ count_content_blocks,
+)
from .base import Provider, TokenCounter
@@ -366,11 +370,26 @@ class OpenAITokenCounter:
self.model = model
encoding_name = _get_encoding_name_for_model(model, custom_encodings)
self._encoding = _get_encoding(encoding_name)
+ # count_text is a pure function of its text, and this counter is a
+ # per-model singleton reused across requests (OpenAIProvider caches it),
+ # so a stable prefix/system prompt or a repeated tool result is otherwise
+ # re-encoded on every turn. Cache the count like AnthropicTokenCounter
+ # already does (the cache only admits large strings, so tiny/rare ones
+ # pay nothing).
+ self._count_cache = TokenCountCache()
def count_text(self, text: str) -> int:
"""Count tokens in text."""
if not text:
return 0
+ cached = self._count_cache.get(text)
+ if cached is not None:
+ return cached
+ count = self._count_text_uncached(text)
+ self._count_cache.put(text, count)
+ return count
+
+ def _count_text_uncached(self, text: str) -> int:
try:
return len(self._encoding.encode(text))
except ValueError:
diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.py
index 6d26993d4..92c0516e0 100644
--- a/tests/test_providers/test_openai.py
+++ b/tests/test_providers/test_openai.py
@@ -33,6 +33,27 @@ class TestOpenAITokenCounting:
count = openai_tokenizer.count_text(text)
assert count > openai_tokenizer.count_text("before after")
+ def test_count_text_caches_large_strings(self):
+ """count_text memoizes large strings so a stable prefix / repeated tool
+ result is not re-encoded on every turn (the counter is a per-model
+ singleton). Cached counts must equal the uncached count, and tiny
+ strings must stay below the admission floor."""
+ from headroom.providers.openai import OpenAITokenCounter
+
+ counter = OpenAITokenCounter("gpt-4o")
+ large = "lorem ipsum dolor sit amet consectetur adipiscing elit " * 8 # > 256 chars
+ assert len(large) >= 256
+
+ direct = counter._count_text_uncached(large)
+ assert counter._count_cache.get(large) is None # cold before first count
+ assert counter.count_text(large) == direct # value-identical to uncached
+ assert counter._count_cache.get(large) == direct # now memoized
+
+ # A short string encodes in microseconds and stays below the floor, so
+ # it never evicts the large entries the cache exists for.
+ counter.count_text("hi there")
+ assert counter._count_cache.get("hi there") is None
+
def test_count_messages_single(self, openai_tokenizer):
messages = [{"role": "user", "content": "Hello"}]
count = openai_tokenizer.count_messages(messages)