Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(tokenizer): memoize OpenAI token counts like the Anthropic counter (#3168)
Abhay Singh0a2c80d52076
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex 5243ebfca..4466d07ec 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -17,7 +17,11 @@ from typing import Any, castfrom headroom import paths as _pathsfrom headroom.pricing.litellm_pricing import estimate_cost_from_tokens-from headroom.tokenizers.base import coerce_countable_text, count_content_blocks+from headroom.tokenizers.base import (+ TokenCountCache,+ coerce_countable_text,+ count_content_blocks,+)from .base import Provider, TokenCounter@@ -366,11 +370,26 @@ class OpenAITokenCounter:self.model = modelencoding_name = _get_encoding_name_for_model(model, custom_encodings)self._encoding = _get_encoding(encoding_name)+ # count_text is a pure function of its text, and this counter is a+ # per-model singleton reused across requests (OpenAIProvider caches it),+ # so a stable prefix/system prompt or a repeated tool result is otherwise+ # re-encoded on every turn. Cache the count like AnthropicTokenCounter+ # already does (the cache only admits large strings, so tiny/rare ones+ # pay nothing).+ self._count_cache = TokenCountCache()def count_text(self, text: str) -> int:"""Count tokens in text."""if not text:return 0+ cached = self._count_cache.get(text)+ if cached is not None:+ return cached+ count = self._count_text_uncached(text)+ self._count_cache.put(text, count)+ return count++ def _count_text_uncached(self, text: str) -> int:try:return len(self._encoding.encode(text))except ValueError:diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.pyindex 6d26993d4..92c0516e0 100644--- a/tests/test_providers/test_openai.py+++ b/tests/test_providers/test_openai.py@@ -33,6 +33,27 @@ class TestOpenAITokenCounting:count = openai_tokenizer.count_text(text)assert count > openai_tokenizer.count_text("before after")+ def test_count_text_caches_large_strings(self):+ """count_text memoizes large strings so a stable prefix / repeated tool+ result is not re-encoded on every turn (the counter is a per-model+ singleton). Cached counts must equal the uncached count, and tiny+ strings must stay below the admission floor."""+ from headroom.providers.openai import OpenAITokenCounter++ counter = OpenAITokenCounter("gpt-4o")+ large = "lorem ipsum dolor sit amet consectetur adipiscing elit " * 8 # > 256 chars+ assert len(large) >= 256++ direct = counter._count_text_uncached(large)+ assert counter._count_cache.get(large) is None # cold before first count+ assert counter.count_text(large) == direct # value-identical to uncached+ assert counter._count_cache.get(large) == direct # now memoized++ # A short string encodes in microseconds and stays below the floor, so+ # it never evicts the large entries the cache exists for.+ counter.count_text("hi there")+ assert counter._count_cache.get("hi there") is None+def test_count_messages_single(self, openai_tokenizer):messages = [{"role": "user", "content": "Hello"}]count = openai_tokenizer.count_messages(messages)