Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): treat literal special-token strings as plain text (#1244)
Umut Keltekd480c464e96d
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex 2851f4802..b4c688b15 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -298,7 +298,14 @@ class OpenAITokenCounter:"""Count tokens in text."""if not text:return 0- return len(self._encoding.encode(text))+ try:+ return len(self._encoding.encode(text))+ except ValueError:+ # Passthrough content can legitimately contain strings that look+ # like tiktoken special tokens (e.g. "<|endoftext|>"). Treat them+ # as ordinary text instead of raising. Matches+ # AnthropicTokenCounter.count_text.+ return len(self._encoding.encode(text, disallowed_special=()))def count_message(self, message: dict[str, Any]) -> int:"""diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex a22fd9f9d..1a093e2df 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -234,7 +234,15 @@ class TiktokenCounter(BaseTokenizer):"""if not text:return 0- return len(self.encoding.encode(text))+ try:+ return len(self.encoding.encode(text))+ except ValueError:+ # Passthrough content can legitimately contain strings that look+ # like tiktoken special tokens (e.g. "<|endoftext|>" or FIM markers+ # in code/tool output). Treat them as ordinary text instead of+ # raising, which would otherwise abort token counting for the whole+ # request. Matches AnthropicTokenCounter.count_text.+ return len(self.encoding.encode(text, disallowed_special=()))def count_messages(self, messages: list[dict[str, Any]]) -> int:"""Count tokens in messages using OpenAI's exact formula.@@ -311,7 +319,13 @@ class TiktokenCounter(BaseTokenizer):Returns:List of token IDs."""- return self.encoding.encode(text)+ try:+ return self.encoding.encode(text)+ except ValueError:+ # See count_text: literal special-token strings in passthrough+ # content must be encoded as ordinary text, not rejected. The+ # round-trip through decode() is unaffected.+ return self.encoding.encode(text, disallowed_special=())def decode(self, tokens: list[int]) -> str:"""Decode token IDs to text.diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.pyindex b9db787b4..3b5857447 100644--- a/tests/test_providers/test_openai.py+++ b/tests/test_providers/test_openai.py@@ -21,6 +21,18 @@ class TestOpenAITokenCounting:count = openai_tokenizer.count_text(text)assert count > 0+ def test_count_text_allows_literal_special_tokens(self, openai_tokenizer):+ """count_text must not raise on literal tiktoken special-token strings.++ Regression: a /v1/responses request whose context contained the literal+ "<|endoftext|>" made tiktoken raise ValueError (default+ disallowed_special="all"), which the proxy turned into an HTTP 413+ compression_refused. Markers must be counted as ordinary text instead.+ """+ text = "before <|endoftext|> after"+ count = openai_tokenizer.count_text(text)+ assert count > openai_tokenizer.count_text("before after")+def test_count_messages_single(self, openai_tokenizer):messages = [{"role": "user", "content": "Hello"}]count = openai_tokenizer.count_messages(messages)diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex 742d5aefb..bc1b3b19b 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -111,6 +111,29 @@ class TestTiktokenCounter:decoded = counter.decode(tokens)assert decoded == text+ def test_count_text_allows_literal_special_tokens(self):+ """count_text must not raise on literal tiktoken special-token strings.++ Regression: passthrough/tool content containing "<|endoftext|>" (or FIM+ markers) made tiktoken raise ValueError under its default+ disallowed_special="all", aborting token counting for the whole request.+ Through the proxy this surfaced as an HTTP 413 compression_refused.+ """+ counter = TiktokenCounter("gpt-4o")+ text = "before <|endoftext|> after <|fim_prefix|> end"+ # Must not raise; markers are counted as ordinary text.+ count = counter.count_text(text)+ assert count > counter.count_text("before after end")++ def test_encode_allows_literal_special_tokens(self):+ """encode must treat literal special-token strings as ordinary text."""+ counter = TiktokenCounter("gpt-4o")+ text = "x <|endoftext|> y"+ tokens = counter.encode(text)+ assert isinstance(tokens, list) and len(tokens) > 0+ # Encoding as ordinary text round-trips back to the original literal.+ assert counter.decode(tokens) == text+def test_repr(self):"""Test string representation."""counter = TiktokenCounter("gpt-4o")