Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): treat literal special-token strings as plain text (#1244)

Umut Keltekd480c464e96d
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index 2851f4802..b4c688b15 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -298,7 +298,14 @@ class OpenAITokenCounter:
"""Count tokens in text."""
if not text:
return 0
- return len(self._encoding.encode(text))
+ try:
+ return len(self._encoding.encode(text))
+ except ValueError:
+ # Passthrough content can legitimately contain strings that look
+ # like tiktoken special tokens (e.g. "<|endoftext|>"). Treat them
+ # as ordinary text instead of raising. Matches
+ # AnthropicTokenCounter.count_text.
+ return len(self._encoding.encode(text, disallowed_special=()))
def count_message(self, message: dict[str, Any]) -> int:
"""
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py
index a22fd9f9d..1a093e2df 100644
--- a/headroom/tokenizers/tiktoken_counter.py
+++ b/headroom/tokenizers/tiktoken_counter.py
@@ -234,7 +234,15 @@ class TiktokenCounter(BaseTokenizer):
"""
if not text:
return 0
- return len(self.encoding.encode(text))
+ try:
+ return len(self.encoding.encode(text))
+ except ValueError:
+ # Passthrough content can legitimately contain strings that look
+ # like tiktoken special tokens (e.g. "<|endoftext|>" or FIM markers
+ # in code/tool output). Treat them as ordinary text instead of
+ # raising, which would otherwise abort token counting for the whole
+ # request. Matches AnthropicTokenCounter.count_text.
+ return len(self.encoding.encode(text, disallowed_special=()))
def count_messages(self, messages: list[dict[str, Any]]) -> int:
"""Count tokens in messages using OpenAI's exact formula.
@@ -311,7 +319,13 @@ class TiktokenCounter(BaseTokenizer):
Returns:
List of token IDs.
"""
- return self.encoding.encode(text)
+ try:
+ return self.encoding.encode(text)
+ except ValueError:
+ # See count_text: literal special-token strings in passthrough
+ # content must be encoded as ordinary text, not rejected. The
+ # round-trip through decode() is unaffected.
+ return self.encoding.encode(text, disallowed_special=())
def decode(self, tokens: list[int]) -> str:
"""Decode token IDs to text.
diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.py
index b9db787b4..3b5857447 100644
--- a/tests/test_providers/test_openai.py
+++ b/tests/test_providers/test_openai.py
@@ -21,6 +21,18 @@ class TestOpenAITokenCounting:
count = openai_tokenizer.count_text(text)
assert count > 0
+ def test_count_text_allows_literal_special_tokens(self, openai_tokenizer):
+ """count_text must not raise on literal tiktoken special-token strings.
+
+ Regression: a /v1/responses request whose context contained the literal
+ "<|endoftext|>" made tiktoken raise ValueError (default
+ disallowed_special="all"), which the proxy turned into an HTTP 413
+ compression_refused. Markers must be counted as ordinary text instead.
+ """
+ text = "before <|endoftext|> after"
+ count = openai_tokenizer.count_text(text)
+ assert count > openai_tokenizer.count_text("before after")
+
def test_count_messages_single(self, openai_tokenizer):
messages = [{"role": "user", "content": "Hello"}]
count = openai_tokenizer.count_messages(messages)
diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py
index 742d5aefb..bc1b3b19b 100644
--- a/tests/test_tokenizers.py
+++ b/tests/test_tokenizers.py
@@ -111,6 +111,29 @@ class TestTiktokenCounter:
decoded = counter.decode(tokens)
assert decoded == text
+ def test_count_text_allows_literal_special_tokens(self):
+ """count_text must not raise on literal tiktoken special-token strings.
+
+ Regression: passthrough/tool content containing "<|endoftext|>" (or FIM
+ markers) made tiktoken raise ValueError under its default
+ disallowed_special="all", aborting token counting for the whole request.
+ Through the proxy this surfaced as an HTTP 413 compression_refused.
+ """
+ counter = TiktokenCounter("gpt-4o")
+ text = "before <|endoftext|> after <|fim_prefix|> end"
+ # Must not raise; markers are counted as ordinary text.
+ count = counter.count_text(text)
+ assert count > counter.count_text("before after end")
+
+ def test_encode_allows_literal_special_tokens(self):
+ """encode must treat literal special-token strings as ordinary text."""
+ counter = TiktokenCounter("gpt-4o")
+ text = "x <|endoftext|> y"
+ tokens = counter.encode(text)
+ assert isinstance(tokens, list) and len(tokens) > 0
+ # Encoding as ordinary text round-trips back to the original literal.
+ assert counter.decode(tokens) == text
+
def test_repr(self):
"""Test string representation."""
counter = TiktokenCounter("gpt-4o")