Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): resolve gpt-5 and mixed-case model names to the right encoding (#2776)

Tejas Choprafc4680b37af1
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py
index 5c05f308d..87a4ef0fa 100644
--- a/headroom/tokenizers/tiktoken_counter.py
+++ b/headroom/tokenizers/tiktoken_counter.py
@@ -164,6 +164,14 @@ def get_encoding_for_model(model: str) -> str:
Returns:
Encoding name (e.g., 'o200k_base', 'cl100k_base').
"""
+ # Case-insensitive: TokenizerRegistry lowercases only its *cache key*, then
+ # constructs the counter from the caller's original string. An uppercase
+ # deployment name ("GPT-4o", routine on Azure) therefore arrived here
+ # verbatim, matched no prefix, and silently took DEFAULT_ENCODING -- so the
+ # encoding a model got depended on the casing of whichever request warmed
+ # the cache first, and flipped across restarts.
+ model = model.lower()
+
# Direct lookup
if model in MODEL_TO_ENCODING:
return MODEL_TO_ENCODING[model]
@@ -180,6 +188,9 @@ def get_encoding_for_model(model: str) -> str:
# which they would otherwise match and be mis-encoded as cl100k_base.
("gpt-4.1", "o200k_base"),
("gpt-4.5", "o200k_base"),
+ # gpt-5 uses o200k_base. Without this it fell through to the
+ # cl100k_base default, over-counting CJK by ~33%.
+ ("gpt-5", "o200k_base"),
("gpt-4-turbo", "cl100k_base"),
("gpt-4", "cl100k_base"),
("gpt-3.5", "cl100k_base"),
diff --git a/tests/test_tokenizer_encoding_resolution.py b/tests/test_tokenizer_encoding_resolution.py
new file mode 100644
index 000000000..3dd1b6be8
--- /dev/null
+++ b/tests/test_tokenizer_encoding_resolution.py
@@ -0,0 +1,84 @@
+"""``get_encoding_for_model`` must not depend on casing, and must know gpt-5.
+
+Two defects, both reachable through the normal ``get_tokenizer()`` path:
+
+1. **gpt-5 had no prefix entry**, so it fell through to ``DEFAULT_ENCODING``
+ (``cl100k_base``) instead of ``o200k_base``. On CJK text cl100k emits ~33%
+ more tokens than o200k, so every gpt-5 count was inflated.
+
+2. **Resolution was case-sensitive.** ``TokenizerRegistry.get`` lowercases only
+ its *cache key*, then builds the counter from the caller's original string
+ (``_create_tokenizer(model, ...)``). An uppercase deployment name -- routine
+ on Azure, where the deployment name is user-chosen -- reached the resolver
+ verbatim, matched nothing, and took the default encoding.
+
+ The cache made (2) genuinely nasty: because the key is lowercased but
+ construction is not, the encoding a model ends up with depended on the
+ casing of whichever request warmed the cache first, and could differ across
+ restarts. The tests below call ``clear_cache()`` so the uppercase spelling is
+ resolved cold, which is the failing order.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.tokenizers import get_tokenizer
+from headroom.tokenizers.registry import TokenizerRegistry
+from headroom.tokenizers.tiktoken_counter import get_encoding_for_model
+
+CJK = "这是一个测试文档,用于验证分词器的差异。" * 30
+
+
+ ("model", "expected"),
+ [
+ # gpt-5 family: the missing entry.
+ ("gpt-5", "o200k_base"),
+ ("gpt-5-mini", "o200k_base"),
+ ("gpt-5-nano", "o200k_base"),
+ ("gpt-5-2025-08-07", "o200k_base"),
+ # Casing must not change the answer.
+ ("GPT-4o", "o200k_base"),
+ ("GPT-4.1", "o200k_base"),
+ ("Gpt-4O-Mini", "o200k_base"),
+ ("GPT-5", "o200k_base"),
+ ("O4-Mini", "o200k_base"),
+ ("GPT-4", "cl100k_base"),
+ ("GPT-4-Turbo", "cl100k_base"),
+ # Must not regress.
+ ("gpt-4o", "o200k_base"),
+ ("gpt-4.1", "o200k_base"),
+ ("gpt-4", "cl100k_base"),
+ ("gpt-4-turbo", "cl100k_base"),
+ ("gpt-3.5-turbo", "cl100k_base"),
+ ("o4-mini", "o200k_base"),
+ ],
+)
+def test_encoding_resolution(model: str, expected: str) -> None:
+ assert get_encoding_for_model(model) == expected
+
+
[email protected]("model", ["gpt-5", "GPT-4o", "GPT-4.1"])
+def test_cold_cache_uppercase_still_gets_the_right_encoding(model: str) -> None:
+ """End-to-end through the registry, with the uppercase spelling resolved first.
+
+ Without clear_cache() a preceding lowercase lookup would populate the shared
+ (lowercased) cache key and mask the defect entirely.
+ """
+ tiktoken = pytest.importorskip("tiktoken")
+ o200k = len(tiktoken.get_encoding("o200k_base").encode(CJK))
+
+ TokenizerRegistry.clear_cache()
+ assert get_tokenizer(model).count_text(CJK) == o200k
+
+
+def test_casing_is_not_load_order_dependent() -> None:
+ """The same model must count identically whichever spelling arrives first."""
+ TokenizerRegistry.clear_cache()
+ upper_first = get_tokenizer("GPT-4o").count_text(CJK)
+
+ TokenizerRegistry.clear_cache()
+ lower_first = get_tokenizer("gpt-4o").count_text(CJK)
+
+ assert upper_first == lower_first