Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): resolve tiktoken encoding for gateway-prefixed model ids (#4637)

Alvi Rahmane9331b6b963b
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py
index 4cd4261d2..61682ca54 100644
--- a/headroom/tokenizers/tiktoken_counter.py
+++ b/headroom/tokenizers/tiktoken_counter.py
@@ -170,7 +170,10 @@ def get_encoding_for_model(model: str) -> str:
# verbatim, matched no prefix, and silently took DEFAULT_ENCODING -- so the
# encoding a model got depended on the casing of whichever request warmed
# the cache first, and flipped across restarts.
- model = model.lower()
+ # Gateway-wrapped ids ("azure/gpt-4o", "openrouter/openai/gpt-5") are picked
+ # for tiktoken by the registry's unwrapping but arrive here as-is, so match on
+ # the bare id or they all take DEFAULT_ENCODING.
+ model = model.lower().rsplit("/", 1)[-1]
# Direct lookup
if model in MODEL_TO_ENCODING:
diff --git a/tests/test_tokenizer_encoding_resolution.py b/tests/test_tokenizer_encoding_resolution.py
index 3dd1b6be8..255acbc52 100644
--- a/tests/test_tokenizer_encoding_resolution.py
+++ b/tests/test_tokenizer_encoding_resolution.py
@@ -46,6 +46,12 @@ CJK = "这是一个测试文档,用于验证分词器的差异。" * 30
("O4-Mini", "o200k_base"),
("GPT-4", "cl100k_base"),
("GPT-4-Turbo", "cl100k_base"),
+ # Gateway prefixes must resolve like the bare id.
+ ("azure/gpt-4o", "o200k_base"),
+ ("openai/gpt-4.1-mini", "o200k_base"),
+ ("openrouter/openai/gpt-5", "o200k_base"),
+ ("Azure/GPT-4o", "o200k_base"),
+ ("openai/gpt-4", "cl100k_base"),
# Must not regress.
("gpt-4o", "o200k_base"),
("gpt-4.1", "o200k_base"),
@@ -73,6 +79,15 @@ def test_cold_cache_uppercase_still_gets_the_right_encoding(model: str) -> None:
assert get_tokenizer(model).count_text(CJK) == o200k
[email protected]("model", ["azure/gpt-4o", "openrouter/openai/gpt-5"])
+def test_gateway_wrapped_model_counts_like_bare_id(model: str) -> None:
+ tiktoken = pytest.importorskip("tiktoken")
+ o200k = len(tiktoken.get_encoding("o200k_base").encode(CJK))
+
+ TokenizerRegistry.clear_cache()
+ assert get_tokenizer(model).count_text(CJK) == o200k
+
+
def test_casing_is_not_load_order_dependent() -> None:
"""The same model must count identically whichever spelling arrives first."""
TokenizerRegistry.clear_cache()