Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): resolve tiktoken encoding for gateway-prefixed model ids (#4637)
Alvi Rahmane9331b6b963b
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex 4cd4261d2..61682ca54 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -170,7 +170,10 @@ def get_encoding_for_model(model: str) -> str:# verbatim, matched no prefix, and silently took DEFAULT_ENCODING -- so the# encoding a model got depended on the casing of whichever request warmed# the cache first, and flipped across restarts.- model = model.lower()+ # Gateway-wrapped ids ("azure/gpt-4o", "openrouter/openai/gpt-5") are picked+ # for tiktoken by the registry's unwrapping but arrive here as-is, so match on+ # the bare id or they all take DEFAULT_ENCODING.+ model = model.lower().rsplit("/", 1)[-1]# Direct lookupif model in MODEL_TO_ENCODING:diff --git a/tests/test_tokenizer_encoding_resolution.py b/tests/test_tokenizer_encoding_resolution.pyindex 3dd1b6be8..255acbc52 100644--- a/tests/test_tokenizer_encoding_resolution.py+++ b/tests/test_tokenizer_encoding_resolution.py@@ -46,6 +46,12 @@ CJK = "这是一个测试文档,用于验证分词器的差异。" * 30("O4-Mini", "o200k_base"),("GPT-4", "cl100k_base"),("GPT-4-Turbo", "cl100k_base"),+ # Gateway prefixes must resolve like the bare id.+ ("azure/gpt-4o", "o200k_base"),+ ("openai/gpt-4.1-mini", "o200k_base"),+ ("openrouter/openai/gpt-5", "o200k_base"),+ ("Azure/GPT-4o", "o200k_base"),+ ("openai/gpt-4", "cl100k_base"),# Must not regress.("gpt-4o", "o200k_base"),("gpt-4.1", "o200k_base"),@@ -73,6 +79,15 @@ def test_cold_cache_uppercase_still_gets_the_right_encoding(model: str) -> None:assert get_tokenizer(model).count_text(CJK) == o200k[email protected]("model", ["azure/gpt-4o", "openrouter/openai/gpt-5"])+def test_gateway_wrapped_model_counts_like_bare_id(model: str) -> None:+ tiktoken = pytest.importorskip("tiktoken")+ o200k = len(tiktoken.get_encoding("o200k_base").encode(CJK))++ TokenizerRegistry.clear_cache()+ assert get_tokenizer(model).count_text(CJK) == o200k++def test_casing_is_not_load_order_dependent() -> None:"""The same model must count identically whichever spelling arrives first."""TokenizerRegistry.clear_cache()