Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(providers/openai): bound tiktoken vocab loads with the guarded loader (#2554)
gglucass0805e8e41054
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex 5b41bc8e1..891b13559 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -32,12 +32,7 @@ _UNKNOWN_MODEL_WARNINGS: set[str] = set()# Models whose price came from the built-in table rather than LiteLLM._PRICING_FALLBACK_WARNINGS: set[str] = set()-try:- import tiktoken-- TIKTOKEN_AVAILABLE = True-except ImportError:- TIKTOKEN_AVAILABLE = False+TIKTOKEN_AVAILABLE = importlib.util.find_spec("tiktoken") is not NoneLITELLM_AVAILABLE = importlib.util.find_spec("litellm") is not None@@ -285,12 +280,21 @@ def _check_pricing_staleness() -> str | None:@lru_cache(maxsize=8)def _get_encoding(encoding_name: str) -> Any:- """Get tiktoken encoding, cached."""+ """Get tiktoken encoding, cached.++ Routes through the bounded loader so a stalled vocab download raises+ :class:`~headroom.tokenizers.tiktoken_counter.TiktokenLoadError` after a+ timeout instead of hanging the caller indefinitely — ``tiktoken`` fetches+ vocabularies with no network timeout, and this runs on whatever thread+ first counts tokens for a model, including proxy startup (GH #956).+ """if not TIKTOKEN_AVAILABLE:raise RuntimeError("tiktoken is required for OpenAI provider. Install with: pip install tiktoken")- return tiktoken.get_encoding(encoding_name)+ from ..tokenizers.tiktoken_counter import load_encoding++ return load_encoding(encoding_name)def _lookup_encoding_name(model: str, custom_encodings: dict[str, str] | None = None) -> str | None:@@ -343,6 +347,8 @@ class OpenAITokenCounter:Raises:RuntimeError: If tiktoken is not installed.+ TiktokenLoadError: If the encoding's vocabulary can't be loaded+ within the bounded timeout (e.g. stalled download)."""self.model = modelencoding_name = _get_encoding_name_for_model(model, custom_encodings)@@ -507,7 +513,9 @@ class OpenAIProvider(Provider):the proxy pipeline resolves its tokenizer through this provider whilehandlers resolve through the tokenizer registry, the two disagree aboutthe same request — savings become a difference of two rulers. Defer to- the registry so each model has exactly one tokenizer.+ the registry so each model has exactly one tokenizer. For OpenAI models,+ fall back to estimation when a vocabulary cannot load within the bounded+ timeout; the cached fallback prevents subsequent requests from blocking."""if model not in self._token_counters:if _lookup_encoding_name(model, self._encodings) is None:@@ -515,9 +523,19 @@ class OpenAIProvider(Provider):self._token_counters[model] = cast(Any, get_tokenizer(model))else:- self._token_counters[model] = OpenAITokenCounter(- model=model, custom_encodings=self._encodings- )+ from ..tokenizers.tiktoken_counter import TiktokenLoadError++ try:+ self._token_counters[model] = OpenAITokenCounter(+ model=model, custom_encodings=self._encodings+ )+ except TiktokenLoadError as exc:+ logger.warning(+ "tiktoken unavailable for %s (%s); using estimation.", model, exc+ )+ from ..tokenizers.estimator import EstimatingTokenCounter++ self._token_counters[model] = EstimatingTokenCounter()return self._token_counters[model]def get_context_limit(self, model: str) -> int:diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.pyindex 3b5857447..6d26993d4 100644--- a/tests/test_providers/test_openai.py+++ b/tests/test_providers/test_openai.py@@ -130,3 +130,52 @@ class TestEncodingSelection:# Unknown models now get a fallback encoding instead of raisingencoding = _get_encoding_name_for_model("completely-unknown")assert encoding == "o200k_base" # Default fallback+++class TestGuardedEncodingLoad:+ """The provider must never hang on tiktoken's unbounded vocab download.++ Regression for the OpenAI-provider hole in GH #956: `_get_encoding` called+ `tiktoken.get_encoding` directly, so a stalled vocab download blocked the+ calling thread (proxy startup included) forever instead of timing out.+ """++ @pytest.fixture(autouse=True)+ def _clear_encoding_cache(self):+ from headroom.providers import openai as openai_module++ openai_module._get_encoding.cache_clear()+ yield+ openai_module._get_encoding.cache_clear()++ def test_get_encoding_routes_through_bounded_loader(self, monkeypatch):+ from headroom.providers.openai import OpenAITokenCounter+ from headroom.tokenizers import tiktoken_counter++ seen: list[str] = []++ def fake_load_encoding(name: str):+ seen.append(name)+ raise tiktoken_counter.TiktokenLoadError(f"{name} load timed out")++ monkeypatch.setattr(tiktoken_counter, "load_encoding", fake_load_encoding)+ with pytest.raises(tiktoken_counter.TiktokenLoadError):+ OpenAITokenCounter(model="gpt-4o")+ assert seen == ["o200k_base"]++ def test_get_token_counter_falls_back_to_estimation(self, monkeypatch):+ from headroom.providers.openai import OpenAIProvider+ from headroom.tokenizers import tiktoken_counter+ from headroom.tokenizers.estimator import EstimatingTokenCounter++ def fake_load_encoding(name: str):+ raise tiktoken_counter.TiktokenLoadError(f"{name} load timed out")++ monkeypatch.setattr(tiktoken_counter, "load_encoding", fake_load_encoding)+ provider = OpenAIProvider()+ counter = provider.get_token_counter("gpt-4o")+ assert isinstance(counter, EstimatingTokenCounter)+ assert counter.count_text("hello world") > 0+ # Cached per model: later requests reuse the fallback instead of+ # re-blocking on the failed download.+ assert provider.get_token_counter("gpt-4o") is counter