Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(providers/openai): bound tiktoken vocab loads with the guarded loader (#2554)

gglucass0805e8e41054
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index 5b41bc8e1..891b13559 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -32,12 +32,7 @@ _UNKNOWN_MODEL_WARNINGS: set[str] = set()
# Models whose price came from the built-in table rather than LiteLLM.
_PRICING_FALLBACK_WARNINGS: set[str] = set()
-try:
- import tiktoken
-
- TIKTOKEN_AVAILABLE = True
-except ImportError:
- TIKTOKEN_AVAILABLE = False
+TIKTOKEN_AVAILABLE = importlib.util.find_spec("tiktoken") is not None
LITELLM_AVAILABLE = importlib.util.find_spec("litellm") is not None
@@ -285,12 +280,21 @@ def _check_pricing_staleness() -> str | None:
@lru_cache(maxsize=8)
def _get_encoding(encoding_name: str) -> Any:
- """Get tiktoken encoding, cached."""
+ """Get tiktoken encoding, cached.
+
+ Routes through the bounded loader so a stalled vocab download raises
+ :class:`~headroom.tokenizers.tiktoken_counter.TiktokenLoadError` after a
+ timeout instead of hanging the caller indefinitely — ``tiktoken`` fetches
+ vocabularies with no network timeout, and this runs on whatever thread
+ first counts tokens for a model, including proxy startup (GH #956).
+ """
if not TIKTOKEN_AVAILABLE:
raise RuntimeError(
"tiktoken is required for OpenAI provider. Install with: pip install tiktoken"
)
- return tiktoken.get_encoding(encoding_name)
+ from ..tokenizers.tiktoken_counter import load_encoding
+
+ return load_encoding(encoding_name)
def _lookup_encoding_name(model: str, custom_encodings: dict[str, str] | None = None) -> str | None:
@@ -343,6 +347,8 @@ class OpenAITokenCounter:
Raises:
RuntimeError: If tiktoken is not installed.
+ TiktokenLoadError: If the encoding's vocabulary can't be loaded
+ within the bounded timeout (e.g. stalled download).
"""
self.model = model
encoding_name = _get_encoding_name_for_model(model, custom_encodings)
@@ -507,7 +513,9 @@ class OpenAIProvider(Provider):
the proxy pipeline resolves its tokenizer through this provider while
handlers resolve through the tokenizer registry, the two disagree about
the same request — savings become a difference of two rulers. Defer to
- the registry so each model has exactly one tokenizer.
+ the registry so each model has exactly one tokenizer. For OpenAI models,
+ fall back to estimation when a vocabulary cannot load within the bounded
+ timeout; the cached fallback prevents subsequent requests from blocking.
"""
if model not in self._token_counters:
if _lookup_encoding_name(model, self._encodings) is None:
@@ -515,9 +523,19 @@ class OpenAIProvider(Provider):
self._token_counters[model] = cast(Any, get_tokenizer(model))
else:
- self._token_counters[model] = OpenAITokenCounter(
- model=model, custom_encodings=self._encodings
- )
+ from ..tokenizers.tiktoken_counter import TiktokenLoadError
+
+ try:
+ self._token_counters[model] = OpenAITokenCounter(
+ model=model, custom_encodings=self._encodings
+ )
+ except TiktokenLoadError as exc:
+ logger.warning(
+ "tiktoken unavailable for %s (%s); using estimation.", model, exc
+ )
+ from ..tokenizers.estimator import EstimatingTokenCounter
+
+ self._token_counters[model] = EstimatingTokenCounter()
return self._token_counters[model]
def get_context_limit(self, model: str) -> int:
diff --git a/tests/test_providers/test_openai.py b/tests/test_providers/test_openai.py
index 3b5857447..6d26993d4 100644
--- a/tests/test_providers/test_openai.py
+++ b/tests/test_providers/test_openai.py
@@ -130,3 +130,52 @@ class TestEncodingSelection:
# Unknown models now get a fallback encoding instead of raising
encoding = _get_encoding_name_for_model("completely-unknown")
assert encoding == "o200k_base" # Default fallback
+
+
+class TestGuardedEncodingLoad:
+ """The provider must never hang on tiktoken's unbounded vocab download.
+
+ Regression for the OpenAI-provider hole in GH #956: `_get_encoding` called
+ `tiktoken.get_encoding` directly, so a stalled vocab download blocked the
+ calling thread (proxy startup included) forever instead of timing out.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _clear_encoding_cache(self):
+ from headroom.providers import openai as openai_module
+
+ openai_module._get_encoding.cache_clear()
+ yield
+ openai_module._get_encoding.cache_clear()
+
+ def test_get_encoding_routes_through_bounded_loader(self, monkeypatch):
+ from headroom.providers.openai import OpenAITokenCounter
+ from headroom.tokenizers import tiktoken_counter
+
+ seen: list[str] = []
+
+ def fake_load_encoding(name: str):
+ seen.append(name)
+ raise tiktoken_counter.TiktokenLoadError(f"{name} load timed out")
+
+ monkeypatch.setattr(tiktoken_counter, "load_encoding", fake_load_encoding)
+ with pytest.raises(tiktoken_counter.TiktokenLoadError):
+ OpenAITokenCounter(model="gpt-4o")
+ assert seen == ["o200k_base"]
+
+ def test_get_token_counter_falls_back_to_estimation(self, monkeypatch):
+ from headroom.providers.openai import OpenAIProvider
+ from headroom.tokenizers import tiktoken_counter
+ from headroom.tokenizers.estimator import EstimatingTokenCounter
+
+ def fake_load_encoding(name: str):
+ raise tiktoken_counter.TiktokenLoadError(f"{name} load timed out")
+
+ monkeypatch.setattr(tiktoken_counter, "load_encoding", fake_load_encoding)
+ provider = OpenAIProvider()
+ counter = provider.get_token_counter("gpt-4o")
+ assert isinstance(counter, EstimatingTokenCounter)
+ assert counter.count_text("hello world") > 0
+ # Cached per model: later requests reuse the fallback instead of
+ # re-blocking on the failed download.
+ assert provider.get_token_counter("gpt-4o") is counter