Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): bound tiktoken vocab load so a stalled download cannot hang requests (#956) (#994)
Eyal Mizrachi7e86bafb9004
diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.pyindex b34878f2d..3e341bd5d 100644--- a/headroom/providers/anthropic.py+++ b/headroom/providers/anthropic.py@@ -274,11 +274,18 @@ class AnthropicTokenCounter(TokenCounter):)_FALLBACK_WARNING_SHOWN = True- # Load tiktoken as fallback+ # Load tiktoken as fallback — bounded, so a stalled vocab download can't+ # hang token counting inside a request (tiktoken's downloader has no+ # network timeout); on timeout we estimate by characters instead (GH #956).try:- import tiktoken+ from headroom.tokenizers.tiktoken_counter import (+ TiktokenLoadError,+ load_encoding,+ )- self._encoding = tiktoken.get_encoding("cl100k_base")+ self._encoding = load_encoding("cl100k_base")+ except TiktokenLoadError:+ self._encoding = None # count_text() falls back to a character estimateexcept ImportError:if not self._use_api:warnings.warn(diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.pyindex 797f0b642..a2579c0a0 100644--- a/headroom/tokenizers/registry.py+++ b/headroom/tokenizers/registry.py@@ -287,10 +287,24 @@ class TokenizerRegistry:return "estimation"def _create_tiktoken(self, model: str) -> TokenCounter:- """Create tiktoken-based tokenizer."""+ """Create tiktoken-based tokenizer.++ Forces the (bounded) encoding load up front so a stalled vocab download+ falls back to estimation instead of hanging later inside a request (GH #956).+ """try:- from .tiktoken_counter import TiktokenCounter+ from .tiktoken_counter import (+ TiktokenCounter,+ TiktokenLoadError,+ get_encoding_for_model,+ load_encoding,+ )+ try:+ load_encoding(get_encoding_for_model(model))+ except TiktokenLoadError as exc:+ logger.warning("tiktoken unavailable (%s); using estimation.", exc)+ return EstimatingTokenCounter()return TiktokenCounter(model)except ImportError:logger.warning("tiktoken not installed. Install with: pip install tiktoken")diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex a3ccab025..a22fd9f9d 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -10,11 +10,38 @@ It supports multiple encodings:from __future__ import annotations+import logging+import os+import threadingfrom functools import lru_cachefrom typing import Anyfrom .base import BaseTokenizer+logger = logging.getLogger(__name__)+++class TiktokenLoadError(RuntimeError):+ """Raised when a tiktoken encoding can't be loaded in time.++ tiktoken downloads its BPE vocab on first use via ``requests.get`` with no+ timeout, so a stalled/firewalled connection can block indefinitely. We bound+ that load and raise this instead, so callers fall back to estimation rather+ than hanging the request (see GH #956).+ """+++# Encoding names whose bounded load already timed out — don't block on them again.+_load_failed: set[str] = set()+++def _load_timeout_seconds() -> float:+ try:+ return float(os.environ.get("HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS", "10"))+ except (TypeError, ValueError):+ return 10.0++# Model to encoding mappingMODEL_TO_ENCODING = {# GPT-4o family (o200k_base)@@ -78,10 +105,54 @@ DEFAULT_ENCODING = "cl100k_base"@lru_cache(maxsize=8)def _get_encoding(encoding_name: str):- """Get tiktoken encoding, cached for performance."""+ """Get a tiktoken encoding, cached for performance.++ Bounded by ``HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS`` (default 10s): tiktoken's+ vocab download has no network timeout, so we run the load on a worker thread+ and raise :class:`TiktokenLoadError` if it doesn't finish in time, letting+ callers fall back to estimation rather than hang the request (GH #956). The+ first timed-out encoding is remembered so later calls fail fast instead of+ re-blocking on every request.+ """import tiktoken- return tiktoken.get_encoding(encoding_name)+ if encoding_name in _load_failed:+ raise TiktokenLoadError(f"tiktoken encoding {encoding_name!r} previously failed to load")++ box: dict[str, Any] = {}++ def _load() -> None:+ try:+ box["enc"] = tiktoken.get_encoding(encoding_name)+ except BaseException as exc: # noqa: BLE001 - re-raised in the calling thread+ box["err"] = exc++ worker = threading.Thread(target=_load, name=f"tiktoken-load-{encoding_name}", daemon=True)+ worker.start()+ worker.join(_load_timeout_seconds())++ if worker.is_alive():+ _load_failed.add(encoding_name)+ logger.warning(+ "tiktoken encoding %r did not load within %.1fs (likely a stalled vocab "+ "download); falling back to token estimation. Pre-populate TIKTOKEN_CACHE_DIR "+ "or tune HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS.",+ encoding_name,+ _load_timeout_seconds(),+ )+ raise TiktokenLoadError(f"tiktoken encoding {encoding_name!r} load timed out")+ if "err" in box:+ raise box["err"]+ return box["enc"]+++def load_encoding(encoding_name: str) -> Any:+ """Public, bounded tiktoken-encoding loader.++ Returns the tiktoken encoding, or raises :class:`TiktokenLoadError` if the+ vocab can't be loaded within the timeout (see :func:`_get_encoding`, GH #956).+ """+ return _get_encoding(encoding_name)def get_encoding_for_model(model: str) -> str:diff --git a/tests/test_tokenizers/test_tiktoken_load_timeout.py b/tests/test_tokenizers/test_tiktoken_load_timeout.pynew file mode 100644index 000000000..4e1970f31--- /dev/null+++ b/tests/test_tokenizers/test_tiktoken_load_timeout.py@@ -0,0 +1,81 @@+"""tiktoken vocab loading must be bounded (GH #956).++tiktoken downloads its BPE vocab via ``requests.get`` with no timeout, so a+stalled/firewalled connection blocks indefinitely. The proxy calls this lazily+inside a request worker, so the only bound was the 30s compression timeout —+yielding "every request times out, 0 compression". The bounded loader caps the+wait and falls back to estimation instead.+"""++from __future__ import annotations++import time++import pytest++from headroom.tokenizers import tiktoken_counter as tc+from headroom.tokenizers.estimator import EstimatingTokenCounter+from headroom.tokenizers.registry import TokenizerRegistry++[email protected](autouse=True)+def _reset_encoding_state():+ tc._get_encoding.cache_clear()+ tc._load_failed.clear()+ yield+ tc._get_encoding.cache_clear()+ tc._load_failed.clear()+++def _stalled_get_encoding(_name: str):+ # Simulates tiktoken's unbounded network download stalling.+ time.sleep(2.0)+ return object()+++def test_load_encoding_is_bounded_on_stall(monkeypatch: pytest.MonkeyPatch) -> None:+ import tiktoken++ monkeypatch.setattr(tiktoken, "get_encoding", _stalled_get_encoding)+ monkeypatch.setenv("HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS", "0.2")++ start = time.perf_counter()+ with pytest.raises(tc.TiktokenLoadError):+ tc.load_encoding("stall-enc")+ elapsed = time.perf_counter() - start+ assert elapsed < 1.5, f"load was not bounded (took {elapsed:.2f}s vs the 2s stall)"+++def test_failed_encoding_short_circuits(monkeypatch: pytest.MonkeyPatch) -> None:+ import tiktoken++ monkeypatch.setattr(tiktoken, "get_encoding", _stalled_get_encoding)+ monkeypatch.setenv("HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS", "0.2")++ with pytest.raises(tc.TiktokenLoadError):+ tc.load_encoding("stall-enc-2")++ # A second request must fail instantly via the _load_failed short-circuit,+ # not wait out the timeout again (this is what makes it not "every request").+ start = time.perf_counter()+ with pytest.raises(tc.TiktokenLoadError):+ tc.load_encoding("stall-enc-2")+ assert time.perf_counter() - start < 0.1+++def test_fast_load_returns_encoding(monkeypatch: pytest.MonkeyPatch) -> None:+ import tiktoken++ sentinel = object()+ monkeypatch.setattr(tiktoken, "get_encoding", lambda _name: sentinel)+ assert tc.load_encoding("fast-enc") is sentinel+++def test_registry_falls_back_to_estimator_on_stall(monkeypatch: pytest.MonkeyPatch) -> None:+ import tiktoken++ monkeypatch.setattr(tiktoken, "get_encoding", _stalled_get_encoding)+ monkeypatch.setenv("HEADROOM_TIKTOKEN_LOAD_TIMEOUT_SECONDS", "0.2")++ counter = TokenizerRegistry()._create_tiktoken("gpt-4")+ assert isinstance(counter, EstimatingTokenCounter)