Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): honor registered model factories (#4437)
Ehsan-hashfe01961b29da
diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.pyindex d057474f9..fe89bc28c 100644--- a/headroom/tokenizers/registry.py+++ b/headroom/tokenizers/registry.py@@ -141,6 +141,9 @@ class TokenizerRegistry:# Registered factories (backend -> factory function)_factories: dict[str, Callable[[str], TokenCounter]] = {}+ # Registered model factories (model -> factory function)+ _model_factories: dict[str, Callable[[str], TokenCounter]] = {}+# Cache for auto-detected tokenizers_cache: dict[str, TokenCounter] = {}@@ -163,6 +166,7 @@ class TokenizerRegistry:"moonshot": self._create_moonshot,"estimation": self._create_estimation,}+ self._model_factories = {}@classmethoddef get(@@ -197,9 +201,14 @@ class TokenizerRegistry:if cache_key in registry._cache:return registry._cache[cache_key]+ uses_model_factory = backend is None and model_lower in registry._model_factories+# Create tokenizertry:- tokenizer = registry._create_tokenizer(model, backend)+ if uses_model_factory:+ tokenizer = registry._model_factories[model_lower](model)+ else:+ tokenizer = registry._create_tokenizer(model, backend)registry._cache[cache_key] = tokenizerreturn tokenizerexcept Exception as e:@@ -208,7 +217,8 @@ class TokenizerRegistry:f"Failed to create tokenizer for {model}: {e}. Falling back to estimation.")tokenizer = EstimatingTokenCounter()- registry._cache[cache_key] = tokenizer+ if not uses_model_factory:+ registry._cache[cache_key] = tokenizerreturn tokenizerraise ValueError(f"No tokenizer available for {model}: {e}") from e@@ -234,8 +244,10 @@ class TokenizerRegistry:if tokenizer is not None:registry._tokenizers[model_lower] = tokenizer+ registry._model_factories.pop(model_lower, None)elif factory is not None:- registry._factories[model_lower] = factory+ registry._model_factories[model_lower] = factory+ registry._tokenizers.pop(model_lower, None)else:raise ValueError("Must provide either tokenizer or factory")@@ -269,7 +281,7 @@ class TokenizerRegistry:def list_registered(cls) -> list[str]:"""List explicitly registered models."""registry = cls()- return list(registry._tokenizers.keys())+ return list(dict.fromkeys((*registry._tokenizers, *registry._model_factories)))@classmethoddef clear_cache(cls) -> None:diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex 38d9e0236..d77927c0e 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -412,6 +412,117 @@ class TestTokenizerRegistry:retrieved = get_tokenizer("my-custom-model")assert retrieved is custom+ def test_register_custom_factory_is_lazy_case_insensitive_and_replaceable(self, monkeypatch):+ """Model factories should override auto detection without polluting backends."""+ registry = TokenizerRegistry()+ for attr in ("_tokenizers", "_factories", "_cache"):+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())+ monkeypatch.setattr(+ registry,+ "_model_factories",+ getattr(registry, "_model_factories", {}).copy(),+ raising=False,+ )++ model = "Custom-Factory-Model"+ calls: list[str] = []+ custom = CharacterCounter(chars_per_token=1.0)++ def factory(requested_model: str) -> CharacterCounter:+ calls.append(requested_model)+ return custom++ register_tokenizer(model, factory=factory)++ assert model.lower() not in registry.list_backends()+ first = get_tokenizer(model.upper())+ assert first is custom+ assert calls == [model.upper()]+ assert get_tokenizer(model.lower()) is first+ assert calls == [model.upper()]++ forced = get_tokenizer(model, backend="estimation")+ assert isinstance(forced, EstimatingTokenCounter)+ assert calls == [model.upper()]++ replacement = EstimatingTokenCounter(chars_per_token=2.0)+ register_tokenizer(model, tokenizer=replacement)+ assert get_tokenizer(model.lower()) is replacement++ replacement_calls: list[str] = []+ replacement_custom = CharacterCounter(chars_per_token=2.0)++ def replacement_factory(requested_model: str) -> CharacterCounter:+ replacement_calls.append(requested_model)+ return replacement_custom++ register_tokenizer(model.lower(), factory=replacement_factory)+ recreated = get_tokenizer(model)+ assert recreated is replacement_custom+ assert replacement_calls == [model]+ assert get_tokenizer(model.upper()) is recreated+ assert replacement_calls == [model]++ def test_model_factory_failure_does_not_poison_strict_or_recovery(self, monkeypatch):+ """A fallback from a model factory must not hide later retries."""+ registry = TokenizerRegistry()+ for attr in ("_tokenizers", "_factories", "_cache"):+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())+ monkeypatch.setattr(+ registry,+ "_model_factories",+ getattr(registry, "_model_factories", {}).copy(),+ raising=False,+ )++ model = "Transient-Factory-Model"+ state = {"available": False}+ calls: list[str] = []+ custom = CharacterCounter(chars_per_token=1.0)++ def factory(requested_model: str) -> CharacterCounter:+ calls.append(requested_model)+ if not state["available"]:+ raise RuntimeError("temporary factory outage")+ return custom++ register_tokenizer(model, factory=factory)++ first = get_tokenizer(model)+ with pytest.raises(ValueError, match="No tokenizer available"):+ get_tokenizer(model, fallback=False)++ state["available"] = True+ recovered = get_tokenizer(model, fallback=False)++ assert isinstance(first, EstimatingTokenCounter)+ assert recovered is custom+ assert calls == [model, model, model]++ def test_list_registered_includes_model_factories_and_replacements(self, monkeypatch):+ """Registered model names should include both instances and factories."""+ registry = TokenizerRegistry()+ for attr in ("_tokenizers", "_factories", "_cache"):+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())+ monkeypatch.setattr(+ registry,+ "_model_factories",+ getattr(registry, "_model_factories", {}).copy(),+ raising=False,+ )++ model = "Listed-Factory-Model"+ register_tokenizer(model, factory=lambda _: CharacterCounter(chars_per_token=1.0))+ assert model.lower() in registry.list_registered()++ register_tokenizer(model, tokenizer=CharacterCounter(chars_per_token=2.0))+ assert model.lower() in registry.list_registered()++ register_tokenizer(model, factory=lambda _: CharacterCounter(chars_per_token=3.0))+ registered = registry.list_registered()+ assert model.lower() in registered+ assert registered.count(model.lower()) == 1+def test_list_supported_models(self):"""Test listing supported models."""models = list_supported_models()