Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): honor registered model factories (#4437)

Ehsan-hashfe01961b29da
diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.py
index d057474f9..fe89bc28c 100644
--- a/headroom/tokenizers/registry.py
+++ b/headroom/tokenizers/registry.py
@@ -141,6 +141,9 @@ class TokenizerRegistry:
# Registered factories (backend -> factory function)
_factories: dict[str, Callable[[str], TokenCounter]] = {}
+ # Registered model factories (model -> factory function)
+ _model_factories: dict[str, Callable[[str], TokenCounter]] = {}
+
# Cache for auto-detected tokenizers
_cache: dict[str, TokenCounter] = {}
@@ -163,6 +166,7 @@ class TokenizerRegistry:
"moonshot": self._create_moonshot,
"estimation": self._create_estimation,
}
+ self._model_factories = {}
@classmethod
def get(
@@ -197,9 +201,14 @@ class TokenizerRegistry:
if cache_key in registry._cache:
return registry._cache[cache_key]
+ uses_model_factory = backend is None and model_lower in registry._model_factories
+
# Create tokenizer
try:
- tokenizer = registry._create_tokenizer(model, backend)
+ if uses_model_factory:
+ tokenizer = registry._model_factories[model_lower](model)
+ else:
+ tokenizer = registry._create_tokenizer(model, backend)
registry._cache[cache_key] = tokenizer
return tokenizer
except Exception as e:
@@ -208,7 +217,8 @@ class TokenizerRegistry:
f"Failed to create tokenizer for {model}: {e}. Falling back to estimation."
)
tokenizer = EstimatingTokenCounter()
- registry._cache[cache_key] = tokenizer
+ if not uses_model_factory:
+ registry._cache[cache_key] = tokenizer
return tokenizer
raise ValueError(f"No tokenizer available for {model}: {e}") from e
@@ -234,8 +244,10 @@ class TokenizerRegistry:
if tokenizer is not None:
registry._tokenizers[model_lower] = tokenizer
+ registry._model_factories.pop(model_lower, None)
elif factory is not None:
- registry._factories[model_lower] = factory
+ registry._model_factories[model_lower] = factory
+ registry._tokenizers.pop(model_lower, None)
else:
raise ValueError("Must provide either tokenizer or factory")
@@ -269,7 +281,7 @@ class TokenizerRegistry:
def list_registered(cls) -> list[str]:
"""List explicitly registered models."""
registry = cls()
- return list(registry._tokenizers.keys())
+ return list(dict.fromkeys((*registry._tokenizers, *registry._model_factories)))
@classmethod
def clear_cache(cls) -> None:
diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py
index 38d9e0236..d77927c0e 100644
--- a/tests/test_tokenizers.py
+++ b/tests/test_tokenizers.py
@@ -412,6 +412,117 @@ class TestTokenizerRegistry:
retrieved = get_tokenizer("my-custom-model")
assert retrieved is custom
+ def test_register_custom_factory_is_lazy_case_insensitive_and_replaceable(self, monkeypatch):
+ """Model factories should override auto detection without polluting backends."""
+ registry = TokenizerRegistry()
+ for attr in ("_tokenizers", "_factories", "_cache"):
+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())
+ monkeypatch.setattr(
+ registry,
+ "_model_factories",
+ getattr(registry, "_model_factories", {}).copy(),
+ raising=False,
+ )
+
+ model = "Custom-Factory-Model"
+ calls: list[str] = []
+ custom = CharacterCounter(chars_per_token=1.0)
+
+ def factory(requested_model: str) -> CharacterCounter:
+ calls.append(requested_model)
+ return custom
+
+ register_tokenizer(model, factory=factory)
+
+ assert model.lower() not in registry.list_backends()
+ first = get_tokenizer(model.upper())
+ assert first is custom
+ assert calls == [model.upper()]
+ assert get_tokenizer(model.lower()) is first
+ assert calls == [model.upper()]
+
+ forced = get_tokenizer(model, backend="estimation")
+ assert isinstance(forced, EstimatingTokenCounter)
+ assert calls == [model.upper()]
+
+ replacement = EstimatingTokenCounter(chars_per_token=2.0)
+ register_tokenizer(model, tokenizer=replacement)
+ assert get_tokenizer(model.lower()) is replacement
+
+ replacement_calls: list[str] = []
+ replacement_custom = CharacterCounter(chars_per_token=2.0)
+
+ def replacement_factory(requested_model: str) -> CharacterCounter:
+ replacement_calls.append(requested_model)
+ return replacement_custom
+
+ register_tokenizer(model.lower(), factory=replacement_factory)
+ recreated = get_tokenizer(model)
+ assert recreated is replacement_custom
+ assert replacement_calls == [model]
+ assert get_tokenizer(model.upper()) is recreated
+ assert replacement_calls == [model]
+
+ def test_model_factory_failure_does_not_poison_strict_or_recovery(self, monkeypatch):
+ """A fallback from a model factory must not hide later retries."""
+ registry = TokenizerRegistry()
+ for attr in ("_tokenizers", "_factories", "_cache"):
+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())
+ monkeypatch.setattr(
+ registry,
+ "_model_factories",
+ getattr(registry, "_model_factories", {}).copy(),
+ raising=False,
+ )
+
+ model = "Transient-Factory-Model"
+ state = {"available": False}
+ calls: list[str] = []
+ custom = CharacterCounter(chars_per_token=1.0)
+
+ def factory(requested_model: str) -> CharacterCounter:
+ calls.append(requested_model)
+ if not state["available"]:
+ raise RuntimeError("temporary factory outage")
+ return custom
+
+ register_tokenizer(model, factory=factory)
+
+ first = get_tokenizer(model)
+ with pytest.raises(ValueError, match="No tokenizer available"):
+ get_tokenizer(model, fallback=False)
+
+ state["available"] = True
+ recovered = get_tokenizer(model, fallback=False)
+
+ assert isinstance(first, EstimatingTokenCounter)
+ assert recovered is custom
+ assert calls == [model, model, model]
+
+ def test_list_registered_includes_model_factories_and_replacements(self, monkeypatch):
+ """Registered model names should include both instances and factories."""
+ registry = TokenizerRegistry()
+ for attr in ("_tokenizers", "_factories", "_cache"):
+ monkeypatch.setattr(registry, attr, getattr(registry, attr).copy())
+ monkeypatch.setattr(
+ registry,
+ "_model_factories",
+ getattr(registry, "_model_factories", {}).copy(),
+ raising=False,
+ )
+
+ model = "Listed-Factory-Model"
+ register_tokenizer(model, factory=lambda _: CharacterCounter(chars_per_token=1.0))
+ assert model.lower() in registry.list_registered()
+
+ register_tokenizer(model, tokenizer=CharacterCounter(chars_per_token=2.0))
+ assert model.lower() in registry.list_registered()
+
+ register_tokenizer(model, factory=lambda _: CharacterCounter(chars_per_token=3.0))
+ registered = registry.list_registered()
+ assert model.lower() in registered
+ assert registered.count(model.lower()) == 1
+
def test_list_supported_models(self):
"""Test listing supported models."""
models = list_supported_models()