Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cache the native-template markup profile per tokenizer in renderable_tool_catalog (#17288)

Daniel Handdba6f722d0e
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 6233cd62d9d..0faa49c188f 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -1534,14 +1534,8 @@ def renderable_tool_catalog(
return safe if active_renders_tools else _unadvertised()
if not active_renders_tools and not _template_reads_tools(native_tpl, tools):
return _unadvertised()
- # Include special tokens too: native render resolves bos_token; else a dropped tool stays
- # listed.
- native = model_markup(
- native_tpl,
- _vocabulary_of(tokenizer),
- tools,
- specials = _special_token_strings(getattr(tokenizer, "tokenizer", tokenizer)),
- )
+ # Profiled with special tokens: the native render resolves bos_token.
+ native = _native_markup_for_tokenizer(tokenizer, native_tpl, tools)
if native is None:
return safe
kept = catalog_tool_names(neutralize_tool_descriptions(tools, cache, native))
@@ -2337,6 +2331,68 @@ def _special_token_strings(tokenizer) -> dict:
return specials
+_NATIVE_MARKUP_BY_TOKENIZER: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()
+
+
+def _template_key(template):
+ """Hashable form of a template; a named-template dict is not."""
+ if isinstance(template, str):
+ return template
+ try:
+ return json.dumps(template, sort_keys = True, default = str)
+ except (TypeError, ValueError):
+ return repr(template)
+
+
+def _cached_markup(store, tokenizer, key, build):
+ """build() memoised per tokenizer under *key*, at most 4 keys each; unweakrefable tokenizers are not cached."""
+ try:
+ entry = store.get(tokenizer)
+ except TypeError:
+ entry = None
+ if isinstance(entry, dict):
+ hit = entry.get(key, _UNPARSED)
+ if hit is not _UNPARSED:
+ return hit
+ profile = build()
+ try:
+ entry = entry if isinstance(entry, dict) else {}
+ if len(entry) >= 4:
+ entry.clear()
+ entry[key] = profile
+ store[tokenizer] = entry
+ except TypeError:
+ pass
+ return profile
+
+
+def _vocabulary_size_key(inner):
+ # Not len(): on a fast tokenizer it builds the whole vocabulary map, the cost this cache avoids.
+ try:
+ return (inner.vocab_size, len(getattr(inner, "added_tokens_decoder", None) or ()))
+ except Exception:
+ return None
+
+
+def _native_markup_for_tokenizer(tokenizer, native_tpl, tools) -> Optional[ModelMarkup]:
+ """model_markup of the native fallback template, which walks the whole vocabulary, cached per tokenizer.
+ model_markup reads tools only for truthiness."""
+ inner = getattr(tokenizer, "tokenizer", tokenizer)
+ specials = _special_token_strings(inner)
+ key = (
+ _template_key(native_tpl),
+ bool(tools),
+ _vocabulary_size_key(inner),
+ tuple(sorted(specials.items())),
+ )
+ return _cached_markup(
+ _NATIVE_MARKUP_BY_TOKENIZER,
+ tokenizer,
+ key,
+ lambda: model_markup(native_tpl, _vocabulary_of(tokenizer), tools, specials = specials),
+ )
+
+
def markup_for_tokenizer(
tokenizer,
tools = None,
@@ -2345,10 +2401,6 @@ def markup_for_tokenizer(
"""Profile tokenizer markers, cached per tokenizer; None when unreadable, so curated patterns apply."""
if tokenizer is None:
return None
- try:
- cached = _MARKUP_BY_TOKENIZER.get(tokenizer)
- except TypeError: # not weak-referenceable
- cached = None
# Vision models: chat_template on the processor, vocabulary on the inner tokenizer.
inner = getattr(tokenizer, "tokenizer", tokenizer)
# An explicit template is what this request renders with; load-time profiling lagged it (#7066).
@@ -2356,47 +2408,24 @@ def markup_for_tokenizer(
template = getattr(tokenizer, "chat_template", None)
if not template:
template = getattr(inner, "chat_template", None)
- # Keyed on template too (a mapped template lands on the same object at generate time) and on tools presence, since named dicts emit different literals.
is_processor = _is_processor(tokenizer)
# A processor always renders default, so tools do not change its profile or the cache key.
selector = bool(tools) and not is_processor
- # A named-template dict is unhashable, so the key is a stable serialization of it; without this the cache write
- # raised TypeError and every call rebuilt the profile.
- if not isinstance(template, str):
- try:
- template_key = json.dumps(template, sort_keys = True, default = str)
- except (TypeError, ValueError):
- template_key = repr(template)
- else:
- template_key = template
- if isinstance(cached, dict):
- hit = cached.get((template_key, selector), _UNPARSED)
- if hit is not _UNPARSED:
- return hit
- else:
- cached = None
- tokens = None
- tokens = _tokenizer_strings(inner)
# ProcessorMixin.apply_chat_template does NOT switch to "tool_use" implicitly; it renders "default" unless
# chat_template= names another. Profiling with the tokenizer rule left a processor's own default-template boundary
# unswept while the render emitted it (#7066).
- profile = model_markup(
- template,
- tokens,
- tools,
- prefer_tool_use = not is_processor,
- specials = _special_token_strings(inner),
+ return _cached_markup(
+ _MARKUP_BY_TOKENIZER,
+ tokenizer,
+ (_template_key(template), selector),
+ lambda: model_markup(
+ template,
+ _tokenizer_strings(inner),
+ tools,
+ prefer_tool_use = not is_processor,
+ specials = _special_token_strings(inner),
+ ),
)
- try:
- entry = cached if isinstance(cached, dict) else {}
- # Two selectors per template; trim if a tokenizer somehow cycles templates.
- if len(entry) >= 4:
- entry.clear()
- entry[(template_key, selector)] = profile
- _MARKUP_BY_TOKENIZER[tokenizer] = entry
- except TypeError:
- pass
- return profile
def trailing_assistant_text(messages: list) -> Optional[str]:
diff --git a/studio/backend/tests/test_native_markup_cache.py b/studio/backend/tests/test_native_markup_cache.py
new file mode 100644
index 00000000000..e828dec1b6e
--- /dev/null
+++ b/studio/backend/tests/test_native_markup_cache.py
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The native-template profile in renderable_tool_catalog is cached per tokenizer, not rebuilt per request."""
+
+import core.inference.chat_template_helpers as helpers
+from core.inference.chat_template_helpers import (
+ catalog_tool_names,
+ model_markup,
+ renderable_tool_catalog,
+)
+
+_NATIVE = (
+ "{% for t in tools %}{{ t }}{% endfor %}"
+ "{% for m in messages %}<zeta_turn>{{ m }}<|im_end|>{% endfor %}"
+)
+
+
+class _Tok:
+ chat_template = (
+ "{% for t in tools %}{{ t }}{% endfor %}{% for m in messages %}{{ m }}{% endfor %}"
+ )
+ eos_token = "<|im_end|>"
+
+ def __init__(self, vocab):
+ self.vocab = list(vocab)
+ self.added_tokens_decoder: dict = {}
+
+ def get_vocab(self):
+ return {token: i for i, token in enumerate(self.vocab)}
+
+ @property
+ def vocab_size(self):
+ return len(self.vocab)
+
+
+def _tool(name, prop = "x"):
+ return {
+ "type": "function",
+ "function": {
+ "name": name,
+ "description": "d",
+ "parameters": {"type": "object", "properties": {prop: {"type": "string"}}},
+ },
+ }
+
+
+def _native_calls(monkeypatch):
+ calls = []
+ real = helpers.model_markup
+
+ def counting(chat_template, *args, **kwargs):
+ if chat_template is not _Tok.chat_template:
+ calls.append(chat_template)
+ return real(chat_template, *args, **kwargs)
+
+ monkeypatch.setattr(helpers, "model_markup", counting)
+ return calls
+
+
+def _fields(markup):
+ if markup is None:
+ return None
+ return (
+ sorted(markup.markers),
+ markup.control and markup.control.pattern,
+ markup.boundary and markup.boundary.pattern,
+ markup.selected_with_tools,
+ )
+
+
+def test_native_markup_is_built_once_for_repeated_requests(monkeypatch):
+ calls = _native_calls(monkeypatch)
+ tok = _Tok(["a", "<zeta_turn>", "<|im_end|>"])
+ info = {"native_chat_template": _NATIVE}
+ tools = [_tool("ok"), _tool("bad", prop = "<zeta_turn>")]
+ first = renderable_tool_catalog(tools, tok, info)
+ second = renderable_tool_catalog([_tool("other"), _tool("ok")], tok, info)
+ assert catalog_tool_names(first) == {"ok"}
+ assert catalog_tool_names(second) == {"other", "ok"}
+ assert len(calls) == 1
+ fresh = model_markup(
+ _NATIVE,
+ helpers._vocabulary_of(tok),
+ tools,
+ specials = helpers._special_token_strings(tok),
+ )
+ assert _fields(helpers._native_markup_for_tokenizer(tok, _NATIVE, tools)) == _fields(fresh)
+
+
+def test_a_different_template_or_vocabulary_is_not_served_from_the_cache(monkeypatch):
+ calls = _native_calls(monkeypatch)
+ tok = _Tok(["a", "<|im_end|>"])
+ tools = [_tool("ok"), _tool("bad", prop = "<zeta_turn>")]
+ # "<zeta_turn>" is structure only once the vocabulary holds it.
+ before = renderable_tool_catalog(tools, tok, {"native_chat_template": _NATIVE})
+ assert catalog_tool_names(before) == {"ok", "bad"}
+ tok.vocab.append("<zeta_turn>")
+ after = renderable_tool_catalog(tools, tok, {"native_chat_template": _NATIVE})
+ assert catalog_tool_names(after) == {"ok"}
+ other = _NATIVE.replace("<zeta_turn>", "<|im_start|>")
+ renderable_tool_catalog(tools, tok, {"native_chat_template": other})
+ assert calls == [_NATIVE, _NATIVE, other]
+ named = {"default": _NATIVE, "tool_use": _NATIVE}
+ renamed = {"default": _NATIVE, "tool_use": other}
+ renderable_tool_catalog(tools, tok, {"native_chat_template": named})
+ renderable_tool_catalog(tools, tok, {"native_chat_template": renamed})
+ assert calls[3:] == [named, renamed]
+ renderable_tool_catalog(tools, _Tok(["a", "<|im_end|>"]), {"native_chat_template": _NATIVE})
+ assert len(calls) == 6