Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: apply the mapped chat template once per loaded tokenizer (#16377)

Daniel Han2f1cc712e9c1
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index dd6da42150d..71fcfad5b78 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -809,6 +809,19 @@ def _tokenize_prompt(
return tokenizer(prompt, return_tensors = "pt", add_special_tokens = add_special_tokens)
+_MAPPED_TEMPLATE_LOCK = threading.Lock()
+
+
+def _mapped_template_tokenizer(model_info, tokenizer, template_name):
+ with _MAPPED_TEMPLATE_LOCK:
+ cached = model_info.get("_mapped_template_tokenizer")
+ if cached is not None and cached[0] is tokenizer and cached[1] == template_name:
+ return cached[2]
+ remapped = get_chat_template(tokenizer, chat_template = template_name)
+ model_info["_mapped_template_tokenizer"] = (tokenizer, template_name, remapped)
+ return remapped
+
+
_MISSING = object()
@@ -1894,10 +1907,7 @@ class InferenceBackend:
f"Applying chat template '{template_name}' for {self.active_model_name}"
)
- tokenizer = get_chat_template(
- tokenizer,
- chat_template = template_name,
- )
+ tokenizer = _mapped_template_tokenizer(model_info, tokenizer, template_name)
# The mapper installs the template only at generate time: UNION into the load-time cache.
# Take marker strings from the mapped template but resolve ids on the original tokenizer.
try:
diff --git a/studio/backend/tests/test_mapped_template_tokenizer_cache.py b/studio/backend/tests/test_mapped_template_tokenizer_cache.py
new file mode 100644
index 00000000000..2e25ab691ec
--- /dev/null
+++ b/studio/backend/tests/test_mapped_template_tokenizer_cache.py
@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+try:
+ from core.inference import inference as inf_mod # noqa: E402
+ from core.inference.inference import InferenceBackend # noqa: E402
+except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent
+ pytest.skip(
+ f"full inference backend unavailable ({type(exc).__name__}: {exc})",
+ allow_module_level = True,
+ )
+
+_CHATML = "{% for m in messages %}<|im_start|>{{m.role}}\n{{m.content}}<|im_end|>{% endfor %}"
+_NAME = "unsloth/gemma-2-2b-it"
+
+
+class _FakeTokenizer:
+ def __init__(
+ self,
+ eos_id,
+ chat_template = "",
+ token_ids = None,
+ ):
+ self.eos_token_id = eos_id
+ self.chat_template = chat_template
+ self.pad_token_id = eos_id
+ self.unk_token_id = None
+ self._ids = dict(token_ids or {})
+
+ def convert_tokens_to_ids(self, tok):
+ return self._ids.get(tok)
+
+
+def _backend(monkeypatch, model_info, get_chat_template):
+ import utils.datasets as ds
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend.active_model_name = _NAME
+ backend.models = {_NAME: model_info}
+ monkeypatch.setattr(inf_mod, "get_chat_template", get_chat_template)
+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "gemma2"}, raising = False)
+ monkeypatch.setattr(backend, "_normalize_top_k", lambda k: k, raising = False)
+ rendered_with = []
+
+ def _render(tok, *a, **k):
+ rendered_with.append(tok)
+ return "PROMPT"
+
+ monkeypatch.setattr(backend, "_apply_chat_template_for_generation", _render, raising = False)
+ monkeypatch.setattr(backend, "generate_stream", lambda *a, **k: iter(()), raising = False)
+ return backend, rendered_with
+
+
+def _send(backend, text = "hi"):
+ list(backend._generate_chat_response_inner(messages = [{"role": "user", "content": text}]))
+
+
+class _Counter:
+ def __init__(self, make):
+ self.calls = []
+ self.make = make
+
+ def __call__(
+ self,
+ tok,
+ chat_template = None,
+ ):
+ self.calls.append((tok, chat_template))
+ return self.make(tok)
+
+
+def _remapped(tok):
+ return _FakeTokenizer(1, chat_template = _CHATML, token_ids = {"<|im_end|>": 1})
+
+
+def test_template_applied_once_and_reused_across_messages(monkeypatch):
+ orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})
+ model_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [1]}
+ gct = _Counter(_remapped)
+ backend, rendered_with = _backend(monkeypatch, model_info, gct)
+
+ for i in range(5):
+ _send(backend, f"msg {i}")
+
+ assert len(gct.calls) == 1
+ assert gct.calls[0] == (orig, "gemma2")
+ assert len(rendered_with) == 5
+ assert all(t is rendered_with[0] for t in rendered_with)
+ assert rendered_with[0] is not orig
+ assert model_info["chat_turn_end_eos_ids"] == [1, 107]
+
+
+def test_eos_union_matches_uncached_repeat(monkeypatch):
+ orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})
+ cached_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [5]}
+ backend, _ = _backend(monkeypatch, cached_info, _Counter(_remapped))
+ for _ in range(3):
+ _send(backend)
+
+ fresh_ids = [5]
+ for _ in range(3):
+ info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": fresh_ids}
+ backend.models = {_NAME: info}
+ _send(backend)
+ fresh_ids = info["chat_turn_end_eos_ids"]
+ assert cached_info["chat_turn_end_eos_ids"] == fresh_ids == [1, 5, 107]
+
+
+def test_reload_and_tokenizer_swap_reapply(monkeypatch):
+ orig = _FakeTokenizer(1)
+ model_info = {"tokenizer": orig, "is_vision": False}
+ gct = _Counter(_remapped)
+ backend, rendered_with = _backend(monkeypatch, model_info, gct)
+ _send(backend)
+ _send(backend)
+ assert len(gct.calls) == 1
+
+ new_tok = _FakeTokenizer(1)
+ model_info["tokenizer"] = new_tok
+ _send(backend)
+ assert len(gct.calls) == 2 and gct.calls[-1][0] is new_tok
+
+ backend.models = {_NAME: {"tokenizer": orig, "is_vision": False}}
+ _send(backend)
+ assert len(gct.calls) == 3
+ assert rendered_with[-1] is not rendered_with[0]
+
+
+def test_template_name_change_reapplies(monkeypatch):
+ import utils.datasets as ds
+
+ orig = _FakeTokenizer(1)
+ model_info = {"tokenizer": orig, "is_vision": False}
+ gct = _Counter(_remapped)
+ backend, _ = _backend(monkeypatch, model_info, gct)
+ _send(backend)
+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "chatml"}, raising = False)
+ _send(backend)
+ _send(backend)
+ assert [c[1] for c in gct.calls] == ["gemma2", "chatml"]
+
+
+def test_failure_is_not_cached_and_falls_back(monkeypatch):
+ orig = _FakeTokenizer(1, chat_template = _CHATML)
+ model_info = {"tokenizer": orig, "is_vision": False}
+ state = {"fail": True}
+
+ def make(tok):
+ if state["fail"]:
+ raise KeyError("qwen3.5")
+ return _remapped(tok)
+
+ gct = _Counter(make)
+ backend, rendered_with = _backend(monkeypatch, model_info, gct)
+ _send(backend)
+ _send(backend)
+ assert rendered_with == [orig, orig]
+ assert len(gct.calls) == 2
+ assert "_mapped_template_tokenizer" not in model_info
+ state["fail"] = False
+ _send(backend)
+ _send(backend)
+ assert len(gct.calls) == 3
+ assert rendered_with[2] is rendered_with[3] is not orig
+
+
+def test_concurrent_first_messages_apply_once(monkeypatch):
+ orig = _FakeTokenizer(1)
+ model_info = {"tokenizer": orig, "is_vision": False}
+ gate = threading.Event()
+
+ def slow(tok):
+ gate.wait(5)
+ return _remapped(tok)
+
+ gct = _Counter(slow)
+ backend, rendered_with = _backend(monkeypatch, model_info, gct)
+ threads = [threading.Thread(target = _send, args = (backend,)) for _ in range(4)]
+ for t in threads:
+ t.start()
+ gate.set()
+ for t in threads:
+ t.join(10)
+ assert len(gct.calls) == 1
+ assert len(rendered_with) == 4 and all(t is rendered_with[0] for t in rendered_with)
+
+
+def test_unmapped_model_never_calls_get_chat_template(monkeypatch):
+ import utils.datasets as ds
+
+ orig = _FakeTokenizer(1, chat_template = _CHATML)
+ model_info = {"tokenizer": orig, "is_vision": False}
+ gct = _Counter(_remapped)
+ backend, rendered_with = _backend(monkeypatch, model_info, gct)
+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {}, raising = False)
+ _send(backend)
+ assert gct.calls == [] and rendered_with == [orig]
+ assert "_mapped_template_tokenizer" not in model_info