Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: apply the mapped chat template once per loaded tokenizer (#16377)
Daniel Han2f1cc712e9c1
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex dd6da42150d..71fcfad5b78 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -809,6 +809,19 @@ def _tokenize_prompt(return tokenizer(prompt, return_tensors = "pt", add_special_tokens = add_special_tokens)+_MAPPED_TEMPLATE_LOCK = threading.Lock()+++def _mapped_template_tokenizer(model_info, tokenizer, template_name):+ with _MAPPED_TEMPLATE_LOCK:+ cached = model_info.get("_mapped_template_tokenizer")+ if cached is not None and cached[0] is tokenizer and cached[1] == template_name:+ return cached[2]+ remapped = get_chat_template(tokenizer, chat_template = template_name)+ model_info["_mapped_template_tokenizer"] = (tokenizer, template_name, remapped)+ return remapped++_MISSING = object()@@ -1894,10 +1907,7 @@ class InferenceBackend:f"Applying chat template '{template_name}' for {self.active_model_name}")- tokenizer = get_chat_template(- tokenizer,- chat_template = template_name,- )+ tokenizer = _mapped_template_tokenizer(model_info, tokenizer, template_name)# The mapper installs the template only at generate time: UNION into the load-time cache.# Take marker strings from the mapped template but resolve ids on the original tokenizer.try:diff --git a/studio/backend/tests/test_mapped_template_tokenizer_cache.py b/studio/backend/tests/test_mapped_template_tokenizer_cache.pynew file mode 100644index 00000000000..2e25ab691ec--- /dev/null+++ b/studio/backend/tests/test_mapped_template_tokenizer_cache.py@@ -0,0 +1,208 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import sys+import threading+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++try:+ from core.inference import inference as inf_mod # noqa: E402+ from core.inference.inference import InferenceBackend # noqa: E402+except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent+ pytest.skip(+ f"full inference backend unavailable ({type(exc).__name__}: {exc})",+ allow_module_level = True,+ )++_CHATML = "{% for m in messages %}<|im_start|>{{m.role}}\n{{m.content}}<|im_end|>{% endfor %}"+_NAME = "unsloth/gemma-2-2b-it"+++class _FakeTokenizer:+ def __init__(+ self,+ eos_id,+ chat_template = "",+ token_ids = None,+ ):+ self.eos_token_id = eos_id+ self.chat_template = chat_template+ self.pad_token_id = eos_id+ self.unk_token_id = None+ self._ids = dict(token_ids or {})++ def convert_tokens_to_ids(self, tok):+ return self._ids.get(tok)+++def _backend(monkeypatch, model_info, get_chat_template):+ import utils.datasets as ds++ backend = InferenceBackend.__new__(InferenceBackend)+ backend.active_model_name = _NAME+ backend.models = {_NAME: model_info}+ monkeypatch.setattr(inf_mod, "get_chat_template", get_chat_template)+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "gemma2"}, raising = False)+ monkeypatch.setattr(backend, "_normalize_top_k", lambda k: k, raising = False)+ rendered_with = []++ def _render(tok, *a, **k):+ rendered_with.append(tok)+ return "PROMPT"++ monkeypatch.setattr(backend, "_apply_chat_template_for_generation", _render, raising = False)+ monkeypatch.setattr(backend, "generate_stream", lambda *a, **k: iter(()), raising = False)+ return backend, rendered_with+++def _send(backend, text = "hi"):+ list(backend._generate_chat_response_inner(messages = [{"role": "user", "content": text}]))+++class _Counter:+ def __init__(self, make):+ self.calls = []+ self.make = make++ def __call__(+ self,+ tok,+ chat_template = None,+ ):+ self.calls.append((tok, chat_template))+ return self.make(tok)+++def _remapped(tok):+ return _FakeTokenizer(1, chat_template = _CHATML, token_ids = {"<|im_end|>": 1})+++def test_template_applied_once_and_reused_across_messages(monkeypatch):+ orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})+ model_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [1]}+ gct = _Counter(_remapped)+ backend, rendered_with = _backend(monkeypatch, model_info, gct)++ for i in range(5):+ _send(backend, f"msg {i}")++ assert len(gct.calls) == 1+ assert gct.calls[0] == (orig, "gemma2")+ assert len(rendered_with) == 5+ assert all(t is rendered_with[0] for t in rendered_with)+ assert rendered_with[0] is not orig+ assert model_info["chat_turn_end_eos_ids"] == [1, 107]+++def test_eos_union_matches_uncached_repeat(monkeypatch):+ orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})+ cached_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [5]}+ backend, _ = _backend(monkeypatch, cached_info, _Counter(_remapped))+ for _ in range(3):+ _send(backend)++ fresh_ids = [5]+ for _ in range(3):+ info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": fresh_ids}+ backend.models = {_NAME: info}+ _send(backend)+ fresh_ids = info["chat_turn_end_eos_ids"]+ assert cached_info["chat_turn_end_eos_ids"] == fresh_ids == [1, 5, 107]+++def test_reload_and_tokenizer_swap_reapply(monkeypatch):+ orig = _FakeTokenizer(1)+ model_info = {"tokenizer": orig, "is_vision": False}+ gct = _Counter(_remapped)+ backend, rendered_with = _backend(monkeypatch, model_info, gct)+ _send(backend)+ _send(backend)+ assert len(gct.calls) == 1++ new_tok = _FakeTokenizer(1)+ model_info["tokenizer"] = new_tok+ _send(backend)+ assert len(gct.calls) == 2 and gct.calls[-1][0] is new_tok++ backend.models = {_NAME: {"tokenizer": orig, "is_vision": False}}+ _send(backend)+ assert len(gct.calls) == 3+ assert rendered_with[-1] is not rendered_with[0]+++def test_template_name_change_reapplies(monkeypatch):+ import utils.datasets as ds++ orig = _FakeTokenizer(1)+ model_info = {"tokenizer": orig, "is_vision": False}+ gct = _Counter(_remapped)+ backend, _ = _backend(monkeypatch, model_info, gct)+ _send(backend)+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "chatml"}, raising = False)+ _send(backend)+ _send(backend)+ assert [c[1] for c in gct.calls] == ["gemma2", "chatml"]+++def test_failure_is_not_cached_and_falls_back(monkeypatch):+ orig = _FakeTokenizer(1, chat_template = _CHATML)+ model_info = {"tokenizer": orig, "is_vision": False}+ state = {"fail": True}++ def make(tok):+ if state["fail"]:+ raise KeyError("qwen3.5")+ return _remapped(tok)++ gct = _Counter(make)+ backend, rendered_with = _backend(monkeypatch, model_info, gct)+ _send(backend)+ _send(backend)+ assert rendered_with == [orig, orig]+ assert len(gct.calls) == 2+ assert "_mapped_template_tokenizer" not in model_info+ state["fail"] = False+ _send(backend)+ _send(backend)+ assert len(gct.calls) == 3+ assert rendered_with[2] is rendered_with[3] is not orig+++def test_concurrent_first_messages_apply_once(monkeypatch):+ orig = _FakeTokenizer(1)+ model_info = {"tokenizer": orig, "is_vision": False}+ gate = threading.Event()++ def slow(tok):+ gate.wait(5)+ return _remapped(tok)++ gct = _Counter(slow)+ backend, rendered_with = _backend(monkeypatch, model_info, gct)+ threads = [threading.Thread(target = _send, args = (backend,)) for _ in range(4)]+ for t in threads:+ t.start()+ gate.set()+ for t in threads:+ t.join(10)+ assert len(gct.calls) == 1+ assert len(rendered_with) == 4 and all(t is rendered_with[0] for t in rendered_with)+++def test_unmapped_model_never_calls_get_chat_template(monkeypatch):+ import utils.datasets as ds++ orig = _FakeTokenizer(1, chat_template = _CHATML)+ model_info = {"tokenizer": orig, "is_vision": False}+ gct = _Counter(_remapped)+ backend, rendered_with = _backend(monkeypatch, model_info, gct)+ monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {}, raising = False)+ _send(backend)+ assert gct.calls == [] and rendered_with == [orig]+ assert "_mapped_template_tokenizer" not in model_info