Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Accept a multimodal processor in get_chat_template and construct_chat_template (#10152)
Aditya Malikba7a539a545c
diff --git a/tests/test_construct_chat_template_processor.py b/tests/test_construct_chat_template_processor.pynew file mode 100644index 00000000000..f8cf07c3bd8--- /dev/null+++ b/tests/test_construct_chat_template_processor.py@@ -0,0 +1,113 @@+"""construct_chat_template accepts a multimodal processor.++A multimodal checkpoint loads as a processor, which carries the text tokenizer in+`.tokenizer`. construct_chat_template is tokenizer-shaped throughout -- `get_vocab()`,+`name_or_path`, `bos_token`, and calling the object on a string -- and a processor has+none of that, so `apply_chat_template(dataset, tokenizer = processor, ...)`, which+reaches it, died on `vocab = tokenizer.get_vocab()`.++It now unwraps once at the top. Unlike get_chat_template there is nothing to re-attach:+this returns a template tuple, never the tokenizer. The unwrap therefore only has to+happen before the first tokenizer-shaped use, which is what the ordering test pins.++Importing unsloth needs a GPU, so the statement is pulled out of the source with ast and+run over stand-ins, as tests/test_map_eos_token.py does.+"""++import ast+import os++CHAT_TEMPLATES_PATH = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "chat_templates.py",+)++_BODY = next(+ node.body+ for node in ast.parse(open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()).body+ if isinstance(node, ast.FunctionDef) and node.name == "construct_chat_template"+)+++def _unwrap_branch():+ """The one top-level `if ...:` that rebinds tokenizer to its `.tokenizer`."""+ found = [+ node+ for node in _BODY+ if isinstance(node, ast.If)+ and any(+ isinstance(stmt, ast.Assign)+ and any(getattr(target, "id", None) == "tokenizer" for target in stmt.targets)+ for stmt in node.body+ )+ ]+ assert len(found) == 1, "could not find the processor unwrap in construct_chat_template"+ return found[0]+++def _run(tokenizer):+ namespace = {"tokenizer": tokenizer, "ProcessorMixin": _ProcessorMixin}+ exec(+ compile(ast.Module(body = [_unwrap_branch()], type_ignores = []), CHAT_TEMPLATES_PATH, "exec"),+ namespace,+ )+ return namespace["tokenizer"]+++class _ProcessorMixin:+ pass+++class _FakeTokenizer:+ """A tokenizer as this function uses one: it can produce a vocab, and has no `.tokenizer`."""++ def get_vocab(self):+ return {"<eos>": 0}+++class _FakeProcessor(_ProcessorMixin):+ """A processor as this function sees one: a `.tokenizer`, and no get_vocab."""++ def __init__(self, tokenizer):+ self.tokenizer = tokenizer+++class _FakeTokenizerBackend(_FakeTokenizer):+ """A tokenizer backend may expose `.tokenizer` without being a processor."""++ def __init__(self):+ self.tokenizer = object()+++def test_processor_is_unwrapped_to_its_inner_tokenizer():+ tokenizer = _FakeTokenizer()+ unwrapped = _run(_FakeProcessor(tokenizer))+ assert unwrapped is tokenizer+ # The get_vocab() call that used to raise now lands on the tokenizer.+ assert unwrapped.get_vocab() == {"<eos>": 0}+++def test_a_plain_tokenizer_is_left_alone():+ tokenizer = _FakeTokenizer()+ assert _run(tokenizer) is tokenizer+++def test_a_tokenizer_backend_with_a_tokenizer_attribute_is_left_alone():+ tokenizer = _FakeTokenizerBackend()+ assert _run(tokenizer) is tokenizer+++def test_the_unwrap_precedes_every_tokenizer_shaped_use():+ # get_vocab() is the first of them and the one that raised. If the unwrap ever drifts+ # below it, the processor reaches get_vocab again and the AttributeError is back.+ unwrap = _unwrap_branch()+ uses = [+ node.lineno+ for node in ast.walk(ast.Module(body = _BODY, type_ignores = []))+ if isinstance(node, ast.Attribute)+ and getattr(node.value, "id", None) == "tokenizer"+ and node.attr in ("get_vocab", "name_or_path", "bos_token", "eos_token")+ ]+ assert uses, "construct_chat_template no longer uses the tokenizer as a tokenizer"+ assert unwrap.lineno < min(uses)diff --git a/tests/test_get_chat_template_processor.py b/tests/test_get_chat_template_processor.pynew file mode 100644index 00000000000..7e0e3447356--- /dev/null+++ b/tests/test_get_chat_template_processor.py@@ -0,0 +1,194 @@+"""get_chat_template accepts a multimodal processor. See issue #10146.++gemma-4-E2B-it and other multimodal checkpoints load as a processor, not a tokenizer:+the text tokenizer sits in `processor.tokenizer`. On MLX the processor has no+padding_side of its own, so `old_padding_side = tokenizer.padding_side` raised+AttributeError before the function did anything. (On CUDA the loader sets padding_side+on the processor at models/vision.py, which is why this only bit Apple Silicon, but the+processor still has no Rust backend, so the vocab-editing templates broke there too.)++It now unwraps the processor, works on the inner tokenizer and re-attaches on return.+Three parts of that are quiet if broken: `old_tokenizer` must be bound after the unwrap,+or the pad/bos/unk restore near the end reads None off the processor and blanks all+three; the processor must come back carrying the new chat_template, since+ProcessorMixin.apply_chat_template renders off its own attribute; and the bos/eos/pad+copies the loader mirrored onto the processor must be refreshed, since chatml and+gemma_chatml rebuild the tokenizer with a remapped eos.++Importing unsloth needs a GPU, so the statements are pulled out of the source with ast+and run over stand-ins, as tests/test_map_eos_token.py does.+"""++import ast+import os++CHAT_TEMPLATES_PATH = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "chat_templates.py",+)++_BODY = next(+ node.body+ for node in ast.parse(open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()).body+ if isinstance(node, ast.FunctionDef) and node.name == "get_chat_template"+)+++def _only(found, what):+ assert len(found) == 1, f"could not find {what} in get_chat_template"+ return found[0]+++def _assign(name):+ """The one top-level `<name> = ...` in get_chat_template."""+ return _only(+ [+ node+ for node in _BODY+ if isinstance(node, ast.Assign)+ and any(getattr(target, "id", None) == name for target in node.targets)+ ],+ f"the {name} binding",+ )+++def _unwrap_branch():+ """The top-level `if ...` whose body claims the processor."""+ return _only(+ [+ node+ for node in _BODY+ if isinstance(node, ast.If)+ and any(+ isinstance(stmt, ast.Assign)+ and any(getattr(target, "id", None) == "_processor" for target in stmt.targets)+ for stmt in node.body+ )+ ],+ "the processor unwrap",+ )+++def _reattach_branch():+ """The top-level `if _processor is not None:` that restores it before return."""+ return _only(+ [+ node+ for node in _BODY+ if isinstance(node, ast.If) and "_processor" in ast.dump(node.test)+ ],+ "the processor re-attach",+ )+++def _run(statements, **namespace):+ namespace.setdefault("ProcessorMixin", _ProcessorMixin)+ exec(+ compile(ast.Module(body = statements, type_ignores = []), CHAT_TEMPLATES_PATH, "exec"),+ namespace,+ )+ return namespace+++class _ProcessorMixin:+ pass+++class _FakeTokenizer:+ """A tokenizer as this function sees one: special tokens, no `.tokenizer`."""++ def __init__(+ self,+ eos_token = "<eos>",+ eos_token_id = 1,+ ):+ self.padding_side = "right"+ self.bos_token, self.bos_token_id = "<bos>", 0+ self.eos_token, self.eos_token_id = eos_token, eos_token_id+ self.pad_token, self.pad_token_id = "<pad>", 2+++class _FakeProcessor(_ProcessorMixin):+ """A processor as this function sees one: a `.tokenizer`, plus the loader's mirror."""++ def __init__(self, tokenizer):+ self.tokenizer = tokenizer+ for token in ("bos_token", "eos_token", "pad_token"):+ setattr(self, token, getattr(tokenizer, token))+ setattr(self, token + "_id", getattr(tokenizer, token + "_id"))+++class _FakeTokenizerBackend(_FakeTokenizer):+ """A tokenizer backend may expose `.tokenizer` without being a processor."""++ def __init__(self):+ super().__init__()+ self.tokenizer = object()+++def _unwrap(tokenizer):+ return _run(+ [_assign("_processor"), _unwrap_branch(), _assign("old_tokenizer")],+ tokenizer = tokenizer,+ )+++def _reattach(processor, tokenizer):+ return _run(+ [_reattach_branch()],+ _processor = processor,+ tokenizer = tokenizer,+ chat_template = "TEMPLATE",+ )+++def test_processor_is_unwrapped_to_its_inner_tokenizer():+ tokenizer = _FakeTokenizer()+ namespace = _unwrap(_FakeProcessor(tokenizer))+ assert namespace["tokenizer"] is tokenizer+ # The padding_side read that used to raise now lands on the tokenizer.+ assert namespace["tokenizer"].padding_side == "right"+++def test_old_tokenizer_is_the_inner_tokenizer_not_the_processor():+ # getattr(processor, "pad_token", None) is None on MLX, so binding old_tokenizer+ # before the unwrap makes the restore at the end blank pad/bos/unk on the tokenizer.+ tokenizer = _FakeTokenizer()+ assert _unwrap(_FakeProcessor(tokenizer))["old_tokenizer"] is tokenizer+++def test_a_plain_tokenizer_is_left_alone():+ tokenizer = _FakeTokenizer()+ namespace = _unwrap(tokenizer)+ assert namespace["_processor"] is None+ assert namespace["tokenizer"] is tokenizer+ assert namespace["old_tokenizer"] is tokenizer+++def test_a_tokenizer_backend_with_a_tokenizer_attribute_is_left_alone():+ tokenizer = _FakeTokenizerBackend()+ namespace = _unwrap(tokenizer)+ assert namespace["_processor"] is None+ assert namespace["tokenizer"] is tokenizer+ assert namespace["old_tokenizer"] is tokenizer+++def test_processor_is_returned_carrying_the_new_tokenizer_and_template():+ processor = _FakeProcessor(_FakeTokenizer())+ rebuilt = _FakeTokenizer() # the vocab surgery replaces the tokenizer object+ namespace = _reattach(processor, rebuilt)+ assert namespace["tokenizer"] is processor+ assert processor.tokenizer is rebuilt+ assert processor.chat_template == "TEMPLATE"+++def test_a_remapped_eos_reaches_the_processors_mirrored_copy():+ # chatml/gemma_chatml rebuild the tokenizer with eos remapped to the stop word. The+ # loader copied the old eos onto the processor, and that copy is what the collators+ # and save paths read, so it has to follow the rebuild.+ processor = _FakeProcessor(_FakeTokenizer(eos_token = "<eos>", eos_token_id = 1))+ rebuilt = _FakeTokenizer(eos_token = "<|im_end|>", eos_token_id = 107)+ _reattach(processor, rebuilt)+ assert processor.eos_token == "<|im_end|>"+ assert processor.eos_token_id == 107diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex 15d081e5c08..82ae96f6259 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -27,6 +27,8 @@ __all__ = ["test_construct_chat_template",]+from transformers import ProcessorMixin+from transformers.utils import loggingtry:from torch import LongTensor, FloatTensor@@ -1894,7 +1896,6 @@ def get_chat_template(is_mlx_backend = getattr(sys.modules.get("unsloth"), "DEVICE_TYPE", None) == "mlx"if use_zoo_tokenizer_patch is None:use_zoo_tokenizer_patch = is_mlx_backend- old_tokenizer = tokenizer# mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust backend the vocab-edit# paths below need; unwrap here, re-wrap before return.@@ -1905,6 +1906,21 @@ def get_chat_template(_mlx_tokenizer_wrapper = tokenizertokenizer = _inner_tokenizer+ # Multimodal checkpoints (gemma-4-E2B, Llava, Qwen-VL, ...) load as a processor,+ # which carries the text tokenizer in `.tokenizer` -- the same unwrap the save and+ # loader paths do. Everything below needs the tokenizer itself: the processor never+ # has the Rust backend the vocab edits use, nor, on MLX, a padding_side of its own.+ # Match the processor type rather than the attribute: MistralCommonBackend is itself+ # a tokenizer but also exposes its non-HF backend as `.tokenizer`.+ _processor = None+ if isinstance(tokenizer, ProcessorMixin):+ _processor = tokenizer+ tokenizer = tokenizer.tokenizer++ # Bind after unwrapping: the pad/bos/unk restore below reads it, and a+ # processor answers None for all three.+ old_tokenizer = tokenizer+IS_GEMMA = Falseif tokenizer.__class__.__name__.startswith("Gemma"):if chat_template == "chatml": chat_template = "gemma_chatml"@@ -2115,9 +2131,22 @@ def get_chat_template(if old_pad_token != new_pad_token: tokenizer.pad_token = old_pad_token+ # Hand the processor back, not the tokenizer it was carrying: it renders chat+ # templates off its own attribute, and the vocab edits above may have rebuilt the+ # tokenizer and remapped eos. The loader mirrors these onto the processor+ # (models/vision.py), so refresh them here or that copy goes stale.+ if _processor is not None:+ _processor.tokenizer = tokenizer+ _processor.chat_template = chat_template+ for _token in ("bos_token", "eos_token", "pad_token",):+ if hasattr(tokenizer, _token):+ setattr(_processor, _token, getattr(tokenizer, _token))+ setattr(_processor, _token + "_id", getattr(tokenizer, _token + "_id"))+ tokenizer = _processor+if patch_saving and not is_mlx_backend:from .save import patch_saving_functions- tokenizer = patch_saving_functions(tokenizer)+ tokenizer = patch_saving_functions(tokenizer, vision = _processor is not None)tokenizer._ollama_modelfile = ollama_modelfiletokenizer._system_message = system_message@@ -2413,6 +2442,14 @@ extra_eos_tokens = None,assert(tokenizer is not None)+ # A multimodal checkpoint hands over a processor, which carries the text tokenizer+ # in `.tokenizer`. Everything below is tokenizer-shaped -- get_vocab, name_or_path,+ # calling it on a string -- and this returns a template, never the tokenizer, so+ # unwrap one way and there is nothing to re-attach. Match ProcessorMixin explicitly:+ # tokenizer backends such as MistralCommonBackend also expose `.tokenizer`.+ if isinstance(tokenizer, ProcessorMixin):+ tokenizer = tokenizer.tokenizer+if extra_eos_tokens is None: extra_eos_tokens = []elif type(extra_eos_tokens) is str: extra_eos_tokens = [extra_eos_tokens,]original_extra_eos_tokens = list(extra_eos_tokens)