Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Accept a multimodal processor in get_chat_template and construct_chat_template (#10152)

Aditya Malikba7a539a545c
diff --git a/tests/test_construct_chat_template_processor.py b/tests/test_construct_chat_template_processor.py
new file mode 100644
index 00000000000..f8cf07c3bd8
--- /dev/null
+++ b/tests/test_construct_chat_template_processor.py
@@ -0,0 +1,113 @@
+"""construct_chat_template accepts a multimodal processor.
+
+A multimodal checkpoint loads as a processor, which carries the text tokenizer in
+`.tokenizer`. construct_chat_template is tokenizer-shaped throughout -- `get_vocab()`,
+`name_or_path`, `bos_token`, and calling the object on a string -- and a processor has
+none of that, so `apply_chat_template(dataset, tokenizer = processor, ...)`, which
+reaches it, died on `vocab = tokenizer.get_vocab()`.
+
+It now unwraps once at the top. Unlike get_chat_template there is nothing to re-attach:
+this returns a template tuple, never the tokenizer. The unwrap therefore only has to
+happen before the first tokenizer-shaped use, which is what the ordering test pins.
+
+Importing unsloth needs a GPU, so the statement is pulled out of the source with ast and
+run over stand-ins, as tests/test_map_eos_token.py does.
+"""
+
+import ast
+import os
+
+CHAT_TEMPLATES_PATH = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "chat_templates.py",
+)
+
+_BODY = next(
+ node.body
+ for node in ast.parse(open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()).body
+ if isinstance(node, ast.FunctionDef) and node.name == "construct_chat_template"
+)
+
+
+def _unwrap_branch():
+ """The one top-level `if ...:` that rebinds tokenizer to its `.tokenizer`."""
+ found = [
+ node
+ for node in _BODY
+ if isinstance(node, ast.If)
+ and any(
+ isinstance(stmt, ast.Assign)
+ and any(getattr(target, "id", None) == "tokenizer" for target in stmt.targets)
+ for stmt in node.body
+ )
+ ]
+ assert len(found) == 1, "could not find the processor unwrap in construct_chat_template"
+ return found[0]
+
+
+def _run(tokenizer):
+ namespace = {"tokenizer": tokenizer, "ProcessorMixin": _ProcessorMixin}
+ exec(
+ compile(ast.Module(body = [_unwrap_branch()], type_ignores = []), CHAT_TEMPLATES_PATH, "exec"),
+ namespace,
+ )
+ return namespace["tokenizer"]
+
+
+class _ProcessorMixin:
+ pass
+
+
+class _FakeTokenizer:
+ """A tokenizer as this function uses one: it can produce a vocab, and has no `.tokenizer`."""
+
+ def get_vocab(self):
+ return {"<eos>": 0}
+
+
+class _FakeProcessor(_ProcessorMixin):
+ """A processor as this function sees one: a `.tokenizer`, and no get_vocab."""
+
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+
+
+class _FakeTokenizerBackend(_FakeTokenizer):
+ """A tokenizer backend may expose `.tokenizer` without being a processor."""
+
+ def __init__(self):
+ self.tokenizer = object()
+
+
+def test_processor_is_unwrapped_to_its_inner_tokenizer():
+ tokenizer = _FakeTokenizer()
+ unwrapped = _run(_FakeProcessor(tokenizer))
+ assert unwrapped is tokenizer
+ # The get_vocab() call that used to raise now lands on the tokenizer.
+ assert unwrapped.get_vocab() == {"<eos>": 0}
+
+
+def test_a_plain_tokenizer_is_left_alone():
+ tokenizer = _FakeTokenizer()
+ assert _run(tokenizer) is tokenizer
+
+
+def test_a_tokenizer_backend_with_a_tokenizer_attribute_is_left_alone():
+ tokenizer = _FakeTokenizerBackend()
+ assert _run(tokenizer) is tokenizer
+
+
+def test_the_unwrap_precedes_every_tokenizer_shaped_use():
+ # get_vocab() is the first of them and the one that raised. If the unwrap ever drifts
+ # below it, the processor reaches get_vocab again and the AttributeError is back.
+ unwrap = _unwrap_branch()
+ uses = [
+ node.lineno
+ for node in ast.walk(ast.Module(body = _BODY, type_ignores = []))
+ if isinstance(node, ast.Attribute)
+ and getattr(node.value, "id", None) == "tokenizer"
+ and node.attr in ("get_vocab", "name_or_path", "bos_token", "eos_token")
+ ]
+ assert uses, "construct_chat_template no longer uses the tokenizer as a tokenizer"
+ assert unwrap.lineno < min(uses)
diff --git a/tests/test_get_chat_template_processor.py b/tests/test_get_chat_template_processor.py
new file mode 100644
index 00000000000..7e0e3447356
--- /dev/null
+++ b/tests/test_get_chat_template_processor.py
@@ -0,0 +1,194 @@
+"""get_chat_template accepts a multimodal processor. See issue #10146.
+
+gemma-4-E2B-it and other multimodal checkpoints load as a processor, not a tokenizer:
+the text tokenizer sits in `processor.tokenizer`. On MLX the processor has no
+padding_side of its own, so `old_padding_side = tokenizer.padding_side` raised
+AttributeError before the function did anything. (On CUDA the loader sets padding_side
+on the processor at models/vision.py, which is why this only bit Apple Silicon, but the
+processor still has no Rust backend, so the vocab-editing templates broke there too.)
+
+It now unwraps the processor, works on the inner tokenizer and re-attaches on return.
+Three parts of that are quiet if broken: `old_tokenizer` must be bound after the unwrap,
+or the pad/bos/unk restore near the end reads None off the processor and blanks all
+three; the processor must come back carrying the new chat_template, since
+ProcessorMixin.apply_chat_template renders off its own attribute; and the bos/eos/pad
+copies the loader mirrored onto the processor must be refreshed, since chatml and
+gemma_chatml rebuild the tokenizer with a remapped eos.
+
+Importing unsloth needs a GPU, so the statements are pulled out of the source with ast
+and run over stand-ins, as tests/test_map_eos_token.py does.
+"""
+
+import ast
+import os
+
+CHAT_TEMPLATES_PATH = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "chat_templates.py",
+)
+
+_BODY = next(
+ node.body
+ for node in ast.parse(open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()).body
+ if isinstance(node, ast.FunctionDef) and node.name == "get_chat_template"
+)
+
+
+def _only(found, what):
+ assert len(found) == 1, f"could not find {what} in get_chat_template"
+ return found[0]
+
+
+def _assign(name):
+ """The one top-level `<name> = ...` in get_chat_template."""
+ return _only(
+ [
+ node
+ for node in _BODY
+ if isinstance(node, ast.Assign)
+ and any(getattr(target, "id", None) == name for target in node.targets)
+ ],
+ f"the {name} binding",
+ )
+
+
+def _unwrap_branch():
+ """The top-level `if ...` whose body claims the processor."""
+ return _only(
+ [
+ node
+ for node in _BODY
+ if isinstance(node, ast.If)
+ and any(
+ isinstance(stmt, ast.Assign)
+ and any(getattr(target, "id", None) == "_processor" for target in stmt.targets)
+ for stmt in node.body
+ )
+ ],
+ "the processor unwrap",
+ )
+
+
+def _reattach_branch():
+ """The top-level `if _processor is not None:` that restores it before return."""
+ return _only(
+ [
+ node
+ for node in _BODY
+ if isinstance(node, ast.If) and "_processor" in ast.dump(node.test)
+ ],
+ "the processor re-attach",
+ )
+
+
+def _run(statements, **namespace):
+ namespace.setdefault("ProcessorMixin", _ProcessorMixin)
+ exec(
+ compile(ast.Module(body = statements, type_ignores = []), CHAT_TEMPLATES_PATH, "exec"),
+ namespace,
+ )
+ return namespace
+
+
+class _ProcessorMixin:
+ pass
+
+
+class _FakeTokenizer:
+ """A tokenizer as this function sees one: special tokens, no `.tokenizer`."""
+
+ def __init__(
+ self,
+ eos_token = "<eos>",
+ eos_token_id = 1,
+ ):
+ self.padding_side = "right"
+ self.bos_token, self.bos_token_id = "<bos>", 0
+ self.eos_token, self.eos_token_id = eos_token, eos_token_id
+ self.pad_token, self.pad_token_id = "<pad>", 2
+
+
+class _FakeProcessor(_ProcessorMixin):
+ """A processor as this function sees one: a `.tokenizer`, plus the loader's mirror."""
+
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+ for token in ("bos_token", "eos_token", "pad_token"):
+ setattr(self, token, getattr(tokenizer, token))
+ setattr(self, token + "_id", getattr(tokenizer, token + "_id"))
+
+
+class _FakeTokenizerBackend(_FakeTokenizer):
+ """A tokenizer backend may expose `.tokenizer` without being a processor."""
+
+ def __init__(self):
+ super().__init__()
+ self.tokenizer = object()
+
+
+def _unwrap(tokenizer):
+ return _run(
+ [_assign("_processor"), _unwrap_branch(), _assign("old_tokenizer")],
+ tokenizer = tokenizer,
+ )
+
+
+def _reattach(processor, tokenizer):
+ return _run(
+ [_reattach_branch()],
+ _processor = processor,
+ tokenizer = tokenizer,
+ chat_template = "TEMPLATE",
+ )
+
+
+def test_processor_is_unwrapped_to_its_inner_tokenizer():
+ tokenizer = _FakeTokenizer()
+ namespace = _unwrap(_FakeProcessor(tokenizer))
+ assert namespace["tokenizer"] is tokenizer
+ # The padding_side read that used to raise now lands on the tokenizer.
+ assert namespace["tokenizer"].padding_side == "right"
+
+
+def test_old_tokenizer_is_the_inner_tokenizer_not_the_processor():
+ # getattr(processor, "pad_token", None) is None on MLX, so binding old_tokenizer
+ # before the unwrap makes the restore at the end blank pad/bos/unk on the tokenizer.
+ tokenizer = _FakeTokenizer()
+ assert _unwrap(_FakeProcessor(tokenizer))["old_tokenizer"] is tokenizer
+
+
+def test_a_plain_tokenizer_is_left_alone():
+ tokenizer = _FakeTokenizer()
+ namespace = _unwrap(tokenizer)
+ assert namespace["_processor"] is None
+ assert namespace["tokenizer"] is tokenizer
+ assert namespace["old_tokenizer"] is tokenizer
+
+
+def test_a_tokenizer_backend_with_a_tokenizer_attribute_is_left_alone():
+ tokenizer = _FakeTokenizerBackend()
+ namespace = _unwrap(tokenizer)
+ assert namespace["_processor"] is None
+ assert namespace["tokenizer"] is tokenizer
+ assert namespace["old_tokenizer"] is tokenizer
+
+
+def test_processor_is_returned_carrying_the_new_tokenizer_and_template():
+ processor = _FakeProcessor(_FakeTokenizer())
+ rebuilt = _FakeTokenizer() # the vocab surgery replaces the tokenizer object
+ namespace = _reattach(processor, rebuilt)
+ assert namespace["tokenizer"] is processor
+ assert processor.tokenizer is rebuilt
+ assert processor.chat_template == "TEMPLATE"
+
+
+def test_a_remapped_eos_reaches_the_processors_mirrored_copy():
+ # chatml/gemma_chatml rebuild the tokenizer with eos remapped to the stop word. The
+ # loader copied the old eos onto the processor, and that copy is what the collators
+ # and save paths read, so it has to follow the rebuild.
+ processor = _FakeProcessor(_FakeTokenizer(eos_token = "<eos>", eos_token_id = 1))
+ rebuilt = _FakeTokenizer(eos_token = "<|im_end|>", eos_token_id = 107)
+ _reattach(processor, rebuilt)
+ assert processor.eos_token == "<|im_end|>"
+ assert processor.eos_token_id == 107
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 15d081e5c08..82ae96f6259 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -27,6 +27,8 @@ __all__ = [
"test_construct_chat_template",
]
+from transformers import ProcessorMixin
+
from transformers.utils import logging
try:
from torch import LongTensor, FloatTensor
@@ -1894,7 +1896,6 @@ def get_chat_template(
is_mlx_backend = getattr(sys.modules.get("unsloth"), "DEVICE_TYPE", None) == "mlx"
if use_zoo_tokenizer_patch is None:
use_zoo_tokenizer_patch = is_mlx_backend
- old_tokenizer = tokenizer
# mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust backend the vocab-edit
# paths below need; unwrap here, re-wrap before return.
@@ -1905,6 +1906,21 @@ def get_chat_template(
_mlx_tokenizer_wrapper = tokenizer
tokenizer = _inner_tokenizer
+ # Multimodal checkpoints (gemma-4-E2B, Llava, Qwen-VL, ...) load as a processor,
+ # which carries the text tokenizer in `.tokenizer` -- the same unwrap the save and
+ # loader paths do. Everything below needs the tokenizer itself: the processor never
+ # has the Rust backend the vocab edits use, nor, on MLX, a padding_side of its own.
+ # Match the processor type rather than the attribute: MistralCommonBackend is itself
+ # a tokenizer but also exposes its non-HF backend as `.tokenizer`.
+ _processor = None
+ if isinstance(tokenizer, ProcessorMixin):
+ _processor = tokenizer
+ tokenizer = tokenizer.tokenizer
+
+ # Bind after unwrapping: the pad/bos/unk restore below reads it, and a
+ # processor answers None for all three.
+ old_tokenizer = tokenizer
+
IS_GEMMA = False
if tokenizer.__class__.__name__.startswith("Gemma"):
if chat_template == "chatml": chat_template = "gemma_chatml"
@@ -2115,9 +2131,22 @@ def get_chat_template(
if old_pad_token != new_pad_token: tokenizer.pad_token = old_pad_token
+ # Hand the processor back, not the tokenizer it was carrying: it renders chat
+ # templates off its own attribute, and the vocab edits above may have rebuilt the
+ # tokenizer and remapped eos. The loader mirrors these onto the processor
+ # (models/vision.py), so refresh them here or that copy goes stale.
+ if _processor is not None:
+ _processor.tokenizer = tokenizer
+ _processor.chat_template = chat_template
+ for _token in ("bos_token", "eos_token", "pad_token",):
+ if hasattr(tokenizer, _token):
+ setattr(_processor, _token, getattr(tokenizer, _token))
+ setattr(_processor, _token + "_id", getattr(tokenizer, _token + "_id"))
+ tokenizer = _processor
+
if patch_saving and not is_mlx_backend:
from .save import patch_saving_functions
- tokenizer = patch_saving_functions(tokenizer)
+ tokenizer = patch_saving_functions(tokenizer, vision = _processor is not None)
tokenizer._ollama_modelfile = ollama_modelfile
tokenizer._system_message = system_message
@@ -2413,6 +2442,14 @@ extra_eos_tokens = None,
assert(tokenizer is not None)
+ # A multimodal checkpoint hands over a processor, which carries the text tokenizer
+ # in `.tokenizer`. Everything below is tokenizer-shaped -- get_vocab, name_or_path,
+ # calling it on a string -- and this returns a template, never the tokenizer, so
+ # unwrap one way and there is nothing to re-attach. Match ProcessorMixin explicitly:
+ # tokenizer backends such as MistralCommonBackend also expose `.tokenizer`.
+ if isinstance(tokenizer, ProcessorMixin):
+ tokenizer = tokenizer.tokenizer
+
if extra_eos_tokens is None: extra_eos_tokens = []
elif type(extra_eos_tokens) is str: extra_eos_tokens = [extra_eos_tokens,]
original_extra_eos_tokens = list(extra_eos_tokens)