Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Skip the duplicate tokenizer load on transformers 5 when it can only copy the fast tokenizer (#17318)

Daniel Han4fd58f0e4add
diff --git a/tests/test_tokenizer_skip_unused_slow_load.py b/tests/test_tokenizer_skip_unused_slow_load.py
index 052c6025660..07670482d05 100644
--- a/tests/test_tokenizer_skip_unused_slow_load.py
+++ b/tests/test_tokenizer_skip_unused_slow_load.py
@@ -7,9 +7,15 @@ import types
from unittest.mock import patch
import pytest
+import transformers
import unsloth.tokenizer_utils as tu
+_needs_v5 = pytest.mark.skipif(
+ int(transformers.__version__.split(".")[0]) < 5,
+ reason = "transformers 4 loads a real slow tokenizer before the fast one",
+)
+
class _Tok:
def __init__(self, kind):
@@ -65,10 +71,183 @@ def test_early_return_skips_slow_tokenizer(name, fix_tokenizer):
@pytest.mark.parametrize("same", [True, False])
def test_compared_path_still_loads_slow_tokenizer(same):
result, calls, repaired = _run("unsloth/Llama-3.2-1B-Instruct", True, same)
- assert calls == ["slow", "fast"]
+ assert sorted(calls) == ["fast", "slow"]
assert repaired.count("slow") == 1
assert result.kind == ("fast" if same else "converted")
+class _V5Tok:
+ bos_token = "<s>"
+ all_special_tokens = ["<s>"]
+
+ def __init__(
+ self,
+ kind,
+ legacy = False,
+ sync_changes_ids = False,
+ ):
+ self.kind = kind
+ self.legacy = legacy
+ self.sync_changes_ids = sync_changes_ids
+ self.rebuilds = 0
+ self._add_bos_token = False
+
+ @property
+ def add_bos_token(self):
+ return self._add_bos_token
+
+ @add_bos_token.setter
+ def add_bos_token(self, value):
+ self._add_bos_token = value
+ self.rebuilds += 1
+
+ @property
+ def add_eos_token(self):
+ return False
+
+ @add_eos_token.setter
+ def add_eos_token(self, value):
+ self.rebuilds += 1
+
+ def __call__(self, text):
+ drop_bos = self.sync_changes_ids and self.rebuilds > 0
+ return types.SimpleNamespace(input_ids = [len(text)] + ([] if drop_bos else [1]))
+
+
+def _v5_class(name, module = "transformers.models.qwen2.tokenization_qwen2"):
+ return type(name, (_V5Tok,), {"__module__": module})
+
+
+def _run_v5(
+ cls,
+ monkeypatch,
+ v5 = True,
+ **tok_kwargs,
+):
+ monkeypatch.setattr(tu, "_TRANSFORMERS_V5", v5, raising = False)
+ calls, made = [], []
+
+ def from_pretrained(tokenizer_name, **kwargs):
+ kind = "slow" if kwargs.get("use_fast") is False else "fast"
+ calls.append(kind)
+ made.append(cls(kind, **tok_kwargs))
+ return made[-1]
+
+ with (
+ patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)),
+ patch.object(tu, "_repair_tokenizer_backend_from_json", lambda tok, **k: tok),
+ patch.object(tu, "convert_to_fast_tokenizer", lambda slow, **k: slow),
+ ):
+ result = tu._load_correct_tokenizer("org/model", fix_tokenizer = True)
+ return result, calls
+
+
+ "name",
+ [
+ "Qwen2Tokenizer",
+ "Qwen3_5Tokenizer",
+ "GemmaTokenizer",
+ "GPT2Tokenizer",
+ "CohereTokenizer",
+ "TokenizersBackend",
+ ],
+)
+def test_v5_skips_slow_load_that_copies_the_fast_tokenizer(name, monkeypatch):
+ result, calls = _run_v5(_v5_class(name), monkeypatch)
+ assert calls == [
+ "fast"
+ ], f"{name}: transformers 5 slow load only rebuilt the fast tokenizer ({calls})"
+ assert result.kind == "fast"
+ assert result.rebuilds == 2
+
+
+def test_v5_llama_skips_only_at_legacy_false(monkeypatch):
+ result, calls = _run_v5(_v5_class("LlamaTokenizer"), monkeypatch, legacy = False)
+ assert calls == ["fast"] and result.kind == "fast"
+ result, calls = _run_v5(_v5_class("LlamaTokenizer"), monkeypatch, legacy = True)
+ assert calls == ["fast", "slow"]
+
+
+def test_v5_sync_changing_ids_returns_the_slow_copy(monkeypatch):
+ result, calls = _run_v5(_v5_class("GemmaTokenizer"), monkeypatch, sync_changes_ids = True)
+ assert calls == ["fast", "slow"]
+ assert result.kind == "slow"
+
+
+ "cls, v5",
+ [
+ (_v5_class("Qwen2Tokenizer"), False),
+ (_v5_class("Qwen2Tokenizer", module = "transformers_modules.org.tokenization"), True),
+ (_v5_class("NllbTokenizer"), True),
+ ],
+)
+def test_slow_load_kept_when_it_may_differ(cls, v5, monkeypatch):
+ result, calls = _run_v5(cls, monkeypatch, v5 = v5)
+ assert sorted(calls) == ["fast", "slow"]
+ assert result.kind == "fast"
+
+
+@_needs_v5
+def test_v5_real_qwen2_tokenizer_constructed_once(tmp_path):
+ vocab = {c: i for i, c in enumerate("abcdefghijklmnopqrstuvwxyz\u0120!")}
+ transformers.Qwen2Tokenizer(
+ vocab = vocab, merges = [], eos_token = "<|endoftext|>", unk_token = None, bos_token = None
+ ).save_pretrained(tmp_path)
+ calls = []
+ original = tu.AutoTokenizer.from_pretrained
+
+ def from_pretrained(*args, **kwargs):
+ calls.append("slow" if kwargs.get("use_fast") is False else "fast")
+ return original(*args, **kwargs)
+
+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):
+ tokenizer = tu.load_correct_tokenizer(str(tmp_path))
+ assert calls == ["fast"]
+ assert type(tokenizer).__name__ == "Qwen2Tokenizer"
+ assert tokenizer("hello world!", add_special_tokens = False).input_ids == [
+ 7,
+ 4,
+ 11,
+ 11,
+ 14,
+ 26,
+ 22,
+ 14,
+ 17,
+ 11,
+ 3,
+ 27,
+ ]
+
+
+@_needs_v5
+def test_v5_failed_slow_load_keeps_the_tokenizer_as_loaded(tmp_path):
+ from tokenizers import processors
+
+ vocab = {c: i for i, c in enumerate("abcdefghijklmnopqrstuvwxyz\u0120!")}
+ vocab["<s>"] = len(vocab)
+ tokenizer = transformers.Qwen2Tokenizer(
+ vocab = vocab, merges = [], eos_token = "<|endoftext|>", unk_token = None, bos_token = "<s>"
+ )
+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(
+ single = "<s>:0 $A:0", pair = "<s>:0 $A:0 $B:1", special_tokens = [("<s>", vocab["<s>"])]
+ )
+ tokenizer.save_pretrained(tmp_path)
+ original = tu.AutoTokenizer.from_pretrained
+ expected = original(str(tmp_path))("ab").input_ids
+ assert expected[0] == vocab["<s>"]
+
+ def from_pretrained(*args, **kwargs):
+ if kwargs.get("use_fast") is False:
+ raise OSError("slow load failed")
+ return original(*args, **kwargs)
+
+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):
+ loaded = tu.load_correct_tokenizer(str(tmp_path))
+ assert loaded("ab").input_ids == expected
+
+
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-q"]))
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index cf5bb278703..ceef1e5e72f 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -12,6 +12,7 @@
from transformers import AutoTokenizer
from transformers.convert_slow_tokenizer import convert_slow_tokenizer
from transformers import PreTrainedTokenizerFast
+from transformers import __version__ as transformers_version
import re
import os
import json
@@ -757,9 +758,7 @@ llama_template = (
)
-def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
- if not hasattr(slow_tokenizer, "all_special_tokens"):
- return True
+def _same_tokenization_special_tokens(slow_tokenizer):
dir_names = dir(slow_tokenizer)
special_tokens = list(
filter(
@@ -774,7 +773,21 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))
replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")
- all_special_tokens = [x for x in all_special_tokens if x != replacement_char]
+ return [x for x in all_special_tokens if x != replacement_char]
+
+
+def _same_tokenization_text(all_special_tokens):
+ return (
+ "\n".join(all_special_tokens)
+ + "A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n"
+ + "".join(all_special_tokens)
+ )
+
+
+def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
+ if not hasattr(slow_tokenizer, "all_special_tokens"):
+ return True
+ all_special_tokens = _same_tokenization_special_tokens(slow_tokenizer)
check_chat_template1 = True
check_chat_template2 = True
@@ -820,11 +833,7 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3
try:
- string = (
- "\n".join(all_special_tokens)
- + "A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n"
- + "".join(all_special_tokens)
- )
+ string = _same_tokenization_text(all_special_tokens)
check_special_tokens = slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids
return check_chat_template and check_special_tokens
@@ -1022,6 +1031,67 @@ def fix_sentencepiece_gguf(saved_location):
return
+def _sync_add_special_tokens(fast_tokenizer, slow_tokenizer):
+ if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
+ fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
+ if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):
+ fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
+
+
+# v5 ignores use_fast/from_slow: the "slow" load is the fast one plus legacy = False, unread by these classes.
+_TRANSFORMERS_V5 = int(transformers_version.split(".")[0]) >= 5
+_V5_LEGACY_FREE_TOKENIZERS = frozenset(
+ (
+ "TokenizersBackend",
+ "Qwen2Tokenizer",
+ "Qwen3_5Tokenizer",
+ "GemmaTokenizer",
+ "GPT2Tokenizer",
+ "CohereTokenizer",
+ )
+)
+
+
+def _slow_load_copies_fast(fast_tokenizer):
+ cls = type(fast_tokenizer)
+ if not _TRANSFORMERS_V5 or not cls.__module__.startswith("transformers."):
+ return False
+ if cls.__name__ in _V5_LEGACY_FREE_TOKENIZERS:
+ return True
+ # LlamaTokenizer builds its pre-tokenizer from legacy, so the copy only matches at legacy = False.
+ return cls.__name__ == "LlamaTokenizer" and getattr(fast_tokenizer, "legacy", None) is False
+
+
+def _same_tokenization_as_own_copy(fast_tokenizer):
+ try:
+ string = _same_tokenization_text(_same_tokenization_special_tokens(fast_tokenizer))
+ before = fast_tokenizer(string).input_ids
+ except Exception:
+ return False
+ # The setters rebuild the post_processor in place; undo them if ids move so the slow path starts as loaded.
+ backend = getattr(fast_tokenizer, "_tokenizer", None)
+ saved_post_processor = getattr(backend, "post_processor", None)
+ saved_flags = {
+ k: fast_tokenizer.__dict__[k]
+ for k in ("_add_bos_token", "_add_eos_token")
+ if k in fast_tokenizer.__dict__
+ }
+ try:
+ _sync_add_special_tokens(fast_tokenizer, fast_tokenizer)
+ same = fast_tokenizer(string).input_ids == before
+ except Exception:
+ same = False
+ if not same:
+ for k in ("_add_bos_token", "_add_eos_token"):
+ if k in saved_flags:
+ object.__setattr__(fast_tokenizer, k, saved_flags[k])
+ else:
+ fast_tokenizer.__dict__.pop(k, None)
+ if backend is not None:
+ backend.post_processor = saved_post_processor
+ return same
+
+
def _load_correct_tokenizer(
tokenizer_name,
model_max_length = None,
@@ -1065,6 +1135,19 @@ def _load_correct_tokenizer(
fast_tokenizer, fix_tokenizer, config, **_repair_kwargs
)
+ fast_tokenizer = None
+ if _TRANSFORMERS_V5:
+ fast_tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs
+ )
+ _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)
+ if _slow_load_copies_fast(fast_tokenizer) and _same_tokenization_as_own_copy(
+ fast_tokenizer
+ ):
+ return _apply_post_load_tokenizer_fixes(
+ fast_tokenizer, fix_tokenizer, config, **_repair_kwargs
+ )
+
# Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no tokenizer.model file.
slow_tokenizer = None
try:
@@ -1083,19 +1166,19 @@ def _load_correct_tokenizer(
if type(slow_tokenizer) is bool:
slow_tokenizer = None
- fast_tokenizer = AutoTokenizer.from_pretrained(
- tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs
- )
+ load_fast = fast_tokenizer is None
+ if load_fast:
+ fast_tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs
+ )
# Repair both before comparing them, or a v5 byte-level mismatch detours into the legacy path.
if slow_tokenizer is not None:
_repair_tokenizer_backend_from_json(slow_tokenizer, **_repair_kwargs)
- _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)
+ if load_fast:
+ _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)
if slow_tokenizer is not None:
- if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
- fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
- if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):
- fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
+ _sync_add_special_tokens(fast_tokenizer, slow_tokenizer)
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
return _apply_post_load_tokenizer_fixes(