Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip the duplicate tokenizer load on transformers 5 when it can only copy the fast tokenizer (#17318)
Daniel Han4fd58f0e4add
diff --git a/tests/test_tokenizer_skip_unused_slow_load.py b/tests/test_tokenizer_skip_unused_slow_load.pyindex 052c6025660..07670482d05 100644--- a/tests/test_tokenizer_skip_unused_slow_load.py+++ b/tests/test_tokenizer_skip_unused_slow_load.py@@ -7,9 +7,15 @@ import typesfrom unittest.mock import patchimport pytest+import transformersimport unsloth.tokenizer_utils as tu+_needs_v5 = pytest.mark.skipif(+ int(transformers.__version__.split(".")[0]) < 5,+ reason = "transformers 4 loads a real slow tokenizer before the fast one",+)+class _Tok:def __init__(self, kind):@@ -65,10 +71,183 @@ def test_early_return_skips_slow_tokenizer(name, fix_tokenizer):@pytest.mark.parametrize("same", [True, False])def test_compared_path_still_loads_slow_tokenizer(same):result, calls, repaired = _run("unsloth/Llama-3.2-1B-Instruct", True, same)- assert calls == ["slow", "fast"]+ assert sorted(calls) == ["fast", "slow"]assert repaired.count("slow") == 1assert result.kind == ("fast" if same else "converted")+class _V5Tok:+ bos_token = "<s>"+ all_special_tokens = ["<s>"]++ def __init__(+ self,+ kind,+ legacy = False,+ sync_changes_ids = False,+ ):+ self.kind = kind+ self.legacy = legacy+ self.sync_changes_ids = sync_changes_ids+ self.rebuilds = 0+ self._add_bos_token = False++ @property+ def add_bos_token(self):+ return self._add_bos_token++ @add_bos_token.setter+ def add_bos_token(self, value):+ self._add_bos_token = value+ self.rebuilds += 1++ @property+ def add_eos_token(self):+ return False++ @add_eos_token.setter+ def add_eos_token(self, value):+ self.rebuilds += 1++ def __call__(self, text):+ drop_bos = self.sync_changes_ids and self.rebuilds > 0+ return types.SimpleNamespace(input_ids = [len(text)] + ([] if drop_bos else [1]))+++def _v5_class(name, module = "transformers.models.qwen2.tokenization_qwen2"):+ return type(name, (_V5Tok,), {"__module__": module})+++def _run_v5(+ cls,+ monkeypatch,+ v5 = True,+ **tok_kwargs,+):+ monkeypatch.setattr(tu, "_TRANSFORMERS_V5", v5, raising = False)+ calls, made = [], []++ def from_pretrained(tokenizer_name, **kwargs):+ kind = "slow" if kwargs.get("use_fast") is False else "fast"+ calls.append(kind)+ made.append(cls(kind, **tok_kwargs))+ return made[-1]++ with (+ patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)),+ patch.object(tu, "_repair_tokenizer_backend_from_json", lambda tok, **k: tok),+ patch.object(tu, "convert_to_fast_tokenizer", lambda slow, **k: slow),+ ):+ result = tu._load_correct_tokenizer("org/model", fix_tokenizer = True)+ return result, calls+++ "name",+ [+ "Qwen2Tokenizer",+ "Qwen3_5Tokenizer",+ "GemmaTokenizer",+ "GPT2Tokenizer",+ "CohereTokenizer",+ "TokenizersBackend",+ ],+)+def test_v5_skips_slow_load_that_copies_the_fast_tokenizer(name, monkeypatch):+ result, calls = _run_v5(_v5_class(name), monkeypatch)+ assert calls == [+ "fast"+ ], f"{name}: transformers 5 slow load only rebuilt the fast tokenizer ({calls})"+ assert result.kind == "fast"+ assert result.rebuilds == 2+++def test_v5_llama_skips_only_at_legacy_false(monkeypatch):+ result, calls = _run_v5(_v5_class("LlamaTokenizer"), monkeypatch, legacy = False)+ assert calls == ["fast"] and result.kind == "fast"+ result, calls = _run_v5(_v5_class("LlamaTokenizer"), monkeypatch, legacy = True)+ assert calls == ["fast", "slow"]+++def test_v5_sync_changing_ids_returns_the_slow_copy(monkeypatch):+ result, calls = _run_v5(_v5_class("GemmaTokenizer"), monkeypatch, sync_changes_ids = True)+ assert calls == ["fast", "slow"]+ assert result.kind == "slow"+++ "cls, v5",+ [+ (_v5_class("Qwen2Tokenizer"), False),+ (_v5_class("Qwen2Tokenizer", module = "transformers_modules.org.tokenization"), True),+ (_v5_class("NllbTokenizer"), True),+ ],+)+def test_slow_load_kept_when_it_may_differ(cls, v5, monkeypatch):+ result, calls = _run_v5(cls, monkeypatch, v5 = v5)+ assert sorted(calls) == ["fast", "slow"]+ assert result.kind == "fast"+++@_needs_v5+def test_v5_real_qwen2_tokenizer_constructed_once(tmp_path):+ vocab = {c: i for i, c in enumerate("abcdefghijklmnopqrstuvwxyz\u0120!")}+ transformers.Qwen2Tokenizer(+ vocab = vocab, merges = [], eos_token = "<|endoftext|>", unk_token = None, bos_token = None+ ).save_pretrained(tmp_path)+ calls = []+ original = tu.AutoTokenizer.from_pretrained++ def from_pretrained(*args, **kwargs):+ calls.append("slow" if kwargs.get("use_fast") is False else "fast")+ return original(*args, **kwargs)++ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):+ tokenizer = tu.load_correct_tokenizer(str(tmp_path))+ assert calls == ["fast"]+ assert type(tokenizer).__name__ == "Qwen2Tokenizer"+ assert tokenizer("hello world!", add_special_tokens = False).input_ids == [+ 7,+ 4,+ 11,+ 11,+ 14,+ 26,+ 22,+ 14,+ 17,+ 11,+ 3,+ 27,+ ]+++@_needs_v5+def test_v5_failed_slow_load_keeps_the_tokenizer_as_loaded(tmp_path):+ from tokenizers import processors++ vocab = {c: i for i, c in enumerate("abcdefghijklmnopqrstuvwxyz\u0120!")}+ vocab["<s>"] = len(vocab)+ tokenizer = transformers.Qwen2Tokenizer(+ vocab = vocab, merges = [], eos_token = "<|endoftext|>", unk_token = None, bos_token = "<s>"+ )+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(+ single = "<s>:0 $A:0", pair = "<s>:0 $A:0 $B:1", special_tokens = [("<s>", vocab["<s>"])]+ )+ tokenizer.save_pretrained(tmp_path)+ original = tu.AutoTokenizer.from_pretrained+ expected = original(str(tmp_path))("ab").input_ids+ assert expected[0] == vocab["<s>"]++ def from_pretrained(*args, **kwargs):+ if kwargs.get("use_fast") is False:+ raise OSError("slow load failed")+ return original(*args, **kwargs)++ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):+ loaded = tu.load_correct_tokenizer(str(tmp_path))+ assert loaded("ab").input_ids == expected++if __name__ == "__main__":raise SystemExit(pytest.main([__file__, "-q"]))diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex cf5bb278703..ceef1e5e72f 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -12,6 +12,7 @@from transformers import AutoTokenizerfrom transformers.convert_slow_tokenizer import convert_slow_tokenizerfrom transformers import PreTrainedTokenizerFast+from transformers import __version__ as transformers_versionimport reimport osimport json@@ -757,9 +758,7 @@ llama_template = ()-def assert_same_tokenization(slow_tokenizer, fast_tokenizer):- if not hasattr(slow_tokenizer, "all_special_tokens"):- return True+def _same_tokenization_special_tokens(slow_tokenizer):dir_names = dir(slow_tokenizer)special_tokens = list(filter(@@ -774,7 +773,21 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")- all_special_tokens = [x for x in all_special_tokens if x != replacement_char]+ return [x for x in all_special_tokens if x != replacement_char]+++def _same_tokenization_text(all_special_tokens):+ return (+ "\n".join(all_special_tokens)+ + "A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n"+ + "".join(all_special_tokens)+ )+++def assert_same_tokenization(slow_tokenizer, fast_tokenizer):+ if not hasattr(slow_tokenizer, "all_special_tokens"):+ return True+ all_special_tokens = _same_tokenization_special_tokens(slow_tokenizer)check_chat_template1 = Truecheck_chat_template2 = True@@ -820,11 +833,7 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3try:- string = (- "\n".join(all_special_tokens)- + "A quick brown fox jumps over the lazy dog!!\n\nHi</s>\n\n"- + "".join(all_special_tokens)- )+ string = _same_tokenization_text(all_special_tokens)check_special_tokens = slow_tokenizer(string).input_ids == fast_tokenizer(string).input_idsreturn check_chat_template and check_special_tokens@@ -1022,6 +1031,67 @@ def fix_sentencepiece_gguf(saved_location):return+def _sync_add_special_tokens(fast_tokenizer, slow_tokenizer):+ if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):+ fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token+ if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):+ fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token+++# v5 ignores use_fast/from_slow: the "slow" load is the fast one plus legacy = False, unread by these classes.+_TRANSFORMERS_V5 = int(transformers_version.split(".")[0]) >= 5+_V5_LEGACY_FREE_TOKENIZERS = frozenset(+ (+ "TokenizersBackend",+ "Qwen2Tokenizer",+ "Qwen3_5Tokenizer",+ "GemmaTokenizer",+ "GPT2Tokenizer",+ "CohereTokenizer",+ )+)+++def _slow_load_copies_fast(fast_tokenizer):+ cls = type(fast_tokenizer)+ if not _TRANSFORMERS_V5 or not cls.__module__.startswith("transformers."):+ return False+ if cls.__name__ in _V5_LEGACY_FREE_TOKENIZERS:+ return True+ # LlamaTokenizer builds its pre-tokenizer from legacy, so the copy only matches at legacy = False.+ return cls.__name__ == "LlamaTokenizer" and getattr(fast_tokenizer, "legacy", None) is False+++def _same_tokenization_as_own_copy(fast_tokenizer):+ try:+ string = _same_tokenization_text(_same_tokenization_special_tokens(fast_tokenizer))+ before = fast_tokenizer(string).input_ids+ except Exception:+ return False+ # The setters rebuild the post_processor in place; undo them if ids move so the slow path starts as loaded.+ backend = getattr(fast_tokenizer, "_tokenizer", None)+ saved_post_processor = getattr(backend, "post_processor", None)+ saved_flags = {+ k: fast_tokenizer.__dict__[k]+ for k in ("_add_bos_token", "_add_eos_token")+ if k in fast_tokenizer.__dict__+ }+ try:+ _sync_add_special_tokens(fast_tokenizer, fast_tokenizer)+ same = fast_tokenizer(string).input_ids == before+ except Exception:+ same = False+ if not same:+ for k in ("_add_bos_token", "_add_eos_token"):+ if k in saved_flags:+ object.__setattr__(fast_tokenizer, k, saved_flags[k])+ else:+ fast_tokenizer.__dict__.pop(k, None)+ if backend is not None:+ backend.post_processor = saved_post_processor+ return same++def _load_correct_tokenizer(tokenizer_name,model_max_length = None,@@ -1065,6 +1135,19 @@ def _load_correct_tokenizer(fast_tokenizer, fix_tokenizer, config, **_repair_kwargs)+ fast_tokenizer = None+ if _TRANSFORMERS_V5:+ fast_tokenizer = AutoTokenizer.from_pretrained(+ tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs+ )+ _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)+ if _slow_load_copies_fast(fast_tokenizer) and _same_tokenization_as_own_copy(+ fast_tokenizer+ ):+ return _apply_post_load_tokenizer_fixes(+ fast_tokenizer, fix_tokenizer, config, **_repair_kwargs+ )+# Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no tokenizer.model file.slow_tokenizer = Nonetry:@@ -1083,19 +1166,19 @@ def _load_correct_tokenizer(if type(slow_tokenizer) is bool:slow_tokenizer = None- fast_tokenizer = AutoTokenizer.from_pretrained(- tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs- )+ load_fast = fast_tokenizer is None+ if load_fast:+ fast_tokenizer = AutoTokenizer.from_pretrained(+ tokenizer_name, cache_dir = cache_dir, revision = revision, **_load_kwargs+ )# Repair both before comparing them, or a v5 byte-level mismatch detours into the legacy path.if slow_tokenizer is not None:_repair_tokenizer_backend_from_json(slow_tokenizer, **_repair_kwargs)- _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)+ if load_fast:+ _repair_tokenizer_backend_from_json(fast_tokenizer, **_repair_kwargs)if slow_tokenizer is not None:- if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):- fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token- if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):- fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token+ _sync_add_special_tokens(fast_tokenizer, slow_tokenizer)if assert_same_tokenization(slow_tokenizer, fast_tokenizer):return _apply_post_load_tokenizer_fixes(