Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add opt-in tokenizers 1.0 release candidate encode path (#13214)
Daniel Hana5a992be2da2
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex 42f9d51234c..0e87065b9be 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -690,6 +690,14 @@"count": 1,"reason": "Reads back a .pt file this function just wrote with torch.save into temporary_location; never a downloaded file."},+ {+ "file": "unsloth/tokenizers_v1.py",+ "rule": "dynamic-import",+ "sink": "spec_from_file_location",+ "digest": "71f24ac068b052d3",+ "count": 1,+ "reason": "Opt-in only (UNSLOTH_TOKENIZERS_V1=1): loads the tokenizers 1.0 extension module from the directory the user installed it into (UNSLOTH_TOKENIZERS_V1_PATH, default ~/.cache/unsloth/tokenizers_v1); never a downloaded or model-supplied file."+ },{"file": "unsloth/trainer.py","rule": "dynamic-import",diff --git a/tests/test_tokenizers_v1.py b/tests/test_tokenizers_v1.pynew file mode 100644index 00000000000..386cdf7ea9d--- /dev/null+++ b/tests/test_tokenizers_v1.py@@ -0,0 +1,492 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import copy+import glob+import os+import pickle++import pytest+from tokenizers import (+ Regex,+ Tokenizer,+ decoders,+ models,+ normalizers,+ pre_tokenizers,+ processors,+ trainers,+)+from transformers import PreTrainedTokenizerFast++import unsloth.tokenizers_v1 as tv1++RC_DIR = os.path.expanduser(os.environ.get(tv1._PATH_ENV) or tv1._DEFAULT_DIR)+HAS_RC = bool(glob.glob(os.path.join(RC_DIR, "tokenizers", "tokenizers*")))+needs_rc = pytest.mark.skipif(+ not HAS_RC, reason = f"tokenizers release candidate not installed in {RC_DIR}"+)++CORPUS = [+ "Hello world! This is a tiny corpus for a test tokenizer.",+ "def f(x):\n return x ** 2 # code",+ "The quick brown fox jumps over the lazy dog. 😁 café 中文",+ "Numbers 0 12 345 6789 and spaces\tand\nnewlines",+] * 25+TEXTS = [+ "Hello world!",+ " spaced out ",+ "<s> special inside </s> text",+ "Unseen 🚀 emoji and Zürich",+ "",+ "def g(y):\n\treturn y",+]+++# Qwen2's pretokenizer; the RC implements real models' patterns, not arbitrary regex.+QWEN2_PATTERN = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"+++def _byte_level(pattern = QWEN2_PATTERN):+ tok = Tokenizer(models.BPE())+ tok.pre_tokenizer = pre_tokenizers.Sequence(+ [+ pre_tokenizers.Split(Regex(pattern), behavior = "isolated"),+ pre_tokenizers.ByteLevel(add_prefix_space = False, use_regex = False),+ ]+ )+ tok.decoder = decoders.ByteLevel()+ trainer = trainers.BpeTrainer(+ vocab_size = 400,+ special_tokens = ["<unk>", "<s>", "</s>", "<pad>"],+ initial_alphabet = pre_tokenizers.ByteLevel.alphabet(),+ )+ tok.train_from_iterator(CORPUS, trainer = trainer)+ tok.post_processor = processors.TemplateProcessing(+ single = "<s> $A", pair = "<s> $A </s> $B", special_tokens = [("<s>", 1), ("</s>", 2)]+ )+ return tok+++def _unigram():+ tok = Tokenizer(models.Unigram())+ tok.normalizer = normalizers.Sequence([normalizers.NFKC(), normalizers.Replace(" ", "▁")])+ tok.pre_tokenizer = pre_tokenizers.Metaspace()+ tok.decoder = decoders.Metaspace()+ trainer = trainers.UnigramTrainer(+ vocab_size = 300, special_tokens = ["<unk>", "<s>", "</s>", "<pad>"], unk_token = "<unk>"+ )+ tok.train_from_iterator(CORPUS, trainer = trainer)+ return tok+++def _hf(backend):+ return PreTrainedTokenizerFast(+ tokenizer_object = backend,+ bos_token = "<s>",+ eos_token = "</s>",+ unk_token = "<unk>",+ pad_token = "<pad>",+ )+++def rc_on(monkeypatch):+ monkeypatch.setenv(tv1._ENV, "1")+ monkeypatch.setenv(tv1._VERIFY_ENV, "1")+ monkeypatch.setattr(tv1, "_v1_tried", False)+ monkeypatch.setattr(tv1, "_v1_module", None)+ monkeypatch.setattr(tv1, "_snapshots", {})+++def _pair(build):+ # Unigram training is not deterministic: clone one trained backend instead of training twice.+ backend = build()+ plain = _hf(Tokenizer.from_str(backend.to_str()))+ fast = tv1.enable_tokenizers_v1(_hf(Tokenizer.from_str(backend.to_str())))+ assert isinstance(fast._tokenizer, tv1.TokenizersV1Backend)+ return plain, fast+++def test_off_by_default_changes_nothing(monkeypatch):+ monkeypatch.delenv(tv1._ENV, raising = False)+ tok = _hf(_byte_level())+ backend = tok._tokenizer+ assert tv1.enable_tokenizers_v1(tok) is tok+ assert tok._tokenizer is backend+++def test_missing_rc_keeps_0x(monkeypatch, tmp_path):+ monkeypatch.setenv(tv1._ENV, "1")+ monkeypatch.setenv(tv1._PATH_ENV, str(tmp_path))+ monkeypatch.setattr(tv1, "_v1_tried", False)+ monkeypatch.setattr(tv1, "_v1_module", None)+ tok = _hf(_byte_level())+ backend = tok._tokenizer+ tv1.enable_tokenizers_v1(tok)+ assert tok._tokenizer is backend+++@needs_rc[email protected]("build", [_byte_level, _unigram], ids = ["bytelevel_bpe", "unigram"])+def test_plain_calls_take_rc_and_match(rc_on, build):+ plain, fast = _pair(build)+ for kwargs in ({}, {"add_special_tokens": False}, {"split_special_tokens": True}):+ want, got = plain(TEXTS, **kwargs), fast(TEXTS, **kwargs)+ assert got["input_ids"] == want["input_ids"]+ assert got["attention_mask"] == want["attention_mask"]+ assert all(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+ assert fast(TEXTS[0])["input_ids"] == plain(TEXTS[0])["input_ids"]+ assert fast.encode(TEXTS[3]) == plain.encode(TEXTS[3])+ assert fast(TEXTS, return_tensors = "np", padding = True)["input_ids"].tolist() == (+ plain(TEXTS, return_tensors = "np", padding = True)["input_ids"].tolist()+ )+++@needs_rc[email protected]("side", ["right", "left"])+def test_padding_and_truncation_take_rc_and_match(rc_on, side):+ plain, fast = _pair(_byte_level)+ for tok in (plain, fast):+ tok.padding_side = tok.truncation_side = side+ cases = [+ {"padding": True},+ {"padding": "max_length", "max_length": 24, "pad_to_multiple_of": 8},+ {"truncation": True, "max_length": 4},+ # unsloth_zoo's dataset tokenizer call+ {"truncation": True, "max_length": 6, "add_special_tokens": False},+ {"padding": True, "truncation": True, "max_length": 5, "return_tensors": "np"},+ ]+ for kwargs in cases:+ want, got = plain(TEXTS, **kwargs), fast(TEXTS, **kwargs)+ assert {k: v.tolist() if hasattr(v, "tolist") else v for k, v in got.items()} == (+ {k: v.tolist() if hasattr(v, "tolist") else v for k, v in want.items()}+ )+ assert all(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+++@needs_rc+def test_max_length_below_special_tokens_matches_0x(rc_on):+ # 0.x leaves such a row untruncated; the RC would keep only the special tokens.+ plain, fast = _pair(_byte_level)+ for kwargs in (+ {"truncation": True, "max_length": 0},+ {"truncation": True, "max_length": 0, "add_special_tokens": False},+ ):+ assert dict(fast(TEXTS, **kwargs)) == dict(plain(TEXTS, **kwargs))+++@needs_rc+def test_overflow_and_fields_the_rc_lacks_come_from_0x(rc_on):+ plain, fast = _pair(_byte_level)+ for kwargs in (+ {"return_offsets_mapping": True, "return_special_tokens_mask": True},+ {"padding": True, "return_offsets_mapping": True, "return_special_tokens_mask": True},+ {"truncation": True, "max_length": 4, "stride": 1, "return_overflowing_tokens": True},+ ):+ want, got = plain(TEXTS, **kwargs), fast(TEXTS, **kwargs)+ assert dict(got) == dict(want)+ for i in range(len(want["input_ids"])):+ assert got.word_ids(i) == want.word_ids(i)+ assert got.tokens(i) == want.tokens(i)+++@needs_rc+def test_pairs_and_presplit_words_fall_back_to_0x(rc_on):+ plain, fast = _pair(_byte_level)+ got = fast(TEXTS[:2], TEXTS[2:4])+ assert dict(got) == dict(plain(TEXTS[:2], TEXTS[2:4]))+ assert not any(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+ words = [["Hello", "world"], ["a", "b", "c"]]+ got = fast(words, is_split_into_words = True)+ assert dict(got) == dict(plain(words, is_split_into_words = True))+ assert not any(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+++@needs_rc+def test_added_tokens_rebuild_the_rc_copy(rc_on):+ plain, fast = _pair(_byte_level)+ fast("warm up")+ for tok in (plain, fast):+ tok.add_tokens(["<new_token>"])+ tok.add_special_tokens({"additional_special_tokens": ["<|special|>"]})+ text = ["a <new_token> b <|special|> c"]+ got = fast(text)+ assert got["input_ids"] == plain(text)["input_ids"]+ assert isinstance(got.encodings[0], tv1._CompatEncoding)+++@needs_rc+def test_probe_mismatch_keeps_0x(rc_on, monkeypatch):+ plain, fast = _pair(_byte_level)++ def _refuse(base, text):+ raise ValueError("forced mismatch")++ monkeypatch.setattr(tv1, "_build_snapshot", _refuse)+ got = fast(TEXTS)+ assert got["input_ids"] == plain(TEXTS)["input_ids"]+ assert not any(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+ assert fast._tokenizer._unsloth_state.disabled+++@needs_rc+def test_tokenizer_the_rc_cannot_load_stays_on_0x(rc_on):+ build = lambda: _byte_level(r"\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+|\s+")+ plain, fast = _pair(build)+ got = fast(TEXTS)+ assert got["input_ids"] == plain(TEXTS)["input_ids"]+ assert not any(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+ assert fast._tokenizer._unsloth_state.disabled+++@needs_rc+def test_copy_pickle_save_and_idempotence(rc_on, tmp_path):+ plain, fast = _pair(_byte_level)+ assert tv1.enable_tokenizers_v1(fast)._tokenizer is fast._tokenizer+ for clone in (copy.deepcopy(fast), pickle.loads(pickle.dumps(fast))):+ assert isinstance(clone._tokenizer, tv1.TokenizersV1Backend)+ assert clone(TEXTS)["input_ids"] == plain(TEXTS)["input_ids"]+ encodings = fast(TEXTS).encodings+ assert [e.ids for e in pickle.loads(pickle.dumps(encodings))] == [+ e.ids for e in plain(TEXTS).encodings+ ]+ fast.save_pretrained(tmp_path / "fast")+ plain.save_pretrained(tmp_path / "plain")+ assert (tmp_path / "fast" / "tokenizer.json").read_text(encoding = "utf-8") == (+ tmp_path / "plain" / "tokenizer.json"+ ).read_text(encoding = "utf-8")+++@needs_rc+def test_positional_arguments_follow_0x(rc_on):+ plain, fast = _pair(_byte_level)+ for args in ((False,), (False, False)):+ want = plain._tokenizer.encode_batch(TEXTS, *args)+ got = fast._tokenizer.encode_batch(TEXTS, *args)+ assert [e.ids for e in got] == [e.ids for e in want]+ assert (+ fast._tokenizer.encode(TEXTS[0], None, False, False).ids+ == plain._tokenizer.encode(TEXTS[0], None, False, False).ids+ )+ fast_out = fast._tokenizer.encode_batch_fast(TEXTS)+ assert not any(isinstance(e, tv1._CompatEncoding) for e in fast_out)+++@needs_rc+def test_encoding_mutation_and_padded_overflow_match_0x(rc_on):+ plain, fast = _pair(_byte_level)+ want, got = plain._tokenizer.encode(TEXTS[3]), fast._tokenizer.encode(TEXTS[3])+ assert isinstance(got, tv1._CompatEncoding)+ for e in (want, got):+ e.pad(32, pad_id = 3, pad_token = "<pad>")+ assert (got.ids, got.attention_mask, len(got)) == (want.ids, want.attention_mask, len(want))+ kwargs = {+ "padding": True,+ "truncation": True,+ "max_length": 6,+ "stride": 1,+ "return_overflowing_tokens": True,+ }+ for kw in (kwargs, dict(kwargs, padding = "max_length", pad_to_multiple_of = 3)):+ assert dict(fast(TEXTS, **kw)) == dict(plain(TEXTS, **kw))+++@needs_rc+def test_mutations_outside_the_wrapper_are_seen(rc_on):+ plain, fast = _pair(_byte_level)+ raw = fast._tokenizer._unsloth_base+ fast(TEXTS)+ new = processors.TemplateProcessing(single = "$A </s>", special_tokens = [("</s>", 2)])+ raw.post_processor = new+ plain._tokenizer.post_processor = new+ got = fast(TEXTS)+ assert got["input_ids"] == plain(TEXTS)["input_ids"]+ assert all(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+++@needs_rc+def test_custom_component_and_rc_failure_fall_back(rc_on):+ plain, fast = _pair(_byte_level)+ fast(TEXTS)+ snapshot = fast._tokenizer._unsloth_state.snapshot++ class _Raises:+ def encode_batch(self, *args, **kwargs):+ raise RuntimeError("rc failure")++ snapshot_v1, snapshot.v1 = snapshot.v1, _Raises()+ try:+ got = fast(TEXTS)+ finally:+ snapshot.v1 = snapshot_v1+ assert got["input_ids"] == plain(TEXTS)["input_ids"]+ assert not any(isinstance(e, tv1._CompatEncoding) for e in got.encodings)++ class _Split:+ def pre_tokenize(self, pretok):+ pretok.split(lambda i, s: [s])++ custom = _hf(_byte_level())+ custom._tokenizer.pre_tokenizer = pre_tokenizers.PreTokenizer.custom(_Split())+ reference = custom(TEXTS)["input_ids"]+ tv1.enable_tokenizers_v1(custom)+ assert custom(TEXTS)["input_ids"] == reference+ assert custom._tokenizer._unsloth_state.disabled+++@needs_rc[email protected](not hasattr(os, "fork"), reason = "os.fork is POSIX only")+def test_fork_while_another_thread_holds_a_lock(rc_on, monkeypatch):+ import threading+ import time++ # Verify mode would materialize in the parent, so the child would never touch the lock.+ monkeypatch.delenv(tv1._VERIFY_ENV)+ plain, fast = _pair(_byte_level)+ got = fast(TEXTS)+ want_offsets = plain(TEXTS).encodings[0].offsets+ snapshot = fast._tokenizer._unsloth_state.snapshot+ assert snapshot is not None+ held, release = threading.Event(), threading.Event()++ def hold():+ with snapshot.lock:+ held.set()+ release.wait()++ thread = threading.Thread(target = hold)+ thread.start()+ assert held.wait(30)+ pid = os.fork()+ if pid == 0:+ os._exit(0 if got.encodings[0].offsets == want_offsets else 1)+ release.set()+ thread.join()+ deadline = time.time() + 60+ while True:+ done, status = os.waitpid(pid, os.WNOHANG)+ if done:+ break+ if time.time() > deadline:+ os.kill(pid, 9)+ pytest.fail("child hung on a lock inherited from another thread")+ time.sleep(0.1)+ assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0+++CHAT_TEMPLATE = (+ "{% for m in messages %}{{ bos_token }}[{{ m['role'] }}] {{ m['content'] }}{{ eos_token }}{% endfor %}"+ "{% if add_generation_prompt %}[assistant] {% endif %}"+)+CONVERSATION = [+ {"role": "user", "content": "Hello, y'all! 😁 What is 12*7?"},+ {"role": "assistant", "content": "It is 84."},+]+++@needs_rc+def test_tokenizer_api_surface_unchanged(rc_on):+ plain, fast = _pair(_byte_level)+ for tok in (plain, fast):+ tok.chat_template = CHAT_TEMPLATE+ assert type(fast) is type(plain)+ assert set(dir(fast._tokenizer)) == set(dir(plain._tokenizer))+ for name in (a for a in dir(plain) if not a.startswith("_")):+ value = getattr(plain, name)+ if not callable(value) and name != "backend_tokenizer":+ other = getattr(fast, name)+ assert other == value or repr(other) == repr(value), name+ cases = [+ {"tokenize": False},+ {"tokenize": True},+ {"tokenize": True, "return_dict": True},+ {"tokenize": True, "return_dict": True, "return_tensors": "np"},+ {"tokenize": True, "add_generation_prompt": True},+ {"tokenize": True, "truncation": True, "max_length": 6},+ ]+ for kwargs in cases:+ want, got = (+ plain.apply_chat_template(CONVERSATION, **kwargs),+ fast.apply_chat_template(CONVERSATION, **kwargs),+ )+ norm = (+ lambda x: {k: v.tolist() if hasattr(v, "tolist") else v for k, v in x.items()}+ if hasattr(x, "keys")+ else x+ )+ assert norm(got) == norm(want), kwargs+ batch = [CONVERSATION, CONVERSATION[:1]]+ want = plain.apply_chat_template(batch, tokenize = True, padding = True, return_dict = True)+ got = fast.apply_chat_template(batch, tokenize = True, padding = True, return_dict = True)+ assert dict(got) == dict(want)+ assert all(isinstance(e, tv1._CompatEncoding) for e in got.encodings)+++@needs_rc+def test_verify_mode_catches_a_mismatch_and_post_process_takes_rc_encodings(rc_on):+ plain, fast = _pair(_byte_level)+ raw = fast._tokenizer.encode(TEXTS[0], add_special_tokens = False)+ want = plain._tokenizer.post_process(+ plain._tokenizer.encode(TEXTS[0], add_special_tokens = False)+ )+ assert isinstance(raw, tv1._CompatEncoding)+ assert fast._tokenizer.post_process(raw).ids == want.ids+ _, other = _pair(_unigram)+ other("warm up")+ snapshot = fast._tokenizer._unsloth_state.snapshot+ snapshot_v1, snapshot.v1 = snapshot.v1, other._tokenizer._unsloth_state.snapshot.v1+ try:+ with pytest.raises(RuntimeError, match = "differs from 0.x"):+ fast(TEXTS)+ finally:+ snapshot.v1 = snapshot_v1+++def _map(+ ds,+ tok,+ mutate = False,+):+ def fn(rows):+ if mutate:+ tok.add_tokens(["<worker_token>"])+ out = tok(rows["text"])+ return {+ "input_ids": out["input_ids"],+ "rc": [isinstance(e, tv1._CompatEncoding) for e in out.encodings],+ }++ return ds.map(fn, batched = True, batch_size = 10, num_proc = 2, load_from_cache_file = False)+++@needs_rc+def test_dataset_map_multiprocess(rc_on):+ # Building an RC tokenizer in a child forked after a build deadlocks; workers must reuse.+ datasets = pytest.importorskip("datasets")+ ds = datasets.Dataset.from_dict({"text": CORPUS})+ plain, used = _pair(_byte_level)+ used("encoded in the parent before the fork")+ plain_u, unused = _pair(_unigram)+ for tok, ref in ((used, plain), (unused, plain_u)):+ got = _map(ds, tok)+ assert got["input_ids"] == _map(ds, ref)["input_ids"]+ assert all(got["rc"])+ got = _map(ds, used, mutate = True)+ plain.add_tokens(["<worker_token>"])+ assert got["input_ids"] == _map(ds, plain)["input_ids"]+ import multiprocess++ if multiprocess.get_start_method() == "fork":+ # Forked after a build: the worker cannot rebuild, so it stays on 0.x. Spawned workers rebuild.+ assert not any(got["rc"])+++def test_decorator_only_touches_model_tokenizer_pairs(monkeypatch):+ monkeypatch.delenv(tv1._ENV, raising = False)+ sentinel = object()+ assert tv1.tokenizers_v1_on_return(lambda: sentinel)() is sentinel+ tok = _hf(_byte_level())+ backend = tok._tokenizer+ model, out = tv1.tokenizers_v1_on_return(lambda: ("model", tok))()+ assert out is tok and tok._tokenizer is backenddiff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex 3c2ed422417..b9036656ee1 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -36,6 +36,7 @@ import osimport shutilimport refrom .ollama_template_mappers import OLLAMA_TEMPLATES+from .tokenizers_v1 import enable_tokenizers_v1try:from unsloth_zoo.dataset_utils import (train_on_responses_only as _zoo_train_on_responses_only,@@ -2236,7 +2237,7 @@ def get_chat_template(getattr(tokenizer, "chat_template", None) is not None)tokenizer = _mlx_tokenizer_wrapper- return tokenizer+ return enable_tokenizers_v1(tokenizer)def remove_special_tokens(tokenizer, prompt):diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex f3fd14cdc77..a2f770c3daa 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -42,6 +42,7 @@ from .mistral_format import (prepare_mistral_format_checkpoint,)from .lora_init import adapter_used_fast_pissa, fast_lora_init, record_fast_pissa+from ..tokenizers_v1 import tokenizers_v1_on_returnfrom .loader_utils import (DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -956,6 +957,7 @@ def _vllm_unavailable_error():class FastLanguageModel(FastLlamaModel):@staticmethod+ @tokenizers_v1_on_return@_offline_aware_load@mistral_format_redirect@track_explicit_4bit_request@@ -1830,6 +1832,7 @@ class FastModel(FastBaseModel):return FastBaseModel.for_training(model, use_gradient_checkpointing)@staticmethod+ @tokenizers_v1_on_return@_restore_load_scoped_env@_offline_aware_load@mistral_format_redirectdiff --git a/unsloth/tokenizers_v1.py b/unsloth/tokenizers_v1.pynew file mode 100644index 00000000000..4eb8c0f3431--- /dev/null+++ b/unsloth/tokenizers_v1.py@@ -0,0 +1,678 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Opt-in encoding through the ``tokenizers`` 1.0 release candidate.++transformers requires ``tokenizers<0.24`` and builds its tokenizers from the 0.x component API,+which the RC drops, so the RC cannot replace 0.x. Instead it is installed beside it:++ pip install --pre --no-deps --target ~/.cache/unsloth/tokenizers_v1 "tokenizers>=1.0.0rc2"+ UNSLOTH_TOKENIZERS_V1=1 python train.py # UNSLOTH_TOKENIZERS_V1_PATH overrides the directory++and loaded under another module name. The fast tokenizer's 0.x backend is then wrapped: plain+encodes (str input, no pair, no pre-split words; padding and truncation included) run on the RC,+everything else goes to 0.x unchanged. Each RC copy is built from the backend's own JSON and must+match 0.x on a probe set first, else the backend stays on 0.x. Fields the RC's ``Encoding`` lacks+(offsets, tokens, word ids, special tokens mask, ...) are recomputed by 0.x on first access.+"""++import contextlib+import copy+import functools+import glob+import hashlib+import importlib.machinery+import importlib.util+import os+import sys+import tempfile+import threading+import weakref++from transformers.utils import logging++logger = logging.get_logger(__name__)++__all__ = ["enable_tokenizers_v1", "tokenizers_v1_enabled", "TokenizersV1Backend"]++_ENV = "UNSLOTH_TOKENIZERS_V1"+_PATH_ENV = "UNSLOTH_TOKENIZERS_V1_PATH"+_VERIFY_ENV = "UNSLOTH_TOKENIZERS_V1_VERIFY"+_DEFAULT_DIR = os.path.join(os.path.expanduser("~"), ".cache", "unsloth", "tokenizers_v1")+# The last component must stay "tokenizers": CPython calls PyInit_<last component>.+_MODULE_NAME = "unsloth_tokenizers_v1.tokenizers"+# Mutations the per-encode fingerprint (vocab size + component states) cannot see.+_MUTATING = frozenset(("model", "add_tokens", "add_special_tokens", "train", "train_from_iterator"))+_MAX_REBUILDS = 16+_PROBES = (+ "Hello world! How are you?",+ " leading and trailing spaces ",+ "Tabs\tand\nnewlines\r\n\n\nend",+ "Numbers 0 12 345 6789 3.14159 -42 1,000,000",+ "Emoji 😁🚀👍🏽 and accents: café naïve Zürich",+ "中文测试,日本語のテキスト、한국어 텍스트",+ "def f(x):\n return x ** 2 # code\n",+ "",+)++_v1_module = None+_v1_tried = False+_v1_lock = threading.Lock()+# RC from_file deadlocks in a child forked after any RC build (encoding inherited ones is fine):+# snapshots are keyed by JSON, refreshed before fork, and such children only reuse them.+_MAX_SNAPSHOTS = 8+_snapshots = {}+_snapshots_lock = threading.Lock()+_built_here = False+_no_build = False+_live = weakref.WeakSet()+_all_snapshots = weakref.WeakSet()+_fork_hooks = False+++def _fatal(error):+ # PyO3 panics derive from BaseException; only these must propagate.+ return isinstance(error, (KeyboardInterrupt, SystemExit, GeneratorExit))+++def tokenizers_v1_enabled():+ return os.environ.get(_ENV, "0").strip().lower() in ("1", "true", "yes", "on")+++def _load_v1():+ global _v1_module, _v1_tried+ if _v1_tried:+ return _v1_module+ with _v1_lock:+ if _v1_tried:+ return _v1_module+ directory = os.path.expanduser(os.environ.get(_PATH_ENV) or _DEFAULT_DIR)+ candidates = [+ path+ for suffix in importlib.machinery.EXTENSION_SUFFIXES+ for path in glob.glob(os.path.join(directory, "tokenizers", "tokenizers*" + suffix))+ if os.path.basename(path) == "tokenizers" + suffix+ ]+ module = None+ loaded = sys.modules.get(_MODULE_NAME)+ loaded_file = os.path.realpath(getattr(loaded, "__file__", None) or "")+ if loaded is not None and any(os.path.realpath(path) == loaded_file for path in candidates):+ # An extension initialises once per process; only tests reset the cache.+ candidates = []+ module = loaded+ elif not candidates:+ logger.warning_once(+ f"Unsloth: {_ENV}=1 but no tokenizers release candidate was found in {directory}. "+ f'Install it with `pip install --pre --no-deps --target {directory} "tokenizers>=1.0.0rc2"`. '+ "Using tokenizers 0.x."+ )+ else:+ try:+ loader = importlib.machinery.ExtensionFileLoader(_MODULE_NAME, candidates[0])+ spec = importlib.util.spec_from_file_location(+ _MODULE_NAME, candidates[0], loader = loader+ )+ module = importlib.util.module_from_spec(spec)+ loader.exec_module(module)+ major = int(str(module.__version__).split(".")[0])+ if major < 1:+ raise ImportError(f"found tokenizers {module.__version__}, need >= 1.0")+ sys.modules[_MODULE_NAME] = module+ except BaseException as error:+ if _fatal(error):+ raise+ logger.warning_once(+ f"Unsloth: could not load the tokenizers release candidate from {candidates[0]}: {error}. "+ "Using tokenizers 0.x."+ )+ module = None+ _v1_module = module+ _v1_tried = True+ return module+++def _same(a, b):+ return a.ids == b.ids and a.type_ids == b.type_ids and a.attention_mask == b.attention_mask+++class _Snapshot:+ """An RC tokenizer and a 0.x twin, both built from one backend JSON (no padding / truncation)."""++ __slots__ = ("v1", "reference", "lock", "option_cache", "n_special", "__weakref__")++ def __init__(self, v1, reference):+ self.v1 = v1+ self.reference = reference+ self.lock = threading.Lock()+ self.option_cache = {}+ self.n_special = reference.num_special_tokens_to_add(False)+ _all_snapshots.add(self)++ @contextlib.contextmanager+ def configured(+ self,+ padding,+ truncation,+ encode_special_tokens = False,+ ):+ with self.lock:+ reference = self.reference+ try:+ reference.encode_special_tokens = encode_special_tokens+ if truncation is not None:+ reference.enable_truncation(**truncation)+ if padding is not None:+ reference.enable_padding(**padding)+ yield reference+ finally:+ reference.no_padding()+ reference.no_truncation()+ reference.encode_special_tokens = False++ def reference_encode(+ self,+ text,+ add_special_tokens,+ encode_special_tokens,+ padding = None,+ truncation = None,+ ):+ with self.configured(padding, truncation, encode_special_tokens) as reference:+ return reference.encode(text, add_special_tokens = add_special_tokens)++ def options(self, padding, truncation):+ """0.x padding / truncation dicts -> RC objects; None if the RC cannot express them."""+ key = (+ None if padding is None else tuple(sorted(padding.items())),+ None if truncation is None else tuple(sorted(truncation.items())),+ )+ cache = self.option_cache+ if key in cache:+ return cache[key]+ module = sys.modules.get(_MODULE_NAME)+ try:+ v1_padding = (+ None+ if padding is None+ else module.Padding(+ direction = padding["direction"],+ pad_id = padding["pad_id"],+ pad_type_id = padding["pad_type_id"],+ pad_token = padding["pad_token"],+ length = padding["length"],+ pad_to_multiple_of = padding["pad_to_multiple_of"],+ )+ )+ # stride only shapes 0.x overflow, which the RC lacks and 0.x recomputes on demand.+ v1_truncation = (+ None+ if truncation is None+ else module.Truncation(+ max_length = truncation["max_length"],+ strategy = truncation["strategy"],+ direction = truncation["direction"],+ )+ )+ result = (v1_padding, v1_truncation)+ except Exception:+ result = None+ if len(cache) < 64:+ cache[key] = result+ return result+++class _State:+ __slots__ = (+ "dirty",+ "generation",+ "fingerprint",+ "digest",+ "snapshot",+ "disabled",+ "rebuilds",+ "lock",+ )++ def __init__(self):+ self.dirty = True+ self.generation = 0+ self.fingerprint = None+ self.digest = None+ self.snapshot = None+ self.disabled = False+ self.rebuilds = 0+ self.lock = threading.Lock()+++def _build_snapshot(base, text):+ v1_module = _load_v1()+ # The RC loads only from a file or the Hub.+ fd, path = tempfile.mkstemp(suffix = ".json", prefix = "unsloth_tokenizers_v1_")+ try:+ with os.fdopen(fd, "w", encoding = "utf-8") as file:+ file.write(text)+ global _built_here+ _built_here = True+ v1 = v1_module.Tokenizer.from_file(path)+ finally:+ try:+ os.remove(path)+ except OSError:+ pass+ reference = type(base).from_str(text)+ added = [token.content for token in base.get_added_tokens_decoder().values()][:64]+ probes = list(_PROBES) + ["".join(added), " ".join(added) + " Hello"]+ for encode_special_tokens in (False, True):+ reference.encode_special_tokens = encode_special_tokens+ for add_special_tokens in (True, False):+ want = reference.encode_batch(probes, add_special_tokens = add_special_tokens)+ got = v1.encode_batch(+ probes,+ add_special_tokens = add_special_tokens,+ encode_special_tokens = encode_special_tokens,+ )+ if len(want) != len(got) or not all(_same(a, b) for a, b in zip(want, got)):+ raise ValueError("token ids differ from tokenizers 0.x on the probe set")+ snapshot = _Snapshot(v1, reference)+ pad_id = (base.token_to_id(added[0]) if added else None) or 0+ for direction in ("right", "left"):+ padding = dict(+ length = None,+ pad_to_multiple_of = None,+ pad_id = pad_id,+ pad_type_id = 0,+ pad_token = added[0] if added else "[PAD]",+ direction = direction,+ )+ truncation = dict(max_length = 5, stride = 0, strategy = "longest_first", direction = direction)+ for pad, trunc in ((padding, None), (None, truncation), (padding, truncation)):+ v1_padding, v1_truncation = snapshot.options(pad, trunc)+ with snapshot.configured(pad, trunc) as configured:+ want = configured.encode_batch(probes)+ got = v1.encode_batch(probes, padding = v1_padding, truncation = v1_truncation)+ if len(want) != len(got) or not all(_same(a, b) for a, b in zip(want, got)):+ raise ValueError(f"padded / truncated ids differ from tokenizers 0.x ({direction})")+ return snapshot+++def _rebuild_encoding(encoding):+ return encoding+++class _CompatEncoding:+ """An RC ``Encoding``; any field the RC lacks is answered by an exact 0.x re-encode."""++ __slots__ = ("_v1", "_text", "_add", "_split", "_snapshot", "_padding", "_truncation", "_real")++ def __init__(+ self, v1, text, add_special_tokens, encode_special_tokens, snapshot, padding, truncation+ ):+ self._v1 = v1+ self._text = text+ self._add = add_special_tokens+ self._split = encode_special_tokens+ self._snapshot = snapshot+ self._padding = padding+ self._truncation = truncation+ self._real = None++ # Once materialized the 0.x encoding answers everything: Encoding.pad / truncate mutate it.+ @property+ def ids(self):+ return (self._v1 if self._real is None else self._real).ids++ @property+ def type_ids(self):+ return (self._v1 if self._real is None else self._real).type_ids++ @property+ def attention_mask(self):+ return (self._v1 if self._real is None else self._real).attention_mask++ @property+ def n_sequences(self):+ # Only single sequences are routed here; BatchEncoding reads this on every call.+ return 1 if self._real is None else self._real.n_sequences++ def __len__(self):+ return len(self._v1 if self._real is None else self._real)++ def _materialize(self):+ if self._real is None:+ padding = self._padding+ if padding is not None:+ # Pad-to-longest depends on the batch: replay the length this row got.+ padding = dict(padding, length = len(self._v1), pad_to_multiple_of = None)+ self._real = self._snapshot.reference_encode(+ self._text, self._add, self._split, padding, self._truncation+ )+ return self._real++ def __getattr__(self, name):+ return getattr(self._materialize(), name)++ def __reduce__(self):+ return (_rebuild_encoding, (self._materialize(),))++ def __repr__(self):+ return repr(self._materialize())+++def _rewrap(base):+ return enable_tokenizers_v1(base) if tokenizers_v1_enabled() else base+++class TokenizersV1Backend:+ """Stands in for a ``tokenizers`` 0.x ``Tokenizer``; only plain encodes take the RC."""++ __slots__ = ("_unsloth_base", "_unsloth_state", "__weakref__")++ def __init__(self, base):+ object.__setattr__(self, "_unsloth_base", base)+ object.__setattr__(self, "_unsloth_state", _State())+ _live.add(self)+ _register_fork_hooks()++ def __getattr__(self, name):+ if name in _MUTATING:+ self._unsloth_mark()+ return getattr(self._unsloth_base, name)++ def __setattr__(self, name, value):+ setattr(self._unsloth_base, name, value)+ if name != "encode_special_tokens":+ self._unsloth_mark()++ def _unsloth_mark(self):+ state = self._unsloth_state+ state.generation += 1+ state.dirty = True++ def _unsloth_fingerprint(self):+ base = self._unsloth_base+ return (base.get_vocab_size(True),) + tuple(+ None if part is None else part.__getstate__()+ for part in (base.normalizer, base.pre_tokenizer, base.post_processor, base.decoder)+ )++ def __dir__(self):+ return dir(self._unsloth_base)++ def __repr__(self):+ return repr(self._unsloth_base)++ def __reduce__(self):+ return (_rewrap, (self._unsloth_base,))++ def post_process(+ self,+ encoding,+ pair = None,+ add_special_tokens = True,+ ):+ # The native method only takes real 0.x encodings.+ unwrap = lambda e: e._materialize() if isinstance(e, _CompatEncoding) else e+ return self._unsloth_base.post_process(unwrap(encoding), unwrap(pair), add_special_tokens)++ def __copy__(self):+ return _rewrap(copy.copy(self._unsloth_base))++ def __deepcopy__(self, memo):+ return _rewrap(copy.deepcopy(self._unsloth_base, memo))++ def _unsloth_refresh(self):+ state = self._unsloth_state+ if state.disabled:+ return None+ # Catches in-place component edits and edits through other references to the backend.+ try:+ fingerprint = self._unsloth_fingerprint()+ except BaseException as error:+ if _fatal(error):+ raise+ # Custom Python components cannot be serialized, so the RC cannot copy them either.+ state.disabled = True+ state.snapshot = None+ return None+ if not state.dirty and fingerprint == state.fingerprint:+ return state.snapshot+ with state.lock:+ if state.disabled:+ return None+ generation = state.generation+ base = self._unsloth_base+ try:+ text = base.to_str()+ if base.padding is not None or base.truncation is not None:+ # Padding / truncation change per call and are passed per encode instead.+ clone = type(base).from_str(text)+ clone.no_padding()+ clone.no_truncation()+ text = clone.to_str()+ except BaseException as error:+ if _fatal(error):+ raise+ state.disabled = True+ state.snapshot = None+ return None+ digest = hashlib.blake2b(text.encode("utf-8"), digest_size = 16).digest()+ if digest != state.digest:+ snapshot = self._unsloth_lookup_or_build(state, base, text, digest)+ if snapshot is None:+ state.disabled = True+ state.snapshot = None+ return None+ state.snapshot = snapshot+ state.digest = digest+ state.fingerprint = fingerprint+ if state.generation == generation:+ state.dirty = False+ return state.snapshot++ @staticmethod+ def _unsloth_lookup_or_build(state, base, text, digest):+ with _snapshots_lock:+ snapshot = _snapshots.get(digest)+ if snapshot is not None:+ return snapshot+ if _no_build:+ logger.warning_once(+ "Unsloth: a tokenizer changed inside a forked worker, so that worker stays on tokenizers 0.x."+ )+ return None+ state.rebuilds += 1+ if state.rebuilds > _MAX_REBUILDS:+ logger.warning_once(+ "Unsloth: the tokenizer keeps changing between encodes, so it stays on tokenizers 0.x."+ )+ return None+ try:+ snapshot = _build_snapshot(base, text)+ except BaseException as error:+ if _fatal(error):+ raise+ logger.warning_once(+ f"Unsloth: this tokenizer stays on tokenizers 0.x, the release candidate cannot match it: {error}"+ )+ return None+ with _snapshots_lock:+ _snapshots[digest] = snapshot+ while len(_snapshots) > _MAX_SNAPSHOTS:+ _snapshots.pop(next(iter(_snapshots)))+ return snapshot++ def _unsloth_encode(self, snapshot, texts, add_special_tokens):+ base = self._unsloth_base+ split = base.encode_special_tokens+ padding, truncation = base.padding, base.truncation+ # 0.x skips truncation when max_length cannot fit the added special tokens; the RC drops content.+ if (+ truncation is not None+ and add_special_tokens+ and truncation["max_length"] < snapshot.n_special+ ):+ return None+ options = snapshot.options(padding, truncation)+ if options is None:+ return None+ try:+ encodings = snapshot.v1.encode_batch(+ texts,+ add_special_tokens = add_special_tokens,+ encode_special_tokens = split,+ padding = options[0],+ truncation = options[1],+ )+ except BaseException as error:+ if _fatal(error):+ raise+ return None+ out = [+ _CompatEncoding(+ encoding, text, add_special_tokens, split, snapshot, padding, truncation+ )+ for encoding, text in zip(encodings, texts)+ ]+ if os.environ.get(_VERIFY_ENV, "0") == "1":+ for text, encoding in zip(texts, out):+ # Against the RC encoding itself: once materialized, the wrapper reads 0.x.+ if not _same(encoding._materialize(), encoding._v1):+ raise RuntimeError(+ f"Unsloth: tokenizers release candidate differs from 0.x on {text[:200]!r}"+ )+ return out++ def _unsloth_try(self, texts, add_special_tokens, is_pretokenized, kwargs):+ if kwargs or is_pretokenized or not isinstance(texts, (list, tuple)):+ return None+ if not all(type(text) is str for text in texts):+ return None+ snapshot = self._unsloth_refresh()+ return (+ None+ if snapshot is None+ else self._unsloth_encode(snapshot, list(texts), add_special_tokens)+ )++ # Signatures match tokenizers 0.x. encode_batch_fast is not routed: 0.x returns zero offsets there.+ def encode_batch(+ self,+ input,+ is_pretokenized = False,+ add_special_tokens = True,+ **kwargs,+ ):+ out = self._unsloth_try(input, add_special_tokens, is_pretokenized, kwargs)+ if out is not None:+ return out+ return self._unsloth_base.encode_batch(+ input, is_pretokenized = is_pretokenized, add_special_tokens = add_special_tokens, **kwargs+ )++ def encode(+ self,+ sequence,+ pair = None,+ is_pretokenized = False,+ add_special_tokens = True,+ **kwargs,+ ):+ if pair is None and type(sequence) is str:+ out = self._unsloth_try([sequence], add_special_tokens, is_pretokenized, kwargs)+ if out is not None:+ return out[0]+ return self._unsloth_base.encode(+ sequence,+ pair,+ is_pretokenized = is_pretokenized,+ add_special_tokens = add_special_tokens,+ **kwargs,+ )+++def _refresh_before_fork():+ if _no_build:+ return+ for proxy in list(_live):+ try:+ snapshot = proxy._unsloth_refresh()+ if snapshot is not None:+ with _snapshots_lock:+ _snapshots.setdefault(proxy._unsloth_state.digest, snapshot)+ except BaseException as error:+ if _fatal(error):+ raise+++def _after_fork_in_child():+ global _no_build, _v1_lock, _snapshots_lock+ _no_build = _no_build or _built_here+ # A lock another thread held at fork time stays locked forever in the child.+ _v1_lock = threading.Lock()+ _snapshots_lock = threading.Lock()+ for proxy in list(_live):+ proxy._unsloth_state.lock = threading.Lock()+ for snapshot in list(_all_snapshots):+ snapshot.lock = threading.Lock()+++def _register_fork_hooks():+ global _fork_hooks+ if not _fork_hooks and hasattr(os, "register_at_fork"):+ _fork_hooks = True+ os.register_at_fork(before = _refresh_before_fork, after_in_child = _after_fork_in_child)+++def _is_tokenizers_0x(backend):+ try:+ import tokenizers+ except ImportError:+ return False+ # Under tokenizers >= 1.0 the backend is already the fast engine.+ return isinstance(backend, tokenizers.Tokenizer) and str(tokenizers.__version__).startswith(+ "0."+ )+++def enable_tokenizers_v1(tokenizer):+ """Route a fast tokenizer's (or processor's inner tokenizer's) plain encodes to the RC.++ Accepts a raw 0.x backend too, returning the wrapper. Returns its argument unchanged when+ ``UNSLOTH_TOKENIZERS_V1`` is off, the RC is missing, or there is no 0.x backend; safe to repeat.+ """+ if not tokenizers_v1_enabled() or tokenizer is None:+ return tokenizer+ if isinstance(tokenizer, TokenizersV1Backend):+ return tokenizer+ if _is_tokenizers_0x(tokenizer):+ return TokenizersV1Backend(tokenizer) if _load_v1() is not None else tokenizer+ for obj in (tokenizer, getattr(tokenizer, "tokenizer", None)):+ backend = getattr(obj, "_tokenizer", None) if obj is not None else None+ if (+ backend is None+ or isinstance(backend, TokenizersV1Backend)+ or not _is_tokenizers_0x(backend)+ ):+ continue+ if _load_v1() is None:+ break+ obj._tokenizer = TokenizersV1Backend(backend)+ return tokenizer+++def tokenizers_v1_on_return(fn):+ @functools.wraps(fn)+ def _wrapper(*args, **kwargs):+ result = fn(*args, **kwargs)+ if not tokenizers_v1_enabled():+ return result+ if isinstance(result, tuple) and len(result) == 2:+ try:+ enable_tokenizers_v1(result[1])+ except Exception as error:+ logger.warning_once(+ f"Unsloth: could not enable the tokenizers release candidate: {error}"+ )+ return result++ return _wrapper