Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
check_tokenizer: repair out-of-vocab added tokens on transformers 5.x, and raise when one can't be repaired (#14096)
Leo Borcherding734100330cdb
diff --git a/tests/test_check_tokenizer_unrepairable_raises.py b/tests/test_check_tokenizer_unrepairable_raises.pynew file mode 100644index 00000000000..c63dfed5e77--- /dev/null+++ b/tests/test_check_tokenizer_unrepairable_raises.py@@ -0,0 +1,174 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""check_tokenizer repairs a <sep> past the embedding (Starling) and raises if it can't."""++import os+import types+from unittest.mock import patch++import pytest+from tokenizers import Tokenizer, models, pre_tokenizers, processors+from transformers import AutoTokenizer, PreTrainedTokenizerFast++import unsloth.tokenizer_utils as tu++VOCAB = {"<unk>": 0, "<s>": 1, "</s>": 2, "hello": 3, "world": 4}+++def _model(rows):+ weight = types.SimpleNamespace(shape = (rows, 4))+ return types.SimpleNamespace(+ model = types.SimpleNamespace(embed_tokens = types.SimpleNamespace(weight = weight))+ )+++def _tiny_tokenizer(path, **special):+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))+ backend.pre_tokenizer = pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(+ tokenizer_object = backend, unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>"+ )+ if special:+ tokenizer.add_special_tokens(special)+ tokenizer.save_pretrained(path)+ return str(path)+++def _max_added_id(tokenizer):+ return max(tokenizer.added_tokens_decoder)+++def test_sep_token_past_embedding_is_repaired(tmp_path):+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")+ tokenizer = AutoTokenizer.from_pretrained(path)+ tokenizer = tu.check_tokenizer(_model(len(VOCAB)), tokenizer, model_name = path)+ assert _max_added_id(tokenizer) < len(VOCAB)+ assert tokenizer.sep_token is None+ assert max(tokenizer("hello <sep> world").input_ids) < len(VOCAB)+++def test_unrepairable_tokenizer_raises(tmp_path):+ # An extra special token has no named *_token slot, so it can't be removed.+ path = _tiny_tokenizer(tmp_path, additional_special_tokens = ["<x>"])+ with pytest.raises(RuntimeError, match = "out of bounds memory accesses"):+ tu.check_tokenizer(_model(len(VOCAB)), AutoTokenizer.from_pretrained(path), model_name = path)+++def test_in_range_tokenizer_is_returned_unchanged(tmp_path):+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")+ original = AutoTokenizer.from_pretrained(path)+ tokenizer = tu.check_tokenizer(_model(len(VOCAB) + 1), original, model_name = path)+ assert tokenizer is original+ assert tokenizer.sep_token_id == len(VOCAB)+++def _no_slow_reload():+ def from_pretrained(*a, **k):+ raise OSError("no slow tokenizer")++ return patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained))+++def test_failed_slow_reload_still_repairs(tmp_path):+ # transformers 4.x can't load a fast-only repo with use_fast = False.+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")+ original = AutoTokenizer.from_pretrained(path)+ with _no_slow_reload():+ tokenizer = tu.check_tokenizer(_model(len(VOCAB)), original, model_name = path)+ assert _max_added_id(tokenizer) < len(VOCAB)+ assert tokenizer.sep_token is None+++def test_failed_slow_reload_still_raises_when_unrepairable(tmp_path):+ path = _tiny_tokenizer(tmp_path, additional_special_tokens = ["<x>"])+ original = AutoTokenizer.from_pretrained(path)+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):+ tu.check_tokenizer(_model(len(VOCAB)), original, model_name = path)+++def test_post_processor_still_emitting_removed_id_raises(tmp_path):+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))+ backend.pre_tokenizer = pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(+ tokenizer_object = backend, unk_token = "<unk>", eos_token = "</s>"+ )+ tokenizer.add_special_tokens({"bos_token": "<bos>"})+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(+ single = "<bos> $A", special_tokens = [("<bos>", len(VOCAB))]+ )+ tokenizer.save_pretrained(tmp_path)+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):+ tu.check_tokenizer(+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)+ )+++def test_pair_template_still_emitting_removed_id_raises(tmp_path):+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))+ backend.pre_tokenizer = pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend, unk_token = "<unk>")+ tokenizer.add_special_tokens({"sep_token": "<sep>"})+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(+ single = "$A", pair = "$A <sep> $B:1", special_tokens = [("<sep>", len(VOCAB))]+ )+ tokenizer.save_pretrained(tmp_path)+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):+ tu.check_tokenizer(+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)+ )+++def test_bad_token_left_in_base_vocab_raises(tmp_path):+ backend = Tokenizer(models.WordLevel({**VOCAB, "<sep>": len(VOCAB)}, unk_token = "<unk>"))+ backend.pre_tokenizer = pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(+ tokenizer_object = backend, unk_token = "<unk>", sep_token = "<sep>"+ )+ tokenizer.save_pretrained(tmp_path)+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):+ tu.check_tokenizer(+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)+ )+++def test_removing_the_only_added_token_is_repaired(tmp_path):+ backend = Tokenizer(models.WordLevel({"hello": 0, "world": 1, "<unk>": 2}, unk_token = "<unk>"))+ backend.pre_tokenizer = pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend)+ tokenizer.add_tokens(["<x>"])+ tokenizer.save_pretrained(tmp_path)+ with _no_slow_reload():+ tokenizer = tu.check_tokenizer(+ _model(3), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)+ )+ assert len(tokenizer.added_tokens_decoder) == 0++[email protected](scope = "module")+def starling():+ # Tokenizer files only: the model this check was written for, <sep> at id 32002.+ try:+ from huggingface_hub import hf_hub_download++ files = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]+ files.append("tokenizer.model")+ paths = [hf_hub_download("berkeley-nest/Starling-LM-7B-alpha", f) for f in files]+ except Exception as e:+ pytest.skip(reason = f"needs the Hub for the Starling tokenizer files (offline run): {e}")+ return os.path.dirname(paths[0])+++def test_starling_sep_is_repaired(starling):+ tokenizer = AutoTokenizer.from_pretrained(starling)+ tokenizer = tu.check_tokenizer(_model(32002), tokenizer, model_name = starling)+ assert _max_added_id(tokenizer) < 32002+ assert tokenizer.sep_token is None+ assert max(tokenizer("Hello <sep> world").input_ids) < 32002+ assert tokenizer.eos_token == "<|end_of_turn|>"+++def test_starling_with_room_is_unchanged(starling):+ original = AutoTokenizer.from_pretrained(starling)+ assert tu.check_tokenizer(_model(32003), original, model_name = starling) is original+ assert original.sep_token_id == 32002diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 9d130be12d5..dd768ff08c3 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -20,7 +20,6 @@ import weakreffrom transformers.models.llama.modeling_llama import loggerfrom peft import PeftModelForCausalLMimport torch-import itertoolsimport collectionsimport numpy as npimport gc@@ -1720,17 +1719,11 @@ def check_tokenizer(bad_tokens = list(added_tokens_fast.values())[j:]if not _reload:added_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]- special_tokens = tokenizer.special_tokens_map- import itertools-- special_tokens = frozenset(- itertools.chain.from_iterable(- [x] if type(x) is str else x for x in special_tokens.values()- )- )+ # special_tokens_map drops the extra special tokens on transformers 5.x+ special_tokens = frozenset(str(x) for x in tokenizer.all_special_tokens)can_be_removed1 = [x for x in bad_tokens if x not in special_tokens]can_be_removed2 = [- x for x in can_be_removed1 if x in tokenizer._added_tokens_encoder.keys()+ x for x in can_be_removed1 if x in tokenizer.added_tokens_encoder.keys()]can_be_removed = (len(can_be_removed1) == len(bad_tokens)) and (@@ -1740,9 +1733,10 @@ def check_tokenizer(remove_generic = Falsetry_mapper = []if not can_be_removed:- names = dir(tokenizer)- names = (x for x in names if x.endswith("_token") and x.count("_") == 1)- generic_tokens = [(x, getattr(tokenizer, x, None)) for x in names]+ # dir(tokenizer) misses the *_token attributes on transformers 5.x+ generic_tokens = [+ (x, y) for x, y in tokenizer.special_tokens_map.items() if type(y) is str+ ]try_removal = []for token in bad_tokens:@@ -1757,15 +1751,32 @@ def check_tokenizer(can_be_removed1 = bad_tokensif can_be_removed:+ if getattr(tokenizer, "is_fast", False):+ # Fast tokenizers read added tokens from the Rust backend, so drop them there.+ import json++ state = json.loads(tokenizer._tokenizer.to_str())+ state["added_tokens"] = [+ x for x in state["added_tokens"] if x["content"] not in can_be_removed1+ ]+ tokenizer._tokenizer = tokenizer._tokenizer.from_str(json.dumps(state))+ else:+ for bad_token in can_be_removed1:+ remove_id = tokenizer._added_tokens_encoder[bad_token]+ del tokenizer._added_tokens_decoder[remove_id]+ del tokenizer._added_tokens_encoder[bad_token]for j, bad_token in enumerate(can_be_removed1):- remove_id = tokenizer._added_tokens_encoder[bad_token]- del tokenizer._added_tokens_decoder[remove_id]- del tokenizer._added_tokens_encoder[bad_token]-if remove_generic and (try_removal[j] == bad_token):setattr(tokenizer, try_mapper[j], None)setattr(tokenizer, try_mapper[j] + "_id", None)- if max(tokenizer.added_tokens_decoder.keys()) < max_embedding_size:+ # Ids left in the base vocab or emitted by a post-processor template must fit too.+ probe_ids = [max(tokenizer.get_vocab().values(), default = -1)]+ probe_ids += tokenizer("a").input_ids+ try:+ probe_ids += tokenizer("a", "a").input_ids+ except Exception:+ pass+ if max(probe_ids, default = -1) < max_embedding_size:logger.warning_once(f"Unsloth loaded a broken tokenizer `{model_name}`, but managed to repair it!\n"f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"@@ -1797,24 +1808,20 @@ def check_tokenizer(from_slow = True,cache_dir = reload_cache_dir,)- return check_tokenizer(- model = model,- tokenizer = tokenizer,- model_name = model_name,- model_max_length = model_max_length,- padding_side = padding_side,- token = token,- _reload = False,- cache_dir = cache_dir,- )- break- except:- logger.warning_once(- "Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n"- "It will still work, but beware of out of bounds memory accesses.\n"- "Please file an issue on the model owner's repo about this issue."- )- return tokenizer+ except Exception:+ # No usable slow tokenizer (fast-only repos on transformers 4.x): repair the one we have.+ pass+ # Outside the try: an unrepairable tokenizer must raise, not come back with a warning.+ return check_tokenizer(+ model = model,+ tokenizer = tokenizer,+ model_name = model_name,+ model_max_length = model_max_length,+ padding_side = padding_side,+ token = token,+ _reload = False,+ cache_dir = cache_dir,+ )return convert_to_fast_tokenizer(tokenizer)