Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

check_tokenizer: repair out-of-vocab added tokens on transformers 5.x, and raise when one can't be repaired (#14096)

Leo Borcherding734100330cdb
diff --git a/tests/test_check_tokenizer_unrepairable_raises.py b/tests/test_check_tokenizer_unrepairable_raises.py
new file mode 100644
index 00000000000..c63dfed5e77
--- /dev/null
+++ b/tests/test_check_tokenizer_unrepairable_raises.py
@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""check_tokenizer repairs a <sep> past the embedding (Starling) and raises if it can't."""
+
+import os
+import types
+from unittest.mock import patch
+
+import pytest
+from tokenizers import Tokenizer, models, pre_tokenizers, processors
+from transformers import AutoTokenizer, PreTrainedTokenizerFast
+
+import unsloth.tokenizer_utils as tu
+
+VOCAB = {"<unk>": 0, "<s>": 1, "</s>": 2, "hello": 3, "world": 4}
+
+
+def _model(rows):
+ weight = types.SimpleNamespace(shape = (rows, 4))
+ return types.SimpleNamespace(
+ model = types.SimpleNamespace(embed_tokens = types.SimpleNamespace(weight = weight))
+ )
+
+
+def _tiny_tokenizer(path, **special):
+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object = backend, unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>"
+ )
+ if special:
+ tokenizer.add_special_tokens(special)
+ tokenizer.save_pretrained(path)
+ return str(path)
+
+
+def _max_added_id(tokenizer):
+ return max(tokenizer.added_tokens_decoder)
+
+
+def test_sep_token_past_embedding_is_repaired(tmp_path):
+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")
+ tokenizer = AutoTokenizer.from_pretrained(path)
+ tokenizer = tu.check_tokenizer(_model(len(VOCAB)), tokenizer, model_name = path)
+ assert _max_added_id(tokenizer) < len(VOCAB)
+ assert tokenizer.sep_token is None
+ assert max(tokenizer("hello <sep> world").input_ids) < len(VOCAB)
+
+
+def test_unrepairable_tokenizer_raises(tmp_path):
+ # An extra special token has no named *_token slot, so it can't be removed.
+ path = _tiny_tokenizer(tmp_path, additional_special_tokens = ["<x>"])
+ with pytest.raises(RuntimeError, match = "out of bounds memory accesses"):
+ tu.check_tokenizer(_model(len(VOCAB)), AutoTokenizer.from_pretrained(path), model_name = path)
+
+
+def test_in_range_tokenizer_is_returned_unchanged(tmp_path):
+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")
+ original = AutoTokenizer.from_pretrained(path)
+ tokenizer = tu.check_tokenizer(_model(len(VOCAB) + 1), original, model_name = path)
+ assert tokenizer is original
+ assert tokenizer.sep_token_id == len(VOCAB)
+
+
+def _no_slow_reload():
+ def from_pretrained(*a, **k):
+ raise OSError("no slow tokenizer")
+
+ return patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained))
+
+
+def test_failed_slow_reload_still_repairs(tmp_path):
+ # transformers 4.x can't load a fast-only repo with use_fast = False.
+ path = _tiny_tokenizer(tmp_path, sep_token = "<sep>")
+ original = AutoTokenizer.from_pretrained(path)
+ with _no_slow_reload():
+ tokenizer = tu.check_tokenizer(_model(len(VOCAB)), original, model_name = path)
+ assert _max_added_id(tokenizer) < len(VOCAB)
+ assert tokenizer.sep_token is None
+
+
+def test_failed_slow_reload_still_raises_when_unrepairable(tmp_path):
+ path = _tiny_tokenizer(tmp_path, additional_special_tokens = ["<x>"])
+ original = AutoTokenizer.from_pretrained(path)
+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):
+ tu.check_tokenizer(_model(len(VOCAB)), original, model_name = path)
+
+
+def test_post_processor_still_emitting_removed_id_raises(tmp_path):
+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object = backend, unk_token = "<unk>", eos_token = "</s>"
+ )
+ tokenizer.add_special_tokens({"bos_token": "<bos>"})
+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(
+ single = "<bos> $A", special_tokens = [("<bos>", len(VOCAB))]
+ )
+ tokenizer.save_pretrained(tmp_path)
+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):
+ tu.check_tokenizer(
+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)
+ )
+
+
+def test_pair_template_still_emitting_removed_id_raises(tmp_path):
+ backend = Tokenizer(models.WordLevel(VOCAB, unk_token = "<unk>"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend, unk_token = "<unk>")
+ tokenizer.add_special_tokens({"sep_token": "<sep>"})
+ tokenizer._tokenizer.post_processor = processors.TemplateProcessing(
+ single = "$A", pair = "$A <sep> $B:1", special_tokens = [("<sep>", len(VOCAB))]
+ )
+ tokenizer.save_pretrained(tmp_path)
+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):
+ tu.check_tokenizer(
+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)
+ )
+
+
+def test_bad_token_left_in_base_vocab_raises(tmp_path):
+ backend = Tokenizer(models.WordLevel({**VOCAB, "<sep>": len(VOCAB)}, unk_token = "<unk>"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object = backend, unk_token = "<unk>", sep_token = "<sep>"
+ )
+ tokenizer.save_pretrained(tmp_path)
+ with _no_slow_reload(), pytest.raises(RuntimeError, match = "out of bounds memory accesses"):
+ tu.check_tokenizer(
+ _model(len(VOCAB)), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)
+ )
+
+
+def test_removing_the_only_added_token_is_repaired(tmp_path):
+ backend = Tokenizer(models.WordLevel({"hello": 0, "world": 1, "<unk>": 2}, unk_token = "<unk>"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend)
+ tokenizer.add_tokens(["<x>"])
+ tokenizer.save_pretrained(tmp_path)
+ with _no_slow_reload():
+ tokenizer = tu.check_tokenizer(
+ _model(3), AutoTokenizer.from_pretrained(tmp_path), model_name = str(tmp_path)
+ )
+ assert len(tokenizer.added_tokens_decoder) == 0
+
+
[email protected](scope = "module")
+def starling():
+ # Tokenizer files only: the model this check was written for, <sep> at id 32002.
+ try:
+ from huggingface_hub import hf_hub_download
+
+ files = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
+ files.append("tokenizer.model")
+ paths = [hf_hub_download("berkeley-nest/Starling-LM-7B-alpha", f) for f in files]
+ except Exception as e:
+ pytest.skip(reason = f"needs the Hub for the Starling tokenizer files (offline run): {e}")
+ return os.path.dirname(paths[0])
+
+
+def test_starling_sep_is_repaired(starling):
+ tokenizer = AutoTokenizer.from_pretrained(starling)
+ tokenizer = tu.check_tokenizer(_model(32002), tokenizer, model_name = starling)
+ assert _max_added_id(tokenizer) < 32002
+ assert tokenizer.sep_token is None
+ assert max(tokenizer("Hello <sep> world").input_ids) < 32002
+ assert tokenizer.eos_token == "<|end_of_turn|>"
+
+
+def test_starling_with_room_is_unchanged(starling):
+ original = AutoTokenizer.from_pretrained(starling)
+ assert tu.check_tokenizer(_model(32003), original, model_name = starling) is original
+ assert original.sep_token_id == 32002
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 9d130be12d5..dd768ff08c3 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -20,7 +20,6 @@ import weakref
from transformers.models.llama.modeling_llama import logger
from peft import PeftModelForCausalLM
import torch
-import itertools
import collections
import numpy as np
import gc
@@ -1720,17 +1719,11 @@ def check_tokenizer(
bad_tokens = list(added_tokens_fast.values())[j:]
if not _reload:
added_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]
- special_tokens = tokenizer.special_tokens_map
- import itertools
-
- special_tokens = frozenset(
- itertools.chain.from_iterable(
- [x] if type(x) is str else x for x in special_tokens.values()
- )
- )
+ # special_tokens_map drops the extra special tokens on transformers 5.x
+ special_tokens = frozenset(str(x) for x in tokenizer.all_special_tokens)
can_be_removed1 = [x for x in bad_tokens if x not in special_tokens]
can_be_removed2 = [
- x for x in can_be_removed1 if x in tokenizer._added_tokens_encoder.keys()
+ x for x in can_be_removed1 if x in tokenizer.added_tokens_encoder.keys()
]
can_be_removed = (len(can_be_removed1) == len(bad_tokens)) and (
@@ -1740,9 +1733,10 @@ def check_tokenizer(
remove_generic = False
try_mapper = []
if not can_be_removed:
- names = dir(tokenizer)
- names = (x for x in names if x.endswith("_token") and x.count("_") == 1)
- generic_tokens = [(x, getattr(tokenizer, x, None)) for x in names]
+ # dir(tokenizer) misses the *_token attributes on transformers 5.x
+ generic_tokens = [
+ (x, y) for x, y in tokenizer.special_tokens_map.items() if type(y) is str
+ ]
try_removal = []
for token in bad_tokens:
@@ -1757,15 +1751,32 @@ def check_tokenizer(
can_be_removed1 = bad_tokens
if can_be_removed:
+ if getattr(tokenizer, "is_fast", False):
+ # Fast tokenizers read added tokens from the Rust backend, so drop them there.
+ import json
+
+ state = json.loads(tokenizer._tokenizer.to_str())
+ state["added_tokens"] = [
+ x for x in state["added_tokens"] if x["content"] not in can_be_removed1
+ ]
+ tokenizer._tokenizer = tokenizer._tokenizer.from_str(json.dumps(state))
+ else:
+ for bad_token in can_be_removed1:
+ remove_id = tokenizer._added_tokens_encoder[bad_token]
+ del tokenizer._added_tokens_decoder[remove_id]
+ del tokenizer._added_tokens_encoder[bad_token]
for j, bad_token in enumerate(can_be_removed1):
- remove_id = tokenizer._added_tokens_encoder[bad_token]
- del tokenizer._added_tokens_decoder[remove_id]
- del tokenizer._added_tokens_encoder[bad_token]
-
if remove_generic and (try_removal[j] == bad_token):
setattr(tokenizer, try_mapper[j], None)
setattr(tokenizer, try_mapper[j] + "_id", None)
- if max(tokenizer.added_tokens_decoder.keys()) < max_embedding_size:
+ # Ids left in the base vocab or emitted by a post-processor template must fit too.
+ probe_ids = [max(tokenizer.get_vocab().values(), default = -1)]
+ probe_ids += tokenizer("a").input_ids
+ try:
+ probe_ids += tokenizer("a", "a").input_ids
+ except Exception:
+ pass
+ if max(probe_ids, default = -1) < max_embedding_size:
logger.warning_once(
f"Unsloth loaded a broken tokenizer `{model_name}`, but managed to repair it!\n"
f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"
@@ -1797,24 +1808,20 @@ def check_tokenizer(
from_slow = True,
cache_dir = reload_cache_dir,
)
- return check_tokenizer(
- model = model,
- tokenizer = tokenizer,
- model_name = model_name,
- model_max_length = model_max_length,
- padding_side = padding_side,
- token = token,
- _reload = False,
- cache_dir = cache_dir,
- )
- break
- except:
- logger.warning_once(
- "Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n"
- "It will still work, but beware of out of bounds memory accesses.\n"
- "Please file an issue on the model owner's repo about this issue."
- )
- return tokenizer
+ except Exception:
+ # No usable slow tokenizer (fast-only repos on transformers 4.x): repair the one we have.
+ pass
+ # Outside the try: an unrepairable tokenizer must raise, not come back with a warning.
+ return check_tokenizer(
+ model = model,
+ tokenizer = tokenizer,
+ model_name = model_name,
+ model_max_length = model_max_length,
+ padding_side = padding_side,
+ token = token,
+ _reload = False,
+ cache_dir = cache_dir,
+ )
return convert_to_fast_tokenizer(tokenizer)