Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep slow tokenizers transformers cannot convert instead of crashing the load (#13073)

Daniel Hanfa8e70ed9720
diff --git a/tests/test_remote_code_slow_tokenizer_conversion.py b/tests/test_remote_code_slow_tokenizer_conversion.py
new file mode 100644
index 00000000000..fdafe9ae269
--- /dev/null
+++ b/tests/test_remote_code_slow_tokenizer_conversion.py
@@ -0,0 +1,60 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A slow tokenizer transformers cannot convert (remote-code SPTokenizer in openGPT-X/Teuken-7B) must be kept, not crash the load."""
+
+from transformers import PreTrainedTokenizer
+from transformers.convert_slow_tokenizer import convert_slow_tokenizer
+
+import unsloth.tokenizer_utils as tu
+
+
+class SPTokenizer(PreTrainedTokenizer):
+ # Like Teuken's remote-code tokenizer: a custom slow class with no vocab_file attribute.
+ def __init__(self, **kwargs):
+ self._vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "hello": 3, "world": 4}
+ super().__init__(unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>", **kwargs)
+
+ @property
+ def vocab_size(self):
+ return len(self._vocab)
+
+ def get_vocab(self):
+ return dict(self._vocab)
+
+ def _tokenize(self, text, **kwargs):
+ return text.split()
+
+ def _convert_token_to_id(self, token):
+ return self._vocab.get(token, 0)
+
+ def _convert_id_to_token(self, index):
+ return {v: k for k, v in self._vocab.items()}.get(index, "<unk>")
+
+
+def test_unconvertible_slow_tokenizer_is_returned_unchanged():
+ tok = SPTokenizer()
+ try:
+ convert_slow_tokenizer(tok)
+ except Exception:
+ pass
+ else:
+ raise AssertionError("premise: transformers must fail to convert this tokenizer")
+
+ assert tu.convert_to_fast_tokenizer(tok) is tok
+
+
+if __name__ == "__main__":
+ test_unconvertible_slow_tokenizer_is_returned_unchanged()
+ print("ok")
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 26c564dc1d1..0d592ab73d5 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -471,8 +471,13 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_sen
kwargs = {}
for arg in args:
kwargs[arg] = getattr(slow_tokenizer, arg, None)
- kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)
- fast_tokenizer = FastTokenizer(**kwargs)
+ # Remote-code slow tokenizers (e.g. Teuken's SPTokenizer, no vocab_file) can't always be converted:
+ # keep the slow one, as the vocab / tokenization mismatch checks below already do.
+ try:
+ kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)
+ fast_tokenizer = FastTokenizer(**kwargs)
+ except Exception:
+ return slow_tokenizer
sorted_slow_tokenizer = get_sorted_dict(slow_tokenizer.get_vocab())
sorted_fast_tokenizer = get_sorted_dict(fast_tokenizer.get_vocab())