Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep slow tokenizers transformers cannot convert instead of crashing the load (#13073)
Daniel Hanfa8e70ed9720
diff --git a/tests/test_remote_code_slow_tokenizer_conversion.py b/tests/test_remote_code_slow_tokenizer_conversion.pynew file mode 100644index 00000000000..fdafe9ae269--- /dev/null+++ b/tests/test_remote_code_slow_tokenizer_conversion.py@@ -0,0 +1,60 @@+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""A slow tokenizer transformers cannot convert (remote-code SPTokenizer in openGPT-X/Teuken-7B) must be kept, not crash the load."""++from transformers import PreTrainedTokenizer+from transformers.convert_slow_tokenizer import convert_slow_tokenizer++import unsloth.tokenizer_utils as tu+++class SPTokenizer(PreTrainedTokenizer):+ # Like Teuken's remote-code tokenizer: a custom slow class with no vocab_file attribute.+ def __init__(self, **kwargs):+ self._vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "hello": 3, "world": 4}+ super().__init__(unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>", **kwargs)++ @property+ def vocab_size(self):+ return len(self._vocab)++ def get_vocab(self):+ return dict(self._vocab)++ def _tokenize(self, text, **kwargs):+ return text.split()++ def _convert_token_to_id(self, token):+ return self._vocab.get(token, 0)++ def _convert_id_to_token(self, index):+ return {v: k for k, v in self._vocab.items()}.get(index, "<unk>")+++def test_unconvertible_slow_tokenizer_is_returned_unchanged():+ tok = SPTokenizer()+ try:+ convert_slow_tokenizer(tok)+ except Exception:+ pass+ else:+ raise AssertionError("premise: transformers must fail to convert this tokenizer")++ assert tu.convert_to_fast_tokenizer(tok) is tok+++if __name__ == "__main__":+ test_unconvertible_slow_tokenizer_is_returned_unchanged()+ print("ok")diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 26c564dc1d1..0d592ab73d5 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -471,8 +471,13 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_senkwargs = {}for arg in args:kwargs[arg] = getattr(slow_tokenizer, arg, None)- kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)- fast_tokenizer = FastTokenizer(**kwargs)+ # Remote-code slow tokenizers (e.g. Teuken's SPTokenizer, no vocab_file) can't always be converted:+ # keep the slow one, as the vocab / tokenization mismatch checks below already do.+ try:+ kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)+ fast_tokenizer = FastTokenizer(**kwargs)+ except Exception:+ return slow_tokenizersorted_slow_tokenizer = get_sorted_dict(slow_tokenizer.get_vocab())sorted_fast_tokenizer = get_sorted_dict(fast_tokenizer.get_vocab())