Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Make Unsloth-patched tokenizers and processors picklable (#13209)
Daniel Han810efe4d2e22
diff --git a/tests/test_tokenizer_pickle.py b/tests/test_tokenizer_pickle.pynew file mode 100644index 00000000000..ca0abd9467f--- /dev/null+++ b/tests/test_tokenizer_pickle.py@@ -0,0 +1,86 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Patched tokenizers must pickle (spawn DataLoader, datasets.map(num_proc), TRL AsyncGRPO)."""++import copy+import inspect+import os+import pickle+import subprocess+import sys++import pytest++TOKENIZER = os.environ.get(+ "UNSLOTH_TEST_TINY_TOKENIZER", "trl-internal-testing/tiny-Qwen3ForCausalLM"+)+++def patched():+ transformers = pytest.importorskip("transformers")+ save = pytest.importorskip("unsloth.save")+ try:+ tokenizer = transformers.AutoTokenizer.from_pretrained(TOKENIZER)+ except Exception as error:+ pytest.skip(f"could not load {TOKENIZER}: {error}")+ return save, save.patch_saving_functions(tokenizer)+++def test_patched_tokenizer_pickles_to_a_stock_tokenizer(patched):+ _, tokenizer = patched+ assert tokenizer.save_pretrained.__name__ == "unsloth_tokenizer_save_pretrained"+ loaded = pickle.loads(pickle.dumps(tokenizer))+ assert loaded("hello world")["input_ids"] == tokenizer("hello world")["input_ids"]+ assert loaded.save_pretrained.__func__ is type(loaded).save_pretrained+ assert loaded.push_to_hub.__func__ is type(loaded).push_to_hub+ assert loaded.original_save_pretrained.__func__ is type(loaded).save_pretrained+++def test_live_wrapper_still_behaves_like_a_bound_method(patched, tmp_path):+ _, tokenizer = patched+ wrapper = tokenizer.save_pretrained+ assert wrapper.__self__ is tokenizer+ assert "save_directory" in inspect.signature(wrapper).parameters+ assert "self" not in inspect.signature(wrapper).parameters+ wrapper(str(tmp_path))+ assert (tmp_path / "tokenizer_config.json").is_file()+ clone = copy.deepcopy(tokenizer)+ assert clone.save_pretrained.__self__ is clone+ assert clone.save_pretrained.__name__ == "unsloth_tokenizer_save_pretrained"+++def test_patching_twice_keeps_one_wrapper(patched):+ save, tokenizer = patched+ original = tokenizer.original_save_pretrained+ save.patch_saving_functions(tokenizer)+ assert tokenizer.original_save_pretrained is original+ assert tokenizer.push_to_hub.__name__ == "unsloth_push_to_hub"+++def test_unsloth_only_methods_unpickle_as_none(patched):+ save, tokenizer = patched+ save.patch_saving_functions(tokenizer, vision = True)+ assert callable(tokenizer.save_pretrained_merged)+ loaded = pickle.loads(pickle.dumps(tokenizer))+ assert loaded.save_pretrained_merged is None+ assert loaded.push_to_hub_gguf is None+++def test_reader_needs_no_unsloth(patched):+ _, tokenizer = patched+ code = (+ "import pickle, sys\n"+ "t = pickle.loads(sys.stdin.buffer.read())\n"+ "assert 'unsloth' not in sys.modules, 'unpickling imported unsloth'\n"+ "print(t('hello world')['input_ids'])\n"+ )+ result = subprocess.run(+ [sys.executable, "-c", code],+ input = pickle.dumps(tokenizer),+ capture_output = True,+ timeout = 300,+ )+ assert result.returncode == 0, result.stderr.decode()[-2000:]+ assert result.stdout.decode().strip() == str(tokenizer("hello world")["input_ids"])diff --git a/unsloth/save.py b/unsloth/save.pyindex f83990eff5a..0274aa171d1 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -7303,9 +7303,93 @@ def not_implemented_save(*args, **kwargs):raise NotImplementedError("Unsloth: Sorry GGUF is currently not supported for vision models!")+class _TokenizerBoundMethod:+ """Picklable `types.MethodType`: a bound method pickles as getattr(obj, func.__name__), which no+ tokenizer has. Unpickled it becomes the class's own method (or None), so readers need no Unsloth."""++ def __init__(self, func, obj, name):+ self.__func__ = func+ self.__self__ = obj+ self._unsloth_name = name++ def __call__(self, *args, **kwargs):+ return self.__func__(self.__self__, *args, **kwargs)++ def __getattr__(self, name):+ if name.startswith("_unsloth") or name in ("__func__", "__self__"):+ raise AttributeError(name)+ return getattr(self.__func__, name)++ @property+ def __doc__(self):+ return self.__func__.__doc__++ @property+ def __signature__(self):+ import inspect+ import types+ return inspect.signature(types.MethodType(self.__func__, self.__self__))++ def __reduce__(self):+ # During unpickling __dict__ is still empty, so this resolves the class attribute.+ if hasattr(type(self.__self__), self._unsloth_name):+ return (getattr, (self.__self__, self._unsloth_name))+ return (type(None), ())++ def __copy__(self):+ return self++ def __deepcopy__(self, memo):+ import copy+ return type(self)(self.__func__, copy.deepcopy(self.__self__, memo), self._unsloth_name)+++def _bind_saving_method(func, obj, name):+ import types+ if isinstance(obj, (PreTrainedTokenizerBase, ProcessorMixin)):+ return _TokenizerBoundMethod(func, obj, name)+ return types.MethodType(func, obj)+++def unsloth_tokenizer_save_pretrained(+ self,+ save_directory,+ legacy_format = None,+ filename_prefix = None,+ push_to_hub = False,+ **kwargs,+):+ result = self.original_save_pretrained(+ save_directory,+ legacy_format = legacy_format,+ filename_prefix = filename_prefix,+ push_to_hub = False,+ **kwargs,+ )+ _preserve_sentencepiece_tokenizer_assets(+ self,+ save_directory,+ token = kwargs.get("token", None),+ )+ _preserve_tokenizer_eos_token(+ self,+ save_directory,+ filename_prefix = filename_prefix,+ )+ _preserve_repaired_tokenizer_class(+ self,+ save_directory,+ filename_prefix = filename_prefix,+ )+ if push_to_hub:+ push_kwargs = dict(kwargs)+ repo_id = push_kwargs.pop("repo_id", save_directory)+ self.push_to_hub(repo_id, **push_kwargs)+ return result++def patch_saving_functions(model, vision = False):import inspect- import typesfrom typing import Callable, Optional, Union, Listif model.push_to_hub.__name__ == "unsloth_push_to_hub":@@ -7378,42 +7462,6 @@ def patch_saving_functions(model, vision = False):'''exec(push_to_hub_text, globals())- def unsloth_tokenizer_save_pretrained(- self,- save_directory,- legacy_format = None,- filename_prefix = None,- push_to_hub = False,- **kwargs,- ):- result = self.original_save_pretrained(- save_directory,- legacy_format = legacy_format,- filename_prefix = filename_prefix,- push_to_hub = False,- **kwargs,- )- _preserve_sentencepiece_tokenizer_assets(- self,- save_directory,- token = kwargs.get("token", None),- )- _preserve_tokenizer_eos_token(- self,- save_directory,- filename_prefix = filename_prefix,- )- _preserve_repaired_tokenizer_class(- self,- save_directory,- filename_prefix = filename_prefix,- )- if push_to_hub:- push_kwargs = dict(kwargs)- repo_id = push_kwargs.pop("repo_id", save_directory)- self.push_to_hub(repo_id, **push_kwargs)- return result-def unsloth_model_save_pretrained(self, *args, **kwargs):"""`safe_serialization = None` means the safetensors default, not a pickle.@@ -7453,7 +7501,9 @@ def patch_saving_functions(model, vision = False):and model.save_pretrained.__name__ != "unsloth_tokenizer_save_pretrained"):model.original_save_pretrained = model.save_pretrained- model.save_pretrained = types.MethodType(unsloth_tokenizer_save_pretrained, model)+ model.save_pretrained = _bind_saving_method(+ unsloth_tokenizer_save_pretrained, model, "save_pretrained"+ )elif getattr(model, "tokenizer", None) is not None:patch_saving_functions(model.tokenizer)@@ -7466,7 +7516,9 @@ def patch_saving_functions(model, vision = False):and getattr(model.save_pretrained, "__name__", "") != "unsloth_model_save_pretrained"):model.original_model_save_pretrained = model.save_pretrained- model.save_pretrained = types.MethodType(unsloth_model_save_pretrained, model)+ model.save_pretrained = _bind_saving_method(+ unsloth_model_save_pretrained, model, "save_pretrained"+ )original_model = modelwhile True:@@ -7475,7 +7527,9 @@ def patch_saving_functions(model, vision = False):and original_model.push_to_hub.__name__ != "unsloth_push_to_hub"):original_model.original_push_to_hub = original_model.push_to_hub- original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)+ original_model.push_to_hub = _bind_saving_method(+ unsloth_push_to_hub, original_model, "push_to_hub"+ )if hasattr(original_model, "add_model_tags"):original_model.add_model_tags([@@ -7490,33 +7544,55 @@ def patch_saving_functions(model, vision = False):if not vision:if hasattr(model, "config"):- model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)- model.save_pretrained_merged = types.MethodType(- unsloth_generic_save_pretrained_merged, model+ model.push_to_hub_merged = _bind_saving_method(+ unsloth_generic_push_to_hub_merged, model, "push_to_hub_merged"+ )+ model.save_pretrained_merged = _bind_saving_method(+ unsloth_generic_save_pretrained_merged, model, "save_pretrained_merged"+ )+ model.push_to_hub_gguf = _bind_saving_method(+ unsloth_push_to_hub_gguf, model, "push_to_hub_gguf"+ )+ model.save_pretrained_gguf = _bind_saving_method(+ unsloth_save_pretrained_gguf, model, "save_pretrained_gguf")- model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)- model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)- model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)- model.save_pretrained_openvino = types.MethodType(- unsloth_save_pretrained_openvino, model+ model.save_pretrained_torchao = _bind_saving_method(+ unsloth_save_pretrained_torchao, model, "save_pretrained_torchao")- model.push_to_hub_openvino = types.MethodType(unsloth_push_to_hub_openvino, model)- model.push_to_hub_ggml = types.MethodType(- unsloth_convert_lora_to_ggml_and_push_to_hub, model+ model.save_pretrained_openvino = _bind_saving_method(+ unsloth_save_pretrained_openvino, model, "save_pretrained_openvino")- model.save_pretrained_ggml = types.MethodType(- unsloth_convert_lora_to_ggml_and_save_locally, model+ model.push_to_hub_openvino = _bind_saving_method(+ unsloth_push_to_hub_openvino, model, "push_to_hub_openvino"+ )+ model.push_to_hub_ggml = _bind_saving_method(+ unsloth_convert_lora_to_ggml_and_push_to_hub, model, "push_to_hub_ggml"+ )+ model.save_pretrained_ggml = _bind_saving_method(+ unsloth_convert_lora_to_ggml_and_save_locally, model, "save_pretrained_ggml")else:- model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)- model.save_pretrained_merged = types.MethodType(- unsloth_generic_save_pretrained_merged, model- )- model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)- model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)- model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)- model.save_pretrained_openvino = types.MethodType(unsloth_save_pretrained_openvino, model)- model.push_to_hub_openvino = types.MethodType(unsloth_push_to_hub_openvino, model)+ model.push_to_hub_merged = _bind_saving_method(+ unsloth_generic_push_to_hub_merged, model, "push_to_hub_merged"+ )+ model.save_pretrained_merged = _bind_saving_method(+ unsloth_generic_save_pretrained_merged, model, "save_pretrained_merged"+ )+ model.push_to_hub_gguf = _bind_saving_method(+ unsloth_push_to_hub_gguf, model, "push_to_hub_gguf"+ )+ model.save_pretrained_gguf = _bind_saving_method(+ unsloth_save_pretrained_gguf, model, "save_pretrained_gguf"+ )+ model.save_pretrained_torchao = _bind_saving_method(+ unsloth_save_pretrained_torchao, model, "save_pretrained_torchao"+ )+ model.save_pretrained_openvino = _bind_saving_method(+ unsloth_save_pretrained_openvino, model, "save_pretrained_openvino"+ )+ model.push_to_hub_openvino = _bind_saving_method(+ unsloth_push_to_hub_openvino, model, "push_to_hub_openvino"+ )return model