Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Make Unsloth-patched tokenizers and processors picklable (#13209)

Daniel Han810efe4d2e22
diff --git a/tests/test_tokenizer_pickle.py b/tests/test_tokenizer_pickle.py
new file mode 100644
index 00000000000..ca0abd9467f
--- /dev/null
+++ b/tests/test_tokenizer_pickle.py
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Patched tokenizers must pickle (spawn DataLoader, datasets.map(num_proc), TRL AsyncGRPO)."""
+
+import copy
+import inspect
+import os
+import pickle
+import subprocess
+import sys
+
+import pytest
+
+TOKENIZER = os.environ.get(
+ "UNSLOTH_TEST_TINY_TOKENIZER", "trl-internal-testing/tiny-Qwen3ForCausalLM"
+)
+
+
+def patched():
+ transformers = pytest.importorskip("transformers")
+ save = pytest.importorskip("unsloth.save")
+ try:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(TOKENIZER)
+ except Exception as error:
+ pytest.skip(f"could not load {TOKENIZER}: {error}")
+ return save, save.patch_saving_functions(tokenizer)
+
+
+def test_patched_tokenizer_pickles_to_a_stock_tokenizer(patched):
+ _, tokenizer = patched
+ assert tokenizer.save_pretrained.__name__ == "unsloth_tokenizer_save_pretrained"
+ loaded = pickle.loads(pickle.dumps(tokenizer))
+ assert loaded("hello world")["input_ids"] == tokenizer("hello world")["input_ids"]
+ assert loaded.save_pretrained.__func__ is type(loaded).save_pretrained
+ assert loaded.push_to_hub.__func__ is type(loaded).push_to_hub
+ assert loaded.original_save_pretrained.__func__ is type(loaded).save_pretrained
+
+
+def test_live_wrapper_still_behaves_like_a_bound_method(patched, tmp_path):
+ _, tokenizer = patched
+ wrapper = tokenizer.save_pretrained
+ assert wrapper.__self__ is tokenizer
+ assert "save_directory" in inspect.signature(wrapper).parameters
+ assert "self" not in inspect.signature(wrapper).parameters
+ wrapper(str(tmp_path))
+ assert (tmp_path / "tokenizer_config.json").is_file()
+ clone = copy.deepcopy(tokenizer)
+ assert clone.save_pretrained.__self__ is clone
+ assert clone.save_pretrained.__name__ == "unsloth_tokenizer_save_pretrained"
+
+
+def test_patching_twice_keeps_one_wrapper(patched):
+ save, tokenizer = patched
+ original = tokenizer.original_save_pretrained
+ save.patch_saving_functions(tokenizer)
+ assert tokenizer.original_save_pretrained is original
+ assert tokenizer.push_to_hub.__name__ == "unsloth_push_to_hub"
+
+
+def test_unsloth_only_methods_unpickle_as_none(patched):
+ save, tokenizer = patched
+ save.patch_saving_functions(tokenizer, vision = True)
+ assert callable(tokenizer.save_pretrained_merged)
+ loaded = pickle.loads(pickle.dumps(tokenizer))
+ assert loaded.save_pretrained_merged is None
+ assert loaded.push_to_hub_gguf is None
+
+
+def test_reader_needs_no_unsloth(patched):
+ _, tokenizer = patched
+ code = (
+ "import pickle, sys\n"
+ "t = pickle.loads(sys.stdin.buffer.read())\n"
+ "assert 'unsloth' not in sys.modules, 'unpickling imported unsloth'\n"
+ "print(t('hello world')['input_ids'])\n"
+ )
+ result = subprocess.run(
+ [sys.executable, "-c", code],
+ input = pickle.dumps(tokenizer),
+ capture_output = True,
+ timeout = 300,
+ )
+ assert result.returncode == 0, result.stderr.decode()[-2000:]
+ assert result.stdout.decode().strip() == str(tokenizer("hello world")["input_ids"])
diff --git a/unsloth/save.py b/unsloth/save.py
index f83990eff5a..0274aa171d1 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -7303,9 +7303,93 @@ def not_implemented_save(*args, **kwargs):
raise NotImplementedError("Unsloth: Sorry GGUF is currently not supported for vision models!")
+class _TokenizerBoundMethod:
+ """Picklable `types.MethodType`: a bound method pickles as getattr(obj, func.__name__), which no
+ tokenizer has. Unpickled it becomes the class's own method (or None), so readers need no Unsloth."""
+
+ def __init__(self, func, obj, name):
+ self.__func__ = func
+ self.__self__ = obj
+ self._unsloth_name = name
+
+ def __call__(self, *args, **kwargs):
+ return self.__func__(self.__self__, *args, **kwargs)
+
+ def __getattr__(self, name):
+ if name.startswith("_unsloth") or name in ("__func__", "__self__"):
+ raise AttributeError(name)
+ return getattr(self.__func__, name)
+
+ @property
+ def __doc__(self):
+ return self.__func__.__doc__
+
+ @property
+ def __signature__(self):
+ import inspect
+ import types
+ return inspect.signature(types.MethodType(self.__func__, self.__self__))
+
+ def __reduce__(self):
+ # During unpickling __dict__ is still empty, so this resolves the class attribute.
+ if hasattr(type(self.__self__), self._unsloth_name):
+ return (getattr, (self.__self__, self._unsloth_name))
+ return (type(None), ())
+
+ def __copy__(self):
+ return self
+
+ def __deepcopy__(self, memo):
+ import copy
+ return type(self)(self.__func__, copy.deepcopy(self.__self__, memo), self._unsloth_name)
+
+
+def _bind_saving_method(func, obj, name):
+ import types
+ if isinstance(obj, (PreTrainedTokenizerBase, ProcessorMixin)):
+ return _TokenizerBoundMethod(func, obj, name)
+ return types.MethodType(func, obj)
+
+
+def unsloth_tokenizer_save_pretrained(
+ self,
+ save_directory,
+ legacy_format = None,
+ filename_prefix = None,
+ push_to_hub = False,
+ **kwargs,
+):
+ result = self.original_save_pretrained(
+ save_directory,
+ legacy_format = legacy_format,
+ filename_prefix = filename_prefix,
+ push_to_hub = False,
+ **kwargs,
+ )
+ _preserve_sentencepiece_tokenizer_assets(
+ self,
+ save_directory,
+ token = kwargs.get("token", None),
+ )
+ _preserve_tokenizer_eos_token(
+ self,
+ save_directory,
+ filename_prefix = filename_prefix,
+ )
+ _preserve_repaired_tokenizer_class(
+ self,
+ save_directory,
+ filename_prefix = filename_prefix,
+ )
+ if push_to_hub:
+ push_kwargs = dict(kwargs)
+ repo_id = push_kwargs.pop("repo_id", save_directory)
+ self.push_to_hub(repo_id, **push_kwargs)
+ return result
+
+
def patch_saving_functions(model, vision = False):
import inspect
- import types
from typing import Callable, Optional, Union, List
if model.push_to_hub.__name__ == "unsloth_push_to_hub":
@@ -7378,42 +7462,6 @@ def patch_saving_functions(model, vision = False):
'''
exec(push_to_hub_text, globals())
- def unsloth_tokenizer_save_pretrained(
- self,
- save_directory,
- legacy_format = None,
- filename_prefix = None,
- push_to_hub = False,
- **kwargs,
- ):
- result = self.original_save_pretrained(
- save_directory,
- legacy_format = legacy_format,
- filename_prefix = filename_prefix,
- push_to_hub = False,
- **kwargs,
- )
- _preserve_sentencepiece_tokenizer_assets(
- self,
- save_directory,
- token = kwargs.get("token", None),
- )
- _preserve_tokenizer_eos_token(
- self,
- save_directory,
- filename_prefix = filename_prefix,
- )
- _preserve_repaired_tokenizer_class(
- self,
- save_directory,
- filename_prefix = filename_prefix,
- )
- if push_to_hub:
- push_kwargs = dict(kwargs)
- repo_id = push_kwargs.pop("repo_id", save_directory)
- self.push_to_hub(repo_id, **push_kwargs)
- return result
-
def unsloth_model_save_pretrained(self, *args, **kwargs):
"""`safe_serialization = None` means the safetensors default, not a pickle.
@@ -7453,7 +7501,9 @@ def patch_saving_functions(model, vision = False):
and model.save_pretrained.__name__ != "unsloth_tokenizer_save_pretrained"
):
model.original_save_pretrained = model.save_pretrained
- model.save_pretrained = types.MethodType(unsloth_tokenizer_save_pretrained, model)
+ model.save_pretrained = _bind_saving_method(
+ unsloth_tokenizer_save_pretrained, model, "save_pretrained"
+ )
elif getattr(model, "tokenizer", None) is not None:
patch_saving_functions(model.tokenizer)
@@ -7466,7 +7516,9 @@ def patch_saving_functions(model, vision = False):
and getattr(model.save_pretrained, "__name__", "") != "unsloth_model_save_pretrained"
):
model.original_model_save_pretrained = model.save_pretrained
- model.save_pretrained = types.MethodType(unsloth_model_save_pretrained, model)
+ model.save_pretrained = _bind_saving_method(
+ unsloth_model_save_pretrained, model, "save_pretrained"
+ )
original_model = model
while True:
@@ -7475,7 +7527,9 @@ def patch_saving_functions(model, vision = False):
and original_model.push_to_hub.__name__ != "unsloth_push_to_hub"
):
original_model.original_push_to_hub = original_model.push_to_hub
- original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)
+ original_model.push_to_hub = _bind_saving_method(
+ unsloth_push_to_hub, original_model, "push_to_hub"
+ )
if hasattr(original_model, "add_model_tags"):
original_model.add_model_tags(
[
@@ -7490,33 +7544,55 @@ def patch_saving_functions(model, vision = False):
if not vision:
if hasattr(model, "config"):
- model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)
- model.save_pretrained_merged = types.MethodType(
- unsloth_generic_save_pretrained_merged, model
+ model.push_to_hub_merged = _bind_saving_method(
+ unsloth_generic_push_to_hub_merged, model, "push_to_hub_merged"
+ )
+ model.save_pretrained_merged = _bind_saving_method(
+ unsloth_generic_save_pretrained_merged, model, "save_pretrained_merged"
+ )
+ model.push_to_hub_gguf = _bind_saving_method(
+ unsloth_push_to_hub_gguf, model, "push_to_hub_gguf"
+ )
+ model.save_pretrained_gguf = _bind_saving_method(
+ unsloth_save_pretrained_gguf, model, "save_pretrained_gguf"
)
- model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)
- model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)
- model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
- model.save_pretrained_openvino = types.MethodType(
- unsloth_save_pretrained_openvino, model
+ model.save_pretrained_torchao = _bind_saving_method(
+ unsloth_save_pretrained_torchao, model, "save_pretrained_torchao"
)
- model.push_to_hub_openvino = types.MethodType(unsloth_push_to_hub_openvino, model)
- model.push_to_hub_ggml = types.MethodType(
- unsloth_convert_lora_to_ggml_and_push_to_hub, model
+ model.save_pretrained_openvino = _bind_saving_method(
+ unsloth_save_pretrained_openvino, model, "save_pretrained_openvino"
)
- model.save_pretrained_ggml = types.MethodType(
- unsloth_convert_lora_to_ggml_and_save_locally, model
+ model.push_to_hub_openvino = _bind_saving_method(
+ unsloth_push_to_hub_openvino, model, "push_to_hub_openvino"
+ )
+ model.push_to_hub_ggml = _bind_saving_method(
+ unsloth_convert_lora_to_ggml_and_push_to_hub, model, "push_to_hub_ggml"
+ )
+ model.save_pretrained_ggml = _bind_saving_method(
+ unsloth_convert_lora_to_ggml_and_save_locally, model, "save_pretrained_ggml"
)
else:
- model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)
- model.save_pretrained_merged = types.MethodType(
- unsloth_generic_save_pretrained_merged, model
- )
- model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)
- model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)
- model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
- model.save_pretrained_openvino = types.MethodType(unsloth_save_pretrained_openvino, model)
- model.push_to_hub_openvino = types.MethodType(unsloth_push_to_hub_openvino, model)
+ model.push_to_hub_merged = _bind_saving_method(
+ unsloth_generic_push_to_hub_merged, model, "push_to_hub_merged"
+ )
+ model.save_pretrained_merged = _bind_saving_method(
+ unsloth_generic_save_pretrained_merged, model, "save_pretrained_merged"
+ )
+ model.push_to_hub_gguf = _bind_saving_method(
+ unsloth_push_to_hub_gguf, model, "push_to_hub_gguf"
+ )
+ model.save_pretrained_gguf = _bind_saving_method(
+ unsloth_save_pretrained_gguf, model, "save_pretrained_gguf"
+ )
+ model.save_pretrained_torchao = _bind_saving_method(
+ unsloth_save_pretrained_torchao, model, "save_pretrained_torchao"
+ )
+ model.save_pretrained_openvino = _bind_saving_method(
+ unsloth_save_pretrained_openvino, model, "save_pretrained_openvino"
+ )
+ model.push_to_hub_openvino = _bind_saving_method(
+ unsloth_push_to_hub_openvino, model, "push_to_hub_openvino"
+ )
return model