Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fine-tune and export native MTP heads and DSpark drafters (#13719)

Daniel Han65d3ca519590
diff --git a/tests/saving/test_modelscope_repo_id.py b/tests/saving/test_modelscope_repo_id.py
index b7155ff9e42..c0432f478c2 100644
--- a/tests/saving/test_modelscope_repo_id.py
+++ b/tests/saving/test_modelscope_repo_id.py
@@ -134,7 +134,12 @@ def test_gguf_still_converts_the_modelscope_snapshot(tmp_path):
reuses = _load(
"unsloth/save.py",
"_gguf_reuses_loaded_checkpoint",
- {"os": os, "PeftModel": _Peft, "PeftModelForCausalLM": _Peft},
+ {
+ "os": os,
+ "PeftModel": _Peft,
+ "PeftModelForCausalLM": _Peft,
+ "_attached_mtp_head": lambda m: None,
+ },
)
model = types.SimpleNamespace(config = types.SimpleNamespace(_name_or_path = "unsloth/x"))
assert not reuses(model)
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index f401126abce..d0482d3d102 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -206,6 +206,12 @@ _GENERIC_SAVE_STUBBED_HELPERS = frozenset(
"_qwen3_5_vlm_state_dict_for_save",
"_determine_username",
"unsloth_save_model",
+ # Trained MTP head hooks: no head is attached in these fixtures.
+ "_refuse_mtp_variant",
+ "_attached_mtp_head",
+ "_internal_mtp_save",
+ "_stitch_mtp_head",
+ "_mtp_head_tensor_names",
}
)
@@ -238,6 +244,11 @@ def _routing_environment(monkeypatch, model):
_qwen3_5_vlm_state_dict_for_save = _not_reached,
_determine_username = lambda repo, old, token: (repo, "owner"),
unsloth_save_model = lambda *args, **kwargs: calls["adapter"].append(kwargs),
+ _refuse_mtp_variant = lambda model, variant: None,
+ _attached_mtp_head = lambda model: None,
+ _internal_mtp_save = contextlib.nullcontext,
+ _stitch_mtp_head = lambda model, directory: None,
+ _mtp_head_tensor_names = lambda model: [],
lora_relative_to_original_base = lambda model: (
calls["convert"].append(model) or contextlib.nullcontext()
),
@@ -406,6 +417,13 @@ def _adapter_save_environment(monkeypatch):
),
fast_save_pickle = lambda *args, **kwargs: None,
clean_gpu_cache = lambda: None,
+ # Trained MTP head hooks: no head is attached in these fixtures.
+ _refuse_mtp_variant = lambda model, variant: None,
+ _attached_mtp_head = lambda model: None,
+ _internal_mtp_save = contextlib.nullcontext,
+ _save_mtp_head_beside_adapter = lambda model, directory: None,
+ _stitch_mtp_head = lambda model, directory: None,
+ _mtp_head_tensor_names = lambda model: [],
), uploads
@@ -711,7 +729,7 @@ def test_the_docstrings_describe_none_as_the_stronger_safetensors_request():
assert (
source.count("`None` is stronger than the default") == 4
), "all four save_method docstrings have to describe None the same way"
- assert "elif safe_serialization and (n_cpus <= 2):" in source
+ assert "elif safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head:" in source
assert "if _force_safe_serialization:" in source
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index d3381bd32e9..c61157b567f 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -167,6 +167,12 @@ def saving(monkeypatch, tmp_path):
_normalize_compressed_method = lambda method: None,
_normalize_torchao_method = lambda method: None,
_is_qwen3_5_vlm = lambda model: False,
+ # Trained MTP head hooks: no head is attached in these fixtures.
+ _refuse_mtp_variant = lambda model, variant: None,
+ _attached_mtp_head = lambda model: None,
+ _internal_mtp_save = contextlib.nullcontext,
+ _stitch_mtp_head = lambda model, directory: None,
+ _mtp_head_tensor_names = lambda model: [],
raise_if_merging_mistral_format_view = lambda model, save_method: None,
lora_relative_to_original_base = lambda model: contextlib.nullcontext(),
nullcontext = contextlib.nullcontext,
diff --git a/tests/test_save_mtp_head_stitch.py b/tests/test_save_mtp_head_stitch.py
new file mode 100644
index 00000000000..0d54b074179
--- /dev/null
+++ b/tests/test_save_mtp_head_stitch.py
@@ -0,0 +1,634 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A trained MTP head attached as `_unsloth_mtp_head` must reach every export: the merge
+copies the STOCK head through from the original shards, so without the stitch a fine-tuned
+model ships a head that was never aligned to it."""
+
+import ast
+from types import SimpleNamespace
+import json
+from pathlib import Path
+
+import pytest
+
+SAVE_PY = Path(__file__).resolve().parents[1] / "unsloth" / "save.py"
+
+
[email protected](scope = "module")
+def tree():
+ return ast.parse(SAVE_PY.read_text(encoding = "utf-8"))
+
+
+def _func(tree, name):
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name:
+ return node
+ raise AssertionError(f"{name} is not defined in save.py")
+
+
+def _calls(node):
+ out = set()
+ for sub in ast.walk(node):
+ if isinstance(sub, ast.Call):
+ f = sub.func
+ out.add(f.id if isinstance(f, ast.Name) else getattr(f, "attr", None))
+ return out
+
+
+ "function, expected",
+ [
+ (
+ "unsloth_save_model",
+ {
+ "_stitch_mtp_head",
+ "_save_mtp_head_beside_adapter",
+ "_push_with_mtp_head",
+ "_mtp_head_tensor_names",
+ },
+ ),
+ (
+ "unsloth_generic_save",
+ {"_stitch_mtp_head", "_attached_mtp_head", "_push_merged_to_hub_revision"},
+ ),
+ ("unsloth_save_pretrained_gguf", {"_stitch_mtp_head"}),
+ ("_gguf_reuses_loaded_checkpoint", {"_attached_mtp_head"}),
+ # torchao's reload drops native MTP tensors: say so instead of exporting it silently.
+ ("_unsloth_save_torchao", {"_attached_mtp_head"}),
+ ],
+)
+def test_save_paths_are_wired(tree, function, expected):
+ assert expected <= _calls(_func(tree, function))
+
+
[email protected](scope = "module")
+def save_module():
+ pytest.importorskip(
+ "unsloth_zoo.speculative.stitch", reason = "unsloth_zoo has no speculative.stitch"
+ )
+ pytest.importorskip("unsloth", reason = "unsloth is not importable on this runner")
+ try:
+ import unsloth.save as module
+ except ImportError as error:
+ pytest.skip(f"unsloth.save is not importable on this runner: {error}")
+ return module
+
+
+def _head(torch):
+ head = torch.nn.Module()
+ head.fc = torch.nn.Linear(16, 8, bias = False)
+ head.norm = torch.nn.Module()
+ head.norm.weight = torch.nn.Parameter(torch.randn(8))
+ object.__setattr__(head, "_unsloth_mtp_info", {"prefix": "mtp.", "source": "tiny"})
+ return head
+
+
+def _wrapped(torch, head, registered):
+ """PeftModel -> LoraModel -> base model, with the head on the innermost model."""
+ base = torch.nn.Module()
+ base.config = None
+ if registered:
+ base._unsloth_mtp_head = head
+ else:
+ object.__setattr__(base, "_unsloth_mtp_head", head)
+ lora = torch.nn.Module()
+ lora.model = base
+ peft = torch.nn.Module()
+ peft.base_model = lora
+ return peft
+
+
[email protected]("registered", [True, False])
+def test_head_is_found_through_wrappers(save_module, registered):
+ import torch
+
+ head = _head(torch)
+ assert save_module._attached_mtp_head(_wrapped(torch, head, registered)) is head
+ assert save_module._attached_mtp_head(torch.nn.Linear(2, 2)) is None
+ assert save_module._mtp_head_tensor_names(torch.nn.Linear(2, 2)) == []
+ assert sorted(save_module._mtp_head_tensor_names(_wrapped(torch, head, registered))) == [
+ "mtp.fc.weight",
+ "mtp.norm.weight",
+ ]
+
+
+def test_no_head_is_a_no_op(save_module, tmp_path):
+ import torch
+
+ model = torch.nn.Linear(2, 2)
+ assert save_module._stitch_mtp_head(model, tmp_path) is None
+ assert save_module._save_mtp_head_beside_adapter(model, tmp_path) is None
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_stitch_appends_and_declares(save_module, tmp_path):
+ import torch
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+
+ save_file(
+ {"model.norm.weight": torch.ones(8, dtype = torch.bfloat16)}, tmp_path / "model.safetensors"
+ )
+ (tmp_path / "config.json").write_text(json.dumps({"text_config": {"num_hidden_layers": 2}}))
+ head = _head(torch)
+ head.layers = torch.nn.ModuleList([torch.nn.Module()])
+ head.layers[0].up = torch.nn.Linear(8, 8, bias = False)
+ report = save_module._stitch_mtp_head(_wrapped(torch, head, registered = False), tmp_path)
+ assert report["mode"] == "appended"
+ index = json.loads((tmp_path / "model.safetensors.index.json").read_text())
+ assert {"mtp.fc.weight", "mtp.norm.weight", "mtp.layers.0.up.weight"} <= set(
+ index["weight_map"]
+ )
+ assert (
+ json.loads((tmp_path / "config.json").read_text())["text_config"]["mtp_num_hidden_layers"]
+ == 1
+ )
+ with safe_open(str(tmp_path / index["weight_map"]["mtp.fc.weight"]), "pt") as f:
+ assert torch.equal(
+ f.get_tensor("mtp.fc.weight"), head.fc.weight.detach().to(torch.bfloat16)
+ )
+
+
+def test_adapter_save_writes_head_file(save_module, tmp_path):
+ import torch
+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head
+
+ head = _head(torch)
+ save_module._save_mtp_head_beside_adapter(_wrapped(torch, head, registered = True), tmp_path)
+ loaded, info = load_saved_mtp_head(tmp_path)
+ assert set(loaded) == {"fc.weight", "norm.weight"} and info["source"] == "tiny"
+
+
+def test_gguf_does_not_reuse_checkpoint_with_head(save_module, tmp_path):
+ import torch
+
+ model = torch.nn.Module()
+ model.config = type("C", (), {"_name_or_path": str(tmp_path)})()
+ assert save_module._gguf_reuses_loaded_checkpoint(model) is True
+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))
+ assert save_module._gguf_reuses_loaded_checkpoint(model) is False
+
+
+def test_plain_save_pretrained_keeps_the_head(save_module, tmp_path):
+ import torch
+ from unsloth.models.speculative import _wrap_save_pretrained
+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head
+
+ calls = []
+
+ class Adapter(torch.nn.Module):
+ def save_pretrained(self, save_directory, **kwargs):
+ calls.append(save_directory)
+ (Path(save_directory) / "adapter_config.json").write_text("{}")
+
+ model = Adapter()
+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))
+ _wrap_save_pretrained(model)
+ _wrap_save_pretrained(model)
+ model.save_pretrained(str(tmp_path))
+ assert calls == [str(tmp_path)]
+ loaded, _ = load_saved_mtp_head(tmp_path)
+ assert set(loaded) == {"fc.weight", "norm.weight"}
+
+
+def test_internal_savers_stitch_once(save_module, tmp_path, monkeypatch):
+ """Inside `_internal_mtp_save` the plain-save wrapper does not stitch, even under an outer wrapper
+ (an export reload wraps `save_pretrained` again), so one save writes the head once."""
+ import torch
+ from unsloth.models.speculative import _wrap_save_pretrained
+
+ stitched = []
+ monkeypatch.setattr(save_module, "_stitch_mtp_head", lambda m, d: stitched.append(d))
+
+ class Plain(torch.nn.Module):
+ def save_pretrained(self, save_directory, **kwargs):
+ (Path(save_directory) / "config.json").write_text("{}")
+
+ model = Plain()
+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))
+ _wrap_save_pretrained(model)
+ inner = model.save_pretrained
+ object.__setattr__(model, "save_pretrained", lambda *a, **k: inner(*a, **k))
+ with save_module._internal_mtp_save():
+ model.save_pretrained(str(tmp_path))
+ assert stitched == []
+ model.save_pretrained(str(tmp_path))
+ assert stitched == [str(tmp_path)]
+
+
+def test_reattach_is_a_no_op_without_a_saved_head(tmp_path):
+ loader_tree = ast.parse((SAVE_PY.parent / "models" / "loader.py").read_text(encoding = "utf-8"))
+ calls = [
+ _calls(n)
+ for n in ast.walk(loader_tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_reattach_saved_mtp_head"
+ ]
+ assert calls, "loader.py lost _reattach_saved_mtp_head"
+ sites = [
+ n
+ for n in ast.walk(loader_tree)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "_reattach_saved_mtp_head"
+ ]
+ assert len(sites) == 2
+ for site in sites:
+ assert {k.arg for k in site.keywords} == {
+ "token",
+ "revision",
+ "cache_dir",
+ "local_files_only",
+ }
+
+
+def _tiny_glm(tmp_path, torch):
+ pytest.importorskip("transformers.models.glm4_moe")
+ from transformers import Glm4MoeForCausalLM
+ from transformers.models.glm4_moe.configuration_glm4_moe import Glm4MoeConfig
+
+ torch.manual_seed(0)
+ config = Glm4MoeConfig(
+ vocab_size = 97,
+ hidden_size = 64,
+ intermediate_size = 128,
+ moe_intermediate_size = 32,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ num_hidden_layers = 2,
+ n_routed_experts = 4,
+ num_experts_per_tok = 2,
+ n_shared_experts = 1,
+ first_k_dense_replace = 1,
+ num_nextn_predict_layers = 1,
+ partial_rotary_factor = 0.5,
+ )
+ return Glm4MoeForCausalLM(config).eval()
+
+
+def test_nextn_head_round_trips_through_save_pretrained(save_module, tmp_path):
+ """A GLM-style head (model.layers.{N}.*) is appended on a plain save and reloads identically."""
+ import torch
+ from safetensors.torch import load_file
+
+ pytest.importorskip("unsloth_zoo.speculative.nextn")
+ from unsloth.models.speculative import get_mtp_head
+
+ model = _tiny_glm(tmp_path, torch)
+ head = get_mtp_head(model, random_init = True)
+ assert type(head).__name__ == "NextNHead"
+ with torch.no_grad():
+ for p in head.parameters():
+ p.add_(0.01)
+ expected = {k: v.detach().clone() for k, v in head.mtp_state_dict().items()}
+ assert all(name.startswith("model.layers.2.") for name in expected)
+ assert sorted(save_module._mtp_head_tensor_names(model)) == sorted(expected)
+
+ out = tmp_path / "saved"
+ model.save_pretrained(str(out))
+ config = json.loads((out / "config.json").read_text())
+ assert config["num_nextn_predict_layers"] == 1 and "mtp_num_hidden_layers" not in config
+ written = {}
+ for shard in out.glob("*.safetensors"):
+ written.update(load_file(str(shard)))
+ for name, tensor in expected.items():
+ assert torch.equal(written[name], tensor.to(written[name].dtype)), name
+
+ fresh = _tiny_glm(tmp_path, torch)
+ reloaded = get_mtp_head(fresh, source = str(out))
+ assert all(torch.equal(reloaded.mtp_state_dict()[k], v) for k, v in expected.items())
+
+ # Saving again over a checkpoint that now carries the head rewrites it in place.
+ report = save_module._stitch_mtp_head(fresh, str(out))
+ assert report["mode"] == "replaced"
+
+
+class _AdapterSaver:
+ """A model whose save_pretrained writes an adapter folder, like PEFT's."""
+
+ def __init__(self, torch):
+ self.module = torch.nn.Module()
+ self.saved = []
+
+ def save_pretrained(self, save_directory, **kwargs):
+ self.saved.append(kwargs)
+ Path(save_directory).mkdir(parents = True, exist_ok = True)
+ (Path(save_directory) / "adapter_config.json").write_text("{}")
+
+
+def _wrapped_adapter(torch, pushes = True):
+ """`pushes`: transformers' save_pretrained (takes push_to_hub), else PEFT's (does not)."""
+ from unsloth.models.speculative import _wrap_save_pretrained
+
+ model = torch.nn.Module()
+ calls = []
+
+ def transformers_save(
+ save_directory,
+ push_to_hub = False,
+ **kwargs,
+ ):
+ calls.append(dict(kwargs, push_to_hub = push_to_hub))
+ Path(save_directory).mkdir(parents = True, exist_ok = True)
+ (Path(save_directory) / "adapter_config.json").write_text("{}")
+
+ def save_pretrained(save_directory, **kwargs):
+ calls.append(kwargs)
+ Path(save_directory).mkdir(parents = True, exist_ok = True)
+ (Path(save_directory) / "adapter_config.json").write_text("{}")
+
+ object.__setattr__(model, "save_pretrained", transformers_save if pushes else save_pretrained)
+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))
+ _wrap_save_pretrained(model)
+ return model, calls
+
+
+def test_plain_save_refuses_a_variant_before_writing(save_module, tmp_path):
+ import torch
+
+ model, calls = _wrapped_adapter(torch)
+ with pytest.raises(NotImplementedError):
+ model.save_pretrained(str(tmp_path / "v"), variant = "foo")
+ assert calls == []
+
+
+def test_plain_save_push_uploads_the_head(save_module, tmp_path, monkeypatch):
+ import huggingface_hub
+ import torch
+
+ uploads = []
+
+ class Api:
+ def __init__(self, token = None):
+ self.token = token
+
+ def upload_folder(self, **kwargs):
+ uploads.append(kwargs)
+
+ monkeypatch.setattr(huggingface_hub, "HfApi", Api)
+ model, _ = _wrapped_adapter(torch)
+ out = tmp_path / "me-adapter"
+ model.save_pretrained(str(out), push_to_hub = True, repo_id = "me/adapter", token = "t")
+ assert (out / "mtp_head.safetensors").is_file()
+ assert (
+ uploads
+ and uploads[0]["repo_id"] == "me/adapter"
+ and uploads[0]["allow_patterns"] == ["mtp_head.safetensors"]
+ )
+
+
+def test_plain_save_push_skips_peft_and_refuses_create_pr(save_module, tmp_path, monkeypatch):
+ import huggingface_hub
+ import torch
+
+ uploads = []
+
+ class Api:
+ def __init__(self, token = None):
+ pass
+
+ def upload_folder(self, **kwargs):
+ uploads.append(kwargs)
+
+ monkeypatch.setattr(huggingface_hub, "HfApi", Api)
+ peft_model, _ = _wrapped_adapter(torch, pushes = False)
+ peft_model.save_pretrained(str(tmp_path / "a"), push_to_hub = True, repo_id = "me/adapter")
+ assert (tmp_path / "a" / "mtp_head.safetensors").is_file() and uploads == []
+ model, calls = _wrapped_adapter(torch)
+ with pytest.raises(ValueError, match = "create_pr"):
+ model.save_pretrained(str(tmp_path / "b"), push_to_hub = True, repo_id = "me/m", create_pr = True)
+ assert calls == [] and uploads == []
+
+
+def test_adapter_save_without_a_head_drops_a_stale_one(save_module, tmp_path):
+ import torch
+
+ (tmp_path / "mtp_head.safetensors").write_bytes(b"old")
+ save_module._save_mtp_head_beside_adapter(torch.nn.Module(), str(tmp_path))
+ assert not (tmp_path / "mtp_head.safetensors").exists()
+
+
+def test_modelscope_snapshot_is_the_head_source(tmp_path):
+ import torch
+ from unsloth.models.speculative import _checkpoint_source
+
+ model = torch.nn.Module()
+ model.config = type("C", (), {"_name_or_path": "Org/Model"})()
+ assert _checkpoint_source(model) == "Org/Model"
+ model._unsloth_modelscope_snapshot = str(tmp_path)
+ assert _checkpoint_source(model) == str(tmp_path)
+
+
+def test_hub_adapter_reattaches_its_head(tmp_path, monkeypatch):
+ import huggingface_hub
+ from unsloth.models import loader, speculative
+
+ head_file = tmp_path / "snap" / "mtp_head.safetensors"
+ head_file.parent.mkdir()
+ head_file.write_bytes(b"h")
+ seen = []
+ monkeypatch.setattr(
+ huggingface_hub, "hf_hub_download", lambda repo, name, **k: seen.append(k) or str(head_file)
+ )
+ attached = []
+ monkeypatch.setattr(speculative, "attach_saved_mtp_head", lambda model, d: attached.append(d))
+ loader._reattach_saved_mtp_head(
+ object(), "me/adapter", token = "t", cache_dir = "/c", local_files_only = True
+ )
+ assert attached == [str(head_file.parent)]
+ # The adapter's own cache and offline mode, else an offline load reads the default cache.
+ assert seen[0]["cache_dir"] == "/c" and seen[0]["local_files_only"] is True
+
+ def missing(*a, **k):
+ raise FileNotFoundError("no head")
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", missing)
+ loader._reattach_saved_mtp_head(object(), "me/other")
+ assert attached == [str(head_file.parent)]
+
+
+def test_mtp_push_keeps_the_datasets_metadata(save_module, monkeypatch):
+ """The attribution is staged in the README, so it lands in the same commit / revision / PR."""
+ import os
+
+ import huggingface_hub
+
+ calls = []
+
+ class Api:
+ def __init__(self, token = None):
+ pass
+
+ def create_repo(self, **kwargs):
+ pass
+
+ def create_branch(self, **kwargs):
+ pass
+
+ def upload_folder(self, **kwargs):
+ readme = os.path.join(kwargs["folder_path"], "README.md")
+ calls.append((kwargs["revision"], kwargs["create_pr"], open(readme).read()))
+
+ def no_card(*args, **kwargs):
+ raise huggingface_hub.errors.EntryNotFoundError("no README")
+
+ monkeypatch.setattr(save_module, "HfApi", Api)
+ monkeypatch.setattr(save_module, "_determine_username", lambda d, u, t: ("me/m", "me"))
+ monkeypatch.setattr(save_module, "unsloth_save_model", lambda **kwargs: None)
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", no_card)
+ monkeypatch.setattr(
+ huggingface_hub, "metadata_update", lambda *a, **k: pytest.fail("edits the default branch")
+ )
+ save_module._push_with_mtp_head(
+ dict(
+ is_main_process = True,
+ save_directory = "m",
+ token = "t",
+ private = False,
+ revision = "dev",
+ create_pr = True,
+ commit_message = None,
+ datasets = ["org/data"],
+ tags = ["my-tag"],
+ model = SimpleNamespace(
+ config = SimpleNamespace(_name_or_path = "org/base", model_type = "qwen3_5")
+ ),
+ )
+ )
+ assert len(calls) == 1 and calls[0][:2] == ("dev", True) and "org/data" in calls[0][2]
+ # A new repo still gets the hub path's model card, with the caller's tags.
+ assert "unsloth" in calls[0][2] and "my-tag" in calls[0][2] and "org/base" in calls[0][2]
+
+
+def test_unsloth_wrapped_save_still_pushes_the_head(save_module, tmp_path, monkeypatch):
+ """Unsloth's loaders wrap save_pretrained as (self, *args, **kwargs), hiding push_to_hub."""
+ from unsloth.models.speculative import _takes_push_to_hub
+
+ class Model:
+ def original_model_save_pretrained(
+ self,
+ save_directory,
+ push_to_hub = False,
+ **kwargs,
+ ):
+ pass
+
+ def unsloth_model_save_pretrained(self, *args, **kwargs):
+ pass
+
+ assert _takes_push_to_hub(Model().unsloth_model_save_pretrained)
+
+
+def test_merged_save_with_a_head_refuses_pickles(tree):
+ fn = ast.get_source_segment(
+ SAVE_PY.read_text(encoding = "utf-8"), _func(tree, "unsloth_save_model")
+ )
+ assert "if _keeps_mtp_head and not safe_serialization:" in fn
+ assert "safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head" in fn
+
+
+def test_get_mtp_head_returns_the_attached_head(save_module):
+ import torch
+ from unsloth.models.speculative import get_mtp_head
+
+ head = _head(torch)
+ model = _wrapped(torch, head, registered = False)
+ assert get_mtp_head(model) is head
+
+
+def test_plain_save_refuses_pickles_before_writing(save_module, tmp_path):
+ import torch
+
+ model, calls = _wrapped_adapter(torch)
+ with pytest.raises(ValueError, match = "safetensors"):
+ model.save_pretrained(str(tmp_path / "p"), safe_serialization = False)
+ assert calls == []
+
+
+def test_modelscope_snapshot_wins_under_a_peft_wrapper(tmp_path):
+ import torch
+ from unsloth.models.speculative import _checkpoint_source
+
+ inner = torch.nn.Module()
+ inner.config = SimpleNamespace(_name_or_path = "modelscope/only-id")
+ object.__setattr__(inner, "_unsloth_modelscope_snapshot", str(tmp_path))
+ peft = torch.nn.Module()
+ peft.config = SimpleNamespace(_name_or_path = "modelscope/only-id")
+ peft.base_model = torch.nn.Module()
+ peft.base_model.model = inner
+ assert _checkpoint_source(peft) == str(tmp_path)
+
+
+def test_mtp_push_never_replaces_a_card_it_could_not_read(save_module, monkeypatch):
+ import huggingface_hub
+
+ uploads = []
+
+ class Api:
+ def __init__(self, token = None):
+ pass
+
+ def create_repo(self, **kwargs):
+ pass
+
+ def upload_folder(self, **kwargs):
+ uploads.append(kwargs)
+
+ def unreachable(*args, **kwargs):
+ raise huggingface_hub.errors.LocalEntryNotFoundError("offline")
+
+ monkeypatch.setattr(save_module, "HfApi", Api)
+ monkeypatch.setattr(save_module, "_determine_username", lambda d, u, t: ("me/m", "me"))
+ monkeypatch.setattr(save_module, "unsloth_save_model", lambda **kwargs: None)
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", unreachable)
+ with pytest.raises(huggingface_hub.errors.LocalEntryNotFoundError):
+ save_module._push_with_mtp_head(
+ dict(
+ is_main_process = True,
+ save_directory = "m",
+ token = "t",
+ private = False,
+ revision = None,
+ create_pr = False,
+ commit_message = None,
+ datasets = None,
+ tags = None,
+ model = SimpleNamespace(
+ config = SimpleNamespace(_name_or_path = "org/base", model_type = "qwen3_5")
+ ),
+ )
+ )
+ assert uploads == []
+
+
+def test_stale_adapter_config_does_not_turn_a_full_save_into_a_sidecar(
+ save_module, tmp_path, monkeypatch
+):
+ import os
+
+ import torch
+ from unsloth.models.speculative import _wrap_save_pretrained
+
+ out = tmp_path / "reused"
+ out.mkdir()
+ (out / "adapter_config.json").write_text("{}")
+ os.utime(out / "adapter_config.json", (1, 1))
+ stitched, sidecars = [], []
+ monkeypatch.setattr(save_module, "_stitch_mtp_head", lambda m, d: stitched.append(d))
+ monkeypatch.setattr(
+ save_module, "_save_mtp_head_beside_adapter", lambda m, d: sidecars.append(d)
+ )
+ model = torch.nn.Module()
+ object.__setattr__(model, "save_pretrained", lambda save_directory, **kwargs: None)
+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))
+ _wrap_save_pretrained(model)
+ model.save_pretrained(str(out))
+ assert stitched == [str(out)] and sidecars == []
+
+
+def test_get_mtp_head_reads_the_loaded_commit(tmp_path):
+ from unsloth.models.speculative import _loaded_revision
+
+ model = SimpleNamespace(config = SimpleNamespace(_name_or_path = "org/m", _commit_hash = "abc123"))
+ assert _loaded_revision(model, "org/m", None) == "abc123"
+ assert _loaded_revision(model, "org/m", "dev") == "dev"
+ assert _loaded_revision(model, str(tmp_path), None) is None
diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py
index 53ec1d02849..57086371551 100644
--- a/unsloth/models/__init__.py
+++ b/unsloth/models/__init__.py
@@ -27,6 +27,13 @@ except:
# falcon_h1 absent before transformers 4.53.0; skip
pass
from .dpo import PatchDPOTrainer, PatchKTOTrainer
+from .speculative import get_mtp_head
+
+try:
+ from .speculative import UnslothMTPTrainer, MTPTrainingArguments
+ from .speculative import UnslothDSparkTrainer, DSparkTrainingArguments, Qwen3DSparkDrafter
+except ImportError:
+ pass
from ._utils import is_bfloat16_supported, is_vLLM_available, __version__
from .rl import PatchFastRL, vLLMSamplingParams
from .lora_init import patch_peft_calibration_eager as _patch_peft_calibration_eager
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 6dffa18f3f9..671a13cb25d 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -973,6 +973,44 @@ def _vllm_unavailable_error():
)
+def _reattach_saved_mtp_head(
+ model,
+ adapter_dir,
+ token = None,
+ revision = None,
+ cache_dir = None,
+ local_files_only = False,
+):
+ """A LoRA folder (local or on the Hub) saved with a trained MTP head gets it back, so a later
+ merge keeps it."""
+ if not isinstance(adapter_dir, str):
+ return
+ if not os.path.isfile(os.path.join(adapter_dir, "mtp_head.safetensors")):
+ if os.path.isdir(adapter_dir):
+ return
+ try:
+ from huggingface_hub import hf_hub_download
+ path = hf_hub_download(
+ adapter_dir,
+ "mtp_head.safetensors",
+ token = token,
+ revision = revision,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ )
+ except Exception:
+ return # no head uploaded beside this adapter
+ adapter_dir = os.path.dirname(path)
+ try:
+ from .speculative import attach_saved_mtp_head
+ attach_saved_mtp_head(model, adapter_dir)
+ print(f"Unsloth: Re-attached the trained MTP head saved in '{adapter_dir}'.")
+ except Exception as error:
+ print(
+ f"Unsloth: Could not re-attach the MTP head in '{adapter_dir}' ({error}); merges will keep the stock head."
+ )
+
+
def _diffusion_peft_kwargs(
model,
args,
@@ -1042,6 +1080,11 @@ class FastLanguageModel(FastLlamaModel):
return FastDiffusionModel.for_training(model, use_gradient_checkpointing)
return FastLlamaModel.for_training(model, use_gradient_checkpointing)
+ @staticmethod
+ def get_mtp_head(model, *args, **kwargs):
+ from .speculative import get_mtp_head
+ return get_mtp_head(model, *args, **kwargs)
+
@staticmethod
@tokenizers_v1_on_return
@_offline_aware_load
@@ -1856,6 +1899,14 @@ class FastLanguageModel(FastLlamaModel):
)
if fast_pissa:
record_fast_pissa(model)
+ _reattach_saved_mtp_head(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = local_files_only,
+ )
model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)
try:
from .vision import _lift_endpoint_hooks_onto_adapters
@@ -1910,6 +1961,11 @@ except:
class FastModel(FastBaseModel):
+ @staticmethod
+ def get_mtp_head(model, *args, **kwargs):
+ from .speculative import get_mtp_head
+ return get_mtp_head(model, *args, **kwargs)
+
@staticmethod
def _prepare_for_qat(model, qat_scheme):
model = _prepare_model_for_qat(model, qat_scheme)
@@ -3182,6 +3238,14 @@ class FastModel(FastBaseModel):
)
if fast_pissa:
record_fast_pissa(model)
+ _reattach_saved_mtp_head(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = local_files_only,
+ )
try:
from .vision import _lift_endpoint_hooks_onto_adapters
_lift_endpoint_hooks_onto_adapters(model)
diff --git a/unsloth/models/speculative.py b/unsloth/models/speculative.py
new file mode 100644
index 00000000000..8224741695f
--- /dev/null
+++ b/unsloth/models/speculative.py
@@ -0,0 +1,255 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Fine-tune a model's native MTP head so its speculative drafts match the fine-tuned model.
+
+transformers drops `mtp.*` on load, and the merged export copies the stock head through
+unchanged, so after fine-tuning the head drafts for the old model. `get_mtp_head` loads the
+head from the checkpoint and attaches it; `UnslothMTPTrainer` trains it against the frozen
+model; every merged / GGUF save then writes the trained head instead of the stock one.
+"""
+
+# The trainer names resolve lazily through __getattr__ (they need a recent unsloth_zoo).
+__all__ = ["get_mtp_head", "attach_saved_mtp_head"]
+
+import functools
+import inspect
+import os
+import time
+
+
+def _zoo_speculative():
+ try:
+ import unsloth_zoo.speculative as speculative
+ from unsloth_zoo.speculative import mtp # noqa: F401
+ except ImportError as error:
+ raise ImportError(
+ "Unsloth: MTP head training needs a newer unsloth_zoo. "
+ "Update via `pip install --upgrade --no-cache-dir unsloth_zoo`."
+ ) from error
+ return speculative
+
+
+def _checkpoint_source(model):
+ """The repo id / folder the weights came from, looked up through PEFT wrappers."""
+ holders, seen, stack = [], set(), [model]
+ while stack:
+ holder = stack.pop()
+ if holder is None or id(holder) in seen:
+ continue
+ seen.add(id(holder))
+ holders.append(holder)
+ for attribute in ("base_model", "model"):
+ child = holder.__dict__.get("_modules", {}).get(attribute) or holder.__dict__.get(
+ attribute
+ )
+ if child is not None:
+ stack.append(child)
+ # ModelScope loads keep the repo id in the config and the downloaded weights on the inner
+ # model, under any PEFT wrapper: the snapshot wins over every holder's config.
+ for holder in holders:
+ snapshot = holder.__dict__.get("_unsloth_modelscope_snapshot")
+ if isinstance(snapshot, str) and os.path.isdir(snapshot):
+ return snapshot
+ for holder in holders:
+ name = getattr(getattr(holder, "config", None), "_name_or_path", None)
+ if name:
+ return name
+ return None
+
+
+def _loaded_revision(model, source, revision):
+ """The commit the weights were loaded from, so a Hub lookup reads that head, not the default branch's."""
+ if revision is not None or not isinstance(source, str) or os.path.isdir(source):
+ return revision
+ return getattr(getattr(model, "config", None), "_commit_hash", None)
+
+
+def get_mtp_head(
+ model,
+ source = None,
+ revision = None,
+ random_init = False,
+ dtype = None,
+):
+ """Load the model's native MTP head, attach it to `model`, and return it.
+
+ source: repo id or folder holding the original `mtp.*` tensors. Defaults to the
+ checkpoint the model was loaded from; pass the original repo when that checkpoint
+ was re-saved without its head.
+ random_init: start from random weights instead (for checkpoints without a head).
+ dtype: head parameter dtype, float32 by default (the head is small; the target stays as loaded).
+ Without `source` / `random_init`, a head already attached (e.g. re-attached from beside a
+ reloaded adapter) is returned as is.
+ """
+ import torch
+
+ if source is None and not random_init:
+ from ..save import _attached_mtp_head
+ attached = _attached_mtp_head(model)
+ if attached is not None:
+ return attached if dtype is None else attached.to(dtype = dtype)
+ speculative = _zoo_speculative()
+ from unsloth_zoo.speculative.mtp import build_mtp_head, load_mtp_state_dict
+
+ parts = speculative.resolve_text_parts(model)
+ if source is None:
+ source = _checkpoint_source(model)
+ revision = _loaded_revision(model, source, revision)
+ family = speculative.mtp_family_name(parts["config"])
+ if family is None:
+ raise NotImplementedError(
+ f"Unsloth: `{getattr(parts['config'], 'model_type', None)}` has no native MTP head Unsloth can train."
+ )
+ if random_init:
+ state_dict, info = (
+ None,
+ {"prefix": "mtp." if family == "qwen3_5" else "", "num_layers": 1, "source": source},
+ )
+ elif isinstance(source, str) and os.path.isfile(os.path.join(source, "mtp_head.safetensors")):
+ # A head trained earlier and saved beside a LoRA adapter.
+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head
+ state_dict, info = load_saved_mtp_head(source)
+ else:
+ if source is None:
+ raise ValueError(
+ "Unsloth: pass `source=` (the repo id or folder that holds the `mtp.*` tensors)."
+ )
+ state_dict, info = load_mtp_state_dict(source, revision = revision, config = parts["config"])
+ # `files` are local cache paths; never write them into a folder that may be pushed.
+ info = {k: v for k, v in info.items() if k != "files"}
+ info.update(source = source, revision = revision)
+ device = parts["embed_tokens"].weight.device
+ extra = {}
+ if family != "qwen3_5":
+ # Head-serialized embedding / lm_head copies equal to the target's are not held twice.
+ extra = dict(
+ target_embed = parts["embed_tokens"].weight,
+ target_lm_head = parts["lm_head"].weight,
+ ckpt_prefix = info.get("ckpt_prefix"),
+ )
+ head = build_mtp_head(
+ parts["config"],
+ state_dict = state_dict,
+ dtype = dtype or torch.float32,
+ device = device,
+ **extra,
+ )
+ status = getattr(getattr(head, "family", None), "llama_cpp", "drafts")
+ if status != "drafts":
+ print(
+ f"Unsloth: llama.cpp support for `{family}` MTP drafting is `{status}`; the trained head "
+ "is still written into merged / GGUF exports."
+ )
+ head._unsloth_mtp_info = info
+ # Not a submodule: keeps the head out of model.state_dict(), PEFT and the target's optimizer.
+ object.__setattr__(model, "_unsloth_mtp_head", head)
+ _wrap_save_pretrained(model)
+ return head
+
+
+def _takes_push_to_hub(fn):
+ # Unsloth's loaders wrap save_pretrained as (self, *args, **kwargs); read the wrapped one.
+ if getattr(fn, "__name__", "") == "unsloth_model_save_pretrained":
+ fn = getattr(getattr(fn, "__self__", None), "original_model_save_pretrained", fn)
+ try:
+ return "push_to_hub" in inspect.signature(fn).parameters
+ except (TypeError, ValueError):
+ return False
+
+
+def _wrap_save_pretrained(model):
+ """Plain `save_pretrained` keeps the trained head too: beside a LoRA adapter, else in the checkpoint."""
+ original = model.save_pretrained
+ if getattr(original, "_unsloth_saves_mtp_head", False):
+ return
+
+ @functools.wraps(original)
+ def save_pretrained(save_directory, *args, **kwargs):
+ from ..save import (
+ _INTERNAL_MTP_SAVE,
+ _refuse_mtp_variant,
+ _save_mtp_head_beside_adapter,
+ _stitch_mtp_head,
+ )
+
+ internal = _INTERNAL_MTP_SAVE.get()
+ if (
+ not internal
+ and kwargs.get("safe_serialization", True) is False
+ and not hasattr(model, "peft_config")
+ ):
+ # The head is stitched into safetensors checkpoints only; refuse before writing.
+ raise ValueError(
+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."
+ )
+ # PEFT's save_pretrained has no push_to_hub (its push_to_hub saves into a folder, then uploads it).
+ push = bool(kwargs.get("push_to_hub")) and _takes_push_to_hub(original)
+ if not internal:
+ _refuse_mtp_variant(model, kwargs.get("variant"))
+ if push and kwargs.get("create_pr"):
+ # transformers opens the PR and keeps its ref; a second upload would open another.
+ raise ValueError(
+ "Unsloth: `create_pr = True` cannot carry the trained MTP head. Save locally, then "
+ "upload the folder with `HfApi().upload_folder(..., create_pr = True)`."
+ )
+ started = time.time()
+ result = original(save_directory, *args, **kwargs)
+ if internal or not os.path.isdir(save_directory):
+ return result
+ adapter_config = os.path.join(save_directory, "adapter_config.json")
+ # Written by this save, not left over from an adapter saved here before.
+ if os.path.isfile(adapter_config) and os.path.getmtime(adapter_config) >= started - 1:
+ files = (
+ ["mtp_head.safetensors"]
+ if _save_mtp_head_beside_adapter(model, save_directory)
+ else []
+ )
+ else:
+ report = _stitch_mtp_head(model, save_directory)
+ files = list(report["files_touched"]) if report else []
+ if push and files:
+ # transformers uploaded the folder before the head existed; send the head files after it.
+ from huggingface_hub import HfApi
+ HfApi(token = kwargs.get("token")).upload_folder(
+ repo_id = kwargs.get("repo_id")
+ or str(save_directory).rstrip(os.sep).split(os.sep)[-1],
+ folder_path = save_directory,
+ allow_patterns = files,
+ commit_message = "Add the trained MTP head",
+ )
+ return result
+
+ save_pretrained._unsloth_saves_mtp_head = True
+ object.__setattr__(model, "save_pretrained", save_pretrained)
+
+
+def attach_saved_mtp_head(model, directory):
+ """Re-attach a head saved beside an adapter folder; None when the folder has none."""
+ if not isinstance(directory, str) or not os.path.isfile(
+ os.path.join(directory, "mtp_head.safetensors")
+ ):
+ return None
+ return get_mtp_head(model, source = directory)
+
+
+def __getattr__(name):
+ if name in ("UnslothMTPTrainer", "MTPTrainingArguments", "MTPDataCollator"):
+ _zoo_speculative()
+ from unsloth_zoo.speculative import trainer
+ return getattr(trainer, name)
+ if name in ("UnslothDSparkTrainer", "DSparkTrainingArguments", "Qwen3DSparkDrafter"):
+ _zoo_speculative()
+ from unsloth_zoo.speculative import dspark
+ return getattr(dspark, name)
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/unsloth/save.py b/unsloth/save.py
index c282bf21509..f54f25eb52c 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -53,6 +53,7 @@ import json
import shutil
import pickle
import gc
+import contextvars
import functools
from transformers.models.llama.modeling_llama import logger
from .kernels import fast_dequantize, QUANT_STATE, get_lora_parameters_bias
@@ -819,6 +820,202 @@ def _strip_absent_mtp_declaration(config_dict, tensor_names):
return False
+def _attached_mtp_head(model):
+ """The trained MTP head attached as `_unsloth_mtp_head` anywhere under PEFT wrappers, else None.
+
+ Walks `__dict__` / `_modules` directly: `getattr` on a PeftModel forwards misses down the
+ wrapper chain, so a plain lookup would answer for the wrong holder or recurse.
+ """
+ seen = set()
+ stack = [model]
+ while stack:
+ holder = stack.pop()
+ if holder is None or id(holder) in seen:
+ continue
+ seen.add(id(holder))
+ attributes = getattr(holder, "__dict__", {})
+ modules = attributes.get("_modules") or {}
+ head = attributes.get("_unsloth_mtp_head", modules.get("_unsloth_mtp_head"))
+ if head is not None:
+ return head
+ for name in ("base_model", "model", "language_model"):
+ child = modules.get(name, attributes.get(name))
+ if child is not None:
+ stack.append(child)
+ return None
+
+
+def _mtp_head_state_dict(head):
+ """Head-relative checkpoint names (`fc.weight`); `mtp_state_dict` excludes shared target tensors."""
+ getter = getattr(head, "mtp_state_dict", None)
+ return dict(getter() if callable(getter) else head.state_dict())
+
+
+def _mtp_head_info(head):
+ info = getattr(head, "_unsloth_mtp_info", None)
+ return dict(info) if isinstance(info, dict) else {}
+
+
+def _mtp_head_tensor_names(model):
+ """Checkpoint names the attached head will occupy, or [] when none is attached."""
+ head = _attached_mtp_head(model)
+ if head is None:
+ return []
+ # Qwen3.5 heads use relative names under `mtp.`; NextN heads (prefix "") already use full names.
+ prefix = _mtp_head_info(head).get("prefix")
+ prefix = "mtp." if prefix is None else prefix
+ return [prefix + name for name in _mtp_head_state_dict(head)]
+
+
+def _is_main_rank():
+ """Only one process writes the head when several save the same folder."""
+ try:
+ import torch.distributed as dist
+ return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0
+ except Exception:
+ return True
+
+
+def _refuse_mtp_variant(model, variant):
+ """Variant filenames (`model.<variant>.safetensors`) are not stitched; refuse before writing anything."""
+ if variant is not None and _attached_mtp_head(model) is not None:
+ raise NotImplementedError(
+ "Unsloth: saving a trained MTP head with `variant=` is not supported; save without `variant`."
+ )
+
+
+def _stitch_mtp_head(model, save_directory):
+ """Write the attached trained MTP head into a saved checkpoint. No-op when none is attached."""
+ head = _attached_mtp_head(model)
+ if head is None or not _is_main_rank():
+ return None
+ from unsloth_zoo.speculative.stitch import write_mtp_tensors
+
+ info = _mtp_head_info(head)
+ # NextN families (GLM, DeepSeek, Qwen3-Next, ...) say where their tensors live themselves.
+ stitch_kwargs = getattr(head, "stitch_kwargs", None)
+ report = write_mtp_tensors(
+ save_directory,
+ _mtp_head_state_dict(head),
+ append_prefix = info.get("prefix") or "mtp.",
+ **(stitch_kwargs() if callable(stitch_kwargs) else {}),
+ )
+ print(
+ f"Unsloth: Wrote the trained MTP head ({len(report['tensors_written'])} tensors, "
+ f"{report['mode']}) into '{save_directory}'."
+ )
+ return report
+
+
+# Set while an Unsloth saver writes a checkpoint it stitches the head into itself, so the plain
+# `save_pretrained` wrapper (unsloth/models/speculative.py) does not stitch it a second time,
+# however the wrappers are stacked (an export reload wraps it again from outside).
+_INTERNAL_MTP_SAVE = contextvars.ContextVar("unsloth_internal_mtp_save", default = False)
+
+
+@contextmanager
+def _internal_mtp_save():
+ token = _INTERNAL_MTP_SAVE.set(True)
+ try:
+ yield
+ finally:
+ _INTERNAL_MTP_SAVE.reset(token)
+
+
+def _save_mtp_head_beside_adapter(model, save_directory):
+ """LoRA saves keep the trained head as `mtp_head.safetensors` so a later merge can stitch it."""
+ head = _attached_mtp_head(model)
+ if not _is_main_rank():
+ return None
+ if head is None:
+ # A reused folder's earlier head would be re-attached to this unrelated adapter on load.
+ stale = os.path.join(save_directory, "mtp_head.safetensors")
+ if os.path.isfile(stale):
+ os.remove(stale)
+ return None
+ from unsloth_zoo.speculative.stitch import save_mtp_head
+
+ path = save_mtp_head(save_directory, _mtp_head_state_dict(head), _mtp_head_info(head))
+ print(f"Unsloth: Saved the trained MTP head to '{path}'.")
+ return path
+
+
+def _push_with_mtp_head(save_kwargs):
+ """Push for `unsloth_save_model` with a trained MTP head: save locally, stitch, upload the folder.
+
+ Every direct-to-hub writer streams the stock head from the original shards, so the head can
+ only be written into a local copy.
+ """
+ import tempfile
+ from huggingface_hub.errors import HfHubHTTPError
+
+ if not save_kwargs["is_main_process"]:
+ return save_kwargs["save_directory"], None
+ token = save_kwargs["token"] if save_kwargs["token"] is not None else get_token()
+ repo_id, username = _determine_username(save_kwargs["save_directory"], None, token)
+ api = HfApi(token = token)
+ api.create_repo(
+ repo_id = repo_id,
+ repo_type = "model",
+ private = save_kwargs["private"],
+ exist_ok = True,
+ )
+ revision = save_kwargs["revision"]
+ if revision is not None and not revision.startswith("refs/pr/"):
+ try:
+ api.create_branch(repo_id = repo_id, repo_type = "model", branch = revision, exist_ok = True)
+ except HfHubHTTPError as error:
+ if not save_kwargs["create_pr"] or error.response.status_code != 403:
+ raise
+ with tempfile.TemporaryDirectory(prefix = "unsloth-mtp-") as directory:
+ unsloth_save_model(
+ **{**save_kwargs, "save_directory": directory, "push_to_hub": False, "token": token}
+ )
+ # The card the hub path writes, staged here so it lands in this commit on this revision.
+ from huggingface_hub import ModelCard, hf_hub_download, metadata_save
+ from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
+
+ readme = os.path.join(directory, "README.md")
+ if not os.path.isfile(readme):
+ try:
+ shutil.copyfile(
+ hf_hub_download(repo_id, "README.md", revision = revision, token = token),
+ readme,
+ )
+ except LocalEntryNotFoundError:
+ raise # offline or unreachable: never replace a card that may exist
+ except EntryNotFoundError: # no card yet; any other failure must not replace one
+ config = save_kwargs["model"].config
+ ModelCard(
+ MODEL_CARD.format(
+ username = username,
+ base_model = config._name_or_path,
+ model_type = config.model_type,
+ method = "finetuned",
+ extra = "unsloth",
+ )
+ ).save(readme)
+ tags = ModelCard.load(readme).data.tags or []
+ metadata = {
+ "tags": list(dict.fromkeys([*tags, *(save_kwargs.get("tags") or []), "unsloth"]))
+ }
+ if save_kwargs.get("datasets"):
+ metadata["datasets"] = save_kwargs["datasets"]
+ metadata_save(readme, metadata)
+ api.upload_folder(
+ folder_path = directory,
+ path_in_repo = ".",
+ repo_id = repo_id,
+ repo_type = "model",
+ commit_message = save_kwargs["commit_message"] or "Trained with Unsloth",
+ commit_description = save_kwargs.get("commit_description"),
+ revision = save_kwargs["revision"],
+ create_pr = save_kwargs["create_pr"],
+ )
+ print(f"Unsloth: Saved model with its trained MTP head to https://huggingface.co/{repo_id}")
+ return repo_id, username
+
+
def _is_qwen3_5_vlm(model):
config = getattr(model, "config", None)
if config is None or not hasattr(config, "vision_config"):
@@ -943,6 +1140,10 @@ def unsloth_save_model(
maximum_memory_usage: float = 0.9,
datasets: Optional[List[str]] = None,
):
+ _refuse_mtp_variant(model, variant)
+ if push_to_hub and _attached_mtp_head(model) is not None:
+ return _push_with_mtp_head(dict(locals()))
+
if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):
tokenizer = patch_saving_functions(tokenizer)
@@ -1019,7 +1220,11 @@ def unsloth_save_model(
print("This might take 5 minutes...")
if hasattr(model, "merge_and_unload"):
+ mtp_head = _attached_mtp_head(model)
model = model.merge_and_unload()
+ # merge_and_unload returns the inner model; keep the trained head with it.
+ if mtp_head is not None:
+ object.__setattr__(model, "_unsloth_mtp_head", mtp_head)
print("Done.")
if tags is not None:
@@ -1170,7 +1375,12 @@ def unsloth_save_model(
if save_method == "lora":
save_pretrained_settings["selected_adapters"] = None
- model.save_pretrained(**save_pretrained_settings)
+ with _internal_mtp_save():
+ model.save_pretrained(**save_pretrained_settings)
+ if save_method == "lora":
+ _save_mtp_head_beside_adapter(model, save_pretrained_settings["save_directory"])
+ else:
+ _stitch_mtp_head(model, save_pretrained_settings["save_directory"])
if push_to_hub and hasattr(model, "config"):
print("Saved to https://huggingface.co/" + save_pretrained_settings["save_directory"])
@@ -1222,12 +1432,18 @@ def unsloth_save_model(
if n_cpus is None:
n_cpus = 1
+ # The trained MTP head is stitched into safetensors checkpoints only.
+ _keeps_mtp_head = _attached_mtp_head(model) is not None
+ if _keeps_mtp_head and not safe_serialization:
+ raise ValueError(
+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."
+ )
# Already `True` from the normalisation at the top of this function; the caller having
# passed `None` is what keeps the downgrade below from undoing it.
if _force_safe_serialization:
save_pretrained_settings["safe_serialization"] = safe_serialization
- elif safe_serialization and (n_cpus <= 2):
+ elif safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head:
logger.warning_once(
f"Unsloth: You have {n_cpus} CPUs. Using `safe_serialization` is 10x slower.\n"
f"We shall switch to Pytorch saving, which might take 3 minutes and not 30 minutes.\n"
@@ -1435,8 +1651,11 @@ def unsloth_save_model(
new_config = model.config.to_dict()
if "quantization_config" in new_config:
del new_config["quantization_config"]
- # The merged state dict is what gets written, so its keys decide MTP presence.
- _strip_absent_mtp_declaration(new_config, state_dict.keys())
+ # The merged state dict is what gets written, so its keys decide MTP presence; a trained
+ # head attached to the model is stitched in right after the write.
+ _strip_absent_mtp_declaration(
+ new_config, list(state_dict.keys()) + _mtp_head_tensor_names(model)
+ )
original_model = model
new_config = type(model.config).from_dict(new_config)
while hasattr(original_model, "model"):
@@ -1467,13 +1686,17 @@ def unsloth_save_model(
ignore_patterns = "*.md",
)
else:
- internal_model.save_pretrained(**save_pretrained_settings)
+ with _internal_mtp_save():
+ internal_model.save_pretrained(**save_pretrained_settings)
finally:
original_model = model
while hasattr(original_model, "model"):
original_model = original_model.model
original_model.config = old_config
model.config = old_config
+ # Pushes with an attached head were rerouted to a local save at the top.
+ if not save_pretrained_settings["push_to_hub"]:
+ _stitch_mtp_head(model, save_pretrained_settings["save_directory"])
print("Done.")
if push_to_hub and hasattr(model, "config"):
@@ -3390,6 +3613,9 @@ def _gguf_reuses_loaded_checkpoint(model, state_dict = None):
return False
if state_dict is not None or getattr(model, "_unsloth_full_finetuning", False):
return False
+ # The loaded folder holds the stock MTP head; a trained one needs a fresh checkpoint.
+ if _attached_mtp_head(model) is not None:
+ return False
# A ModelScope load keeps its snapshot here, since `_name_or_path` holds the repo id (#3726).
name_or_path = getattr(model, "_unsloth_modelscope_snapshot", None) or getattr(
getattr(model, "config", None), "_name_or_path", None
@@ -4161,7 +4387,9 @@ def unsloth_save_pretrained_gguf(
os.makedirs(save_directory, exist_ok = True)
# `gguf_directory` can point anywhere, and freeing bytes on one filesystem does nothing for a quantize pass writing to another: without this the merge could be deleted for a destination it cannot help, data gone and the export still out of space.
try:
- self.save_pretrained(save_directory, state_dict = state_dict)
+ with _internal_mtp_save():
+ self.save_pretrained(save_directory, state_dict = state_dict)
+ _stitch_mtp_head(self, save_directory)
if tokenizer is not None:
tokenizer.save_pretrained(save_directory)
except Exception as e:
@@ -5666,6 +5894,7 @@ def unsloth_generic_save(
datasets: Optional[List[str]] = None,
dtype = None,
):
+ _refuse_mtp_variant(model, variant)
if save_method == "merged_4bit":
raise RuntimeError(
"Unsloth: Merging into 4bit will cause your model to lose accuracy if you plan\n"
@@ -5679,7 +5908,13 @@ def unsloth_generic_save(
# this function's own keywords.
safe_serialization = _normalize_safe_serialization(safe_serialization)
- if push_to_hub and (create_pr or revision is not None or not isinstance(model, PeftModel)):
+ # A trained MTP head can only be stitched into a local copy, which this route makes first.
+ if push_to_hub and (
+ create_pr
+ or revision is not None
+ or not isinstance(model, PeftModel)
+ or _attached_mtp_head(model) is not None
+ ):
return _push_merged_to_hub_revision(dict(locals()))
if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):
@@ -5708,6 +5943,11 @@ def unsloth_generic_save(
# method, since `patch_saving_functions` wraps it in a `(*args, **kwargs)` passthrough
# that would hide a transformers which does honour the request.
_real_save = getattr(model, "original_model_save_pretrained", model.save_pretrained)
+ if safe_serialization is False and _attached_mtp_head(model) is not None:
+ # The trained head is stitched into safetensors checkpoints only; refuse before writing.
+ raise ValueError(
+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."
+ )
if safe_serialization is False and not _honours_safe_serialization(_real_save):
logger.warning_once(
"Unsloth: this transformers always writes safetensors, so "
@@ -5728,6 +5968,12 @@ def unsloth_generic_save(
state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)
if is_qwen3_5_vlm:
state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)
+ if _attached_mtp_head(model) is not None:
+ # A head registered as a submodule would otherwise be written under its attribute
+ # name; it is stitched in under its checkpoint names after the save instead.
+ if state_dict is None:
+ state_dict = model.state_dict()
+ state_dict = {k: v for k, v in state_dict.items() if "_unsloth_mtp_head." not in k}
if state_dict is not None:
_save_kwargs["state_dict"] = state_dict
@@ -5738,9 +5984,15 @@ def unsloth_generic_save(
_written_tensor_names = list(state_dict.keys()) if state_dict is not None else None
print(f"Unsloth: Saving full fine-tuned model to '{save_directory}' ...")
- model.save_pretrained(save_directory, **_save_kwargs)
+ with _internal_mtp_save():
+ model.save_pretrained(save_directory, **_save_kwargs)
if "16bit" in save_method and _requests_float32(dtype):
_stamp_saved_config_float32(save_directory)
+ if (
+ _stitch_mtp_head(model, save_directory) is not None
+ and _written_tensor_names is not None
+ ):
+ _written_tensor_names = _written_tensor_names + _mtp_head_tensor_names(model)
# Guarded: an older zoo must not raise once the weights are already on disk.
try:
from unsloth_zoo.saving_utils import reconcile_mtp_config
@@ -5828,6 +6080,9 @@ def unsloth_generic_save(
low_disk_space_usage = True,
use_temp_file = False,
)
+ # Pushes with an attached head were rerouted to a local save above.
+ if not push_to_hub:
+ _stitch_mtp_head(model, save_directory)
if push_to_hub and datasets:
try:
@@ -6656,6 +6911,12 @@ def _unsloth_save_compressed_tensors(
# Only the main process installs deps, merges, quantizes and uploads, mirroring the non-PEFT save path; other ranks return at once rather than race on dirs or run pip installs.
if not is_main_process:
return None
+ if _attached_mtp_head(model) is not None:
+ # The quantizer reloads through transformers, which drops native MTP tensors.
+ print(
+ "Unsloth: compressed-tensors exports do not carry the trained MTP head; use merged_16bit "
+ "or GGUF to keep it."
+ )
# Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of writing a full 16bit checkpoint first. Under the llm-compressor-main shadow the subprocess validates itself, so the workspace install / ceiling / scheme checks are skipped.
_shadow_pythonpath = _compressed_quantize_pythonpath()
@@ -6911,6 +7172,12 @@ def _unsloth_save_torchao(
if not is_main_process:
return None
+ if _attached_mtp_head(model) is not None:
+ # The reload below drops native MTP tensors (the reason the head is stitched at all).
+ print(
+ f"Unsloth: torchao {kind} exports do not carry the trained MTP head; use merged_16bit or "
+ "GGUF to keep it."
+ )
from transformers import (
AutoModelForCausalLM,