Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fine-tune and export native MTP heads and DSpark drafters (#13719)
Daniel Han65d3ca519590
diff --git a/tests/saving/test_modelscope_repo_id.py b/tests/saving/test_modelscope_repo_id.pyindex b7155ff9e42..c0432f478c2 100644--- a/tests/saving/test_modelscope_repo_id.py+++ b/tests/saving/test_modelscope_repo_id.py@@ -134,7 +134,12 @@ def test_gguf_still_converts_the_modelscope_snapshot(tmp_path):reuses = _load("unsloth/save.py","_gguf_reuses_loaded_checkpoint",- {"os": os, "PeftModel": _Peft, "PeftModelForCausalLM": _Peft},+ {+ "os": os,+ "PeftModel": _Peft,+ "PeftModelForCausalLM": _Peft,+ "_attached_mtp_head": lambda m: None,+ },)model = types.SimpleNamespace(config = types.SimpleNamespace(_name_or_path = "unsloth/x"))assert not reuses(model)diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex f401126abce..d0482d3d102 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -206,6 +206,12 @@ _GENERIC_SAVE_STUBBED_HELPERS = frozenset("_qwen3_5_vlm_state_dict_for_save","_determine_username","unsloth_save_model",+ # Trained MTP head hooks: no head is attached in these fixtures.+ "_refuse_mtp_variant",+ "_attached_mtp_head",+ "_internal_mtp_save",+ "_stitch_mtp_head",+ "_mtp_head_tensor_names",})@@ -238,6 +244,11 @@ def _routing_environment(monkeypatch, model):_qwen3_5_vlm_state_dict_for_save = _not_reached,_determine_username = lambda repo, old, token: (repo, "owner"),unsloth_save_model = lambda *args, **kwargs: calls["adapter"].append(kwargs),+ _refuse_mtp_variant = lambda model, variant: None,+ _attached_mtp_head = lambda model: None,+ _internal_mtp_save = contextlib.nullcontext,+ _stitch_mtp_head = lambda model, directory: None,+ _mtp_head_tensor_names = lambda model: [],lora_relative_to_original_base = lambda model: (calls["convert"].append(model) or contextlib.nullcontext()),@@ -406,6 +417,13 @@ def _adapter_save_environment(monkeypatch):),fast_save_pickle = lambda *args, **kwargs: None,clean_gpu_cache = lambda: None,+ # Trained MTP head hooks: no head is attached in these fixtures.+ _refuse_mtp_variant = lambda model, variant: None,+ _attached_mtp_head = lambda model: None,+ _internal_mtp_save = contextlib.nullcontext,+ _save_mtp_head_beside_adapter = lambda model, directory: None,+ _stitch_mtp_head = lambda model, directory: None,+ _mtp_head_tensor_names = lambda model: [],), uploads@@ -711,7 +729,7 @@ def test_the_docstrings_describe_none_as_the_stronger_safetensors_request():assert (source.count("`None` is stronger than the default") == 4), "all four save_method docstrings have to describe None the same way"- assert "elif safe_serialization and (n_cpus <= 2):" in source+ assert "elif safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head:" in sourceassert "if _force_safe_serialization:" in sourcediff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex d3381bd32e9..c61157b567f 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -167,6 +167,12 @@ def saving(monkeypatch, tmp_path):_normalize_compressed_method = lambda method: None,_normalize_torchao_method = lambda method: None,_is_qwen3_5_vlm = lambda model: False,+ # Trained MTP head hooks: no head is attached in these fixtures.+ _refuse_mtp_variant = lambda model, variant: None,+ _attached_mtp_head = lambda model: None,+ _internal_mtp_save = contextlib.nullcontext,+ _stitch_mtp_head = lambda model, directory: None,+ _mtp_head_tensor_names = lambda model: [],raise_if_merging_mistral_format_view = lambda model, save_method: None,lora_relative_to_original_base = lambda model: contextlib.nullcontext(),nullcontext = contextlib.nullcontext,diff --git a/tests/test_save_mtp_head_stitch.py b/tests/test_save_mtp_head_stitch.pynew file mode 100644index 00000000000..0d54b074179--- /dev/null+++ b/tests/test_save_mtp_head_stitch.py@@ -0,0 +1,634 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A trained MTP head attached as `_unsloth_mtp_head` must reach every export: the merge+copies the STOCK head through from the original shards, so without the stitch a fine-tuned+model ships a head that was never aligned to it."""++import ast+from types import SimpleNamespace+import json+from pathlib import Path++import pytest++SAVE_PY = Path(__file__).resolve().parents[1] / "unsloth" / "save.py"++[email protected](scope = "module")+def tree():+ return ast.parse(SAVE_PY.read_text(encoding = "utf-8"))+++def _func(tree, name):+ for node in ast.walk(tree):+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name:+ return node+ raise AssertionError(f"{name} is not defined in save.py")+++def _calls(node):+ out = set()+ for sub in ast.walk(node):+ if isinstance(sub, ast.Call):+ f = sub.func+ out.add(f.id if isinstance(f, ast.Name) else getattr(f, "attr", None))+ return out+++ "function, expected",+ [+ (+ "unsloth_save_model",+ {+ "_stitch_mtp_head",+ "_save_mtp_head_beside_adapter",+ "_push_with_mtp_head",+ "_mtp_head_tensor_names",+ },+ ),+ (+ "unsloth_generic_save",+ {"_stitch_mtp_head", "_attached_mtp_head", "_push_merged_to_hub_revision"},+ ),+ ("unsloth_save_pretrained_gguf", {"_stitch_mtp_head"}),+ ("_gguf_reuses_loaded_checkpoint", {"_attached_mtp_head"}),+ # torchao's reload drops native MTP tensors: say so instead of exporting it silently.+ ("_unsloth_save_torchao", {"_attached_mtp_head"}),+ ],+)+def test_save_paths_are_wired(tree, function, expected):+ assert expected <= _calls(_func(tree, function))++[email protected](scope = "module")+def save_module():+ pytest.importorskip(+ "unsloth_zoo.speculative.stitch", reason = "unsloth_zoo has no speculative.stitch"+ )+ pytest.importorskip("unsloth", reason = "unsloth is not importable on this runner")+ try:+ import unsloth.save as module+ except ImportError as error:+ pytest.skip(f"unsloth.save is not importable on this runner: {error}")+ return module+++def _head(torch):+ head = torch.nn.Module()+ head.fc = torch.nn.Linear(16, 8, bias = False)+ head.norm = torch.nn.Module()+ head.norm.weight = torch.nn.Parameter(torch.randn(8))+ object.__setattr__(head, "_unsloth_mtp_info", {"prefix": "mtp.", "source": "tiny"})+ return head+++def _wrapped(torch, head, registered):+ """PeftModel -> LoraModel -> base model, with the head on the innermost model."""+ base = torch.nn.Module()+ base.config = None+ if registered:+ base._unsloth_mtp_head = head+ else:+ object.__setattr__(base, "_unsloth_mtp_head", head)+ lora = torch.nn.Module()+ lora.model = base+ peft = torch.nn.Module()+ peft.base_model = lora+ return peft++[email protected]("registered", [True, False])+def test_head_is_found_through_wrappers(save_module, registered):+ import torch++ head = _head(torch)+ assert save_module._attached_mtp_head(_wrapped(torch, head, registered)) is head+ assert save_module._attached_mtp_head(torch.nn.Linear(2, 2)) is None+ assert save_module._mtp_head_tensor_names(torch.nn.Linear(2, 2)) == []+ assert sorted(save_module._mtp_head_tensor_names(_wrapped(torch, head, registered))) == [+ "mtp.fc.weight",+ "mtp.norm.weight",+ ]+++def test_no_head_is_a_no_op(save_module, tmp_path):+ import torch++ model = torch.nn.Linear(2, 2)+ assert save_module._stitch_mtp_head(model, tmp_path) is None+ assert save_module._save_mtp_head_beside_adapter(model, tmp_path) is None+ assert list(tmp_path.iterdir()) == []+++def test_stitch_appends_and_declares(save_module, tmp_path):+ import torch+ from safetensors import safe_open+ from safetensors.torch import save_file++ save_file(+ {"model.norm.weight": torch.ones(8, dtype = torch.bfloat16)}, tmp_path / "model.safetensors"+ )+ (tmp_path / "config.json").write_text(json.dumps({"text_config": {"num_hidden_layers": 2}}))+ head = _head(torch)+ head.layers = torch.nn.ModuleList([torch.nn.Module()])+ head.layers[0].up = torch.nn.Linear(8, 8, bias = False)+ report = save_module._stitch_mtp_head(_wrapped(torch, head, registered = False), tmp_path)+ assert report["mode"] == "appended"+ index = json.loads((tmp_path / "model.safetensors.index.json").read_text())+ assert {"mtp.fc.weight", "mtp.norm.weight", "mtp.layers.0.up.weight"} <= set(+ index["weight_map"]+ )+ assert (+ json.loads((tmp_path / "config.json").read_text())["text_config"]["mtp_num_hidden_layers"]+ == 1+ )+ with safe_open(str(tmp_path / index["weight_map"]["mtp.fc.weight"]), "pt") as f:+ assert torch.equal(+ f.get_tensor("mtp.fc.weight"), head.fc.weight.detach().to(torch.bfloat16)+ )+++def test_adapter_save_writes_head_file(save_module, tmp_path):+ import torch+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head++ head = _head(torch)+ save_module._save_mtp_head_beside_adapter(_wrapped(torch, head, registered = True), tmp_path)+ loaded, info = load_saved_mtp_head(tmp_path)+ assert set(loaded) == {"fc.weight", "norm.weight"} and info["source"] == "tiny"+++def test_gguf_does_not_reuse_checkpoint_with_head(save_module, tmp_path):+ import torch++ model = torch.nn.Module()+ model.config = type("C", (), {"_name_or_path": str(tmp_path)})()+ assert save_module._gguf_reuses_loaded_checkpoint(model) is True+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))+ assert save_module._gguf_reuses_loaded_checkpoint(model) is False+++def test_plain_save_pretrained_keeps_the_head(save_module, tmp_path):+ import torch+ from unsloth.models.speculative import _wrap_save_pretrained+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head++ calls = []++ class Adapter(torch.nn.Module):+ def save_pretrained(self, save_directory, **kwargs):+ calls.append(save_directory)+ (Path(save_directory) / "adapter_config.json").write_text("{}")++ model = Adapter()+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))+ _wrap_save_pretrained(model)+ _wrap_save_pretrained(model)+ model.save_pretrained(str(tmp_path))+ assert calls == [str(tmp_path)]+ loaded, _ = load_saved_mtp_head(tmp_path)+ assert set(loaded) == {"fc.weight", "norm.weight"}+++def test_internal_savers_stitch_once(save_module, tmp_path, monkeypatch):+ """Inside `_internal_mtp_save` the plain-save wrapper does not stitch, even under an outer wrapper+ (an export reload wraps `save_pretrained` again), so one save writes the head once."""+ import torch+ from unsloth.models.speculative import _wrap_save_pretrained++ stitched = []+ monkeypatch.setattr(save_module, "_stitch_mtp_head", lambda m, d: stitched.append(d))++ class Plain(torch.nn.Module):+ def save_pretrained(self, save_directory, **kwargs):+ (Path(save_directory) / "config.json").write_text("{}")++ model = Plain()+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))+ _wrap_save_pretrained(model)+ inner = model.save_pretrained+ object.__setattr__(model, "save_pretrained", lambda *a, **k: inner(*a, **k))+ with save_module._internal_mtp_save():+ model.save_pretrained(str(tmp_path))+ assert stitched == []+ model.save_pretrained(str(tmp_path))+ assert stitched == [str(tmp_path)]+++def test_reattach_is_a_no_op_without_a_saved_head(tmp_path):+ loader_tree = ast.parse((SAVE_PY.parent / "models" / "loader.py").read_text(encoding = "utf-8"))+ calls = [+ _calls(n)+ for n in ast.walk(loader_tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_reattach_saved_mtp_head"+ ]+ assert calls, "loader.py lost _reattach_saved_mtp_head"+ sites = [+ n+ for n in ast.walk(loader_tree)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "_reattach_saved_mtp_head"+ ]+ assert len(sites) == 2+ for site in sites:+ assert {k.arg for k in site.keywords} == {+ "token",+ "revision",+ "cache_dir",+ "local_files_only",+ }+++def _tiny_glm(tmp_path, torch):+ pytest.importorskip("transformers.models.glm4_moe")+ from transformers import Glm4MoeForCausalLM+ from transformers.models.glm4_moe.configuration_glm4_moe import Glm4MoeConfig++ torch.manual_seed(0)+ config = Glm4MoeConfig(+ vocab_size = 97,+ hidden_size = 64,+ intermediate_size = 128,+ moe_intermediate_size = 32,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ num_hidden_layers = 2,+ n_routed_experts = 4,+ num_experts_per_tok = 2,+ n_shared_experts = 1,+ first_k_dense_replace = 1,+ num_nextn_predict_layers = 1,+ partial_rotary_factor = 0.5,+ )+ return Glm4MoeForCausalLM(config).eval()+++def test_nextn_head_round_trips_through_save_pretrained(save_module, tmp_path):+ """A GLM-style head (model.layers.{N}.*) is appended on a plain save and reloads identically."""+ import torch+ from safetensors.torch import load_file++ pytest.importorskip("unsloth_zoo.speculative.nextn")+ from unsloth.models.speculative import get_mtp_head++ model = _tiny_glm(tmp_path, torch)+ head = get_mtp_head(model, random_init = True)+ assert type(head).__name__ == "NextNHead"+ with torch.no_grad():+ for p in head.parameters():+ p.add_(0.01)+ expected = {k: v.detach().clone() for k, v in head.mtp_state_dict().items()}+ assert all(name.startswith("model.layers.2.") for name in expected)+ assert sorted(save_module._mtp_head_tensor_names(model)) == sorted(expected)++ out = tmp_path / "saved"+ model.save_pretrained(str(out))+ config = json.loads((out / "config.json").read_text())+ assert config["num_nextn_predict_layers"] == 1 and "mtp_num_hidden_layers" not in config+ written = {}+ for shard in out.glob("*.safetensors"):+ written.update(load_file(str(shard)))+ for name, tensor in expected.items():+ assert torch.equal(written[name], tensor.to(written[name].dtype)), name++ fresh = _tiny_glm(tmp_path, torch)+ reloaded = get_mtp_head(fresh, source = str(out))+ assert all(torch.equal(reloaded.mtp_state_dict()[k], v) for k, v in expected.items())++ # Saving again over a checkpoint that now carries the head rewrites it in place.+ report = save_module._stitch_mtp_head(fresh, str(out))+ assert report["mode"] == "replaced"+++class _AdapterSaver:+ """A model whose save_pretrained writes an adapter folder, like PEFT's."""++ def __init__(self, torch):+ self.module = torch.nn.Module()+ self.saved = []++ def save_pretrained(self, save_directory, **kwargs):+ self.saved.append(kwargs)+ Path(save_directory).mkdir(parents = True, exist_ok = True)+ (Path(save_directory) / "adapter_config.json").write_text("{}")+++def _wrapped_adapter(torch, pushes = True):+ """`pushes`: transformers' save_pretrained (takes push_to_hub), else PEFT's (does not)."""+ from unsloth.models.speculative import _wrap_save_pretrained++ model = torch.nn.Module()+ calls = []++ def transformers_save(+ save_directory,+ push_to_hub = False,+ **kwargs,+ ):+ calls.append(dict(kwargs, push_to_hub = push_to_hub))+ Path(save_directory).mkdir(parents = True, exist_ok = True)+ (Path(save_directory) / "adapter_config.json").write_text("{}")++ def save_pretrained(save_directory, **kwargs):+ calls.append(kwargs)+ Path(save_directory).mkdir(parents = True, exist_ok = True)+ (Path(save_directory) / "adapter_config.json").write_text("{}")++ object.__setattr__(model, "save_pretrained", transformers_save if pushes else save_pretrained)+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))+ _wrap_save_pretrained(model)+ return model, calls+++def test_plain_save_refuses_a_variant_before_writing(save_module, tmp_path):+ import torch++ model, calls = _wrapped_adapter(torch)+ with pytest.raises(NotImplementedError):+ model.save_pretrained(str(tmp_path / "v"), variant = "foo")+ assert calls == []+++def test_plain_save_push_uploads_the_head(save_module, tmp_path, monkeypatch):+ import huggingface_hub+ import torch++ uploads = []++ class Api:+ def __init__(self, token = None):+ self.token = token++ def upload_folder(self, **kwargs):+ uploads.append(kwargs)++ monkeypatch.setattr(huggingface_hub, "HfApi", Api)+ model, _ = _wrapped_adapter(torch)+ out = tmp_path / "me-adapter"+ model.save_pretrained(str(out), push_to_hub = True, repo_id = "me/adapter", token = "t")+ assert (out / "mtp_head.safetensors").is_file()+ assert (+ uploads+ and uploads[0]["repo_id"] == "me/adapter"+ and uploads[0]["allow_patterns"] == ["mtp_head.safetensors"]+ )+++def test_plain_save_push_skips_peft_and_refuses_create_pr(save_module, tmp_path, monkeypatch):+ import huggingface_hub+ import torch++ uploads = []++ class Api:+ def __init__(self, token = None):+ pass++ def upload_folder(self, **kwargs):+ uploads.append(kwargs)++ monkeypatch.setattr(huggingface_hub, "HfApi", Api)+ peft_model, _ = _wrapped_adapter(torch, pushes = False)+ peft_model.save_pretrained(str(tmp_path / "a"), push_to_hub = True, repo_id = "me/adapter")+ assert (tmp_path / "a" / "mtp_head.safetensors").is_file() and uploads == []+ model, calls = _wrapped_adapter(torch)+ with pytest.raises(ValueError, match = "create_pr"):+ model.save_pretrained(str(tmp_path / "b"), push_to_hub = True, repo_id = "me/m", create_pr = True)+ assert calls == [] and uploads == []+++def test_adapter_save_without_a_head_drops_a_stale_one(save_module, tmp_path):+ import torch++ (tmp_path / "mtp_head.safetensors").write_bytes(b"old")+ save_module._save_mtp_head_beside_adapter(torch.nn.Module(), str(tmp_path))+ assert not (tmp_path / "mtp_head.safetensors").exists()+++def test_modelscope_snapshot_is_the_head_source(tmp_path):+ import torch+ from unsloth.models.speculative import _checkpoint_source++ model = torch.nn.Module()+ model.config = type("C", (), {"_name_or_path": "Org/Model"})()+ assert _checkpoint_source(model) == "Org/Model"+ model._unsloth_modelscope_snapshot = str(tmp_path)+ assert _checkpoint_source(model) == str(tmp_path)+++def test_hub_adapter_reattaches_its_head(tmp_path, monkeypatch):+ import huggingface_hub+ from unsloth.models import loader, speculative++ head_file = tmp_path / "snap" / "mtp_head.safetensors"+ head_file.parent.mkdir()+ head_file.write_bytes(b"h")+ seen = []+ monkeypatch.setattr(+ huggingface_hub, "hf_hub_download", lambda repo, name, **k: seen.append(k) or str(head_file)+ )+ attached = []+ monkeypatch.setattr(speculative, "attach_saved_mtp_head", lambda model, d: attached.append(d))+ loader._reattach_saved_mtp_head(+ object(), "me/adapter", token = "t", cache_dir = "/c", local_files_only = True+ )+ assert attached == [str(head_file.parent)]+ # The adapter's own cache and offline mode, else an offline load reads the default cache.+ assert seen[0]["cache_dir"] == "/c" and seen[0]["local_files_only"] is True++ def missing(*a, **k):+ raise FileNotFoundError("no head")++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", missing)+ loader._reattach_saved_mtp_head(object(), "me/other")+ assert attached == [str(head_file.parent)]+++def test_mtp_push_keeps_the_datasets_metadata(save_module, monkeypatch):+ """The attribution is staged in the README, so it lands in the same commit / revision / PR."""+ import os++ import huggingface_hub++ calls = []++ class Api:+ def __init__(self, token = None):+ pass++ def create_repo(self, **kwargs):+ pass++ def create_branch(self, **kwargs):+ pass++ def upload_folder(self, **kwargs):+ readme = os.path.join(kwargs["folder_path"], "README.md")+ calls.append((kwargs["revision"], kwargs["create_pr"], open(readme).read()))++ def no_card(*args, **kwargs):+ raise huggingface_hub.errors.EntryNotFoundError("no README")++ monkeypatch.setattr(save_module, "HfApi", Api)+ monkeypatch.setattr(save_module, "_determine_username", lambda d, u, t: ("me/m", "me"))+ monkeypatch.setattr(save_module, "unsloth_save_model", lambda **kwargs: None)+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", no_card)+ monkeypatch.setattr(+ huggingface_hub, "metadata_update", lambda *a, **k: pytest.fail("edits the default branch")+ )+ save_module._push_with_mtp_head(+ dict(+ is_main_process = True,+ save_directory = "m",+ token = "t",+ private = False,+ revision = "dev",+ create_pr = True,+ commit_message = None,+ datasets = ["org/data"],+ tags = ["my-tag"],+ model = SimpleNamespace(+ config = SimpleNamespace(_name_or_path = "org/base", model_type = "qwen3_5")+ ),+ )+ )+ assert len(calls) == 1 and calls[0][:2] == ("dev", True) and "org/data" in calls[0][2]+ # A new repo still gets the hub path's model card, with the caller's tags.+ assert "unsloth" in calls[0][2] and "my-tag" in calls[0][2] and "org/base" in calls[0][2]+++def test_unsloth_wrapped_save_still_pushes_the_head(save_module, tmp_path, monkeypatch):+ """Unsloth's loaders wrap save_pretrained as (self, *args, **kwargs), hiding push_to_hub."""+ from unsloth.models.speculative import _takes_push_to_hub++ class Model:+ def original_model_save_pretrained(+ self,+ save_directory,+ push_to_hub = False,+ **kwargs,+ ):+ pass++ def unsloth_model_save_pretrained(self, *args, **kwargs):+ pass++ assert _takes_push_to_hub(Model().unsloth_model_save_pretrained)+++def test_merged_save_with_a_head_refuses_pickles(tree):+ fn = ast.get_source_segment(+ SAVE_PY.read_text(encoding = "utf-8"), _func(tree, "unsloth_save_model")+ )+ assert "if _keeps_mtp_head and not safe_serialization:" in fn+ assert "safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head" in fn+++def test_get_mtp_head_returns_the_attached_head(save_module):+ import torch+ from unsloth.models.speculative import get_mtp_head++ head = _head(torch)+ model = _wrapped(torch, head, registered = False)+ assert get_mtp_head(model) is head+++def test_plain_save_refuses_pickles_before_writing(save_module, tmp_path):+ import torch++ model, calls = _wrapped_adapter(torch)+ with pytest.raises(ValueError, match = "safetensors"):+ model.save_pretrained(str(tmp_path / "p"), safe_serialization = False)+ assert calls == []+++def test_modelscope_snapshot_wins_under_a_peft_wrapper(tmp_path):+ import torch+ from unsloth.models.speculative import _checkpoint_source++ inner = torch.nn.Module()+ inner.config = SimpleNamespace(_name_or_path = "modelscope/only-id")+ object.__setattr__(inner, "_unsloth_modelscope_snapshot", str(tmp_path))+ peft = torch.nn.Module()+ peft.config = SimpleNamespace(_name_or_path = "modelscope/only-id")+ peft.base_model = torch.nn.Module()+ peft.base_model.model = inner+ assert _checkpoint_source(peft) == str(tmp_path)+++def test_mtp_push_never_replaces_a_card_it_could_not_read(save_module, monkeypatch):+ import huggingface_hub++ uploads = []++ class Api:+ def __init__(self, token = None):+ pass++ def create_repo(self, **kwargs):+ pass++ def upload_folder(self, **kwargs):+ uploads.append(kwargs)++ def unreachable(*args, **kwargs):+ raise huggingface_hub.errors.LocalEntryNotFoundError("offline")++ monkeypatch.setattr(save_module, "HfApi", Api)+ monkeypatch.setattr(save_module, "_determine_username", lambda d, u, t: ("me/m", "me"))+ monkeypatch.setattr(save_module, "unsloth_save_model", lambda **kwargs: None)+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", unreachable)+ with pytest.raises(huggingface_hub.errors.LocalEntryNotFoundError):+ save_module._push_with_mtp_head(+ dict(+ is_main_process = True,+ save_directory = "m",+ token = "t",+ private = False,+ revision = None,+ create_pr = False,+ commit_message = None,+ datasets = None,+ tags = None,+ model = SimpleNamespace(+ config = SimpleNamespace(_name_or_path = "org/base", model_type = "qwen3_5")+ ),+ )+ )+ assert uploads == []+++def test_stale_adapter_config_does_not_turn_a_full_save_into_a_sidecar(+ save_module, tmp_path, monkeypatch+):+ import os++ import torch+ from unsloth.models.speculative import _wrap_save_pretrained++ out = tmp_path / "reused"+ out.mkdir()+ (out / "adapter_config.json").write_text("{}")+ os.utime(out / "adapter_config.json", (1, 1))+ stitched, sidecars = [], []+ monkeypatch.setattr(save_module, "_stitch_mtp_head", lambda m, d: stitched.append(d))+ monkeypatch.setattr(+ save_module, "_save_mtp_head_beside_adapter", lambda m, d: sidecars.append(d)+ )+ model = torch.nn.Module()+ object.__setattr__(model, "save_pretrained", lambda save_directory, **kwargs: None)+ object.__setattr__(model, "_unsloth_mtp_head", _head(torch))+ _wrap_save_pretrained(model)+ model.save_pretrained(str(out))+ assert stitched == [str(out)] and sidecars == []+++def test_get_mtp_head_reads_the_loaded_commit(tmp_path):+ from unsloth.models.speculative import _loaded_revision++ model = SimpleNamespace(config = SimpleNamespace(_name_or_path = "org/m", _commit_hash = "abc123"))+ assert _loaded_revision(model, "org/m", None) == "abc123"+ assert _loaded_revision(model, "org/m", "dev") == "dev"+ assert _loaded_revision(model, str(tmp_path), None) is Nonediff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.pyindex 53ec1d02849..57086371551 100644--- a/unsloth/models/__init__.py+++ b/unsloth/models/__init__.py@@ -27,6 +27,13 @@ except:# falcon_h1 absent before transformers 4.53.0; skippassfrom .dpo import PatchDPOTrainer, PatchKTOTrainer+from .speculative import get_mtp_head++try:+ from .speculative import UnslothMTPTrainer, MTPTrainingArguments+ from .speculative import UnslothDSparkTrainer, DSparkTrainingArguments, Qwen3DSparkDrafter+except ImportError:+ passfrom ._utils import is_bfloat16_supported, is_vLLM_available, __version__from .rl import PatchFastRL, vLLMSamplingParamsfrom .lora_init import patch_peft_calibration_eager as _patch_peft_calibration_eagerdiff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 6dffa18f3f9..671a13cb25d 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -973,6 +973,44 @@ def _vllm_unavailable_error():)+def _reattach_saved_mtp_head(+ model,+ adapter_dir,+ token = None,+ revision = None,+ cache_dir = None,+ local_files_only = False,+):+ """A LoRA folder (local or on the Hub) saved with a trained MTP head gets it back, so a later+ merge keeps it."""+ if not isinstance(adapter_dir, str):+ return+ if not os.path.isfile(os.path.join(adapter_dir, "mtp_head.safetensors")):+ if os.path.isdir(adapter_dir):+ return+ try:+ from huggingface_hub import hf_hub_download+ path = hf_hub_download(+ adapter_dir,+ "mtp_head.safetensors",+ token = token,+ revision = revision,+ cache_dir = cache_dir,+ local_files_only = local_files_only,+ )+ except Exception:+ return # no head uploaded beside this adapter+ adapter_dir = os.path.dirname(path)+ try:+ from .speculative import attach_saved_mtp_head+ attach_saved_mtp_head(model, adapter_dir)+ print(f"Unsloth: Re-attached the trained MTP head saved in '{adapter_dir}'.")+ except Exception as error:+ print(+ f"Unsloth: Could not re-attach the MTP head in '{adapter_dir}' ({error}); merges will keep the stock head."+ )++def _diffusion_peft_kwargs(model,args,@@ -1042,6 +1080,11 @@ class FastLanguageModel(FastLlamaModel):return FastDiffusionModel.for_training(model, use_gradient_checkpointing)return FastLlamaModel.for_training(model, use_gradient_checkpointing)+ @staticmethod+ def get_mtp_head(model, *args, **kwargs):+ from .speculative import get_mtp_head+ return get_mtp_head(model, *args, **kwargs)+@staticmethod@tokenizers_v1_on_return@_offline_aware_load@@ -1856,6 +1899,14 @@ class FastLanguageModel(FastLlamaModel):)if fast_pissa:record_fast_pissa(model)+ _reattach_saved_mtp_head(+ model,+ old_model_name,+ token = token,+ revision = revision,+ cache_dir = kwargs.get("cache_dir"),+ local_files_only = local_files_only,+ )model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)try:from .vision import _lift_endpoint_hooks_onto_adapters@@ -1910,6 +1961,11 @@ except:class FastModel(FastBaseModel):+ @staticmethod+ def get_mtp_head(model, *args, **kwargs):+ from .speculative import get_mtp_head+ return get_mtp_head(model, *args, **kwargs)+@staticmethoddef _prepare_for_qat(model, qat_scheme):model = _prepare_model_for_qat(model, qat_scheme)@@ -3182,6 +3238,14 @@ class FastModel(FastBaseModel):)if fast_pissa:record_fast_pissa(model)+ _reattach_saved_mtp_head(+ model,+ old_model_name,+ token = token,+ revision = revision,+ cache_dir = kwargs.get("cache_dir"),+ local_files_only = local_files_only,+ )try:from .vision import _lift_endpoint_hooks_onto_adapters_lift_endpoint_hooks_onto_adapters(model)diff --git a/unsloth/models/speculative.py b/unsloth/models/speculative.pynew file mode 100644index 00000000000..8224741695f--- /dev/null+++ b/unsloth/models/speculative.py@@ -0,0 +1,255 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""Fine-tune a model's native MTP head so its speculative drafts match the fine-tuned model.++transformers drops `mtp.*` on load, and the merged export copies the stock head through+unchanged, so after fine-tuning the head drafts for the old model. `get_mtp_head` loads the+head from the checkpoint and attaches it; `UnslothMTPTrainer` trains it against the frozen+model; every merged / GGUF save then writes the trained head instead of the stock one.+"""++# The trainer names resolve lazily through __getattr__ (they need a recent unsloth_zoo).+__all__ = ["get_mtp_head", "attach_saved_mtp_head"]++import functools+import inspect+import os+import time+++def _zoo_speculative():+ try:+ import unsloth_zoo.speculative as speculative+ from unsloth_zoo.speculative import mtp # noqa: F401+ except ImportError as error:+ raise ImportError(+ "Unsloth: MTP head training needs a newer unsloth_zoo. "+ "Update via `pip install --upgrade --no-cache-dir unsloth_zoo`."+ ) from error+ return speculative+++def _checkpoint_source(model):+ """The repo id / folder the weights came from, looked up through PEFT wrappers."""+ holders, seen, stack = [], set(), [model]+ while stack:+ holder = stack.pop()+ if holder is None or id(holder) in seen:+ continue+ seen.add(id(holder))+ holders.append(holder)+ for attribute in ("base_model", "model"):+ child = holder.__dict__.get("_modules", {}).get(attribute) or holder.__dict__.get(+ attribute+ )+ if child is not None:+ stack.append(child)+ # ModelScope loads keep the repo id in the config and the downloaded weights on the inner+ # model, under any PEFT wrapper: the snapshot wins over every holder's config.+ for holder in holders:+ snapshot = holder.__dict__.get("_unsloth_modelscope_snapshot")+ if isinstance(snapshot, str) and os.path.isdir(snapshot):+ return snapshot+ for holder in holders:+ name = getattr(getattr(holder, "config", None), "_name_or_path", None)+ if name:+ return name+ return None+++def _loaded_revision(model, source, revision):+ """The commit the weights were loaded from, so a Hub lookup reads that head, not the default branch's."""+ if revision is not None or not isinstance(source, str) or os.path.isdir(source):+ return revision+ return getattr(getattr(model, "config", None), "_commit_hash", None)+++def get_mtp_head(+ model,+ source = None,+ revision = None,+ random_init = False,+ dtype = None,+):+ """Load the model's native MTP head, attach it to `model`, and return it.++ source: repo id or folder holding the original `mtp.*` tensors. Defaults to the+ checkpoint the model was loaded from; pass the original repo when that checkpoint+ was re-saved without its head.+ random_init: start from random weights instead (for checkpoints without a head).+ dtype: head parameter dtype, float32 by default (the head is small; the target stays as loaded).+ Without `source` / `random_init`, a head already attached (e.g. re-attached from beside a+ reloaded adapter) is returned as is.+ """+ import torch++ if source is None and not random_init:+ from ..save import _attached_mtp_head+ attached = _attached_mtp_head(model)+ if attached is not None:+ return attached if dtype is None else attached.to(dtype = dtype)+ speculative = _zoo_speculative()+ from unsloth_zoo.speculative.mtp import build_mtp_head, load_mtp_state_dict++ parts = speculative.resolve_text_parts(model)+ if source is None:+ source = _checkpoint_source(model)+ revision = _loaded_revision(model, source, revision)+ family = speculative.mtp_family_name(parts["config"])+ if family is None:+ raise NotImplementedError(+ f"Unsloth: `{getattr(parts['config'], 'model_type', None)}` has no native MTP head Unsloth can train."+ )+ if random_init:+ state_dict, info = (+ None,+ {"prefix": "mtp." if family == "qwen3_5" else "", "num_layers": 1, "source": source},+ )+ elif isinstance(source, str) and os.path.isfile(os.path.join(source, "mtp_head.safetensors")):+ # A head trained earlier and saved beside a LoRA adapter.+ from unsloth_zoo.speculative.stitch import load_saved_mtp_head+ state_dict, info = load_saved_mtp_head(source)+ else:+ if source is None:+ raise ValueError(+ "Unsloth: pass `source=` (the repo id or folder that holds the `mtp.*` tensors)."+ )+ state_dict, info = load_mtp_state_dict(source, revision = revision, config = parts["config"])+ # `files` are local cache paths; never write them into a folder that may be pushed.+ info = {k: v for k, v in info.items() if k != "files"}+ info.update(source = source, revision = revision)+ device = parts["embed_tokens"].weight.device+ extra = {}+ if family != "qwen3_5":+ # Head-serialized embedding / lm_head copies equal to the target's are not held twice.+ extra = dict(+ target_embed = parts["embed_tokens"].weight,+ target_lm_head = parts["lm_head"].weight,+ ckpt_prefix = info.get("ckpt_prefix"),+ )+ head = build_mtp_head(+ parts["config"],+ state_dict = state_dict,+ dtype = dtype or torch.float32,+ device = device,+ **extra,+ )+ status = getattr(getattr(head, "family", None), "llama_cpp", "drafts")+ if status != "drafts":+ print(+ f"Unsloth: llama.cpp support for `{family}` MTP drafting is `{status}`; the trained head "+ "is still written into merged / GGUF exports."+ )+ head._unsloth_mtp_info = info+ # Not a submodule: keeps the head out of model.state_dict(), PEFT and the target's optimizer.+ object.__setattr__(model, "_unsloth_mtp_head", head)+ _wrap_save_pretrained(model)+ return head+++def _takes_push_to_hub(fn):+ # Unsloth's loaders wrap save_pretrained as (self, *args, **kwargs); read the wrapped one.+ if getattr(fn, "__name__", "") == "unsloth_model_save_pretrained":+ fn = getattr(getattr(fn, "__self__", None), "original_model_save_pretrained", fn)+ try:+ return "push_to_hub" in inspect.signature(fn).parameters+ except (TypeError, ValueError):+ return False+++def _wrap_save_pretrained(model):+ """Plain `save_pretrained` keeps the trained head too: beside a LoRA adapter, else in the checkpoint."""+ original = model.save_pretrained+ if getattr(original, "_unsloth_saves_mtp_head", False):+ return++ @functools.wraps(original)+ def save_pretrained(save_directory, *args, **kwargs):+ from ..save import (+ _INTERNAL_MTP_SAVE,+ _refuse_mtp_variant,+ _save_mtp_head_beside_adapter,+ _stitch_mtp_head,+ )++ internal = _INTERNAL_MTP_SAVE.get()+ if (+ not internal+ and kwargs.get("safe_serialization", True) is False+ and not hasattr(model, "peft_config")+ ):+ # The head is stitched into safetensors checkpoints only; refuse before writing.+ raise ValueError(+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."+ )+ # PEFT's save_pretrained has no push_to_hub (its push_to_hub saves into a folder, then uploads it).+ push = bool(kwargs.get("push_to_hub")) and _takes_push_to_hub(original)+ if not internal:+ _refuse_mtp_variant(model, kwargs.get("variant"))+ if push and kwargs.get("create_pr"):+ # transformers opens the PR and keeps its ref; a second upload would open another.+ raise ValueError(+ "Unsloth: `create_pr = True` cannot carry the trained MTP head. Save locally, then "+ "upload the folder with `HfApi().upload_folder(..., create_pr = True)`."+ )+ started = time.time()+ result = original(save_directory, *args, **kwargs)+ if internal or not os.path.isdir(save_directory):+ return result+ adapter_config = os.path.join(save_directory, "adapter_config.json")+ # Written by this save, not left over from an adapter saved here before.+ if os.path.isfile(adapter_config) and os.path.getmtime(adapter_config) >= started - 1:+ files = (+ ["mtp_head.safetensors"]+ if _save_mtp_head_beside_adapter(model, save_directory)+ else []+ )+ else:+ report = _stitch_mtp_head(model, save_directory)+ files = list(report["files_touched"]) if report else []+ if push and files:+ # transformers uploaded the folder before the head existed; send the head files after it.+ from huggingface_hub import HfApi+ HfApi(token = kwargs.get("token")).upload_folder(+ repo_id = kwargs.get("repo_id")+ or str(save_directory).rstrip(os.sep).split(os.sep)[-1],+ folder_path = save_directory,+ allow_patterns = files,+ commit_message = "Add the trained MTP head",+ )+ return result++ save_pretrained._unsloth_saves_mtp_head = True+ object.__setattr__(model, "save_pretrained", save_pretrained)+++def attach_saved_mtp_head(model, directory):+ """Re-attach a head saved beside an adapter folder; None when the folder has none."""+ if not isinstance(directory, str) or not os.path.isfile(+ os.path.join(directory, "mtp_head.safetensors")+ ):+ return None+ return get_mtp_head(model, source = directory)+++def __getattr__(name):+ if name in ("UnslothMTPTrainer", "MTPTrainingArguments", "MTPDataCollator"):+ _zoo_speculative()+ from unsloth_zoo.speculative import trainer+ return getattr(trainer, name)+ if name in ("UnslothDSparkTrainer", "DSparkTrainingArguments", "Qwen3DSparkDrafter"):+ _zoo_speculative()+ from unsloth_zoo.speculative import dspark+ return getattr(dspark, name)+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/unsloth/save.py b/unsloth/save.pyindex c282bf21509..f54f25eb52c 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -53,6 +53,7 @@ import jsonimport shutilimport pickleimport gc+import contextvarsimport functoolsfrom transformers.models.llama.modeling_llama import loggerfrom .kernels import fast_dequantize, QUANT_STATE, get_lora_parameters_bias@@ -819,6 +820,202 @@ def _strip_absent_mtp_declaration(config_dict, tensor_names):return False+def _attached_mtp_head(model):+ """The trained MTP head attached as `_unsloth_mtp_head` anywhere under PEFT wrappers, else None.++ Walks `__dict__` / `_modules` directly: `getattr` on a PeftModel forwards misses down the+ wrapper chain, so a plain lookup would answer for the wrong holder or recurse.+ """+ seen = set()+ stack = [model]+ while stack:+ holder = stack.pop()+ if holder is None or id(holder) in seen:+ continue+ seen.add(id(holder))+ attributes = getattr(holder, "__dict__", {})+ modules = attributes.get("_modules") or {}+ head = attributes.get("_unsloth_mtp_head", modules.get("_unsloth_mtp_head"))+ if head is not None:+ return head+ for name in ("base_model", "model", "language_model"):+ child = modules.get(name, attributes.get(name))+ if child is not None:+ stack.append(child)+ return None+++def _mtp_head_state_dict(head):+ """Head-relative checkpoint names (`fc.weight`); `mtp_state_dict` excludes shared target tensors."""+ getter = getattr(head, "mtp_state_dict", None)+ return dict(getter() if callable(getter) else head.state_dict())+++def _mtp_head_info(head):+ info = getattr(head, "_unsloth_mtp_info", None)+ return dict(info) if isinstance(info, dict) else {}+++def _mtp_head_tensor_names(model):+ """Checkpoint names the attached head will occupy, or [] when none is attached."""+ head = _attached_mtp_head(model)+ if head is None:+ return []+ # Qwen3.5 heads use relative names under `mtp.`; NextN heads (prefix "") already use full names.+ prefix = _mtp_head_info(head).get("prefix")+ prefix = "mtp." if prefix is None else prefix+ return [prefix + name for name in _mtp_head_state_dict(head)]+++def _is_main_rank():+ """Only one process writes the head when several save the same folder."""+ try:+ import torch.distributed as dist+ return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0+ except Exception:+ return True+++def _refuse_mtp_variant(model, variant):+ """Variant filenames (`model.<variant>.safetensors`) are not stitched; refuse before writing anything."""+ if variant is not None and _attached_mtp_head(model) is not None:+ raise NotImplementedError(+ "Unsloth: saving a trained MTP head with `variant=` is not supported; save without `variant`."+ )+++def _stitch_mtp_head(model, save_directory):+ """Write the attached trained MTP head into a saved checkpoint. No-op when none is attached."""+ head = _attached_mtp_head(model)+ if head is None or not _is_main_rank():+ return None+ from unsloth_zoo.speculative.stitch import write_mtp_tensors++ info = _mtp_head_info(head)+ # NextN families (GLM, DeepSeek, Qwen3-Next, ...) say where their tensors live themselves.+ stitch_kwargs = getattr(head, "stitch_kwargs", None)+ report = write_mtp_tensors(+ save_directory,+ _mtp_head_state_dict(head),+ append_prefix = info.get("prefix") or "mtp.",+ **(stitch_kwargs() if callable(stitch_kwargs) else {}),+ )+ print(+ f"Unsloth: Wrote the trained MTP head ({len(report['tensors_written'])} tensors, "+ f"{report['mode']}) into '{save_directory}'."+ )+ return report+++# Set while an Unsloth saver writes a checkpoint it stitches the head into itself, so the plain+# `save_pretrained` wrapper (unsloth/models/speculative.py) does not stitch it a second time,+# however the wrappers are stacked (an export reload wraps it again from outside).+_INTERNAL_MTP_SAVE = contextvars.ContextVar("unsloth_internal_mtp_save", default = False)+++@contextmanager+def _internal_mtp_save():+ token = _INTERNAL_MTP_SAVE.set(True)+ try:+ yield+ finally:+ _INTERNAL_MTP_SAVE.reset(token)+++def _save_mtp_head_beside_adapter(model, save_directory):+ """LoRA saves keep the trained head as `mtp_head.safetensors` so a later merge can stitch it."""+ head = _attached_mtp_head(model)+ if not _is_main_rank():+ return None+ if head is None:+ # A reused folder's earlier head would be re-attached to this unrelated adapter on load.+ stale = os.path.join(save_directory, "mtp_head.safetensors")+ if os.path.isfile(stale):+ os.remove(stale)+ return None+ from unsloth_zoo.speculative.stitch import save_mtp_head++ path = save_mtp_head(save_directory, _mtp_head_state_dict(head), _mtp_head_info(head))+ print(f"Unsloth: Saved the trained MTP head to '{path}'.")+ return path+++def _push_with_mtp_head(save_kwargs):+ """Push for `unsloth_save_model` with a trained MTP head: save locally, stitch, upload the folder.++ Every direct-to-hub writer streams the stock head from the original shards, so the head can+ only be written into a local copy.+ """+ import tempfile+ from huggingface_hub.errors import HfHubHTTPError++ if not save_kwargs["is_main_process"]:+ return save_kwargs["save_directory"], None+ token = save_kwargs["token"] if save_kwargs["token"] is not None else get_token()+ repo_id, username = _determine_username(save_kwargs["save_directory"], None, token)+ api = HfApi(token = token)+ api.create_repo(+ repo_id = repo_id,+ repo_type = "model",+ private = save_kwargs["private"],+ exist_ok = True,+ )+ revision = save_kwargs["revision"]+ if revision is not None and not revision.startswith("refs/pr/"):+ try:+ api.create_branch(repo_id = repo_id, repo_type = "model", branch = revision, exist_ok = True)+ except HfHubHTTPError as error:+ if not save_kwargs["create_pr"] or error.response.status_code != 403:+ raise+ with tempfile.TemporaryDirectory(prefix = "unsloth-mtp-") as directory:+ unsloth_save_model(+ **{**save_kwargs, "save_directory": directory, "push_to_hub": False, "token": token}+ )+ # The card the hub path writes, staged here so it lands in this commit on this revision.+ from huggingface_hub import ModelCard, hf_hub_download, metadata_save+ from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError++ readme = os.path.join(directory, "README.md")+ if not os.path.isfile(readme):+ try:+ shutil.copyfile(+ hf_hub_download(repo_id, "README.md", revision = revision, token = token),+ readme,+ )+ except LocalEntryNotFoundError:+ raise # offline or unreachable: never replace a card that may exist+ except EntryNotFoundError: # no card yet; any other failure must not replace one+ config = save_kwargs["model"].config+ ModelCard(+ MODEL_CARD.format(+ username = username,+ base_model = config._name_or_path,+ model_type = config.model_type,+ method = "finetuned",+ extra = "unsloth",+ )+ ).save(readme)+ tags = ModelCard.load(readme).data.tags or []+ metadata = {+ "tags": list(dict.fromkeys([*tags, *(save_kwargs.get("tags") or []), "unsloth"]))+ }+ if save_kwargs.get("datasets"):+ metadata["datasets"] = save_kwargs["datasets"]+ metadata_save(readme, metadata)+ api.upload_folder(+ folder_path = directory,+ path_in_repo = ".",+ repo_id = repo_id,+ repo_type = "model",+ commit_message = save_kwargs["commit_message"] or "Trained with Unsloth",+ commit_description = save_kwargs.get("commit_description"),+ revision = save_kwargs["revision"],+ create_pr = save_kwargs["create_pr"],+ )+ print(f"Unsloth: Saved model with its trained MTP head to https://huggingface.co/{repo_id}")+ return repo_id, username++def _is_qwen3_5_vlm(model):config = getattr(model, "config", None)if config is None or not hasattr(config, "vision_config"):@@ -943,6 +1140,10 @@ def unsloth_save_model(maximum_memory_usage: float = 0.9,datasets: Optional[List[str]] = None,):+ _refuse_mtp_variant(model, variant)+ if push_to_hub and _attached_mtp_head(model) is not None:+ return _push_with_mtp_head(dict(locals()))+if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):tokenizer = patch_saving_functions(tokenizer)@@ -1019,7 +1220,11 @@ def unsloth_save_model(print("This might take 5 minutes...")if hasattr(model, "merge_and_unload"):+ mtp_head = _attached_mtp_head(model)model = model.merge_and_unload()+ # merge_and_unload returns the inner model; keep the trained head with it.+ if mtp_head is not None:+ object.__setattr__(model, "_unsloth_mtp_head", mtp_head)print("Done.")if tags is not None:@@ -1170,7 +1375,12 @@ def unsloth_save_model(if save_method == "lora":save_pretrained_settings["selected_adapters"] = None- model.save_pretrained(**save_pretrained_settings)+ with _internal_mtp_save():+ model.save_pretrained(**save_pretrained_settings)+ if save_method == "lora":+ _save_mtp_head_beside_adapter(model, save_pretrained_settings["save_directory"])+ else:+ _stitch_mtp_head(model, save_pretrained_settings["save_directory"])if push_to_hub and hasattr(model, "config"):print("Saved to https://huggingface.co/" + save_pretrained_settings["save_directory"])@@ -1222,12 +1432,18 @@ def unsloth_save_model(if n_cpus is None:n_cpus = 1+ # The trained MTP head is stitched into safetensors checkpoints only.+ _keeps_mtp_head = _attached_mtp_head(model) is not None+ if _keeps_mtp_head and not safe_serialization:+ raise ValueError(+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."+ )# Already `True` from the normalisation at the top of this function; the caller having# passed `None` is what keeps the downgrade below from undoing it.if _force_safe_serialization:save_pretrained_settings["safe_serialization"] = safe_serialization- elif safe_serialization and (n_cpus <= 2):+ elif safe_serialization and (n_cpus <= 2) and not _keeps_mtp_head:logger.warning_once(f"Unsloth: You have {n_cpus} CPUs. Using `safe_serialization` is 10x slower.\n"f"We shall switch to Pytorch saving, which might take 3 minutes and not 30 minutes.\n"@@ -1435,8 +1651,11 @@ def unsloth_save_model(new_config = model.config.to_dict()if "quantization_config" in new_config:del new_config["quantization_config"]- # The merged state dict is what gets written, so its keys decide MTP presence.- _strip_absent_mtp_declaration(new_config, state_dict.keys())+ # The merged state dict is what gets written, so its keys decide MTP presence; a trained+ # head attached to the model is stitched in right after the write.+ _strip_absent_mtp_declaration(+ new_config, list(state_dict.keys()) + _mtp_head_tensor_names(model)+ )original_model = modelnew_config = type(model.config).from_dict(new_config)while hasattr(original_model, "model"):@@ -1467,13 +1686,17 @@ def unsloth_save_model(ignore_patterns = "*.md",)else:- internal_model.save_pretrained(**save_pretrained_settings)+ with _internal_mtp_save():+ internal_model.save_pretrained(**save_pretrained_settings)finally:original_model = modelwhile hasattr(original_model, "model"):original_model = original_model.modeloriginal_model.config = old_configmodel.config = old_config+ # Pushes with an attached head were rerouted to a local save at the top.+ if not save_pretrained_settings["push_to_hub"]:+ _stitch_mtp_head(model, save_pretrained_settings["save_directory"])print("Done.")if push_to_hub and hasattr(model, "config"):@@ -3390,6 +3613,9 @@ def _gguf_reuses_loaded_checkpoint(model, state_dict = None):return Falseif state_dict is not None or getattr(model, "_unsloth_full_finetuning", False):return False+ # The loaded folder holds the stock MTP head; a trained one needs a fresh checkpoint.+ if _attached_mtp_head(model) is not None:+ return False# A ModelScope load keeps its snapshot here, since `_name_or_path` holds the repo id (#3726).name_or_path = getattr(model, "_unsloth_modelscope_snapshot", None) or getattr(getattr(model, "config", None), "_name_or_path", None@@ -4161,7 +4387,9 @@ def unsloth_save_pretrained_gguf(os.makedirs(save_directory, exist_ok = True)# `gguf_directory` can point anywhere, and freeing bytes on one filesystem does nothing for a quantize pass writing to another: without this the merge could be deleted for a destination it cannot help, data gone and the export still out of space.try:- self.save_pretrained(save_directory, state_dict = state_dict)+ with _internal_mtp_save():+ self.save_pretrained(save_directory, state_dict = state_dict)+ _stitch_mtp_head(self, save_directory)if tokenizer is not None:tokenizer.save_pretrained(save_directory)except Exception as e:@@ -5666,6 +5894,7 @@ def unsloth_generic_save(datasets: Optional[List[str]] = None,dtype = None,):+ _refuse_mtp_variant(model, variant)if save_method == "merged_4bit":raise RuntimeError("Unsloth: Merging into 4bit will cause your model to lose accuracy if you plan\n"@@ -5679,7 +5908,13 @@ def unsloth_generic_save(# this function's own keywords.safe_serialization = _normalize_safe_serialization(safe_serialization)- if push_to_hub and (create_pr or revision is not None or not isinstance(model, PeftModel)):+ # A trained MTP head can only be stitched into a local copy, which this route makes first.+ if push_to_hub and (+ create_pr+ or revision is not None+ or not isinstance(model, PeftModel)+ or _attached_mtp_head(model) is not None+ ):return _push_merged_to_hub_revision(dict(locals()))if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):@@ -5708,6 +5943,11 @@ def unsloth_generic_save(# method, since `patch_saving_functions` wraps it in a `(*args, **kwargs)` passthrough# that would hide a transformers which does honour the request._real_save = getattr(model, "original_model_save_pretrained", model.save_pretrained)+ if safe_serialization is False and _attached_mtp_head(model) is not None:+ # The trained head is stitched into safetensors checkpoints only; refuse before writing.+ raise ValueError(+ "Unsloth: a model with a trained MTP head saves as safetensors; drop `safe_serialization = False`."+ )if safe_serialization is False and not _honours_safe_serialization(_real_save):logger.warning_once("Unsloth: this transformers always writes safetensors, so "@@ -5728,6 +5968,12 @@ def unsloth_generic_save(state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)if is_qwen3_5_vlm:state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)+ if _attached_mtp_head(model) is not None:+ # A head registered as a submodule would otherwise be written under its attribute+ # name; it is stitched in under its checkpoint names after the save instead.+ if state_dict is None:+ state_dict = model.state_dict()+ state_dict = {k: v for k, v in state_dict.items() if "_unsloth_mtp_head." not in k}if state_dict is not None:_save_kwargs["state_dict"] = state_dict@@ -5738,9 +5984,15 @@ def unsloth_generic_save(_written_tensor_names = list(state_dict.keys()) if state_dict is not None else Noneprint(f"Unsloth: Saving full fine-tuned model to '{save_directory}' ...")- model.save_pretrained(save_directory, **_save_kwargs)+ with _internal_mtp_save():+ model.save_pretrained(save_directory, **_save_kwargs)if "16bit" in save_method and _requests_float32(dtype):_stamp_saved_config_float32(save_directory)+ if (+ _stitch_mtp_head(model, save_directory) is not None+ and _written_tensor_names is not None+ ):+ _written_tensor_names = _written_tensor_names + _mtp_head_tensor_names(model)# Guarded: an older zoo must not raise once the weights are already on disk.try:from unsloth_zoo.saving_utils import reconcile_mtp_config@@ -5828,6 +6080,9 @@ def unsloth_generic_save(low_disk_space_usage = True,use_temp_file = False,)+ # Pushes with an attached head were rerouted to a local save above.+ if not push_to_hub:+ _stitch_mtp_head(model, save_directory)if push_to_hub and datasets:try:@@ -6656,6 +6911,12 @@ def _unsloth_save_compressed_tensors(# Only the main process installs deps, merges, quantizes and uploads, mirroring the non-PEFT save path; other ranks return at once rather than race on dirs or run pip installs.if not is_main_process:return None+ if _attached_mtp_head(model) is not None:+ # The quantizer reloads through transformers, which drops native MTP tensors.+ print(+ "Unsloth: compressed-tensors exports do not carry the trained MTP head; use merged_16bit "+ "or GGUF to keep it."+ )# Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of writing a full 16bit checkpoint first. Under the llm-compressor-main shadow the subprocess validates itself, so the workspace install / ceiling / scheme checks are skipped._shadow_pythonpath = _compressed_quantize_pythonpath()@@ -6911,6 +7172,12 @@ def _unsloth_save_torchao(if not is_main_process:return None+ if _attached_mtp_head(model) is not None:+ # The reload below drops native MTP tensors (the reason the head is stitched at all).+ print(+ f"Unsloth: torchao {kind} exports do not carry the trained MTP head; use merged_16bit or "+ "GGUF to keep it."+ )from transformers import (AutoModelForCausalLM,