Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
TP / PP: every causal LM architecture trains like one process or is refused up front (#15230)
Daniel Han4f18eb4326a7
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex 1571ed2756e..6f293df21a5 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -219,6 +219,8 @@ _GENERIC_SAVE_STUBBED_HELPERS = frozenset("_stitch_mtp_head","_mtp_head_tensor_names","_unmerge_flags_on_error",+ # Decorator factory; _load strips decorators.+ "_model_parallel_refuses_save",})diff --git a/tests/test_parallel_general_gloo.py b/tests/test_parallel_general_gloo.pynew file mode 100644index 00000000000..18f3b6eab69--- /dev/null+++ b/tests/test_parallel_general_gloo.py@@ -0,0 +1,768 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""apply_tensor_parallel / apply_pipeline_parallel across architectures, gloo CPU, 2 ranks, fp32, vs one+process: every architecture either trains like one process or is refused before the model is touched.++The configs are tiny shrinks of each model type's default config, built offline."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import importlib.machinery # noqa: E402+import importlib.util # noqa: E402+import socket # noqa: E402+import sys # noqa: E402+import tempfile # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+++def _load(name):+ """unsloth/models/<name>.py without importing the unsloth package (which needs a GPU)."""+ full = f"unsloth.models.{name}"+ if full in sys.modules:+ return sys.modules[full]+ pkg = types.ModuleType("unsloth")+ pkg.__path__ = [os.path.join(REPO, "unsloth")]+ models = types.ModuleType("unsloth.models")+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]+ for package in (pkg, models):+ package.__spec__ = importlib.machinery.ModuleSpec(package.__name__, None, is_package = True)+ package.__spec__.submodule_search_locations = package.__path__+ sys.modules.setdefault("unsloth", pkg)+ sys.modules.setdefault("unsloth.models", models)+ spec = importlib.util.spec_from_file_location(+ full, os.path.join(REPO, "unsloth", "models", f"{name}.py")+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[full] = module+ spec.loader.exec_module(module)+ return module+++tpn = _load("tp_native")+ppn = _load("pp_native")++_SMALL = dict(vocab_size = 512, hidden_size = 64, intermediate_size = 128, num_hidden_layers = 4)+_HEADS = dict(num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16)+# arch -> (model_type, config kwargs)+CONFIGS = {+ "llama": ("llama", {**_SMALL, **_HEADS}),+ # GPT-J: parallel attention + MLP, fc_in / fc_out, partial rotary+ "gptj": (+ "gptj",+ dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, rotary_dim = 8, n_inner = 128),+ ),+ # per-head query_layernorm / key_layernorm+ "hunyuan_v1_dense": (+ "hunyuan_v1_dense",+ {+ **_SMALL,+ **_HEADS,+ "use_qk_norm": True,+ "attention_dropout": 0.0,+ "tie_word_embeddings": False,+ },+ ),+ # short-conv layers (replicated) between attention layers with q_layernorm / k_layernorm+ "lfm2": (+ "lfm2",+ {+ **_SMALL,+ **_HEADS,+ "layer_types": ["conv", "full_attention", "conv", "full_attention"],+ "full_attn_idxs": [1, 3],+ },+ ),+ # xIELU activation with scalar parameters inside the MLP+ "apertus": ("apertus", {**_SMALL, "num_attention_heads": 4, "num_key_value_heads": 2}),+ # activation sparsity: mean / std over the intermediate features+ "gemma3n_text": (+ "gemma3n_text",+ {+ **_SMALL,+ **_HEADS,+ "intermediate_size": 128,+ "layer_types": ["sliding_attention"] * 3 + ["full_attention"],+ "hidden_size_per_layer_input": 16,+ "num_kv_shared_layers": 0,+ "activation_sparsity_pattern": [0.95, 0.95, 0.0, 0.0],+ "laurel_rank": 8,+ },+ ),+ # attention output reshaped with self.embed_dim / the input width+ "xglm": (+ "xglm",+ dict(vocab_size = 512, d_model = 64, ffn_dim = 128, num_layers = 4, attention_heads = 4),+ ),+ "trocr": (+ "trocr",+ dict(+ vocab_size = 512,+ d_model = 64,+ decoder_layers = 4,+ decoder_attention_heads = 4,+ decoder_ffn_dim = 128,+ ),+ ),+ # fused qkv_proj in mp_num groups, not [q, k, v] blocks+ "codegen": (+ "codegen",+ dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, rotary_dim = 8, n_inner = 128),+ ),+ # learned position tables before the layers+ "gpt2": ("gpt2", dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, n_inner = 128)),+ "opt": (+ "opt",+ dict(+ vocab_size = 512,+ hidden_size = 64,+ num_hidden_layers = 4,+ ffn_dim = 128,+ word_embed_proj_dim = 64,+ num_attention_heads = 4,+ ),+ ),+ # CrossEntropyLoss mean per call (no num_items_in_batch)+ "bart": (+ "bart",+ dict(+ vocab_size = 512,+ d_model = 64,+ decoder_layers = 4,+ decoder_ffn_dim = 128,+ decoder_attention_heads = 4,+ encoder_layers = 1,+ encoder_ffn_dim = 128,+ encoder_attention_heads = 4,+ ),+ ),+}+++def _zero_dropout(config):+ for key in list(config.to_dict()):+ value = getattr(config, key, None)+ if isinstance(value, float) and ("dropout" in key or "pdrop" in key):+ setattr(config, key, 0.0)+++def build(+ arch,+ lora = True,+ seed = 0,+ share = None,+):+ from transformers import AutoModelForCausalLM+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING++ model_type, kwargs = CONFIGS[arch]+ if model_type not in CONFIG_MAPPING:+ pytest.skip(f"transformers has no {model_type}")+ config = CONFIG_MAPPING[model_type](**kwargs)+ _zero_dropout(config)+ config._attn_implementation = "eager"+ torch.manual_seed(seed)+ try:+ model = AutoModelForCausalLM.from_config(config, dtype = torch.float32)+ except TypeError:+ model = AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+ with torch.no_grad():+ for name, p in model.named_parameters():+ if name.endswith("bias"):+ p.normal_(0, 0.1)+ elif "norm" in name and p.ndim == 1:+ p.add_(torch.randn_like(p) * 0.1)+ if (+ share is not None+ ): # layers[j] is layers[i]: a block reused by several layers (Zamba2 / ALBERT style)+ i, j = share+ layers = tpn._decoder_layers(model)[1]+ layers[j] = layers[i]+ if not lora:+ return model.train()+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(seed + 1)+ kw = dict(+ r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False, lora_dropout = 0.0+ )+ return get_peft_model(model, LoraConfig(**kw)).train()+++def batch(+ model,+ bsz = 4,+ seq = 8,+):+ g = torch.Generator().manual_seed(1)+ ids = torch.randint(0, 500, (bsz, seq), generator = g)+ labels = ids.clone()+ labels[0, :3] = -100+ labels[-1, -2:] = -100+ return ids, labels+++def _clip_fp64(params, max_norm):+ grads = [p.grad for p in params if p.grad is not None]+ total = torch.stack([g.double().square().sum() for g in grads]).sum().sqrt()+ coef = torch.clamp(max_norm / (total + 1e-6), max = 1.0)+ for g in grads:+ g.mul_(coef.to(g.dtype))+ return float(total)+++def _reference(+ model,+ ids,+ labels,+ steps = 2,+):+ params = [p for p in model.parameters() if p.requires_grad]+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)+ losses, norms, grads = [], [], None+ for step in range(steps):+ out = model(input_ids = ids, labels = labels, use_cache = False)+ out.loss.backward()+ if step == 0:+ grads = {+ n: p.grad.detach().clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ norms.append(_clip_fp64(params, 1.0))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(out.loss))+ return {"losses": losses, "norms": norms, "grads": grads}+++def _fingerprint(model):+ """Everything a refusal must leave as it was."""+ return {+ "params": {+ n: (type(p).__name__, p.detach().clone(), p.requires_grad)+ for n, p in model.named_parameters()+ },+ "modules": [(n, type(m).__name__) for n, m in model.named_modules()],+ "marks": sorted(+ (n, k)+ for n, m in model.named_modules()+ for k in m.__dict__+ if k.startswith("_unsloth") or k in ("forward", "generate")+ ),+ "hooks": sum(len(m._forward_hooks) + len(m._forward_pre_hooks) for m in model.modules()),+ }+++def _unchanged(before, after):+ if before["modules"] != after["modules"] or before["marks"] != after["marks"]:+ return False+ if before["hooks"] != after["hooks"] or set(before["params"]) != set(after["params"]):+ return False+ for n, (cls, value, grad) in before["params"].items():+ cls2, value2, grad2 = after["params"][n]+ if cls != cls2 or grad != grad2 or not torch.equal(value, value2):+ return False+ return True+++def _refusal(apply, model):+ before = _fingerprint(model)+ try:+ apply(model)+ except ValueError as e:+ return {"error": str(e), "unchanged": _unchanged(before, _fingerprint(model))}+ except Exception as e:+ return {+ "error": f"crash {type(e).__name__}: {e}",+ "unchanged": _unchanged(before, _fingerprint(model)),+ }+ return {"error": None, "unchanged": None}+++# ---------------------------------------------------------------------------------------------- cases++TP_PASS = {+ "gptj": None,+ "hunyuan_v1_dense": None,+ "lfm2": None,+ "apertus": None,+ "llama_shared": (0, 1),+}+TP_REFUSE = {+ "gemma3n_text": "reduces over its features",+ "xglm": "the global width",+ "trocr": "keeps the input width",+ "codegen": "does not slice the fused 'qkv_proj'",+}+++def _case_tp(rank, world, tmp):+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]+ out = {}+ for name, share in TP_PASS.items():+ try:+ out[name] = _tp_parity(mesh, name.replace("_shared", ""), share)+ except (+ Exception+ ) as e: # recorded per architecture: one failing case must not hide the others+ out[name] = {"crash": f"{type(e).__name__}: {e}"}+ for arch in TP_REFUSE:+ out[arch] = _refusal(lambda m: tpn.apply_tensor_parallel(m, mesh), build(arch))+ return out+++def _tp_parity(mesh, arch, share):+ ref = build(arch, share = share)+ ids, labels = batch(ref, bsz = 2)+ model = build(arch, share = share)+ tpn.apply_tensor_parallel(model, mesh)+ params = [p for p in model.parameters() if p.requires_grad]+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)+ losses, norms, grads = [], [], None+ for step in range(2):+ loss = model(input_ids = ids, labels = labels, use_cache = False).loss+ loss.backward()+ if step == 0:+ grads = {+ n: tpn._full(p.grad, p).detach().clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ norms.append(float(tpn.tp_clip_grad_norm_(params, 1.0)))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(loss))+ return {+ "tp": {"losses": losses, "norms": norms, "grads": grads},+ "ref": _reference(ref, ids, labels),+ "regions": len(model._unsloth_tp_plan.regions),+ }+++PP_PASS = ("gpt2", "opt")+PP_REFUSE = {+ "bart": ("bart", None, "needs the BartForCausalLM loss to take num_items_in_batch"),+ "llama_shared": ("llama", (0, 3), "share trainable parameters across stages"),+}+++def _case_pp(rank, world, tmp):+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]+ out = {}+ for arch in PP_PASS:+ try:+ out[arch] = _pp_parity(mesh, arch)+ except Exception as e:+ out[arch] = {"crash": f"{type(e).__name__}: {e}"}+ for name, (arch, share, _) in PP_REFUSE.items():+ model = build(arch, share = share)+ out[name] = _refusal(+ lambda m: ppn.apply_pipeline_parallel(m, mesh, n_microbatches = 2), model+ )+ return out+++def _pp_parity(mesh, arch):+ ref = build(arch)+ ids, labels = batch(ref)+ model = build(arch)+ pp = ppn.apply_pipeline_parallel(model, mesh, n_microbatches = 2)+ params = pp.parameters()+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)+ losses, norms, grads = [], [], None+ for step in range(2):+ loss = pp.step(ids, labels)+ if step == 0:+ grads = {+ n: p.grad.detach().clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ norms.append(float(pp.clip_grad_norm_(1.0)))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(loss))+ return {+ "pp": {"losses": losses, "norms": norms, "grads": grads},+ "ref": _reference(ref, ids, labels),+ "bounds": pp.bounds,+ }+++def _import_checkout_unsloth():+ """`import unsloth` from this checkout (conftest's device spoof first); None, or why it failed."""+ for name in [n for n in sys.modules if n == "unsloth" or n.startswith("unsloth.")]:+ del sys.modules[name]+ sys.path.insert(0, REPO)+ try:+ spec = importlib.util.spec_from_file_location(+ "_pg_conftest", os.path.join(REPO, "tests", "conftest.py")+ )+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+ import unsloth # noqa: F401+ except Exception as e: # a CPU host that refuses `import unsloth`+ return repr(e)+ return None+++def _no_host_sync():+ """Patches making any device -> host read of a tensor (float(), .item(), bool()) raise."""+ import contextlib++ @contextlib.contextmanager+ def guard():+ saved = {k: getattr(torch.Tensor, k) for k in ("__float__", "item", "__bool__")}++ def refuse(self, *args, **kwargs):+ raise AssertionError("host sync")++ try:+ for k in saved:+ setattr(torch.Tensor, k, refuse)+ yield+ finally:+ for k, v in saved.items():+ setattr(torch.Tensor, k, v)++ return guard()+++def _case_clip(rank, world, tmp):+ """tp_clip_grad_norm_ / _clip_over_world: no host sync, a tensor norm, grads clipped to max_norm."""+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]+ model = build("llama")+ tpn.apply_tensor_parallel(model, mesh)+ ids, labels = batch(model, bsz = 2)+ params = [p for p in model.parameters() if p.requires_grad]+ out = {}+ for name, clip in (+ ("tp_clip_grad_norm_", lambda: tpn.tp_clip_grad_norm_(params, 1e-3)),+ (+ "_clip_over_world",+ lambda: tpn._clip_over_world([p for p in params if p.grad is not None], 1e-3, 1e-6),+ ),+ ("no_clip", lambda: tpn.tp_clip_grad_norm_(params, 1e9)),+ ):+ model.zero_grad(set_to_none = True)+ model(input_ids = ids, labels = labels, use_cache = False).loss.backward()+ before = {+ n: tpn._full(p.grad, p).clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ try:+ with _no_host_sync():+ norm = clip()+ error = None+ except AssertionError as e:+ norm, error = None, str(e)+ after = {+ n: tpn._full(p.grad, p).clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ total = float(torch.stack([g.double().square().sum() for g in after.values()]).sum().sqrt())+ out[name] = {+ "error": error,+ "is_tensor": isinstance(norm, torch.Tensor),+ "norm": None if norm is None else float(norm),+ "before": float(+ torch.stack([g.double().square().sum() for g in before.values()]).sum().sqrt()+ ),+ "after": total,+ "unchanged": all(torch.equal(before[k], after[k]) for k in before),+ }+ return out+++EXPORTS = (+ "unsloth_save_model",+ "unsloth_save_model:lora",+ "unsloth_generic_save_pretrained_merged",+ "unsloth_save_pretrained_merged",+)+++def _case_export(rank, world, tmp):+ """Merged / LoRA exports refuse a TP or PP model on every rank before writing anything."""+ error = _import_checkout_unsloth()+ if error is not None:+ return {"skip": error}+ from torch.distributed.device_mesh import init_device_mesh+ from unsloth import save+ from unsloth.models import _utils, pp_native, tp_native++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]+ guard = getattr(_utils, "_refuse_model_parallel_export", None)+ if guard is not None:+ guard(build("llama"), "save_pretrained_merged") # not parallel: passes through+ tp_model = build("llama")+ tp_native.apply_tensor_parallel(tp_model, mesh)+ pp_model = build("llama")+ pp_native.apply_pipeline_parallel(pp_model, mesh, n_microbatches = 2)+ calls = {+ "unsloth_save_model": lambda m, d: save.unsloth_save_model(+ m, None, d, save_method = "merged_16bit"+ ),+ "unsloth_save_model:lora": lambda m, d: save.unsloth_save_model(+ m, None, d, save_method = "lora"+ ),+ "unsloth_generic_save_pretrained_merged": lambda m,+ d: save.unsloth_generic_save_pretrained_merged(+ m, d, tokenizer = None, save_method = "merged_16bit"+ ),+ "unsloth_save_pretrained_merged": lambda m, d: save.unsloth_save_pretrained_merged(+ m, d, tokenizer = None+ ),+ }+ out = {}+ for kind, model in (("tp", tp_model), ("pp", pp_model)):+ for name in EXPORTS:+ directory = os.path.join(tmp, f"{kind}_{name.replace(':', '_')}_{rank}")+ try:+ calls[name](model, directory)+ error = None+ except Exception as e:+ error = f"{type(e).__name__}: {e}"+ wrote = os.path.exists(directory) and bool(os.listdir(directory))+ out[f"{kind}:{name}"] = {"error": error, "wrote": wrote}+ return out+++CASES = {"tp": _case_tp, "pp": _case_pp, "clip": _case_clip, "export": _case_export}+++def _worker(rank, world, port, case, tmp):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ torch.set_num_threads(2)+ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ torch.save(CASES[case](rank, world, tmp), os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ dist.destroy_process_group()+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _spawn(case, world = 2):+ ctx = multiprocessing.get_context("spawn")+ port = _free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, case, tmp)) for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(600)+ codes = [p.exitcode for p in procs]+ assert codes == [0] * world, f"rank exit codes {codes}"+ return [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+++_SPAWNED = {}+++def _spawn_cached(case):+ if case not in _SPAWNED:+ _SPAWNED[case] = _spawn(case)+ return _SPAWNED[case]+++def _close(a, b, rtol):+ return abs(a - b) <= 1e-6 + rtol * abs(b)+++def _check_run(+ run,+ ref,+ name,+ union = None,+):+ # step 0 at 1e-4; step 1 norms at 1e-3 (after an AdamW step, near-zero grads flip update signs)+ for i, (a, b) in enumerate(zip(run["losses"], ref["losses"])):+ assert _close(a, b, 1e-4), (name, "loss", i, a, b)+ for i, (a, b) in enumerate(zip(run["norms"], ref["norms"])):+ assert _close(a, b, 1e-4 if i == 0 else 1e-3), (name, "grad norm", i, a, b)+ if union is None:+ assert set(run["grads"]) == set(ref["grads"]), (+ name,+ sorted(set(run["grads"]) ^ set(ref["grads"]))[:4],+ )+ for key, g in run["grads"].items():+ r = ref["grads"][key]+ atol = max(+ 1e-7, 1e-4 * float(r.abs().max())+ ) # fp32 reduction order: ~1e-6..1e-5 of the max+ torch.testing.assert_close(g, r, rtol = 1e-4, atol = atol, msg = f"{name}: {key}")++[email protected](scope = "module")+def tp_results():+ return _spawn("tp")++[email protected](scope = "module")+def pp_results():+ return _spawn("pp")++[email protected]("name", sorted(TP_PASS))+def test_tp_trains_like_one_process(tp_results, name):+ for res in tp_results:+ assert "crash" not in res[name], res[name]+ _check_run(res[name]["tp"], res[name]["ref"], name)+ assert res[name]["regions"] > 0+ if name == "llama_shared":+ # a block shared by two layers is sharded once (it used to be re-sharded: DTensor of a DTensor)+ assert tp_results[0][name]["regions"] == 6++[email protected]("arch", sorted(TP_REFUSE))+def test_tp_refuses_before_mutation(tp_results, arch):+ for res in tp_results:+ assert res[arch]["error"] and TP_REFUSE[arch] in res[arch]["error"], res[arch]+ assert res[arch]["unchanged"], arch++[email protected]("arch", PP_PASS)+def test_pp_trains_like_one_process(pp_results, arch):+ for res in pp_results:+ assert "crash" not in res[arch], res[arch]+ _check_run(res[arch]["pp"], res[arch]["ref"], arch, union = True)+ keys = set(pp_results[0][arch]["pp"]["grads"]) | set(pp_results[1][arch]["pp"]["grads"])+ assert keys == set(pp_results[0][arch]["ref"]["grads"])+ assert not set(pp_results[0][arch]["pp"]["grads"]) & set(pp_results[1][arch]["pp"]["grads"])++[email protected]("name", sorted(PP_REFUSE))+def test_pp_refuses_before_mutation(pp_results, name):+ for res in pp_results:+ assert res[name]["error"] and PP_REFUSE[name][2] in res[name]["error"], res[name]+ assert res[name]["unchanged"], name+++def test_tp_head_norm_names_need_one_head():+ # q_layernorm over all heads (not one) is still refused: a norm across heads needs the full projection+ model = build("hunyuan_v1_dense", lora = False)+ attn = model.model.layers[0].self_attn+ attn.query_layernorm = type(attn.query_layernorm)(64)+ with pytest.raises(ValueError, match = "normalizes 64 features, not one head"):+ tpn.derive_tp_plan(model, 2)++[email protected]("name", ["tp_clip_grad_norm_", "_clip_over_world", "no_clip"])+def test_tp_clip_has_no_host_sync(name):+ for res in _spawn_cached("clip"):+ r = res[name]+ assert r["error"] is None and r["is_tensor"], r+ assert r["norm"] == pytest.approx(r["before"], rel = 1e-5), r+ if name == "no_clip":+ assert r[+ "unchanged"+ ], r # a coefficient clamped at 1 leaves the grads bitwise unchanged+ else:+ assert r["after"] == pytest.approx(1e-3, rel = 1e-4), r+++def test_tp_clip_coefficient_is_not_narrowed_to_fp16():+ # A coefficient of ~1.7e-8 rounds to 0 in fp16; the clipped grads (~1e-3) are representable.+ p = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ p.grad = torch.full((4,), 60000.0, dtype = torch.float16)+ tpn._scale_grads_([p], torch.tensor(6e7), 1.0, 1e-6)+ assert torch.all(p.grad > 0), p.grad+ assert p.grad.float() == pytest.approx(torch.full((4,), 1e-3), rel = 1e-2), p.grad++[email protected]("kind", ["tp", "pp"])[email protected]("name", EXPORTS)+def test_exports_refuse_model_parallel_before_writing(kind, name):+ results = _spawn_cached("export")+ if "skip" in results[0]:+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")+ want = "tensor parallel model" if kind == "tp" else "pipeline parallel model"+ for res in results:+ r = res[f"{kind}:{name}"]+ assert r["error"] and "RuntimeError" in r["error"] and want in r["error"], r+ assert not r["wrote"], r+++def test_every_whole_model_export_is_guarded():+ """Static: every export patch_saving_functions binds, and every writer behind them, refuses TP / PP first."""+ import ast++ tree = ast.parse(open(os.path.join(REPO, "unsloth", "save.py")).read())+ functions = {n.name: n for n in tree.body if isinstance(n, ast.FunctionDef)}+ bound = set()+ for node in ast.walk(functions["patch_saving_functions"]):+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "_bind_saving_method":+ bound.add(node.args[0].id)+ adapter_or_tokenizer = {+ "unsloth_model_save_pretrained",+ "unsloth_push_to_hub",+ "unsloth_tokenizer_save_pretrained",+ }+ writers = {+ "unsloth_save_model",+ "unsloth_generic_save",+ "unsloth_save_pretrained_merged",+ "unsloth_push_to_hub_merged",+ "save_to_gguf_generic",+ "_unsloth_save_lora_gguf",+ "save_lora_to_custom_dir",+ "_unsloth_save_compressed_tensors",+ "_unsloth_save_torchao",+ "_unsloth_save_openvino",+ }+ targets = (bound - adapter_or_tokenizer) | writers+ assert len(targets) >= 18, sorted(targets)+ unguarded = []+ for name in sorted(targets):+ decorators = functions[+ name+ ].decorator_list # bound inside patch_saving_functions: defined at module level+ first = decorators[0] if decorators else None+ if not (+ isinstance(first, ast.Call)+ and getattr(first.func, "id", None) == "_model_parallel_refuses_save"+ ):+ unguarded.append(name)+ assert not unguarded, unguardeddiff --git a/tests/test_pp_native_gloo.py b/tests/test_pp_native_gloo.pyindex 1251b9b12ca..4c6a4838a5a 100644--- a/tests/test_pp_native_gloo.py+++ b/tests/test_pp_native_gloo.py@@ -190,6 +190,14 @@ def _read_adapter(path):return load_file(os.path.join(path, "adapter_model.safetensors"))+def _enable_reentrant_gc(model):+ """Reentrant checkpointing only back-propagates into a segment whose input requires grad; with the+ embeddings frozen (LoRA) that needs enable_input_require_grads, which transformers 4.x+ gradient_checkpointing_enable does not do for a PEFT model (5.x does)."""+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})+ model.enable_input_require_grads()++def _case_parity(rank,world,@@ -211,13 +219,13 @@ def _case_parity(ref = build(repo, tie = tie, **extra)ids, labels = batch(ref)if gc:- ref.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})+ _enable_reentrant_gc(ref)ref_res = _reference(ref, ids, labels, steps)if rank == 0:ref.save_pretrained(os.path.join(tmp, "ref"))model = build(repo, tie = tie, **extra)if gc:- model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})+ _enable_reentrant_gc(model)mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))originals = {"embed": weakref.ref(model.get_input_embeddings()),diff --git a/tests/test_save_keep_fp32_modules.py b/tests/test_save_keep_fp32_modules.pyindex 23ac8458a54..0274b683851 100644--- a/tests/test_save_keep_fp32_modules.py+++ b/tests/test_save_keep_fp32_modules.py@@ -125,5 +125,10 @@ def test_generic_save_casts_through_the_helper():c.func.id for c in ast.walk(fn) if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)}assert "_cast_state_dict_16bit" in calls- decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in fn.decorator_list]- assert decorators == ["_normalize_tied_weights_keys_for_save", "inference_mode"]+ decorators = [d.func if isinstance(d, ast.Call) else d for d in fn.decorator_list]+ decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in decorators]+ assert decorators == [+ "_model_parallel_refuses_save",+ "_normalize_tied_weights_keys_for_save",+ "inference_mode",+ ]diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex ad8e6191022..128de54b4d5 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -6216,6 +6216,27 @@ def _tensor_parallel_trainer_pre(model):return tp_trainer_setup(model, phase = "pre")+def _refuse_model_parallel_export(model, method):+ """Merged / GGUF / torchao / OpenVINO / LoRA exports read every weight of the model in this process. Under+ apply_tensor_parallel each rank holds 1 / tp of the attention / MLP weights; under apply_pipeline_parallel+ the layers, embeddings and lm_head of the other stages are parameter-free placeholders. Raise on every rank+ before anything is written (no rank waits on another)."""+ if _is_unsloth_tensor_parallel(model):+ raise RuntimeError(+ f"Unsloth: `{method}` does not support a tensor parallel model (apply_tensor_parallel): each rank "+ "holds 1 / tp of the attention / MLP weights, so the export would be shard sized. Save the adapter "+ "with `save_tp_adapter(model, directory)` (on every tp rank), then in a single process load the "+ f"base model with that adapter and call `{method}` there."+ )+ if _is_unsloth_pipeline_parallel(model):+ raise RuntimeError(+ f"Unsloth: `{method}` does not support a pipeline parallel model (apply_pipeline_parallel): each "+ "rank holds one stage, and the other stages' layers, embeddings and lm_head are placeholders. Save "+ "the adapter with `pp.save_adapter(directory)` (on every pp rank), then in a single process load "+ f"the base model with that adapter and call `{method}` there."+ )++def _refuse_tensor_parallel_for_inference(model):if _is_unsloth_tensor_parallel(model):raise RuntimeError(diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.pyindex f7d295b7a4d..cfef8809c30 100644--- a/unsloth/models/pp_native.py+++ b/unsloth/models/pp_native.py@@ -45,6 +45,7 @@ import collectionsimport dataclassesimport inspectimport math+import reimport sysimport textwrapimport types@@ -301,6 +302,8 @@ def _find_parts(model) -> _Parts:)text = _text_config(model)n = getattr(text, "num_hidden_layers", None)+ # A decoder-only use of an encoder-decoder config (BartForCausalLM): num_hidden_layers is the encoder's.+ counts = {n, getattr(text, "decoder_layers", None)} - {None}pp_plan = getattr(text, "base_model_pp_plan", None) or getattr(getattr(inner, "config", None), "base_model_pp_plan", None)@@ -313,7 +316,11 @@ def _find_parts(model) -> _Parts:layers_name, source = lists[0], "transformers"if layers_name is None:for name, child in named.items():- if isinstance(child, nn.ModuleList) and len(child) and (n is None or len(child) == n):+ if (+ isinstance(child, nn.ModuleList)+ and len(child)+ and (n is None or len(child) in counts)+ ):layers_name = namebreakif layers_name is None:@@ -344,11 +351,25 @@ def _find_parts(model) -> _Parts:continuenorms.append((name, child))hidden = getattr(text, "hidden_size", None)+ vocab = getattr(text, "vocab_size", None)+ order = list(decoder._modules)others = []for name, child in norms:base = getattr(child, "original_module", child) # PEFT modules_to_save wraps itnorm_like = "norm" in name.lower() or "Norm" in type(base).__name__too_big = bool(hidden) and sum(p.numel() for p in base.parameters()) > 4 * hidden+ # A learned position / token-type table before the layers (GPT-2 wpe, OPT / BART / Whisper+ # embed_positions): indexed by positions, not tokens, so later stages (whose pre-layer result is+ # replaced by the received hidden states anyway) need nothing from it. A token-indexed table could+ # feed the layers a side input and stays refused.+ positional = (+ isinstance(base, nn.Embedding)+ and isinstance(vocab, int)+ and base.num_embeddings != vocab+ and order.index(name) < order.index(layers_name)+ )+ if positional:+ continueif not norm_like or too_big:others.append(name)if others:@@ -361,6 +382,12 @@ def _find_parts(model) -> _Parts:continueif _owner(child, lm_head)[0] is not None:continue+ if _owner(child, decoder)[0] is not None:+ # A wrapper around the decoder (OPTModel.decoder, BartDecoderWrapper.decoder): fine when it holds+ # nothing else.+ inside = {id(p) for p in decoder.parameters()}+ if all(id(p) in inside for p in child.parameters()):+ continueraise ValueError(f"Unsloth: pipeline parallelism v1 found an extra module {name} ({type(child).__name__}) "f"next to the decoder of {type(inner).__name__} (multimodal towers / extra heads are v2)."@@ -371,6 +398,44 @@ def _find_parts(model) -> _Parts:return _Parts(inner, decoder, layers, layers_name, embed, lm_head, norms, source, tied)+def _refuse_shared_across_stages(parts, bounds):+ """A trainable parameter used by layers of two stages (a block shared by several layers, Zamba2 /+ ALBERT style) would be trained as two independent copies."""+ stage_of = {i: k for k, (start, end) in enumerate(bounds) for i in range(start, end)}+ owners = {}+ for i, layer in enumerate(parts.layers):+ for name, p in layer.named_parameters():+ stages, first, param = owners.setdefault(+ id(p), (set(), f"{parts.layers_name}.{i}.{name}", p)+ )+ stages.add(stage_of[i])+ shared = [first for stages, first, p in owners.values() if len(stages) > 1 and p.requires_grad]+ if shared:+ raise ValueError(+ f"Unsloth: pipeline parallelism v1 cannot split layers that share trainable parameters across "+ f"stages ({shared[:3]}: a block reused by several layers). Keep the sharing layers on one stage "+ "(layers_per_stage, when they are adjacent), or use native FSDP2 (UNSLOTH_FSDP2=1)."+ )+++def _check_loss_normalization(inner):+ """PipelineParallel sums the micro-batch losses, each normalised by the token count of the whole batch+ (num_items_in_batch). A causal LM forward that drops num_items_in_batch takes a per-micro-batch mean, so the+ sum would be n_microbatches times the one-process loss (BART / Whisper / TrOCR decoders)."""+ try:+ source = inspect.getsource(_forward_function(inner))+ except (OSError, TypeError):+ source = ""+ passes_kwargs = re.search(r"loss_function\([^()]*(?:\([^()]*\)[^()]*)*\*\*\s*\w+", source)+ if "num_items_in_batch" in source or passes_kwargs:+ return+ raise ValueError(+ f"Unsloth: pipeline parallelism v1 needs the {type(inner).__name__} loss to take num_items_in_batch "+ "(self.loss_function(..., **kwargs)); this forward computes a per-micro-batch mean, so micro-batches "+ "cannot be summed like gradient accumulation."+ )++def _refuse_unsupported(model, parts):text = _text_config(model)inner = parts.inner@@ -446,6 +511,7 @@ def _refuse_unsupported(model, parts):"Unsloth: pipeline parallelism v1 cannot train tied input / output embeddings (the first and ""last stage would each update their own copy). Freeze them, or untie them before splitting.")+ _check_loss_normalization(inner)# ---------------------------------------------------------------------------------- layer call format@@ -1373,6 +1439,7 @@ def apply_pipeline_parallel(_check_schedule(model, schedule)tuple_format = _layer_call_format(parts.decoder, parts.layers_name) == "tuple"bounds = derive_pp_split(model, pp, config)+ _refuse_shared_across_stages(parts, bounds)dtype = config.activation_dtype or parts.embed.weight.dtyperank = pp_mesh.get_local_rank()if config.sync_trainable and pp > 1:diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.pyindex bb66b618ea1..619d22690e3 100644--- a/unsloth/models/tp_native.py+++ b/unsloth/models/tp_native.py@@ -74,6 +74,7 @@ _TP_FLAG = "_unsloth_tp"COLWISE = "colwise"ROWWISE = "rowwise"HEAD_NORM = "replicated_with_grad_allreduce"+ELEMENTWISE = "elementwise_replicated" # scalar-parameter activations inside an MLP# transformers TP styles (4.5x and 5.x names) -> what this module does with them._STYLES = {@@ -115,15 +116,36 @@ _MLP_NAMES = ("mlp", "feed_forward", "ffn")_Q, _K, _V = ("q_proj", "query"), ("k_proj", "key"), ("v_proj", "value")_FUSED_QKV = ("qkv_proj",) # [q, k, v] along the output dim (Phi3)_ATTN_ROW = ("o_proj", "out_proj", "dense")-_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc")+_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc", "fc_in", "dense_h_to_4h")_MLP_PACKED = ("gate_up_proj",) # [gate, up] along the output dim-_MLP_ROW = ("down_proj", "w2", "fc2")-_HEAD_NORMS = ("q_norm", "k_norm", "query_norm", "key_norm")+_MLP_ROW = ("down_proj", "w2", "fc2", "fc_out", "dense_4h_to_h", "c_proj")+_HEAD_NORMS = (+ "q_norm",+ "k_norm",+ "query_norm",+ "key_norm",+ "q_layernorm",+ "k_layernorm",+ "query_layernorm",+ "key_layernorm",+)_MLA_NAMES = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "kv_a_layernorm")_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")# An attention forward reading these sizes off the config would reshape to the global head count._GLOBAL_HEAD_READS = re.compile(r"config\.(num_attention_heads|num_key_value_heads)\b")-_GLOBAL_WIDTH_READS = re.compile(r"(config|self)\.hidden_size\b")+_GLOBAL_WIDTH_READS = re.compile(+ r"(config|self)\.(hidden_size|embed_dim|all_head_size|n_embd|d_model|split_size)\b"+)+# An MLP forward reducing over its features (activation sparsity statistics, a norm, a softmax) would see+# only this rank's 1 / tp of the intermediate dimension; one reading the intermediate size splits wrongly.+_FEATURE_REDUCTIONS = re.compile(+ r"(?:\btorch\.|\bF\.|\bfunctional\.|\.)(mean|std|var|var_mean|std_mean|sum|nansum|prod|amax|amin|"+ r"argmax|argmin|topk|kthvalue|sort|argsort|softmax|log_softmax|logsumexp|cumsum|cumprod|norm|normalize|"+ r"layer_norm|rms_norm|group_norm)\s*\("+)+_GLOBAL_INTERMEDIATE_READS = re.compile(r"(config|self)\.(intermediate_size|ffn_dim|n_inner)\b")+# A fused qkv_proj is [q, k, v] blocks only when the forward slices it by the (local) head counts (Phi3).+_LOCAL_QKV_SLICE = re.compile(r"(num_key_value_heads|num_kv_heads)\s*\*\s*self\.head_dim")@dataclasses.dataclass@@ -226,13 +248,19 @@ def _decoder_layers(model):decoder = innertext = _text_config(model)n = getattr(text, "num_hidden_layers", None)+ # A decoder-only use of an encoder-decoder config (BartForCausalLM): num_hidden_layers is the encoder's.+ counts = {n, getattr(text, "decoder_layers", None)} - {None}prefix = ""for name, module in inner.named_modules():if module is decoder:prefix = namebreakfor name, module in decoder.named_modules():- if isinstance(module, nn.ModuleList) and len(module) and (n is None or len(module) == n):+ if (+ isinstance(module, nn.ModuleList)+ and len(module)+ and (n is None or len(module) in counts)+ ):path = ".".join(x for x in (prefix, name) if x)return path, moduleraise ValueError(@@ -336,9 +364,25 @@ def _classify(region_path, name, child, kind, compiled):return COLWISE, "packed"if name in _MLP_ROW:return ROWWISE, None+ if _elementwise_scalars(child):+ return ELEMENTWISE, Nonereturn None, None+def _elementwise_scalars(module):+ """A parameterised activation with scalar parameters only (Apertus xIELU): elementwise on any shard of+ the intermediate dimension, so it stays replicated (gradients summed over tp)."""+ if isinstance(module, nn.Linear) or "lora_A" in module._modules or list(module.children()):+ return False+ if any(p.numel() != 1 for p in module.parameters()):+ return False+ try:+ source = inspect.getsource(type(module))+ except (OSError, TypeError):+ return False+ return not _FEATURE_REDUCTIONS.search(_drop_init(source))++def _attention_sizes(module, projs, text):"""(heads, kv_heads, head_dim) of one attention module, from its own projections."""heads_cfg = getattr(text, "num_attention_heads", None)@@ -393,7 +437,7 @@ def derive_tp_plan(if style not in _STYLES and style not in _REPLICATED_STYLES:raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")text = _text_config(model)- regions, replicated = [], 0+ regions, replicated, seen = [], 0, {}for index, layer in enumerate(layers):found = 0for child_name, child in layer.named_children():@@ -405,8 +449,11 @@ def derive_tp_plan(if kind is None or not _has_params(child):continuepath = f"{layers_path}.{index}.{child_name}"- regions.append(_region(path, child, kind, compiled, text, tp_size))found += 1+ if id(child) in seen:+ continue # a block shared by several layers (Zamba2, ALBERT style): sharded once+ seen[id(child)] = path+ regions.append(_region(path, child, kind, compiled, text, tp_size))# A pipeline parallel pass-through (a layer another stage owns) is not a replicated layer.replicated += found == 0 and "_pp_index" not in layer.__dict__if not regions:@@ -439,7 +486,7 @@ def _region(path, module, kind, compiled, text, tp):"every module with parameters inside a sharded attention / MLP must be in the plan. Pass ""plan = {...} to apply_tensor_parallel.")- if style == HEAD_NORM:+ if style in (HEAD_NORM, ELEMENTWISE):norms.append((name, child))continueweight, _ = _weight(child)@@ -468,15 +515,20 @@ def _region(path, module, kind, compiled, text, tp):)_check_divisible("num_key_value_heads", kv_heads, tp, path)for name, norm in norms:- numel = sum(p.numel() for p in norm.parameters())+ weight = norm._parameters.get("weight")+ numel = (+ weight.numel() if weight is not None else sum(p.numel() for p in norm.parameters())+ )if numel != head_dim:raise ValueError(f"Unsloth: '{path}.{name}' normalizes {numel} features, not one head ({head_dim}): a norm ""across all heads needs the full projection, which tensor parallelism v1 does not gather.")_check_attention_source(module, path)- elif norms:- raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")+ else:+ if any(not _elementwise_scalars(norm) for _, norm in norms):+ raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")+ _check_mlp_source(module, path)for proj in projs:weight, base = _weight(proj.module)out_features, in_features = _logical_shape(weight, base)@@ -485,6 +537,7 @@ def _region(path, module, kind, compiled, text, tp):proj.parts = (out_features,)continueif proj.parts == "qkv":+ _check_fused_qkv_source(module, path, proj.name)q, kv = region.heads * region.head_dim, region.kv_heads * region.head_dimif q + 2 * kv != out_features:raise ValueError(@@ -527,21 +580,103 @@ def _forward_source(module):return inspect.getsource(forward)-def _check_attention_source(module, path):+def _drop_init(source):+ """A class source without its __init__ (sizes read there are construction, not forward math)."""+ return re.sub(r"\n(\s+)def __init__\(.*?(?=\n\1def |\Z)", "\n", source, flags = re.S)+++def _region_source(module, path, what):+ """The forward of a region module plus the methods of its own class that forward calls (self._helper)."""try:source = _forward_source(module)except (OSError, TypeError):raise ValueError(f"Unsloth: tensor parallelism cannot read the forward of '{path}' ({type(module).__name__}) to "- "check it uses this rank's head count."+ f"check {what}."+ )+ seen = set()+ for name in re.findall(r"\bself\.(\w+)\s*\(", source):+ if name in seen or name in module._modules or name == "forward":+ continue+ seen.add(name)+ method = getattr(type(module), name, None)+ if not inspect.isfunction(method):+ continue+ try:+ source += "\n" + inspect.getsource(inspect.unwrap(method))+ except (OSError, TypeError):+ pass+ return source+++def _input_width_names(source, module):+ """Variables a forward reads the full input width into (`bsz, len, dim = hidden_states.size()`)."""+ names = set()+ for x in {_input_name(module), "hidden_states"}:+ x = re.escape(x)+ for m in re.finditer(+ r"\b\w+\s*,\s*\w+\s*,\s*(\w+)\s*=\s*" + x + r"\.(?:size\(\)|shape\b)", source+ ):+ names.add(m.group(1))+ for m in re.finditer(+ r"\b(\w+)\s*=\s*" + x + r"\.(?:shape\[(?:-1|2)\]|size\((?:-1|2)\))", source+ ):+ names.add(m.group(1))+ names.discard("_")+ return sorted(n for n in names if len(re.findall(r"\b" + re.escape(n) + r"\b", source)) > 1)+++def _check_attention_source(module, path):+ source = _region_source(module, path, "it uses this rank's head count")+ if "_unsloth_tp_heads" in source:+ return # an Unsloth forward written for tensor parallelism+ read = _GLOBAL_WIDTH_READS.search(source)+ if read:+ raise ValueError(+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with "+ f"{read.group(0)}, the global width; tensor parallelism v1 needs it to follow the local heads."+ )+ width = _input_width_names(source, module)+ if width:+ raise ValueError(+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) keeps the input width "+ f"({', '.join(width)}) for its output shape; tensor parallelism v1 needs it to follow the local heads."+ )+++def _check_mlp_source(module, path):+ source = _region_source(module, path, "it is elementwise over the intermediate features")+ if "_unsloth_tp" in source:+ return+ reduction = _FEATURE_REDUCTIONS.search(source)+ if reduction:+ raise ValueError(+ f"Unsloth: the MLP forward of '{path}' ({type(module).__name__}) reduces over its features "+ f"({reduction.group(0).strip()}...); with tensor parallelism each rank holds 1 / tp of them, so v1 "+ "cannot shard it.")- if _GLOBAL_WIDTH_READS.search(source) and "_unsloth_tp_heads" not in source:+ read = _GLOBAL_INTERMEDIATE_READS.search(source)+ if read:raise ValueError(- f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with hidden_size, "- "the global width; tensor parallelism v1 needs it to follow the local heads."+ f"Unsloth: the MLP forward of '{path}' ({type(module).__name__}) reads {read.group(0)}, the global "+ "size; tensor parallelism v1 needs it to follow the local shard.")+def _check_fused_qkv_source(module, path, name):+ try:+ source = _forward_source(module)+ except (OSError, TypeError):+ source = ""+ if "_unsloth_tp_heads" in source or _LOCAL_QKV_SLICE.search(source):+ return+ raise ValueError(+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) does not slice the fused "+ f"'{name}' by the key / value head count ([q, k, v] blocks); its layout is unknown to tensor "+ "parallelism v1."+ )++def shard_slice(tensor, dim, parts, rank, tp):"""This rank's slice of `tensor` along `dim`: the rank-th 1/tp of every part, concatenated (a copy)."""pieces, offset = [], 0@@ -1144,6 +1279,11 @@ def apply_tensor_parallel(_refuse_base(path, proj.module, weight, base)_refuse_adapter(path, proj.module)_check_preloaded(path, proj, weight, base, tp_mesh)+ if id(weight) in covered:+ raise ValueError(+ f"Unsloth: '{path}' shares its weight with another sharded projection; tensor "+ "parallelism v1 shards every projection once."+ )covered.update((id(weight), id(base.bias)))if _is_params4bit(weight):_check_params4bit(path, weight, proj.style, tp)@@ -1413,13 +1553,26 @@ def tp_clip_grad_norm_(dist.all_reduce(shard_sq, group = group)total = total + shard_sqtotal = total.sqrt()- coef = float(max_norm) / (float(total) + eps)- if coef < 1:- for p in params:- p.grad.mul_(coef)+ _scale_grads_(params, total, max_norm, eps)return total+def _scale_grads_(params, total, max_norm, eps):+ """torch's clip: a tensor coefficient clamped at 1 and applied unconditionally (no host sync, no+ data-dependent branch). DTensor grads are scaled through their local shard."""+ with torch.no_grad():+ coef = torch.clamp(max_norm / (total + eps), max = 1.0)+ for p in params:+ g = p.grad+ local = g.to_local() if _is_dtensor(g) else g+ c = coef.to(device = local.device, dtype = torch.float32)+ if local.dtype in (torch.float16, torch.bfloat16):+ # A small coefficient rounds to 0 in fp16 (a device tensor is cast to the grad dtype).+ local.copy_(local.float().mul_(c))+ else:+ local.mul_(c.to(local.dtype))++def _clip_over_world(params, max_norm, eps):"""tp_clip_grad_norm_ for gradients on a 2-D (dp, tp) mesh (shard_model over the dp mesh): everyelement's square is counted once, divided by the number of ranks holding a replica of it."""@@ -1451,10 +1604,7 @@ def _clip_over_world(params, max_norm, eps):total = sq if total is None else total + sqdist.all_reduce(total)total = total.sqrt()- coef = float(max_norm) / (float(total) + eps)- if coef < 1:- for p in params:- p.grad.mul_(coef)+ _scale_grads_(params, total, max_norm, eps)return totaldiff --git a/unsloth/save.py b/unsloth/save.pyindex d4e25cacd3d..b123646b075 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -70,6 +70,7 @@ from .models.loader_utils import (_tokenizer_wants_local_only,)from .models._utils import (+ _refuse_model_parallel_export,_convert_torchao_model,_merged_torchao_config_for_peft_qat,_peft_untied_lm_head,@@ -1284,6 +1285,23 @@ def _restore_tied_weights_keys(originals):pass+def _model_parallel_refuses_save(method):+ """Every whole-model export refuses an apply_tensor_parallel / apply_pipeline_parallel model before it+ writes anything (see _refuse_model_parallel_export)."""++ def decorate(fn):+ @functools.wraps(fn)+ def wrapper(*args, **kwargs):+ model = args[0] if args else kwargs.get("model", kwargs.get("self"))+ if model is not None:+ _refuse_model_parallel_export(model, method)+ return fn(*args, **kwargs)++ return wrapper++ return decorate++def _normalize_tied_weights_keys_for_save(save_fn):"""Coerce legacy list-form ``_tied_weights_keys`` to dict for the duration of a save, then restore: transformers >= 5 re-ties from the dict's VALUES, so a persisted ``{k: k}`` self-map would no-op a later resize/re-tie. ``model`` is the first positional arg (bound-method self) or the ``model=`` keyword."""@@ -1303,6 +1321,7 @@ def _normalize_tied_weights_keys_for_save(save_fn):return wrapper+@_model_parallel_refuses_save("unsloth_save_model")@_normalize_tied_weights_keys_for_save@torch.inference_modedef unsloth_save_model(@@ -2738,6 +2757,7 @@ def save_to_gguf(return all_saved_locations, want_full_precision, is_vlm+@_model_parallel_refuses_save("save_pretrained_merged")def unsloth_save_pretrained_merged(self,save_directory: Union[str, os.PathLike],@@ -2877,6 +2897,7 @@ def unsloth_save_pretrained_merged(gc.collect()+@_model_parallel_refuses_save("push_to_hub_merged")def unsloth_push_to_hub_merged(self,repo_id: str,@@ -4484,6 +4505,7 @@ def _fsdp2_refuses_save(method):return decorate+@_model_parallel_refuses_save("save_pretrained_gguf")@_fsdp2_refuses_save("save_pretrained_gguf")@_normalize_tied_weights_keys_for_savedef unsloth_save_pretrained_gguf(@@ -5158,6 +5180,7 @@ def _free_merge_if_disk_is_tight(return freed+@_model_parallel_refuses_save("push_to_hub_gguf")@_fsdp2_refuses_save("push_to_hub_gguf")def unsloth_push_to_hub_gguf(self,@@ -5439,6 +5462,7 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gireturn full_repo_id+@_model_parallel_refuses_save("save_lora_to_custom_dir")def save_lora_to_custom_dir(model, tokenizer, save_directory):os.makedirs(save_directory, exist_ok = True)@@ -5631,6 +5655,7 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):)+@_model_parallel_refuses_save("save_pretrained_gguf")def _unsloth_save_lora_gguf(model,tokenizer,@@ -5797,6 +5822,7 @@ def _unsloth_save_lora_gguf(shutil.rmtree(base_config_dir, ignore_errors = True)+@_model_parallel_refuses_save("push_to_hub_ggml")def unsloth_convert_lora_to_ggml_and_push_to_hub(self,tokenizer,@@ -5827,6 +5853,7 @@ def unsloth_convert_lora_to_ggml_and_push_to_hub()+@_model_parallel_refuses_save("save_pretrained_ggml")def unsloth_convert_lora_to_ggml_and_save_locally(self,save_directory: str,@@ -6025,6 +6052,7 @@ def _prewarm_base_model_hub_cache()+@_model_parallel_refuses_save("save_pretrained_gguf")@torch.inference_modedef save_to_gguf_generic(model,@@ -6281,6 +6309,7 @@ def _unmerge_flags_on_error(streamed):raise+@_model_parallel_refuses_save("save_pretrained_merged")@_normalize_tied_weights_keys_for_save@torch.inference_modedef unsloth_generic_save(@@ -6695,6 +6724,7 @@ def _fsdp2_merged_gather_options(model, args, kwargs):return merged_save_gather_options(model, _merged_save_method(args, kwargs))+@_model_parallel_refuses_save("save_pretrained_merged")@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel)and _is_adapter_save_method(_merged_save_method(args, kwargs)),@@ -6863,6 +6893,7 @@ def unsloth_generic_save_pretrained_merged(# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.+@_model_parallel_refuses_save("push_to_hub_merged")@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel)and _is_adapter_save_method(_merged_save_method(args, kwargs)),@@ -7545,6 +7576,7 @@ def _restore_model_after_quantize_subprocess(model, restore_token) -> None:)+@_model_parallel_refuses_save("save_pretrained_merged")def _unsloth_save_compressed_tensors(model,save_directory: Union[str, os.PathLike],@@ -7814,6 +7846,7 @@ def _unsloth_save_compressed_tensors(clean_gpu_cache()+@_model_parallel_refuses_save("save_pretrained_torchao")def _unsloth_save_torchao(model,save_directory: Union[str, os.PathLike],@@ -8079,6 +8112,7 @@ def _write_executorch_export_config(return path+@_model_parallel_refuses_save("save_pretrained_torchao")@_fsdp2_refuses_save("save_pretrained_torchao")def unsloth_save_pretrained_torchao(self,@@ -8264,6 +8298,7 @@ def _openvino_transformers_mismatch(model_type, task):)+@_model_parallel_refuses_save("save_pretrained_openvino")def _unsloth_save_openvino(model,save_directory: Union[str, os.PathLike],@@ -8406,6 +8441,7 @@ def _unsloth_save_openvino(gc.collect()+@_model_parallel_refuses_save("save_pretrained_openvino")def unsloth_save_pretrained_openvino(self,save_directory: Union[str, os.PathLike],@@ -8440,6 +8476,7 @@ def unsloth_save_pretrained_openvino()+@_model_parallel_refuses_save("push_to_hub_openvino")def unsloth_push_to_hub_openvino(self,repo_id: str,