Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

TP / PP: every causal LM architecture trains like one process or is refused up front (#15230)

Daniel Han4f18eb4326a7
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index 1571ed2756e..6f293df21a5 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -219,6 +219,8 @@ _GENERIC_SAVE_STUBBED_HELPERS = frozenset(
"_stitch_mtp_head",
"_mtp_head_tensor_names",
"_unmerge_flags_on_error",
+ # Decorator factory; _load strips decorators.
+ "_model_parallel_refuses_save",
}
)
diff --git a/tests/test_parallel_general_gloo.py b/tests/test_parallel_general_gloo.py
new file mode 100644
index 00000000000..18f3b6eab69
--- /dev/null
+++ b/tests/test_parallel_general_gloo.py
@@ -0,0 +1,768 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""apply_tensor_parallel / apply_pipeline_parallel across architectures, gloo CPU, 2 ranks, fp32, vs one
+process: every architecture either trains like one process or is refused before the model is touched.
+
+The configs are tiny shrinks of each model type's default config, built offline."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import importlib.machinery # noqa: E402
+import importlib.util # noqa: E402
+import socket # noqa: E402
+import sys # noqa: E402
+import tempfile # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _load(name):
+ """unsloth/models/<name>.py without importing the unsloth package (which needs a GPU)."""
+ full = f"unsloth.models.{name}"
+ if full in sys.modules:
+ return sys.modules[full]
+ pkg = types.ModuleType("unsloth")
+ pkg.__path__ = [os.path.join(REPO, "unsloth")]
+ models = types.ModuleType("unsloth.models")
+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]
+ for package in (pkg, models):
+ package.__spec__ = importlib.machinery.ModuleSpec(package.__name__, None, is_package = True)
+ package.__spec__.submodule_search_locations = package.__path__
+ sys.modules.setdefault("unsloth", pkg)
+ sys.modules.setdefault("unsloth.models", models)
+ spec = importlib.util.spec_from_file_location(
+ full, os.path.join(REPO, "unsloth", "models", f"{name}.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[full] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+tpn = _load("tp_native")
+ppn = _load("pp_native")
+
+_SMALL = dict(vocab_size = 512, hidden_size = 64, intermediate_size = 128, num_hidden_layers = 4)
+_HEADS = dict(num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16)
+# arch -> (model_type, config kwargs)
+CONFIGS = {
+ "llama": ("llama", {**_SMALL, **_HEADS}),
+ # GPT-J: parallel attention + MLP, fc_in / fc_out, partial rotary
+ "gptj": (
+ "gptj",
+ dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, rotary_dim = 8, n_inner = 128),
+ ),
+ # per-head query_layernorm / key_layernorm
+ "hunyuan_v1_dense": (
+ "hunyuan_v1_dense",
+ {
+ **_SMALL,
+ **_HEADS,
+ "use_qk_norm": True,
+ "attention_dropout": 0.0,
+ "tie_word_embeddings": False,
+ },
+ ),
+ # short-conv layers (replicated) between attention layers with q_layernorm / k_layernorm
+ "lfm2": (
+ "lfm2",
+ {
+ **_SMALL,
+ **_HEADS,
+ "layer_types": ["conv", "full_attention", "conv", "full_attention"],
+ "full_attn_idxs": [1, 3],
+ },
+ ),
+ # xIELU activation with scalar parameters inside the MLP
+ "apertus": ("apertus", {**_SMALL, "num_attention_heads": 4, "num_key_value_heads": 2}),
+ # activation sparsity: mean / std over the intermediate features
+ "gemma3n_text": (
+ "gemma3n_text",
+ {
+ **_SMALL,
+ **_HEADS,
+ "intermediate_size": 128,
+ "layer_types": ["sliding_attention"] * 3 + ["full_attention"],
+ "hidden_size_per_layer_input": 16,
+ "num_kv_shared_layers": 0,
+ "activation_sparsity_pattern": [0.95, 0.95, 0.0, 0.0],
+ "laurel_rank": 8,
+ },
+ ),
+ # attention output reshaped with self.embed_dim / the input width
+ "xglm": (
+ "xglm",
+ dict(vocab_size = 512, d_model = 64, ffn_dim = 128, num_layers = 4, attention_heads = 4),
+ ),
+ "trocr": (
+ "trocr",
+ dict(
+ vocab_size = 512,
+ d_model = 64,
+ decoder_layers = 4,
+ decoder_attention_heads = 4,
+ decoder_ffn_dim = 128,
+ ),
+ ),
+ # fused qkv_proj in mp_num groups, not [q, k, v] blocks
+ "codegen": (
+ "codegen",
+ dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, rotary_dim = 8, n_inner = 128),
+ ),
+ # learned position tables before the layers
+ "gpt2": ("gpt2", dict(vocab_size = 512, n_embd = 64, n_layer = 4, n_head = 4, n_inner = 128)),
+ "opt": (
+ "opt",
+ dict(
+ vocab_size = 512,
+ hidden_size = 64,
+ num_hidden_layers = 4,
+ ffn_dim = 128,
+ word_embed_proj_dim = 64,
+ num_attention_heads = 4,
+ ),
+ ),
+ # CrossEntropyLoss mean per call (no num_items_in_batch)
+ "bart": (
+ "bart",
+ dict(
+ vocab_size = 512,
+ d_model = 64,
+ decoder_layers = 4,
+ decoder_ffn_dim = 128,
+ decoder_attention_heads = 4,
+ encoder_layers = 1,
+ encoder_ffn_dim = 128,
+ encoder_attention_heads = 4,
+ ),
+ ),
+}
+
+
+def _zero_dropout(config):
+ for key in list(config.to_dict()):
+ value = getattr(config, key, None)
+ if isinstance(value, float) and ("dropout" in key or "pdrop" in key):
+ setattr(config, key, 0.0)
+
+
+def build(
+ arch,
+ lora = True,
+ seed = 0,
+ share = None,
+):
+ from transformers import AutoModelForCausalLM
+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING
+
+ model_type, kwargs = CONFIGS[arch]
+ if model_type not in CONFIG_MAPPING:
+ pytest.skip(f"transformers has no {model_type}")
+ config = CONFIG_MAPPING[model_type](**kwargs)
+ _zero_dropout(config)
+ config._attn_implementation = "eager"
+ torch.manual_seed(seed)
+ try:
+ model = AutoModelForCausalLM.from_config(config, dtype = torch.float32)
+ except TypeError:
+ model = AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if name.endswith("bias"):
+ p.normal_(0, 0.1)
+ elif "norm" in name and p.ndim == 1:
+ p.add_(torch.randn_like(p) * 0.1)
+ if (
+ share is not None
+ ): # layers[j] is layers[i]: a block reused by several layers (Zamba2 / ALBERT style)
+ i, j = share
+ layers = tpn._decoder_layers(model)[1]
+ layers[j] = layers[i]
+ if not lora:
+ return model.train()
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(seed + 1)
+ kw = dict(
+ r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False, lora_dropout = 0.0
+ )
+ return get_peft_model(model, LoraConfig(**kw)).train()
+
+
+def batch(
+ model,
+ bsz = 4,
+ seq = 8,
+):
+ g = torch.Generator().manual_seed(1)
+ ids = torch.randint(0, 500, (bsz, seq), generator = g)
+ labels = ids.clone()
+ labels[0, :3] = -100
+ labels[-1, -2:] = -100
+ return ids, labels
+
+
+def _clip_fp64(params, max_norm):
+ grads = [p.grad for p in params if p.grad is not None]
+ total = torch.stack([g.double().square().sum() for g in grads]).sum().sqrt()
+ coef = torch.clamp(max_norm / (total + 1e-6), max = 1.0)
+ for g in grads:
+ g.mul_(coef.to(g.dtype))
+ return float(total)
+
+
+def _reference(
+ model,
+ ids,
+ labels,
+ steps = 2,
+):
+ params = [p for p in model.parameters() if p.requires_grad]
+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(steps):
+ out = model(input_ids = ids, labels = labels, use_cache = False)
+ out.loss.backward()
+ if step == 0:
+ grads = {
+ n: p.grad.detach().clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ norms.append(_clip_fp64(params, 1.0))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(out.loss))
+ return {"losses": losses, "norms": norms, "grads": grads}
+
+
+def _fingerprint(model):
+ """Everything a refusal must leave as it was."""
+ return {
+ "params": {
+ n: (type(p).__name__, p.detach().clone(), p.requires_grad)
+ for n, p in model.named_parameters()
+ },
+ "modules": [(n, type(m).__name__) for n, m in model.named_modules()],
+ "marks": sorted(
+ (n, k)
+ for n, m in model.named_modules()
+ for k in m.__dict__
+ if k.startswith("_unsloth") or k in ("forward", "generate")
+ ),
+ "hooks": sum(len(m._forward_hooks) + len(m._forward_pre_hooks) for m in model.modules()),
+ }
+
+
+def _unchanged(before, after):
+ if before["modules"] != after["modules"] or before["marks"] != after["marks"]:
+ return False
+ if before["hooks"] != after["hooks"] or set(before["params"]) != set(after["params"]):
+ return False
+ for n, (cls, value, grad) in before["params"].items():
+ cls2, value2, grad2 = after["params"][n]
+ if cls != cls2 or grad != grad2 or not torch.equal(value, value2):
+ return False
+ return True
+
+
+def _refusal(apply, model):
+ before = _fingerprint(model)
+ try:
+ apply(model)
+ except ValueError as e:
+ return {"error": str(e), "unchanged": _unchanged(before, _fingerprint(model))}
+ except Exception as e:
+ return {
+ "error": f"crash {type(e).__name__}: {e}",
+ "unchanged": _unchanged(before, _fingerprint(model)),
+ }
+ return {"error": None, "unchanged": None}
+
+
+# ---------------------------------------------------------------------------------------------- cases
+
+TP_PASS = {
+ "gptj": None,
+ "hunyuan_v1_dense": None,
+ "lfm2": None,
+ "apertus": None,
+ "llama_shared": (0, 1),
+}
+TP_REFUSE = {
+ "gemma3n_text": "reduces over its features",
+ "xglm": "the global width",
+ "trocr": "keeps the input width",
+ "codegen": "does not slice the fused 'qkv_proj'",
+}
+
+
+def _case_tp(rank, world, tmp):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]
+ out = {}
+ for name, share in TP_PASS.items():
+ try:
+ out[name] = _tp_parity(mesh, name.replace("_shared", ""), share)
+ except (
+ Exception
+ ) as e: # recorded per architecture: one failing case must not hide the others
+ out[name] = {"crash": f"{type(e).__name__}: {e}"}
+ for arch in TP_REFUSE:
+ out[arch] = _refusal(lambda m: tpn.apply_tensor_parallel(m, mesh), build(arch))
+ return out
+
+
+def _tp_parity(mesh, arch, share):
+ ref = build(arch, share = share)
+ ids, labels = batch(ref, bsz = 2)
+ model = build(arch, share = share)
+ tpn.apply_tensor_parallel(model, mesh)
+ params = [p for p in model.parameters() if p.requires_grad]
+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(2):
+ loss = model(input_ids = ids, labels = labels, use_cache = False).loss
+ loss.backward()
+ if step == 0:
+ grads = {
+ n: tpn._full(p.grad, p).detach().clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ norms.append(float(tpn.tp_clip_grad_norm_(params, 1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(loss))
+ return {
+ "tp": {"losses": losses, "norms": norms, "grads": grads},
+ "ref": _reference(ref, ids, labels),
+ "regions": len(model._unsloth_tp_plan.regions),
+ }
+
+
+PP_PASS = ("gpt2", "opt")
+PP_REFUSE = {
+ "bart": ("bart", None, "needs the BartForCausalLM loss to take num_items_in_batch"),
+ "llama_shared": ("llama", (0, 3), "share trainable parameters across stages"),
+}
+
+
+def _case_pp(rank, world, tmp):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]
+ out = {}
+ for arch in PP_PASS:
+ try:
+ out[arch] = _pp_parity(mesh, arch)
+ except Exception as e:
+ out[arch] = {"crash": f"{type(e).__name__}: {e}"}
+ for name, (arch, share, _) in PP_REFUSE.items():
+ model = build(arch, share = share)
+ out[name] = _refusal(
+ lambda m: ppn.apply_pipeline_parallel(m, mesh, n_microbatches = 2), model
+ )
+ return out
+
+
+def _pp_parity(mesh, arch):
+ ref = build(arch)
+ ids, labels = batch(ref)
+ model = build(arch)
+ pp = ppn.apply_pipeline_parallel(model, mesh, n_microbatches = 2)
+ params = pp.parameters()
+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(2):
+ loss = pp.step(ids, labels)
+ if step == 0:
+ grads = {
+ n: p.grad.detach().clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ norms.append(float(pp.clip_grad_norm_(1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(loss))
+ return {
+ "pp": {"losses": losses, "norms": norms, "grads": grads},
+ "ref": _reference(ref, ids, labels),
+ "bounds": pp.bounds,
+ }
+
+
+def _import_checkout_unsloth():
+ """`import unsloth` from this checkout (conftest's device spoof first); None, or why it failed."""
+ for name in [n for n in sys.modules if n == "unsloth" or n.startswith("unsloth.")]:
+ del sys.modules[name]
+ sys.path.insert(0, REPO)
+ try:
+ spec = importlib.util.spec_from_file_location(
+ "_pg_conftest", os.path.join(REPO, "tests", "conftest.py")
+ )
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+ import unsloth # noqa: F401
+ except Exception as e: # a CPU host that refuses `import unsloth`
+ return repr(e)
+ return None
+
+
+def _no_host_sync():
+ """Patches making any device -> host read of a tensor (float(), .item(), bool()) raise."""
+ import contextlib
+
+ @contextlib.contextmanager
+ def guard():
+ saved = {k: getattr(torch.Tensor, k) for k in ("__float__", "item", "__bool__")}
+
+ def refuse(self, *args, **kwargs):
+ raise AssertionError("host sync")
+
+ try:
+ for k in saved:
+ setattr(torch.Tensor, k, refuse)
+ yield
+ finally:
+ for k, v in saved.items():
+ setattr(torch.Tensor, k, v)
+
+ return guard()
+
+
+def _case_clip(rank, world, tmp):
+ """tp_clip_grad_norm_ / _clip_over_world: no host sync, a tensor norm, grads clipped to max_norm."""
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]
+ model = build("llama")
+ tpn.apply_tensor_parallel(model, mesh)
+ ids, labels = batch(model, bsz = 2)
+ params = [p for p in model.parameters() if p.requires_grad]
+ out = {}
+ for name, clip in (
+ ("tp_clip_grad_norm_", lambda: tpn.tp_clip_grad_norm_(params, 1e-3)),
+ (
+ "_clip_over_world",
+ lambda: tpn._clip_over_world([p for p in params if p.grad is not None], 1e-3, 1e-6),
+ ),
+ ("no_clip", lambda: tpn.tp_clip_grad_norm_(params, 1e9)),
+ ):
+ model.zero_grad(set_to_none = True)
+ model(input_ids = ids, labels = labels, use_cache = False).loss.backward()
+ before = {
+ n: tpn._full(p.grad, p).clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ try:
+ with _no_host_sync():
+ norm = clip()
+ error = None
+ except AssertionError as e:
+ norm, error = None, str(e)
+ after = {
+ n: tpn._full(p.grad, p).clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ total = float(torch.stack([g.double().square().sum() for g in after.values()]).sum().sqrt())
+ out[name] = {
+ "error": error,
+ "is_tensor": isinstance(norm, torch.Tensor),
+ "norm": None if norm is None else float(norm),
+ "before": float(
+ torch.stack([g.double().square().sum() for g in before.values()]).sum().sqrt()
+ ),
+ "after": total,
+ "unchanged": all(torch.equal(before[k], after[k]) for k in before),
+ }
+ return out
+
+
+EXPORTS = (
+ "unsloth_save_model",
+ "unsloth_save_model:lora",
+ "unsloth_generic_save_pretrained_merged",
+ "unsloth_save_pretrained_merged",
+)
+
+
+def _case_export(rank, world, tmp):
+ """Merged / LoRA exports refuse a TP or PP model on every rank before writing anything."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ from torch.distributed.device_mesh import init_device_mesh
+ from unsloth import save
+ from unsloth.models import _utils, pp_native, tp_native
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]
+ guard = getattr(_utils, "_refuse_model_parallel_export", None)
+ if guard is not None:
+ guard(build("llama"), "save_pretrained_merged") # not parallel: passes through
+ tp_model = build("llama")
+ tp_native.apply_tensor_parallel(tp_model, mesh)
+ pp_model = build("llama")
+ pp_native.apply_pipeline_parallel(pp_model, mesh, n_microbatches = 2)
+ calls = {
+ "unsloth_save_model": lambda m, d: save.unsloth_save_model(
+ m, None, d, save_method = "merged_16bit"
+ ),
+ "unsloth_save_model:lora": lambda m, d: save.unsloth_save_model(
+ m, None, d, save_method = "lora"
+ ),
+ "unsloth_generic_save_pretrained_merged": lambda m,
+ d: save.unsloth_generic_save_pretrained_merged(
+ m, d, tokenizer = None, save_method = "merged_16bit"
+ ),
+ "unsloth_save_pretrained_merged": lambda m, d: save.unsloth_save_pretrained_merged(
+ m, d, tokenizer = None
+ ),
+ }
+ out = {}
+ for kind, model in (("tp", tp_model), ("pp", pp_model)):
+ for name in EXPORTS:
+ directory = os.path.join(tmp, f"{kind}_{name.replace(':', '_')}_{rank}")
+ try:
+ calls[name](model, directory)
+ error = None
+ except Exception as e:
+ error = f"{type(e).__name__}: {e}"
+ wrote = os.path.exists(directory) and bool(os.listdir(directory))
+ out[f"{kind}:{name}"] = {"error": error, "wrote": wrote}
+ return out
+
+
+CASES = {"tp": _case_tp, "pp": _case_pp, "clip": _case_clip, "export": _case_export}
+
+
+def _worker(rank, world, port, case, tmp):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ torch.save(CASES[case](rank, world, tmp), os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _spawn(case, world = 2):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp)) for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+_SPAWNED = {}
+
+
+def _spawn_cached(case):
+ if case not in _SPAWNED:
+ _SPAWNED[case] = _spawn(case)
+ return _SPAWNED[case]
+
+
+def _close(a, b, rtol):
+ return abs(a - b) <= 1e-6 + rtol * abs(b)
+
+
+def _check_run(
+ run,
+ ref,
+ name,
+ union = None,
+):
+ # step 0 at 1e-4; step 1 norms at 1e-3 (after an AdamW step, near-zero grads flip update signs)
+ for i, (a, b) in enumerate(zip(run["losses"], ref["losses"])):
+ assert _close(a, b, 1e-4), (name, "loss", i, a, b)
+ for i, (a, b) in enumerate(zip(run["norms"], ref["norms"])):
+ assert _close(a, b, 1e-4 if i == 0 else 1e-3), (name, "grad norm", i, a, b)
+ if union is None:
+ assert set(run["grads"]) == set(ref["grads"]), (
+ name,
+ sorted(set(run["grads"]) ^ set(ref["grads"]))[:4],
+ )
+ for key, g in run["grads"].items():
+ r = ref["grads"][key]
+ atol = max(
+ 1e-7, 1e-4 * float(r.abs().max())
+ ) # fp32 reduction order: ~1e-6..1e-5 of the max
+ torch.testing.assert_close(g, r, rtol = 1e-4, atol = atol, msg = f"{name}: {key}")
+
+
[email protected](scope = "module")
+def tp_results():
+ return _spawn("tp")
+
+
[email protected](scope = "module")
+def pp_results():
+ return _spawn("pp")
+
+
[email protected]("name", sorted(TP_PASS))
+def test_tp_trains_like_one_process(tp_results, name):
+ for res in tp_results:
+ assert "crash" not in res[name], res[name]
+ _check_run(res[name]["tp"], res[name]["ref"], name)
+ assert res[name]["regions"] > 0
+ if name == "llama_shared":
+ # a block shared by two layers is sharded once (it used to be re-sharded: DTensor of a DTensor)
+ assert tp_results[0][name]["regions"] == 6
+
+
[email protected]("arch", sorted(TP_REFUSE))
+def test_tp_refuses_before_mutation(tp_results, arch):
+ for res in tp_results:
+ assert res[arch]["error"] and TP_REFUSE[arch] in res[arch]["error"], res[arch]
+ assert res[arch]["unchanged"], arch
+
+
[email protected]("arch", PP_PASS)
+def test_pp_trains_like_one_process(pp_results, arch):
+ for res in pp_results:
+ assert "crash" not in res[arch], res[arch]
+ _check_run(res[arch]["pp"], res[arch]["ref"], arch, union = True)
+ keys = set(pp_results[0][arch]["pp"]["grads"]) | set(pp_results[1][arch]["pp"]["grads"])
+ assert keys == set(pp_results[0][arch]["ref"]["grads"])
+ assert not set(pp_results[0][arch]["pp"]["grads"]) & set(pp_results[1][arch]["pp"]["grads"])
+
+
[email protected]("name", sorted(PP_REFUSE))
+def test_pp_refuses_before_mutation(pp_results, name):
+ for res in pp_results:
+ assert res[name]["error"] and PP_REFUSE[name][2] in res[name]["error"], res[name]
+ assert res[name]["unchanged"], name
+
+
+def test_tp_head_norm_names_need_one_head():
+ # q_layernorm over all heads (not one) is still refused: a norm across heads needs the full projection
+ model = build("hunyuan_v1_dense", lora = False)
+ attn = model.model.layers[0].self_attn
+ attn.query_layernorm = type(attn.query_layernorm)(64)
+ with pytest.raises(ValueError, match = "normalizes 64 features, not one head"):
+ tpn.derive_tp_plan(model, 2)
+
+
[email protected]("name", ["tp_clip_grad_norm_", "_clip_over_world", "no_clip"])
+def test_tp_clip_has_no_host_sync(name):
+ for res in _spawn_cached("clip"):
+ r = res[name]
+ assert r["error"] is None and r["is_tensor"], r
+ assert r["norm"] == pytest.approx(r["before"], rel = 1e-5), r
+ if name == "no_clip":
+ assert r[
+ "unchanged"
+ ], r # a coefficient clamped at 1 leaves the grads bitwise unchanged
+ else:
+ assert r["after"] == pytest.approx(1e-3, rel = 1e-4), r
+
+
+def test_tp_clip_coefficient_is_not_narrowed_to_fp16():
+ # A coefficient of ~1.7e-8 rounds to 0 in fp16; the clipped grads (~1e-3) are representable.
+ p = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ p.grad = torch.full((4,), 60000.0, dtype = torch.float16)
+ tpn._scale_grads_([p], torch.tensor(6e7), 1.0, 1e-6)
+ assert torch.all(p.grad > 0), p.grad
+ assert p.grad.float() == pytest.approx(torch.full((4,), 1e-3), rel = 1e-2), p.grad
+
+
[email protected]("kind", ["tp", "pp"])
[email protected]("name", EXPORTS)
+def test_exports_refuse_model_parallel_before_writing(kind, name):
+ results = _spawn_cached("export")
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ want = "tensor parallel model" if kind == "tp" else "pipeline parallel model"
+ for res in results:
+ r = res[f"{kind}:{name}"]
+ assert r["error"] and "RuntimeError" in r["error"] and want in r["error"], r
+ assert not r["wrote"], r
+
+
+def test_every_whole_model_export_is_guarded():
+ """Static: every export patch_saving_functions binds, and every writer behind them, refuses TP / PP first."""
+ import ast
+
+ tree = ast.parse(open(os.path.join(REPO, "unsloth", "save.py")).read())
+ functions = {n.name: n for n in tree.body if isinstance(n, ast.FunctionDef)}
+ bound = set()
+ for node in ast.walk(functions["patch_saving_functions"]):
+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "_bind_saving_method":
+ bound.add(node.args[0].id)
+ adapter_or_tokenizer = {
+ "unsloth_model_save_pretrained",
+ "unsloth_push_to_hub",
+ "unsloth_tokenizer_save_pretrained",
+ }
+ writers = {
+ "unsloth_save_model",
+ "unsloth_generic_save",
+ "unsloth_save_pretrained_merged",
+ "unsloth_push_to_hub_merged",
+ "save_to_gguf_generic",
+ "_unsloth_save_lora_gguf",
+ "save_lora_to_custom_dir",
+ "_unsloth_save_compressed_tensors",
+ "_unsloth_save_torchao",
+ "_unsloth_save_openvino",
+ }
+ targets = (bound - adapter_or_tokenizer) | writers
+ assert len(targets) >= 18, sorted(targets)
+ unguarded = []
+ for name in sorted(targets):
+ decorators = functions[
+ name
+ ].decorator_list # bound inside patch_saving_functions: defined at module level
+ first = decorators[0] if decorators else None
+ if not (
+ isinstance(first, ast.Call)
+ and getattr(first.func, "id", None) == "_model_parallel_refuses_save"
+ ):
+ unguarded.append(name)
+ assert not unguarded, unguarded
diff --git a/tests/test_pp_native_gloo.py b/tests/test_pp_native_gloo.py
index 1251b9b12ca..4c6a4838a5a 100644
--- a/tests/test_pp_native_gloo.py
+++ b/tests/test_pp_native_gloo.py
@@ -190,6 +190,14 @@ def _read_adapter(path):
return load_file(os.path.join(path, "adapter_model.safetensors"))
+def _enable_reentrant_gc(model):
+ """Reentrant checkpointing only back-propagates into a segment whose input requires grad; with the
+ embeddings frozen (LoRA) that needs enable_input_require_grads, which transformers 4.x
+ gradient_checkpointing_enable does not do for a PEFT model (5.x does)."""
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ model.enable_input_require_grads()
+
+
def _case_parity(
rank,
world,
@@ -211,13 +219,13 @@ def _case_parity(
ref = build(repo, tie = tie, **extra)
ids, labels = batch(ref)
if gc:
- ref.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ _enable_reentrant_gc(ref)
ref_res = _reference(ref, ids, labels, steps)
if rank == 0:
ref.save_pretrained(os.path.join(tmp, "ref"))
model = build(repo, tie = tie, **extra)
if gc:
- model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ _enable_reentrant_gc(model)
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))
originals = {
"embed": weakref.ref(model.get_input_embeddings()),
diff --git a/tests/test_save_keep_fp32_modules.py b/tests/test_save_keep_fp32_modules.py
index 23ac8458a54..0274b683851 100644
--- a/tests/test_save_keep_fp32_modules.py
+++ b/tests/test_save_keep_fp32_modules.py
@@ -125,5 +125,10 @@ def test_generic_save_casts_through_the_helper():
c.func.id for c in ast.walk(fn) if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
}
assert "_cast_state_dict_16bit" in calls
- decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in fn.decorator_list]
- assert decorators == ["_normalize_tied_weights_keys_for_save", "inference_mode"]
+ decorators = [d.func if isinstance(d, ast.Call) else d for d in fn.decorator_list]
+ decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in decorators]
+ assert decorators == [
+ "_model_parallel_refuses_save",
+ "_normalize_tied_weights_keys_for_save",
+ "inference_mode",
+ ]
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index ad8e6191022..128de54b4d5 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -6216,6 +6216,27 @@ def _tensor_parallel_trainer_pre(model):
return tp_trainer_setup(model, phase = "pre")
+def _refuse_model_parallel_export(model, method):
+ """Merged / GGUF / torchao / OpenVINO / LoRA exports read every weight of the model in this process. Under
+ apply_tensor_parallel each rank holds 1 / tp of the attention / MLP weights; under apply_pipeline_parallel
+ the layers, embeddings and lm_head of the other stages are parameter-free placeholders. Raise on every rank
+ before anything is written (no rank waits on another)."""
+ if _is_unsloth_tensor_parallel(model):
+ raise RuntimeError(
+ f"Unsloth: `{method}` does not support a tensor parallel model (apply_tensor_parallel): each rank "
+ "holds 1 / tp of the attention / MLP weights, so the export would be shard sized. Save the adapter "
+ "with `save_tp_adapter(model, directory)` (on every tp rank), then in a single process load the "
+ f"base model with that adapter and call `{method}` there."
+ )
+ if _is_unsloth_pipeline_parallel(model):
+ raise RuntimeError(
+ f"Unsloth: `{method}` does not support a pipeline parallel model (apply_pipeline_parallel): each "
+ "rank holds one stage, and the other stages' layers, embeddings and lm_head are placeholders. Save "
+ "the adapter with `pp.save_adapter(directory)` (on every pp rank), then in a single process load "
+ f"the base model with that adapter and call `{method}` there."
+ )
+
+
def _refuse_tensor_parallel_for_inference(model):
if _is_unsloth_tensor_parallel(model):
raise RuntimeError(
diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.py
index f7d295b7a4d..cfef8809c30 100644
--- a/unsloth/models/pp_native.py
+++ b/unsloth/models/pp_native.py
@@ -45,6 +45,7 @@ import collections
import dataclasses
import inspect
import math
+import re
import sys
import textwrap
import types
@@ -301,6 +302,8 @@ def _find_parts(model) -> _Parts:
)
text = _text_config(model)
n = getattr(text, "num_hidden_layers", None)
+ # A decoder-only use of an encoder-decoder config (BartForCausalLM): num_hidden_layers is the encoder's.
+ counts = {n, getattr(text, "decoder_layers", None)} - {None}
pp_plan = getattr(text, "base_model_pp_plan", None) or getattr(
getattr(inner, "config", None), "base_model_pp_plan", None
)
@@ -313,7 +316,11 @@ def _find_parts(model) -> _Parts:
layers_name, source = lists[0], "transformers"
if layers_name is None:
for name, child in named.items():
- if isinstance(child, nn.ModuleList) and len(child) and (n is None or len(child) == n):
+ if (
+ isinstance(child, nn.ModuleList)
+ and len(child)
+ and (n is None or len(child) in counts)
+ ):
layers_name = name
break
if layers_name is None:
@@ -344,11 +351,25 @@ def _find_parts(model) -> _Parts:
continue
norms.append((name, child))
hidden = getattr(text, "hidden_size", None)
+ vocab = getattr(text, "vocab_size", None)
+ order = list(decoder._modules)
others = []
for name, child in norms:
base = getattr(child, "original_module", child) # PEFT modules_to_save wraps it
norm_like = "norm" in name.lower() or "Norm" in type(base).__name__
too_big = bool(hidden) and sum(p.numel() for p in base.parameters()) > 4 * hidden
+ # A learned position / token-type table before the layers (GPT-2 wpe, OPT / BART / Whisper
+ # embed_positions): indexed by positions, not tokens, so later stages (whose pre-layer result is
+ # replaced by the received hidden states anyway) need nothing from it. A token-indexed table could
+ # feed the layers a side input and stays refused.
+ positional = (
+ isinstance(base, nn.Embedding)
+ and isinstance(vocab, int)
+ and base.num_embeddings != vocab
+ and order.index(name) < order.index(layers_name)
+ )
+ if positional:
+ continue
if not norm_like or too_big:
others.append(name)
if others:
@@ -361,6 +382,12 @@ def _find_parts(model) -> _Parts:
continue
if _owner(child, lm_head)[0] is not None:
continue
+ if _owner(child, decoder)[0] is not None:
+ # A wrapper around the decoder (OPTModel.decoder, BartDecoderWrapper.decoder): fine when it holds
+ # nothing else.
+ inside = {id(p) for p in decoder.parameters()}
+ if all(id(p) in inside for p in child.parameters()):
+ continue
raise ValueError(
f"Unsloth: pipeline parallelism v1 found an extra module {name} ({type(child).__name__}) "
f"next to the decoder of {type(inner).__name__} (multimodal towers / extra heads are v2)."
@@ -371,6 +398,44 @@ def _find_parts(model) -> _Parts:
return _Parts(inner, decoder, layers, layers_name, embed, lm_head, norms, source, tied)
+def _refuse_shared_across_stages(parts, bounds):
+ """A trainable parameter used by layers of two stages (a block shared by several layers, Zamba2 /
+ ALBERT style) would be trained as two independent copies."""
+ stage_of = {i: k for k, (start, end) in enumerate(bounds) for i in range(start, end)}
+ owners = {}
+ for i, layer in enumerate(parts.layers):
+ for name, p in layer.named_parameters():
+ stages, first, param = owners.setdefault(
+ id(p), (set(), f"{parts.layers_name}.{i}.{name}", p)
+ )
+ stages.add(stage_of[i])
+ shared = [first for stages, first, p in owners.values() if len(stages) > 1 and p.requires_grad]
+ if shared:
+ raise ValueError(
+ f"Unsloth: pipeline parallelism v1 cannot split layers that share trainable parameters across "
+ f"stages ({shared[:3]}: a block reused by several layers). Keep the sharing layers on one stage "
+ "(layers_per_stage, when they are adjacent), or use native FSDP2 (UNSLOTH_FSDP2=1)."
+ )
+
+
+def _check_loss_normalization(inner):
+ """PipelineParallel sums the micro-batch losses, each normalised by the token count of the whole batch
+ (num_items_in_batch). A causal LM forward that drops num_items_in_batch takes a per-micro-batch mean, so the
+ sum would be n_microbatches times the one-process loss (BART / Whisper / TrOCR decoders)."""
+ try:
+ source = inspect.getsource(_forward_function(inner))
+ except (OSError, TypeError):
+ source = ""
+ passes_kwargs = re.search(r"loss_function\([^()]*(?:\([^()]*\)[^()]*)*\*\*\s*\w+", source)
+ if "num_items_in_batch" in source or passes_kwargs:
+ return
+ raise ValueError(
+ f"Unsloth: pipeline parallelism v1 needs the {type(inner).__name__} loss to take num_items_in_batch "
+ "(self.loss_function(..., **kwargs)); this forward computes a per-micro-batch mean, so micro-batches "
+ "cannot be summed like gradient accumulation."
+ )
+
+
def _refuse_unsupported(model, parts):
text = _text_config(model)
inner = parts.inner
@@ -446,6 +511,7 @@ def _refuse_unsupported(model, parts):
"Unsloth: pipeline parallelism v1 cannot train tied input / output embeddings (the first and "
"last stage would each update their own copy). Freeze them, or untie them before splitting."
)
+ _check_loss_normalization(inner)
# ---------------------------------------------------------------------------------- layer call format
@@ -1373,6 +1439,7 @@ def apply_pipeline_parallel(
_check_schedule(model, schedule)
tuple_format = _layer_call_format(parts.decoder, parts.layers_name) == "tuple"
bounds = derive_pp_split(model, pp, config)
+ _refuse_shared_across_stages(parts, bounds)
dtype = config.activation_dtype or parts.embed.weight.dtype
rank = pp_mesh.get_local_rank()
if config.sync_trainable and pp > 1:
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index bb66b618ea1..619d22690e3 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -74,6 +74,7 @@ _TP_FLAG = "_unsloth_tp"
COLWISE = "colwise"
ROWWISE = "rowwise"
HEAD_NORM = "replicated_with_grad_allreduce"
+ELEMENTWISE = "elementwise_replicated" # scalar-parameter activations inside an MLP
# transformers TP styles (4.5x and 5.x names) -> what this module does with them.
_STYLES = {
@@ -115,15 +116,36 @@ _MLP_NAMES = ("mlp", "feed_forward", "ffn")
_Q, _K, _V = ("q_proj", "query"), ("k_proj", "key"), ("v_proj", "value")
_FUSED_QKV = ("qkv_proj",) # [q, k, v] along the output dim (Phi3)
_ATTN_ROW = ("o_proj", "out_proj", "dense")
-_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc")
+_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc", "fc_in", "dense_h_to_4h")
_MLP_PACKED = ("gate_up_proj",) # [gate, up] along the output dim
-_MLP_ROW = ("down_proj", "w2", "fc2")
-_HEAD_NORMS = ("q_norm", "k_norm", "query_norm", "key_norm")
+_MLP_ROW = ("down_proj", "w2", "fc2", "fc_out", "dense_4h_to_h", "c_proj")
+_HEAD_NORMS = (
+ "q_norm",
+ "k_norm",
+ "query_norm",
+ "key_norm",
+ "q_layernorm",
+ "k_layernorm",
+ "query_layernorm",
+ "key_layernorm",
+)
_MLA_NAMES = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "kv_a_layernorm")
_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
# An attention forward reading these sizes off the config would reshape to the global head count.
_GLOBAL_HEAD_READS = re.compile(r"config\.(num_attention_heads|num_key_value_heads)\b")
-_GLOBAL_WIDTH_READS = re.compile(r"(config|self)\.hidden_size\b")
+_GLOBAL_WIDTH_READS = re.compile(
+ r"(config|self)\.(hidden_size|embed_dim|all_head_size|n_embd|d_model|split_size)\b"
+)
+# An MLP forward reducing over its features (activation sparsity statistics, a norm, a softmax) would see
+# only this rank's 1 / tp of the intermediate dimension; one reading the intermediate size splits wrongly.
+_FEATURE_REDUCTIONS = re.compile(
+ r"(?:\btorch\.|\bF\.|\bfunctional\.|\.)(mean|std|var|var_mean|std_mean|sum|nansum|prod|amax|amin|"
+ r"argmax|argmin|topk|kthvalue|sort|argsort|softmax|log_softmax|logsumexp|cumsum|cumprod|norm|normalize|"
+ r"layer_norm|rms_norm|group_norm)\s*\("
+)
+_GLOBAL_INTERMEDIATE_READS = re.compile(r"(config|self)\.(intermediate_size|ffn_dim|n_inner)\b")
+# A fused qkv_proj is [q, k, v] blocks only when the forward slices it by the (local) head counts (Phi3).
+_LOCAL_QKV_SLICE = re.compile(r"(num_key_value_heads|num_kv_heads)\s*\*\s*self\.head_dim")
@dataclasses.dataclass
@@ -226,13 +248,19 @@ def _decoder_layers(model):
decoder = inner
text = _text_config(model)
n = getattr(text, "num_hidden_layers", None)
+ # A decoder-only use of an encoder-decoder config (BartForCausalLM): num_hidden_layers is the encoder's.
+ counts = {n, getattr(text, "decoder_layers", None)} - {None}
prefix = ""
for name, module in inner.named_modules():
if module is decoder:
prefix = name
break
for name, module in decoder.named_modules():
- if isinstance(module, nn.ModuleList) and len(module) and (n is None or len(module) == n):
+ if (
+ isinstance(module, nn.ModuleList)
+ and len(module)
+ and (n is None or len(module) in counts)
+ ):
path = ".".join(x for x in (prefix, name) if x)
return path, module
raise ValueError(
@@ -336,9 +364,25 @@ def _classify(region_path, name, child, kind, compiled):
return COLWISE, "packed"
if name in _MLP_ROW:
return ROWWISE, None
+ if _elementwise_scalars(child):
+ return ELEMENTWISE, None
return None, None
+def _elementwise_scalars(module):
+ """A parameterised activation with scalar parameters only (Apertus xIELU): elementwise on any shard of
+ the intermediate dimension, so it stays replicated (gradients summed over tp)."""
+ if isinstance(module, nn.Linear) or "lora_A" in module._modules or list(module.children()):
+ return False
+ if any(p.numel() != 1 for p in module.parameters()):
+ return False
+ try:
+ source = inspect.getsource(type(module))
+ except (OSError, TypeError):
+ return False
+ return not _FEATURE_REDUCTIONS.search(_drop_init(source))
+
+
def _attention_sizes(module, projs, text):
"""(heads, kv_heads, head_dim) of one attention module, from its own projections."""
heads_cfg = getattr(text, "num_attention_heads", None)
@@ -393,7 +437,7 @@ def derive_tp_plan(
if style not in _STYLES and style not in _REPLICATED_STYLES:
raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")
text = _text_config(model)
- regions, replicated = [], 0
+ regions, replicated, seen = [], 0, {}
for index, layer in enumerate(layers):
found = 0
for child_name, child in layer.named_children():
@@ -405,8 +449,11 @@ def derive_tp_plan(
if kind is None or not _has_params(child):
continue
path = f"{layers_path}.{index}.{child_name}"
- regions.append(_region(path, child, kind, compiled, text, tp_size))
found += 1
+ if id(child) in seen:
+ continue # a block shared by several layers (Zamba2, ALBERT style): sharded once
+ seen[id(child)] = path
+ regions.append(_region(path, child, kind, compiled, text, tp_size))
# A pipeline parallel pass-through (a layer another stage owns) is not a replicated layer.
replicated += found == 0 and "_pp_index" not in layer.__dict__
if not regions:
@@ -439,7 +486,7 @@ def _region(path, module, kind, compiled, text, tp):
"every module with parameters inside a sharded attention / MLP must be in the plan. Pass "
"plan = {...} to apply_tensor_parallel."
)
- if style == HEAD_NORM:
+ if style in (HEAD_NORM, ELEMENTWISE):
norms.append((name, child))
continue
weight, _ = _weight(child)
@@ -468,15 +515,20 @@ def _region(path, module, kind, compiled, text, tp):
)
_check_divisible("num_key_value_heads", kv_heads, tp, path)
for name, norm in norms:
- numel = sum(p.numel() for p in norm.parameters())
+ weight = norm._parameters.get("weight")
+ numel = (
+ weight.numel() if weight is not None else sum(p.numel() for p in norm.parameters())
+ )
if numel != head_dim:
raise ValueError(
f"Unsloth: '{path}.{name}' normalizes {numel} features, not one head ({head_dim}): a norm "
"across all heads needs the full projection, which tensor parallelism v1 does not gather."
)
_check_attention_source(module, path)
- elif norms:
- raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")
+ else:
+ if any(not _elementwise_scalars(norm) for _, norm in norms):
+ raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")
+ _check_mlp_source(module, path)
for proj in projs:
weight, base = _weight(proj.module)
out_features, in_features = _logical_shape(weight, base)
@@ -485,6 +537,7 @@ def _region(path, module, kind, compiled, text, tp):
proj.parts = (out_features,)
continue
if proj.parts == "qkv":
+ _check_fused_qkv_source(module, path, proj.name)
q, kv = region.heads * region.head_dim, region.kv_heads * region.head_dim
if q + 2 * kv != out_features:
raise ValueError(
@@ -527,21 +580,103 @@ def _forward_source(module):
return inspect.getsource(forward)
-def _check_attention_source(module, path):
+def _drop_init(source):
+ """A class source without its __init__ (sizes read there are construction, not forward math)."""
+ return re.sub(r"\n(\s+)def __init__\(.*?(?=\n\1def |\Z)", "\n", source, flags = re.S)
+
+
+def _region_source(module, path, what):
+ """The forward of a region module plus the methods of its own class that forward calls (self._helper)."""
try:
source = _forward_source(module)
except (OSError, TypeError):
raise ValueError(
f"Unsloth: tensor parallelism cannot read the forward of '{path}' ({type(module).__name__}) to "
- "check it uses this rank's head count."
+ f"check {what}."
+ )
+ seen = set()
+ for name in re.findall(r"\bself\.(\w+)\s*\(", source):
+ if name in seen or name in module._modules or name == "forward":
+ continue
+ seen.add(name)
+ method = getattr(type(module), name, None)
+ if not inspect.isfunction(method):
+ continue
+ try:
+ source += "\n" + inspect.getsource(inspect.unwrap(method))
+ except (OSError, TypeError):
+ pass
+ return source
+
+
+def _input_width_names(source, module):
+ """Variables a forward reads the full input width into (`bsz, len, dim = hidden_states.size()`)."""
+ names = set()
+ for x in {_input_name(module), "hidden_states"}:
+ x = re.escape(x)
+ for m in re.finditer(
+ r"\b\w+\s*,\s*\w+\s*,\s*(\w+)\s*=\s*" + x + r"\.(?:size\(\)|shape\b)", source
+ ):
+ names.add(m.group(1))
+ for m in re.finditer(
+ r"\b(\w+)\s*=\s*" + x + r"\.(?:shape\[(?:-1|2)\]|size\((?:-1|2)\))", source
+ ):
+ names.add(m.group(1))
+ names.discard("_")
+ return sorted(n for n in names if len(re.findall(r"\b" + re.escape(n) + r"\b", source)) > 1)
+
+
+def _check_attention_source(module, path):
+ source = _region_source(module, path, "it uses this rank's head count")
+ if "_unsloth_tp_heads" in source:
+ return # an Unsloth forward written for tensor parallelism
+ read = _GLOBAL_WIDTH_READS.search(source)
+ if read:
+ raise ValueError(
+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with "
+ f"{read.group(0)}, the global width; tensor parallelism v1 needs it to follow the local heads."
+ )
+ width = _input_width_names(source, module)
+ if width:
+ raise ValueError(
+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) keeps the input width "
+ f"({', '.join(width)}) for its output shape; tensor parallelism v1 needs it to follow the local heads."
+ )
+
+
+def _check_mlp_source(module, path):
+ source = _region_source(module, path, "it is elementwise over the intermediate features")
+ if "_unsloth_tp" in source:
+ return
+ reduction = _FEATURE_REDUCTIONS.search(source)
+ if reduction:
+ raise ValueError(
+ f"Unsloth: the MLP forward of '{path}' ({type(module).__name__}) reduces over its features "
+ f"({reduction.group(0).strip()}...); with tensor parallelism each rank holds 1 / tp of them, so v1 "
+ "cannot shard it."
)
- if _GLOBAL_WIDTH_READS.search(source) and "_unsloth_tp_heads" not in source:
+ read = _GLOBAL_INTERMEDIATE_READS.search(source)
+ if read:
raise ValueError(
- f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with hidden_size, "
- "the global width; tensor parallelism v1 needs it to follow the local heads."
+ f"Unsloth: the MLP forward of '{path}' ({type(module).__name__}) reads {read.group(0)}, the global "
+ "size; tensor parallelism v1 needs it to follow the local shard."
)
+def _check_fused_qkv_source(module, path, name):
+ try:
+ source = _forward_source(module)
+ except (OSError, TypeError):
+ source = ""
+ if "_unsloth_tp_heads" in source or _LOCAL_QKV_SLICE.search(source):
+ return
+ raise ValueError(
+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) does not slice the fused "
+ f"'{name}' by the key / value head count ([q, k, v] blocks); its layout is unknown to tensor "
+ "parallelism v1."
+ )
+
+
def shard_slice(tensor, dim, parts, rank, tp):
"""This rank's slice of `tensor` along `dim`: the rank-th 1/tp of every part, concatenated (a copy)."""
pieces, offset = [], 0
@@ -1144,6 +1279,11 @@ def apply_tensor_parallel(
_refuse_base(path, proj.module, weight, base)
_refuse_adapter(path, proj.module)
_check_preloaded(path, proj, weight, base, tp_mesh)
+ if id(weight) in covered:
+ raise ValueError(
+ f"Unsloth: '{path}' shares its weight with another sharded projection; tensor "
+ "parallelism v1 shards every projection once."
+ )
covered.update((id(weight), id(base.bias)))
if _is_params4bit(weight):
_check_params4bit(path, weight, proj.style, tp)
@@ -1413,13 +1553,26 @@ def tp_clip_grad_norm_(
dist.all_reduce(shard_sq, group = group)
total = total + shard_sq
total = total.sqrt()
- coef = float(max_norm) / (float(total) + eps)
- if coef < 1:
- for p in params:
- p.grad.mul_(coef)
+ _scale_grads_(params, total, max_norm, eps)
return total
+def _scale_grads_(params, total, max_norm, eps):
+ """torch's clip: a tensor coefficient clamped at 1 and applied unconditionally (no host sync, no
+ data-dependent branch). DTensor grads are scaled through their local shard."""
+ with torch.no_grad():
+ coef = torch.clamp(max_norm / (total + eps), max = 1.0)
+ for p in params:
+ g = p.grad
+ local = g.to_local() if _is_dtensor(g) else g
+ c = coef.to(device = local.device, dtype = torch.float32)
+ if local.dtype in (torch.float16, torch.bfloat16):
+ # A small coefficient rounds to 0 in fp16 (a device tensor is cast to the grad dtype).
+ local.copy_(local.float().mul_(c))
+ else:
+ local.mul_(c.to(local.dtype))
+
+
def _clip_over_world(params, max_norm, eps):
"""tp_clip_grad_norm_ for gradients on a 2-D (dp, tp) mesh (shard_model over the dp mesh): every
element's square is counted once, divided by the number of ranks holding a replica of it."""
@@ -1451,10 +1604,7 @@ def _clip_over_world(params, max_norm, eps):
total = sq if total is None else total + sq
dist.all_reduce(total)
total = total.sqrt()
- coef = float(max_norm) / (float(total) + eps)
- if coef < 1:
- for p in params:
- p.grad.mul_(coef)
+ _scale_grads_(params, total, max_norm, eps)
return total
diff --git a/unsloth/save.py b/unsloth/save.py
index d4e25cacd3d..b123646b075 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -70,6 +70,7 @@ from .models.loader_utils import (
_tokenizer_wants_local_only,
)
from .models._utils import (
+ _refuse_model_parallel_export,
_convert_torchao_model,
_merged_torchao_config_for_peft_qat,
_peft_untied_lm_head,
@@ -1284,6 +1285,23 @@ def _restore_tied_weights_keys(originals):
pass
+def _model_parallel_refuses_save(method):
+ """Every whole-model export refuses an apply_tensor_parallel / apply_pipeline_parallel model before it
+ writes anything (see _refuse_model_parallel_export)."""
+
+ def decorate(fn):
+ @functools.wraps(fn)
+ def wrapper(*args, **kwargs):
+ model = args[0] if args else kwargs.get("model", kwargs.get("self"))
+ if model is not None:
+ _refuse_model_parallel_export(model, method)
+ return fn(*args, **kwargs)
+
+ return wrapper
+
+ return decorate
+
+
def _normalize_tied_weights_keys_for_save(save_fn):
"""Coerce legacy list-form ``_tied_weights_keys`` to dict for the duration of a save, then restore: transformers >= 5 re-ties from the dict's VALUES, so a persisted ``{k: k}`` self-map would no-op a later resize/re-tie. ``model`` is the first positional arg (bound-method self) or the ``model=`` keyword."""
@@ -1303,6 +1321,7 @@ def _normalize_tied_weights_keys_for_save(save_fn):
return wrapper
+@_model_parallel_refuses_save("unsloth_save_model")
@_normalize_tied_weights_keys_for_save
@torch.inference_mode
def unsloth_save_model(
@@ -2738,6 +2757,7 @@ def save_to_gguf(
return all_saved_locations, want_full_precision, is_vlm
+@_model_parallel_refuses_save("save_pretrained_merged")
def unsloth_save_pretrained_merged(
self,
save_directory: Union[str, os.PathLike],
@@ -2877,6 +2897,7 @@ def unsloth_save_pretrained_merged(
gc.collect()
+@_model_parallel_refuses_save("push_to_hub_merged")
def unsloth_push_to_hub_merged(
self,
repo_id: str,
@@ -4484,6 +4505,7 @@ def _fsdp2_refuses_save(method):
return decorate
+@_model_parallel_refuses_save("save_pretrained_gguf")
@_fsdp2_refuses_save("save_pretrained_gguf")
@_normalize_tied_weights_keys_for_save
def unsloth_save_pretrained_gguf(
@@ -5158,6 +5180,7 @@ def _free_merge_if_disk_is_tight(
return freed
+@_model_parallel_refuses_save("push_to_hub_gguf")
@_fsdp2_refuses_save("push_to_hub_gguf")
def unsloth_push_to_hub_gguf(
self,
@@ -5439,6 +5462,7 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
return full_repo_id
+@_model_parallel_refuses_save("save_lora_to_custom_dir")
def save_lora_to_custom_dir(model, tokenizer, save_directory):
os.makedirs(save_directory, exist_ok = True)
@@ -5631,6 +5655,7 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):
)
+@_model_parallel_refuses_save("save_pretrained_gguf")
def _unsloth_save_lora_gguf(
model,
tokenizer,
@@ -5797,6 +5822,7 @@ def _unsloth_save_lora_gguf(
shutil.rmtree(base_config_dir, ignore_errors = True)
+@_model_parallel_refuses_save("push_to_hub_ggml")
def unsloth_convert_lora_to_ggml_and_push_to_hub(
self,
tokenizer,
@@ -5827,6 +5853,7 @@ def unsloth_convert_lora_to_ggml_and_push_to_hub(
)
+@_model_parallel_refuses_save("save_pretrained_ggml")
def unsloth_convert_lora_to_ggml_and_save_locally(
self,
save_directory: str,
@@ -6025,6 +6052,7 @@ def _prewarm_base_model_hub_cache(
)
+@_model_parallel_refuses_save("save_pretrained_gguf")
@torch.inference_mode
def save_to_gguf_generic(
model,
@@ -6281,6 +6309,7 @@ def _unmerge_flags_on_error(streamed):
raise
+@_model_parallel_refuses_save("save_pretrained_merged")
@_normalize_tied_weights_keys_for_save
@torch.inference_mode
def unsloth_generic_save(
@@ -6695,6 +6724,7 @@ def _fsdp2_merged_gather_options(model, args, kwargs):
return merged_save_gather_options(model, _merged_save_method(args, kwargs))
+@_model_parallel_refuses_save("save_pretrained_merged")
@_fsdp2_collective_save(
lambda self, args, kwargs: isinstance(self, PeftModel)
and _is_adapter_save_method(_merged_save_method(args, kwargs)),
@@ -6863,6 +6893,7 @@ def unsloth_generic_save_pretrained_merged(
# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.
+@_model_parallel_refuses_save("push_to_hub_merged")
@_fsdp2_collective_save(
lambda self, args, kwargs: isinstance(self, PeftModel)
and _is_adapter_save_method(_merged_save_method(args, kwargs)),
@@ -7545,6 +7576,7 @@ def _restore_model_after_quantize_subprocess(model, restore_token) -> None:
)
+@_model_parallel_refuses_save("save_pretrained_merged")
def _unsloth_save_compressed_tensors(
model,
save_directory: Union[str, os.PathLike],
@@ -7814,6 +7846,7 @@ def _unsloth_save_compressed_tensors(
clean_gpu_cache()
+@_model_parallel_refuses_save("save_pretrained_torchao")
def _unsloth_save_torchao(
model,
save_directory: Union[str, os.PathLike],
@@ -8079,6 +8112,7 @@ def _write_executorch_export_config(
return path
+@_model_parallel_refuses_save("save_pretrained_torchao")
@_fsdp2_refuses_save("save_pretrained_torchao")
def unsloth_save_pretrained_torchao(
self,
@@ -8264,6 +8298,7 @@ def _openvino_transformers_mismatch(model_type, task):
)
+@_model_parallel_refuses_save("save_pretrained_openvino")
def _unsloth_save_openvino(
model,
save_directory: Union[str, os.PathLike],
@@ -8406,6 +8441,7 @@ def _unsloth_save_openvino(
gc.collect()
+@_model_parallel_refuses_save("save_pretrained_openvino")
def unsloth_save_pretrained_openvino(
self,
save_directory: Union[str, os.PathLike],
@@ -8440,6 +8476,7 @@ def unsloth_save_pretrained_openvino(
)
+@_model_parallel_refuses_save("push_to_hub_openvino")
def unsloth_push_to_hub_openvino(
self,
repo_id: str,