Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tensor parallelism v1: apply_tensor_parallel for dense decoders (LoRA / QLoRA) (#15137)

Daniel Hand2ca6515204f
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
new file mode 100644
index 00000000000..396c142ce10
--- /dev/null
+++ b/tests/test_tp_native_gloo.py
@@ -0,0 +1,632 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""apply_tensor_parallel on a gloo CPU world (2 and 4 ranks) vs the same model in one process, fp32."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import socket # noqa: E402
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_oracle import ( # noqa: E402
+ ARCHS,
+ LLAMA,
+ PHI3_PACKED_PLAN,
+ QWEN3,
+ _batch,
+ build,
+ skip_unsupported,
+ tpn,
+)
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _grads(model):
+ return {
+ n: tpn._full(p.grad, p).detach().clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+
+
+def _train(model, ids, steps, clip):
+ params = [p for p in model.parameters() if p.requires_grad]
+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(steps):
+ out = model(input_ids = ids, labels = ids, use_cache = False)
+ out.loss.backward()
+ if step == 0:
+ grads = _grads(model)
+ logits = out.logits.detach().clone()
+ # Replicated factors' Partial grads are summed in backward: grads land in their param's layout.
+ mismatched = [
+ n
+ for n, p in model.named_parameters()
+ if tpn._is_dtensor(p.grad) and tuple(p.grad.placements) != tuple(p.placements)
+ ]
+ assert not mismatched, mismatched[:4]
+ norms.append(float(clip(params, 1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(out.loss))
+ return logits, grads, losses, norms
+
+
+def _clip_fp64(params, max_norm):
+ """clip_grad_norm_ in float64: torch's CPU fp32 vector_norm is ~2e-4 off, above the parity tolerance."""
+ grads = [p.grad for p in params if p.grad is not None]
+ total = torch.stack([g.double().square().sum() for g in grads]).sum().sqrt().float()
+ coef = torch.clamp(max_norm / (total + 1e-6), max = 1.0)
+ for g in grads:
+ g.mul_(coef)
+ return total
+
+
+def _tie_q_norms(model):
+ # Two regions sharing one trainable per-head norm weight: its tp sum must run once, not once per region.
+ layers = tpn._unwrap(model).model.layers
+ layers[1].self_attn.q_norm.weight = layers[0].self_attn.q_norm.weight
+
+
+def _case_parity(
+ rank,
+ world,
+ tmp,
+ arch,
+ overrides = None,
+ lora_kwargs = None,
+ save_kwargs = None,
+ plan = None,
+ freeze_before_save = False,
+ tie_norms = False,
+ fp64_ref_clip = False,
+):
+ from torch.distributed.device_mesh import init_device_mesh
+ from safetensors.torch import load_file
+
+ repo, base_overrides = ARCHS[arch]
+ overrides = {**base_overrides, **(overrides or {})}
+ train_norms = arch in ("qwen3", "gemma3")
+ ref = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))
+ model = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))
+ if tie_norms:
+ _tie_q_norms(ref), _tie_q_norms(model)
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"], plan = plan)
+ assert tpn.is_tensor_parallel(model)
+ assert tpn.apply_tensor_parallel(model, mesh["tp"]) is model # idempotent
+ with pytest.raises(RuntimeError, match = "generation is not supported"):
+ model.generate(input_ids = torch.zeros(1, 2, dtype = torch.long))
+ from test_fsdp2_native_units import _load_native
+
+ try:
+ _load_native()._refuse_tensor_parallel_model(model)
+ fsdp2_refused = None
+ except RuntimeError as e:
+ fsdp2_refused = str(e)
+ ids = _batch(ref)
+ ref_out = _train(ref, ids, 3, _clip_fp64 if fp64_ref_clip else torch.nn.utils.clip_grad_norm_)
+ tp_out = _train(model, ids, 3, tpn.tp_clip_grad_norm_)
+ if freeze_before_save:
+ # Frozen adapter (e.g. after training): PEFT still saves bias = "all" biases, trainable or not.
+ for m in (ref, model):
+ for p in m.parameters():
+ p.requires_grad_(False)
+ ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")
+ ref.save_pretrained(ref_dir, **(save_kwargs or {}))
+ tpn.save_tp_adapter(model, tp_dir, **(save_kwargs or {}))
+ q = tpn._weight(model._unsloth_tp_plan.regions[0].projs[0].module)[0]
+ return {
+ "ref": ref_out,
+ "tp": tp_out,
+ "saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),
+ "saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))
+ if rank == 0
+ else None,
+ "local_q": (tuple(q.to_local().shape), tuple(q.shape), q.placements[0].dim),
+ "fsdp2_refused": fsdp2_refused,
+ }
+
+
+def _import_checkout_unsloth():
+ """`import unsloth` from this checkout (not the stub tp_native is loaded under); None, or why it failed."""
+ import pathlib
+
+ for name in [n for n in sys.modules if n == "unsloth" or n.startswith("unsloth.")]:
+ del sys.modules[name]
+ sys.path.insert(0, REPO)
+ try:
+ # conftest's device spoof (spawned ranks do not run it).
+ import importlib.util
+
+ spec = importlib.util.spec_from_file_location(
+ "_tp_conftest", os.path.join(REPO, "tests", "conftest.py")
+ )
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+ import unsloth
+ except Exception as e: # a CPU host that refuses `import unsloth`
+ return repr(e)
+ # By path: compile setup may put other directories in front of sys.path.
+ root = pathlib.Path(REPO).resolve()
+ assert root in pathlib.Path(unsloth.__file__).resolve().parents, unsloth.__file__
+ return None
+
+
+def _raised(fn):
+ try:
+ fn()
+ except Exception as e:
+ return f"{type(e).__name__}: {e}"
+ return None
+
+
+def _case_guards(rank, world, tmp):
+ """Entry points outside the tp regions are refused on every wrapper level."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import transformers
+ from torch.distributed.device_mesh import init_device_mesh
+ from unsloth import FastLanguageModel, FastModel
+ from unsloth.models import fsdp2_native, tp_native
+ from unsloth.models._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(transformers.Trainer)
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ args = transformers.TrainingArguments(output_dir = tmp, report_to = [], use_cpu = True)
+ out = {}
+ for applied in ("peft", "causal"):
+ model = build(LLAMA)
+ levels = {
+ "peft": model,
+ "lora": model.base_model,
+ "causal": model.base_model.model,
+ "decoder": model.base_model.model.model,
+ }
+ tp_native.apply_tensor_parallel(levels[applied], mesh["tp"])
+ ids = _batch(model)
+ res = {}
+ cached = model(input_ids = ids, use_cache = True)
+ res["forward_without_cache"] = cached.logits is not None
+ past = cached.past_key_values
+ step = ids[:, -1:]
+ for name, level in levels.items():
+ res[f"is_tp:{name}"] = tp_native.is_tensor_parallel(level)
+ res[f"fsdp2:{name}"] = _raised(
+ lambda: fsdp2_native._refuse_tensor_parallel_model(level)
+ )
+ res[f"for_inference:{name}"] = _raised(lambda: FastLanguageModel.for_inference(level))
+ res[f"FastModel.for_inference:{name}"] = _raised(lambda: FastModel.for_inference(level))
+ if name != "decoder":
+ res[f"generate:{name}"] = _raised(lambda: level.generate(input_ids = ids))
+ res[f"trainer:{name}"] = _raised(
+ lambda: transformers.Trainer(model = level, args = args)
+ )
+ res[f"model_init:{name}"] = _raised(
+ lambda: transformers.Trainer(model_init = lambda: level, args = args)
+ )
+ res[f"cached:{name}"] = _raised(
+ lambda: level(input_ids = step, past_key_values = past, use_cache = True)
+ )
+ res["generate:get_base_model"] = _raised(
+ lambda: model.get_base_model().generate(input_ids = ids)
+ )
+ params = [p for p in model.parameters() if p.requires_grad]
+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()
+ norm = tp_native.tp_clip_grad_norm_(params, 1e9)
+ res["clip_mesh_equal"] = float(norm) == float(
+ tp_native.tp_clip_grad_norm_(params, 1e9, mesh = mesh["tp"])
+ )
+ out[applied] = res
+ return out
+
+
+def _case_fused(rank, world, tmp):
+ # The real package from this checkout, not the stub the oracle module loads tp_native under.
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ from unsloth.kernels import fast_lora
+ from unsloth.models import tp_native
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ calls = []
+ apply = fast_lora._apply
+ fast_lora._apply = lambda function, *args: calls.append(function.__name__) or apply(
+ function, *args
+ )
+ out = {}
+ for fused in (False, True):
+ calls.clear()
+ model = build(QWEN3, {"num_attention_heads": 8, "num_key_value_heads": 4})
+ tp_native.apply_tensor_parallel(model, mesh["tp"])
+ attn = tp_native._unwrap(model).model.layers[0].self_attn
+ g = torch.Generator().manual_seed(5)
+ X = torch.randn(
+ 2, 5, attn.q_proj.base_layer.weight.shape[1], generator = g, requires_grad = True
+ )
+ Xr = tp_native.copy_to_tp(X, attn._unsloth_tp_group)
+ if fused:
+ Q, K, V = fast_lora.apply_lora_qkv(attn, Xr, inplace = False)
+ else:
+ Q, K, V = attn.q_proj(Xr), attn.k_proj(Xr), attn.v_proj(Xr)
+ Y = torch.cat([Q, K, V], -1)
+ O_in = Y[..., : Q.shape[-1]].contiguous()
+ O = fast_lora.apply_lora_o(attn, O_in) if fused else attn.o_proj(O_in)
+ O = tp_native.reduce_from_tp(O, attn._unsloth_tp_group)
+ (O.square().sum() + Y.square().sum()).backward()
+ out[fused] = {
+ "O": O.detach(),
+ "dX": X.grad.detach(),
+ "grads": _grads(attn),
+ "calls": list(calls),
+ }
+ return out
+
+
+def _quantize_layers(
+ model,
+ nested,
+ blocksize = 64,
+):
+ """Every Linear in the decoder layers -> a bitsandbytes NF4 Linear4bit (the same bytes in every process)."""
+ import bitsandbytes as bnb
+
+ for name, module in list(model.named_modules()):
+ for child_name, child in list(module.named_children()):
+ if type(child) is not torch.nn.Linear or ".layers." not in f".{name}.":
+ continue
+ q = bnb.nn.Linear4bit(
+ child.in_features,
+ child.out_features,
+ bias = child.bias is not None,
+ compute_dtype = torch.float32,
+ quant_type = "nf4",
+ compress_statistics = nested,
+ )
+ packed, state = bnb.functional.quantize_4bit(
+ child.weight.data, blocksize = blocksize, quant_type = "nf4", compress_statistics = nested
+ )
+ q.weight = bnb.nn.Params4bit(
+ packed,
+ requires_grad = False,
+ quant_state = state,
+ blocksize = blocksize,
+ compress_statistics = nested,
+ quant_type = "nf4",
+ module = q,
+ bnb_quantized = True,
+ )
+ if child.bias is not None:
+ q.bias = child.bias
+ setattr(module, child_name, q)
+ return model
+
+
+QLORA_SIZES = {
+ "hidden_size": 128,
+ "intermediate_size": 256,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 32,
+}
+
+
+def _qlora_model(
+ nested,
+ seed = 0,
+ blocksize = 64,
+):
+ from peft import LoraConfig, get_peft_model
+
+ model = _quantize_layers(build(LLAMA, QLORA_SIZES, lora = False, seed = seed), nested, blocksize)
+ torch.manual_seed(seed + 1)
+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
+ # train(): eval-mode CPU Linear4bit repacks for a CPU gemm that rounds differently.
+ return get_peft_model(model, config).train()
+
+
+def _case_qlora(rank, world, tmp, nested):
+ import bitsandbytes as bnb
+ from torch.distributed.device_mesh import init_device_mesh
+
+ ref, model = _qlora_model(nested), _qlora_model(nested)
+ full = {
+ n: bnb.functional.dequantize_4bit(m.weight.data, m.weight.quant_state)
+ for n, m in model.named_modules()
+ if type(m).__name__ == "Linear4bit"
+ }
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ shard_err = 0.0
+ for region in model._unsloth_tp_plan.regions:
+ for proj in region.projs:
+ base = proj.module.base_layer
+ name = next(n for n, m in model.named_modules() if m is base)
+ local = bnb.functional.dequantize_4bit(base.weight.data, base.weight.quant_state)
+ col = proj.style == tpn.COLWISE
+ want = tpn.shard_slice(
+ full[name],
+ 0 if col else 1,
+ proj.parts if col else (full[name].shape[1],),
+ rank,
+ world,
+ )
+ assert local.shape == want.shape, (name, local.shape, want.shape)
+ shard_err = max(shard_err, float((local - want).abs().max()))
+ ids = _batch(ref)
+ return {
+ "ref": _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_),
+ "tp": _train(model, ids, 3, tpn.tp_clip_grad_norm_),
+ "shard_err": shard_err,
+ }
+
+
+def _case_qlora_misaligned(rank, world, tmp):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ # blocksize 128: o_proj / down_proj row shards (in_features / 2 = 64) would split blocks.
+ model = _qlora_model(False, blocksize = 128)
+ before = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ with pytest.raises(ValueError, match = "row sharding would split blocks"):
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ after = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}
+ return {"unchanged": before == after, "flag": tpn.is_tensor_parallel(model)}
+
+
+def _case_save_subgroup(rank, world, tmp):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ # tp = 1 inside a 2-rank world (data parallel 2): the world-scoped writer / barrier would not match.
+ mesh = init_device_mesh("cpu", (world, 1), mesh_dim_names = ("dp", "tp"))
+ model = tpn.apply_tensor_parallel(build(LLAMA), mesh["tp"])
+ return {"error": _raised(lambda: tpn.save_tp_adapter(model, os.path.join(tmp, "out")))}
+
+
+CASES = {
+ "parity": _case_parity,
+ "save_subgroup": _case_save_subgroup,
+ "fused": _case_fused,
+ "guards": _case_guards,
+ "qlora": _case_qlora,
+ "qlora_misaligned": _case_qlora_misaligned,
+}
+
+
+def _worker(rank, world, port, case, tmp, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = 2,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+def _check_parity(results, world):
+ for rank, res in enumerate(results):
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads), sorted(set(grads) ^ set(ref_grads))[:6]
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6), (rank, losses, ref_losses)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5), (rank, norms, ref_norms)
+ assert "does not compose with Unsloth tensor parallelism" in (res["fsdp2_refused"] or "")
+ local, full, dim = res["local_q"]
+ assert local[dim] * world == full[dim] and local[1 - dim] == full[1 - dim]
+ saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]
+ assert set(saved_tp) == set(saved_ref)
+ for key in saved_ref:
+ assert saved_tp[key].shape == saved_ref[key].shape, key
+ torch.testing.assert_close(saved_tp[key], saved_ref[key], rtol = 1e-4, atol = 1e-5, msg = key)
+
+
[email protected]("arch", ["llama", "llama_bias", "qwen3", "phi3", "stablelm", "gemma2"])
+def test_tp2_matches_one_process(arch):
+ skip_unsupported(arch)
+ _check_parity(_spawn("parity", 2, arch = arch), 2)
+
+
+def test_tp2_refused_4bit_model_is_left_untouched():
+ for res in _spawn("qlora_misaligned", 2):
+ assert res["unchanged"] and not res["flag"]
+
+
+def test_tp2_trainable_biases_match_one_process():
+ # bias = "all": the row bias is added after the all-reduce, so its gradient is complete on every rank.
+ results = _spawn("parity", 2, arch = "llama_bias", lora_kwargs = {"bias": "all"})
+ _check_parity(results, 2)
+ assert any("bias" in key for key in results[0]["saved_tp"])
+
+
+def test_tp2_adapter_save_keeps_requested_embeddings():
+ results = _spawn("parity", 2, arch = "llama", save_kwargs = {"save_embedding_layers": True})
+ _check_parity(results, 2)
+ assert any("embed_tokens" in key for key in results[0]["saved_tp"])
+
+
[email protected]("nested", [False, True])
+def test_tp2_qlora_matches_one_process(nested):
+ results = _spawn("qlora", 2, nested = nested)
+ for res in results:
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ assert res["shard_err"] == 0.0
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads) and grads
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5)
+
+
+def test_tp4_gqa_matches_one_process():
+ overrides = {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 64}
+ _check_parity(_spawn("parity", 4, arch = "qwen3", overrides = overrides), 4)
+
+
+def test_fused_lora_functions_read_local_shards():
+ results = _spawn("fused", 2)
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ for res in results:
+ plain, fused = res[False], res[True]
+ assert plain["calls"] == [] and fused["calls"] == ["LoRA_QKV", "LoRA_W"], (
+ plain["calls"],
+ fused["calls"],
+ )
+ torch.testing.assert_close(fused["O"], plain["O"], rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(fused["dX"], plain["dX"], rtol = 1e-5, atol = 1e-5)
+ assert set(fused["grads"]) == set(plain["grads"]) and plain["grads"]
+ for name in plain["grads"]:
+ torch.testing.assert_close(
+ fused["grads"][name], plain["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name
+ )
+
+
+def test_apply_refuses_a_2d_mesh():
+ model = build(LLAMA)
+
+ class _Mesh:
+ ndim = 2
+
+ with pytest.raises(ValueError, match = "1-D tp mesh"):
+ tpn.apply_tensor_parallel(model, _Mesh())
+
+
+def test_tp2_phi3_fused_qkv_under_a_packed_plan_matches_one_process():
+ # The tiny Phi3 is GQA (4 heads, 2 kv heads): qkv_proj parts are (q, kv, kv), not two packed halves.
+ _check_parity(_spawn("parity", 2, arch = "phi3", plan = PHI3_PACKED_PLAN), 2)
+
+
+ "lora_kwargs,save_kwargs",
+ [
+ ({"bias": "all"}, None), # frozen bias-bearing adapter: every bias, trainable or not
+ ({"bias": "lora_only"}, None),
+ ({"modules_to_save": ["lm_head"]}, None),
+ ({"modules_to_save": ["lm_head"]}, {"save_embedding_layers": True}),
+ ],
+)
+def test_tp2_adapter_save_matches_peft_selection(lora_kwargs, save_kwargs):
+ results = _spawn(
+ "parity",
+ 2,
+ arch = "llama_bias",
+ lora_kwargs = lora_kwargs,
+ save_kwargs = save_kwargs,
+ freeze_before_save = True,
+ fp64_ref_clip = "modules_to_save" in lora_kwargs,
+ )
+ _check_parity(results, 2)
+ saved = results[0]["saved_tp"]
+ if (
+ lora_kwargs.get("bias") == "all"
+ ): # "lora_only" saves no base_layer bias on PEFT 0.18 (1 process too)
+ assert any("o_proj" in k and k.endswith("bias") for k in saved), sorted(saved)[:8]
+ if "modules_to_save" in lora_kwargs:
+ assert any("modules_to_save" in k or "lm_head" in k for k in saved), sorted(saved)[:8]
+
+
+def test_tp2_tied_norm_weight_is_summed_once():
+ _check_parity(_spawn("parity", 2, arch = "qwen3", tie_norms = True), 2)
+
+
+def test_tp2_entry_points_outside_the_regions_are_refused():
+ results = _spawn("guards", 2)
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ for res in results:
+ for applied, got in res.items():
+ assert got["forward_without_cache"] and got["clip_mesh_equal"], (applied, got)
+ for key, value in got.items():
+ if key.startswith("is_tp:"):
+ assert value is True, (applied, key)
+ elif key.startswith("fsdp2:"):
+ assert value and "does not compose with Unsloth tensor parallelism" in value, (
+ applied,
+ key,
+ value,
+ )
+ elif "for_inference" in key:
+ assert value and "for_inference is not supported" in value, (
+ applied,
+ key,
+ value,
+ )
+ elif key.startswith("generate:"):
+ assert value and "generation is not supported" in value, (applied, key, value)
+ elif key.startswith(("trainer:", "model_init:")):
+ assert value and "cannot be trained by the Trainer" in value, (
+ applied,
+ key,
+ value,
+ )
+ elif key.startswith("cached:"):
+ assert value and "cached decoding" in value, (applied, key, value)
+
+
+def test_save_refuses_a_tp_mesh_smaller_than_the_world():
+ for res in _spawn("save_subgroup", 2):
+ assert res["error"] and "data parallel > 1 is outside v1" in res["error"], res
diff --git a/tests/test_tp_native_oracle.py b/tests/test_tp_native_oracle.py
new file mode 100644
index 00000000000..8ad9825aa67
--- /dev/null
+++ b/tests/test_tp_native_oracle.py
@@ -0,0 +1,688 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallel plan derivation, refusals, and a simulated-TP oracle (summed shard copies), fp32 CPU."""
+
+import copy
+import importlib.util
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _load_tp():
+ """tp_native.py without importing the unsloth package (which needs a GPU)."""
+ name = "unsloth.models.tp_native"
+ if name in sys.modules:
+ return sys.modules[name]
+ import importlib.machinery
+
+ pkg = types.ModuleType("unsloth")
+ pkg.__path__ = [os.path.join(REPO, "unsloth")]
+ models = types.ModuleType("unsloth.models")
+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]
+ for package in (pkg, models):
+ # A spec, so a later importlib.util.find_spec("unsloth") in the same session does not raise.
+ package.__spec__ = importlib.machinery.ModuleSpec(package.__name__, None, is_package = True)
+ package.__spec__.submodule_search_locations = package.__path__
+ sys.modules.setdefault("unsloth", pkg)
+ sys.modules.setdefault("unsloth.models", models)
+ spec = importlib.util.spec_from_file_location(
+ name, os.path.join(REPO, "unsloth", "models", "tp_native.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+tpn = _load_tp()
+
+
+def skip_unsupported(arch):
+ """transformers 4.x StableLmAttention reshapes with hidden_size (the global width): apply refuses it."""
+ import transformers
+ if arch == "stablelm" and int(transformers.__version__.split(".")[0]) < 5:
+ pytest.skip("transformers 4.x StableLM attention is refused (test_stablelm_4x_is_refused)")
+
+
+LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"
+QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+GEMMA2 = "trl-internal-testing/tiny-Gemma2ForCausalLM"
+GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"
+MISTRAL = "hf-internal-testing/tiny-random-MistralForCausalLM"
+PHI3 = "trl-internal-testing/tiny-Phi3ForCausalLM"
+QWEN2 = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5-Coder"
+STABLELM = "hf-internal-testing/tiny-random-StableLmForCausalLM" # no transformers TP plan
+OLMO2 = "hf-internal-testing/tiny-random-Olmo2ForCausalLM"
+QWEN3_MOE = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"
+DEEPSEEK_V3 = "trl-internal-testing/tiny-DeepseekV3ForCausalLM"
+
+# Archs that shard at tp = 2 (config overrides make every count divisible where the tiny repo is not).
+ARCHS = {
+ "llama": (LLAMA, {}),
+ "llama_bias": (LLAMA, {"attention_bias": True, "mlp_bias": True}), # row biases: added once
+ "qwen3": (QWEN3, {}),
+ "gemma2": (GEMMA2, {}),
+ "gemma3": (GEMMA3, {"num_attention_heads": 4, "num_key_value_heads": 2}),
+ "mistral": (MISTRAL, {"intermediate_size": 64}),
+ "phi3": (PHI3, {}),
+ "qwen2": (QWEN2, {}),
+ "stablelm": (STABLELM, {"use_qkv_bias": True, "intermediate_size": 64}),
+}
+
+
+def build(
+ repo,
+ overrides = None,
+ lora = True,
+ seed = 0,
+ train_norms = False,
+ **lora_kwargs,
+):
+ from transformers import AutoConfig, AutoModelForCausalLM
+
+ config = AutoConfig.from_pretrained(repo)
+ text = config.get_text_config()
+ for key, value in (overrides or {}).items():
+ setattr(text, key, value)
+ config._attn_implementation = "eager"
+ torch.manual_seed(seed)
+ model = AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if name.endswith("bias"):
+ p.normal_(0, 0.1) # random biases: their placement matters
+ elif "norm" in name and p.ndim == 1:
+ p.add_(torch.randn_like(p) * 0.1)
+ if not lora:
+ return model.eval()
+ from peft import LoraConfig, get_peft_model
+
+ kwargs = dict(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
+ kwargs.update(lora_kwargs)
+ torch.manual_seed(seed + 1)
+ model = get_peft_model(model, LoraConfig(**kwargs))
+ if train_norms:
+ for name, p in model.named_parameters():
+ if name.endswith(("q_norm.weight", "k_norm.weight")):
+ p.requires_grad_(True)
+ return model.eval()
+
+
+def _batch(
+ model,
+ seed = 1,
+ bsz = 2,
+ seq = 7,
+):
+ vocab = tpn._text_config(model).vocab_size
+ g = torch.Generator().manual_seed(seed)
+ return torch.randint(0, vocab, (bsz, seq), generator = g)
+
+
+class _SimRegion(torch.nn.Module):
+ """tp shard copies of one region, outputs summed (reduce_from_tp) plus the row bias added once."""
+
+ def __init__(self, copies, row_bias):
+ super().__init__()
+ self.copies = torch.nn.ModuleList(copies)
+ self.row_bias = row_bias
+
+ def __getattr__(self, name):
+ try:
+ return super().__getattr__(name)
+ except AttributeError:
+ return getattr(self._modules["copies"][0], name)
+
+ def forward(self, *args, **kwargs):
+ outs = [c(*args, **kwargs) for c in self.copies]
+ tuple_out = isinstance(outs[0], tuple)
+ y = sum(o[0] if tuple_out else o for o in outs)
+ if self.row_bias is not None:
+ y = y + self.row_bias
+ return (y,) + tuple(outs[0][1:]) if tuple_out else y
+
+
+def _shard_copy(region, rank, tp):
+ module = copy.deepcopy(region.module)
+ for proj in region.projs:
+ child = getattr(module, proj.name)
+ weight, base = tpn._weight(child)
+ col = proj.style == tpn.COLWISE
+ in_parts = (weight.shape[1],)
+ with torch.no_grad():
+ if col:
+ base.weight = torch.nn.Parameter(
+ tpn.shard_slice(weight, 0, proj.parts, rank, tp), False
+ )
+ if base.bias is not None:
+ base.bias = torch.nn.Parameter(
+ tpn.shard_slice(base.bias, 0, proj.parts, rank, tp), False
+ )
+ else:
+ base.weight = torch.nn.Parameter(
+ tpn.shard_slice(weight, 1, in_parts, rank, tp), False
+ )
+ base.bias = None
+ for adapter in getattr(child, "lora_A", {}):
+ A, B = child.lora_A[adapter], child.lora_B[adapter]
+ if col:
+ B.weight = torch.nn.Parameter(
+ tpn.shard_slice(B.weight, 0, proj.parts, rank, tp)
+ )
+ else:
+ A.weight = torch.nn.Parameter(tpn.shard_slice(A.weight, 1, in_parts, rank, tp))
+ if region.kind == "attention":
+ tpn._set_local_heads(module, region, tp)
+ return module
+
+
+def simulate(
+ model,
+ tp,
+ user_plan = None,
+):
+ """A copy of `model` whose regions run as tp summed shard copies, and the plan used."""
+ sim = copy.deepcopy(model)
+ plan = tpn.derive_tp_plan(sim, tp, user_plan)
+ for region in plan.regions:
+ parent_path, name = region.path.rsplit(".", 1)
+ parent = tpn._unwrap(sim).get_submodule(parent_path)
+ row_bias = None
+ for proj in region.projs:
+ base = tpn._weight(proj.module)[1]
+ if proj.style == tpn.ROWWISE and base.bias is not None:
+ row_bias = base.bias.detach()
+ copies = [_shard_copy(region, r, tp) for r in range(tp)]
+ setattr(parent, name, _SimRegion(copies, row_bias))
+ return sim, plan
+
+
+def _sim_grads(sim, plan, tp):
+ """{param path relative to the unwrapped model: full gradient} reassembled from the shard copies."""
+ grads = {}
+ inner = tpn._unwrap(sim)
+ for region in plan.regions:
+ sim_region = inner.get_submodule(region.path)
+ for proj in region.projs:
+ col = proj.style == tpn.COLWISE
+ children = [getattr(c, proj.name) for c in sim_region.copies]
+ for adapter in getattr(children[0], "lora_A", {}):
+ As = [c.lora_A[adapter].weight.grad for c in children]
+ Bs = [c.lora_B[adapter].weight.grad for c in children]
+ prefix = f"{region.path}.{proj.name}"
+ if col:
+ grads[f"{prefix}.lora_A.{adapter}.weight"] = sum(As)
+ grads[f"{prefix}.lora_B.{adapter}.weight"] = tpn._unpermute(
+ torch.cat(Bs, 0), 0, proj.parts, tp
+ )
+ else:
+ grads[f"{prefix}.lora_A.{adapter}.weight"] = torch.cat(As, 1)
+ grads[f"{prefix}.lora_B.{adapter}.weight"] = sum(Bs)
+ for norm_name, _ in region.norms:
+ for pname, _ in getattr(sim_region.copies[0], norm_name).named_parameters():
+ gs = [getattr(c, norm_name).get_parameter(pname).grad for c in sim_region.copies]
+ if gs[0] is not None:
+ grads[f"{region.path}.{norm_name}.{pname}"] = sum(gs)
+ return grads
+
+
+def _ref_grads(model):
+ inner = tpn._unwrap(model)
+ return {n: p.grad for n, p in inner.named_parameters() if p.grad is not None}
+
+
+def _loss(model, ids):
+ out = model(input_ids = ids, labels = ids, use_cache = False)
+ return out.loss, out.logits
+
+
[email protected]("arch", sorted(ARCHS))
+def test_simulated_tp_matches_unsharded(arch):
+ skip_unsupported(arch)
+ repo, overrides = ARCHS[arch]
+ model = build(repo, overrides, train_norms = arch == "qwen3")
+ ids = _batch(model)
+ ref_loss, ref_logits = _loss(model, ids)
+ ref_loss.backward()
+ ref = _ref_grads(model)
+ tp = 2
+ sim, plan = simulate(model, tp)
+ for p in sim.parameters():
+ p.grad = None
+ loss, logits = _loss(sim, ids)
+ loss.backward()
+ got = _sim_grads(sim, plan, tp)
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(loss, ref_loss, rtol = 1e-6, atol = 1e-6)
+ lora = [k for k in ref if "lora_" in k]
+ assert lora and set(lora) <= set(got), sorted(set(lora) - set(got))[:4]
+ for key in ref:
+ if key in got:
+ torch.testing.assert_close(got[key], ref[key], rtol = 1e-4, atol = 1e-6, msg = key)
+ if arch == "qwen3":
+ assert any(
+ k.endswith("q_norm.weight") for k in got
+ ) # the per-head norm grad sum was compared
+
+
[email protected]("tp", [2, 4])
+def test_simulated_tp_gqa_widths(tp):
+ model = build(
+ QWEN3, {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 32}
+ )
+ ids = _batch(model)
+ ref_loss, ref_logits = _loss(model, ids)
+ sim, _ = simulate(model, tp)
+ _, logits = _loss(sim, ids)
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+
+
+def test_mutation_unsummed_replicated_factor_fails():
+ """Dropping the tp sum of a replicated factor's gradient (taking one copy's partial) is caught."""
+ model = build(LLAMA)
+ ids = _batch(model)
+ _loss(model, ids)[0].backward()
+ ref = _ref_grads(model)
+ sim, plan = simulate(model, 2)
+ _loss(sim, ids)[0].backward()
+ region = plan.regions[0]
+ proj = region.projs[0]
+ child = getattr(tpn._unwrap(sim).get_submodule(region.path).copies[0], proj.name)
+ key = f"{region.path}.{proj.name}.lora_A.default.weight"
+ with pytest.raises(AssertionError):
+ torch.testing.assert_close(
+ child.lora_A["default"].weight.grad, ref[key], rtol = 1e-4, atol = 1e-6
+ )
+
+
+def _styles(plan):
+ return {k.split(".", 3)[-1] if k.count(".") >= 3 else k: v for k, v in plan.styles().items()}
+
+
[email protected]("arch", sorted(ARCHS))
+def test_plan_covers_every_layer(arch):
+ skip_unsupported(arch)
+ repo, overrides = ARCHS[arch]
+ model = build(repo, overrides)
+ plan = tpn.derive_tp_plan(model, 2)
+ layers = tpn._text_config(model).num_hidden_layers
+ assert len(plan.regions) == 2 * layers
+ assert plan.source == ("structural" if arch == "stablelm" else "transformers")
+ styles = {(p.name, p.style) for r in plan.regions for p in r.projs}
+ if arch == "phi3":
+ assert styles == {
+ ("qkv_proj", "colwise"),
+ ("o_proj", "rowwise"),
+ ("gate_up_proj", "colwise"),
+ ("down_proj", "rowwise"),
+ }
+ parts = {p.name: p.parts for r in plan.regions for p in r.projs}
+ text = tpn._text_config(model)
+ hd = text.hidden_size // text.num_attention_heads
+ assert parts["qkv_proj"] == (
+ text.num_attention_heads * hd,
+ text.num_key_value_heads * hd,
+ text.num_key_value_heads * hd,
+ )
+ assert parts["gate_up_proj"] == (text.intermediate_size, text.intermediate_size)
+ else:
+ assert styles == {
+ ("q_proj", "colwise"),
+ ("k_proj", "colwise"),
+ ("v_proj", "colwise"),
+ ("o_proj", "rowwise"),
+ ("gate_proj", "colwise"),
+ ("up_proj", "colwise"),
+ ("down_proj", "rowwise"),
+ }
+ norms = {n for r in plan.regions for n, _ in r.norms}
+ assert norms == ({"q_norm", "k_norm"} if arch in ("qwen3", "gemma3") else set())
+
+
+def test_structural_plan_equals_transformers_plan():
+ model = build(QWEN3)
+ hf = tpn.derive_tp_plan(model, 2)
+ text = tpn._text_config(model)
+ text.base_model_tp_plan = None
+ for module in model.modules():
+ if "_tp_plan" in module.__dict__:
+ module._tp_plan = None
+ structural = tpn.derive_tp_plan(model, 2)
+ assert structural.source == "structural"
+ assert structural.styles() == hf.styles()
+ assert [len(r.norms) for r in structural.regions] == [len(r.norms) for r in hf.regions]
+
+
+def test_user_plan_overrides():
+ model = build(LLAMA, lora = False)
+ plan = {
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ }
+ assert tpn.derive_tp_plan(model, 2, plan).source == "user"
+ with pytest.raises(ValueError, match = "unknown tensor parallel style"):
+ tpn.derive_tp_plan(model, 2, {**plan, "layers.*.mlp.down_proj": "diagonal"})
+ # A plan entry wins over the name rule; names fill what the plan leaves out.
+ swapped = tpn.derive_tp_plan(model, 2, {**plan, "layers.*.mlp.up_proj": "rowwise"}).styles()
+ assert swapped["model.layers.0.mlp.up_proj"] == "rowwise"
+ missing = {k: v for k, v in plan.items() if "down_proj" not in k}
+ assert (
+ tpn.derive_tp_plan(model, 2, missing).styles()
+ == tpn.derive_tp_plan(model, 2, plan).styles()
+ )
+
+
+def test_unknown_module_inside_region_is_refused():
+ model = build(LLAMA, lora = False)
+ model.model.layers[1].mlp.gate_scale = torch.nn.Linear(4, 4)
+ with pytest.raises(ValueError, match = "no rule for 'model.layers.1.mlp.gate_scale'"):
+ tpn.derive_tp_plan(model, 2)
+ model = build(LLAMA, lora = False)
+ model.model.layers[0].self_attn.sinks = torch.nn.Parameter(torch.zeros(4))
+ with pytest.raises(ValueError, match = "owns the parameter 'sinks'"):
+ tpn.derive_tp_plan(model, 2)
+
+
+def test_mla_config_refused():
+ model = build(LLAMA, lora = False)
+ tpn._text_config(model).kv_lora_rank = 16
+ with pytest.raises(ValueError, match = "latent attention"):
+ tpn.derive_tp_plan(model, 2)
+
+
+def test_local_heads_set_per_module_not_config():
+ model = build(PHI3)
+ text = tpn._text_config(model)
+ heads, kv = text.num_attention_heads, text.num_key_value_heads
+ sim, plan = simulate(model, 2)
+ attn = tpn._unwrap(sim).get_submodule(plan.regions[0].path).copies[0]
+ assert attn._unsloth_tp_heads == heads // 2 and attn._unsloth_tp_kv_heads == kv // 2
+ assert attn.config.num_attention_heads == heads // 2 # Phi3 slices qkv with config heads
+ assert tpn._text_config(sim).num_attention_heads == heads # shared config untouched
+ assert tpn._text_config(model).num_attention_heads == heads
+
+
+def test_shard_slice_unpermute_round_trip():
+ t = torch.arange(10 * 3).view(10, 3)
+ parts = (6, 2, 2)
+ shards = [tpn.shard_slice(t, 0, parts, r, 2) for r in range(2)]
+ assert shards[0][:3].tolist() == t[:3].tolist() and shards[0][3].tolist() == t[6].tolist()
+ assert torch.equal(tpn._unpermute(torch.cat(shards), 0, parts, 2), t)
+
+
+ "repo,overrides,tp,match",
+ [
+ (LLAMA, {"num_attention_heads": 4, "num_key_value_heads": 4}, 3, "num_attention_heads = 4"),
+ (QWEN3, {"num_attention_heads": 6, "num_key_value_heads": 3}, 2, "num_key_value_heads = 3"),
+ (GEMMA3, {}, 2, "larger than num_key_value_heads = 1"),
+ (MISTRAL, {}, 2, "intermediate_size = 37"),
+ (OLMO2, {}, 2, "normalizes .* features, not one head"),
+ (QWEN3_MOE, {}, 2, "mixture-of-experts"),
+ (DEEPSEEK_V3, {}, 2, "mixture-of-experts|latent attention"),
+ ],
+)
+def test_refusals_plan(repo, overrides, tp, match):
+ model = build(repo, overrides, lora = False)
+ with pytest.raises(ValueError, match = match):
+ tpn.derive_tp_plan(model, tp)
+
+
+def test_refuses_attention_and_layer_dropout():
+ with pytest.raises(ValueError, match = "attention dropout"):
+ tpn.derive_tp_plan(build(LLAMA, {"attention_dropout": 0.1}), 2)
+ model = build(LLAMA)
+ model.get_base_model().model.layers[0].mlp.add_module("drop", torch.nn.Dropout(0.1))
+ with pytest.raises(ValueError, match = "dropout inside decoder layers"):
+ tpn.derive_tp_plan(model, 2)
+
+
+def test_refuses_dora_and_lora_bias():
+ for kwargs, match in (
+ ({"use_dora": True}, "DoRA"),
+ ({"lora_bias": True}, "lora_bias"),
+ ({"lora_dropout": 0.1}, "lora_dropout"),
+ ):
+ model = build(LLAMA, **kwargs)
+ region = tpn.derive_tp_plan(model, 2).regions[0]
+ with pytest.raises(ValueError, match = match):
+ tpn._refuse_adapter(region.path, region.projs[0].module)
+
+
+def test_refuses_quantized_and_trainable_bases():
+ model = build(LLAMA)
+ region = tpn.derive_tp_plan(model, 2).regions[0]
+ proj = region.projs[0].module
+ weight, base = tpn._weight(proj)
+ with pytest.raises(ValueError, match = "fp8"):
+ tpn._refuse_base("p", proj, weight.to(torch.float8_e4m3fn), base)
+ int8 = type("Int8Params", (torch.nn.Parameter,), {})
+ with pytest.raises(ValueError, match = "8-bit"):
+ tpn._refuse_base("p", proj, int8(weight.detach()), base)
+ gptq = type("QuantLinear", (torch.nn.Module,), {})()
+ with pytest.raises(ValueError, match = "GPTQ"):
+ tpn._refuse_base("p", proj, weight, gptq)
+ with pytest.raises(ValueError, match = "full finetuning"):
+ tpn._refuse_base("p", proj, torch.nn.Parameter(weight.detach().clone()), base)
+
+
+def test_refuses_global_width_attention():
+ model = build(LLAMA, lora = False)
+ attn = model.model.layers[0].self_attn
+
+ def forward(self, hidden_states, **kwargs):
+ return hidden_states.reshape(1, 1, self.config.hidden_size), None
+
+ attn.forward = types.MethodType(forward, attn)
+ with pytest.raises(ValueError, match = "hidden_size"):
+ tpn.derive_tp_plan(model, 2)
+
+
+def test_stablelm_4x_is_refused():
+ import transformers
+
+ if int(transformers.__version__.split(".")[0]) >= 5:
+ pytest.skip("transformers 5 StableLM attention follows the local heads")
+ repo, overrides = ARCHS["stablelm"]
+ with pytest.raises(ValueError, match = "reshapes with hidden_size"):
+ tpn.derive_tp_plan(build(repo, overrides), 2)
+
+
+# Phi3's fused qkv under a plan that calls it packed: the layout is still [q, kv, kv] (GQA: unequal parts).
+PHI3_PACKED_PLAN = {
+ "layers.*.self_attn.qkv_proj": "packed_colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.gate_up_proj": "packed_colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+}
+
+
+def test_fused_qkv_under_a_packed_plan_keeps_the_gqa_layout():
+ model = build(PHI3)
+ text = tpn._text_config(model)
+ assert text.num_key_value_heads < text.num_attention_heads # the tiny Phi3 is GQA (4 / 2)
+ hd = text.hidden_size // text.num_attention_heads
+ q, kv = text.num_attention_heads * hd, text.num_key_value_heads * hd
+ ids = _batch(model)
+ _, ref_logits = _loss(model, ids)
+ sim, plan = simulate(model, 2, PHI3_PACKED_PLAN)
+ parts = {p.name: p.parts for p in plan.regions[0].projs}
+ assert parts["qkv_proj"] == (q, kv, kv), parts
+ _, logits = _loss(sim, ids)
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ with pytest.raises(ValueError, match = "has no known layout"):
+ tpn.derive_tp_plan(
+ build(LLAMA, lora = False), 2, {"layers.*.self_attn.q_proj": "packed_colwise"}
+ )
+
+
+class _FakeLora(torch.nn.Module):
+ def __init__(self, active):
+ super().__init__()
+ self.base_layer = torch.nn.Linear(4, 6, bias = False)
+ self.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})
+ self.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 6, bias = False)})
+ self.scaling = {"default": 2.0}
+ self.disable_adapters = False
+ self.merged = False
+ if active is not None:
+ self.active_adapters = active
+
+
+def test_active_adapter_matches_get_lora_parameters():
+ singular = _FakeLora(None)
+ singular.active_adapter = "default"
+ assert tpn._active_adapter(singular) == ["default"]
+ assert tpn._active_adapter(_FakeLora(None)) == ["default"]
+ assert tpn._active_adapter(_FakeLora("default")) == ["default"]
+ assert tpn._active_adapter(_FakeLora([])) == []
+ for active in (None, "default", ["default"]):
+ W, _, A, B, s = tpn._tp_lora_parameters(_FakeLora(active))
+ assert A is not None and B is not None and s == 2.0
+ for active in ([], ["other"]):
+ assert tpn._tp_lora_parameters(_FakeLora(active))[2:] == (None, None, None)
+ with pytest.raises(ValueError, match = "single active adapter"):
+ tpn._tp_lora_parameters(_FakeLora(["default", "other"]))
+
+
+def test_refuses_forward_qat_and_activation_fake_quantizers():
+ model = build(LLAMA)
+ region = tpn.derive_tp_plan(model, 2).regions[0]
+ proj = region.projs[0].module
+ weight, base = tpn._weight(proj)
+ # torchao's NVFP4 / MX QAT linears fake-quantize inside forward() and carry no weight_fake_quantizer.
+ forward_qat = type("NVFP4FakeQuantizedLinear", (torch.nn.Linear,), {})(4, 4).requires_grad_(
+ False
+ )
+ with pytest.raises(ValueError, match = "QAT fake quantization"):
+ tpn._refuse_base(
+ "p", torch.nn.ModuleDict({"base_layer": forward_qat}), forward_qat.weight, forward_qat
+ )
+ try:
+ from torchao.prototype import qat
+ real_classes = [
+ (qat.MXFakeQuantizedLinear, qat.MXFakeQuantizeConfig),
+ (qat.NVFP4FakeQuantizedLinear, qat.NVFP4FakeQuantizeConfig),
+ ]
+ except Exception:
+ real_classes = []
+ for cls, cfg in real_classes: # the real torchao classes (weights frozen, as under LoRA QAT)
+ real = cls(32, 32, activation_config = cfg(), weight_config = cfg()).requires_grad_(False)
+ assert getattr(real, "weight_fake_quantizer", None) is None
+ with pytest.raises(ValueError, match = "QAT fake quantization"):
+ tpn._refuse_base("p", torch.nn.ModuleDict({"base_layer": real}), real.weight, real)
+ proj.lora_A["default"].activation_fake_quantizer = torch.nn.Identity()
+ with pytest.raises(ValueError, match = "QAT fake quantization"):
+ tpn._refuse_base("p", proj, weight, base)
+ del proj.lora_A["default"].activation_fake_quantizer
+ tpn._refuse_base("p", proj, weight, base) # plain LoRA passes
+
+
+def test_refuses_modules_to_save_inside_a_region():
+ model = build(
+ LLAMA,
+ target_modules = ["k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+ modules_to_save = ["q_proj"],
+ )
+ region = tpn.derive_tp_plan(model, 2).regions[0]
+ q = next(p for p in region.projs if p.name == "q_proj")
+ with pytest.raises(ValueError, match = "modules_to_save"):
+ tpn._refuse_adapter(f"{region.path}.q_proj", q.module)
+
+
+def test_region_hook_takes_the_forward_input_by_name():
+ model = build(LLAMA, lora = False)
+ mlp = model.model.layers[0].mlp
+ mlp._unsloth_tp_group = (
+ None # copy_to_tp's forward is the identity; the group is read in backward
+ )
+ mlp._unsloth_tp_input = "x" # what apply_tensor_parallel records for LlamaMLP.forward(self, x)
+ h = torch.randn(1, 2, 16)
+ args, kwargs = tpn._region_pre_hook(mlp, (), {"x": h}) # mlp(x = h)
+ assert args == () and torch.equal(kwargs["x"], h) and kwargs["x"] is not h
+ attn = model.model.layers[0].self_attn
+ assert tpn._input_name(mlp) == "x" and tpn._input_name(attn) == "hidden_states"
+ args, kwargs = tpn._region_pre_hook(mlp, (h,), {})
+ assert torch.equal(args[0], h)
+ with pytest.raises(RuntimeError, match = "got no tensor input"):
+ tpn._region_pre_hook(mlp, (), {"y": h})
+
+
+def test_clip_takes_a_single_parameter():
+ a = torch.nn.Parameter(torch.randn(5, 3))
+ b = torch.nn.Parameter(a.detach().clone())
+ a.grad = torch.randn(5, 3) * 4
+ b.grad = a.grad.clone()
+ ref = torch.nn.utils.clip_grad_norm_(b, 1.0)
+ got = tpn.tp_clip_grad_norm_(a, 1.0)
+ torch.testing.assert_close(got, ref)
+ torch.testing.assert_close(a.grad, b.grad)
+
+
+def _load_flex():
+ name = "_tp_flex_attention"
+ if name not in sys.modules:
+ spec = importlib.util.spec_from_file_location(
+ name, os.path.join(REPO, "unsloth", "kernels", "flex_attention.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ sys.modules[name] = module
+ return sys.modules[name]
+
+
+def test_gemma2_softcapping_helpers_use_local_heads():
+ """Gemma2 softcapping helpers honour _unsloth_tp_heads over the config's global counts."""
+ flex = _load_flex()
+ import types as _types
+
+ def attn(
+ heads,
+ kv,
+ tp_heads = None,
+ tp_kv = None,
+ ):
+ config = _types.SimpleNamespace(
+ num_attention_heads = heads,
+ num_key_value_heads = kv,
+ query_pre_attn_scalar = 16,
+ attn_logit_softcapping = 50.0,
+ )
+ m = torch.nn.Module()
+ m.config, m.head_dim, m.num_key_value_groups = config, 8, heads // kv
+ if tp_heads is not None:
+ m._unsloth_tp_heads, m._unsloth_tp_kv_heads = tp_heads, tp_kv
+ return m
+
+ bsz, q_len = 2, 5
+ g = torch.Generator().manual_seed(0)
+ Q = torch.randn(bsz, 2, q_len, 8, generator = g)
+ K = torch.randn(bsz, 1, q_len, 8, generator = g)
+ V = torch.randn(bsz, 1, q_len, 8, generator = g)
+ mask = torch.triu(torch.full((q_len, q_len), float("-inf")), 1)
+ local, tp_rank = attn(2, 1), attn(4, 2, 2, 1)
+ eager = flex._compiled_slow_attention_softcapping._torchdynamo_orig_callable
+ for fn in (eager, flex.slow_inference_attention_softcapping):
+ want = fn(Q, K, V, mask, local, bsz, q_len)
+ got = fn(Q, K, V, mask, tp_rank, bsz, q_len)
+ assert got.shape == (bsz, q_len, 2 * 8)
+ torch.testing.assert_close(got, want)
+
+
+def test_stub_packages_keep_valid_specs():
+ # Later tests in the same session (e.g. the fused-loss harness) call find_spec("unsloth").
+ for name in ("unsloth", "unsloth.models"):
+ assert sys.modules[name].__spec__ is not None, name
+ importlib.util.find_spec("unsloth.models")
diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py
index 45b91164bb9..1560c9f3bb2 100644
--- a/unsloth/kernels/flex_attention.py
+++ b/unsloth/kernels/flex_attention.py
@@ -38,9 +38,10 @@ if not HAS_FLEX_ATTENTION:
# Logit softcapping
@torch.compile(fullgraph = True, dynamic = True, options = torch_compile_options)
def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):
- n_heads = self.config.num_attention_heads
+ # This rank's heads under tensor parallelism (models/tp_native.py); the config counts otherwise.
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
head_dim = self.head_dim
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
n_groups = self.num_key_value_groups
actual_q_len = Q.shape[-2]
@@ -90,7 +91,15 @@ if not HAS_FLEX_ATTENTION:
def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):
# Inductor indexes the scores in int32; past 2**31 elements the last rows come back NaN.
- rows = max(1, (2**31 - 1) // (self.config.num_attention_heads * q_len * q_len))
+ rows = max(
+ 1,
+ (2**31 - 1)
+ // (
+ self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
+ * q_len
+ * q_len
+ ),
+ )
if bsz <= rows:
return _softcapping_attention(Q, K, V, causal_mask, self, bsz, q_len)
per_row_mask = causal_mask.dim() == 4 and causal_mask.shape[0] == bsz
@@ -167,7 +176,7 @@ else:
)
def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
head_dim = self.head_dim
s = self.config.query_pre_attn_scalar
t = self.config.attn_logit_softcapping
@@ -184,9 +193,9 @@ torch_nn_functional_softmax = torch.nn.functional.softmax
def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
head_dim = self.head_dim
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
n_groups = self.num_key_value_groups
actual_q_len = Q.shape[-2]
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 94c812ac32b..9f7612b6db3 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -445,7 +445,11 @@ def _is_packed_state(quant_state):
def get_lora_parameters(proj):
"""Return (weight, weight quant_state, lora A, lora B, lora scale).
With QAT enabled, also fake-quantizes the base layer and lora weights.
+ Tensor parallel projections (models/tp_native.py) return local shards of their DTensors.
"""
+ localize = proj.__dict__.get("_unsloth_tp_local")
+ if localize is not None:
+ return localize(proj)
# For DPO or disabled adapters.
base_layer = getattr(proj, "base_layer", proj)
packed = _packed_base(base_layer)
@@ -513,6 +517,9 @@ def get_lora_parameters(proj):
def get_lora_parameters_bias(proj):
+ localize = proj.__dict__.get("_unsloth_tp_local")
+ if localize is not None:
+ return localize(proj, bias = True)
# For DPO or disabled adapters.
base_layer = getattr(proj, "base_layer", proj)
packed = _packed_base(base_layer)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 11f2f824dd5..e735910a6f7 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5698,6 +5698,7 @@ def patch_gradient_accumulation_fix(Trainer):
install_fsdp2_policy_provenance()
except Exception:
pass
+ _refuse_tensor_parallel_for_trainer(model)
from ..fsdp2_launch import unsloth_fsdp2_native_requested
# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.
@@ -5772,6 +5773,7 @@ def patch_gradient_accumulation_fix(Trainer):
# model_init rebuilds the model when training starts: the new instance needs the same
# DDP exclusions as the one __init__ marked.
model = _original_call_model_init(self, *args, **kwargs)
+ _refuse_tensor_parallel_for_trainer(model)
_tensor_subclass_weights_for_ddp(self, model)
return model
@@ -5782,6 +5784,35 @@ def patch_gradient_accumulation_fix(Trainer):
_patch_trainer_train_gc_scope(Trainer)
+def _is_unsloth_tensor_parallel(model):
+ """Walk the wrapper levels for apply_tensor_parallel's marker (dict lookups, no tp_native import)."""
+ m, depth = model, 0
+ while isinstance(getattr(m, "__dict__", None), dict) and depth < 8:
+ if m.__dict__.get("_unsloth_tp"):
+ return True
+ modules = m.__dict__.get("_modules") or {}
+ m, depth = modules.get("base_model") or modules.get("model"), depth + 1
+ return False
+
+
+def _refuse_tensor_parallel_for_trainer(model):
+ if _is_unsloth_tensor_parallel(model):
+ raise RuntimeError(
+ "Unsloth: a tensor parallel model (apply_tensor_parallel) cannot be trained by the "
+ "Trainer yet: accelerate would wrap the tensor parallel ranks in DDP. Use a custom "
+ "training loop (unsloth/models/tp_native.py)."
+ )
+
+
+def _refuse_tensor_parallel_for_inference(model):
+ if _is_unsloth_tensor_parallel(model):
+ raise RuntimeError(
+ "Unsloth: for_inference is not supported on a tensor parallel model in v1 (the fast decode "
+ "paths run outside the tensor parallel regions); use model.eval() for forward evaluation, or "
+ "save the adapter (save_tp_adapter) and generate on one GPU."
+ )
+
+
# Real children only: PEFT's LoraModel forwards `base_model` to the backbone, skipping the loss head.
_LOSS_KWARGS_CHILDREN = ("base_model", "model") + _UNSLOTH_WRAPPED_MODULE_ATTRS
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index d96a6f01224..425324b8331 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -109,9 +109,9 @@ def CohereAttention_fast_forward(
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
assert n_kv_heads * n_groups == n_heads
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index de94b144e33..ff07b6f209d 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -3461,6 +3461,16 @@ def _refuse_tensor_parallel_model(model):
except ImportError:
pass
dtensor = DTensor is not None and any(isinstance(p, DTensor) for p in model.parameters())
+ # Any level or module (the marker sits on every wrapper level apply_tensor_parallel was given or below).
+ if any(
+ m.__dict__.get("_unsloth_tp") or m.__dict__.get("_unsloth_tp_style")
+ for m in model.modules()
+ ):
+ raise RuntimeError(
+ "Unsloth: native FSDP2 does not compose with Unsloth tensor parallelism "
+ "(apply_tensor_parallel) yet; train a tensor parallel model with a custom loop over the tp "
+ "ranks only (data parallel size 1)."
+ )
if (tp_size or 1) > 1 or dtensor:
raise RuntimeError(
"Unsloth: native FSDP2 got a model that is already distributed (tensor-parallel tp_plan, "
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index b5240b2470f..68c76a224b3 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -157,9 +157,9 @@ def Gemma2Attention_fast_forward(
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
assert n_kv_heads * n_groups == n_heads
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 2fea7d7c85d..e280ee90eba 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -106,9 +106,9 @@ def GraniteAttention_fast_forward(
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
dropout_p = self.config.attention_dropout if self.training else 0
assert n_kv_heads * n_groups == n_heads
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 4b5206dee99..277e06e8daa 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -22,6 +22,7 @@ from typing import Optional, Tuple, List, Union
from ._utils import *
from ._utils import apply_unsloth_gradient_checkpointing
from ._utils import _ignore_block_swap_params_for_ddp
+from ._utils import _refuse_tensor_parallel_for_inference
from ._utils import __version__, importlib_version
from ._utils import move_to_device
from ._utils import per_layer_device
@@ -996,9 +997,9 @@ def LlamaAttention_fast_forward(
del self.attention
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
assert n_kv_heads * n_groups == n_heads
@@ -5345,6 +5346,7 @@ class FastLlamaModel:
raise TypeError(
"Unsloth: I think you're passing a tokenizer, not the model to for_inference!"
)
+ _refuse_tensor_parallel_for_inference(model)
def _for_inference(m):
if hasattr(m, "gradient_checkpointing"):
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 812818f2a00..a8d749818cb 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -87,9 +87,9 @@ def MistralAttention_fast_forward(
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
assert n_kv_heads * n_groups == n_heads
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index cef4a20b2f8..3392c438f63 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -91,9 +91,9 @@ def Qwen3Attention_fast_forward(
bsz, q_len, _ = hidden_states.size()
- n_heads = self.config.num_attention_heads
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
n_groups = self.num_key_value_groups
- n_kv_heads = self.config.num_key_value_heads
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
head_dim = self.head_dim
assert n_kv_heads * n_groups == n_heads
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
new file mode 100644
index 00000000000..0d97139218a
--- /dev/null
+++ b/unsloth/models/tp_native.py
@@ -0,0 +1,1341 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallelism (column / row sharding of attention and MLP) for any dense decoder, on DTensor.
+
+ model = FastLanguageModel.get_peft_model(model, ...) # LoRA first: same init as 1 GPU
+ apply_tensor_parallel(model, init_device_mesh("cuda", (tp,), mesh_dim_names = ("tp",))["tp"])
+
+v1: LoRA / QLoRA training forward on dense decoders, tp ranks only (no Trainer / FSDP2 / generate).
+Custom loop (torchrun --nproc_per_node tp):
+
+ params = [p for p in model.parameters() if p.requires_grad] # DTensors (keep plain ones in their own group)
+ optimizer = torch.optim.AdamW(params, lr = 2e-4)
+ loss = model(input_ids = ids, labels = ids).loss # same batch on every tp rank
+ loss.backward()
+ tp_clip_grad_norm_(params, 1.0)
+ optimizer.step(); optimizer.zero_grad()
+ save_tp_adapter(model, "out") # collective: the standard 1-GPU PEFT adapter, written by rank 0
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses
+import inspect
+import re
+import types
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+__all__ = [
+ "TPConfig",
+ "TPPlan",
+ "apply_tensor_parallel",
+ "derive_tp_plan",
+ "is_tensor_parallel",
+ "shard_slice",
+ "tp_clip_grad_norm_",
+ "tp_full_state_dict",
+ "save_tp_adapter",
+]
+
+_TP_FLAG = "_unsloth_tp"
+COLWISE = "colwise"
+ROWWISE = "rowwise"
+HEAD_NORM = "replicated_with_grad_allreduce"
+
+# transformers TP styles (4.5x and 5.x names) -> what this module does with them.
+_STYLES = {
+ "colwise": COLWISE,
+ "local_colwise": COLWISE,
+ "colwise_rep": COLWISE,
+ "colwise_gather_output": COLWISE,
+ "packed_colwise": COLWISE,
+ "local_packed_colwise": COLWISE,
+ "rowwise": ROWWISE,
+ "local_rowwise": ROWWISE,
+ "rowwise_rep": ROWWISE,
+ "rowwise_split_input": ROWWISE,
+ "replicated_with_grad_allreduce": HEAD_NORM,
+}
+# Outside the decoder regions (embeddings, lm_head, final norm): replicated in v1.
+_REPLICATED_STYLES = {
+ "embedding_rowwise",
+ "gather",
+ "sequence_parallel",
+ "all_reduce",
+ "replicate",
+ "isolated",
+}
+_MOE_STYLES = {
+ "grouped_gemm",
+ "ep_router",
+ "megamoe_router",
+ "moe_tp_experts",
+ "megamoe_experts",
+ "moe_identity_expert",
+ "packed_rowwise",
+ "local_packed_rowwise",
+}
+_MLA_STYLES = {"mla_kv_a_proj"}
+
+_ATTN_NAMES = ("self_attn", "attention", "attn", "self_attention")
+_MLP_NAMES = ("mlp", "feed_forward", "ffn")
+_Q, _K, _V = ("q_proj", "query"), ("k_proj", "key"), ("v_proj", "value")
+_FUSED_QKV = ("qkv_proj",) # [q, k, v] along the output dim (Phi3)
+_ATTN_ROW = ("o_proj", "out_proj", "dense")
+_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc")
+_MLP_PACKED = ("gate_up_proj",) # [gate, up] along the output dim
+_MLP_ROW = ("down_proj", "w2", "fc2")
+_HEAD_NORMS = ("q_norm", "k_norm", "query_norm", "key_norm")
+_MLA_NAMES = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "kv_a_layernorm")
+_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
+# An attention forward reading these sizes off the config would reshape to the global head count.
+_GLOBAL_HEAD_READS = re.compile(r"config\.(num_attention_heads|num_key_value_heads)\b")
+_GLOBAL_WIDTH_READS = re.compile(r"(config|self)\.hidden_size\b")
+
+
+class TPConfig:
+ # {"layers.*.self_attn.q_proj": "colwise", ...}; None = the model's TP plan, else structural discovery.
+ plan: Optional[dict] = None
+ # Broadcast trainable parameters from tp rank 0 first (every rank must start from the same adapter).
+ sync_trainable: bool = True
+
+
+class _Proj:
+ name: str
+ module: nn.Module
+ style: str # COLWISE / ROWWISE
+ parts: tuple # output (colwise) part sizes; one part unless fused / packed
+ from_plan: bool = False
+
+
+class _Region:
+ path: str
+ module: nn.Module
+ kind: str # "attention" / "mlp"
+ projs: list
+ norms: list # per-head norms inside the region: replicated, gradients summed over tp
+ heads: int = 0
+ kv_heads: int = 0
+ head_dim: int = 0
+
+
+class TPPlan:
+ tp_size: int
+ regions: list
+ source: str # "user" / "transformers" / "structural"
+ replicated_layers: int = (
+ 0 # decoder layers with no region (e.g. linear attention): run replicated
+ )
+
+ def styles(self):
+ """{projection path: style} (the plan in transformers form, local paths)."""
+ return {f"{r.path}.{p.name}": p.style for r in self.regions for p in r.projs}
+
+
+def _unwrap(model):
+ inner = model
+ if hasattr(inner, "get_base_model"):
+ inner = inner.get_base_model()
+ return inner
+
+
+def _text_config(model):
+ config = getattr(_unwrap(model), "config", None)
+ if config is None:
+ return None
+ get = getattr(config, "get_text_config", None)
+ return get() if callable(get) else config
+
+
+def _hf_plan(model):
+ inner = _unwrap(model)
+ plan = {}
+ text = _text_config(model)
+ for source in (getattr(text, "base_model_tp_plan", None), getattr(inner, "_tp_plan", None)):
+ if isinstance(source, dict):
+ plan.update(source)
+ return plan
+
+
+def _pattern(key):
+ return re.compile(r"(^|\.)" + re.escape(key).replace(r"\*", r"\d+") + r"$")
+
+
+def _style_for(path, compiled):
+ for key, regex, style in compiled:
+ if regex.search(path):
+ return key, style
+ return None, None
+
+
+def _weight(module):
+ base = module._modules.get("base_layer", module)
+ return base._parameters.get("weight", getattr(base, "weight", None)), base
+
+
+def _has_params(module):
+ return next(module.parameters(), None) is not None
+
+
+def _decoder_layers(model):
+ """The ModuleList of decoder layers (len == text config num_hidden_layers)."""
+ inner = _unwrap(model)
+ decoder = inner
+ get_decoder = getattr(inner, "get_decoder", None)
+ if callable(get_decoder):
+ try:
+ decoder = get_decoder() or inner
+ except Exception:
+ decoder = inner
+ text = _text_config(model)
+ n = getattr(text, "num_hidden_layers", None)
+ prefix = ""
+ for name, module in inner.named_modules():
+ if module is decoder:
+ prefix = name
+ break
+ for name, module in decoder.named_modules():
+ if isinstance(module, nn.ModuleList) and len(module) and (n is None or len(module) == n):
+ path = ".".join(x for x in (prefix, name) if x)
+ return path, module
+ raise ValueError(
+ "Unsloth: tensor parallelism found no decoder layer list "
+ f"({type(inner).__name__}: no ModuleList of num_hidden_layers = {n} layers)."
+ )
+
+
+_ATTENTION_DROPOUT_KEYS = ("attention_dropout", "attn_pdrop")
+
+
+def _refuse_dropout(model, layers):
+ """v1 has no TP-aware RNG (masks must differ per shard, match when replicated): refuse dropout."""
+ text = _text_config(model)
+ for key in _ATTENTION_DROPOUT_KEYS:
+ value = getattr(text, key, None)
+ if isinstance(value, (int, float)) and value > 0:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support attention dropout ({key} = {value}); set it to 0."
+ )
+ for name, module in layers.named_modules():
+ if "lora_dropout" in name or not isinstance(module, nn.modules.dropout._DropoutNd):
+ continue
+ if module.p > 0:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support dropout inside decoder layers ({name}: p = "
+ f"{module.p}); set it to 0."
+ )
+
+
+def _refuse_moe_mla(model, layers):
+ text = _text_config(model)
+ for key in _MOE_CONFIG_KEYS:
+ value = getattr(text, key, None)
+ if isinstance(value, int) and value > 0:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts models ({key} = {value}): "
+ "expert sharding is not implemented. Use native FSDP2 (UNSLOTH_FSDP2=1) instead."
+ )
+ if getattr(text, "kv_lora_rank", None):
+ raise ValueError(
+ "Unsloth: tensor parallelism v1 does not support multi-head latent attention (MLA, kv_lora_rank "
+ f"= {text.kv_lora_rank}). Use native FSDP2 (UNSLOTH_FSDP2=1) instead."
+ )
+ for name, module in layers.named_modules():
+ leaf = name.rsplit(".", 1)[-1]
+ cls = type(module).__name__
+ if leaf == "experts" or "Experts" in cls or "SparseMoe" in cls or "MoE" in cls:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ({name}: {cls})."
+ )
+ if leaf in _MLA_NAMES:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ({name})."
+ )
+
+
+def _classify(region_path, name, child, kind, compiled):
+ """(style, parts kind) for one child of an attention / MLP region, from the plan or its name."""
+ key, style = _style_for(f"{region_path}.{name}", compiled) if compiled else (None, None)
+ if style is not None:
+ if style in _MOE_STYLES:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support the MoE style '{style}' ({key})."
+ )
+ if style in _MLA_STYLES:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support the MLA style '{style}' ({key})."
+ )
+ if style in _REPLICATED_STYLES:
+ raise ValueError(
+ f"Unsloth: '{key}': '{style}' inside a decoder {kind} is not supported by tensor parallelism v1."
+ )
+ if style not in _STYLES:
+ raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")
+ raw, style = style, _STYLES[style]
+ # A fused qkv is [q, kv, kv] (GQA: unequal parts) whatever the plan calls it; "packed" means equal halves.
+ if style == COLWISE and kind == "attention" and name in _FUSED_QKV:
+ return COLWISE, "qkv:plan"
+ if style == COLWISE and (raw.endswith("packed_colwise") or name in _MLP_PACKED):
+ if kind == "attention":
+ raise ValueError(
+ f"Unsloth: '{key}': '{raw}' on the attention projection '{name}' has no known layout "
+ "(tensor parallelism v1 knows fused qkv_proj); pass plan = {...} with 'colwise'."
+ )
+ return COLWISE, "packed:plan"
+ return style, "plan"
+ if kind == "attention":
+ if name in _Q + _K + _V:
+ return COLWISE, None
+ if name in _FUSED_QKV:
+ return COLWISE, "qkv"
+ if name in _ATTN_ROW:
+ return ROWWISE, None
+ if name in _HEAD_NORMS:
+ return HEAD_NORM, None
+ else:
+ if name in _MLP_COL:
+ return COLWISE, None
+ if name in _MLP_PACKED:
+ return COLWISE, "packed"
+ if name in _MLP_ROW:
+ return ROWWISE, None
+ return None, None
+
+
+def _attention_sizes(module, projs, text):
+ """(heads, kv_heads, head_dim) of one attention module, from its own projections."""
+ heads_cfg = getattr(text, "num_attention_heads", None)
+ kv_cfg = getattr(text, "num_key_value_heads", None) or heads_cfg
+ head_dim = getattr(module, "head_dim", None)
+ if not isinstance(head_dim, int):
+ head_dim = getattr(text, "head_dim", None) or (text.hidden_size // heads_cfg)
+ by_name = {p.name: p for p in projs}
+ q = next((by_name[n] for n in _Q if n in by_name), None)
+ k = next((by_name[n] for n in _K if n in by_name), None)
+ if q is not None and k is not None:
+ heads = _logical_shape(*_weight(q.module))[0] // head_dim
+ kv_heads = _logical_shape(*_weight(k.module))[0] // head_dim
+ else:
+ heads, kv_heads = heads_cfg, kv_cfg
+ return heads, kv_heads, head_dim
+
+
+def _check_divisible(what, value, tp, path):
+ if value % tp != 0:
+ raise ValueError(
+ f"Unsloth: tensor parallel size {tp} does not divide {what} = {value} ({path}). "
+ f"Choose a tp size that divides it."
+ )
+
+
+def derive_tp_plan(
+ model,
+ tp_size,
+ plan = None,
+) -> TPPlan:
+ """The regions to shard, validated for tp_size; ValueError for anything v1 cannot shard."""
+ tp_size = int(tp_size)
+ if tp_size < 1:
+ raise ValueError(f"Unsloth: tensor parallel size must be >= 1, got {tp_size}.")
+ user = bool(plan)
+ plan = dict(plan) if plan else _hf_plan(model)
+ compiled = [(k, _pattern(k), v) for k, v in plan.items()]
+ layers_path, layers = _decoder_layers(model)
+ # Config / module MoE and MLA first: older transformers plans name expert styles v1 does not know.
+ _refuse_moe_mla(model, layers)
+ _refuse_dropout(model, layers)
+ for key, _, style in compiled:
+ if style in _MOE_STYLES:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ('{key}': '{style}')."
+ )
+ if style in _MLA_STYLES:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ('{key}')."
+ )
+ if style not in _STYLES and style not in _REPLICATED_STYLES:
+ raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")
+ text = _text_config(model)
+ regions, replicated = [], 0
+ for index, layer in enumerate(layers):
+ found = 0
+ for child_name, child in layer.named_children():
+ kind = (
+ "attention"
+ if child_name in _ATTN_NAMES
+ else ("mlp" if child_name in _MLP_NAMES else None)
+ )
+ if kind is None or not _has_params(child):
+ continue
+ path = f"{layers_path}.{index}.{child_name}"
+ regions.append(_region(path, child, kind, compiled, text, tp_size))
+ found += 1
+ replicated += found == 0
+ if not regions:
+ raise ValueError(
+ "Unsloth: tensor parallelism found no attention / MLP modules to shard in the decoder layers."
+ )
+ if user:
+ source = "user"
+ else:
+ source = (
+ "transformers" if any(p.from_plan for r in regions for p in r.projs) else "structural"
+ )
+ return TPPlan(tp_size = tp_size, regions = regions, source = source, replicated_layers = replicated)
+
+
+def _region(path, module, kind, compiled, text, tp):
+ projs, norms = [], []
+ for name, param in module.named_parameters(recurse = False):
+ raise ValueError(
+ f"Unsloth: tensor parallelism cannot shard '{path}': it owns the parameter '{name}' directly "
+ "(not inside a projection), which v1 has no rule for."
+ )
+ for name, child in module.named_children():
+ if not _has_params(child):
+ continue
+ style, packing = _classify(path, name, child, kind, compiled)
+ if style is None:
+ raise ValueError(
+ f"Unsloth: tensor parallelism has no rule for '{path}.{name}' ({type(child).__name__}); "
+ "every module with parameters inside a sharded attention / MLP must be in the plan. Pass "
+ "plan = {...} to apply_tensor_parallel."
+ )
+ if style == HEAD_NORM:
+ norms.append((name, child))
+ continue
+ weight, _ = _weight(child)
+ if weight is None or getattr(weight, "ndim", 0) != 2:
+ raise ValueError(
+ f"Unsloth: tensor parallelism expects a Linear at '{path}.{name}' "
+ f"(got {type(child).__name__})."
+ )
+ from_plan = packing is not None and packing.endswith("plan")
+ packing = packing.split(":")[0] if packing and packing != "plan" else None
+ projs.append(_Proj(name, child, style, packing, from_plan))
+ if not any(p.style == COLWISE for p in projs) or not any(p.style == ROWWISE for p in projs):
+ raise ValueError(
+ f"Unsloth: '{path}' needs column and row sharded projections for tensor parallelism; found "
+ f"{[(p.name, p.style) for p in projs]}."
+ )
+ region = _Region(path, module, kind, projs, norms)
+ if kind == "attention":
+ heads, kv_heads, head_dim = _attention_sizes(module, projs, text)
+ region.heads, region.kv_heads, region.head_dim = heads, kv_heads, head_dim
+ _check_divisible("num_attention_heads", heads, tp, path)
+ if kv_heads < tp:
+ raise ValueError(
+ f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} "
+ f"({path}); replicating KV heads is not supported in v1. Use tp <= {kv_heads}."
+ )
+ _check_divisible("num_key_value_heads", kv_heads, tp, path)
+ for name, norm in norms:
+ numel = sum(p.numel() for p in norm.parameters())
+ if numel != head_dim:
+ raise ValueError(
+ f"Unsloth: '{path}.{name}' normalizes {numel} features, not one head ({head_dim}): a norm "
+ "across all heads needs the full projection, which tensor parallelism v1 does not gather."
+ )
+ _check_attention_source(module, path)
+ elif norms:
+ raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")
+ for proj in projs:
+ weight, base = _weight(proj.module)
+ out_features, in_features = _logical_shape(weight, base)
+ if proj.style == ROWWISE:
+ _check_divisible(f"{proj.name} in_features", in_features, tp, path)
+ proj.parts = (out_features,)
+ continue
+ if proj.parts == "qkv":
+ q, kv = region.heads * region.head_dim, region.kv_heads * region.head_dim
+ if q + 2 * kv != out_features:
+ raise ValueError(
+ f"Unsloth: fused '{path}.{proj.name}' has {out_features} outputs, not q + 2 kv = "
+ f"{q + 2 * kv}; its layout is unknown to tensor parallelism v1."
+ )
+ proj.parts = (q, kv, kv)
+ elif proj.parts == "packed":
+ if out_features % 2:
+ raise ValueError(f"Unsloth: packed '{path}.{proj.name}' has an odd output size.")
+ proj.parts = (out_features // 2, out_features // 2)
+ else:
+ proj.parts = (out_features,)
+ for size in proj.parts:
+ what = "intermediate_size" if kind == "mlp" else f"{proj.name} out_features"
+ _check_divisible(what, size, tp, path)
+ return region
+
+
+def _logical_shape(weight, base):
+ state = getattr(weight, "quant_state", None)
+ if state is not None and getattr(state, "shape", None) is not None:
+ return tuple(state.shape)
+ return tuple(weight.shape)
+
+
+def _forward_source(module):
+ forward = module.forward
+ for _ in range(4):
+ forward = getattr(forward, "__func__", forward)
+ forward = inspect.unwrap(forward)
+ # torch.compile'd forwards (unsloth_zoo temporary patches) keep the original here.
+ inner = getattr(forward, "_torchdynamo_orig_callable", None)
+ if inner is None:
+ break
+ forward = inner
+ return inspect.getsource(forward)
+
+
+def _check_attention_source(module, path):
+ try:
+ source = _forward_source(module)
+ except (OSError, TypeError):
+ raise ValueError(
+ f"Unsloth: tensor parallelism cannot read the forward of '{path}' ({type(module).__name__}) to "
+ "check it uses this rank's head count."
+ )
+ if _GLOBAL_WIDTH_READS.search(source) and "_unsloth_tp_heads" not in source:
+ raise ValueError(
+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with hidden_size, "
+ "the global width; tensor parallelism v1 needs it to follow the local heads."
+ )
+
+
+def shard_slice(tensor, dim, parts, rank, tp):
+ """This rank's slice of `tensor` along `dim`: the rank-th 1/tp of every part, concatenated (a copy)."""
+ pieces, offset = [], 0
+ for size in parts:
+ step = size // tp
+ pieces.append(tensor.narrow(dim, offset + rank * step, step))
+ offset += size
+ return torch.cat(pieces, dim = dim)
+
+
+def _unpermute(tensor, dim, parts, tp):
+ """Inverse of concatenating every rank's shard_slice: the original part order."""
+ if len(parts) == 1:
+ return tensor
+ local = sum(parts) // tp
+ chunks = tensor.split(local, dim = dim) # one per rank
+ pieces = []
+ for i, size in enumerate(parts):
+ step = size // tp
+ start = sum(p // tp for p in parts[:i])
+ pieces.extend(chunk.narrow(dim, start, step) for chunk in chunks)
+ return torch.cat(pieces, dim = dim)
+
+
+def _dtensor_api():
+ from torch.distributed.tensor import DTensor, Partial, Replicate, Shard
+ return DTensor, Partial, Replicate, Shard
+
+
+def _is_dtensor(t):
+ try:
+ from torch.distributed.tensor import DTensor
+ except ImportError:
+ return False
+ return isinstance(t, DTensor)
+
+
+def _local_complete(t):
+ """A local view whose gradient is already complete (e.g. row bias): never summed over tp again."""
+ DTensor, _, Replicate, _ = _dtensor_api()
+ if not isinstance(t, DTensor):
+ return t
+ if t.requires_grad and any(p.is_replicate() for p in t.placements):
+ return t.to_local(grad_placements = [Replicate()])
+ return t.to_local()
+
+
+def _local(t):
+ """A local view of a DTensor for local compute; replicated trainable tensors return Partial grads."""
+ DTensor, Partial, _, _ = _dtensor_api()
+ if not isinstance(t, DTensor):
+ return t
+ if t.requires_grad and any(p.is_replicate() for p in t.placements):
+ return t.to_local(grad_placements = [Partial()])
+ return t.to_local()
+
+
+def _tp_lora_parameters(proj, bias = False):
+ """get_lora_parameters(_bias) on local shards; row bias is None (the region adds it after the reduce)."""
+ weight, base = _weight(proj)
+ W = _local(weight)
+ W_quant = getattr(W, "quant_state", None)
+ b = None
+ if bias and proj.__dict__.get("_unsloth_tp_style") == COLWISE and base.bias is not None:
+ b = _local(base.bias)
+ lora_A = proj._modules.get("lora_A")
+ adapter = None
+ if lora_A is not None and not (
+ getattr(proj, "disable_adapters", True) or getattr(proj, "merged", False)
+ ):
+ adapters = _active_adapter(proj)
+ if len(adapters) > 1:
+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")
+ if adapters and adapters[0] in lora_A:
+ adapter = adapters[0]
+ if adapter is None:
+ out = (W, W_quant, None, None, None)
+ else:
+ out = (
+ W,
+ W_quant,
+ _local(lora_A[adapter].weight),
+ _local(proj.lora_B[adapter].weight),
+ proj.scaling[adapter],
+ )
+ return out + (b,) if bias else out
+
+
+def _replicate_grad(grad):
+ # The Partial gradient of a replicated factor becomes its tp sum here (one all-reduce per backward).
+ if _is_dtensor(grad) and any(p.is_partial() for p in grad.placements):
+ _, _, Replicate, _ = _dtensor_api()
+ return grad.redistribute(grad.device_mesh, [Replicate()])
+ return grad
+
+
+def _is_params4bit(t):
+ return any(cls.__name__ == "Params4bit" for cls in type(t).__mro__)
+
+
+def _refuse_base(path, proj_module, weight, base):
+ cls = type(base).__name__
+ if weight is None:
+ raise ValueError(f"Unsloth: '{path}' ({cls}) has no dense weight (GPTQ / AWQ?).")
+ if getattr(base, "qweight", None) is not None or cls.startswith(("QuantLinear", "WQLinear")):
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support GPTQ / AWQ weights ({path})."
+ )
+ if cls == "Linear8bitLt" or type(weight).__name__ == "Int8Params":
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support bitsandbytes 8-bit weights ({path}); load in "
+ "4-bit or 16-bit."
+ )
+ fp8 = {
+ getattr(torch, n, None)
+ for n in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")
+ }
+ if weight.dtype in fp8 or getattr(base, "weight_scale", None) is not None:
+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support fp8 weights ({path}).")
+ if getattr(type(base), "_unsloth_mxfp4_packed_linear", False):
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support packed MXFP4 weights ({path})."
+ )
+ for module in proj_module.modules():
+ forward_qat = any("FakeQuantized" in c.__name__ for c in type(module).__mro__)
+ if (
+ forward_qat
+ or getattr(module, "weight_fake_quantizer", None) is not None
+ or getattr(module, "activation_fake_quantizer", None) is not None
+ ):
+ # The TP projection forward and get_lora_parameters run plain matmuls on the local shards.
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support QAT fake quantization ({path}: "
+ f"{type(module).__name__})."
+ )
+ if not _is_params4bit(weight) and type(weight) not in (nn.Parameter, torch.Tensor):
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 cannot shard '{path}' weights of type {type(weight).__name__}."
+ )
+ if not _is_params4bit(weight) and not weight.is_floating_point():
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 cannot shard {weight.dtype} weights ({path})."
+ )
+ if weight.requires_grad:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 shards LoRA / QLoRA training only; '{path}' has a trainable base "
+ "weight (full finetuning). Use native FSDP2 for full finetuning."
+ )
+
+
+def _active_adapter(proj):
+ """Active adapter names, resolved like kernels/utils.py get_lora_parameters; empty stays empty."""
+ adapters = getattr(proj, "active_adapters", None)
+ if adapters is None:
+ adapters = getattr(proj, "active_adapter", "default")
+ if adapters is None:
+ return []
+ if isinstance(adapters, str):
+ return [adapters]
+ return list(adapters)
+
+
+def _refuse_adapter(path, proj):
+ if "original_module" in proj._modules or "modules_to_save" in proj._modules:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 cannot shard '{path}': it is a PEFT modules_to_save / trainable "
+ "copy (a full trainable projection). Keep modules_to_save outside the attention / MLP projections."
+ )
+ if "lora_A" not in proj._modules:
+ if "base_layer" in proj._modules:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 supports LoRA adapters only ({path}: {type(proj).__name__})."
+ )
+ return
+ names = list(proj.lora_A.keys())
+ if len(names) > 1:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 supports one LoRA adapter per layer ({path} has {names})."
+ )
+ magnitude = getattr(proj, "lora_magnitude_vector", None)
+ if magnitude is not None and len(magnitude):
+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support DoRA ({path}).")
+ variants = proj.__dict__.get("lora_variant")
+ if variants:
+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support LoRA variants ({path}).")
+ for name in names:
+ dropout = proj.lora_dropout[name] if "lora_dropout" in proj._modules else None
+ if getattr(dropout, "p", 0) > 0:
+ # Masks must match across ranks for replicated inputs and differ for sharded ones.
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support lora_dropout > 0 ({path}); use lora_dropout = 0."
+ )
+ if proj.lora_B[name].bias is not None:
+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support lora_bias ({path}).")
+ if type(proj.lora_A[name]) is not nn.Linear:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 supports Linear LoRA factors only ({path})."
+ )
+
+
+def _as_param(tensor, like):
+ param = nn.Parameter(tensor, requires_grad = like.requires_grad)
+ return param
+
+
+def _set_dtensor(
+ module,
+ name,
+ full,
+ mesh,
+ placement,
+ parts = None,
+ shard_dim = None,
+ rank = 0,
+ tp = 1,
+):
+ """Replace module.<name> by a DTensor parameter (Shard with parts permuted per rank, or Replicate)."""
+ DTensor, _, Replicate, Shard = _dtensor_api()
+ if placement == "shard":
+ local = shard_slice(full.detach(), shard_dim, parts, rank, tp)
+ shape = list(full.shape)
+ dt = DTensor.from_local(
+ local,
+ mesh,
+ [Shard(shard_dim)],
+ run_check = False,
+ shape = torch.Size(shape),
+ stride = torch.empty(shape, device = "meta").stride(),
+ )
+ else:
+ dt = DTensor.from_local(full.detach(), mesh, [Replicate()], run_check = False)
+ param = _as_param(dt, full)
+ if placement == "shard" and len(parts) > 1:
+ param._unsloth_tp_parts = (shard_dim, tuple(parts))
+ module._parameters[name] = param
+ if param.requires_grad and placement != "shard":
+ param.register_hook(_replicate_grad)
+ return param
+
+
+def _check_params4bit(path, weight, style, tp):
+ """The 4-bit blocksize refusals, run before anything is sharded."""
+ state = weight.quant_state
+ blocksize = state.blocksize
+ in_features = state.shape[1]
+ if in_features % blocksize or blocksize % 2:
+ raise ValueError(
+ f"Unsloth: 4-bit '{path}': in_features = {in_features} is not a multiple of the quantization "
+ f"blocksize {blocksize}; its blocks span rows, which tensor parallelism v1 cannot split."
+ )
+ if style == ROWWISE and (in_features // tp) % blocksize:
+ raise ValueError(
+ f"Unsloth: 4-bit '{path}': in_features / tp = {in_features // tp} is not a multiple of the "
+ f"quantization blocksize {blocksize}; row sharding would split blocks."
+ )
+
+
+def _shard_params4bit(path, base, weight, style, parts, rank, tp):
+ """This rank's 4-bit shard (bitwise slice of the 1-GPU weight); double-quant absmax kept as float32."""
+ import bitsandbytes as bnb
+
+ _check_params4bit(path, weight, style, tp)
+ state = weight.quant_state
+ blocksize = state.blocksize
+ out_features, in_features = state.shape
+ codes = weight.data.reshape(-1).view(torch.uint8).view(out_features, in_features // 2)
+ absmax = state.absmax
+ if state.nested:
+ absmax = bnb.functional.dequantize_blockwise(state.absmax, state.state2) + state.offset
+ absmax = absmax.float().view(out_features, in_features // blocksize)
+ if style == COLWISE:
+ codes = shard_slice(codes, 0, parts, rank, tp)
+ absmax = shard_slice(absmax, 0, parts, rank, tp)
+ shape = (sum(parts) // tp, in_features)
+ else:
+ codes = shard_slice(codes, 1, (in_features // 2,), rank, tp)
+ absmax = shard_slice(absmax, 1, (in_features // blocksize,), rank, tp)
+ shape = (out_features, in_features // tp)
+ storage = weight.data.dtype
+ packed = codes.contiguous().view(storage).reshape(-1, 1)
+ local_state = bnb.functional.QuantState(
+ absmax = absmax.contiguous(),
+ shape = torch.Size(shape),
+ code = state.code,
+ blocksize = blocksize,
+ quant_type = state.quant_type,
+ dtype = state.dtype,
+ )
+ new = bnb.nn.Params4bit(
+ packed,
+ requires_grad = False,
+ quant_state = local_state,
+ blocksize = blocksize,
+ compress_statistics = False,
+ quant_type = state.quant_type,
+ quant_storage = storage,
+ module = base,
+ bnb_quantized = True,
+ )
+ new._unsloth_tp_layout = (style, tuple(parts), rank, tp)
+ base._parameters["weight"] = new
+ if "quant_state" in base.__dict__:
+ base.quant_state = local_state
+
+
+def _all_reduce(tensor, group):
+ import torch.distributed._functional_collectives as funcol
+
+ out = funcol.all_reduce(tensor.contiguous(), "sum", group)
+ wait = getattr(out, "wait", None)
+ return wait() if callable(wait) else out
+
+
+class _CopyToTP(torch.autograd.Function):
+ """Identity forward, all-reduce backward: the region input every rank reads in full."""
+
+ @staticmethod
+ def forward(ctx, x, group):
+ ctx.group = group
+ return x.view_as(x)
+
+ @staticmethod
+ def backward(ctx, grad):
+ return _all_reduce(grad, ctx.group), None
+
+
+class _ReduceFromTP(torch.autograd.Function):
+ """All-reduce forward, identity backward: the row projection's partial sums."""
+
+ @staticmethod
+ def forward(ctx, x, group):
+ return _all_reduce(x, group)
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad, None
+
+
+def copy_to_tp(x, group):
+ return _CopyToTP.apply(x, group)
+
+
+def reduce_from_tp(x, group):
+ return _ReduceFromTP.apply(x, group)
+
+
+def _input_name(module):
+ """The name of the region forward's first argument (hidden_states, x, ...), for keyword calls."""
+ try:
+ params = list(inspect.signature(module.forward).parameters.values())
+ except (TypeError, ValueError):
+ return "hidden_states"
+ if params and params[0].kind in (params[0].POSITIONAL_ONLY, params[0].POSITIONAL_OR_KEYWORD):
+ return params[0].name
+ return "hidden_states"
+
+
+def _region_pre_hook(module, args, kwargs):
+ group = module._unsloth_tp_group
+ for name in (module.__dict__.get("_unsloth_tp_input", "hidden_states"), "hidden_states"):
+ if isinstance(kwargs.get(name), torch.Tensor):
+ kwargs = dict(kwargs)
+ kwargs[name] = copy_to_tp(kwargs[name], group)
+ return args, kwargs
+ if args and isinstance(args[0], torch.Tensor):
+ return (copy_to_tp(args[0], group),) + tuple(args[1:]), kwargs
+ raise RuntimeError(
+ f"Unsloth: tensor parallel region {type(module).__name__} got no tensor input "
+ f"({module.__dict__.get('_unsloth_tp_input', 'hidden_states')})."
+ )
+
+
+def _region_finish(module, y):
+ y = reduce_from_tp(y, module._unsloth_tp_group)
+ for proj in module._unsloth_tp_row_projs:
+ bias = _weight(proj)[1].bias
+ if bias is not None:
+ y = y + _local_complete(bias).to(y.dtype)
+ return y
+
+
+def _region_post_hook(module, args, output):
+ if isinstance(output, torch.Tensor):
+ return _region_finish(module, output)
+ if isinstance(output, (tuple, list)) and output and isinstance(output[0], torch.Tensor):
+ return type(output)((_region_finish(module, output[0]), *output[1:]))
+ raise RuntimeError(
+ f"Unsloth: tensor parallel region {type(module).__name__} returned {type(output)}."
+ )
+
+
+def _tp_projection_forward(self, x, *args, **kwargs):
+ weight, base = _weight(self)
+ W = _local(weight)
+ bias = (
+ base.bias if self._unsloth_tp_style == COLWISE else None
+ ) # row bias: once, after the reduce
+ bias = None if bias is None else _local(bias)
+ input_dtype = x.dtype
+ if _is_params4bit(W):
+ import bitsandbytes as bnb
+
+ compute = getattr(base, "compute_dtype", None) or x.dtype
+ out = bnb.matmul_4bit(x.to(compute), W.t(), bias = None, quant_state = W.quant_state)
+ if bias is not None:
+ out = out + bias.to(out.dtype)
+ out = out.to(input_dtype)
+ else:
+ out = F.linear(x, W, bias)
+ lora_A = self._modules.get("lora_A")
+ if lora_A is None or getattr(self, "disable_adapters", True) or getattr(self, "merged", False):
+ return out
+ for adapter in _active_adapter(self) or ():
+ if adapter not in lora_A:
+ continue
+ A = _local(lora_A[adapter].weight)
+ B = _local(self.lora_B[adapter].weight)
+ scaling = self.scaling[adapter]
+ x_lora = x if torch.is_autocast_enabled() else x.to(A.dtype)
+ x_lora = self.lora_dropout[adapter](x_lora)
+ out = out + (F.linear(F.linear(x_lora, A), B) * scaling).to(out.dtype)
+ return out
+
+
+def _decoder_of(inner):
+ get_decoder = getattr(inner, "get_decoder", None)
+ if not callable(get_decoder):
+ return None
+ try:
+ decoder = get_decoder()
+ except Exception:
+ return None
+ return decoder if isinstance(decoder, nn.Module) else None
+
+
+def _tp_chain(model):
+ """`model` and every wrapper level below it (PeftModel -> LoraModel -> CausalLM -> decoder)."""
+ chain, m = [], model
+ while isinstance(m, nn.Module) and all(m is not c for c in chain) and len(chain) < 8:
+ chain.append(m)
+ modules = m.__dict__.get("_modules") or {}
+ m = modules.get("base_model") or modules.get("model")
+ inner = _unwrap(model)
+ for extra in (inner, _decoder_of(inner)):
+ if isinstance(extra, nn.Module) and all(extra is not c for c in chain):
+ chain.append(extra)
+ return chain
+
+
+def is_tensor_parallel(model) -> bool:
+ """True when apply_tensor_parallel sharded `model`, a wrapper level below it, or any module inside it."""
+ if not isinstance(model, nn.Module):
+ return getattr(model, _TP_FLAG, None) is not None
+ try:
+ chain = _tp_chain(model)
+ except Exception:
+ chain = [model]
+ if any(m.__dict__.get(_TP_FLAG) is not None for m in chain):
+ return True
+ return any(m.__dict__.get("_unsloth_tp_style") is not None for m in model.modules())
+
+
+def _default_mesh(model):
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+
+ if not dist.is_initialized():
+ raise RuntimeError(
+ "Unsloth: apply_tensor_parallel needs an initialized process group (torchrun) or a tp_mesh."
+ )
+ device = next(model.parameters()).device
+ return init_device_mesh(device.type, (dist.get_world_size(),), mesh_dim_names = ("tp",))
+
+
+def _refuse_generate(*args, **kwargs):
+ raise RuntimeError(
+ "Unsloth: generation is not supported on a tensor parallel model in v1; save the adapter "
+ "(save_tp_adapter) and generate on one GPU."
+ )
+
+
+def _cache_in_use(past):
+ if past is None:
+ return False
+ length = getattr(past, "get_seq_length", None)
+ if callable(length):
+ try:
+ return int(length()) > 0
+ except Exception:
+ return True
+ try:
+ return len(past) > 0
+ except TypeError:
+ return True
+
+
+def _refuse_cached_forward(module):
+ """Refuse a filled KV cache (instance attribute: LoraModel calls self.model.forward directly)."""
+ forward = module.forward
+ try:
+ names = list(inspect.signature(forward).parameters)
+ except (TypeError, ValueError):
+ names = []
+ index = names.index("past_key_values") if "past_key_values" in names else None
+
+ def forward_without_cache(*args, **kwargs):
+ past = kwargs.get("past_key_values", kwargs.get("past_key_value"))
+ if past is None and index is not None and len(args) > index:
+ past = args[index]
+ if _cache_in_use(past):
+ raise RuntimeError(
+ "Unsloth: cached decoding (past_key_values) is not supported on a tensor parallel model in "
+ "v1; save the adapter (save_tp_adapter) and generate on one GPU."
+ )
+ return forward(*args, **kwargs)
+
+ forward_without_cache.__wrapped__ = forward
+ module.__dict__["forward"] = forward_without_cache
+
+
+def _sync_trainable(model, group, src):
+ import torch.distributed as dist
+ with torch.no_grad():
+ for param in model.parameters():
+ if param.requires_grad:
+ dist.broadcast(param.data, src = src, group = group)
+
+
+def _set_local_heads(module, region, tp):
+ heads, kv_heads = region.heads // tp, region.kv_heads // tp
+ module._unsloth_tp_heads = heads
+ module._unsloth_tp_kv_heads = kv_heads
+ for name, value, always in (
+ ("num_heads", heads, True),
+ ("num_key_value_heads", kv_heads, True),
+ ("num_attention_heads", heads, False),
+ ("num_kv_heads", kv_heads, False),
+ ):
+ if always or isinstance(module.__dict__.get(name), int):
+ module.__dict__[name] = value
+ config = module.__dict__.get("config")
+ if config is None:
+ return
+ try:
+ source = _forward_source(module)
+ except (OSError, TypeError):
+ source = ""
+ if _GLOBAL_HEAD_READS.search(source) and "_unsloth_tp_heads" not in source:
+ # Per-module config copy with local counts (Phi3 fused qkv); the shared config is left alone.
+ local = copy.copy(config)
+ local.num_attention_heads = heads
+ if getattr(config, "num_key_value_heads", None) is not None:
+ local.num_key_value_heads = kv_heads
+ module.__dict__["config"] = local
+
+
+def apply_tensor_parallel(
+ model,
+ tp_mesh = None,
+ plan = None,
+ config: Optional[TPConfig] = None,
+) -> nn.Module:
+ """Shard every decoder attention / MLP of `model` over `tp_mesh` (None = all ranks), in place."""
+ if is_tensor_parallel(model):
+ return model
+ config = config or TPConfig()
+ if plan is None:
+ plan = config.plan
+ if tp_mesh is None:
+ tp_mesh = _default_mesh(model)
+ if tp_mesh.ndim != 1:
+ raise ValueError("Unsloth: apply_tensor_parallel takes a 1-D tp mesh (pass mesh['tp']).")
+ tp = tp_mesh.size()
+ rank = tp_mesh.get_local_rank()
+ group = tp_mesh.get_group()
+ if (
+ any(_is_dtensor(p) for p in model.parameters())
+ or (getattr(model, "tp_size", None) or 1) > 1
+ ):
+ raise ValueError(
+ "Unsloth: this model is already distributed (DTensor parameters or a transformers tp_plan); "
+ "load it without tp_plan before apply_tensor_parallel."
+ )
+ tp_plan = derive_tp_plan(model, tp, plan)
+ for region in tp_plan.regions:
+ for proj in region.projs:
+ path = f"{region.path}.{proj.name}"
+ weight, base = _weight(proj.module)
+ _refuse_base(path, proj.module, weight, base)
+ _refuse_adapter(path, proj.module)
+ if _is_params4bit(weight):
+ _check_params4bit(path, weight, proj.style, tp)
+ for name, norm in region.norms:
+ for p in norm.parameters():
+ if type(p) not in (nn.Parameter, torch.Tensor):
+ raise ValueError(
+ f"Unsloth: cannot replicate {region.path}.{name} ({type(p).__name__})."
+ )
+ chain = _tp_chain(model)
+ if config.sync_trainable and tp > 1:
+ import torch.distributed as dist
+ _sync_trainable(model, group, dist.get_global_rank(group, 0))
+ hooked_norms = (
+ set()
+ ) # a norm weight shared by several regions gets one tp sum, not one per region
+ for region in tp_plan.regions:
+ for proj in region.projs:
+ _shard_projection(region, proj, tp_mesh, rank, tp)
+ for _, norm in region.norms:
+ for p in norm.parameters():
+ if p.requires_grad and id(p) not in hooked_norms:
+ hooked_norms.add(id(p))
+ # Each rank normalizes its own heads with the shared weight: the grad is a partial sum.
+ p.register_hook(lambda g, group = group: _all_reduce(g, group))
+ module = region.module
+ module._unsloth_tp_group = group
+ module._unsloth_tp_row_projs = [p.module for p in region.projs if p.style == ROWWISE]
+ module._unsloth_tp_input = _input_name(module)
+ if region.kind == "attention":
+ _set_local_heads(module, region, tp)
+ module.register_forward_pre_hook(_region_pre_hook, with_kwargs = True)
+ module.register_forward_hook(_region_post_hook)
+ # Unsloth decode paths bypass the regions: mark and guard every wrapper level.
+ info = {"size": tp, "rank": rank, "regions": len(tp_plan.regions), "source": tp_plan.source}
+ inner = _unwrap(model)
+ decoder = _decoder_of(inner)
+ for level in chain:
+ level.__dict__[_TP_FLAG] = info
+ level.__dict__["_unsloth_tp_mesh"] = tp_mesh
+ level.__dict__["_unsloth_tp_plan"] = tp_plan
+ if callable(getattr(level, "generate", None)):
+ level.__dict__["generate"] = _refuse_generate
+ if level is inner or level is decoder:
+ _refuse_cached_forward(level)
+ if rank == 0:
+ print(
+ f"Unsloth: tensor parallel over {tp} ranks: {len(tp_plan.regions)} attention / MLP regions "
+ f"({tp_plan.source} plan), embeddings / lm_head / norms replicated"
+ + (
+ f", {tp_plan.replicated_layers} layers replicated"
+ if tp_plan.replicated_layers
+ else ""
+ )
+ + "."
+ )
+ return model
+
+
+def _shard_projection(region, proj, mesh, rank, tp):
+ module = proj.module
+ weight, base = _weight(module)
+ col = proj.style == COLWISE
+ parts = proj.parts
+ in_parts = (_logical_shape(weight, base)[1],)
+ if _is_params4bit(weight):
+ _shard_params4bit(f"{region.path}.{proj.name}", base, weight, proj.style, parts, rank, tp)
+ else:
+ _set_dtensor(
+ base,
+ "weight",
+ weight,
+ mesh,
+ "shard",
+ parts if col else in_parts,
+ 0 if col else 1,
+ rank,
+ tp,
+ )
+ if base.bias is not None:
+ if col:
+ _set_dtensor(base, "bias", base.bias, mesh, "shard", parts, 0, rank, tp)
+ else:
+ _set_dtensor(base, "bias", base.bias, mesh, "replicate")
+ lora_A = module._modules.get("lora_A")
+ if lora_A is not None:
+ for adapter in lora_A.keys():
+ A, B = lora_A[adapter], module.lora_B[adapter]
+ if col:
+ _set_dtensor(A, "weight", A.weight, mesh, "replicate")
+ _set_dtensor(B, "weight", B.weight, mesh, "shard", parts, 0, rank, tp)
+ else:
+ _set_dtensor(A, "weight", A.weight, mesh, "shard", in_parts, 1, rank, tp)
+ _set_dtensor(B, "weight", B.weight, mesh, "replicate")
+ module._unsloth_tp_style = proj.style
+ module._unsloth_tp_local = (
+ _tp_lora_parameters # read by kernels/utils.py get_lora_parameters(_bias)
+ )
+ module.forward = types.MethodType(_tp_projection_forward, module)
+
+
+def tp_clip_grad_norm_(
+ parameters,
+ max_norm,
+ mesh = None,
+ eps = 1e-6,
+):
+ """Clip by the global 2-norm: sharded grads summed over tp, replicated ones counted once."""
+ import torch.distributed as dist
+
+ if isinstance(parameters, torch.Tensor):
+ parameters = [parameters]
+ params = [p for p in parameters if p.grad is not None]
+ if not params:
+ return torch.tensor(0.0)
+ sharded = replicated = None
+ group = mesh.get_group() if mesh is not None else None
+ for p in params:
+ g = p.grad
+ if _is_dtensor(g):
+ g = _replicate_grad(g)
+ if g is not p.grad:
+ p.grad = g
+ group = g.device_mesh.get_group() if group is None else group
+ local = g.to_local().float()
+ sq = local.square().sum()
+ if any(pl.is_shard() for pl in g.placements):
+ sharded = sq if sharded is None else sharded + sq
+ continue
+ else:
+ sq = g.float().square().sum()
+ replicated = sq if replicated is None else replicated + sq
+ device = (
+ params[0].grad.device
+ if not _is_dtensor(params[0].grad)
+ else params[0].grad.to_local().device
+ )
+ total = torch.zeros((), device = device) if replicated is None else replicated.to(device)
+ if group is not None:
+ shard_sq = torch.zeros((), device = device) if sharded is None else sharded.to(device)
+ dist.all_reduce(shard_sq, group = group)
+ total = total + shard_sq
+ total = total.sqrt()
+ coef = float(max_norm) / (float(total) + eps)
+ if coef < 1:
+ for p in params:
+ p.grad.mul_(coef)
+ return total
+
+
+def _full(tensor, param = None):
+ if not _is_dtensor(tensor):
+ return tensor
+ full = tensor.full_tensor()
+ layout = getattr(param, "_unsloth_tp_parts", None) if param is not None else None
+ if layout is not None:
+ dim, parts = layout
+ full = _unpermute(full, dim, parts, tensor.device_mesh.size())
+ return full
+
+
+def tp_full_state_dict(model, adapters_only = False):
+ """State dict with DTensors gathered (collective); 4-bit shards stay local."""
+ state = model.state_dict(keep_vars = True)
+ # PEFT picks embedding weights out of the given state dict (save_embedding_layers, resized vocab).
+ embeddings = set()
+ for get in ("get_input_embeddings", "get_output_embeddings"):
+ module = getattr(model, get, lambda: None)()
+ weight = getattr(module, "weight", None)
+ if weight is not None:
+ embeddings.add(id(weight))
+ out = {}
+ with torch.no_grad():
+ for key, value in state.items():
+ if (
+ adapters_only
+ and "lora_" not in key
+ and not getattr(value, "requires_grad", False)
+ and id(value) not in embeddings
+ ):
+ continue
+ out[key] = _full(value, value).detach()
+ return out
+
+
+def _peft_state_dict(model, **kwargs):
+ """What PEFT selects for save_pretrained, DTensors gathered; local tensors by reference (PEFT reruns
+ its selection inside save_pretrained and must see the same model)."""
+ from peft import get_peft_model_state_dict
+
+ raw = model.state_dict(keep_vars = True)
+ adapters = kwargs.get("selected_adapters") or list(model.peft_config)
+ known = {id(value) for value in raw.values()}
+ wanted, unmapped = set(), 0
+ for adapter in adapters:
+ selected = get_peft_model_state_dict(
+ model,
+ state_dict = raw,
+ adapter_name = adapter,
+ save_embedding_layers = kwargs.get("save_embedding_layers", "auto"),
+ )
+ wanted.update(id(value) for value in selected.values())
+ unmapped += sum(_is_dtensor(v) and id(v) not in known for v in selected.values())
+ if unmapped:
+ raise RuntimeError(
+ "Unsloth: PEFT selected tensor parallel tensors that are not in the model state dict; "
+ "save_tp_adapter cannot gather them."
+ )
+ state = {}
+ with torch.no_grad():
+ for key, value in raw.items():
+ if _is_dtensor(value):
+ if id(value) in wanted:
+ state[key] = _full(value, value).detach()
+ else:
+ state[key] = value.detach()
+ return state
+
+
+def save_tp_adapter(model, save_directory, **kwargs):
+ """Write the standard 1-GPU PEFT adapter. Collective (tp mesh = whole world in v1); rank 0 writes."""
+ import torch.distributed as dist
+
+ mesh = None
+ for level in _tp_chain(model):
+ mesh = mesh or level.__dict__.get("_unsloth_tp_mesh")
+ if dist.is_initialized() and mesh is not None and mesh.size() != dist.get_world_size():
+ raise ValueError(
+ f"Unsloth: save_tp_adapter writes on global rank 0 behind a world barrier, but the tensor parallel "
+ f"mesh has {mesh.size()} of {dist.get_world_size()} ranks (data parallel > 1 is outside v1). "
+ "Gather with tp_full_state_dict and write from one rank yourself."
+ )
+ state = _peft_state_dict(model, **kwargs)
+ rank = dist.get_rank() if dist.is_initialized() else 0
+ kwargs.setdefault("is_main_process", rank == 0)
+ if kwargs["is_main_process"]:
+ model.save_pretrained(save_directory, state_dict = state, **kwargs)
+ if dist.is_initialized():
+ dist.barrier()
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 6aafd48ffa6..9500b1a2d90 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -191,6 +191,7 @@ from ._utils import (
)
from ._utils import *
from ._utils import _ignore_block_swap_params_for_ddp
+from ._utils import _refuse_tensor_parallel_for_inference
from ._uma_safetensors import is_integrated_unified_memory_gpu
from ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimate
from ._remote_code_buffers import restore_remote_code_non_persistent_buffers
@@ -5501,6 +5502,7 @@ class FastBaseModel:
raise TypeError(
"Unsloth: I think you're passing a tokenizer, not the model to for_inference!"
)
+ _refuse_tensor_parallel_for_inference(model)
def _for_inference(m):
if hasattr(m, "gradient_checkpointing"):