Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Tensor parallelism v1: apply_tensor_parallel for dense decoders (LoRA / QLoRA) (#15137)
Daniel Hand2ca6515204f
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.pynew file mode 100644index 00000000000..396c142ce10--- /dev/null+++ b/tests/test_tp_native_gloo.py@@ -0,0 +1,632 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""apply_tensor_parallel on a gloo CPU world (2 and 4 ranks) vs the same model in one process, fp32."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import socket # noqa: E402+import sys # noqa: E402+import tempfile # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_tp_native_oracle import ( # noqa: E402+ ARCHS,+ LLAMA,+ PHI3_PACKED_PLAN,+ QWEN3,+ _batch,+ build,+ skip_unsupported,+ tpn,+)++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _grads(model):+ return {+ n: tpn._full(p.grad, p).detach().clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+++def _train(model, ids, steps, clip):+ params = [p for p in model.parameters() if p.requires_grad]+ opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)+ losses, norms, grads = [], [], None+ for step in range(steps):+ out = model(input_ids = ids, labels = ids, use_cache = False)+ out.loss.backward()+ if step == 0:+ grads = _grads(model)+ logits = out.logits.detach().clone()+ # Replicated factors' Partial grads are summed in backward: grads land in their param's layout.+ mismatched = [+ n+ for n, p in model.named_parameters()+ if tpn._is_dtensor(p.grad) and tuple(p.grad.placements) != tuple(p.placements)+ ]+ assert not mismatched, mismatched[:4]+ norms.append(float(clip(params, 1.0)))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(out.loss))+ return logits, grads, losses, norms+++def _clip_fp64(params, max_norm):+ """clip_grad_norm_ in float64: torch's CPU fp32 vector_norm is ~2e-4 off, above the parity tolerance."""+ grads = [p.grad for p in params if p.grad is not None]+ total = torch.stack([g.double().square().sum() for g in grads]).sum().sqrt().float()+ coef = torch.clamp(max_norm / (total + 1e-6), max = 1.0)+ for g in grads:+ g.mul_(coef)+ return total+++def _tie_q_norms(model):+ # Two regions sharing one trainable per-head norm weight: its tp sum must run once, not once per region.+ layers = tpn._unwrap(model).model.layers+ layers[1].self_attn.q_norm.weight = layers[0].self_attn.q_norm.weight+++def _case_parity(+ rank,+ world,+ tmp,+ arch,+ overrides = None,+ lora_kwargs = None,+ save_kwargs = None,+ plan = None,+ freeze_before_save = False,+ tie_norms = False,+ fp64_ref_clip = False,+):+ from torch.distributed.device_mesh import init_device_mesh+ from safetensors.torch import load_file++ repo, base_overrides = ARCHS[arch]+ overrides = {**base_overrides, **(overrides or {})}+ train_norms = arch in ("qwen3", "gemma3")+ ref = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))+ model = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))+ if tie_norms:+ _tie_q_norms(ref), _tie_q_norms(model)+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ tpn.apply_tensor_parallel(model, mesh["tp"], plan = plan)+ assert tpn.is_tensor_parallel(model)+ assert tpn.apply_tensor_parallel(model, mesh["tp"]) is model # idempotent+ with pytest.raises(RuntimeError, match = "generation is not supported"):+ model.generate(input_ids = torch.zeros(1, 2, dtype = torch.long))+ from test_fsdp2_native_units import _load_native++ try:+ _load_native()._refuse_tensor_parallel_model(model)+ fsdp2_refused = None+ except RuntimeError as e:+ fsdp2_refused = str(e)+ ids = _batch(ref)+ ref_out = _train(ref, ids, 3, _clip_fp64 if fp64_ref_clip else torch.nn.utils.clip_grad_norm_)+ tp_out = _train(model, ids, 3, tpn.tp_clip_grad_norm_)+ if freeze_before_save:+ # Frozen adapter (e.g. after training): PEFT still saves bias = "all" biases, trainable or not.+ for m in (ref, model):+ for p in m.parameters():+ p.requires_grad_(False)+ ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")+ ref.save_pretrained(ref_dir, **(save_kwargs or {}))+ tpn.save_tp_adapter(model, tp_dir, **(save_kwargs or {}))+ q = tpn._weight(model._unsloth_tp_plan.regions[0].projs[0].module)[0]+ return {+ "ref": ref_out,+ "tp": tp_out,+ "saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),+ "saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))+ if rank == 0+ else None,+ "local_q": (tuple(q.to_local().shape), tuple(q.shape), q.placements[0].dim),+ "fsdp2_refused": fsdp2_refused,+ }+++def _import_checkout_unsloth():+ """`import unsloth` from this checkout (not the stub tp_native is loaded under); None, or why it failed."""+ import pathlib++ for name in [n for n in sys.modules if n == "unsloth" or n.startswith("unsloth.")]:+ del sys.modules[name]+ sys.path.insert(0, REPO)+ try:+ # conftest's device spoof (spawned ranks do not run it).+ import importlib.util++ spec = importlib.util.spec_from_file_location(+ "_tp_conftest", os.path.join(REPO, "tests", "conftest.py")+ )+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+ import unsloth+ except Exception as e: # a CPU host that refuses `import unsloth`+ return repr(e)+ # By path: compile setup may put other directories in front of sys.path.+ root = pathlib.Path(REPO).resolve()+ assert root in pathlib.Path(unsloth.__file__).resolve().parents, unsloth.__file__+ return None+++def _raised(fn):+ try:+ fn()+ except Exception as e:+ return f"{type(e).__name__}: {e}"+ return None+++def _case_guards(rank, world, tmp):+ """Entry points outside the tp regions are refused on every wrapper level."""+ error = _import_checkout_unsloth()+ if error is not None:+ return {"skip": error}+ import transformers+ from torch.distributed.device_mesh import init_device_mesh+ from unsloth import FastLanguageModel, FastModel+ from unsloth.models import fsdp2_native, tp_native+ from unsloth.models._utils import patch_gradient_accumulation_fix++ patch_gradient_accumulation_fix(transformers.Trainer)+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ args = transformers.TrainingArguments(output_dir = tmp, report_to = [], use_cpu = True)+ out = {}+ for applied in ("peft", "causal"):+ model = build(LLAMA)+ levels = {+ "peft": model,+ "lora": model.base_model,+ "causal": model.base_model.model,+ "decoder": model.base_model.model.model,+ }+ tp_native.apply_tensor_parallel(levels[applied], mesh["tp"])+ ids = _batch(model)+ res = {}+ cached = model(input_ids = ids, use_cache = True)+ res["forward_without_cache"] = cached.logits is not None+ past = cached.past_key_values+ step = ids[:, -1:]+ for name, level in levels.items():+ res[f"is_tp:{name}"] = tp_native.is_tensor_parallel(level)+ res[f"fsdp2:{name}"] = _raised(+ lambda: fsdp2_native._refuse_tensor_parallel_model(level)+ )+ res[f"for_inference:{name}"] = _raised(lambda: FastLanguageModel.for_inference(level))+ res[f"FastModel.for_inference:{name}"] = _raised(lambda: FastModel.for_inference(level))+ if name != "decoder":+ res[f"generate:{name}"] = _raised(lambda: level.generate(input_ids = ids))+ res[f"trainer:{name}"] = _raised(+ lambda: transformers.Trainer(model = level, args = args)+ )+ res[f"model_init:{name}"] = _raised(+ lambda: transformers.Trainer(model_init = lambda: level, args = args)+ )+ res[f"cached:{name}"] = _raised(+ lambda: level(input_ids = step, past_key_values = past, use_cache = True)+ )+ res["generate:get_base_model"] = _raised(+ lambda: model.get_base_model().generate(input_ids = ids)+ )+ params = [p for p in model.parameters() if p.requires_grad]+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()+ norm = tp_native.tp_clip_grad_norm_(params, 1e9)+ res["clip_mesh_equal"] = float(norm) == float(+ tp_native.tp_clip_grad_norm_(params, 1e9, mesh = mesh["tp"])+ )+ out[applied] = res+ return out+++def _case_fused(rank, world, tmp):+ # The real package from this checkout, not the stub the oracle module loads tp_native under.+ error = _import_checkout_unsloth()+ if error is not None:+ return {"skip": error}+ from unsloth.kernels import fast_lora+ from unsloth.models import tp_native+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ calls = []+ apply = fast_lora._apply+ fast_lora._apply = lambda function, *args: calls.append(function.__name__) or apply(+ function, *args+ )+ out = {}+ for fused in (False, True):+ calls.clear()+ model = build(QWEN3, {"num_attention_heads": 8, "num_key_value_heads": 4})+ tp_native.apply_tensor_parallel(model, mesh["tp"])+ attn = tp_native._unwrap(model).model.layers[0].self_attn+ g = torch.Generator().manual_seed(5)+ X = torch.randn(+ 2, 5, attn.q_proj.base_layer.weight.shape[1], generator = g, requires_grad = True+ )+ Xr = tp_native.copy_to_tp(X, attn._unsloth_tp_group)+ if fused:+ Q, K, V = fast_lora.apply_lora_qkv(attn, Xr, inplace = False)+ else:+ Q, K, V = attn.q_proj(Xr), attn.k_proj(Xr), attn.v_proj(Xr)+ Y = torch.cat([Q, K, V], -1)+ O_in = Y[..., : Q.shape[-1]].contiguous()+ O = fast_lora.apply_lora_o(attn, O_in) if fused else attn.o_proj(O_in)+ O = tp_native.reduce_from_tp(O, attn._unsloth_tp_group)+ (O.square().sum() + Y.square().sum()).backward()+ out[fused] = {+ "O": O.detach(),+ "dX": X.grad.detach(),+ "grads": _grads(attn),+ "calls": list(calls),+ }+ return out+++def _quantize_layers(+ model,+ nested,+ blocksize = 64,+):+ """Every Linear in the decoder layers -> a bitsandbytes NF4 Linear4bit (the same bytes in every process)."""+ import bitsandbytes as bnb++ for name, module in list(model.named_modules()):+ for child_name, child in list(module.named_children()):+ if type(child) is not torch.nn.Linear or ".layers." not in f".{name}.":+ continue+ q = bnb.nn.Linear4bit(+ child.in_features,+ child.out_features,+ bias = child.bias is not None,+ compute_dtype = torch.float32,+ quant_type = "nf4",+ compress_statistics = nested,+ )+ packed, state = bnb.functional.quantize_4bit(+ child.weight.data, blocksize = blocksize, quant_type = "nf4", compress_statistics = nested+ )+ q.weight = bnb.nn.Params4bit(+ packed,+ requires_grad = False,+ quant_state = state,+ blocksize = blocksize,+ compress_statistics = nested,+ quant_type = "nf4",+ module = q,+ bnb_quantized = True,+ )+ if child.bias is not None:+ q.bias = child.bias+ setattr(module, child_name, q)+ return model+++QLORA_SIZES = {+ "hidden_size": 128,+ "intermediate_size": 256,+ "num_attention_heads": 4,+ "num_key_value_heads": 2,+ "head_dim": 32,+}+++def _qlora_model(+ nested,+ seed = 0,+ blocksize = 64,+):+ from peft import LoraConfig, get_peft_model++ model = _quantize_layers(build(LLAMA, QLORA_SIZES, lora = False, seed = seed), nested, blocksize)+ torch.manual_seed(seed + 1)+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)+ # train(): eval-mode CPU Linear4bit repacks for a CPU gemm that rounds differently.+ return get_peft_model(model, config).train()+++def _case_qlora(rank, world, tmp, nested):+ import bitsandbytes as bnb+ from torch.distributed.device_mesh import init_device_mesh++ ref, model = _qlora_model(nested), _qlora_model(nested)+ full = {+ n: bnb.functional.dequantize_4bit(m.weight.data, m.weight.quant_state)+ for n, m in model.named_modules()+ if type(m).__name__ == "Linear4bit"+ }+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ tpn.apply_tensor_parallel(model, mesh["tp"])+ shard_err = 0.0+ for region in model._unsloth_tp_plan.regions:+ for proj in region.projs:+ base = proj.module.base_layer+ name = next(n for n, m in model.named_modules() if m is base)+ local = bnb.functional.dequantize_4bit(base.weight.data, base.weight.quant_state)+ col = proj.style == tpn.COLWISE+ want = tpn.shard_slice(+ full[name],+ 0 if col else 1,+ proj.parts if col else (full[name].shape[1],),+ rank,+ world,+ )+ assert local.shape == want.shape, (name, local.shape, want.shape)+ shard_err = max(shard_err, float((local - want).abs().max()))+ ids = _batch(ref)+ return {+ "ref": _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_),+ "tp": _train(model, ids, 3, tpn.tp_clip_grad_norm_),+ "shard_err": shard_err,+ }+++def _case_qlora_misaligned(rank, world, tmp):+ from torch.distributed.device_mesh import init_device_mesh++ # blocksize 128: o_proj / down_proj row shards (in_features / 2 = 64) would split blocks.+ model = _qlora_model(False, blocksize = 128)+ before = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ with pytest.raises(ValueError, match = "row sharding would split blocks"):+ tpn.apply_tensor_parallel(model, mesh["tp"])+ after = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}+ return {"unchanged": before == after, "flag": tpn.is_tensor_parallel(model)}+++def _case_save_subgroup(rank, world, tmp):+ from torch.distributed.device_mesh import init_device_mesh++ # tp = 1 inside a 2-rank world (data parallel 2): the world-scoped writer / barrier would not match.+ mesh = init_device_mesh("cpu", (world, 1), mesh_dim_names = ("dp", "tp"))+ model = tpn.apply_tensor_parallel(build(LLAMA), mesh["tp"])+ return {"error": _raised(lambda: tpn.save_tp_adapter(model, os.path.join(tmp, "out")))}+++CASES = {+ "parity": _case_parity,+ "save_subgroup": _case_save_subgroup,+ "fused": _case_fused,+ "guards": _case_guards,+ "qlora": _case_qlora,+ "qlora_misaligned": _case_qlora_misaligned,+}+++def _worker(rank, world, port, case, tmp, kwargs):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ torch.set_num_threads(2)+ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ res = CASES[case](rank, world, tmp, **kwargs)+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ dist.destroy_process_group()+++def _spawn(+ case,+ world = 2,+ **kwargs,+):+ ctx = multiprocessing.get_context("spawn")+ port = _free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))+ for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(600)+ codes = [p.exitcode for p in procs]+ assert codes == [0] * world, f"rank exit codes {codes}"+ return [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+++def _check_parity(results, world):+ for rank, res in enumerate(results):+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]+ logits, grads, losses, norms = res["tp"]+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+ assert set(grads) == set(ref_grads), sorted(set(grads) ^ set(ref_grads))[:6]+ for name in ref_grads:+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6), (rank, losses, ref_losses)+ assert norms == pytest.approx(ref_norms, rel = 1e-5), (rank, norms, ref_norms)+ assert "does not compose with Unsloth tensor parallelism" in (res["fsdp2_refused"] or "")+ local, full, dim = res["local_q"]+ assert local[dim] * world == full[dim] and local[1 - dim] == full[1 - dim]+ saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]+ assert set(saved_tp) == set(saved_ref)+ for key in saved_ref:+ assert saved_tp[key].shape == saved_ref[key].shape, key+ torch.testing.assert_close(saved_tp[key], saved_ref[key], rtol = 1e-4, atol = 1e-5, msg = key)++[email protected]("arch", ["llama", "llama_bias", "qwen3", "phi3", "stablelm", "gemma2"])+def test_tp2_matches_one_process(arch):+ skip_unsupported(arch)+ _check_parity(_spawn("parity", 2, arch = arch), 2)+++def test_tp2_refused_4bit_model_is_left_untouched():+ for res in _spawn("qlora_misaligned", 2):+ assert res["unchanged"] and not res["flag"]+++def test_tp2_trainable_biases_match_one_process():+ # bias = "all": the row bias is added after the all-reduce, so its gradient is complete on every rank.+ results = _spawn("parity", 2, arch = "llama_bias", lora_kwargs = {"bias": "all"})+ _check_parity(results, 2)+ assert any("bias" in key for key in results[0]["saved_tp"])+++def test_tp2_adapter_save_keeps_requested_embeddings():+ results = _spawn("parity", 2, arch = "llama", save_kwargs = {"save_embedding_layers": True})+ _check_parity(results, 2)+ assert any("embed_tokens" in key for key in results[0]["saved_tp"])++[email protected]("nested", [False, True])+def test_tp2_qlora_matches_one_process(nested):+ results = _spawn("qlora", 2, nested = nested)+ for res in results:+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]+ logits, grads, losses, norms = res["tp"]+ assert res["shard_err"] == 0.0+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+ assert set(grads) == set(ref_grads) and grads+ for name in ref_grads:+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)+ assert norms == pytest.approx(ref_norms, rel = 1e-5)+++def test_tp4_gqa_matches_one_process():+ overrides = {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 64}+ _check_parity(_spawn("parity", 4, arch = "qwen3", overrides = overrides), 4)+++def test_fused_lora_functions_read_local_shards():+ results = _spawn("fused", 2)+ if "skip" in results[0]:+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")+ for res in results:+ plain, fused = res[False], res[True]+ assert plain["calls"] == [] and fused["calls"] == ["LoRA_QKV", "LoRA_W"], (+ plain["calls"],+ fused["calls"],+ )+ torch.testing.assert_close(fused["O"], plain["O"], rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(fused["dX"], plain["dX"], rtol = 1e-5, atol = 1e-5)+ assert set(fused["grads"]) == set(plain["grads"]) and plain["grads"]+ for name in plain["grads"]:+ torch.testing.assert_close(+ fused["grads"][name], plain["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name+ )+++def test_apply_refuses_a_2d_mesh():+ model = build(LLAMA)++ class _Mesh:+ ndim = 2++ with pytest.raises(ValueError, match = "1-D tp mesh"):+ tpn.apply_tensor_parallel(model, _Mesh())+++def test_tp2_phi3_fused_qkv_under_a_packed_plan_matches_one_process():+ # The tiny Phi3 is GQA (4 heads, 2 kv heads): qkv_proj parts are (q, kv, kv), not two packed halves.+ _check_parity(_spawn("parity", 2, arch = "phi3", plan = PHI3_PACKED_PLAN), 2)+++ "lora_kwargs,save_kwargs",+ [+ ({"bias": "all"}, None), # frozen bias-bearing adapter: every bias, trainable or not+ ({"bias": "lora_only"}, None),+ ({"modules_to_save": ["lm_head"]}, None),+ ({"modules_to_save": ["lm_head"]}, {"save_embedding_layers": True}),+ ],+)+def test_tp2_adapter_save_matches_peft_selection(lora_kwargs, save_kwargs):+ results = _spawn(+ "parity",+ 2,+ arch = "llama_bias",+ lora_kwargs = lora_kwargs,+ save_kwargs = save_kwargs,+ freeze_before_save = True,+ fp64_ref_clip = "modules_to_save" in lora_kwargs,+ )+ _check_parity(results, 2)+ saved = results[0]["saved_tp"]+ if (+ lora_kwargs.get("bias") == "all"+ ): # "lora_only" saves no base_layer bias on PEFT 0.18 (1 process too)+ assert any("o_proj" in k and k.endswith("bias") for k in saved), sorted(saved)[:8]+ if "modules_to_save" in lora_kwargs:+ assert any("modules_to_save" in k or "lm_head" in k for k in saved), sorted(saved)[:8]+++def test_tp2_tied_norm_weight_is_summed_once():+ _check_parity(_spawn("parity", 2, arch = "qwen3", tie_norms = True), 2)+++def test_tp2_entry_points_outside_the_regions_are_refused():+ results = _spawn("guards", 2)+ if "skip" in results[0]:+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")+ for res in results:+ for applied, got in res.items():+ assert got["forward_without_cache"] and got["clip_mesh_equal"], (applied, got)+ for key, value in got.items():+ if key.startswith("is_tp:"):+ assert value is True, (applied, key)+ elif key.startswith("fsdp2:"):+ assert value and "does not compose with Unsloth tensor parallelism" in value, (+ applied,+ key,+ value,+ )+ elif "for_inference" in key:+ assert value and "for_inference is not supported" in value, (+ applied,+ key,+ value,+ )+ elif key.startswith("generate:"):+ assert value and "generation is not supported" in value, (applied, key, value)+ elif key.startswith(("trainer:", "model_init:")):+ assert value and "cannot be trained by the Trainer" in value, (+ applied,+ key,+ value,+ )+ elif key.startswith("cached:"):+ assert value and "cached decoding" in value, (applied, key, value)+++def test_save_refuses_a_tp_mesh_smaller_than_the_world():+ for res in _spawn("save_subgroup", 2):+ assert res["error"] and "data parallel > 1 is outside v1" in res["error"], resdiff --git a/tests/test_tp_native_oracle.py b/tests/test_tp_native_oracle.pynew file mode 100644index 00000000000..8ad9825aa67--- /dev/null+++ b/tests/test_tp_native_oracle.py@@ -0,0 +1,688 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Tensor parallel plan derivation, refusals, and a simulated-TP oracle (summed shard copies), fp32 CPU."""++import copy+import importlib.util+import os+import sys+import types++import pytest+import torch++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+++def _load_tp():+ """tp_native.py without importing the unsloth package (which needs a GPU)."""+ name = "unsloth.models.tp_native"+ if name in sys.modules:+ return sys.modules[name]+ import importlib.machinery++ pkg = types.ModuleType("unsloth")+ pkg.__path__ = [os.path.join(REPO, "unsloth")]+ models = types.ModuleType("unsloth.models")+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]+ for package in (pkg, models):+ # A spec, so a later importlib.util.find_spec("unsloth") in the same session does not raise.+ package.__spec__ = importlib.machinery.ModuleSpec(package.__name__, None, is_package = True)+ package.__spec__.submodule_search_locations = package.__path__+ sys.modules.setdefault("unsloth", pkg)+ sys.modules.setdefault("unsloth.models", models)+ spec = importlib.util.spec_from_file_location(+ name, os.path.join(REPO, "unsloth", "models", "tp_native.py")+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++tpn = _load_tp()+++def skip_unsupported(arch):+ """transformers 4.x StableLmAttention reshapes with hidden_size (the global width): apply refuses it."""+ import transformers+ if arch == "stablelm" and int(transformers.__version__.split(".")[0]) < 5:+ pytest.skip("transformers 4.x StableLM attention is refused (test_stablelm_4x_is_refused)")+++LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"+QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"+GEMMA2 = "trl-internal-testing/tiny-Gemma2ForCausalLM"+GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"+MISTRAL = "hf-internal-testing/tiny-random-MistralForCausalLM"+PHI3 = "trl-internal-testing/tiny-Phi3ForCausalLM"+QWEN2 = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5-Coder"+STABLELM = "hf-internal-testing/tiny-random-StableLmForCausalLM" # no transformers TP plan+OLMO2 = "hf-internal-testing/tiny-random-Olmo2ForCausalLM"+QWEN3_MOE = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"+DEEPSEEK_V3 = "trl-internal-testing/tiny-DeepseekV3ForCausalLM"++# Archs that shard at tp = 2 (config overrides make every count divisible where the tiny repo is not).+ARCHS = {+ "llama": (LLAMA, {}),+ "llama_bias": (LLAMA, {"attention_bias": True, "mlp_bias": True}), # row biases: added once+ "qwen3": (QWEN3, {}),+ "gemma2": (GEMMA2, {}),+ "gemma3": (GEMMA3, {"num_attention_heads": 4, "num_key_value_heads": 2}),+ "mistral": (MISTRAL, {"intermediate_size": 64}),+ "phi3": (PHI3, {}),+ "qwen2": (QWEN2, {}),+ "stablelm": (STABLELM, {"use_qkv_bias": True, "intermediate_size": 64}),+}+++def build(+ repo,+ overrides = None,+ lora = True,+ seed = 0,+ train_norms = False,+ **lora_kwargs,+):+ from transformers import AutoConfig, AutoModelForCausalLM++ config = AutoConfig.from_pretrained(repo)+ text = config.get_text_config()+ for key, value in (overrides or {}).items():+ setattr(text, key, value)+ config._attn_implementation = "eager"+ torch.manual_seed(seed)+ model = AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+ with torch.no_grad():+ for name, p in model.named_parameters():+ if name.endswith("bias"):+ p.normal_(0, 0.1) # random biases: their placement matters+ elif "norm" in name and p.ndim == 1:+ p.add_(torch.randn_like(p) * 0.1)+ if not lora:+ return model.eval()+ from peft import LoraConfig, get_peft_model++ kwargs = dict(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)+ kwargs.update(lora_kwargs)+ torch.manual_seed(seed + 1)+ model = get_peft_model(model, LoraConfig(**kwargs))+ if train_norms:+ for name, p in model.named_parameters():+ if name.endswith(("q_norm.weight", "k_norm.weight")):+ p.requires_grad_(True)+ return model.eval()+++def _batch(+ model,+ seed = 1,+ bsz = 2,+ seq = 7,+):+ vocab = tpn._text_config(model).vocab_size+ g = torch.Generator().manual_seed(seed)+ return torch.randint(0, vocab, (bsz, seq), generator = g)+++class _SimRegion(torch.nn.Module):+ """tp shard copies of one region, outputs summed (reduce_from_tp) plus the row bias added once."""++ def __init__(self, copies, row_bias):+ super().__init__()+ self.copies = torch.nn.ModuleList(copies)+ self.row_bias = row_bias++ def __getattr__(self, name):+ try:+ return super().__getattr__(name)+ except AttributeError:+ return getattr(self._modules["copies"][0], name)++ def forward(self, *args, **kwargs):+ outs = [c(*args, **kwargs) for c in self.copies]+ tuple_out = isinstance(outs[0], tuple)+ y = sum(o[0] if tuple_out else o for o in outs)+ if self.row_bias is not None:+ y = y + self.row_bias+ return (y,) + tuple(outs[0][1:]) if tuple_out else y+++def _shard_copy(region, rank, tp):+ module = copy.deepcopy(region.module)+ for proj in region.projs:+ child = getattr(module, proj.name)+ weight, base = tpn._weight(child)+ col = proj.style == tpn.COLWISE+ in_parts = (weight.shape[1],)+ with torch.no_grad():+ if col:+ base.weight = torch.nn.Parameter(+ tpn.shard_slice(weight, 0, proj.parts, rank, tp), False+ )+ if base.bias is not None:+ base.bias = torch.nn.Parameter(+ tpn.shard_slice(base.bias, 0, proj.parts, rank, tp), False+ )+ else:+ base.weight = torch.nn.Parameter(+ tpn.shard_slice(weight, 1, in_parts, rank, tp), False+ )+ base.bias = None+ for adapter in getattr(child, "lora_A", {}):+ A, B = child.lora_A[adapter], child.lora_B[adapter]+ if col:+ B.weight = torch.nn.Parameter(+ tpn.shard_slice(B.weight, 0, proj.parts, rank, tp)+ )+ else:+ A.weight = torch.nn.Parameter(tpn.shard_slice(A.weight, 1, in_parts, rank, tp))+ if region.kind == "attention":+ tpn._set_local_heads(module, region, tp)+ return module+++def simulate(+ model,+ tp,+ user_plan = None,+):+ """A copy of `model` whose regions run as tp summed shard copies, and the plan used."""+ sim = copy.deepcopy(model)+ plan = tpn.derive_tp_plan(sim, tp, user_plan)+ for region in plan.regions:+ parent_path, name = region.path.rsplit(".", 1)+ parent = tpn._unwrap(sim).get_submodule(parent_path)+ row_bias = None+ for proj in region.projs:+ base = tpn._weight(proj.module)[1]+ if proj.style == tpn.ROWWISE and base.bias is not None:+ row_bias = base.bias.detach()+ copies = [_shard_copy(region, r, tp) for r in range(tp)]+ setattr(parent, name, _SimRegion(copies, row_bias))+ return sim, plan+++def _sim_grads(sim, plan, tp):+ """{param path relative to the unwrapped model: full gradient} reassembled from the shard copies."""+ grads = {}+ inner = tpn._unwrap(sim)+ for region in plan.regions:+ sim_region = inner.get_submodule(region.path)+ for proj in region.projs:+ col = proj.style == tpn.COLWISE+ children = [getattr(c, proj.name) for c in sim_region.copies]+ for adapter in getattr(children[0], "lora_A", {}):+ As = [c.lora_A[adapter].weight.grad for c in children]+ Bs = [c.lora_B[adapter].weight.grad for c in children]+ prefix = f"{region.path}.{proj.name}"+ if col:+ grads[f"{prefix}.lora_A.{adapter}.weight"] = sum(As)+ grads[f"{prefix}.lora_B.{adapter}.weight"] = tpn._unpermute(+ torch.cat(Bs, 0), 0, proj.parts, tp+ )+ else:+ grads[f"{prefix}.lora_A.{adapter}.weight"] = torch.cat(As, 1)+ grads[f"{prefix}.lora_B.{adapter}.weight"] = sum(Bs)+ for norm_name, _ in region.norms:+ for pname, _ in getattr(sim_region.copies[0], norm_name).named_parameters():+ gs = [getattr(c, norm_name).get_parameter(pname).grad for c in sim_region.copies]+ if gs[0] is not None:+ grads[f"{region.path}.{norm_name}.{pname}"] = sum(gs)+ return grads+++def _ref_grads(model):+ inner = tpn._unwrap(model)+ return {n: p.grad for n, p in inner.named_parameters() if p.grad is not None}+++def _loss(model, ids):+ out = model(input_ids = ids, labels = ids, use_cache = False)+ return out.loss, out.logits++[email protected]("arch", sorted(ARCHS))+def test_simulated_tp_matches_unsharded(arch):+ skip_unsupported(arch)+ repo, overrides = ARCHS[arch]+ model = build(repo, overrides, train_norms = arch == "qwen3")+ ids = _batch(model)+ ref_loss, ref_logits = _loss(model, ids)+ ref_loss.backward()+ ref = _ref_grads(model)+ tp = 2+ sim, plan = simulate(model, tp)+ for p in sim.parameters():+ p.grad = None+ loss, logits = _loss(sim, ids)+ loss.backward()+ got = _sim_grads(sim, plan, tp)+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(loss, ref_loss, rtol = 1e-6, atol = 1e-6)+ lora = [k for k in ref if "lora_" in k]+ assert lora and set(lora) <= set(got), sorted(set(lora) - set(got))[:4]+ for key in ref:+ if key in got:+ torch.testing.assert_close(got[key], ref[key], rtol = 1e-4, atol = 1e-6, msg = key)+ if arch == "qwen3":+ assert any(+ k.endswith("q_norm.weight") for k in got+ ) # the per-head norm grad sum was compared++[email protected]("tp", [2, 4])+def test_simulated_tp_gqa_widths(tp):+ model = build(+ QWEN3, {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 32}+ )+ ids = _batch(model)+ ref_loss, ref_logits = _loss(model, ids)+ sim, _ = simulate(model, tp)+ _, logits = _loss(sim, ids)+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+++def test_mutation_unsummed_replicated_factor_fails():+ """Dropping the tp sum of a replicated factor's gradient (taking one copy's partial) is caught."""+ model = build(LLAMA)+ ids = _batch(model)+ _loss(model, ids)[0].backward()+ ref = _ref_grads(model)+ sim, plan = simulate(model, 2)+ _loss(sim, ids)[0].backward()+ region = plan.regions[0]+ proj = region.projs[0]+ child = getattr(tpn._unwrap(sim).get_submodule(region.path).copies[0], proj.name)+ key = f"{region.path}.{proj.name}.lora_A.default.weight"+ with pytest.raises(AssertionError):+ torch.testing.assert_close(+ child.lora_A["default"].weight.grad, ref[key], rtol = 1e-4, atol = 1e-6+ )+++def _styles(plan):+ return {k.split(".", 3)[-1] if k.count(".") >= 3 else k: v for k, v in plan.styles().items()}++[email protected]("arch", sorted(ARCHS))+def test_plan_covers_every_layer(arch):+ skip_unsupported(arch)+ repo, overrides = ARCHS[arch]+ model = build(repo, overrides)+ plan = tpn.derive_tp_plan(model, 2)+ layers = tpn._text_config(model).num_hidden_layers+ assert len(plan.regions) == 2 * layers+ assert plan.source == ("structural" if arch == "stablelm" else "transformers")+ styles = {(p.name, p.style) for r in plan.regions for p in r.projs}+ if arch == "phi3":+ assert styles == {+ ("qkv_proj", "colwise"),+ ("o_proj", "rowwise"),+ ("gate_up_proj", "colwise"),+ ("down_proj", "rowwise"),+ }+ parts = {p.name: p.parts for r in plan.regions for p in r.projs}+ text = tpn._text_config(model)+ hd = text.hidden_size // text.num_attention_heads+ assert parts["qkv_proj"] == (+ text.num_attention_heads * hd,+ text.num_key_value_heads * hd,+ text.num_key_value_heads * hd,+ )+ assert parts["gate_up_proj"] == (text.intermediate_size, text.intermediate_size)+ else:+ assert styles == {+ ("q_proj", "colwise"),+ ("k_proj", "colwise"),+ ("v_proj", "colwise"),+ ("o_proj", "rowwise"),+ ("gate_proj", "colwise"),+ ("up_proj", "colwise"),+ ("down_proj", "rowwise"),+ }+ norms = {n for r in plan.regions for n, _ in r.norms}+ assert norms == ({"q_norm", "k_norm"} if arch in ("qwen3", "gemma3") else set())+++def test_structural_plan_equals_transformers_plan():+ model = build(QWEN3)+ hf = tpn.derive_tp_plan(model, 2)+ text = tpn._text_config(model)+ text.base_model_tp_plan = None+ for module in model.modules():+ if "_tp_plan" in module.__dict__:+ module._tp_plan = None+ structural = tpn.derive_tp_plan(model, 2)+ assert structural.source == "structural"+ assert structural.styles() == hf.styles()+ assert [len(r.norms) for r in structural.regions] == [len(r.norms) for r in hf.regions]+++def test_user_plan_overrides():+ model = build(LLAMA, lora = False)+ plan = {+ "layers.*.self_attn.q_proj": "colwise",+ "layers.*.self_attn.k_proj": "colwise",+ "layers.*.self_attn.v_proj": "colwise",+ "layers.*.self_attn.o_proj": "rowwise",+ "layers.*.mlp.gate_proj": "colwise",+ "layers.*.mlp.up_proj": "colwise",+ "layers.*.mlp.down_proj": "rowwise",+ }+ assert tpn.derive_tp_plan(model, 2, plan).source == "user"+ with pytest.raises(ValueError, match = "unknown tensor parallel style"):+ tpn.derive_tp_plan(model, 2, {**plan, "layers.*.mlp.down_proj": "diagonal"})+ # A plan entry wins over the name rule; names fill what the plan leaves out.+ swapped = tpn.derive_tp_plan(model, 2, {**plan, "layers.*.mlp.up_proj": "rowwise"}).styles()+ assert swapped["model.layers.0.mlp.up_proj"] == "rowwise"+ missing = {k: v for k, v in plan.items() if "down_proj" not in k}+ assert (+ tpn.derive_tp_plan(model, 2, missing).styles()+ == tpn.derive_tp_plan(model, 2, plan).styles()+ )+++def test_unknown_module_inside_region_is_refused():+ model = build(LLAMA, lora = False)+ model.model.layers[1].mlp.gate_scale = torch.nn.Linear(4, 4)+ with pytest.raises(ValueError, match = "no rule for 'model.layers.1.mlp.gate_scale'"):+ tpn.derive_tp_plan(model, 2)+ model = build(LLAMA, lora = False)+ model.model.layers[0].self_attn.sinks = torch.nn.Parameter(torch.zeros(4))+ with pytest.raises(ValueError, match = "owns the parameter 'sinks'"):+ tpn.derive_tp_plan(model, 2)+++def test_mla_config_refused():+ model = build(LLAMA, lora = False)+ tpn._text_config(model).kv_lora_rank = 16+ with pytest.raises(ValueError, match = "latent attention"):+ tpn.derive_tp_plan(model, 2)+++def test_local_heads_set_per_module_not_config():+ model = build(PHI3)+ text = tpn._text_config(model)+ heads, kv = text.num_attention_heads, text.num_key_value_heads+ sim, plan = simulate(model, 2)+ attn = tpn._unwrap(sim).get_submodule(plan.regions[0].path).copies[0]+ assert attn._unsloth_tp_heads == heads // 2 and attn._unsloth_tp_kv_heads == kv // 2+ assert attn.config.num_attention_heads == heads // 2 # Phi3 slices qkv with config heads+ assert tpn._text_config(sim).num_attention_heads == heads # shared config untouched+ assert tpn._text_config(model).num_attention_heads == heads+++def test_shard_slice_unpermute_round_trip():+ t = torch.arange(10 * 3).view(10, 3)+ parts = (6, 2, 2)+ shards = [tpn.shard_slice(t, 0, parts, r, 2) for r in range(2)]+ assert shards[0][:3].tolist() == t[:3].tolist() and shards[0][3].tolist() == t[6].tolist()+ assert torch.equal(tpn._unpermute(torch.cat(shards), 0, parts, 2), t)+++ "repo,overrides,tp,match",+ [+ (LLAMA, {"num_attention_heads": 4, "num_key_value_heads": 4}, 3, "num_attention_heads = 4"),+ (QWEN3, {"num_attention_heads": 6, "num_key_value_heads": 3}, 2, "num_key_value_heads = 3"),+ (GEMMA3, {}, 2, "larger than num_key_value_heads = 1"),+ (MISTRAL, {}, 2, "intermediate_size = 37"),+ (OLMO2, {}, 2, "normalizes .* features, not one head"),+ (QWEN3_MOE, {}, 2, "mixture-of-experts"),+ (DEEPSEEK_V3, {}, 2, "mixture-of-experts|latent attention"),+ ],+)+def test_refusals_plan(repo, overrides, tp, match):+ model = build(repo, overrides, lora = False)+ with pytest.raises(ValueError, match = match):+ tpn.derive_tp_plan(model, tp)+++def test_refuses_attention_and_layer_dropout():+ with pytest.raises(ValueError, match = "attention dropout"):+ tpn.derive_tp_plan(build(LLAMA, {"attention_dropout": 0.1}), 2)+ model = build(LLAMA)+ model.get_base_model().model.layers[0].mlp.add_module("drop", torch.nn.Dropout(0.1))+ with pytest.raises(ValueError, match = "dropout inside decoder layers"):+ tpn.derive_tp_plan(model, 2)+++def test_refuses_dora_and_lora_bias():+ for kwargs, match in (+ ({"use_dora": True}, "DoRA"),+ ({"lora_bias": True}, "lora_bias"),+ ({"lora_dropout": 0.1}, "lora_dropout"),+ ):+ model = build(LLAMA, **kwargs)+ region = tpn.derive_tp_plan(model, 2).regions[0]+ with pytest.raises(ValueError, match = match):+ tpn._refuse_adapter(region.path, region.projs[0].module)+++def test_refuses_quantized_and_trainable_bases():+ model = build(LLAMA)+ region = tpn.derive_tp_plan(model, 2).regions[0]+ proj = region.projs[0].module+ weight, base = tpn._weight(proj)+ with pytest.raises(ValueError, match = "fp8"):+ tpn._refuse_base("p", proj, weight.to(torch.float8_e4m3fn), base)+ int8 = type("Int8Params", (torch.nn.Parameter,), {})+ with pytest.raises(ValueError, match = "8-bit"):+ tpn._refuse_base("p", proj, int8(weight.detach()), base)+ gptq = type("QuantLinear", (torch.nn.Module,), {})()+ with pytest.raises(ValueError, match = "GPTQ"):+ tpn._refuse_base("p", proj, weight, gptq)+ with pytest.raises(ValueError, match = "full finetuning"):+ tpn._refuse_base("p", proj, torch.nn.Parameter(weight.detach().clone()), base)+++def test_refuses_global_width_attention():+ model = build(LLAMA, lora = False)+ attn = model.model.layers[0].self_attn++ def forward(self, hidden_states, **kwargs):+ return hidden_states.reshape(1, 1, self.config.hidden_size), None++ attn.forward = types.MethodType(forward, attn)+ with pytest.raises(ValueError, match = "hidden_size"):+ tpn.derive_tp_plan(model, 2)+++def test_stablelm_4x_is_refused():+ import transformers++ if int(transformers.__version__.split(".")[0]) >= 5:+ pytest.skip("transformers 5 StableLM attention follows the local heads")+ repo, overrides = ARCHS["stablelm"]+ with pytest.raises(ValueError, match = "reshapes with hidden_size"):+ tpn.derive_tp_plan(build(repo, overrides), 2)+++# Phi3's fused qkv under a plan that calls it packed: the layout is still [q, kv, kv] (GQA: unequal parts).+PHI3_PACKED_PLAN = {+ "layers.*.self_attn.qkv_proj": "packed_colwise",+ "layers.*.self_attn.o_proj": "rowwise",+ "layers.*.mlp.gate_up_proj": "packed_colwise",+ "layers.*.mlp.down_proj": "rowwise",+}+++def test_fused_qkv_under_a_packed_plan_keeps_the_gqa_layout():+ model = build(PHI3)+ text = tpn._text_config(model)+ assert text.num_key_value_heads < text.num_attention_heads # the tiny Phi3 is GQA (4 / 2)+ hd = text.hidden_size // text.num_attention_heads+ q, kv = text.num_attention_heads * hd, text.num_key_value_heads * hd+ ids = _batch(model)+ _, ref_logits = _loss(model, ids)+ sim, plan = simulate(model, 2, PHI3_PACKED_PLAN)+ parts = {p.name: p.parts for p in plan.regions[0].projs}+ assert parts["qkv_proj"] == (q, kv, kv), parts+ _, logits = _loss(sim, ids)+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+ with pytest.raises(ValueError, match = "has no known layout"):+ tpn.derive_tp_plan(+ build(LLAMA, lora = False), 2, {"layers.*.self_attn.q_proj": "packed_colwise"}+ )+++class _FakeLora(torch.nn.Module):+ def __init__(self, active):+ super().__init__()+ self.base_layer = torch.nn.Linear(4, 6, bias = False)+ self.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})+ self.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 6, bias = False)})+ self.scaling = {"default": 2.0}+ self.disable_adapters = False+ self.merged = False+ if active is not None:+ self.active_adapters = active+++def test_active_adapter_matches_get_lora_parameters():+ singular = _FakeLora(None)+ singular.active_adapter = "default"+ assert tpn._active_adapter(singular) == ["default"]+ assert tpn._active_adapter(_FakeLora(None)) == ["default"]+ assert tpn._active_adapter(_FakeLora("default")) == ["default"]+ assert tpn._active_adapter(_FakeLora([])) == []+ for active in (None, "default", ["default"]):+ W, _, A, B, s = tpn._tp_lora_parameters(_FakeLora(active))+ assert A is not None and B is not None and s == 2.0+ for active in ([], ["other"]):+ assert tpn._tp_lora_parameters(_FakeLora(active))[2:] == (None, None, None)+ with pytest.raises(ValueError, match = "single active adapter"):+ tpn._tp_lora_parameters(_FakeLora(["default", "other"]))+++def test_refuses_forward_qat_and_activation_fake_quantizers():+ model = build(LLAMA)+ region = tpn.derive_tp_plan(model, 2).regions[0]+ proj = region.projs[0].module+ weight, base = tpn._weight(proj)+ # torchao's NVFP4 / MX QAT linears fake-quantize inside forward() and carry no weight_fake_quantizer.+ forward_qat = type("NVFP4FakeQuantizedLinear", (torch.nn.Linear,), {})(4, 4).requires_grad_(+ False+ )+ with pytest.raises(ValueError, match = "QAT fake quantization"):+ tpn._refuse_base(+ "p", torch.nn.ModuleDict({"base_layer": forward_qat}), forward_qat.weight, forward_qat+ )+ try:+ from torchao.prototype import qat+ real_classes = [+ (qat.MXFakeQuantizedLinear, qat.MXFakeQuantizeConfig),+ (qat.NVFP4FakeQuantizedLinear, qat.NVFP4FakeQuantizeConfig),+ ]+ except Exception:+ real_classes = []+ for cls, cfg in real_classes: # the real torchao classes (weights frozen, as under LoRA QAT)+ real = cls(32, 32, activation_config = cfg(), weight_config = cfg()).requires_grad_(False)+ assert getattr(real, "weight_fake_quantizer", None) is None+ with pytest.raises(ValueError, match = "QAT fake quantization"):+ tpn._refuse_base("p", torch.nn.ModuleDict({"base_layer": real}), real.weight, real)+ proj.lora_A["default"].activation_fake_quantizer = torch.nn.Identity()+ with pytest.raises(ValueError, match = "QAT fake quantization"):+ tpn._refuse_base("p", proj, weight, base)+ del proj.lora_A["default"].activation_fake_quantizer+ tpn._refuse_base("p", proj, weight, base) # plain LoRA passes+++def test_refuses_modules_to_save_inside_a_region():+ model = build(+ LLAMA,+ target_modules = ["k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],+ modules_to_save = ["q_proj"],+ )+ region = tpn.derive_tp_plan(model, 2).regions[0]+ q = next(p for p in region.projs if p.name == "q_proj")+ with pytest.raises(ValueError, match = "modules_to_save"):+ tpn._refuse_adapter(f"{region.path}.q_proj", q.module)+++def test_region_hook_takes_the_forward_input_by_name():+ model = build(LLAMA, lora = False)+ mlp = model.model.layers[0].mlp+ mlp._unsloth_tp_group = (+ None # copy_to_tp's forward is the identity; the group is read in backward+ )+ mlp._unsloth_tp_input = "x" # what apply_tensor_parallel records for LlamaMLP.forward(self, x)+ h = torch.randn(1, 2, 16)+ args, kwargs = tpn._region_pre_hook(mlp, (), {"x": h}) # mlp(x = h)+ assert args == () and torch.equal(kwargs["x"], h) and kwargs["x"] is not h+ attn = model.model.layers[0].self_attn+ assert tpn._input_name(mlp) == "x" and tpn._input_name(attn) == "hidden_states"+ args, kwargs = tpn._region_pre_hook(mlp, (h,), {})+ assert torch.equal(args[0], h)+ with pytest.raises(RuntimeError, match = "got no tensor input"):+ tpn._region_pre_hook(mlp, (), {"y": h})+++def test_clip_takes_a_single_parameter():+ a = torch.nn.Parameter(torch.randn(5, 3))+ b = torch.nn.Parameter(a.detach().clone())+ a.grad = torch.randn(5, 3) * 4+ b.grad = a.grad.clone()+ ref = torch.nn.utils.clip_grad_norm_(b, 1.0)+ got = tpn.tp_clip_grad_norm_(a, 1.0)+ torch.testing.assert_close(got, ref)+ torch.testing.assert_close(a.grad, b.grad)+++def _load_flex():+ name = "_tp_flex_attention"+ if name not in sys.modules:+ spec = importlib.util.spec_from_file_location(+ name, os.path.join(REPO, "unsloth", "kernels", "flex_attention.py")+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ sys.modules[name] = module+ return sys.modules[name]+++def test_gemma2_softcapping_helpers_use_local_heads():+ """Gemma2 softcapping helpers honour _unsloth_tp_heads over the config's global counts."""+ flex = _load_flex()+ import types as _types++ def attn(+ heads,+ kv,+ tp_heads = None,+ tp_kv = None,+ ):+ config = _types.SimpleNamespace(+ num_attention_heads = heads,+ num_key_value_heads = kv,+ query_pre_attn_scalar = 16,+ attn_logit_softcapping = 50.0,+ )+ m = torch.nn.Module()+ m.config, m.head_dim, m.num_key_value_groups = config, 8, heads // kv+ if tp_heads is not None:+ m._unsloth_tp_heads, m._unsloth_tp_kv_heads = tp_heads, tp_kv+ return m++ bsz, q_len = 2, 5+ g = torch.Generator().manual_seed(0)+ Q = torch.randn(bsz, 2, q_len, 8, generator = g)+ K = torch.randn(bsz, 1, q_len, 8, generator = g)+ V = torch.randn(bsz, 1, q_len, 8, generator = g)+ mask = torch.triu(torch.full((q_len, q_len), float("-inf")), 1)+ local, tp_rank = attn(2, 1), attn(4, 2, 2, 1)+ eager = flex._compiled_slow_attention_softcapping._torchdynamo_orig_callable+ for fn in (eager, flex.slow_inference_attention_softcapping):+ want = fn(Q, K, V, mask, local, bsz, q_len)+ got = fn(Q, K, V, mask, tp_rank, bsz, q_len)+ assert got.shape == (bsz, q_len, 2 * 8)+ torch.testing.assert_close(got, want)+++def test_stub_packages_keep_valid_specs():+ # Later tests in the same session (e.g. the fused-loss harness) call find_spec("unsloth").+ for name in ("unsloth", "unsloth.models"):+ assert sys.modules[name].__spec__ is not None, name+ importlib.util.find_spec("unsloth.models")diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex 45b91164bb9..1560c9f3bb2 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -38,9 +38,10 @@ if not HAS_FLEX_ATTENTION:# Logit softcapping@torch.compile(fullgraph = True, dynamic = True, options = torch_compile_options)def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):- n_heads = self.config.num_attention_heads+ # This rank's heads under tensor parallelism (models/tp_native.py); the config counts otherwise.+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)head_dim = self.head_dim- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)n_groups = self.num_key_value_groupsactual_q_len = Q.shape[-2]@@ -90,7 +91,15 @@ if not HAS_FLEX_ATTENTION:def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):# Inductor indexes the scores in int32; past 2**31 elements the last rows come back NaN.- rows = max(1, (2**31 - 1) // (self.config.num_attention_heads * q_len * q_len))+ rows = max(+ 1,+ (2**31 - 1)+ // (+ self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)+ * q_len+ * q_len+ ),+ )if bsz <= rows:return _softcapping_attention(Q, K, V, causal_mask, self, bsz, q_len)per_row_mask = causal_mask.dim() == 4 and causal_mask.shape[0] == bsz@@ -167,7 +176,7 @@ else:)def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)head_dim = self.head_dims = self.config.query_pre_attn_scalart = self.config.attn_logit_softcapping@@ -184,9 +193,9 @@ torch_nn_functional_softmax = torch.nn.functional.softmaxdef slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)head_dim = self.head_dim- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)n_groups = self.num_key_value_groupsactual_q_len = Q.shape[-2]diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 94c812ac32b..9f7612b6db3 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -445,7 +445,11 @@ def _is_packed_state(quant_state):def get_lora_parameters(proj):"""Return (weight, weight quant_state, lora A, lora B, lora scale).With QAT enabled, also fake-quantizes the base layer and lora weights.+ Tensor parallel projections (models/tp_native.py) return local shards of their DTensors."""+ localize = proj.__dict__.get("_unsloth_tp_local")+ if localize is not None:+ return localize(proj)# For DPO or disabled adapters.base_layer = getattr(proj, "base_layer", proj)packed = _packed_base(base_layer)@@ -513,6 +517,9 @@ def get_lora_parameters(proj):def get_lora_parameters_bias(proj):+ localize = proj.__dict__.get("_unsloth_tp_local")+ if localize is not None:+ return localize(proj, bias = True)# For DPO or disabled adapters.base_layer = getattr(proj, "base_layer", proj)packed = _packed_base(base_layer)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 11f2f824dd5..e735910a6f7 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5698,6 +5698,7 @@ def patch_gradient_accumulation_fix(Trainer):install_fsdp2_policy_provenance()except Exception:pass+ _refuse_tensor_parallel_for_trainer(model)from ..fsdp2_launch import unsloth_fsdp2_native_requested# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.@@ -5772,6 +5773,7 @@ def patch_gradient_accumulation_fix(Trainer):# model_init rebuilds the model when training starts: the new instance needs the same# DDP exclusions as the one __init__ marked.model = _original_call_model_init(self, *args, **kwargs)+ _refuse_tensor_parallel_for_trainer(model)_tensor_subclass_weights_for_ddp(self, model)return model@@ -5782,6 +5784,35 @@ def patch_gradient_accumulation_fix(Trainer):_patch_trainer_train_gc_scope(Trainer)+def _is_unsloth_tensor_parallel(model):+ """Walk the wrapper levels for apply_tensor_parallel's marker (dict lookups, no tp_native import)."""+ m, depth = model, 0+ while isinstance(getattr(m, "__dict__", None), dict) and depth < 8:+ if m.__dict__.get("_unsloth_tp"):+ return True+ modules = m.__dict__.get("_modules") or {}+ m, depth = modules.get("base_model") or modules.get("model"), depth + 1+ return False+++def _refuse_tensor_parallel_for_trainer(model):+ if _is_unsloth_tensor_parallel(model):+ raise RuntimeError(+ "Unsloth: a tensor parallel model (apply_tensor_parallel) cannot be trained by the "+ "Trainer yet: accelerate would wrap the tensor parallel ranks in DDP. Use a custom "+ "training loop (unsloth/models/tp_native.py)."+ )+++def _refuse_tensor_parallel_for_inference(model):+ if _is_unsloth_tensor_parallel(model):+ raise RuntimeError(+ "Unsloth: for_inference is not supported on a tensor parallel model in v1 (the fast decode "+ "paths run outside the tensor parallel regions); use model.eval() for forward evaluation, or "+ "save the adapter (save_tp_adapter) and generate on one GPU."+ )++# Real children only: PEFT's LoraModel forwards `base_model` to the backbone, skipping the loss head._LOSS_KWARGS_CHILDREN = ("base_model", "model") + _UNSLOTH_WRAPPED_MODULE_ATTRSdiff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex d96a6f01224..425324b8331 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -109,9 +109,9 @@ def CohereAttention_fast_forward(bsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimassert n_kv_heads * n_groups == n_headsdiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex de94b144e33..ff07b6f209d 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -3461,6 +3461,16 @@ def _refuse_tensor_parallel_model(model):except ImportError:passdtensor = DTensor is not None and any(isinstance(p, DTensor) for p in model.parameters())+ # Any level or module (the marker sits on every wrapper level apply_tensor_parallel was given or below).+ if any(+ m.__dict__.get("_unsloth_tp") or m.__dict__.get("_unsloth_tp_style")+ for m in model.modules()+ ):+ raise RuntimeError(+ "Unsloth: native FSDP2 does not compose with Unsloth tensor parallelism "+ "(apply_tensor_parallel) yet; train a tensor parallel model with a custom loop over the tp "+ "ranks only (data parallel size 1)."+ )if (tp_size or 1) > 1 or dtensor:raise RuntimeError("Unsloth: native FSDP2 got a model that is already distributed (tensor-parallel tp_plan, "diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex b5240b2470f..68c76a224b3 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -157,9 +157,9 @@ def Gemma2Attention_fast_forward(bsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimassert n_kv_heads * n_groups == n_headsdiff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 2fea7d7c85d..e280ee90eba 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -106,9 +106,9 @@ def GraniteAttention_fast_forward(bsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimdropout_p = self.config.attention_dropout if self.training else 0assert n_kv_heads * n_groups == n_headsdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 4b5206dee99..277e06e8daa 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -22,6 +22,7 @@ from typing import Optional, Tuple, List, Unionfrom ._utils import *from ._utils import apply_unsloth_gradient_checkpointingfrom ._utils import _ignore_block_swap_params_for_ddp+from ._utils import _refuse_tensor_parallel_for_inferencefrom ._utils import __version__, importlib_versionfrom ._utils import move_to_devicefrom ._utils import per_layer_device@@ -996,9 +997,9 @@ def LlamaAttention_fast_forward(del self.attentionbsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimassert n_kv_heads * n_groups == n_heads@@ -5345,6 +5346,7 @@ class FastLlamaModel:raise TypeError("Unsloth: I think you're passing a tokenizer, not the model to for_inference!")+ _refuse_tensor_parallel_for_inference(model)def _for_inference(m):if hasattr(m, "gradient_checkpointing"):diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 812818f2a00..a8d749818cb 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -87,9 +87,9 @@ def MistralAttention_fast_forward(bsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimassert n_kv_heads * n_groups == n_headsdiff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex cef4a20b2f8..3392c438f63 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -91,9 +91,9 @@ def Qwen3Attention_fast_forward(bsz, q_len, _ = hidden_states.size()- n_heads = self.config.num_attention_heads+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)n_groups = self.num_key_value_groups- n_kv_heads = self.config.num_key_value_heads+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)head_dim = self.head_dimassert n_kv_heads * n_groups == n_headsdiff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.pynew file mode 100644index 00000000000..0d97139218a--- /dev/null+++ b/unsloth/models/tp_native.py@@ -0,0 +1,1341 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Tensor parallelism (column / row sharding of attention and MLP) for any dense decoder, on DTensor.++ model = FastLanguageModel.get_peft_model(model, ...) # LoRA first: same init as 1 GPU+ apply_tensor_parallel(model, init_device_mesh("cuda", (tp,), mesh_dim_names = ("tp",))["tp"])++v1: LoRA / QLoRA training forward on dense decoders, tp ranks only (no Trainer / FSDP2 / generate).+Custom loop (torchrun --nproc_per_node tp):++ params = [p for p in model.parameters() if p.requires_grad] # DTensors (keep plain ones in their own group)+ optimizer = torch.optim.AdamW(params, lr = 2e-4)+ loss = model(input_ids = ids, labels = ids).loss # same batch on every tp rank+ loss.backward()+ tp_clip_grad_norm_(params, 1.0)+ optimizer.step(); optimizer.zero_grad()+ save_tp_adapter(model, "out") # collective: the standard 1-GPU PEFT adapter, written by rank 0+"""++from __future__ import annotations++import copy+import dataclasses+import inspect+import re+import types+from typing import Optional++import torch+import torch.nn as nn+import torch.nn.functional as F++__all__ = [+ "TPConfig",+ "TPPlan",+ "apply_tensor_parallel",+ "derive_tp_plan",+ "is_tensor_parallel",+ "shard_slice",+ "tp_clip_grad_norm_",+ "tp_full_state_dict",+ "save_tp_adapter",+]++_TP_FLAG = "_unsloth_tp"+COLWISE = "colwise"+ROWWISE = "rowwise"+HEAD_NORM = "replicated_with_grad_allreduce"++# transformers TP styles (4.5x and 5.x names) -> what this module does with them.+_STYLES = {+ "colwise": COLWISE,+ "local_colwise": COLWISE,+ "colwise_rep": COLWISE,+ "colwise_gather_output": COLWISE,+ "packed_colwise": COLWISE,+ "local_packed_colwise": COLWISE,+ "rowwise": ROWWISE,+ "local_rowwise": ROWWISE,+ "rowwise_rep": ROWWISE,+ "rowwise_split_input": ROWWISE,+ "replicated_with_grad_allreduce": HEAD_NORM,+}+# Outside the decoder regions (embeddings, lm_head, final norm): replicated in v1.+_REPLICATED_STYLES = {+ "embedding_rowwise",+ "gather",+ "sequence_parallel",+ "all_reduce",+ "replicate",+ "isolated",+}+_MOE_STYLES = {+ "grouped_gemm",+ "ep_router",+ "megamoe_router",+ "moe_tp_experts",+ "megamoe_experts",+ "moe_identity_expert",+ "packed_rowwise",+ "local_packed_rowwise",+}+_MLA_STYLES = {"mla_kv_a_proj"}++_ATTN_NAMES = ("self_attn", "attention", "attn", "self_attention")+_MLP_NAMES = ("mlp", "feed_forward", "ffn")+_Q, _K, _V = ("q_proj", "query"), ("k_proj", "key"), ("v_proj", "value")+_FUSED_QKV = ("qkv_proj",) # [q, k, v] along the output dim (Phi3)+_ATTN_ROW = ("o_proj", "out_proj", "dense")+_MLP_COL = ("gate_proj", "up_proj", "w1", "w3", "fc1", "c_fc")+_MLP_PACKED = ("gate_up_proj",) # [gate, up] along the output dim+_MLP_ROW = ("down_proj", "w2", "fc2")+_HEAD_NORMS = ("q_norm", "k_norm", "query_norm", "key_norm")+_MLA_NAMES = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "kv_a_layernorm")+_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")+# An attention forward reading these sizes off the config would reshape to the global head count.+_GLOBAL_HEAD_READS = re.compile(r"config\.(num_attention_heads|num_key_value_heads)\b")+_GLOBAL_WIDTH_READS = re.compile(r"(config|self)\.hidden_size\b")+++class TPConfig:+ # {"layers.*.self_attn.q_proj": "colwise", ...}; None = the model's TP plan, else structural discovery.+ plan: Optional[dict] = None+ # Broadcast trainable parameters from tp rank 0 first (every rank must start from the same adapter).+ sync_trainable: bool = True+++class _Proj:+ name: str+ module: nn.Module+ style: str # COLWISE / ROWWISE+ parts: tuple # output (colwise) part sizes; one part unless fused / packed+ from_plan: bool = False+++class _Region:+ path: str+ module: nn.Module+ kind: str # "attention" / "mlp"+ projs: list+ norms: list # per-head norms inside the region: replicated, gradients summed over tp+ heads: int = 0+ kv_heads: int = 0+ head_dim: int = 0+++class TPPlan:+ tp_size: int+ regions: list+ source: str # "user" / "transformers" / "structural"+ replicated_layers: int = (+ 0 # decoder layers with no region (e.g. linear attention): run replicated+ )++ def styles(self):+ """{projection path: style} (the plan in transformers form, local paths)."""+ return {f"{r.path}.{p.name}": p.style for r in self.regions for p in r.projs}+++def _unwrap(model):+ inner = model+ if hasattr(inner, "get_base_model"):+ inner = inner.get_base_model()+ return inner+++def _text_config(model):+ config = getattr(_unwrap(model), "config", None)+ if config is None:+ return None+ get = getattr(config, "get_text_config", None)+ return get() if callable(get) else config+++def _hf_plan(model):+ inner = _unwrap(model)+ plan = {}+ text = _text_config(model)+ for source in (getattr(text, "base_model_tp_plan", None), getattr(inner, "_tp_plan", None)):+ if isinstance(source, dict):+ plan.update(source)+ return plan+++def _pattern(key):+ return re.compile(r"(^|\.)" + re.escape(key).replace(r"\*", r"\d+") + r"$")+++def _style_for(path, compiled):+ for key, regex, style in compiled:+ if regex.search(path):+ return key, style+ return None, None+++def _weight(module):+ base = module._modules.get("base_layer", module)+ return base._parameters.get("weight", getattr(base, "weight", None)), base+++def _has_params(module):+ return next(module.parameters(), None) is not None+++def _decoder_layers(model):+ """The ModuleList of decoder layers (len == text config num_hidden_layers)."""+ inner = _unwrap(model)+ decoder = inner+ get_decoder = getattr(inner, "get_decoder", None)+ if callable(get_decoder):+ try:+ decoder = get_decoder() or inner+ except Exception:+ decoder = inner+ text = _text_config(model)+ n = getattr(text, "num_hidden_layers", None)+ prefix = ""+ for name, module in inner.named_modules():+ if module is decoder:+ prefix = name+ break+ for name, module in decoder.named_modules():+ if isinstance(module, nn.ModuleList) and len(module) and (n is None or len(module) == n):+ path = ".".join(x for x in (prefix, name) if x)+ return path, module+ raise ValueError(+ "Unsloth: tensor parallelism found no decoder layer list "+ f"({type(inner).__name__}: no ModuleList of num_hidden_layers = {n} layers)."+ )+++_ATTENTION_DROPOUT_KEYS = ("attention_dropout", "attn_pdrop")+++def _refuse_dropout(model, layers):+ """v1 has no TP-aware RNG (masks must differ per shard, match when replicated): refuse dropout."""+ text = _text_config(model)+ for key in _ATTENTION_DROPOUT_KEYS:+ value = getattr(text, key, None)+ if isinstance(value, (int, float)) and value > 0:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support attention dropout ({key} = {value}); set it to 0."+ )+ for name, module in layers.named_modules():+ if "lora_dropout" in name or not isinstance(module, nn.modules.dropout._DropoutNd):+ continue+ if module.p > 0:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support dropout inside decoder layers ({name}: p = "+ f"{module.p}); set it to 0."+ )+++def _refuse_moe_mla(model, layers):+ text = _text_config(model)+ for key in _MOE_CONFIG_KEYS:+ value = getattr(text, key, None)+ if isinstance(value, int) and value > 0:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts models ({key} = {value}): "+ "expert sharding is not implemented. Use native FSDP2 (UNSLOTH_FSDP2=1) instead."+ )+ if getattr(text, "kv_lora_rank", None):+ raise ValueError(+ "Unsloth: tensor parallelism v1 does not support multi-head latent attention (MLA, kv_lora_rank "+ f"= {text.kv_lora_rank}). Use native FSDP2 (UNSLOTH_FSDP2=1) instead."+ )+ for name, module in layers.named_modules():+ leaf = name.rsplit(".", 1)[-1]+ cls = type(module).__name__+ if leaf == "experts" or "Experts" in cls or "SparseMoe" in cls or "MoE" in cls:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ({name}: {cls})."+ )+ if leaf in _MLA_NAMES:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ({name})."+ )+++def _classify(region_path, name, child, kind, compiled):+ """(style, parts kind) for one child of an attention / MLP region, from the plan or its name."""+ key, style = _style_for(f"{region_path}.{name}", compiled) if compiled else (None, None)+ if style is not None:+ if style in _MOE_STYLES:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support the MoE style '{style}' ({key})."+ )+ if style in _MLA_STYLES:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support the MLA style '{style}' ({key})."+ )+ if style in _REPLICATED_STYLES:+ raise ValueError(+ f"Unsloth: '{key}': '{style}' inside a decoder {kind} is not supported by tensor parallelism v1."+ )+ if style not in _STYLES:+ raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")+ raw, style = style, _STYLES[style]+ # A fused qkv is [q, kv, kv] (GQA: unequal parts) whatever the plan calls it; "packed" means equal halves.+ if style == COLWISE and kind == "attention" and name in _FUSED_QKV:+ return COLWISE, "qkv:plan"+ if style == COLWISE and (raw.endswith("packed_colwise") or name in _MLP_PACKED):+ if kind == "attention":+ raise ValueError(+ f"Unsloth: '{key}': '{raw}' on the attention projection '{name}' has no known layout "+ "(tensor parallelism v1 knows fused qkv_proj); pass plan = {...} with 'colwise'."+ )+ return COLWISE, "packed:plan"+ return style, "plan"+ if kind == "attention":+ if name in _Q + _K + _V:+ return COLWISE, None+ if name in _FUSED_QKV:+ return COLWISE, "qkv"+ if name in _ATTN_ROW:+ return ROWWISE, None+ if name in _HEAD_NORMS:+ return HEAD_NORM, None+ else:+ if name in _MLP_COL:+ return COLWISE, None+ if name in _MLP_PACKED:+ return COLWISE, "packed"+ if name in _MLP_ROW:+ return ROWWISE, None+ return None, None+++def _attention_sizes(module, projs, text):+ """(heads, kv_heads, head_dim) of one attention module, from its own projections."""+ heads_cfg = getattr(text, "num_attention_heads", None)+ kv_cfg = getattr(text, "num_key_value_heads", None) or heads_cfg+ head_dim = getattr(module, "head_dim", None)+ if not isinstance(head_dim, int):+ head_dim = getattr(text, "head_dim", None) or (text.hidden_size // heads_cfg)+ by_name = {p.name: p for p in projs}+ q = next((by_name[n] for n in _Q if n in by_name), None)+ k = next((by_name[n] for n in _K if n in by_name), None)+ if q is not None and k is not None:+ heads = _logical_shape(*_weight(q.module))[0] // head_dim+ kv_heads = _logical_shape(*_weight(k.module))[0] // head_dim+ else:+ heads, kv_heads = heads_cfg, kv_cfg+ return heads, kv_heads, head_dim+++def _check_divisible(what, value, tp, path):+ if value % tp != 0:+ raise ValueError(+ f"Unsloth: tensor parallel size {tp} does not divide {what} = {value} ({path}). "+ f"Choose a tp size that divides it."+ )+++def derive_tp_plan(+ model,+ tp_size,+ plan = None,+) -> TPPlan:+ """The regions to shard, validated for tp_size; ValueError for anything v1 cannot shard."""+ tp_size = int(tp_size)+ if tp_size < 1:+ raise ValueError(f"Unsloth: tensor parallel size must be >= 1, got {tp_size}.")+ user = bool(plan)+ plan = dict(plan) if plan else _hf_plan(model)+ compiled = [(k, _pattern(k), v) for k, v in plan.items()]+ layers_path, layers = _decoder_layers(model)+ # Config / module MoE and MLA first: older transformers plans name expert styles v1 does not know.+ _refuse_moe_mla(model, layers)+ _refuse_dropout(model, layers)+ for key, _, style in compiled:+ if style in _MOE_STYLES:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ('{key}': '{style}')."+ )+ if style in _MLA_STYLES:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ('{key}')."+ )+ if style not in _STYLES and style not in _REPLICATED_STYLES:+ raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")+ text = _text_config(model)+ regions, replicated = [], 0+ for index, layer in enumerate(layers):+ found = 0+ for child_name, child in layer.named_children():+ kind = (+ "attention"+ if child_name in _ATTN_NAMES+ else ("mlp" if child_name in _MLP_NAMES else None)+ )+ if kind is None or not _has_params(child):+ continue+ path = f"{layers_path}.{index}.{child_name}"+ regions.append(_region(path, child, kind, compiled, text, tp_size))+ found += 1+ replicated += found == 0+ if not regions:+ raise ValueError(+ "Unsloth: tensor parallelism found no attention / MLP modules to shard in the decoder layers."+ )+ if user:+ source = "user"+ else:+ source = (+ "transformers" if any(p.from_plan for r in regions for p in r.projs) else "structural"+ )+ return TPPlan(tp_size = tp_size, regions = regions, source = source, replicated_layers = replicated)+++def _region(path, module, kind, compiled, text, tp):+ projs, norms = [], []+ for name, param in module.named_parameters(recurse = False):+ raise ValueError(+ f"Unsloth: tensor parallelism cannot shard '{path}': it owns the parameter '{name}' directly "+ "(not inside a projection), which v1 has no rule for."+ )+ for name, child in module.named_children():+ if not _has_params(child):+ continue+ style, packing = _classify(path, name, child, kind, compiled)+ if style is None:+ raise ValueError(+ f"Unsloth: tensor parallelism has no rule for '{path}.{name}' ({type(child).__name__}); "+ "every module with parameters inside a sharded attention / MLP must be in the plan. Pass "+ "plan = {...} to apply_tensor_parallel."+ )+ if style == HEAD_NORM:+ norms.append((name, child))+ continue+ weight, _ = _weight(child)+ if weight is None or getattr(weight, "ndim", 0) != 2:+ raise ValueError(+ f"Unsloth: tensor parallelism expects a Linear at '{path}.{name}' "+ f"(got {type(child).__name__})."+ )+ from_plan = packing is not None and packing.endswith("plan")+ packing = packing.split(":")[0] if packing and packing != "plan" else None+ projs.append(_Proj(name, child, style, packing, from_plan))+ if not any(p.style == COLWISE for p in projs) or not any(p.style == ROWWISE for p in projs):+ raise ValueError(+ f"Unsloth: '{path}' needs column and row sharded projections for tensor parallelism; found "+ f"{[(p.name, p.style) for p in projs]}."+ )+ region = _Region(path, module, kind, projs, norms)+ if kind == "attention":+ heads, kv_heads, head_dim = _attention_sizes(module, projs, text)+ region.heads, region.kv_heads, region.head_dim = heads, kv_heads, head_dim+ _check_divisible("num_attention_heads", heads, tp, path)+ if kv_heads < tp:+ raise ValueError(+ f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} "+ f"({path}); replicating KV heads is not supported in v1. Use tp <= {kv_heads}."+ )+ _check_divisible("num_key_value_heads", kv_heads, tp, path)+ for name, norm in norms:+ numel = sum(p.numel() for p in norm.parameters())+ if numel != head_dim:+ raise ValueError(+ f"Unsloth: '{path}.{name}' normalizes {numel} features, not one head ({head_dim}): a norm "+ "across all heads needs the full projection, which tensor parallelism v1 does not gather."+ )+ _check_attention_source(module, path)+ elif norms:+ raise ValueError(f"Unsloth: unexpected per-head norm in MLP '{path}'.")+ for proj in projs:+ weight, base = _weight(proj.module)+ out_features, in_features = _logical_shape(weight, base)+ if proj.style == ROWWISE:+ _check_divisible(f"{proj.name} in_features", in_features, tp, path)+ proj.parts = (out_features,)+ continue+ if proj.parts == "qkv":+ q, kv = region.heads * region.head_dim, region.kv_heads * region.head_dim+ if q + 2 * kv != out_features:+ raise ValueError(+ f"Unsloth: fused '{path}.{proj.name}' has {out_features} outputs, not q + 2 kv = "+ f"{q + 2 * kv}; its layout is unknown to tensor parallelism v1."+ )+ proj.parts = (q, kv, kv)+ elif proj.parts == "packed":+ if out_features % 2:+ raise ValueError(f"Unsloth: packed '{path}.{proj.name}' has an odd output size.")+ proj.parts = (out_features // 2, out_features // 2)+ else:+ proj.parts = (out_features,)+ for size in proj.parts:+ what = "intermediate_size" if kind == "mlp" else f"{proj.name} out_features"+ _check_divisible(what, size, tp, path)+ return region+++def _logical_shape(weight, base):+ state = getattr(weight, "quant_state", None)+ if state is not None and getattr(state, "shape", None) is not None:+ return tuple(state.shape)+ return tuple(weight.shape)+++def _forward_source(module):+ forward = module.forward+ for _ in range(4):+ forward = getattr(forward, "__func__", forward)+ forward = inspect.unwrap(forward)+ # torch.compile'd forwards (unsloth_zoo temporary patches) keep the original here.+ inner = getattr(forward, "_torchdynamo_orig_callable", None)+ if inner is None:+ break+ forward = inner+ return inspect.getsource(forward)+++def _check_attention_source(module, path):+ try:+ source = _forward_source(module)+ except (OSError, TypeError):+ raise ValueError(+ f"Unsloth: tensor parallelism cannot read the forward of '{path}' ({type(module).__name__}) to "+ "check it uses this rank's head count."+ )+ if _GLOBAL_WIDTH_READS.search(source) and "_unsloth_tp_heads" not in source:+ raise ValueError(+ f"Unsloth: the attention forward of '{path}' ({type(module).__name__}) reshapes with hidden_size, "+ "the global width; tensor parallelism v1 needs it to follow the local heads."+ )+++def shard_slice(tensor, dim, parts, rank, tp):+ """This rank's slice of `tensor` along `dim`: the rank-th 1/tp of every part, concatenated (a copy)."""+ pieces, offset = [], 0+ for size in parts:+ step = size // tp+ pieces.append(tensor.narrow(dim, offset + rank * step, step))+ offset += size+ return torch.cat(pieces, dim = dim)+++def _unpermute(tensor, dim, parts, tp):+ """Inverse of concatenating every rank's shard_slice: the original part order."""+ if len(parts) == 1:+ return tensor+ local = sum(parts) // tp+ chunks = tensor.split(local, dim = dim) # one per rank+ pieces = []+ for i, size in enumerate(parts):+ step = size // tp+ start = sum(p // tp for p in parts[:i])+ pieces.extend(chunk.narrow(dim, start, step) for chunk in chunks)+ return torch.cat(pieces, dim = dim)+++def _dtensor_api():+ from torch.distributed.tensor import DTensor, Partial, Replicate, Shard+ return DTensor, Partial, Replicate, Shard+++def _is_dtensor(t):+ try:+ from torch.distributed.tensor import DTensor+ except ImportError:+ return False+ return isinstance(t, DTensor)+++def _local_complete(t):+ """A local view whose gradient is already complete (e.g. row bias): never summed over tp again."""+ DTensor, _, Replicate, _ = _dtensor_api()+ if not isinstance(t, DTensor):+ return t+ if t.requires_grad and any(p.is_replicate() for p in t.placements):+ return t.to_local(grad_placements = [Replicate()])+ return t.to_local()+++def _local(t):+ """A local view of a DTensor for local compute; replicated trainable tensors return Partial grads."""+ DTensor, Partial, _, _ = _dtensor_api()+ if not isinstance(t, DTensor):+ return t+ if t.requires_grad and any(p.is_replicate() for p in t.placements):+ return t.to_local(grad_placements = [Partial()])+ return t.to_local()+++def _tp_lora_parameters(proj, bias = False):+ """get_lora_parameters(_bias) on local shards; row bias is None (the region adds it after the reduce)."""+ weight, base = _weight(proj)+ W = _local(weight)+ W_quant = getattr(W, "quant_state", None)+ b = None+ if bias and proj.__dict__.get("_unsloth_tp_style") == COLWISE and base.bias is not None:+ b = _local(base.bias)+ lora_A = proj._modules.get("lora_A")+ adapter = None+ if lora_A is not None and not (+ getattr(proj, "disable_adapters", True) or getattr(proj, "merged", False)+ ):+ adapters = _active_adapter(proj)+ if len(adapters) > 1:+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")+ if adapters and adapters[0] in lora_A:+ adapter = adapters[0]+ if adapter is None:+ out = (W, W_quant, None, None, None)+ else:+ out = (+ W,+ W_quant,+ _local(lora_A[adapter].weight),+ _local(proj.lora_B[adapter].weight),+ proj.scaling[adapter],+ )+ return out + (b,) if bias else out+++def _replicate_grad(grad):+ # The Partial gradient of a replicated factor becomes its tp sum here (one all-reduce per backward).+ if _is_dtensor(grad) and any(p.is_partial() for p in grad.placements):+ _, _, Replicate, _ = _dtensor_api()+ return grad.redistribute(grad.device_mesh, [Replicate()])+ return grad+++def _is_params4bit(t):+ return any(cls.__name__ == "Params4bit" for cls in type(t).__mro__)+++def _refuse_base(path, proj_module, weight, base):+ cls = type(base).__name__+ if weight is None:+ raise ValueError(f"Unsloth: '{path}' ({cls}) has no dense weight (GPTQ / AWQ?).")+ if getattr(base, "qweight", None) is not None or cls.startswith(("QuantLinear", "WQLinear")):+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support GPTQ / AWQ weights ({path})."+ )+ if cls == "Linear8bitLt" or type(weight).__name__ == "Int8Params":+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support bitsandbytes 8-bit weights ({path}); load in "+ "4-bit or 16-bit."+ )+ fp8 = {+ getattr(torch, n, None)+ for n in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")+ }+ if weight.dtype in fp8 or getattr(base, "weight_scale", None) is not None:+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support fp8 weights ({path}).")+ if getattr(type(base), "_unsloth_mxfp4_packed_linear", False):+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support packed MXFP4 weights ({path})."+ )+ for module in proj_module.modules():+ forward_qat = any("FakeQuantized" in c.__name__ for c in type(module).__mro__)+ if (+ forward_qat+ or getattr(module, "weight_fake_quantizer", None) is not None+ or getattr(module, "activation_fake_quantizer", None) is not None+ ):+ # The TP projection forward and get_lora_parameters run plain matmuls on the local shards.+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support QAT fake quantization ({path}: "+ f"{type(module).__name__})."+ )+ if not _is_params4bit(weight) and type(weight) not in (nn.Parameter, torch.Tensor):+ raise ValueError(+ f"Unsloth: tensor parallelism v1 cannot shard '{path}' weights of type {type(weight).__name__}."+ )+ if not _is_params4bit(weight) and not weight.is_floating_point():+ raise ValueError(+ f"Unsloth: tensor parallelism v1 cannot shard {weight.dtype} weights ({path})."+ )+ if weight.requires_grad:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 shards LoRA / QLoRA training only; '{path}' has a trainable base "+ "weight (full finetuning). Use native FSDP2 for full finetuning."+ )+++def _active_adapter(proj):+ """Active adapter names, resolved like kernels/utils.py get_lora_parameters; empty stays empty."""+ adapters = getattr(proj, "active_adapters", None)+ if adapters is None:+ adapters = getattr(proj, "active_adapter", "default")+ if adapters is None:+ return []+ if isinstance(adapters, str):+ return [adapters]+ return list(adapters)+++def _refuse_adapter(path, proj):+ if "original_module" in proj._modules or "modules_to_save" in proj._modules:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 cannot shard '{path}': it is a PEFT modules_to_save / trainable "+ "copy (a full trainable projection). Keep modules_to_save outside the attention / MLP projections."+ )+ if "lora_A" not in proj._modules:+ if "base_layer" in proj._modules:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 supports LoRA adapters only ({path}: {type(proj).__name__})."+ )+ return+ names = list(proj.lora_A.keys())+ if len(names) > 1:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 supports one LoRA adapter per layer ({path} has {names})."+ )+ magnitude = getattr(proj, "lora_magnitude_vector", None)+ if magnitude is not None and len(magnitude):+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support DoRA ({path}).")+ variants = proj.__dict__.get("lora_variant")+ if variants:+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support LoRA variants ({path}).")+ for name in names:+ dropout = proj.lora_dropout[name] if "lora_dropout" in proj._modules else None+ if getattr(dropout, "p", 0) > 0:+ # Masks must match across ranks for replicated inputs and differ for sharded ones.+ raise ValueError(+ f"Unsloth: tensor parallelism v1 does not support lora_dropout > 0 ({path}); use lora_dropout = 0."+ )+ if proj.lora_B[name].bias is not None:+ raise ValueError(f"Unsloth: tensor parallelism v1 does not support lora_bias ({path}).")+ if type(proj.lora_A[name]) is not nn.Linear:+ raise ValueError(+ f"Unsloth: tensor parallelism v1 supports Linear LoRA factors only ({path})."+ )+++def _as_param(tensor, like):+ param = nn.Parameter(tensor, requires_grad = like.requires_grad)+ return param+++def _set_dtensor(+ module,+ name,+ full,+ mesh,+ placement,+ parts = None,+ shard_dim = None,+ rank = 0,+ tp = 1,+):+ """Replace module.<name> by a DTensor parameter (Shard with parts permuted per rank, or Replicate)."""+ DTensor, _, Replicate, Shard = _dtensor_api()+ if placement == "shard":+ local = shard_slice(full.detach(), shard_dim, parts, rank, tp)+ shape = list(full.shape)+ dt = DTensor.from_local(+ local,+ mesh,+ [Shard(shard_dim)],+ run_check = False,+ shape = torch.Size(shape),+ stride = torch.empty(shape, device = "meta").stride(),+ )+ else:+ dt = DTensor.from_local(full.detach(), mesh, [Replicate()], run_check = False)+ param = _as_param(dt, full)+ if placement == "shard" and len(parts) > 1:+ param._unsloth_tp_parts = (shard_dim, tuple(parts))+ module._parameters[name] = param+ if param.requires_grad and placement != "shard":+ param.register_hook(_replicate_grad)+ return param+++def _check_params4bit(path, weight, style, tp):+ """The 4-bit blocksize refusals, run before anything is sharded."""+ state = weight.quant_state+ blocksize = state.blocksize+ in_features = state.shape[1]+ if in_features % blocksize or blocksize % 2:+ raise ValueError(+ f"Unsloth: 4-bit '{path}': in_features = {in_features} is not a multiple of the quantization "+ f"blocksize {blocksize}; its blocks span rows, which tensor parallelism v1 cannot split."+ )+ if style == ROWWISE and (in_features // tp) % blocksize:+ raise ValueError(+ f"Unsloth: 4-bit '{path}': in_features / tp = {in_features // tp} is not a multiple of the "+ f"quantization blocksize {blocksize}; row sharding would split blocks."+ )+++def _shard_params4bit(path, base, weight, style, parts, rank, tp):+ """This rank's 4-bit shard (bitwise slice of the 1-GPU weight); double-quant absmax kept as float32."""+ import bitsandbytes as bnb++ _check_params4bit(path, weight, style, tp)+ state = weight.quant_state+ blocksize = state.blocksize+ out_features, in_features = state.shape+ codes = weight.data.reshape(-1).view(torch.uint8).view(out_features, in_features // 2)+ absmax = state.absmax+ if state.nested:+ absmax = bnb.functional.dequantize_blockwise(state.absmax, state.state2) + state.offset+ absmax = absmax.float().view(out_features, in_features // blocksize)+ if style == COLWISE:+ codes = shard_slice(codes, 0, parts, rank, tp)+ absmax = shard_slice(absmax, 0, parts, rank, tp)+ shape = (sum(parts) // tp, in_features)+ else:+ codes = shard_slice(codes, 1, (in_features // 2,), rank, tp)+ absmax = shard_slice(absmax, 1, (in_features // blocksize,), rank, tp)+ shape = (out_features, in_features // tp)+ storage = weight.data.dtype+ packed = codes.contiguous().view(storage).reshape(-1, 1)+ local_state = bnb.functional.QuantState(+ absmax = absmax.contiguous(),+ shape = torch.Size(shape),+ code = state.code,+ blocksize = blocksize,+ quant_type = state.quant_type,+ dtype = state.dtype,+ )+ new = bnb.nn.Params4bit(+ packed,+ requires_grad = False,+ quant_state = local_state,+ blocksize = blocksize,+ compress_statistics = False,+ quant_type = state.quant_type,+ quant_storage = storage,+ module = base,+ bnb_quantized = True,+ )+ new._unsloth_tp_layout = (style, tuple(parts), rank, tp)+ base._parameters["weight"] = new+ if "quant_state" in base.__dict__:+ base.quant_state = local_state+++def _all_reduce(tensor, group):+ import torch.distributed._functional_collectives as funcol++ out = funcol.all_reduce(tensor.contiguous(), "sum", group)+ wait = getattr(out, "wait", None)+ return wait() if callable(wait) else out+++class _CopyToTP(torch.autograd.Function):+ """Identity forward, all-reduce backward: the region input every rank reads in full."""++ @staticmethod+ def forward(ctx, x, group):+ ctx.group = group+ return x.view_as(x)++ @staticmethod+ def backward(ctx, grad):+ return _all_reduce(grad, ctx.group), None+++class _ReduceFromTP(torch.autograd.Function):+ """All-reduce forward, identity backward: the row projection's partial sums."""++ @staticmethod+ def forward(ctx, x, group):+ return _all_reduce(x, group)++ @staticmethod+ def backward(ctx, grad):+ return grad, None+++def copy_to_tp(x, group):+ return _CopyToTP.apply(x, group)+++def reduce_from_tp(x, group):+ return _ReduceFromTP.apply(x, group)+++def _input_name(module):+ """The name of the region forward's first argument (hidden_states, x, ...), for keyword calls."""+ try:+ params = list(inspect.signature(module.forward).parameters.values())+ except (TypeError, ValueError):+ return "hidden_states"+ if params and params[0].kind in (params[0].POSITIONAL_ONLY, params[0].POSITIONAL_OR_KEYWORD):+ return params[0].name+ return "hidden_states"+++def _region_pre_hook(module, args, kwargs):+ group = module._unsloth_tp_group+ for name in (module.__dict__.get("_unsloth_tp_input", "hidden_states"), "hidden_states"):+ if isinstance(kwargs.get(name), torch.Tensor):+ kwargs = dict(kwargs)+ kwargs[name] = copy_to_tp(kwargs[name], group)+ return args, kwargs+ if args and isinstance(args[0], torch.Tensor):+ return (copy_to_tp(args[0], group),) + tuple(args[1:]), kwargs+ raise RuntimeError(+ f"Unsloth: tensor parallel region {type(module).__name__} got no tensor input "+ f"({module.__dict__.get('_unsloth_tp_input', 'hidden_states')})."+ )+++def _region_finish(module, y):+ y = reduce_from_tp(y, module._unsloth_tp_group)+ for proj in module._unsloth_tp_row_projs:+ bias = _weight(proj)[1].bias+ if bias is not None:+ y = y + _local_complete(bias).to(y.dtype)+ return y+++def _region_post_hook(module, args, output):+ if isinstance(output, torch.Tensor):+ return _region_finish(module, output)+ if isinstance(output, (tuple, list)) and output and isinstance(output[0], torch.Tensor):+ return type(output)((_region_finish(module, output[0]), *output[1:]))+ raise RuntimeError(+ f"Unsloth: tensor parallel region {type(module).__name__} returned {type(output)}."+ )+++def _tp_projection_forward(self, x, *args, **kwargs):+ weight, base = _weight(self)+ W = _local(weight)+ bias = (+ base.bias if self._unsloth_tp_style == COLWISE else None+ ) # row bias: once, after the reduce+ bias = None if bias is None else _local(bias)+ input_dtype = x.dtype+ if _is_params4bit(W):+ import bitsandbytes as bnb++ compute = getattr(base, "compute_dtype", None) or x.dtype+ out = bnb.matmul_4bit(x.to(compute), W.t(), bias = None, quant_state = W.quant_state)+ if bias is not None:+ out = out + bias.to(out.dtype)+ out = out.to(input_dtype)+ else:+ out = F.linear(x, W, bias)+ lora_A = self._modules.get("lora_A")+ if lora_A is None or getattr(self, "disable_adapters", True) or getattr(self, "merged", False):+ return out+ for adapter in _active_adapter(self) or ():+ if adapter not in lora_A:+ continue+ A = _local(lora_A[adapter].weight)+ B = _local(self.lora_B[adapter].weight)+ scaling = self.scaling[adapter]+ x_lora = x if torch.is_autocast_enabled() else x.to(A.dtype)+ x_lora = self.lora_dropout[adapter](x_lora)+ out = out + (F.linear(F.linear(x_lora, A), B) * scaling).to(out.dtype)+ return out+++def _decoder_of(inner):+ get_decoder = getattr(inner, "get_decoder", None)+ if not callable(get_decoder):+ return None+ try:+ decoder = get_decoder()+ except Exception:+ return None+ return decoder if isinstance(decoder, nn.Module) else None+++def _tp_chain(model):+ """`model` and every wrapper level below it (PeftModel -> LoraModel -> CausalLM -> decoder)."""+ chain, m = [], model+ while isinstance(m, nn.Module) and all(m is not c for c in chain) and len(chain) < 8:+ chain.append(m)+ modules = m.__dict__.get("_modules") or {}+ m = modules.get("base_model") or modules.get("model")+ inner = _unwrap(model)+ for extra in (inner, _decoder_of(inner)):+ if isinstance(extra, nn.Module) and all(extra is not c for c in chain):+ chain.append(extra)+ return chain+++def is_tensor_parallel(model) -> bool:+ """True when apply_tensor_parallel sharded `model`, a wrapper level below it, or any module inside it."""+ if not isinstance(model, nn.Module):+ return getattr(model, _TP_FLAG, None) is not None+ try:+ chain = _tp_chain(model)+ except Exception:+ chain = [model]+ if any(m.__dict__.get(_TP_FLAG) is not None for m in chain):+ return True+ return any(m.__dict__.get("_unsloth_tp_style") is not None for m in model.modules())+++def _default_mesh(model):+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh++ if not dist.is_initialized():+ raise RuntimeError(+ "Unsloth: apply_tensor_parallel needs an initialized process group (torchrun) or a tp_mesh."+ )+ device = next(model.parameters()).device+ return init_device_mesh(device.type, (dist.get_world_size(),), mesh_dim_names = ("tp",))+++def _refuse_generate(*args, **kwargs):+ raise RuntimeError(+ "Unsloth: generation is not supported on a tensor parallel model in v1; save the adapter "+ "(save_tp_adapter) and generate on one GPU."+ )+++def _cache_in_use(past):+ if past is None:+ return False+ length = getattr(past, "get_seq_length", None)+ if callable(length):+ try:+ return int(length()) > 0+ except Exception:+ return True+ try:+ return len(past) > 0+ except TypeError:+ return True+++def _refuse_cached_forward(module):+ """Refuse a filled KV cache (instance attribute: LoraModel calls self.model.forward directly)."""+ forward = module.forward+ try:+ names = list(inspect.signature(forward).parameters)+ except (TypeError, ValueError):+ names = []+ index = names.index("past_key_values") if "past_key_values" in names else None++ def forward_without_cache(*args, **kwargs):+ past = kwargs.get("past_key_values", kwargs.get("past_key_value"))+ if past is None and index is not None and len(args) > index:+ past = args[index]+ if _cache_in_use(past):+ raise RuntimeError(+ "Unsloth: cached decoding (past_key_values) is not supported on a tensor parallel model in "+ "v1; save the adapter (save_tp_adapter) and generate on one GPU."+ )+ return forward(*args, **kwargs)++ forward_without_cache.__wrapped__ = forward+ module.__dict__["forward"] = forward_without_cache+++def _sync_trainable(model, group, src):+ import torch.distributed as dist+ with torch.no_grad():+ for param in model.parameters():+ if param.requires_grad:+ dist.broadcast(param.data, src = src, group = group)+++def _set_local_heads(module, region, tp):+ heads, kv_heads = region.heads // tp, region.kv_heads // tp+ module._unsloth_tp_heads = heads+ module._unsloth_tp_kv_heads = kv_heads+ for name, value, always in (+ ("num_heads", heads, True),+ ("num_key_value_heads", kv_heads, True),+ ("num_attention_heads", heads, False),+ ("num_kv_heads", kv_heads, False),+ ):+ if always or isinstance(module.__dict__.get(name), int):+ module.__dict__[name] = value+ config = module.__dict__.get("config")+ if config is None:+ return+ try:+ source = _forward_source(module)+ except (OSError, TypeError):+ source = ""+ if _GLOBAL_HEAD_READS.search(source) and "_unsloth_tp_heads" not in source:+ # Per-module config copy with local counts (Phi3 fused qkv); the shared config is left alone.+ local = copy.copy(config)+ local.num_attention_heads = heads+ if getattr(config, "num_key_value_heads", None) is not None:+ local.num_key_value_heads = kv_heads+ module.__dict__["config"] = local+++def apply_tensor_parallel(+ model,+ tp_mesh = None,+ plan = None,+ config: Optional[TPConfig] = None,+) -> nn.Module:+ """Shard every decoder attention / MLP of `model` over `tp_mesh` (None = all ranks), in place."""+ if is_tensor_parallel(model):+ return model+ config = config or TPConfig()+ if plan is None:+ plan = config.plan+ if tp_mesh is None:+ tp_mesh = _default_mesh(model)+ if tp_mesh.ndim != 1:+ raise ValueError("Unsloth: apply_tensor_parallel takes a 1-D tp mesh (pass mesh['tp']).")+ tp = tp_mesh.size()+ rank = tp_mesh.get_local_rank()+ group = tp_mesh.get_group()+ if (+ any(_is_dtensor(p) for p in model.parameters())+ or (getattr(model, "tp_size", None) or 1) > 1+ ):+ raise ValueError(+ "Unsloth: this model is already distributed (DTensor parameters or a transformers tp_plan); "+ "load it without tp_plan before apply_tensor_parallel."+ )+ tp_plan = derive_tp_plan(model, tp, plan)+ for region in tp_plan.regions:+ for proj in region.projs:+ path = f"{region.path}.{proj.name}"+ weight, base = _weight(proj.module)+ _refuse_base(path, proj.module, weight, base)+ _refuse_adapter(path, proj.module)+ if _is_params4bit(weight):+ _check_params4bit(path, weight, proj.style, tp)+ for name, norm in region.norms:+ for p in norm.parameters():+ if type(p) not in (nn.Parameter, torch.Tensor):+ raise ValueError(+ f"Unsloth: cannot replicate {region.path}.{name} ({type(p).__name__})."+ )+ chain = _tp_chain(model)+ if config.sync_trainable and tp > 1:+ import torch.distributed as dist+ _sync_trainable(model, group, dist.get_global_rank(group, 0))+ hooked_norms = (+ set()+ ) # a norm weight shared by several regions gets one tp sum, not one per region+ for region in tp_plan.regions:+ for proj in region.projs:+ _shard_projection(region, proj, tp_mesh, rank, tp)+ for _, norm in region.norms:+ for p in norm.parameters():+ if p.requires_grad and id(p) not in hooked_norms:+ hooked_norms.add(id(p))+ # Each rank normalizes its own heads with the shared weight: the grad is a partial sum.+ p.register_hook(lambda g, group = group: _all_reduce(g, group))+ module = region.module+ module._unsloth_tp_group = group+ module._unsloth_tp_row_projs = [p.module for p in region.projs if p.style == ROWWISE]+ module._unsloth_tp_input = _input_name(module)+ if region.kind == "attention":+ _set_local_heads(module, region, tp)+ module.register_forward_pre_hook(_region_pre_hook, with_kwargs = True)+ module.register_forward_hook(_region_post_hook)+ # Unsloth decode paths bypass the regions: mark and guard every wrapper level.+ info = {"size": tp, "rank": rank, "regions": len(tp_plan.regions), "source": tp_plan.source}+ inner = _unwrap(model)+ decoder = _decoder_of(inner)+ for level in chain:+ level.__dict__[_TP_FLAG] = info+ level.__dict__["_unsloth_tp_mesh"] = tp_mesh+ level.__dict__["_unsloth_tp_plan"] = tp_plan+ if callable(getattr(level, "generate", None)):+ level.__dict__["generate"] = _refuse_generate+ if level is inner or level is decoder:+ _refuse_cached_forward(level)+ if rank == 0:+ print(+ f"Unsloth: tensor parallel over {tp} ranks: {len(tp_plan.regions)} attention / MLP regions "+ f"({tp_plan.source} plan), embeddings / lm_head / norms replicated"+ + (+ f", {tp_plan.replicated_layers} layers replicated"+ if tp_plan.replicated_layers+ else ""+ )+ + "."+ )+ return model+++def _shard_projection(region, proj, mesh, rank, tp):+ module = proj.module+ weight, base = _weight(module)+ col = proj.style == COLWISE+ parts = proj.parts+ in_parts = (_logical_shape(weight, base)[1],)+ if _is_params4bit(weight):+ _shard_params4bit(f"{region.path}.{proj.name}", base, weight, proj.style, parts, rank, tp)+ else:+ _set_dtensor(+ base,+ "weight",+ weight,+ mesh,+ "shard",+ parts if col else in_parts,+ 0 if col else 1,+ rank,+ tp,+ )+ if base.bias is not None:+ if col:+ _set_dtensor(base, "bias", base.bias, mesh, "shard", parts, 0, rank, tp)+ else:+ _set_dtensor(base, "bias", base.bias, mesh, "replicate")+ lora_A = module._modules.get("lora_A")+ if lora_A is not None:+ for adapter in lora_A.keys():+ A, B = lora_A[adapter], module.lora_B[adapter]+ if col:+ _set_dtensor(A, "weight", A.weight, mesh, "replicate")+ _set_dtensor(B, "weight", B.weight, mesh, "shard", parts, 0, rank, tp)+ else:+ _set_dtensor(A, "weight", A.weight, mesh, "shard", in_parts, 1, rank, tp)+ _set_dtensor(B, "weight", B.weight, mesh, "replicate")+ module._unsloth_tp_style = proj.style+ module._unsloth_tp_local = (+ _tp_lora_parameters # read by kernels/utils.py get_lora_parameters(_bias)+ )+ module.forward = types.MethodType(_tp_projection_forward, module)+++def tp_clip_grad_norm_(+ parameters,+ max_norm,+ mesh = None,+ eps = 1e-6,+):+ """Clip by the global 2-norm: sharded grads summed over tp, replicated ones counted once."""+ import torch.distributed as dist++ if isinstance(parameters, torch.Tensor):+ parameters = [parameters]+ params = [p for p in parameters if p.grad is not None]+ if not params:+ return torch.tensor(0.0)+ sharded = replicated = None+ group = mesh.get_group() if mesh is not None else None+ for p in params:+ g = p.grad+ if _is_dtensor(g):+ g = _replicate_grad(g)+ if g is not p.grad:+ p.grad = g+ group = g.device_mesh.get_group() if group is None else group+ local = g.to_local().float()+ sq = local.square().sum()+ if any(pl.is_shard() for pl in g.placements):+ sharded = sq if sharded is None else sharded + sq+ continue+ else:+ sq = g.float().square().sum()+ replicated = sq if replicated is None else replicated + sq+ device = (+ params[0].grad.device+ if not _is_dtensor(params[0].grad)+ else params[0].grad.to_local().device+ )+ total = torch.zeros((), device = device) if replicated is None else replicated.to(device)+ if group is not None:+ shard_sq = torch.zeros((), device = device) if sharded is None else sharded.to(device)+ dist.all_reduce(shard_sq, group = group)+ total = total + shard_sq+ total = total.sqrt()+ coef = float(max_norm) / (float(total) + eps)+ if coef < 1:+ for p in params:+ p.grad.mul_(coef)+ return total+++def _full(tensor, param = None):+ if not _is_dtensor(tensor):+ return tensor+ full = tensor.full_tensor()+ layout = getattr(param, "_unsloth_tp_parts", None) if param is not None else None+ if layout is not None:+ dim, parts = layout+ full = _unpermute(full, dim, parts, tensor.device_mesh.size())+ return full+++def tp_full_state_dict(model, adapters_only = False):+ """State dict with DTensors gathered (collective); 4-bit shards stay local."""+ state = model.state_dict(keep_vars = True)+ # PEFT picks embedding weights out of the given state dict (save_embedding_layers, resized vocab).+ embeddings = set()+ for get in ("get_input_embeddings", "get_output_embeddings"):+ module = getattr(model, get, lambda: None)()+ weight = getattr(module, "weight", None)+ if weight is not None:+ embeddings.add(id(weight))+ out = {}+ with torch.no_grad():+ for key, value in state.items():+ if (+ adapters_only+ and "lora_" not in key+ and not getattr(value, "requires_grad", False)+ and id(value) not in embeddings+ ):+ continue+ out[key] = _full(value, value).detach()+ return out+++def _peft_state_dict(model, **kwargs):+ """What PEFT selects for save_pretrained, DTensors gathered; local tensors by reference (PEFT reruns+ its selection inside save_pretrained and must see the same model)."""+ from peft import get_peft_model_state_dict++ raw = model.state_dict(keep_vars = True)+ adapters = kwargs.get("selected_adapters") or list(model.peft_config)+ known = {id(value) for value in raw.values()}+ wanted, unmapped = set(), 0+ for adapter in adapters:+ selected = get_peft_model_state_dict(+ model,+ state_dict = raw,+ adapter_name = adapter,+ save_embedding_layers = kwargs.get("save_embedding_layers", "auto"),+ )+ wanted.update(id(value) for value in selected.values())+ unmapped += sum(_is_dtensor(v) and id(v) not in known for v in selected.values())+ if unmapped:+ raise RuntimeError(+ "Unsloth: PEFT selected tensor parallel tensors that are not in the model state dict; "+ "save_tp_adapter cannot gather them."+ )+ state = {}+ with torch.no_grad():+ for key, value in raw.items():+ if _is_dtensor(value):+ if id(value) in wanted:+ state[key] = _full(value, value).detach()+ else:+ state[key] = value.detach()+ return state+++def save_tp_adapter(model, save_directory, **kwargs):+ """Write the standard 1-GPU PEFT adapter. Collective (tp mesh = whole world in v1); rank 0 writes."""+ import torch.distributed as dist++ mesh = None+ for level in _tp_chain(model):+ mesh = mesh or level.__dict__.get("_unsloth_tp_mesh")+ if dist.is_initialized() and mesh is not None and mesh.size() != dist.get_world_size():+ raise ValueError(+ f"Unsloth: save_tp_adapter writes on global rank 0 behind a world barrier, but the tensor parallel "+ f"mesh has {mesh.size()} of {dist.get_world_size()} ranks (data parallel > 1 is outside v1). "+ "Gather with tp_full_state_dict and write from one rank yourself."+ )+ state = _peft_state_dict(model, **kwargs)+ rank = dist.get_rank() if dist.is_initialized() else 0+ kwargs.setdefault("is_main_process", rank == 0)+ if kwargs["is_main_process"]:+ model.save_pretrained(save_directory, state_dict = state, **kwargs)+ if dist.is_initialized():+ dist.barrier()diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 6aafd48ffa6..9500b1a2d90 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -191,6 +191,7 @@ from ._utils import ()from ._utils import *from ._utils import _ignore_block_swap_params_for_ddp+from ._utils import _refuse_tensor_parallel_for_inferencefrom ._uma_safetensors import is_integrated_unified_memory_gpufrom ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimatefrom ._remote_code_buffers import restore_remote_code_non_persistent_buffers@@ -5501,6 +5502,7 @@ class FastBaseModel:raise TypeError("Unsloth: I think you're passing a tokenizer, not the model to for_inference!")+ _refuse_tensor_parallel_for_inference(model)def _for_inference(m):if hasattr(m, "gradient_checkpointing"):