Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tensor parallel: support few KV heads, full finetuning, weight-only QAT, eval, MoE and dropout instead of refusing (#15722)

Daniel Hanc06ae15b0f17
diff --git a/tests/test_tp_dropout_mask_precision.py b/tests/test_tp_dropout_mask_precision.py
new file mode 100644
index 00000000000..7a245750128
--- /dev/null
+++ b/tests/test_tp_dropout_mask_precision.py
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallel dropout masks round as one process's F.dropout: each tp rank's slice of the output and of
+the input gradient is bitwise the matching slice of F.dropout on the whole tensor under the same RNG state, for
+bf16 / fp16 / fp32 activations. On CUDA F.dropout is ATen's fused native_dropout, which applies 1 / (1 - p) in
+accumulator precision and rounds once; on CPU it multiplies by a noise tensor holding 1 / (1 - p) rounded to the
+activation dtype. Ranks are emulated in one process: a mask depends only on the RNG state and the shape."""
+
+import os
+import sys
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from real_accelerator import has_real_cuda # noqa: E402
+from test_tp_native_oracle import tpn # noqa: E402 (tp_native without the GPU-only unsloth import)
+
+TP = 2
+DEVICES = ["cpu"] + (["cuda"] if torch.cuda.is_available() else [])
+
+
+def _bits(t):
+ return t.view(torch.int16) if t.element_size() == 2 else t.view(torch.int32)
+
+
+def _assert_bitwise(a, b, what):
+ assert a.dtype == b.dtype and a.shape == b.shape, what
+ bad = (_bits(a) != _bits(b)).sum().item()
+ assert bad == 0, f"{what}: {bad} of {a.numel()} elements differ"
+
+
+def _reference(x, g, p, seed):
+ torch.manual_seed(seed)
+ x = x.clone().requires_grad_()
+ y = F.dropout(x, p, True)
+ y.backward(g)
+ return y.detach(), x.grad
+
+
+def _one_rank(drop, x, g, rank, p, seed):
+ width = x.shape[-1] // TP
+ xr = x[..., rank * width : (rank + 1) * width].clone().requires_grad_()
+ torch.manual_seed(seed)
+ y = drop(xr, rank)
+ y.backward(g[..., rank * width : (rank + 1) * width])
+ return y.detach(), xr.grad
+
+
+def _tp_dropout(p, width):
+ def drop(xr, rank):
+ state = SimpleNamespace(
+ kind = "mlp",
+ rank = rank,
+ tp = TP,
+ local_heads = None,
+ local_width = width,
+ row_bias = False,
+ phase = 1,
+ bypass = 0,
+ path = "mlp",
+ )
+ return tpn._tp_dropout(state, xr, p, True)
+
+ return drop
+
+
+def _lora_dropout(p):
+ def drop(xr, rank):
+ dropout = nn.Dropout(p).train()
+ proj = SimpleNamespace(
+ lora_dropout = {"default": dropout},
+ _unsloth_tp_style = tpn.ROWWISE,
+ _unsloth_tp_coord = (rank, TP),
+ )
+ return tpn._lora_dropout(proj, "default", xr)
+
+ return drop
+
+
[email protected]("device", DEVICES)
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
[email protected]("p", [0.1, 0.15, 0.25, 0.3])
[email protected]("path", ["tp_dropout", "row_lora_dropout"])
+def test_tp_dropout_slices_round_as_one_process(device, dtype, p, path):
+ gen = torch.Generator().manual_seed(7)
+ x = (torch.randn(4, 16, 256, generator = gen) * 3).to(device, dtype)
+ g = torch.randn(4, 16, 256, generator = gen).to(device, dtype)
+ seed = 1234
+ y_ref, dx_ref = _reference(x, g, p, seed)
+ width = x.shape[-1] // TP
+ drop = _tp_dropout(p, width) if path == "tp_dropout" else _lora_dropout(p)
+ for rank in range(TP):
+ y, dx = _one_rank(drop, x, g, rank, p, seed)
+ cols = slice(rank * width, (rank + 1) * width)
+ _assert_bitwise(y, y_ref[..., cols], f"{path} rank {rank} output")
+ _assert_bitwise(dx, dx_ref[..., cols], f"{path} rank {rank} input grad")
+
+
[email protected](not has_real_cuda(), reason = "CUDA native_dropout rounding")
+def test_bf16_p025_example_rounds_in_accumulator_precision():
+ # 1.015625 / 0.75 in fp32 rounds to 1.3515625 in bf16; with 1 / 0.75 first rounded to bf16 it is 1.359375.
+ x = torch.full((1, 64), 1.015625, dtype = torch.bfloat16, device = "cuda")
+ mask = torch.ones(1, 64, dtype = torch.bool, device = "cuda")
+ y = tpn._apply_mask(x, mask, 0.25)
+ assert y.unique().tolist() == [1.3515625]
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.py
index b8c18d0beb8..485c2b5fb90 100644
--- a/tests/test_tp_env_gloo.py
+++ b/tests/test_tp_env_gloo.py
@@ -121,7 +121,6 @@ def _case_units(rank, world, tmp):
layout(UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 2)
wrapped = tp_env.tensor_parallel_env_load(loader)
out["refuse_vllm"] = _raised(lambda: wrapped("m", fast_inference = True))
- out["refuse_full_ft"] = _raised(lambda: wrapped("m", False, full_finetuning = True))
out["refuse_8bit"] = _raised(lambda: wrapped("m", load_in_8bit = True))
layout(UNSLOTH_TP_SIZE = 2, UNSLOTH_FSDP2 = 1, UNSLOTH_FSDP2_SHARDED_LOAD = 1, WORLD_SIZE = 2)
out["refuse_fsdp2_sharded_load"] = _raised(lambda: wrapped("m", False))
@@ -315,7 +314,9 @@ def _case_env(
eval_dataset = data,
)
)
- res["evaluate"] = _raised(lambda: trainer.evaluate(data))
+ res["evaluate"] = _raised(
+ lambda: res.__setitem__("eval_loss", trainer.evaluate(data)["eval_loss"])
+ )
res["lion"] = _raised(
lambda: SequentialTrainer(
model = model,
@@ -612,7 +613,6 @@ def test_env_layout_parsing_and_refusals():
assert res["fsdp2"] == (2, 2, 4, True)
assert "native FSDP2 only" in res["accelerate_fsdp"]
assert "vLLM" in res["refuse_vllm"] and "UNSLOTH_TP_SIZE=2" in res["refuse_vllm"]
- assert "full finetuning" in res["refuse_full_ft"]
assert "8-bit weights" in res["refuse_8bit"]
assert "UNSLOTH_FSDP2_SHARDED_LOAD=1" in (res["refuse_fsdp2_sharded_load"] or ""), res[
"refuse_fsdp2_sharded_load"
@@ -639,11 +639,11 @@ def test_env_tp2_trainer_matches_one_process(four_bit):
_close(res["a"], ref["a"], what = rank)
hint = "UNSLOTH_TP_SIZE=2: unset it"
assert "map-style dataset" in res["iterable"] and hint in res["iterable"], res["iterable"]
- assert "evaluation during training" in res["eval"] and hint in res["eval"], res["eval"]
- assert "Trainer.evaluate is not supported" in res["evaluate"] and hint in res["evaluate"]
+ assert res["eval"] is None and res["evaluate"] is None, (res["eval"], res["evaluate"])
assert "lion_8bit" in res["lion"], res["lion"]
assert res["generate"] is None, res["generate"] # a whole replica per rank (tp_replica.py)
assert results[0]["seen"] == results[1]["seen"] # both tp ranks read the same batches
+ assert results[0]["eval_loss"] == results[1]["eval_loss"]
def test_env_tp2_with_fsdp2_flag_and_no_dp_dimension():
diff --git a/tests/test_tp_eval_gloo.py b/tests/test_tp_eval_gloo.py
new file mode 100644
index 00000000000..906c966e8ae
--- /dev/null
+++ b/tests/test_tp_eval_gloo.py
@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallel evaluation and cached generate on a gloo CPU world vs one process, fp32.
+
+- The HF Trainer with eval_strategy = "steps" (every step), Trainer.evaluate and Trainer.predict on a tp model
+ (and dp x tp after shard_model): the gather covers the dp replicas only, so eval_loss / predictions equal
+ one process (the tp ranks of a replica hold identical results; a world gather would count each tp times).
+- Cached greedy generate after a forward wrapper is installed over the tensor parallel forward guard (what the
+ GRPO hidden states wrapper in models/rl.py does to the model the GRPO trainer generates with): the gathered
+ replica decodes with a KV cache instead of being refused mid-run."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import tempfile # noqa: E402
+
+from test_tp_native_gloo import _free_port, _import_checkout_unsloth, _Rows, _trainer_args # noqa: E402
+from test_tp_native_oracle import LLAMA, _batch, build # noqa: E402
+
+
+def _case_eval(rank, world, tmp, dp, tp):
+ """3 Trainer steps evaluating every step, then evaluate() and predict() on 7 rows (a dp remainder)."""
+ import transformers
+
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ from torch.distributed.device_mesh import init_device_mesh
+ from unsloth.models import fsdp2_native, tp_native
+ from unsloth.models._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(transformers.Trainer)
+
+ class SequentialTrainer(transformers.Trainer):
+ def _get_train_sampler(self, *args, **kwargs):
+ return torch.utils.data.SequentialSampler(self.train_dataset)
+
+ model = build(LLAMA)
+ data = _Rows(list(_batch(model, bsz = 12 * dp, seq = 9)))
+ held_out = _Rows(list(_batch(model, seed = 2, bsz = 7, seq = 9)))
+ if tp > 1:
+ mesh = init_device_mesh("cpu", (dp, tp), mesh_dim_names = ("dp", "tp"))
+ tp_native.apply_tensor_parallel(model, mesh["tp"])
+ if dp > 1:
+ fsdp2_native.shard_model(
+ model, fsdp2_native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
+ )
+ args = _trainer_args(
+ tmp,
+ 2 * dp if tp == 1 else 2,
+ eval_strategy = "steps",
+ eval_steps = 1,
+ per_device_eval_batch_size = 1,
+ dataloader_drop_last = False,
+ )
+ trainer = SequentialTrainer(model = model, args = args, train_dataset = data, eval_dataset = held_out)
+ trainer.train()
+ history = trainer.state.log_history
+ final = trainer.evaluate()
+ predicted = trainer.predict(held_out)
+ return {
+ "losses": [h["loss"] for h in history if "loss" in h],
+ "eval_losses": [h["eval_loss"] for h in history if "eval_loss" in h],
+ "final_eval": final["eval_loss"],
+ "predictions": torch.as_tensor(predicted.predictions),
+ "label_ids": torch.as_tensor(predicted.label_ids),
+ "predict_loss": predicted.metrics["test_loss"],
+ }
+
+
+def _case_generate(rank, world, tmp, tp):
+ """Greedy cached generate after the GRPO hidden states forward wrapper wraps the tp forward guard."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ from torch.distributed.device_mesh import init_device_mesh
+ from unsloth.models import rl, tp_native, tp_replica
+
+ model = build(LLAMA)
+ prompt = _batch(model, seed = 3, bsz = 2, seq = 5)
+ if tp > 1:
+ mesh = init_device_mesh("cpu", (tp,), mesh_dim_names = ("tp",))
+ tp_native.apply_tensor_parallel(model, mesh["tp"])
+ wrapped = rl._install_grpo_hidden_states_forward_wrapper(model)
+ model.eval()
+ try:
+ with torch.no_grad():
+ out = model.generate(
+ input_ids = prompt,
+ attention_mask = torch.ones_like(prompt),
+ max_new_tokens = 6,
+ do_sample = False,
+ use_cache = True,
+ pad_token_id = 0,
+ )
+ error = None
+ except Exception as e:
+ out, error = None, f"{type(e).__name__}: {e}"
+ active = tp_replica.replica_active(model) if tp > 1 else False
+ return {"out": out, "error": error, "wrapped": wrapped, "replica_after": active}
+
+
+CASES = {"eval": _case_eval, "generate": _case_generate}
+
+
+def _worker(rank, world, port, case, tmp, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = 2,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+def _torch_below_2_7():
+ from packaging.version import Version
+ return Version(torch.__version__.split("+")[0]) < Version("2.7")
+
+
[email protected]("dp, tp", [(1, 2), (2, 2)])
+def test_trainer_evaluates_a_tensor_parallel_model_like_one_process(dp, tp):
+ if dp > 1 and _torch_below_2_7():
+ pytest.skip(
+ "torch < 2.7: tensor parallel x FSDP2 is refused (no fully_shard(ignored_params))"
+ )
+ ref = _spawn("eval", 1, dp = dp, tp = 1)[0]
+ if "skip" in ref:
+ pytest.skip(f"import unsloth failed on this host: {ref['skip']}")
+ results = _spawn("eval", dp * tp, dp = dp, tp = tp)
+ assert (
+ len(ref["losses"]) == 3 and len(ref["eval_losses"]) == 4
+ ) # 3 during training + evaluate()
+ assert ref["predictions"].shape[0] == 7
+ for rank, res in enumerate(results):
+ assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5), (
+ rank,
+ res["losses"],
+ ref["losses"],
+ )
+ assert res["eval_losses"] == pytest.approx(ref["eval_losses"], rel = 1e-5), (
+ rank,
+ res["eval_losses"],
+ ref["eval_losses"],
+ )
+ assert res["final_eval"] == pytest.approx(ref["final_eval"], rel = 1e-5)
+ assert res["predict_loss"] == pytest.approx(ref["predict_loss"], rel = 1e-5)
+ assert res["predictions"].shape == ref["predictions"].shape, (
+ rank,
+ res["predictions"].shape,
+ )
+ torch.testing.assert_close(res["predictions"], ref["predictions"], rtol = 1e-4, atol = 1e-5)
+ assert torch.equal(res["label_ids"], ref["label_ids"])
+
+
+def test_cached_generate_through_a_wrapped_tp_forward_matches_one_process():
+ ref = _spawn("generate", 1, tp = 1)[0]
+ if "skip" in ref:
+ pytest.skip(f"import unsloth failed on this host: {ref['skip']}")
+ assert ref["error"] is None, ref["error"]
+ results = _spawn("generate", 2, tp = 2)
+ for rank, res in enumerate(results):
+ assert res[
+ "wrapped"
+ ], "the GRPO hidden states wrapper was not installed: the test proves nothing"
+ assert res["error"] is None, (rank, res["error"])
+ assert torch.equal(res["out"], ref["out"]), (rank, res["out"], ref["out"])
+ assert res["replica_after"] is False
diff --git a/tests/test_tp_full_finetune_gloo.py b/tests/test_tp_full_finetune_gloo.py
new file mode 100644
index 00000000000..a77c03e86e4
--- /dev/null
+++ b/tests/test_tp_full_finetune_gloo.py
@@ -0,0 +1,459 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Full finetuning (every base weight trainable, no LoRA) under tensor parallel on a gloo CPU world vs one
+process: the direct API (apply_tensor_parallel + a custom loop) and UNSLOTH_TP_SIZE=2 with the HF Trainer
+(loader flag full_finetuning = True, tensor parallel applied at Trainer init, Trainer.save_model)."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_gloo import ( # noqa: E402
+ _Rows,
+ _clip_fp64,
+ _free_port,
+ _grads,
+ _import_checkout_unsloth,
+ _save_checkpoint,
+ _trainer_args,
+)
+from test_tp_native_oracle import ARCHS, _batch, build # noqa: E402
+
+_ENV_KEYS = ("UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "UNSLOTH_FSDP2", "UNSLOTH_FSDP2_BACKEND")
+# Adam's m / (sqrt(v) + eps) turns fp32 reduction-order noise on near-zero gradients into visible weight
+# drift at eps 1e-8; at 1e-4 a real tp error (a missing all-reduce) still moves the weights far more.
+_ADAM_EPS = 1e-4
+
+
+def _full_ft_train(model, ids, steps, clip):
+ params = [p for p in model.parameters() if p.requires_grad]
+ opt = torch.optim.AdamW(params, lr = 1e-3, eps = _ADAM_EPS, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(steps):
+ out = model(input_ids = ids, labels = ids, use_cache = False)
+ out.loss.backward()
+ if step == 0:
+ grads = _grads(model)
+ norms.append(float(clip(params, 1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(out.loss))
+ return grads, losses, norms
+
+
+def _case_api(rank, world, tmp, arch):
+ from safetensors.torch import load_file
+ from torch.distributed.device_mesh import init_device_mesh
+
+ from unsloth.models import tp_native as tpn
+
+ repo, overrides = ARCHS[arch]
+ ref = build(repo, overrides, lora = False)
+ model = build(repo, overrides, lora = False)
+ for m in (ref, model):
+ m.requires_grad_(True)
+ m.generation_config.pad_token_id = None # tiny repos ship -1, which save_pretrained rejects
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ trainable_shards = sum(
+ tpn._is_dtensor(p) and p.requires_grad and p.placements[0].is_shard()
+ for p in model.parameters()
+ )
+ ids = _batch(ref)
+ ref_out = _full_ft_train(ref, ids, 3, _clip_fp64)
+ tp_out = _full_ft_train(model, ids, 3, tpn.tp_clip_grad_norm_)
+ final = tpn.tp_full_state_dict(model)
+ ref_dir, tp_dir = os.path.join(tmp, "ref"), os.path.join(tmp, "tp")
+ if rank == 0:
+ ref.save_pretrained(ref_dir)
+ from unsloth.models.tp_replica import tp_collective
+
+ tp_collective(model, "save_pretrained", lambda: model.save_pretrained(tp_dir), "all")
+ return {
+ "ref": ref_out,
+ "tp": tp_out,
+ "ref_final": {k: v.detach().clone() for k, v in ref.state_dict().items()},
+ "tp_final": final,
+ "trainable_shards": trainable_shards,
+ "saved_ref": load_file(os.path.join(ref_dir, "model.safetensors")) if rank == 0 else None,
+ "saved_tp": load_file(os.path.join(tp_dir, "model.safetensors")) if rank == 0 else None,
+ "is_tp_after_save": tpn.is_tensor_parallel(model)
+ and any(tpn._is_dtensor(p) for p in model.parameters()),
+ }
+
+
+def _case_env(rank, world, tmp, ckpt):
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import transformers
+ from safetensors.torch import load_file
+ from transformers import AutoModelForCausalLM
+
+ from unsloth.models import tp_env, tp_native
+ from unsloth.models._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(transformers.Trainer)
+
+ class SequentialTrainer(transformers.Trainer):
+ def _get_train_sampler(self, *args, **kwargs):
+ return torch.utils.data.SequentialSampler(self.train_dataset)
+
+ def load(
+ model_name,
+ load_in_4bit = True,
+ full_finetuning = False,
+ **kwargs,
+ ):
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name, torch_dtype = torch.float32, attn_implementation = "eager"
+ )
+ model.requires_grad_(full_finetuning)
+ return model
+
+ load = tp_env.tensor_parallel_env_load(load)
+ refused = None
+ try:
+ model = load(ckpt, load_in_4bit = False, full_finetuning = True)
+ except Exception as e:
+ refused = f"{type(e).__name__}: {e}"
+ return {"refused": refused}
+ g = torch.Generator().manual_seed(1)
+ data = _Rows(list(torch.randint(0, 1000, (3 * 4, 9), generator = g)))
+ out_dir = os.path.join(tmp, f"out{world}")
+ args = _trainer_args(
+ out_dir,
+ 2,
+ max_steps = 3,
+ adam_epsilon = _ADAM_EPS,
+ learning_rate = 1e-3,
+ save_strategy = "steps",
+ save_steps = 2,
+ )
+ try:
+ trainer = SequentialTrainer(model = model, args = args, train_dataset = data)
+ except RuntimeError as e:
+ if "full finetuning on a tensor parallel x native FSDP2" not in str(e):
+ raise
+ return {"refused": None, "trainer_refused": str(e)}
+ trainer.train()
+ is_tp = tp_native.is_tensor_parallel(trainer.model)
+ state = (
+ tp_native.tp_full_state_dict(trainer.model)
+ if is_tp
+ else {k: v.detach().clone() for k, v in trainer.model.state_dict().items()}
+ )
+ save_dir = os.path.join(tmp, f"saved{world}")
+ trainer.save_model(save_dir)
+ import torch.distributed as dist
+
+ if dist.is_initialized():
+ dist.barrier()
+ logs = [h for h in trainer.state.log_history if "loss" in h]
+ return {
+ "refused": refused,
+ "is_tp": is_tp,
+ "losses": [h["loss"] for h in logs],
+ "grad_norms": [h.get("grad_norm") for h in logs],
+ "state": state,
+ "saved": load_file(os.path.join(save_dir, "model.safetensors")) if rank == 0 else None,
+ "ckpt_files": sorted(os.listdir(os.path.join(out_dir, "checkpoint-2"))),
+ }
+
+
+def _qat_config(kind):
+ import torch as _t
+ from torchao.quantization import (
+ Int4WeightOnlyConfig,
+ Int8DynamicActivationIntxWeightConfig,
+ IntxWeightOnlyConfig,
+ )
+ from torchao.quantization.granularity import PerAxis, PerGroup
+
+ return {
+ "int4": lambda: Int4WeightOnlyConfig(group_size = 16),
+ "int8_group": lambda: IntxWeightOnlyConfig(weight_dtype = _t.int8, granularity = PerGroup(16)),
+ "int8_axis": lambda: IntxWeightOnlyConfig(weight_dtype = _t.int8, granularity = PerAxis(0)),
+ "int8_int4": lambda: Int8DynamicActivationIntxWeightConfig(
+ weight_dtype = _t.int4, weight_granularity = PerGroup(16)
+ ),
+ }[kind]()
+
+
+def _case_qat(
+ rank,
+ world,
+ tmp,
+ kind,
+ lora,
+ kv_heads = 2,
+):
+ """QAT (torchao QATConfig prepare, as Unsloth's qat_scheme) on full finetuning or under LoRA."""
+ from peft import LoraConfig, get_peft_model
+ from torch.distributed.device_mesh import init_device_mesh
+ from torchao.quantization import quantize_
+ from torchao.quantization.qat import QATConfig
+
+ from unsloth.models import tp_native as tpn
+
+ repo, overrides = ARCHS["llama"]
+ overrides = {
+ **overrides,
+ "hidden_size": 32,
+ "intermediate_size": 64,
+ "num_attention_heads": 4,
+ "num_key_value_heads": kv_heads,
+ "head_dim": 8,
+ }
+
+ def make():
+ model = build(repo, overrides, lora = False)
+ decoder = model.model.layers
+ quantize_(decoder, QATConfig(_qat_config(kind), step = "prepare"))
+ if lora:
+ torch.manual_seed(1)
+ config = LoraConfig(
+ r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False
+ )
+ model = get_peft_model(model, config)
+ else:
+ model.requires_grad_(True)
+ return model
+
+ ref, model = make(), make()
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ try:
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ except ValueError as e:
+ return {"refused": str(e)}
+ fq = sum(type(m).__name__ == "FakeQuantizedLinear" for m in model.modules())
+ ids = _batch(ref)
+ return {
+ "refused": None,
+ "fake_quantized": fq,
+ "ref": _full_ft_train(ref, ids, 3, _clip_fp64),
+ "tp": _full_ft_train(model, ids, 3, tpn.tp_clip_grad_norm_),
+ }
+
+
+CASES = {"api": _case_api, "env": _case_env, "qat": _case_qat}
+
+
+def _worker(rank, world, port, case, tmp, env, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ for key in _ENV_KEYS:
+ os.environ.pop(key, None)
+ os.environ.update(env)
+ torch.set_num_threads(2)
+ import datetime
+
+ import torch.distributed as dist
+
+ if case in ("api", "qat"):
+ error = _import_checkout_unsloth()
+ if error is not None:
+ torch.save({"skip": error}, os.path.join(tmp, f"result_{rank}.pt"))
+ return
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 300)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world,
+ env = None,
+ tmp = None,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(900)
+ codes = [p.exitcode for p in procs]
+ for p in procs:
+ if p.is_alive():
+ p.kill()
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ results = [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False) for r in range(world)
+ ]
+ for r in range(world):
+ os.remove(os.path.join(tmp, f"result_{r}.pt"))
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ return results
+
+
+def _assert_state(got, want, what):
+ assert set(got) == set(want), sorted(set(got) ^ set(want))[:6]
+ for key, value in want.items():
+ torch.testing.assert_close(got[key], value, rtol = 1e-4, atol = 2e-6, msg = f"{what} {key}")
+
+
[email protected]("arch", ["llama", "llama_bias", "qwen3", "gemma2"])
+def test_tp2_full_finetuning_matches_one_process(arch):
+ with tempfile.TemporaryDirectory() as tmp:
+ results = _spawn("api", 2, tmp = tmp, arch = arch)
+ for rank, res in enumerate(results):
+ ref_grads, ref_losses, ref_norms = res["ref"]
+ grads, losses, norms = res["tp"]
+ assert res["trainable_shards"] > 0 # the sharded base weights themselves train
+ assert set(grads) == set(ref_grads) and len(ref_grads) > 0
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6), (rank, losses, ref_losses)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5), (rank, norms, ref_norms)
+ _assert_state(res["tp_final"], res["ref_final"], f"rank {rank}")
+ assert res["is_tp_after_save"]
+ saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]
+ assert set(saved_tp) == set(saved_ref)
+ for key in saved_ref:
+ assert saved_tp[key].shape == saved_ref[key].shape, key
+ torch.testing.assert_close(saved_tp[key], saved_ref[key], rtol = 1e-4, atol = 2e-6, msg = key)
+
+
+def test_env_tp2_full_finetuning_trainer_matches_one_process():
+ with tempfile.TemporaryDirectory() as tmp:
+ ckpt = _save_checkpoint("llama", os.path.join(tmp, "ckpt"))
+ (ref,) = _spawn("env", 1, tmp = tmp, ckpt = ckpt)
+ results = _spawn("env", 2, {"UNSLOTH_TP_SIZE": "2"}, tmp = tmp, ckpt = ckpt)
+ assert ref["refused"] is None and not ref["is_tp"]
+ for rank, res in enumerate(results):
+ assert res["refused"] is None, res["refused"] # main: "does not support full finetuning"
+ assert res["is_tp"]
+ assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5), (
+ res["losses"],
+ ref["losses"],
+ )
+ assert res["grad_norms"] == pytest.approx(ref["grad_norms"], rel = 1e-4)
+ _assert_state(res["state"], ref["state"], f"rank {rank}")
+ assert "unsloth_tp" in res["ckpt_files"] and "model.safetensors" in res["ckpt_files"]
+ saved = results[0]["saved"]
+ assert set(saved) == set(ref["saved"])
+ for key, value in ref["saved"].items():
+ torch.testing.assert_close(saved[key], value, rtol = 1e-4, atol = 2e-6, msg = key)
+
+
+def test_env_tp2_x_fsdp2_full_finetuning_is_refused_before_training():
+ # tp x native FSDP2 cannot gather the whole model for save_model / Trainer checkpoints: before the fix the
+ # run trained and then died at its first checkpoint (save_model refused, pointing at a missing adapter).
+ if _torch_below_2_7():
+ pytest.skip("torch < 2.7: tensor parallel x FSDP2 is refused at load")
+ with tempfile.TemporaryDirectory() as tmp:
+ ckpt = _save_checkpoint("llama", os.path.join(tmp, "ckpt"))
+ results = _spawn(
+ "env", 4, {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1"}, tmp = tmp, ckpt = ckpt
+ )
+ for res in results:
+ assert res["refused"] is None, res["refused"] # the load itself is fine
+ assert "UNSLOTH_TP_SIZE without UNSLOTH_FSDP2" in res.get("trainer_refused", ""), res
+
+
+def _torch_below_2_7():
+ from packaging.version import Version
+ return Version(torch.__version__.split("+")[0]) < Version("2.7")
+
+
+def _torchao():
+ try:
+ from torchao.quantization.qat import QATConfig # noqa: F401
+ except Exception:
+ return False
+ return True
+
+
+def _skip_unsupported_qat(kind):
+ """Older torchao (0.13) cannot QATConfig-prepare every scheme (IntxWeightOnlyConfig, ...): skip there."""
+ from torchao.quantization import quantize_
+ from torchao.quantization.qat import QATConfig
+
+ try:
+ quantize_(
+ torch.nn.Sequential(torch.nn.Linear(32, 32)),
+ QATConfig(_qat_config(kind), step = "prepare"),
+ )
+ except Exception as e:
+ pytest.skip(f"this torchao cannot QAT-prepare {kind}: {type(e).__name__}: {e}")
+
+
[email protected](not _torchao(), reason = "torchao QAT not installed")
+ "kind,lora,kv_heads",
+ [
+ ("int4", False, 2),
+ ("int8_group", False, 2),
+ ("int4", True, 2),
+ ("int8_group", True, 2),
+ # num_key_value_heads < tp: the replicated key / value projections fake quantize whole rows.
+ ("int4", True, 1),
+ ("int4", False, 1),
+ ],
+)
+def test_tp2_qat_weight_only_matches_one_process(kind, lora, kv_heads):
+ _skip_unsupported_qat(kind)
+ with tempfile.TemporaryDirectory() as tmp:
+ results = _spawn("qat", 2, tmp = tmp, kind = kind, lora = lora, kv_heads = kv_heads)
+ for rank, res in enumerate(results):
+ assert res["refused"] is None, res[
+ "refused"
+ ] # main: "does not support QAT fake quantization"
+ assert res["fake_quantized"] > 0
+ ref_grads, ref_losses, ref_norms = res["ref"]
+ grads, losses, norms = res["tp"]
+ assert set(grads) == set(ref_grads) and ref_grads
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ # Step 1 is exact; afterwards a weight that differs by fp32 reduction-order noise can round to the
+ # neighbouring quantization level (a step function), which shows up as ~1e-4 in the later losses.
+ assert losses[0] == pytest.approx(ref_losses[0], rel = 1e-6), (rank, losses, ref_losses)
+ assert norms[0] == pytest.approx(ref_norms[0], rel = 1e-5), (rank, norms, ref_norms)
+ assert losses == pytest.approx(ref_losses, rel = 5e-4), (rank, losses, ref_losses)
+ assert norms == pytest.approx(ref_norms, rel = 5e-3), (rank, norms, ref_norms)
+
+
[email protected](not _torchao(), reason = "torchao QAT not installed")
+ "kind,needle",
+ [
+ ("int8_axis", "per-row (PerAxis) scale spans the in_features"),
+ ("int8_int4", "QAT fake quantization of activations"),
+ ],
+)
+def test_tp2_qat_inexact_schemes_are_refused(kind, needle):
+ _skip_unsupported_qat(kind)
+ with tempfile.TemporaryDirectory() as tmp:
+ results = _spawn("qat", 2, tmp = tmp, kind = kind, lora = False)
+ for res in results:
+ assert needle in (res["refused"] or ""), res["refused"]
diff --git a/tests/test_tp_kv_replicated_gloo.py b/tests/test_tp_kv_replicated_gloo.py
new file mode 100644
index 00000000000..bcf6aa6113c
--- /dev/null
+++ b/tests/test_tp_kv_replicated_gloo.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallelism with num_key_value_heads < tp (Gemma3 270M / 1B: one KV head) on a gloo CPU world vs one
+process: the key / value projections are replicated, each rank computes the head its query heads read."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_env_gloo import _ENV_KEYS, _case_env, _close # noqa: E402
+from test_tp_native_gloo import ( # noqa: E402
+ _batch,
+ _case_parity,
+ _case_sharded_load,
+ _check_parity,
+ _free_port,
+ _quantize_layers,
+ _save_checkpoint,
+ _train,
+ build,
+ tpn,
+)
+from test_tp_native_oracle import LLAMA # noqa: E402
+
+KV1 = {"num_attention_heads": 4, "num_key_value_heads": 1}
+
+
+def _case_layout(rank, world, tmp, arch, overrides):
+ """The derived plan and the per-rank attention attributes (local heads, GQA groups, the KV head)."""
+ from test_tp_native_oracle import ARCHS
+ from torch.distributed.device_mesh import init_device_mesh
+
+ repo, base = ARCHS[arch]
+ model = build(repo, {**base, **overrides})
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ region = model._unsloth_tp_plan.regions[0]
+ attn = region.module
+ styles = {p.name: p.style for p in region.projs}
+ k = next(p.module for p in region.projs if p.name == "k_proj")
+ return {
+ "styles": styles,
+ "kv_replicated": region.kv_replicated,
+ "heads": (attn.num_heads, attn.num_key_value_heads, attn.num_key_value_groups),
+ "rows": k.__dict__.get("_unsloth_tp_kv_rows"),
+ "k_placements": [p.is_replicate() for p in tpn._weight(k)[0].placements],
+ "saved_groups": attn.__dict__["_unsloth_tp_saved_attrs"].get("num_key_value_groups"),
+ }
+
+
+def _qlora(seed = 0):
+ from peft import LoraConfig, get_peft_model
+
+ sizes = {
+ "hidden_size": 128,
+ "intermediate_size": 256,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 1,
+ "head_dim": 32,
+ }
+ model = _quantize_layers(build(LLAMA, sizes, lora = False, seed = seed), False)
+ torch.manual_seed(seed + 1)
+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
+ return get_peft_model(model, config).train()
+
+
+def _case_qlora(rank, world, tmp):
+ import bitsandbytes as bnb
+ from torch.distributed.device_mesh import init_device_mesh
+
+ ref, model = _qlora(), _qlora()
+ k_full = model.base_model.model.model.layers[0].self_attn.k_proj.base_layer
+ k_full = bnb.functional.dequantize_4bit(k_full.weight.data, k_full.weight.quant_state)
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ k = model.base_model.model.model.layers[0].self_attn.k_proj
+ k_local = bnb.functional.dequantize_4bit(
+ k.base_layer.weight.data, k.base_layer.weight.quant_state
+ )
+ generated = model.generate(
+ input_ids = torch.zeros(1, 2, dtype = torch.long), max_new_tokens = 2, pad_token_id = 0
+ )
+ ids = _batch(ref)
+ return {
+ "k_local_equal": torch.equal(k_local, k_full), # one KV head: every rank holds all rows
+ "generated": tuple(generated.shape),
+ "ref": _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_),
+ "tp": _train(model, ids, 3, tpn.tp_clip_grad_norm_),
+ }
+
+
+CASES = {
+ "parity": _case_parity,
+ "layout": _case_layout,
+ "qlora": _case_qlora,
+ "sharded_load": _case_sharded_load,
+ "env": _case_env,
+}
+
+
+def _worker(rank, world, port, case, tmp, env, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ for key in _ENV_KEYS:
+ os.environ.pop(key, None)
+ os.environ.update(env)
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ if case != "env": # tp_env creates the process group from the environment
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = 2,
+ env = None,
+ ckpt_overrides = None,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ if ckpt_overrides is not None:
+ kwargs["ckpt"] = _save_checkpoint("llama", os.path.join(tmp, "ckpt"), ckpt_overrides)
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ for p in procs:
+ if p.is_alive():
+ p.kill()
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ results = [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ return results
+
+
+def test_kv_heads_below_tp_are_replicated_per_rank():
+ # tp 4 over 2 KV heads: ranks 0, 1 compute head 0 and ranks 2, 3 head 1.
+ overrides = {"num_attention_heads": 8, "num_key_value_heads": 2, "intermediate_size": 64}
+ results = _spawn("layout", 4, arch = "qwen3", overrides = overrides)
+ for rank, res in enumerate(results):
+ assert res["kv_replicated"]
+ assert res["styles"]["k_proj"] == res["styles"]["v_proj"] == tpn.KV_REPLICATED
+ assert res["styles"]["q_proj"] == tpn.COLWISE and res["styles"]["o_proj"] == tpn.ROWWISE
+ assert res["heads"] == (2, 1, 2), res["heads"] # 8 / 4 query heads over 1 local KV head
+ head_dim = 128
+ assert res["rows"] == ((rank // 2) * head_dim, head_dim), res["rows"]
+ assert res["k_placements"] == [True]
+ assert res["saved_groups"] == 4 # tp_replica restores the 1-GPU grouping
+
+
+ "arch,world,overrides",
+ [
+ (
+ "gemma3",
+ 2,
+ KV1,
+ ), # one KV head, trainable q_norm / k_norm (k_norm on the replicated head)
+ ("llama", 2, KV1),
+ ("llama_bias", 2, KV1), # k / v biases narrowed to the head
+ ("llama", 4, KV1), # 4 ranks reading one KV head
+ ("qwen3", 4, {"num_attention_heads": 8, "num_key_value_heads": 2, "intermediate_size": 64}),
+ ],
+)
+def test_kv_heads_below_tp_match_one_process(arch, world, overrides):
+ _check_parity(_spawn("parity", world, arch = arch, overrides = overrides), world)
+
+
+def test_kv_heads_below_tp_with_trainable_biases():
+ results = _spawn("parity", 2, arch = "llama_bias", overrides = KV1, lora_kwargs = {"bias": "all"})
+ _check_parity(results, 2)
+ assert any("k_proj.base_layer.bias" in key for key in results[0]["saved_tp"])
+
+
+def test_kv_heads_below_tp_sharded_loading_reads_kv_whole():
+ with tempfile.TemporaryDirectory() as ckpt:
+ _save_checkpoint("llama", ckpt, KV1)
+ results = _spawn("sharded_load", 2, arch = "llama", ckpt = ckpt)
+ _check_parity(results, 2)
+ for res in results:
+ assert res["diffs"] == [], res["diffs"][:6]
+ assert res["tp"][2] == res["v1"][2] and res["tp"][3] == res["v1"][3]
+ stats = res["stats"]
+ assert stats["read_bytes_sliced"] * 2 == stats["sharded_numel"] * 4, stats
+ assert stats["read_bytes_full"] == stats["replicated_numel"] * 4, stats
+
+
+def _bnb_cpu():
+ try:
+ import bitsandbytes # noqa: F401
+ except Exception:
+ return False
+ return True
+
+
[email protected](not _bnb_cpu(), reason = "bitsandbytes not installed")
+def test_kv_heads_below_tp_qlora_matches_one_process():
+ for res in _spawn("qlora", 2):
+ assert res["k_local_equal"] and res["generated"] == (1, 4)
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads) and grads
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5)
+
+
+def test_kv_heads_below_tp_env_trainer_matches_one_process():
+ ref = _spawn("env", 1, ckpt_overrides = KV1, bs = 2)[0]
+ assert ref["layout"] is None and not ref["is_tp"]
+ results = _spawn("env", 2, {"UNSLOTH_TP_SIZE": "2"}, ckpt_overrides = KV1, bs = 2)
+ for rank, res in enumerate(results):
+ assert res["layout"] == "tp 2" and res["is_tp"] and res["sharded_load"], res
+ _close(res["a"], ref["a"], what = rank)
+
+
+def test_kv_heads_not_dividing_tp_are_refused():
+ from test_tp_native_oracle import ARCHS
+
+ repo, base = ARCHS["qwen3"]
+ model = build(repo, {**base, "num_attention_heads": 12, "num_key_value_heads": 3})
+ with pytest.raises(ValueError, match = "not a multiple of it"):
+ tpn.derive_tp_plan(model, 4)
diff --git a/tests/test_tp_moe_dropout_gloo.py b/tests/test_tp_moe_dropout_gloo.py
new file mode 100644
index 00000000000..8cf12aa70f6
--- /dev/null
+++ b/tests/test_tp_moe_dropout_gloo.py
@@ -0,0 +1,274 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallel on a gloo CPU world vs one process: mixture-of-experts blocks run replicated (attention and
+dense MLPs still sharded), and dropout draws the masks one process would draw."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_gloo import _free_port, _train # noqa: E402
+from test_tp_native_oracle import LLAMA, QWEN3_MOE, _batch, build, tpn # noqa: E402
+
+MOE_OVERRIDES = {"mlp_only_layers": [1]}
+MOE_LORA = dict(target_parameters = ["mlp.experts.gate_up_proj", "mlp.experts.down_proj"])
+
+
+def _seeded_train(model, ids, steps, clip, seed):
+ torch.manual_seed(seed) # the same RNG state on every rank and in the one-process reference
+ model.train()
+ return _train(model, ids, steps, clip)
+
+
+class _FusedO:
+ """o_proj through Unsloth's fused LoRA_W (apply_lora_o, as llama.py's attention forward calls it): its
+ lora_dropout runs inside the fused Function, on this rank's row shard of the input."""
+
+ def __init__(self, attn):
+ self.attn, self.inner = attn, attn.o_proj.forward
+ self.busy = False
+ attn.o_proj.forward = self
+
+ def __call__(self, X, *args, **kwargs):
+ if self.busy: # apply_lora_o's own fallback: the projection's forward
+ return self.inner(X, *args, **kwargs)
+ from unsloth.kernels.fast_lora import apply_lora_o
+
+ self.busy = True
+ try:
+ return apply_lora_o(self.attn, X)
+ finally:
+ self.busy = False
+
+
+def _case_parity(
+ rank,
+ world,
+ tmp,
+ repo,
+ overrides,
+ lora_kwargs,
+ dropout_seed = None,
+ reentrant_checkpointing = False,
+ fused_lora_o = False,
+):
+ from safetensors.torch import load_file
+ from torch.distributed.device_mesh import init_device_mesh
+
+ if fused_lora_o:
+ from test_tp_native_gloo import _import_checkout_unsloth
+
+ error = _import_checkout_unsloth()
+ assert error is None, error
+ from unsloth.models import (
+ tp_native as T,
+ ) # one tp_native for the fused kernels and tp_replica
+ else:
+ T = tpn
+ ref = build(repo, overrides, **lora_kwargs)
+ model = build(repo, overrides, **lora_kwargs)
+ for m in (ref, model):
+ if reentrant_checkpointing:
+ # The first forward of a reentrant checkpoint runs under no_grad, the recompute with grad.
+ m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ m.enable_input_require_grads()
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ T.apply_tensor_parallel(model, mesh["tp"])
+ if fused_lora_o:
+ for m in (ref, model):
+ for layer in T._unwrap(m).model.layers:
+ _FusedO(layer.self_attn)
+ plan = model._unsloth_tp_plan
+ generated = model.generate(
+ input_ids = torch.zeros(1, 2, dtype = torch.long), max_new_tokens = 2, pad_token_id = 0
+ )
+ ids = _batch(ref)
+ if dropout_seed is None:
+ ref_out = _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_)
+ tp_out = _train(model, ids, 3, T.tp_clip_grad_norm_)
+ else:
+ ref_out = _seeded_train(ref, ids, 3, torch.nn.utils.clip_grad_norm_, dropout_seed)
+ tp_out = _seeded_train(model, ids, 3, T.tp_clip_grad_norm_, dropout_seed)
+ other = build(repo, overrides, **lora_kwargs)
+ other_out = _seeded_train(other, ids, 1, torch.nn.utils.clip_grad_norm_, dropout_seed + 1)
+ ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")
+ ref.save_pretrained(ref_dir)
+ T.save_tp_adapter(model, tp_dir)
+ return {
+ "ref": ref_out,
+ "tp": tp_out,
+ "other_loss": None if dropout_seed is None else other_out[2][0],
+ "regions": [(r.path, r.kind) for r in plan.regions],
+ "replicated_moe": plan.replicated_moe,
+ "moe_synced": all(b.__dict__.get("_unsloth_tp_size") == world for b in plan.moe_blocks),
+ "generated": tuple(generated.shape),
+ "saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),
+ "saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))
+ if rank == 0
+ else None,
+ }
+
+
+def _case_mean_half(rank, world, tmp, dtype):
+ import torch.distributed as dist
+
+ out = {}
+ for value in (40000.0, 3.0):
+ x = torch.full((4,), value, dtype = getattr(torch, dtype))
+ out[value] = tpn._MeanFromTP.apply(x, dist.group.WORLD, world)
+ return out
+
+
+CASES = {"parity": _case_parity, "mean_half": _case_mean_half}
+
+
+def _worker(rank, world, port, case, tmp, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = 2,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+def _check(results):
+ for rank, res in enumerate(results):
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads), sorted(set(grads) ^ set(ref_grads))[:6]
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6), (rank, losses, ref_losses)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5), (rank, norms, ref_norms)
+ assert res["generated"] == (1, 4)
+ saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]
+ assert set(saved_tp) == set(saved_ref)
+ for key in saved_ref:
+ # 3 AdamW steps (eps 1e-8) turn fp32 reduction-order noise on tiny expert grads into ~5e-5 here; a
+ # missed or doubled reduction moves weights by ~lr = 1e-2.
+ torch.testing.assert_close(saved_tp[key], saved_ref[key], rtol = 1e-3, atol = 2e-4, msg = key)
+ return results[0]
+
+
[email protected]("dtype", ["float16", "bfloat16"])
+def test_tp2_replicated_mean_does_not_overflow_half(dtype):
+ # 40000 is finite in fp16 but two replicas summed in fp16 are inf: the mean must not overflow.
+ for res in _spawn("mean_half", 2, dtype = dtype):
+ for value, y in res.items():
+ assert y.dtype == getattr(torch, dtype)
+ assert torch.equal(y, torch.full_like(y, value)), (value, y)
+
+
+def test_tp2_moe_runs_replicated_and_matches_one_process():
+ res = _check(_spawn("parity", 2, repo = QWEN3_MOE, overrides = MOE_OVERRIDES, lora_kwargs = MOE_LORA))
+ assert res["replicated_moe"] == 1 and res["moe_synced"]
+ assert sorted(k for _, k in res["regions"]) == ["attention", "attention", "mlp"], res["regions"]
+ assert any("experts" in k for k in res["saved_tp"]), sorted(res["saved_tp"])[:8]
+
+
+ "overrides,lora_kwargs",
+ [
+ ({}, {"lora_dropout": 0.25}), # lora_dropout: column input replicated, row input sharded
+ (
+ {"attention_dropout": 0.25},
+ {"lora_dropout": 0.25},
+ ), # + eager attention probabilities, per head
+ ],
+ ids = ["lora_dropout", "attention_and_lora_dropout"],
+)
+def test_tp2_dropout_draws_one_process_masks(overrides, lora_kwargs):
+ results = _spawn(
+ "parity", 2, repo = LLAMA, overrides = overrides, lora_kwargs = lora_kwargs, dropout_seed = 1234
+ )
+ res = _check(results)
+ assert res["other_loss"] != pytest.approx(res["ref"][2][0], rel = 1e-6), (
+ res["other_loss"],
+ res["ref"][2],
+ )
+
+
+def test_tp2_attention_dropout_in_a_fused_kernel_is_refused():
+ model = build(LLAMA, {"attention_dropout": 0.25})
+ tpn._text_config(model)._attn_implementation = "sdpa"
+ with pytest.raises(ValueError, match = "eager attention only"):
+ tpn.derive_tp_plan(model, 2)
+
+
+ "overrides,kwargs",
+ [
+ # Reentrant gradient checkpointing: the attention dropout masks of the no-grad first forward and of the
+ # recompute must both be one process's masks.
+ ({"attention_dropout": 0.25}, {"reentrant_checkpointing": True}),
+ # One KV head < tp 2: the replicated key / value projections read the whole (replicated) input.
+ ({"num_attention_heads": 4, "num_key_value_heads": 1}, {}),
+ # Unsloth's fused LoRA o_proj (LoRA_W) with lora_dropout: its input is sharded over tp.
+ ({}, {"fused_lora_o": True}),
+ ],
+ ids = ["reentrant_checkpointing", "kv_replicated_lora_dropout", "fused_lora_o_dropout"],
+)
+def test_tp2_dropout_paths_draw_one_process_masks(overrides, kwargs):
+ results = _spawn(
+ "parity",
+ 2,
+ repo = LLAMA,
+ overrides = overrides,
+ lora_kwargs = {"lora_dropout": 0.25},
+ dropout_seed = 1234,
+ **kwargs,
+ )
+ res = _check(results)
+ assert res["other_loss"] != pytest.approx(res["ref"][2][0], rel = 1e-6)
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index 5bb8df5bb9d..6e1dfee2f7d 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -390,6 +390,47 @@ def _case_qlora(rank, world, tmp, nested):
}
+class _DropoutMLP(torch.nn.Module):
+ """LlamaMLP with dropout between the column (gate / up) and row (down) projections."""
+
+ def forward(self, x):
+ return self.down_proj(self.drop(self.act_fn(self.gate_proj(x)) * self.up_proj(x)))
+
+
+def _add_mlp_dropout(model, p = 0.25):
+ for module in list(model.modules()):
+ if type(module).__name__ == "LlamaMLP":
+ module.add_module("drop", torch.nn.Dropout(p))
+ module.__class__ = type("LlamaMLP", (_DropoutMLP, type(module)), {})
+ return model
+
+
+def _case_mlp_dropout(rank, world, tmp, quantized):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ def make():
+ if quantized:
+ return _add_mlp_dropout(_qlora_model(False))
+ return _add_mlp_dropout(build(LLAMA, QLORA_SIZES).train())
+
+ ref, model = make(), make()
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ widths = [
+ r.module._unsloth_tp_local_width for r in model._unsloth_tp_plan.regions if r.kind == "mlp"
+ ]
+ ids = _batch(ref)
+ out = {}
+ for name, m, clip in (
+ ("ref", ref, torch.nn.utils.clip_grad_norm_),
+ ("tp", model, tpn.tp_clip_grad_norm_),
+ ):
+ torch.manual_seed(1234) # the same dropout RNG state on every rank and in the reference
+ out[name] = _train(m, ids, 2, clip)
+ out["widths"] = widths
+ return out
+
+
def _case_qlora_misaligned(rank, world, tmp):
from torch.distributed.device_mesh import init_device_mesh
@@ -710,7 +751,10 @@ def _case_trainer(
)
for key, overrides in {
"checkpoints": {"save_strategy": "steps", "save_steps": 1},
- "eval": {"eval_strategy": "steps", "eval_steps": 1},
+ "eval": {
+ "eval_strategy": "steps",
+ "eval_steps": 1,
+ }, # evaluates over dp (test_tp_eval_gloo.py)
"optim_8bit": {"optim": "adamw_bnb_8bit"},
}.items():
refusals[key] = _raised(
@@ -737,7 +781,6 @@ def _case_trainer(
model = model, args = _trainer_args(tmp, 2 * dp if tp == 1 else 2), train_dataset = data
)
trainer.train()
- refusals["evaluate"] = _raised(lambda: trainer.evaluate(data)) if tp > 1 else None
logs = [h for h in trainer.state.log_history if "loss" in h]
out_dir = os.path.join(tmp, f"adapter_{world}_{rank}")
trainer.save_model(out_dir)
@@ -932,6 +975,7 @@ CASES = {
"guards": _case_guards,
"qlora": _case_qlora,
"qlora_misaligned": _case_qlora_misaligned,
+ "mlp_dropout": _case_mlp_dropout,
"sharded_load": _case_sharded_load,
"sharded_lora_init": _case_sharded_lora_init,
"tp_fsdp2": _case_tp_fsdp2,
@@ -1049,6 +1093,24 @@ def test_tp2_qlora_matches_one_process(nested):
assert norms == pytest.approx(ref_norms, rel = 1e-5)
[email protected]("quantized", [False, True], ids = ["bf16_weights", "4bit_weights"])
+def test_tp2_mlp_dropout_between_column_and_row_matches_one_process(quantized):
+ # The row projection's local in_features is intermediate_size / tp, also once a 4-bit weight is sharded.
+ results = _spawn("mlp_dropout", 2, quantized = quantized)
+ for res in results:
+ assert res["widths"] and set(res["widths"]) == {QLORA_SIZES["intermediate_size"] // 2}, res[
+ "widths"
+ ]
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads) and grads
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5)
+
+
def test_tp4_gqa_matches_one_process():
overrides = {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 64}
_check_parity(_spawn("parity", 4, arch = "qwen3", overrides = overrides), 4)
@@ -1359,9 +1421,8 @@ def test_trainer_trains_a_tensor_parallel_model_like_one_process(dp, tp):
refusals = res["refusals"]
# Checkpoints are per rank, bitsandbytes AdamW steps local shards (or switches to torch's AdamW).
assert refusals["checkpoints"] is None, refusals
- assert "evaluation during training" in (refusals["eval"] or ""), refusals
+ assert refusals["eval"] is None, refusals
assert refusals["optim_8bit"] is None, refusals
- assert "Trainer.evaluate is not supported" in (refusals["evaluate"] or ""), refusals
if dp > 1:
assert "covers 2 of 4 ranks" in (refusals["no_shard_model"] or ""), refusals
# The tp ranks of a replica read the same batches; dp replicas read different ones.
diff --git a/tests/test_tp_native_oracle.py b/tests/test_tp_native_oracle.py
index 6a69ebcfb85..bd3910ead55 100644
--- a/tests/test_tp_native_oracle.py
+++ b/tests/test_tp_native_oracle.py
@@ -427,11 +427,11 @@ def test_shard_slice_unpermute_round_trip():
[
(LLAMA, {"num_attention_heads": 4, "num_key_value_heads": 4}, 3, "num_attention_heads = 4"),
(QWEN3, {"num_attention_heads": 6, "num_key_value_heads": 3}, 2, "num_key_value_heads = 3"),
- (GEMMA3, {}, 2, "larger than num_key_value_heads = 1"),
+ # num_key_value_heads < tp is replicated (test_tp_kv_replicated_gloo.py) only when tp is a multiple of it.
+ (GEMMA3, {"num_attention_heads": 12, "num_key_value_heads": 3}, 4, "not a multiple of it"),
(MISTRAL, {}, 2, "intermediate_size = 37"),
(OLMO2, {}, 2, "normalizes .* features, not one head"),
- (QWEN3_MOE, {}, 2, "mixture-of-experts"),
- (DEEPSEEK_V3, {}, 2, "mixture-of-experts|latent attention"),
+ (DEEPSEEK_V3, {}, 2, "latent attention"),
],
)
def test_refusals_plan(repo, overrides, tp, match):
@@ -440,20 +440,27 @@ def test_refusals_plan(repo, overrides, tp, match):
tpn.derive_tp_plan(model, tp)
-def test_refuses_attention_and_layer_dropout():
- with pytest.raises(ValueError, match = "attention dropout"):
- tpn.derive_tp_plan(build(LLAMA, {"attention_dropout": 0.1}), 2)
+def test_moe_blocks_run_replicated():
+ plan = tpn.derive_tp_plan(build(QWEN3_MOE, {"mlp_only_layers": [1]}, lora = False), 2)
+ assert plan.replicated_moe == 1
+ assert [r.kind for r in plan.regions] == ["attention", "attention", "mlp"]
+
+
+def test_attention_dropout_needs_the_eager_attention():
+ tpn.derive_tp_plan(build(LLAMA, {"attention_dropout": 0.1}), 2) # eager: supported
+ model = build(LLAMA, {"attention_dropout": 0.1})
+ tpn._text_config(model)._attn_implementation = "sdpa"
+ with pytest.raises(ValueError, match = "eager attention only"):
+ tpn.derive_tp_plan(model, 2)
model = build(LLAMA)
model.get_base_model().model.layers[0].mlp.add_module("drop", torch.nn.Dropout(0.1))
- with pytest.raises(ValueError, match = "dropout inside decoder layers"):
- tpn.derive_tp_plan(model, 2)
+ tpn.derive_tp_plan(model, 2) # dropout modules: supported
def test_refuses_dora_and_lora_bias():
for kwargs, match in (
({"use_dora": True}, "DoRA"),
({"lora_bias": True}, "lora_bias"),
- ({"lora_dropout": 0.1}, "lora_dropout"),
):
model = build(LLAMA, **kwargs)
region = tpn.derive_tp_plan(model, 2).regions[0]
@@ -461,7 +468,7 @@ def test_refuses_dora_and_lora_bias():
tpn._refuse_adapter(region.path, region.projs[0].module)
-def test_refuses_quantized_and_trainable_bases():
+def test_refuses_quantized_bases_and_accepts_trainable_ones():
model = build(LLAMA)
region = tpn.derive_tp_plan(model, 2).regions[0]
proj = region.projs[0].module
@@ -474,8 +481,8 @@ def test_refuses_quantized_and_trainable_bases():
gptq = type("QuantLinear", (torch.nn.Module,), {})()
with pytest.raises(ValueError, match = "GPTQ"):
tpn._refuse_base("p", proj, weight, gptq)
- with pytest.raises(ValueError, match = "full finetuning"):
- tpn._refuse_base("p", proj, torch.nn.Parameter(weight.detach().clone()), base)
+ # Full finetuning: a trainable base weight shards like any other (tests/test_tp_full_finetune_gloo.py).
+ tpn._refuse_base("p", proj, torch.nn.Parameter(weight.detach().clone()), base)
def test_refuses_global_width_attention():
diff --git a/tests/test_tp_replica_gloo.py b/tests/test_tp_replica_gloo.py
index 3680a94730b..6ec02ec6c1f 100644
--- a/tests/test_tp_replica_gloo.py
+++ b/tests/test_tp_replica_gloo.py
@@ -476,6 +476,45 @@ def _case_workspace(rank, world, tmp):
return out
+def _case_save_lora(rank, world, tmp):
+ """model.save_lora (save_lora_adapter, what the GRPO notebooks call) on every rank after a generate: rank 0
+ writes the one-process adapter (its state_dict is built from the gathered LoRA tensors, not the DTensors)."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import torch.distributed as dist
+ from unsloth.models import tp_native, tp_replica
+ from unsloth.models._utils import save_lora_adapter
+ from unsloth.save import patch_saving_functions
+
+ model, ref = build(LLAMA), build(LLAMA)
+ ids = _batch(model)
+ tp_native.apply_tensor_parallel(model, _mesh(world))
+ _train(model, ids, 2, tp_native.tp_clip_grad_norm_)
+ ref.load_state_dict(tp_native.tp_full_state_dict(model))
+ patch_saving_functions(model)
+ with torch.no_grad():
+ model.generate(
+ input_ids = ids[:, :4], max_new_tokens = 3, do_sample = False, use_cache = True, pad_token_id = 0
+ )
+ out = {}
+ tp_dir, ref_dir = os.path.join(tmp, "tp"), os.path.join(tmp, "ref")
+ _slow_rank0(model, "original_model_save_pretrained")
+ try:
+ save_lora_adapter(model, tp_dir)
+ out["error"] = None
+ except Exception as e:
+ out["error"] = f"{type(e).__name__}: {e}"
+ out["still_sharded"] = not _all_plain(model) and not tp_replica.replica_active(model)
+ if rank == 0:
+ patch_saving_functions(ref)
+ save_lora_adapter(ref, ref_dir)
+ dist.barrier()
+ out["files_tp"] = _files(tp_dir) if out["error"] is None else {}
+ out["files_ref"] = _files(ref_dir)
+ return out
+
+
CASES = {
"workspace": _case_workspace,
"destructive": _case_destructive,
@@ -483,6 +522,7 @@ CASES = {
"generate": _case_generate,
"qlora": _case_qlora,
"save": _case_save,
+ "save_lora": _case_save_lora,
"merged": _case_merged,
"refuse": _case_refuse,
}
@@ -629,6 +669,19 @@ def test_tp2_adapter_and_tokenizer_saves_equal_one_process():
assert "adapter_model.safetensors" in results[0]["files_ref"]
+def test_tp2_save_lora_after_generate_equals_one_process():
+ for rank, res in enumerate(_spawn("save_lora", 2)):
+ assert res["error"] is None, (rank, res["error"])
+ assert res["still_sharded"], rank
+ assert "adapter_model.safetensors" in res["files_ref"]
+ assert res["files_tp"].keys() == res["files_ref"].keys(), (
+ sorted(res["files_tp"]),
+ sorted(res["files_ref"]),
+ )
+ for name in res["files_ref"]:
+ assert res["files_tp"][name] == res["files_ref"][name], name
+
+
def test_tp2_merged_save_equals_one_process():
results = _spawn("merged", 2)
ref = results[0]
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 758b5a94ed1..3d0ab6dcc5f 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -268,6 +268,10 @@ def _lora_dropout_and_bias(projs, X):
p = _lora_dropout_p(proj, X, adapter)
if p is None:
return None
+ if p and proj.__dict__.get("_unsloth_tp_style") == "rowwise":
+ # Tensor parallel row projection: X is this rank's in_features shard, so the mask must be the
+ # rank's slice of one process's mask; the projection's own forward draws that (models/tp_native).
+ return None
dropout_p.append(p)
biases.append(proj.lora_B[adapter].bias)
if any(lora_bias_blocks_fused(proj) for proj in projs):
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 2bdb7d97719..b8c677eae28 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8419,13 +8419,23 @@ def save_lora_adapter(model, save_directory, *args, **kwargs):
"""`save_pretrained` over the adapter, cast to the embedding dtype. PEFT's own selection decides what an adapter contains, so it is handed the whole state dict and only the adapter tensors are cast: filtering down to `.lora_A.`/`.lora_B.` first, as the Zoo helper does, makes PEFT raise `KeyError` looking up `modules_to_save.<adapter>.weight`, and a DoRA run loses its `lora_magnitude_vector` the same way. Both are reachable without vLLM, since `get_peft_model` adds `embed_tokens` and `lm_head` to `modules_to_save` once new tokens are trained. Non-adapter entries pass through by reference."""
from .compressed_tensors_int4 import adapter_only_state_dict
- dtype = model.get_input_embeddings().weight.dtype
- with adapter_only_state_dict():
- kwargs["state_dict"] = {
- key: (value.to(dtype) if "lora_" in key else value)
- for key, value in model.state_dict().items()
- }
- return model.save_pretrained(save_directory, *args, **kwargs)
+ def write():
+ dtype = model.get_input_embeddings().weight.dtype
+ with adapter_only_state_dict():
+ kwargs["state_dict"] = {
+ key: (value.to(dtype) if "lora_" in key else value)
+ for key, value in model.state_dict().items()
+ }
+ return model.save_pretrained(save_directory, *args, **kwargs)
+
+ if _is_unsloth_tensor_parallel(model):
+ # A state_dict built here would hold the sharded DTensors (safetensors cannot read their storage) and
+ # bypass the tensor parallel save: build it on rank 0 from the gathered adapter instead.
+ from .tp_replica import collective_applies, tp_collective
+ if collective_applies(model):
+ return tp_collective(model, "save_lora", write, "adapters")
+ return model.save_pretrained(save_directory, *args, **kwargs)
+ return write()
def patch_peft_fast_inference(model):
diff --git a/unsloth/models/tp_env.py b/unsloth/models/tp_env.py
index 5cbd524ea02..52c40c9dd6b 100644
--- a/unsloth/models/tp_env.py
+++ b/unsloth/models/tp_env.py
@@ -182,10 +182,8 @@ def _rank0():
_REFUSED_LOAD_FLAGS = (
("fast_inference", "vLLM (fast_inference = True)"),
- ("full_finetuning", "full finetuning (full_finetuning = True; use UNSLOTH_FSDP2=1 instead)"),
("load_in_8bit", "8-bit weights (load_in_8bit = True; load in 4-bit or 16-bit)"),
("load_in_fp8", "fp8 weights (load_in_fp8)"),
- ("qat_scheme", "QAT (qat_scheme)"),
)
@@ -244,7 +242,9 @@ def tensor_parallel_env_load(fn):
mesh = layout.tp_mesh
_DEPTH["load"] += 1
try:
- if _four_bit(values, fn) or not layout.sharded_loading or _is_adapter(values):
+ # QAT swaps the loaded Linears for torchao FakeQuantizedLinear copies: load in full, shard later.
+ qat = values.get("qat_scheme") is not None
+ if _four_bit(values, fn) or not layout.sharded_loading or _is_adapter(values) or qat:
return fn(
*args, **kwargs
) # full load; get_peft_model shards it (Params4bit included)
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index acd0fa62eee..a5d357fb478 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -6,7 +6,8 @@
model = FastLanguageModel.get_peft_model(model, ...) # LoRA first: same init as 1 GPU
apply_tensor_parallel(model, init_device_mesh("cuda", (tp,), mesh_dim_names = ("tp",))["tp"])
-v1: LoRA / QLoRA training forward on dense decoders, tp ranks only (no Trainer / FSDP2 / generate).
+LoRA / QLoRA or full finetuning (trainable base weights as Shard DTensors) on dense decoders; QAT with
+weight-only fake quantization whose scales stay inside a shard (models/tp_native._check_qat).
Custom loop (torchrun --nproc_per_node tp):
params = [p for p in model.parameters() if p.requires_grad] # DTensors (keep plain ones in their own group)
@@ -36,8 +37,8 @@ reads the same batch, dp replicas read different ones:
The HF / TRL Trainer trains a tensor parallel model (tp only, or dp x tp after shard_model above): no DDP over
the tp ranks, batches sharded over dp only, tp-aware clipping and float16 scaler. Trainer checkpoints are per
rank (unsloth_tp/rank*.pt: local shards of the trainable weights and optimizer state, resumed on the same
-layout) next to the 1-GPU adapter; bitsandbytes Adam / AdamW step the local shards; no evaluation;
-trainer.save_model() writes the 1-GPU adapter.
+layout) next to the 1-GPU adapter; bitsandbytes Adam / AdamW step the local shards; evaluate / predict gather over dp only;
+trainer.save_model() writes the 1-GPU adapter (full finetuning: the whole 1-GPU model, gathered to rank 0).
Unmodified scripts: UNSLOTH_TP_SIZE=k torchrun ... applies all of the above from the environment
(models/tp_env.py), with UNSLOTH_FSDP2=1 (native FSDP2) or DDP-style gradient averaging for the dp dimension.
@@ -79,6 +80,9 @@ COLWISE = "colwise"
ROWWISE = "rowwise"
HEAD_NORM = "replicated_with_grad_allreduce"
ELEMENTWISE = "elementwise_replicated" # scalar-parameter activations inside an MLP
+# A key / value projection when num_key_value_heads < tp: kept whole on every rank (Replicate DTensors, 4-bit:
+# the rows of this rank's key / value head), each rank computing the one key / value head its query heads use.
+KV_REPLICATED = "kv_replicated"
# transformers TP styles (4.5x and 5.x names) -> what this module does with them.
_STYLES = {
@@ -134,7 +138,6 @@ _HEAD_NORMS = (
"key_layernorm",
)
_MLA_NAMES = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "kv_a_layernorm")
-_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
# An attention forward reading these sizes off the config would reshape to the global head count.
_GLOBAL_HEAD_READS = re.compile(r"config\.(num_attention_heads|num_key_value_heads)\b")
_GLOBAL_WIDTH_READS = re.compile(
@@ -164,7 +167,7 @@ class TPConfig:
class _Proj:
name: str
module: nn.Module
- style: str # COLWISE / ROWWISE
+ style: str # COLWISE / ROWWISE / KV_REPLICATED
parts: tuple # output (colwise) part sizes; one part unless fused / packed
from_plan: bool = False
@@ -179,6 +182,7 @@ class _Region:
heads: int = 0
kv_heads: int = 0
head_dim: int = 0
+ kv_replicated: bool = False # num_key_value_heads < tp: k / v projections replicated
@dataclasses.dataclass
@@ -189,6 +193,12 @@ class TPPlan:
replicated_layers: int = (
0 # decoder layers with no region (e.g. linear attention): run replicated
)
+ # Mixture-of-experts blocks: run replicated (experts not sharded), outputs averaged over tp.
+ moe_blocks: list = dataclasses.field(default_factory = list)
+
+ @property
+ def replicated_moe(self):
+ return len(self.moe_blocks)
def styles(self):
"""{projection path: style} (the plan in transformers form, local paths)."""
@@ -276,47 +286,68 @@ def _decoder_layers(model):
_ATTENTION_DROPOUT_KEYS = ("attention_dropout", "attn_pdrop")
-def _refuse_dropout(model, layers):
- """v1 has no TP-aware RNG (masks must differ per shard, match when replicated): refuse dropout."""
+def _attention_dropout(model):
text = _text_config(model)
for key in _ATTENTION_DROPOUT_KEYS:
value = getattr(text, key, None)
if isinstance(value, (int, float)) and value > 0:
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support attention dropout ({key} = {value}); set it to 0."
- )
- for name, module in layers.named_modules():
- if "lora_dropout" in name or not isinstance(module, nn.modules.dropout._DropoutNd):
+ return key, value
+ return None
+
+
+def _refuse_dropout(model, layers):
+ """Dropout runs on tensor parallel ranks with the masks one process draws (_TPDropoutMode); attention
+ dropout inside a fused kernel (SDPA / flash attention) draws its own mask and cannot be split per head."""
+ found = _attention_dropout(model)
+ if found is None:
+ return
+ key, value = found
+ implementation = getattr(_text_config(model), "_attn_implementation", None)
+ if implementation not in (None, "eager"):
+ raise ValueError(
+ f"Unsloth: tensor parallelism supports attention dropout ({key} = {value}) with the eager attention "
+ f"only: '{implementation}' draws the dropout mask inside a fused kernel, which cannot be split over "
+ "the tp ranks' heads. Load with attn_implementation = 'eager', or set it to 0."
+ )
+
+
+def _region_dropout_p(module, model):
+ """Whether a region can draw dropout in training (other than lora_dropout, which the projection handles)."""
+ if (
+ module.__dict__.get("_unsloth_tp_kind") == "attention"
+ and _attention_dropout(model) is not None
+ ):
+ return True
+ for name, sub in module.named_modules():
+ if "lora_dropout" in name:
continue
- if module.p > 0:
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support dropout inside decoder layers ({name}: p = "
- f"{module.p}); set it to 0."
- )
+ if isinstance(sub, nn.modules.dropout._DropoutNd) and sub.p > 0:
+ return True
+ value = sub.__dict__.get("attention_dropout", sub.__dict__.get("dropout"))
+ if isinstance(value, float) and value > 0:
+ return True
+ return False
+
+
+def _is_moe_block(module):
+ """A mixture-of-experts block (by structure: an `experts` child or an experts / sparse-MoE class)."""
+ for name, sub in module.named_modules():
+ leaf = name.rsplit(".", 1)[-1]
+ cls = type(sub).__name__
+ if leaf == "experts" or "Experts" in cls or "SparseMoe" in cls or "MoE" in cls:
+ return True
+ return False
-def _refuse_moe_mla(model, layers):
+def _refuse_mla(model, layers):
text = _text_config(model)
- for key in _MOE_CONFIG_KEYS:
- value = getattr(text, key, None)
- if isinstance(value, int) and value > 0:
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support mixture-of-experts models ({key} = {value}): "
- "expert sharding is not implemented. Use native FSDP2 (UNSLOTH_FSDP2=1) instead."
- )
if getattr(text, "kv_lora_rank", None):
raise ValueError(
"Unsloth: tensor parallelism v1 does not support multi-head latent attention (MLA, kv_lora_rank "
f"= {text.kv_lora_rank}). Use native FSDP2 (UNSLOTH_FSDP2=1) instead."
)
for name, module in layers.named_modules():
- leaf = name.rsplit(".", 1)[-1]
- cls = type(module).__name__
- if leaf == "experts" or "Experts" in cls or "SparseMoe" in cls or "MoE" in cls:
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ({name}: {cls})."
- )
- if leaf in _MLA_NAMES:
+ if name.rsplit(".", 1)[-1] in _MLA_NAMES:
raise ValueError(
f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ({name})."
)
@@ -426,22 +457,18 @@ def derive_tp_plan(
plan = dict(plan) if plan else _hf_plan(model)
compiled = [(k, _pattern(k), v) for k, v in plan.items()]
layers_path, layers = _decoder_layers(model)
- # Config / module MoE and MLA first: older transformers plans name expert styles v1 does not know.
- _refuse_moe_mla(model, layers)
+ _refuse_mla(model, layers)
_refuse_dropout(model, layers)
+ # MoE blocks run replicated, not sharded; an expert style inside a sharded region is refused in _classify.
for key, _, style in compiled:
- if style in _MOE_STYLES:
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support mixture-of-experts layers ('{key}': '{style}')."
- )
if style in _MLA_STYLES:
raise ValueError(
f"Unsloth: tensor parallelism v1 does not support multi-head latent attention ('{key}')."
)
- if style not in _STYLES and style not in _REPLICATED_STYLES:
+ if style not in _STYLES and style not in _REPLICATED_STYLES and style not in _MOE_STYLES:
raise ValueError(f"Unsloth: unknown tensor parallel style '{style}' for '{key}'.")
text = _text_config(model)
- regions, replicated, seen = [], 0, {}
+ regions, replicated, seen, moe = [], 0, {}, []
for index, layer in enumerate(layers):
found = 0
for child_name, child in layer.named_children():
@@ -450,7 +477,16 @@ def derive_tp_plan(
if child_name in _ATTN_NAMES
else ("mlp" if child_name in _MLP_NAMES else None)
)
- if kind is None or not _has_params(child):
+ if not _has_params(child):
+ continue
+ if kind != "attention" and _is_moe_block(child):
+ # Replicated: identical input and weights on every tp rank, so identical outputs and grads.
+ if id(child) not in seen:
+ moe.append(child)
+ seen[id(child)] = None
+ found += kind is not None
+ continue
+ if kind is None:
continue
path = f"{layers_path}.{index}.{child_name}"
found += 1
@@ -470,7 +506,13 @@ def derive_tp_plan(
source = (
"transformers" if any(p.from_plan for r in regions for p in r.projs) else "structural"
)
- return TPPlan(tp_size = tp_size, regions = regions, source = source, replicated_layers = replicated)
+ return TPPlan(
+ tp_size = tp_size,
+ regions = regions,
+ source = source,
+ replicated_layers = replicated,
+ moe_blocks = moe,
+ )
def _region(path, module, kind, compiled, text, tp):
@@ -513,11 +555,9 @@ def _region(path, module, kind, compiled, text, tp):
region.heads, region.kv_heads, region.head_dim = heads, kv_heads, head_dim
_check_divisible("num_attention_heads", heads, tp, path)
if kv_heads < tp:
- raise ValueError(
- f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} "
- f"({path}); replicating KV heads is not supported in v1. Use tp <= {kv_heads}."
- )
- _check_divisible("num_key_value_heads", kv_heads, tp, path)
+ _replicate_kv_heads(region, path, tp)
+ else:
+ _check_divisible("num_key_value_heads", kv_heads, tp, path)
for name, norm in norms:
weight = norm._parameters.get("weight")
numel = (
@@ -536,6 +576,9 @@ def _region(path, module, kind, compiled, text, tp):
for proj in projs:
weight, base = _weight(proj.module)
out_features, in_features = _logical_shape(weight, base)
+ if proj.style == KV_REPLICATED:
+ proj.parts = (out_features,)
+ continue
if proj.style == ROWWISE:
_check_divisible(f"{proj.name} in_features", in_features, tp, path)
proj.parts = (out_features,)
@@ -561,6 +604,36 @@ def _region(path, module, kind, compiled, text, tp):
return region
+def _replicate_kv_heads(region, path, tp):
+ """num_key_value_heads < tp: every rank keeps the key / value projections whole and computes the one key /
+ value head its query heads read (rank r: head r // (tp / num_key_value_heads)). Their gradients are partial
+ sums over tp (each rank backpropagates its own query heads only), reduced like any replicated factor."""
+ kv_heads = region.kv_heads
+ if kv_heads < 1 or tp % kv_heads:
+ raise ValueError(
+ f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} ({path}) and not "
+ f"a multiple of it, so the query heads of one rank would read several key / value heads. Use a tp "
+ f"size that divides or is a multiple of {kv_heads}."
+ )
+ names = {p.name for p in region.projs}
+ fused = names.intersection(_FUSED_QKV)
+ if fused:
+ raise ValueError(
+ f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} ({path}) and the "
+ f"key / value heads are fused into '{sorted(fused)[0]}'; replicating them needs separate key / value "
+ f"projections. Use tp <= {kv_heads}."
+ )
+ kv = [p for p in region.projs if p.name in _K + _V]
+ if not any(p.name in _K for p in kv) or not any(p.name in _V for p in kv):
+ raise ValueError(
+ f"Unsloth: tensor parallel size {tp} is larger than num_key_value_heads = {kv_heads} ({path}), and "
+ f"no separate key and value projections ({_K + _V}) were found to replicate."
+ )
+ for proj in kv:
+ proj.style = KV_REPLICATED
+ region.kv_replicated = True
+
+
def _logical_shape(weight, base):
full = getattr(weight, "_unsloth_tp_full_shape", None) # a sharded_loading slice
if full is not None:
@@ -742,10 +815,23 @@ def _tp_lora_parameters(proj, bias = False):
"""get_lora_parameters(_bias) on local shards; row bias is None (the region adds it after the reduce)."""
weight, base = _weight(proj)
W = _local(weight)
+ if (
+ not bias
+ ): # as kernels/utils.py get_lora_parameters (get_lora_parameters_bias does not fake quantize)
+ W = _fake_quantized(base, W)
W_quant = getattr(W, "quant_state", None)
b = None
- if bias and proj.__dict__.get("_unsloth_tp_style") == COLWISE and base.bias is not None:
+ rows = proj.__dict__.get("_unsloth_tp_kv_rows")
+ if rows is not None and not _is_params4bit(W):
+ W = W.narrow(0, *rows)
+ if (
+ bias
+ and proj.__dict__.get("_unsloth_tp_style") in (COLWISE, KV_REPLICATED)
+ and base.bias is not None
+ ):
b = _local(base.bias)
+ if rows is not None:
+ b = b.narrow(0, *rows)
lora_A = proj._modules.get("lora_A")
adapter = None
if lora_A is not None and not (
@@ -759,11 +845,12 @@ def _tp_lora_parameters(proj, bias = False):
if adapter is None:
out = (W, W_quant, None, None, None)
else:
+ B = _local(proj.lora_B[adapter].weight)
out = (
W,
W_quant,
_local(lora_A[adapter].weight),
- _local(proj.lora_B[adapter].weight),
+ B if rows is None else B.narrow(0, *rows),
proj.scaling[adapter],
)
return out + (b,) if bias else out
@@ -781,7 +868,14 @@ def _is_params4bit(t):
return any(cls.__name__ == "Params4bit" for cls in type(t).__mro__)
-def _refuse_base(path, proj_module, weight, base):
+def _refuse_base(
+ path,
+ proj_module,
+ weight,
+ base,
+ style = None,
+ tp = 1,
+):
cls = type(base).__name__
if weight is None:
raise ValueError(f"Unsloth: '{path}' ({cls}) has no dense weight (GPTQ / AWQ?).")
@@ -811,11 +905,7 @@ def _refuse_base(path, proj_module, weight, base):
or getattr(module, "weight_fake_quantizer", None) is not None
or getattr(module, "activation_fake_quantizer", None) is not None
):
- # The TP projection forward and get_lora_parameters run plain matmuls on the local shards.
- raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support QAT fake quantization ({path}: "
- f"{type(module).__name__})."
- )
+ _check_qat(path, module, base, style, tp)
if (
not _is_params4bit(weight)
and _preloaded(weight) is None
@@ -828,13 +918,79 @@ def _refuse_base(path, proj_module, weight, base):
raise ValueError(
f"Unsloth: tensor parallelism v1 cannot shard {weight.dtype} weights ({path})."
)
- if weight.requires_grad:
+ # A trainable base weight (full finetuning) is a Shard DTensor like the LoRA B / A shards: its local
+ # gradient is already complete, and a replicated row bias gets the whole gradient on every rank.
+
+
+def _qat_split_problem(fq, style, local_in):
+ """None when weight fake quantizer `fq` gives on this rank's shard exactly the rows / columns it gives on the
+ whole weight (its scales never reduce across the sharded dimension), else why not."""
+ cls = type(fq).__name__
+ config = getattr(fq, "config", None)
+ if next(fq.parameters(), None) is not None or next(fq.buffers(), None) is not None:
+ return f"{cls} keeps learned / static quantization parameters for the whole weight"
+ if cls == "Int4WeightFakeQuantizer":
+ group = getattr(config, "group_size", None)
+ if style == COLWISE:
+ return None # per-row fp8 scale and per-group int4 scales: rows are whole on every rank
+ if getattr(config, "activation_dtype", None) != torch.bfloat16:
+ return "its per-row float8 weight scale spans the in_features row projections shard"
+ if not isinstance(group, int) or local_in % group:
+ return f"in_features / tp = {local_in} is not a multiple of its group size {group}"
+ return None
+ if cls == "IntxFakeQuantizer":
+ if getattr(config, "range_learning", False) or not getattr(config, "is_dynamic", True):
+ return "range learning / static scales cover the whole weight"
+ granularity = getattr(config, "granularity", None)
+ kind = type(granularity).__name__
+ if kind == "PerGroup":
+ if style == COLWISE or local_in % granularity.group_size == 0:
+ return None
+ return f"in_features / tp = {local_in} is not a multiple of its group size {granularity.group_size}"
+ if kind == "PerRow" or (kind == "PerAxis" and getattr(granularity, "axis", None) == 0):
+ if style == COLWISE:
+ return None
+ return f"its per-row ({kind}) scale spans the in_features row projections shard"
+ return f"granularity {kind} reduces across the sharded dimension"
+ return f"{cls} is not a weight fake quantizer tensor parallelism knows the numerics of"
+
+
+def _check_qat(path, module, base, style, tp):
+ """QAT under tensor parallelism: dynamic weight-only fake quantization whose scales stay inside a shard
+ (int4 / int8 per-group, per-row on column projections) runs on the local shard, exactly as on one GPU;
+ anything that would need the whole weight or a whole activation row is refused."""
+ if getattr(module, "activation_fake_quantizer", None) is not None:
+ raise ValueError(
+ f"Unsloth: tensor parallelism does not support QAT fake quantization of activations ({path}: "
+ f"{type(module).__name__}): its per-token scales need the whole input row, which row parallel "
+ "projections hold 1 / tp of. Use a weight-only qat_scheme ('int4', 'int8', 'cactus')."
+ )
+ fq = getattr(module, "weight_fake_quantizer", None)
+ if module is not base or fq is None or type(module).__name__ != "FakeQuantizedLinear":
+ raise ValueError(
+ f"Unsloth: tensor parallelism does not support QAT fake quantization in {type(module).__name__} "
+ f"({path}): only torchao FakeQuantizedLinear base weights with a weight fake quantizer."
+ )
+ if style is None:
+ return
+ if style == KV_REPLICATED:
+ style = COLWISE # whole rows and whole in_features on every rank: column-like for the fake quantizer
+ weight = base._parameters.get("weight")
+ in_features = _logical_shape(weight, base)[1] if weight is not None else 0
+ problem = _qat_split_problem(fq, style, in_features // max(int(tp), 1))
+ if problem is not None:
raise ValueError(
- f"Unsloth: tensor parallelism v1 shards LoRA / QLoRA training only; '{path}' has a trainable base "
- "weight (full finetuning). Use native FSDP2 for full finetuning."
+ f"Unsloth: tensor parallelism cannot shard the QAT weight fake quantization of '{path}' "
+ f"({type(fq).__name__}) exactly: {problem}."
)
+def _fake_quantized(base, W):
+ """The local shard through the base layer's QAT weight fake quantizer (as FakeQuantizedLinear.forward)."""
+ fq = base._modules.get("weight_fake_quantizer")
+ return W if fq is None else fq(W)
+
+
def _active_adapter(proj):
"""Active adapter names, resolved like kernels/utils.py get_lora_parameters; empty stays empty."""
adapters = getattr(proj, "active_adapters", None)
@@ -872,10 +1028,10 @@ def _refuse_adapter(path, proj):
raise ValueError(f"Unsloth: tensor parallelism v1 does not support LoRA variants ({path}).")
for name in names:
dropout = proj.lora_dropout[name] if "lora_dropout" in proj._modules else None
- if getattr(dropout, "p", 0) > 0:
- # Masks must match across ranks for replicated inputs and differ for sharded ones.
+ if getattr(dropout, "p", 0) > 0 and type(dropout) is not nn.Dropout:
raise ValueError(
- f"Unsloth: tensor parallelism v1 does not support lora_dropout > 0 ({path}); use lora_dropout = 0."
+ f"Unsloth: tensor parallelism supports nn.Dropout as lora_dropout only ({path}: "
+ f"{type(dropout).__name__})."
)
if proj.lora_B[name].bias is not None:
raise ValueError(f"Unsloth: tensor parallelism v1 does not support lora_bias ({path}).")
@@ -1035,6 +1191,227 @@ class _ReduceFromTP(torch.autograd.Function):
return grad, None
+class _MeanFromTP(torch.autograd.Function):
+ """All-reduce mean forward, identity backward: a replicated block's output made bitwise equal on every tp
+ rank (CUDA scatter / index_add kernels are not deterministic), its gradient already the same everywhere."""
+
+ @staticmethod
+ def forward(ctx, x, group, size):
+ if x.element_size() >= 4:
+ return _all_reduce(x, group) / size
+ # fp16 / bf16: sum in float32, or tp copies of a large finite value overflow to inf before the divide
+ return (_all_reduce(x.float(), group) / size).to(x.dtype)
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad, None, None
+
+
+def _moe_post_hook(module, args, output):
+ if module.__dict__.get("_unsloth_tp_gathered"):
+ return None # tp_replica: one rank may run alone
+ group, size = module._unsloth_tp_group, module._unsloth_tp_size
+
+ def sync(y):
+ return _MeanFromTP.apply(y, group, size) if y.is_floating_point() else y
+
+ if isinstance(output, torch.Tensor):
+ return sync(output)
+ if isinstance(output, (tuple, list)) and output and isinstance(output[0], torch.Tensor):
+ return type(output)((sync(output[0]), *output[1:]))
+ raise RuntimeError(
+ f"Unsloth: tensor parallel replicated mixture-of-experts block {type(module).__name__} returned "
+ f"{type(output)}."
+ )
+
+
+class _DropoutState:
+ """One training forward of a region: where the tensor parallel layout of its activations stands."""
+
+ def __init__(self, module):
+ self.kind = module._unsloth_tp_kind
+ self.rank, self.tp = module._unsloth_tp_coord
+ self.local_heads = module.__dict__.get("_unsloth_tp_heads")
+ self.local_width = module._unsloth_tp_local_width # row projections' local in_features
+ self.row_bias = any(_weight(p)[1].bias is not None for p in module._unsloth_tp_row_projs)
+ self.phase = 0 # 0: replicated input, 1: column sharded, 2: row partial sums
+ self.bypass = 0
+ self.path = type(module).__name__
+
+
+def _full_mask(shape, dim, rank, tp, p, like):
+ """The rank-th 1 / tp of the boolean keep mask one process draws for the whole tensor (same RNG draws)."""
+ full = list(shape)
+ full[dim] *= tp
+ with torch._C.DisableTorchFunction():
+ mask = F.dropout(torch.ones(full, dtype = like.dtype, device = like.device), p, True) != 0
+ return mask.narrow(dim, rank * shape[dim], shape[dim]).contiguous()
+
+
+class _NativeDropoutScale(torch.autograd.Function):
+ """x * mask * scale rounded once from accumulator precision, as ATen's fused native_dropout kernel: its
+ forward scale is 1 / (accscalar)(1 - p), its backward native_dropout_backward(grad, mask, 1 / (1 - p))."""
+
+ @staticmethod
+ def forward(ctx, x, mask, p):
+ acc = torch.float64 if x.dtype == torch.float64 else torch.float32
+ ctx.save_for_backward(mask)
+ ctx.scale = 1.0 / (1.0 - p)
+ scale = 1.0 / torch.tensor(1.0 - p, dtype = acc).item()
+ return torch.ops.aten.native_dropout_backward(x, mask, scale)
+
+ @staticmethod
+ def backward(ctx, grad):
+ (mask,) = ctx.saved_tensors
+ return torch.ops.aten.native_dropout_backward(grad, mask, ctx.scale), None, None
+
+
+def _apply_mask(
+ x,
+ mask,
+ p,
+ inplace = False,
+):
+ """x after dropout with this boolean keep mask, rounded as F.dropout rounds on x's device: ATen's fused
+ native_dropout on CUDA / XPU / lazy / privateuse1 (scale applied in accumulator precision), else a noise
+ tensor in x's dtype holding the rounded 1 / (1 - p) (Dropout.cpp _dropout_impl)."""
+ if x.device.type in ("cuda", "xpu", "lazy", "privateuseone") and 0 < p < 1 and x.numel() > 0:
+ y = _NativeDropoutScale.apply(x, mask, p)
+ return x.copy_(y) if inplace else y
+ noise = mask.to(x.dtype)
+ if p < 1:
+ noise.div_(1 - p)
+ return x.mul_(noise) if inplace else x * noise
+
+
+def _tp_dropout(
+ state,
+ input,
+ p = 0.5,
+ training = True,
+ inplace = False,
+):
+ if not training or p == 0 or state.bypass or state.phase == 0:
+ # Replicated activations: every rank draws the same mask as one process (same RNG, same shape).
+ return F.dropout(input, p, training, inplace)
+ if state.phase == 2:
+ if state.row_bias:
+ raise RuntimeError(
+ f"Unsloth: dropout on the row projection output inside {state.path} runs before its bias, which "
+ "tensor parallelism adds after the reduce; set that dropout to 0."
+ )
+ return F.dropout(
+ input, p, training, inplace
+ ) # linear in the partial sums: same mask everywhere
+ if state.kind == "attention" and input.dim() >= 3 and input.shape[1] == state.local_heads:
+ dim = 1 # [batch, local heads, ...]: attention probabilities
+ elif state.kind == "mlp" and input.shape[-1] == state.local_width:
+ dim = input.dim() - 1 # [..., intermediate / tp]
+ else:
+ raise RuntimeError(
+ f"Unsloth: tensor parallelism cannot tell how the dropout input {tuple(input.shape)} inside "
+ f"{state.path} is sharded; set that dropout to 0."
+ )
+ return _apply_mask(
+ input, _full_mask(input.shape, dim, state.rank, state.tp, p, input), p, inplace
+ )
+
+
+def _dropout_mode_class():
+ from torch.overrides import TorchFunctionMode
+ class _TPDropoutMode(TorchFunctionMode):
+ def __init__(self, state):
+ super().__init__()
+ self.state = state
+
+ def __torch_function__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ kwargs = kwargs or {}
+ if func is F.dropout:
+ return _tp_dropout(self.state, *args, **kwargs)
+ if func is torch.dropout:
+ return _tp_dropout(self.state, *args, **kwargs)
+ if (
+ func is F.scaled_dot_product_attention
+ and (kwargs.get("dropout_p", args[4] if len(args) > 4 else 0.0) or 0.0) > 0
+ ):
+ raise RuntimeError(
+ f"Unsloth: {self.state.path} passes dropout_p > 0 to scaled_dot_product_attention, which "
+ "draws the mask inside the kernel and cannot be split over the tp ranks' heads; use the eager "
+ "attention or set the attention dropout to 0."
+ )
+ if (
+ func in _UNSPLIT_DROPOUTS
+ and (args[2] if len(args) > 2 else kwargs.get("training", True))
+ and (args[1] if len(args) > 1 else kwargs.get("p", 0.5)) > 0
+ ):
+ raise RuntimeError(
+ f"Unsloth: tensor parallelism supports plain dropout only; {func.__name__} inside "
+ f"{self.state.path} is not supported."
+ )
+ return func(*args, **kwargs)
+
+ return _TPDropoutMode
+
+
+_UNSPLIT_DROPOUTS = (
+ F.dropout1d,
+ F.dropout2d,
+ F.dropout3d,
+ F.alpha_dropout,
+ F.feature_alpha_dropout,
+)
+
+
+def _dropout_region_forward(module, forward):
+ """The region forward under _TPDropoutMode in training: masks that slice / match one process's masks."""
+ mode_class = _dropout_mode_class()
+
+ def forward_with_tp_dropout(*args, **kwargs):
+ if module.__dict__.get("_unsloth_tp_gathered") or not module.training:
+ # Not torch.is_grad_enabled(): a reentrant checkpoint runs its first forward under no_grad and
+ # recomputes with grad; both must draw one process's masks.
+ return forward(*args, **kwargs)
+ state = _DropoutState(module)
+ module.__dict__["_unsloth_tp_dropout"] = state
+ try:
+ with mode_class(state):
+ return forward(*args, **kwargs)
+ finally:
+ module.__dict__.pop("_unsloth_tp_dropout", None)
+
+ forward_with_tp_dropout.__wrapped__ = forward
+ return forward_with_tp_dropout
+
+
+def _lora_dropout(self, adapter, x):
+ """lora_dropout on this projection's input: replicated (column) or this rank's in_features (row)."""
+ drop = self.lora_dropout[adapter]
+ p = getattr(drop, "p", 0)
+ if not (isinstance(drop, nn.Dropout) and drop.training and p > 0):
+ return drop(x)
+ state = self.__dict__.get("_unsloth_tp_state_of")
+ state = state() if state is not None else None
+ if state is not None:
+ state.bypass += 1
+ try:
+ if self._unsloth_tp_style in (
+ COLWISE,
+ KV_REPLICATED,
+ ): # whole (replicated) input on every rank
+ return F.dropout(x, p, True)
+ rank, tp = self._unsloth_tp_coord
+ return _apply_mask(x, _full_mask(x.shape, x.dim() - 1, rank, tp, p, x), p)
+ finally:
+ if state is not None:
+ state.bypass -= 1
+
+
def copy_to_tp(x, group):
return _CopyToTP.apply(x, group)
@@ -1093,12 +1470,29 @@ def _region_post_hook(module, args, output):
def _tp_projection_forward(self, x, *args, **kwargs):
+ state_of = self.__dict__.get("_unsloth_tp_state_of")
+ state = state_of() if state_of is not None else None
+ if state is not None and self._unsloth_tp_style == COLWISE:
+ state.phase = max(state.phase, 1)
+ out = _tp_projection_compute(self, x)
+ if state is not None and self._unsloth_tp_style == ROWWISE:
+ state.phase = 2
+ return out
+
+
+def _tp_projection_compute(self, x):
weight, base = _weight(self)
- W = _local(weight)
+ W = _fake_quantized(base, _local(weight))
bias = (
- base.bias if self._unsloth_tp_style == COLWISE else None
+ base.bias if self._unsloth_tp_style != ROWWISE else None
) # row bias: once, after the reduce
bias = None if bias is None else _local(bias)
+ rows = self.__dict__.get("_unsloth_tp_kv_rows") # KV_REPLICATED: this rank's key / value head
+ if rows is not None:
+ if not _is_params4bit(W): # a 4-bit weight already holds only these rows
+ W = W.narrow(0, *rows)
+ if bias is not None:
+ bias = bias.narrow(0, *rows)
input_dtype = x.dtype
if _is_params4bit(W):
import bitsandbytes as bnb
@@ -1118,9 +1512,11 @@ def _tp_projection_forward(self, x, *args, **kwargs):
continue
A = _local(lora_A[adapter].weight)
B = _local(self.lora_B[adapter].weight)
+ if rows is not None:
+ B = B.narrow(0, *rows)
scaling = self.scaling[adapter]
x_lora = x if torch.is_autocast_enabled() else x.to(A.dtype)
- x_lora = self.lora_dropout[adapter](x_lora)
+ x_lora = _lora_dropout(self, adapter, x_lora)
out = out + (F.linear(F.linear(x_lora, A), B) * scaling).to(out.dtype)
return out
@@ -1204,6 +1600,18 @@ def _cache_in_use(past):
return True
+def _replica_in(module):
+ """A whole replica is gathered (tp_replica: generate / for_inference): the forward runs as on one GPU. A
+ wrapper installed over forward_without_cache after apply_tensor_parallel (the GRPO hidden states forward)
+ keeps calling it, so the check lives here and not only in the forward swap of tp_replica."""
+ root = module.__dict__.get("_unsloth_tp_root")
+ if root is None:
+ return False
+ from .tp_replica import replica_active
+
+ return replica_active(root)
+
+
def _refuse_cached_forward(module):
"""Refuse a filled KV cache (instance attribute: LoraModel calls self.model.forward directly)."""
forward = module.forward
@@ -1217,7 +1625,7 @@ def _refuse_cached_forward(module):
past = kwargs.get("past_key_values", kwargs.get("past_key_value"))
if past is None and index is not None and len(args) > index:
past = args[index]
- if _cache_in_use(past):
+ if _cache_in_use(past) and not _replica_in(module):
raise RuntimeError(
"Unsloth: cached decoding (past_key_values) is not supported on a tensor parallel model in "
"v1; save the adapter (save_tp_adapter) and generate on one GPU."
@@ -1237,6 +1645,30 @@ def _mark_tokenizers(chain):
mark_world_save(tokenizer)
+def _has_lora_dropout(model):
+ return any(
+ "lora_dropout" in name and isinstance(m, nn.Dropout) and m.p > 0
+ for name, m in model.named_modules()
+ )
+
+
+def _global_rank0(group):
+ import torch.distributed as dist
+ return dist.get_global_rank(group, 0)
+
+
+def _sync_rng(group, dist_rank0):
+ import torch.distributed as dist
+
+ states = [torch.get_rng_state()]
+ if torch.cuda.is_available() and torch.cuda.is_initialized():
+ states.append(torch.cuda.get_rng_state())
+ dist.broadcast_object_list(states, src = dist_rank0, group = group)
+ torch.set_rng_state(states[0])
+ if len(states) > 1 and torch.cuda.is_available():
+ torch.cuda.set_rng_state(states[1].cpu())
+
+
def _sync_trainable(model, group, src):
import torch.distributed as dist
with torch.no_grad():
@@ -1247,7 +1679,8 @@ def _sync_trainable(model, group, src):
def _set_local_heads(module, region, tp):
- heads, kv_heads = region.heads // tp, region.kv_heads // tp
+ heads = region.heads // tp
+ kv_heads = 1 if region.kv_replicated else region.kv_heads // tp
# What tp_replica puts back while a whole replica is gathered.
module.__dict__["_unsloth_tp_saved_attrs"] = {
name: module.__dict__.get(name, _MISSING)
@@ -1256,6 +1689,7 @@ def _set_local_heads(module, region, tp):
"num_key_value_heads",
"num_attention_heads",
"num_kv_heads",
+ "num_key_value_groups",
"config",
)
}
@@ -1269,6 +1703,9 @@ def _set_local_heads(module, region, tp):
):
if always or isinstance(module.__dict__.get(name), int):
module.__dict__[name] = value
+ if isinstance(getattr(module, "num_key_value_groups", None), int):
+ # repeat_kv / GQA grouping: local query heads per local key / value head (changes when KV is replicated).
+ module.__dict__["num_key_value_groups"] = heads // kv_heads
config = module.__dict__.get("config")
if config is None:
return
@@ -1322,7 +1759,7 @@ def apply_tensor_parallel(
for proj in region.projs:
path = f"{region.path}.{proj.name}"
weight, base = _weight(proj.module)
- _refuse_base(path, proj.module, weight, base)
+ _refuse_base(path, proj.module, weight, base, proj.style, tp)
_refuse_adapter(path, proj.module)
_check_preloaded(path, proj, weight, base, tp_mesh)
if id(weight) in covered:
@@ -1356,7 +1793,17 @@ def apply_tensor_parallel(
hooked_norms = (
set()
) # a norm weight shared by several regions gets one tp sum, not one per region
+ dropout_regions = 0
for region in tp_plan.regions:
+ # Row projections' in_features, read before sharding: a sharded Params4bit's quant_state is already local.
+ local_width = next(
+ (
+ _logical_shape(*_weight(p.module))[1] // tp
+ for p in region.projs
+ if p.style == ROWWISE
+ ),
+ None,
+ )
for proj in region.projs:
_shard_projection(region, proj, tp_mesh, rank, tp)
for _, norm in region.norms:
@@ -1371,8 +1818,28 @@ def apply_tensor_parallel(
module._unsloth_tp_input = _input_name(module)
if region.kind == "attention":
_set_local_heads(module, region, tp)
+ module._unsloth_tp_kind = region.kind
+ module._unsloth_tp_coord = (rank, tp)
+ module._unsloth_tp_local_width = local_width
+
+ def state_of(module = module):
+ return module.__dict__.get("_unsloth_tp_dropout")
+
+ for proj in region.projs:
+ proj.module.__dict__["_unsloth_tp_state_of"] = state_of
+ proj.module._unsloth_tp_coord = (rank, tp)
+ if _region_dropout_p(module, model):
+ dropout_regions += 1
+ module.__dict__["forward"] = _dropout_region_forward(module, module.forward)
module.register_forward_pre_hook(_region_pre_hook, with_kwargs = True)
module.register_forward_hook(_region_post_hook)
+ if tp > 1 and (dropout_regions or _has_lora_dropout(model)):
+ # Dropout masks are drawn from the ranks' own RNG in lockstep: start every tp rank from rank 0's state.
+ _sync_rng(group, dist_rank0 = _global_rank0(group))
+ for block in tp_plan.moe_blocks:
+ block._unsloth_tp_group = group
+ block._unsloth_tp_size = tp
+ block.register_forward_hook(_moe_post_hook)
# Unsloth decode paths bypass the regions: mark and guard every wrapper level.
info = {"size": tp, "rank": rank, "regions": len(tp_plan.regions), "source": tp_plan.source}
inner = _unwrap(model)
@@ -1396,18 +1863,72 @@ def apply_tensor_parallel(
if tp_plan.replicated_layers
else ""
)
+ + _kv_replicated_note(tp_plan)
+ + (
+ f", {tp_plan.replicated_moe} mixture-of-experts blocks replicated (experts not sharded)"
+ if tp_plan.replicated_moe
+ else ""
+ )
+ "."
)
return model
+def _kv_replicated_note(tp_plan):
+ regions = [r for r in tp_plan.regions if r.kv_replicated]
+ if not regions:
+ return ""
+ kv = sorted({r.kv_heads for r in regions})
+ return (
+ f", key / value heads replicated in {len(regions)} attention regions (num_key_value_heads = "
+ f"{'/'.join(map(str, kv))} < tp {tp_plan.tp_size})"
+ )
+
+
+def _shard_kv_replicated(region, proj, mesh, rank, tp):
+ """KV_REPLICATED: weight / bias / LoRA kept whole (Replicate); the forward narrows to this rank's head."""
+ module = proj.module
+ weight, base = _weight(module)
+ path = f"{region.path}.{proj.name}"
+ if _preloaded(weight) is not None or _preloaded(base.bias) is not None:
+ raise ValueError(
+ f"Unsloth: '{path}' was sliced by sharded_loading, but its key / value heads are replicated "
+ "(num_key_value_heads < tp); pass the same plan and tp mesh to both."
+ )
+ out_features = proj.parts[0]
+ kv_heads = region.kv_heads
+ if out_features != kv_heads * region.head_dim:
+ raise ValueError(
+ f"Unsloth: '{path}' has {out_features} outputs, not num_key_value_heads * head_dim = "
+ f"{kv_heads * region.head_dim}; its key / value heads cannot be replicated."
+ )
+ head = rank // (tp // kv_heads)
+ if _is_params4bit(weight):
+ # The rows of this rank's head (bitwise a slice of the 1-GPU codes); tp_replica regathers the whole.
+ _shard_params4bit(path, base, weight, COLWISE, (out_features,), head, kv_heads)
+ base.weight._unsloth_tp_layout = (KV_REPLICATED, (out_features,), rank, tp)
+ else:
+ _set_dtensor(base, "weight", weight, mesh, "replicate")
+ if base.bias is not None:
+ _set_dtensor(base, "bias", base.bias, mesh, "replicate")
+ lora_A = module._modules.get("lora_A")
+ if lora_A is not None:
+ for adapter in lora_A.keys():
+ A, B = lora_A[adapter], module.lora_B[adapter]
+ _set_dtensor(A, "weight", A.weight, mesh, "replicate")
+ _set_dtensor(B, "weight", B.weight, mesh, "replicate")
+ module.__dict__["_unsloth_tp_kv_rows"] = (head * region.head_dim, region.head_dim)
+
+
def _shard_projection(region, proj, mesh, rank, tp):
module = proj.module
weight, base = _weight(module)
col = proj.style == COLWISE
parts = proj.parts
in_parts = (_logical_shape(weight, base)[1],)
- if _preloaded(weight) is not None:
+ if proj.style == KV_REPLICATED:
+ _shard_kv_replicated(region, proj, mesh, rank, tp)
+ elif _preloaded(weight) is not None:
_wrap_preloaded(base, "weight", weight, mesh, 0 if col else 1)
elif _is_params4bit(weight):
_shard_params4bit(f"{region.path}.{proj.name}", base, weight, proj.style, parts, rank, tp)
@@ -1423,7 +1944,7 @@ def _shard_projection(region, proj, mesh, rank, tp):
rank,
tp,
)
- if base.bias is not None:
+ if base.bias is not None and proj.style != KV_REPLICATED:
if col:
if _preloaded(base.bias) is not None:
_wrap_preloaded(base, "bias", base.bias, mesh, 0)
@@ -1436,7 +1957,7 @@ def _shard_projection(region, proj, mesh, rank, tp):
if guard is not None:
guard.remove()
lora_A = module._modules.get("lora_A")
- if lora_A is not None:
+ if lora_A is not None and proj.style != KV_REPLICATED:
for adapter in lora_A.keys():
A, B = lora_A[adapter], module.lora_B[adapter]
if col:
@@ -2139,6 +2660,8 @@ def _materialize_sharded(model, name, kwargs, mesh, plan, device):
shards = {}
for region in tp_plan.regions:
for proj in region.projs:
+ if proj.style == KV_REPLICATED:
+ continue # read whole: every rank keeps the replicated key / value projections
weight, base = _weight(proj.module)
col = proj.style == COLWISE
shards[id(weight)] = (proj, 0 if col else 1, proj.parts if col else (weight.shape[1],))
@@ -2324,6 +2847,7 @@ def load_tensor_parallel(
_TP_DATA = "_unsloth_tp_data"
_DDP_MESH = "_unsloth_tp_ddp_mesh"
_DP_SYNC = "_unsloth_tp_dp_sync"
+_TP_GATHER = "_unsloth_tp_gather_mesh"
_UNSUPPORTED_OPTIMIZERS = (
"8bit",
"bnb",
@@ -2420,11 +2944,6 @@ def _refuse_tp_trainer_args(args, cpu_shards = False):
problems.append(
"accelerate FSDP / ParallelismConfig (use shard_model over mesh['dp'] for data parallel)"
)
- if (
- _strategy(getattr(args, "eval_strategy", getattr(args, "evaluation_strategy", "no")))
- != "no"
- ):
- problems.append("evaluation during training (set eval_strategy = 'no')")
for flag in ("load_best_model_at_end", "push_to_hub"):
if getattr(args, flag, False):
problems.append(flag)
@@ -2443,6 +2962,22 @@ def _refuse_tp_trainer_args(args, cpu_shards = False):
)
+def _refuse_full_finetuning_fsdp2(model):
+ """Full finetuning on tp x native FSDP2 trains, but save_model (also every Trainer checkpoint) cannot gather
+ the whole model there yet: refuse before training instead of at the first save."""
+ if _has_peft_config(model):
+ return
+ from .tp_replica import _fsdp2_units, tp_root
+
+ if _fsdp2_units(tp_root(model)):
+ raise RuntimeError(
+ "Unsloth: full finetuning on a tensor parallel x native FSDP2 model cannot save the whole model "
+ "yet (Trainer.save_model and Trainer checkpoints). Use tensor parallel alone, or its DDP-style data "
+ "parallel (UNSLOTH_TP_SIZE without UNSLOTH_FSDP2), or native FSDP2 alone (UNSLOTH_FSDP2=1 without "
+ "UNSLOTH_TP_SIZE)." + _env_hint()
+ )
+
+
def _tp_fix_train_sampler(sampler, args, dp_size, dp_rank):
"""Make a Trainer train sampler identical on the tp ranks of one data parallel replica."""
from transformers import trainer_pt_utils
@@ -2564,17 +3099,17 @@ def tp_trainer_setup(
cpu_shards = tp_mesh.device_type == "cpu"
_tp_bnb_optim(trainer.args, cpu_shards)
_refuse_tp_trainer_args(trainer.args, cpu_shards)
+ _refuse_full_finetuning_fsdp2(model)
ddp_mesh = _ddp_mesh(model)
dp_size, dp_rank = (dp_mesh.size(), dp_mesh.get_local_rank()) if dp_mesh is not None else (1, 0)
trainer.accelerator.__dict__[_TP_DATA] = (dp_size, dp_rank)
+ trainer.accelerator.__dict__[_TP_GATHER] = dp_mesh
_tp_train_sampler(trainer, dp_size, dp_rank)
trainer.__dict__["_unsloth_tensor_parallel"] = (
True # the generated TRL trainers skip for_inference()
)
_tp_scaler(trainer, dp_size)
trainer.__dict__["save_model"] = types.MethodType(_tp_trainer_save_model, trainer)
- for name in ("evaluate", "predict"):
- trainer.__dict__[name] = _refuse_tp_trainer_call(name)
# Trainer checkpoints: per-rank local shards (instance level: before native FSDP2's class hooks, whose
# full-state gathers do not restore the packed tp part order).
for name, method in (
@@ -2861,15 +3396,33 @@ def _tp_load_optimizer_and_scheduler(self, checkpoint, *args, **kwargs):
return None
-def _refuse_tp_trainer_call(name):
- def refuse(*args, **kwargs):
- raise RuntimeError(
- f"Unsloth: Trainer.{name} is not supported on a tensor parallel model (its gather would count every "
- "tp rank); run the forward in your own loop, or save the adapter and evaluate on one GPU."
- + _env_hint()
- )
+def _dp_gather(data, dp_mesh):
+ """accelerate's gather over the dp replicas only (the tp ranks of a replica hold identical results). dp = 1:
+ the one-process result (accelerate returns its input unchanged on one process)."""
+ if dp_mesh is None or dp_mesh.size() == 1:
+ return data
+ import torch.distributed as dist
+ from accelerate.utils import recursively_apply
- return refuse
+ group, size = dp_mesh.get_group(), dp_mesh.size()
+
+ def one(t):
+ t = t.clone()[None] if t.ndim == 0 else t.contiguous()
+ out = [torch.empty_like(t) for _ in range(size)]
+ dist.all_gather(out, t, group = group)
+ return torch.cat(out, dim = 0)
+
+ return recursively_apply(one, data, error_on_other_type = True)
+
+
+def _dp_gather_object(obj, dp_mesh):
+ if dp_mesh is None or dp_mesh.size() == 1:
+ return obj
+ import torch.distributed as dist
+
+ out = [None] * dp_mesh.size()
+ dist.all_gather_object(out, obj, group = dp_mesh.get_group())
+ return [x for y in out for x in y]
def _tp_trainer_save_model(
@@ -2877,14 +3430,24 @@ def _tp_trainer_save_model(
output_dir = None,
_internal_call = False,
):
- """Trainer.save_model for a tensor parallel model: the 1-GPU PEFT adapter (collective, rank 0 writes)."""
+ """Trainer.save_model for a tensor parallel model: the 1-GPU PEFT adapter, or for full finetuning the whole
+ 1-GPU model (collective, rank 0 writes)."""
output_dir = output_dir if output_dir is not None else self.args.output_dir
- save_tp_adapter(self.model, output_dir)
+ if _has_peft_config(self.model):
+ save_tp_adapter(self.model, output_dir)
+ else:
+ from .tp_replica import tp_collective
+ model = self.model
+ tp_collective(model, "save_model", lambda: model.save_pretrained(output_dir), "all")
processing = getattr(self, "processing_class", None) or getattr(self, "tokenizer", None)
if processing is not None and self.args.should_save:
processing.save_pretrained(output_dir)
+def _has_peft_config(model):
+ return any(getattr(level, "peft_config", None) for level in _tp_chain(model))
+
+
def _tp_scaler(trainer, dp):
"""float16: one GradScaler inf decision for all ranks (each tp rank sees other gradient shards)."""
from .fsdp2_native import _scaler_for_native
@@ -2907,6 +3470,7 @@ def _install_tp_accelerate_hooks():
original_prepare_model = Accelerator.prepare_model
original_data_loader = Accelerator.prepare_data_loader
original_clip = Accelerator.clip_grad_norm_
+ original_gather_for_metrics = Accelerator.gather_for_metrics
def _tp_model(self, model):
if self.__dict__.get(_TP_DATA) is None or not is_tensor_parallel(model):
@@ -2989,6 +3553,42 @@ def _install_tp_accelerate_hooks():
sync["synced"] = True
return tp_clip_grad_norm_(parameters, max_norm)
+ def gather_for_metrics(
+ self,
+ input_data,
+ use_gather_object = False,
+ ):
+ # Gather over dp only: the eval loader is sharded over dp, so remainder truncation is the dp world's.
+ if _TP_GATHER not in self.__dict__:
+ return original_gather_for_metrics(
+ self, input_data, use_gather_object = use_gather_object
+ )
+ from accelerate.utils import recursively_apply
+
+ dp_mesh = self.__dict__[_TP_GATHER]
+ try:
+ recursively_apply(lambda x: x, input_data, error_on_other_type = True)
+ all_tensors = True
+ except TypeError:
+ all_tensors = False
+ use_gather_object = use_gather_object or not all_tensors
+ if use_gather_object:
+ data = _dp_gather_object(input_data, dp_mesh)
+ else:
+ data = _dp_gather(input_data, dp_mesh)
+ try:
+ remainder = self.gradient_state.remainder
+ if not self.gradient_state.end_of_dataloader or remainder <= 0:
+ return data
+
+ def adjust(tensor):
+ return tensor[:remainder]
+
+ return adjust(data) if use_gather_object else recursively_apply(adjust, data)
+ except Exception:
+ return data
+
+ Accelerator.gather_for_metrics = gather_for_metrics
Accelerator.prepare_model = prepare_model
Accelerator.prepare_data_loader = prepare_data_loader
Accelerator.clip_grad_norm_ = clip_grad_norm_
diff --git a/unsloth/models/tp_replica.py b/unsloth/models/tp_replica.py
index 59efac453ed..2b98c9e7d9c 100644
--- a/unsloth/models/tp_replica.py
+++ b/unsloth/models/tp_replica.py
@@ -208,7 +208,8 @@ def _full_params4bit(weight, base, group, device, keep):
blocksize = state.blocksize
out_local, in_local = state.shape
codes = weight.data.reshape(-1).view(torch.uint8).view(out_local, in_local // 2)
- col = style == tpn.COLWISE
+ kv = style == tpn.KV_REPLICATED
+ col = style == tpn.COLWISE or kv
dim = 0 if col else 1
codes = _gather_cat(codes, dim, group, tp)
full_state = getattr(weight, "_unsloth_tp_quant_state", None)
@@ -217,6 +218,12 @@ def _full_params4bit(weight, base, group, device, keep):
absmax = _gather_cat(
state.absmax.float().view(out_local, in_local // blocksize), dim, group, tp
)
+ if kv:
+ # Replicated key / value heads: tp / num_key_value_heads consecutive ranks hold the same head's rows.
+ step = tp // (sum(parts) // out_local)
+ codes = torch.cat(codes.split(out_local)[::step])
+ absmax = None if absmax is None else torch.cat(absmax.split(out_local)[::step])
+ tp = 1
if not keep:
return None
if col:
@@ -398,6 +405,8 @@ def _switch_to_one_gpu(root, replica):
replica.set(p, "forward", p.__dict__["_unsloth_tp_prior_forward"])
if "_unsloth_tp_local" in p.__dict__:
replica.set(p, "_unsloth_tp_local", tpn._MISSING)
+ for block in plan.moe_blocks if plan is not None else ():
+ replica.set(block, "_unsloth_tp_gathered", True) # replicated MoE: no tp average
for level in tpn._tp_chain(root):
forward = level.__dict__.get("forward")
original = getattr(forward, "__wrapped__", None)