Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""apply_tensor_parallel on a gloo CPU world (2 and 4 ranks) vs the same model in one process, fp32."""
import multiprocessing
import os
if os.environ.get("UNSLOTH_TEST_RANKS_ON_GPU") != "1" and (
multiprocessing.parent_process() is not None
or getattr(multiprocessing.current_process(), "_inheriting", False)
):
os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
import sys # noqa: E402
import tempfile # noqa: E402
import pytest # noqa: E402
import torch # noqa: E402
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_tp_native_oracle import ( # noqa: E402
ARCHS,
LLAMA,
PHI3_PACKED_PLAN,
QWEN3,
_batch,
build,
skip_unsupported,
tpn,
)
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Small next to a typical trained-weight gradient (~6e-6 on the tiny llama lm_head), so Adam still moves it.
_ADAM_EPS = 1e-6
def _free_port():
from tests._rendezvous_port import free_port
return free_port()
def _grads(model):
return {
n: tpn._full(p.grad, p).detach().clone()
for n, p in model.named_parameters()
if p.grad is not None
}
def _train(model, ids, steps, clip):
params = [p for p in model.parameters() if p.requires_grad]
opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
losses, norms, grads = [], [], None
for step in range(steps):
out = model(input_ids = ids, labels = ids, use_cache = False)
out.loss.backward()
if step == 0:
grads = _grads(model)
logits = out.logits.detach().clone()
# Replicated factors' Partial grads are summed in backward: grads land in their param's layout.
mismatched = [
n
for n, p in model.named_parameters()
if tpn._is_dtensor(p.grad) and tuple(p.grad.placements) != tuple(p.placements)
]
assert not mismatched, mismatched[:4]
norms.append(float(clip(params, 1.0)))
opt.step()
opt.zero_grad(set_to_none = True)
losses.append(float(out.loss))
return logits, grads, losses, norms
def _clip_fp64(params, max_norm):
"""clip_grad_norm_ in float64: torch's CPU fp32 vector_norm is ~2e-4 off, above the parity tolerance."""
grads = [p.grad for p in params if p.grad is not None]
total = torch.stack([g.double().square().sum() for g in grads]).sum().sqrt().float()
coef = torch.clamp(max_norm / (total + 1e-6), max = 1.0)
for g in grads:
g.mul_(coef)
return total
def _tie_q_norms(model):
# Two regions sharing one trainable per-head norm weight: its tp sum must run once, not once per region.
layers = tpn._unwrap(model).model.layers
layers[1].self_attn.q_norm.weight = layers[0].self_attn.q_norm.weight
def _case_parity(
rank,
world,
tmp,
arch,
overrides = None,
lora_kwargs = None,
save_kwargs = None,
plan = None,
freeze_before_save = False,
tie_norms = False,
fp64_ref_clip = False,
):
from torch.distributed.device_mesh import init_device_mesh
from safetensors.torch import load_file
repo, base_overrides = ARCHS[arch]
overrides = {**base_overrides, **(overrides or {})}
train_norms = arch in ("qwen3", "gemma3")
ref = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))
model = build(repo, overrides, train_norms = train_norms, **(lora_kwargs or {}))
if tie_norms:
_tie_q_norms(ref), _tie_q_norms(model)
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
tpn.apply_tensor_parallel(model, mesh["tp"], plan = plan)
assert tpn.is_tensor_parallel(model)
assert tpn.apply_tensor_parallel(model, mesh["tp"]) is model # idempotent
# The training forward below releases the generate replica, so parity also covers that release.
generated = model.generate(
input_ids = torch.zeros(1, 2, dtype = torch.long), max_new_tokens = 2, pad_token_id = 0
)
assert generated.shape == (1, 4), generated.shape
from test_fsdp2_native_units import _load_native
try:
_load_native()._refuse_tensor_parallel_model(model)
fsdp2_refused = None
except RuntimeError as e:
fsdp2_refused = str(e)
ids = _batch(ref)
ref_out = _train(ref, ids, 3, _clip_fp64 if fp64_ref_clip else torch.nn.utils.clip_grad_norm_)
tp_out = _train(model, ids, 3, tpn.tp_clip_grad_norm_)
if freeze_before_save:
# Frozen adapter (e.g. after training): PEFT still saves bias = "all" biases, trainable or not.
for m in (ref, model):
for p in m.parameters():
p.requires_grad_(False)
ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")
ref.save_pretrained(ref_dir, **(save_kwargs or {}))
tpn.save_tp_adapter(model, tp_dir, **(save_kwargs or {}))
q = tpn._weight(model._unsloth_tp_plan.regions[0].projs[0].module)[0]
return {
"ref": ref_out,
"tp": tp_out,
"saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),
"saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))
if rank == 0
else None,
"local_q": (tuple(q.to_local().shape), tuple(q.shape), q.placements[0].dim),
"fsdp2_refused": fsdp2_refused,
}
def _import_checkout_unsloth():
"""`import unsloth` from this checkout (not the stub tp_native is loaded under); None, or why it failed."""
import pathlib
for name in [n for n in sys.modules if n == "unsloth" or n.startswith("unsloth.")]:
del sys.modules[name]
sys.path.insert(0, REPO)
try:
# conftest's device spoof (spawned ranks do not run it).
import importlib.util
spec = importlib.util.spec_from_file_location(
"_tp_conftest", os.path.join(REPO, "tests", "conftest.py")
)
spec.loader.exec_module(importlib.util.module_from_spec(spec))
import unsloth
except Exception as e: # a CPU host that refuses `import unsloth`
return repr(e)
# By path: compile setup may put other directories in front of sys.path.
root = pathlib.Path(REPO).resolve()
assert root in pathlib.Path(unsloth.__file__).resolve().parents, unsloth.__file__
return None
def _raised(fn):
try:
fn()
except Exception as e:
return f"{type(e).__name__}: {e}"
return None
def _case_guards(rank, world, tmp):
"""Entry points outside the tp regions are refused on every wrapper level."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import transformers
from torch.distributed.device_mesh import init_device_mesh
from unsloth import FastLanguageModel, FastModel
from unsloth.models import fsdp2_native, tp_native, tp_replica
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
args = transformers.TrainingArguments(output_dir = tmp, report_to = [], use_cpu = True)
out = {}
for applied in ("peft", "causal"):
model = build(LLAMA)
levels = {
"peft": model,
"lora": model.base_model,
"causal": model.base_model.model,
"decoder": model.base_model.model.model,
}
tp_native.apply_tensor_parallel(levels[applied], mesh["tp"])
ids = _batch(model)
res = {}
cached = model(input_ids = ids, use_cache = True)
res["forward_without_cache"] = cached.logits is not None
past = cached.past_key_values
step = ids[:, -1:]
for name, level in levels.items():
res[f"is_tp:{name}"] = tp_native.is_tensor_parallel(level)
res[f"fsdp2:{name}"] = _raised(
lambda: fsdp2_native._refuse_tensor_parallel_model(level)
)
res[f"for_inference:{name}"] = _raised(lambda: FastLanguageModel.for_inference(level))
res[f"replica_released:{name}"] = tp_replica.release_replica(level)
res[f"FastModel.for_inference:{name}"] = _raised(lambda: FastModel.for_inference(level))
tp_replica.release_replica(level)
if name != "decoder":
res[f"generate:{name}"] = _raised(
lambda: level.generate(input_ids = ids, max_new_tokens = 2, pad_token_id = 0)
)
tp_replica.release_replica(level)
res[f"trainer:{name}"] = _raised(
lambda: transformers.Trainer(model = level, args = args)
)
res[f"model_init:{name}"] = _raised(
lambda: transformers.Trainer(model_init = lambda: level, args = args)
)
res[f"cached:{name}"] = _raised(
lambda: level(input_ids = step, past_key_values = past, use_cache = True)
)
res["generate:get_base_model"] = _raised(
lambda: model.get_base_model().generate(input_ids = ids)
)
params = [p for p in model.parameters() if p.requires_grad]
model(input_ids = ids, labels = ids, use_cache = False).loss.backward()
norm = tp_native.tp_clip_grad_norm_(params, 1e9)
res["clip_mesh_equal"] = float(norm) == float(
tp_native.tp_clip_grad_norm_(params, 1e9, mesh = mesh["tp"])
)
out[applied] = res
return out
def _case_fused(rank, world, tmp):
# The real package from this checkout, not the stub the oracle module loads tp_native under.
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
from unsloth.kernels import fast_lora
from unsloth.models import tp_native
from torch.distributed.device_mesh import init_device_mesh
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
calls = []
apply = fast_lora._apply
fast_lora._apply = lambda function, *args: calls.append(function.__name__) or apply(
function, *args
)
out = {}
for fused in (False, True):
calls.clear()
model = build(QWEN3, {"num_attention_heads": 8, "num_key_value_heads": 4})
tp_native.apply_tensor_parallel(model, mesh["tp"])
attn = tp_native._unwrap(model).model.layers[0].self_attn
g = torch.Generator().manual_seed(5)
X = torch.randn(
2, 5, attn.q_proj.base_layer.weight.shape[1], generator = g, requires_grad = True
)
Xr = tp_native.copy_to_tp(X, attn._unsloth_tp_group)
if fused:
Q, K, V = fast_lora.apply_lora_qkv(attn, Xr, inplace = False)
else:
Q, K, V = attn.q_proj(Xr), attn.k_proj(Xr), attn.v_proj(Xr)
Y = torch.cat([Q, K, V], -1)
O_in = Y[..., : Q.shape[-1]].contiguous()
O = fast_lora.apply_lora_o(attn, O_in) if fused else attn.o_proj(O_in)
O = tp_native.reduce_from_tp(O, attn._unsloth_tp_group)
(O.square().sum() + Y.square().sum()).backward()
out[fused] = {
"O": O.detach(),
"dX": X.grad.detach(),
"grads": _grads(attn),
"calls": list(calls),
}
return out
def _quantize_layers(
model,
nested,
blocksize = 64,
):
"""Every Linear in the decoder layers -> a bitsandbytes NF4 Linear4bit (the same bytes in every process)."""
import bitsandbytes as bnb
for name, module in list(model.named_modules()):
for child_name, child in list(module.named_children()):
if type(child) is not torch.nn.Linear or ".layers." not in f".{name}.":
continue
q = bnb.nn.Linear4bit(
child.in_features,
child.out_features,
bias = child.bias is not None,
compute_dtype = torch.float32,
quant_type = "nf4",
compress_statistics = nested,
)
packed, state = bnb.functional.quantize_4bit(
child.weight.data, blocksize = blocksize, quant_type = "nf4", compress_statistics = nested
)
q.weight = bnb.nn.Params4bit(
packed,
requires_grad = False,
quant_state = state,
blocksize = blocksize,
compress_statistics = nested,
quant_type = "nf4",
module = q,
bnb_quantized = True,
)
if child.bias is not None:
q.bias = child.bias
setattr(module, child_name, q)
return model
QLORA_SIZES = {
"hidden_size": 128,
"intermediate_size": 256,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"head_dim": 32,
}
def _qlora_model(
nested,
seed = 0,
blocksize = 64,
):
from peft import LoraConfig, get_peft_model
model = _quantize_layers(build(LLAMA, QLORA_SIZES, lora = False, seed = seed), nested, blocksize)
torch.manual_seed(seed + 1)
config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
# train(): eval-mode CPU Linear4bit repacks for a CPU gemm that rounds differently.
return get_peft_model(model, config).train()
def _case_qlora(rank, world, tmp, nested):
import bitsandbytes as bnb
from torch.distributed.device_mesh import init_device_mesh
ref, model = _qlora_model(nested), _qlora_model(nested)
full = {
n: bnb.functional.dequantize_4bit(m.weight.data, m.weight.quant_state)
for n, m in model.named_modules()
if type(m).__name__ == "Linear4bit"
}
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
tpn.apply_tensor_parallel(model, mesh["tp"])
shard_err = 0.0
for region in model._unsloth_tp_plan.regions:
for proj in region.projs:
base = proj.module.base_layer
name = next(n for n, m in model.named_modules() if m is base)
local = bnb.functional.dequantize_4bit(base.weight.data, base.weight.quant_state)
col = proj.style == tpn.COLWISE
want = tpn.shard_slice(
full[name],
0 if col else 1,
proj.parts if col else (full[name].shape[1],),
rank,
world,
)
assert local.shape == want.shape, (name, local.shape, want.shape)
shard_err = max(shard_err, float((local - want).abs().max()))
ids = _batch(ref)
return {
"ref": _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_),
"tp": _train(model, ids, 3, tpn.tp_clip_grad_norm_),
"shard_err": shard_err,
}
class _DropoutMLP(torch.nn.Module):
"""LlamaMLP with dropout between the column (gate / up) and row (down) projections."""
def forward(self, x):
return self.down_proj(self.drop(self.act_fn(self.gate_proj(x)) * self.up_proj(x)))
def _add_mlp_dropout(model, p = 0.25):
for module in list(model.modules()):
if type(module).__name__ == "LlamaMLP":
module.add_module("drop", torch.nn.Dropout(p))
module.__class__ = type("LlamaMLP", (_DropoutMLP, type(module)), {})
return model
def _case_mlp_dropout(rank, world, tmp, quantized):
from torch.distributed.device_mesh import init_device_mesh
def make():
if quantized:
return _add_mlp_dropout(_qlora_model(False))
return _add_mlp_dropout(build(LLAMA, QLORA_SIZES).train())
ref, model = make(), make()
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
tpn.apply_tensor_parallel(model, mesh["tp"])
widths = [
r.module._unsloth_tp_local_width for r in model._unsloth_tp_plan.regions if r.kind == "mlp"
]
ids = _batch(ref)
out = {}
for name, m, clip in (
("ref", ref, torch.nn.utils.clip_grad_norm_),
("tp", model, tpn.tp_clip_grad_norm_),
):
torch.manual_seed(1234) # the same dropout RNG state on every rank and in the reference
out[name] = _train(m, ids, 2, clip)
out["widths"] = widths
return out
def _case_qlora_misaligned(rank, world, tmp):
from torch.distributed.device_mesh import init_device_mesh
# blocksize 128: o_proj / down_proj row shards (in_features / 2 = 64) would split blocks.
model = _qlora_model(False, blocksize = 128)
before = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
with pytest.raises(ValueError, match = "row sharding would split blocks"):
tpn.apply_tensor_parallel(model, mesh["tp"])
after = {n: (type(p), p.data_ptr()) for n, p in model.named_parameters()}
return {"unchanged": before == after, "flag": tpn.is_tensor_parallel(model)}
def _case_save_subgroup(rank, world, tmp):
from torch.distributed.device_mesh import init_device_mesh
# tp = 1 inside a 2-rank world (data parallel 2): the world-scoped writer / barrier would not match.
mesh = init_device_mesh("cpu", (world, 1), mesh_dim_names = ("dp", "tp"))
model = tpn.apply_tensor_parallel(build(LLAMA), mesh["tp"])
return {"error": _raised(lambda: tpn.save_tp_adapter(model, os.path.join(tmp, "out")))}
def _save_checkpoint(
arch,
directory,
overrides = None,
):
"""A tiny random model saved as several safetensors files (index + shards), like a Hub checkpoint."""
repo, base_overrides = ARCHS[arch]
model = build(repo, {**base_overrides, **(overrides or {})}, lora = False)
model.generation_config.pad_token_id = None # tiny repos ship -1, which save_pretrained rejects
model.save_pretrained(directory, max_shard_size = "1MB", safe_serialization = True)
return directory
def _case_sharded_load(
rank,
world,
tmp,
arch,
ckpt,
lora_kwargs = None,
):
"""sharded_loading + get_peft_model + apply_tensor_parallel vs a full load (1 process, and v1 TP)."""
from peft import LoraConfig, get_peft_model
from safetensors.torch import load_file
from torch.distributed.device_mesh import init_device_mesh
from transformers import AutoModelForCausalLM
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
def peft(model):
torch.manual_seed(1)
config = dict(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
config.update(lora_kwargs or {})
return get_peft_model(model, LoraConfig(**config)).eval()
ref = peft(AutoModelForCausalLM.from_pretrained(ckpt, **kw))
full_tp = peft(AutoModelForCausalLM.from_pretrained(ckpt, **kw))
tpn.apply_tensor_parallel(full_tp, mesh["tp"])
model = tpn.load_tensor_parallel(
ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, **kw
)
stats = dict(model._unsloth_tp_load_stats)
ids = _batch(ref)
unapplied = _raised(lambda: model(input_ids = ids))
model = peft(model)
tpn.apply_tensor_parallel(model, mesh["tp"])
# Every local tensor equals the v1 shard bitwise (weights, biases, LoRA, buffers).
a, b = full_tp.state_dict(), model.state_dict()
local = lambda t: t.to_local() if tpn._is_dtensor(t) else t # noqa: E731
diffs = sorted(set(a) ^ set(b)) + [
k for k in a if k in b and not torch.equal(local(a[k]), local(b[k]))
]
full_local = sum(local(p).numel() for p in full_tp.parameters())
sharded_local = sum(local(p).numel() for p in model.parameters())
from test_fsdp2_native_units import _load_native
try:
_load_native()._refuse_tensor_parallel_model(model)
fsdp2_refused = None
except RuntimeError as e:
fsdp2_refused = str(e)
ref_out = _train(ref, ids, 3, torch.nn.utils.clip_grad_norm_)
v1_out = _train(full_tp, ids, 3, tpn.tp_clip_grad_norm_)
tp_out = _train(model, ids, 3, tpn.tp_clip_grad_norm_)
ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")
ref.save_pretrained(ref_dir)
tpn.save_tp_adapter(model, tp_dir)
q = tpn._weight(model._unsloth_tp_plan.regions[0].projs[0].module)[0]
return {
"ref": ref_out,
"tp": tp_out,
"v1": v1_out,
"diffs": diffs,
"stats": stats,
"unapplied": unapplied,
"local_numel": (full_local, sharded_local),
"saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),
"saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))
if rank == 0
else None,
"local_q": (tuple(q.to_local().shape), tuple(q.shape), q.placements[0].dim),
"fsdp2_refused": fsdp2_refused,
}
def _case_tp_fsdp2(
rank,
world,
tmp,
arch,
fit_layout = True,
lora_kwargs = None,
):
"""dp 2 x tp 2: apply_tensor_parallel over mesh['tp'], native FSDP2 shard_model over mesh['dp'], each dp
replica on its own half of the batch, vs one process on the whole batch."""
import torch.distributed as dist
from safetensors.torch import load_file
from torch.distributed.device_mesh import init_device_mesh
from test_fsdp2_native_units import _load_native
native = _load_native()
repo, overrides = ARCHS[arch]
mesh = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))
dp = mesh["dp"].get_local_rank()
ref = build(repo, overrides, **(lora_kwargs or {}))
model = build(repo, overrides, **(lora_kwargs or {}))
ids = _batch(ref, bsz = 4)
mine = ids[2 * dp : 2 * dp + 2]
tpn.apply_tensor_parallel(model, mesh["tp"])
if not native._supports_ignored_params(native._fsdp_api()[2]):
# torch < 2.7: no fully_shard(ignored_params) to keep the frozen tp shards out of FSDP2.
config = native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
return {"unsupported": _raised(lambda: native.shard_model(model, config))}
refusals = {
"no_mesh": _raised(
lambda: native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32))
),
"tp_mesh": _raised(
lambda: native.shard_model(
model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["tp"])
)
),
"shard_frozen": _raised(
lambda: native.shard_model(
model,
native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], shard_frozen = True),
)
),
}
native.shard_model(
model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout)
)
layouts = tpn._param_layouts(model)
params = [p for p in model.parameters() if p.requires_grad]
ref_params = [p for p in ref.parameters() if p.requires_grad]
# Adam's g / (sqrt(v) + eps) has slope lr / eps at g = 0. A full lm_head copy (modules_to_save) holds
# hundreds of entries with |g| ~ 1e-9, where the default eps = 1e-8 turns ~1e-12 of reduction-order
# noise into ~5e-6 of weight, half the saved-weight tolerance, and over it on some CI CPUs.
opt = torch.optim.AdamW(params, lr = 1e-2, eps = _ADAM_EPS, foreach = False)
ref_opt = torch.optim.AdamW(ref_params, lr = 1e-2, eps = _ADAM_EPS, foreach = False)
losses, ref_losses, norms, native_norms, ref_norms = [], [], [], [], []
for step in range(3):
out = model(input_ids = mine, labels = mine, use_cache = False)
out.loss.backward()
r = ref(input_ids = ids, labels = ids, use_cache = False)
r.loss.backward()
if step == 0:
grads = {
n: tpn._full(p.grad, layout = layouts.get(id(p))).detach().clone()
for n, p in model.named_parameters()
if p.grad is not None
}
ref_grads = {
n: p.grad.detach().clone() for n, p in ref.named_parameters() if p.grad is not None
}
mismatched = [
n
for n, p in model.named_parameters()
if p.grad is not None and tuple(p.grad.placements) != tuple(p.placements)
]
native_norms.append(float(native.native_clip_grad_norm_(params, 1e9)))
norms.append(float(tpn.tp_clip_grad_norm_(params, 1.0)))
ref_norms.append(float(_clip_fp64(ref_params, 1.0)))
opt.step(), opt.zero_grad(set_to_none = True)
ref_opt.step(), ref_opt.zero_grad(set_to_none = True)
loss = out.loss.detach().clone()
dist.all_reduce(loss) # mean over the 4 ranks = mean over the 2 dp halves (tp ranks agree)
losses.append(float(loss) / world)
ref_losses.append(float(r.loss))
ref_dir, tp_dir = os.path.join(tmp, f"ref{rank}"), os.path.join(tmp, "tp")
ref.save_pretrained(ref_dir)
tpn.save_tp_adapter(model, tp_dir)
dist.barrier()
return {
"grads": grads,
"ref_grads": ref_grads,
"mismatched": mismatched,
"losses": losses,
"ref_losses": ref_losses,
"norms": norms,
"native_norms": native_norms,
"ref_norms": ref_norms,
"refusals": refusals,
"placements": {
n: tuple(p.placements) for n, p in model.named_parameters() if p.requires_grad
},
"saved_ref": load_file(os.path.join(ref_dir, "adapter_model.safetensors")),
"saved_tp": load_file(os.path.join(tp_dir, "adapter_model.safetensors"))
if rank == 0
else None,
}
def _case_tp_fsdp2_trainable_norms(rank, world, tmp):
from torch.distributed.device_mesh import init_device_mesh
from test_fsdp2_native_units import _load_native
native = _load_native()
mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("dp", "tp"))
model = build(QWEN3, train_norms = True)
tpn.apply_tensor_parallel(model, mesh["tp"])
return {
"error": _raised(
lambda: native.shard_model(
model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
)
)
}
def _case_tp_fsdp2_writer(
rank,
world,
tmp,
head_only = False,
compiled = False,
):
"""dp 2 x tp 2: the collective saves name one writer for the model, not one per tp group."""
import importlib
from torch.distributed.device_mesh import init_device_mesh
from test_fsdp2_native_units import _load_native
native = _load_native()
save = importlib.import_module("unsloth.models.fsdp2_save")
mesh = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))
model = build(*ARCHS["llama"], **({"target_modules": ["lm_head"]} if head_only else {}))
expected = {name: p.detach().clone() for name, p in model.named_parameters()}
tpn.apply_tensor_parallel(model, mesh["tp"])
if not native._supports_ignored_params(native._fsdp_api()[2]):
return {"unsupported": "torch 2.7+"}
native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))
unwrapped_model = model
if compiled:
model = torch.compile(model, backend = "eager")
res = {"writer_rank": save.mesh_writer_rank(model)}
with save.fsdp2_gathered(model) as writer:
res["writer"] = writer
if writer:
for name, param in unwrapped_model.named_parameters():
torch.testing.assert_close(param, expected[name])
state = native.native_full_state_dict(model)
res["full_keys"] = len(state)
if rank == 0:
for name, param in expected.items():
torch.testing.assert_close(state[name], param)
return res
def _case_tp_fsdp2_writer_pipeline_slice(rank, world, tmp):
import importlib
from torch.distributed.device_mesh import init_device_mesh
from test_fsdp2_native_units import _load_native
from torch.distributed.tensor.parallel import ColwiseParallel, parallelize_module
native = _load_native()
save = importlib.import_module("unsloth.models.fsdp2_save")
mesh = init_device_mesh("cpu", (2, 1, 2), mesh_dim_names = ("pp", "dp", "tp"))
# One pipeline stage per pp rank: a tp layer sharded by FSDP2 over the stage's dp slice.
model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4))
model[0].weight.requires_grad_(False)
parallelize_module(model[0], mesh["tp"], ColwiseParallel())
parallelize_module(model[1], mesh["tp"], ColwiseParallel())
fully_shard = native._fsdp_api()[2]
if not native._supports_ignored_params(fully_shard):
return {"unsupported": "torch 2.7+"}
fully_shard(model, mesh = mesh["dp"], ignored_params = set(model[1].parameters()))
return {"writer_rank": save.mesh_writer_rank(model)}
class _Rows(torch.utils.data.Dataset):
def __init__(self, ids):
self.ids = ids
def __len__(self):
return len(self.ids)
def __getitem__(self, i):
return {"input_ids": self.ids[i], "labels": self.ids[i]}
def _trainer_args(tmp, bs, **overrides):
import transformers
kwargs = dict(
output_dir = tmp,
per_device_train_batch_size = bs,
gradient_accumulation_steps = 2,
max_steps = 3,
learning_rate = 1e-2,
optim = "adamw_torch",
lr_scheduler_type = "constant",
warmup_steps = 0,
weight_decay = 0.0,
max_grad_norm = 1.0,
logging_steps = 1,
save_strategy = "no",
report_to = [],
use_cpu = True,
seed = 0,
dataloader_drop_last = True,
remove_unused_columns = False,
disable_tqdm = True,
)
kwargs.update(overrides)
return transformers.TrainingArguments(**kwargs)
def _case_trainer(
rank,
world,
tmp,
dp,
tp,
lora_kwargs = None,
):
"""The HF Trainer on a tp (dp = 1) or dp x tp (shard_model over mesh['dp']) model; tp = 1: the 1-process
reference with a dp times larger batch. Sequential batches, so both read the same rows per step."""
import torch.distributed as dist
import transformers
from safetensors.torch import load_file
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
from torch.distributed.device_mesh import init_device_mesh
from unsloth.models import fsdp2_native, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
seen = []
class SequentialTrainer(transformers.Trainer):
def _get_train_sampler(self, *args, **kwargs):
return torch.utils.data.SequentialSampler(self.train_dataset)
def compute_loss(self, model, inputs, *args, **kwargs):
seen.append(inputs["input_ids"].clone())
return super().compute_loss(model, inputs, *args, **kwargs)
model = build(LLAMA, **(lora_kwargs or {}))
data = _Rows(list(_batch(model, bsz = 12 * dp, seq = 9)))
refusals = {}
if tp > 1:
mesh = init_device_mesh("cpu", (dp, tp), mesh_dim_names = ("dp", "tp"))
tp_native.apply_tensor_parallel(model, mesh["tp"])
if dp > 1:
refusals["no_shard_model"] = _raised(
lambda: SequentialTrainer(
model = model, args = _trainer_args(tmp, 2), train_dataset = data
)
)
fsdp2_native.shard_model(
model, fsdp2_native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
)
for key, overrides in {
"checkpoints": {"save_strategy": "steps", "save_steps": 1},
"eval": {
"eval_strategy": "steps",
"eval_steps": 1,
}, # evaluates over dp (test_tp_eval_gloo.py)
"optim_8bit": {"optim": "adamw_bnb_8bit"},
}.items():
refusals[key] = _raised(
lambda: SequentialTrainer(
model = model,
args = _trainer_args(tmp, 2, **overrides),
train_dataset = data,
eval_dataset = data,
)
)
if tp > 1 and lora_kwargs:
# A prebuilt scheduler already recorded the mixed groups: refused, not split under it.
opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)
sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda step: 1.0)
refusals["prebuilt_scheduler"] = _raised(
lambda: SequentialTrainer(
model = model,
args = _trainer_args(tmp, 2),
train_dataset = data,
optimizers = (opt, sched),
).train()
)
trainer = SequentialTrainer(
model = model, args = _trainer_args(tmp, 2 * dp if tp == 1 else 2), train_dataset = data
)
trainer.train()
logs = [h for h in trainer.state.log_history if "loss" in h]
out_dir = os.path.join(tmp, f"adapter_{world}_{rank}")
trainer.save_model(out_dir)
if dist.is_initialized():
dist.barrier()
state = (
tp_native.tp_full_state_dict(trainer.model, adapters_only = True)
if tp > 1
else {k: v.detach().clone() for k, v in trainer.model.state_dict().items() if "lora_" in k}
)
wrapped = type(trainer.model_wrapped).__name__
return {
"losses": [h["loss"] for h in logs],
"grad_norms": [h.get("grad_norm") for h in logs],
"seen": [x.sum().item() for x in seen],
"state": {k: v for k, v in state.items() if "lora_" in k},
"saved": load_file(os.path.join(out_dir, "adapter_model.safetensors"))
if rank == 0 or tp == 1
else None,
"wrapped": wrapped,
"refusals": refusals,
"mixed_groups": fsdp2_native._mixed_param_groups(trainer.optimizer),
}
def _case_sharded_lora_init(rank, world, tmp, ckpt):
"""Weight-dependent LoRA init (PiSSA) on sharded_loading slices is refused before any mutation;
orthogonal init only reads the full in / out features and the dtype, so it is allowed like gaussian."""
from peft import LoraConfig, get_peft_model
from torch.distributed.device_mesh import init_device_mesh
from transformers import AutoModelForCausalLM
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
out = {}
for init in ("pissa", "gaussian", "orthogonal"):
model = tpn.load_tensor_parallel(
ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, torch_dtype = torch.float32
)
config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = init)
model = get_peft_model(model, config)
out[init] = _raised(lambda: tpn.apply_tensor_parallel(model, mesh["tp"]))
out[init + "_tp"] = tpn.is_tensor_parallel(model)
return out
def _case_sharded_random_lora_init(rank, world, tmp, ckpt, init, sync_trainable, seed_per_rank):
"""A random LoRA init on sharded_loading slices vs the same init on a full load in one process. Each rank
seeds get_peft_model differently when seed_per_rank, so only the tp rank 0 broadcast can make them agree."""
from peft import LoraConfig, get_peft_model
from torch.distributed.device_mesh import init_device_mesh
from transformers import AutoModelForCausalLM
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
def peft(model, seed):
torch.manual_seed(seed)
config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = init)
return get_peft_model(model, config)
def lora(model):
return {
n: tpn._full(p.detach(), p).clone() for n, p in model.named_parameters() if "lora_" in n
}
ref = peft(AutoModelForCausalLM.from_pretrained(ckpt, **kw), 7)
model = tpn.load_tensor_parallel(
ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, **kw
)
model = peft(model, 7 + rank if seed_per_rank else 7)
error = _raised(
lambda: tpn.apply_tensor_parallel(
model, mesh["tp"], config = tpn.TPConfig(sync_trainable = sync_trainable)
)
)
if error is not None:
return {"error": error}
# Replicated factors hold this rank's own copy: the raw local tensors show any per-rank divergence.
local = {
n: (p.to_local() if tpn._is_dtensor(p) else p).detach().clone()
for n, p in model.named_parameters()
if "lora_" in n
}
init_ref, init_tp = lora(ref), lora(model)
ids = _batch(ref)
ref_out = _train(ref, ids, 1, torch.nn.utils.clip_grad_norm_)
tp_out = _train(model, ids, 1, tpn.tp_clip_grad_norm_)
return {
"error": None,
"local": local,
"init_ref": init_ref,
"init_tp": init_tp,
"ref": ref_out,
"tp": tp_out,
"step_ref": lora(ref),
"step_tp": lora(model),
}
def _case_sampler(rank, world, tmp, dp, strategy):
"""Rows each rank's dataloader yields under a length-aware sampler, after a rank-local RNG draw."""
import transformers
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
fields = transformers.TrainingArguments.__dataclass_fields__
if strategy == "batch_rebalance" and "train_sampling_strategy" not in fields:
return {"skip": "this transformers has no train_sampling_strategy = 'batch_rebalance'"}
from torch.distributed.device_mesh import init_device_mesh
from unsloth.models import fsdp2_native, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
tp = world // dp
model = build(LLAMA)
mesh = init_device_mesh("cpu", (dp, tp), mesh_dim_names = ("dp", "tp"))
tp_native.apply_tensor_parallel(model, mesh["tp"])
if dp > 1:
try:
fsdp2_native.shard_model(
model, fsdp2_native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
)
except Exception as e:
if "torch < 2.7" in str(e) or "torch 2.7+" in str(e):
return {"skip": "torch < 2.7: tensor parallel x FSDP2 is refused"}
raise
g = torch.Generator().manual_seed(1)
rows = []
for i in range(48):
ids = torch.randint(3, 100, (4 + int(torch.randint(0, 9, (1,), generator = g)),), generator = g)
ids[0] = 3 + i
rows.append(ids)
def collate(features):
n = max(len(f["input_ids"]) for f in features)
ids = torch.zeros(len(features), n, dtype = torch.long)
for i, f in enumerate(features):
ids[i, : len(f["input_ids"])] = f["input_ids"]
return {"input_ids": ids, "labels": ids.clone()}
if strategy == "group_by_length" and "train_sampling_strategy" not in fields:
overrides = {"group_by_length": True}
else:
overrides = {"train_sampling_strategy": strategy}
trainer = transformers.Trainer(
model = model,
args = _trainer_args(tmp, 2, dataloader_drop_last = False, **overrides),
train_dataset = _Rows(rows),
data_collator = collate,
)
torch.rand(1 + 5 * rank)
seen = []
for step, batch in enumerate(trainer.get_train_dataloader()):
if step == 6:
break
seen.append([int(r[0]) - 3 for r in batch["input_ids"]])
return {"seen": seen}
def _case_sampler_resume(rank, world, tmp, dp, strategy):
"""Rows per rank for an uninterrupted run vs resumes at epoch boundaries and mid epoch."""
import transformers
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
fields = transformers.TrainingArguments.__dataclass_fields__
if strategy == "batch_rebalance" and "train_sampling_strategy" not in fields:
return {"skip": "this transformers has no train_sampling_strategy = 'batch_rebalance'"}
from torch.distributed.device_mesh import init_device_mesh
from unsloth.models import fsdp2_native, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
tp = world // dp
mesh = init_device_mesh("cpu", (dp, tp), mesh_dim_names = ("dp", "tp"))
g = torch.Generator().manual_seed(1)
rows = []
for i in range(12 * dp): # 3 optimizer steps per epoch on every layout
ids = torch.randint(3, 100, (4 + int(torch.randint(0, 9, (1,), generator = g)),), generator = g)
ids[0] = 3 + i
rows.append(ids)
def collate(features):
n = max(len(f["input_ids"]) for f in features)
ids = torch.zeros(len(features), n, dtype = torch.long)
for i, f in enumerate(features):
ids[i, : len(f["input_ids"])] = f["input_ids"]
return {"input_ids": ids, "labels": ids.clone()}
if strategy == "group_by_length" and "train_sampling_strategy" not in fields:
overrides = {"group_by_length": True}
else:
overrides = {"train_sampling_strategy": strategy}
ckpt = os.path.join(tmp, "ckpt")
def run(resume = None):
model = build(LLAMA)
tp_native.apply_tensor_parallel(model, mesh["tp"])
if dp > 1:
try:
fsdp2_native.shard_model(
model, fsdp2_native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
)
except Exception as e:
if "torch < 2.7" in str(e) or "torch 2.7+" in str(e):
return None
raise
args = _trainer_args(
ckpt,
2,
max_steps = 8,
dataloader_drop_last = False,
save_strategy = "steps",
save_steps = 1,
logging_steps = 100,
**overrides,
)
trainer = transformers.Trainer(
model = model, args = args, train_dataset = _Rows(rows), data_collator = collate
)
seen = []
training_step = trainer.training_step
def record(model, inputs, *args, **kwargs):
seen.append([int(r[0]) - 3 for r in inputs["input_ids"]])
return training_step(model, inputs, *args, **kwargs)
trainer.training_step = record
torch.rand(1 + 5 * rank) # a rank-local draw on the global RNG
trainer.train(resume_from_checkpoint = resume)
return seen
full = run()
if full is None:
return {"skip": "torch < 2.7: tensor parallel x FSDP2 is refused"}
# 3 steps per epoch, 2 micro-batches per step: 3 / 6 are epoch boundaries, 4 / 7 mid epoch 1 / 2.
resumed = {k: run(os.path.join(ckpt, f"checkpoint-{k}")) for k in (3, 4, 6, 7)}
return {"full": full, "resumed": resumed}
def _case_teardown_abort(rank, world, tmp):
"""Stands in for a C++ thread that aborts the interpreter's teardown after the work is done."""
import atexit
atexit.register(os.abort)
return {"rank": rank}
CASES = {
"parity": _case_parity,
"save_subgroup": _case_save_subgroup,
"fused": _case_fused,
"guards": _case_guards,
"qlora": _case_qlora,
"qlora_misaligned": _case_qlora_misaligned,
"mlp_dropout": _case_mlp_dropout,
"sharded_load": _case_sharded_load,
"sharded_lora_init": _case_sharded_lora_init,
"tp_fsdp2": _case_tp_fsdp2,
"tp_fsdp2_trainable_norms": _case_tp_fsdp2_trainable_norms,
"tp_fsdp2_one_writer": _case_tp_fsdp2_writer,
"tp_fsdp2_writer_pipeline_slice": _case_tp_fsdp2_writer_pipeline_slice,
"trainer": _case_trainer,
"sharded_random_lora_init": _case_sharded_random_lora_init,
"sampler": _case_sampler,
"sampler_resume": _case_sampler_resume,
"teardown_abort": _case_teardown_abort,
}
def _worker(rank, world, port, case, tmp, kwargs):
os.environ.update(
MASTER_ADDR = "127.0.0.1",
MASTER_PORT = str(port),
RANK = str(rank),
WORLD_SIZE = str(world),
LOCAL_RANK = str(rank),
LOCAL_WORLD_SIZE = str(world),
)
import datetime
import torch.distributed as dist
torch.set_num_threads(2)
dist.init_process_group(
"gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
)
finished = False
try:
res = CASES[case](rank, world, tmp, **kwargs)
torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
finished = True
finally:
dist.destroy_process_group()
if finished:
# Result saved, group torn down. Interpreter teardown can still hit std::terminate on a
# joinable C++ thread ("terminate called without an active exception", exit -6), which
# failed ranks that had finished; skip teardown so only the case itself decides.
sys.stdout.flush()
sys.stderr.flush()
os._exit(0)
def _spawn(
case,
world = 2,
**kwargs,
):
ctx = multiprocessing.get_context("spawn")
port = _free_port()
with tempfile.TemporaryDirectory() as tmp:
procs = [
ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
for r in range(world)
]
for p in procs:
p.start()
for p in procs:
p.join(600)
codes = [p.exitcode for p in procs]
assert codes == [0] * world, f"rank exit codes {codes}"
return [
torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
for r in range(world)
]
def _check_parity(results, world):
for rank, res in enumerate(results):
ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
logits, grads, losses, norms = res["tp"]
torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
assert set(grads) == set(ref_grads), sorted(set(grads) ^ set(ref_grads))[:6]
for name in ref_grads:
torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6), (rank, losses, ref_losses)
assert norms == pytest.approx(ref_norms, rel = 1e-5), (rank, norms, ref_norms)
assert "does not compose with Unsloth tensor parallelism" in (res["fsdp2_refused"] or "")
local, full, dim = res["local_q"]
assert local[dim] * world == full[dim] and local[1 - dim] == full[1 - dim]
saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]
assert set(saved_tp) == set(saved_ref)
for key in saved_ref:
assert saved_tp[key].shape == saved_ref[key].shape, key
torch.testing.assert_close(saved_tp[key], saved_ref[key], rtol = 1e-4, atol = 1e-5, msg = key)
@pytest.mark.parametrize("arch", ["llama", "llama_bias", "qwen3", "phi3", "stablelm", "gemma2"])
def test_tp2_matches_one_process(arch):
skip_unsupported(arch)
_check_parity(_spawn("parity", 2, arch = arch), 2)
def test_tp2_refused_4bit_model_is_left_untouched():
for res in _spawn("qlora_misaligned", 2):
assert res["unchanged"] and not res["flag"]
def test_tp2_trainable_biases_match_one_process():
# bias = "all": the row bias is added after the all-reduce, so its gradient is complete on every rank.
results = _spawn("parity", 2, arch = "llama_bias", lora_kwargs = {"bias": "all"})
_check_parity(results, 2)
assert any("bias" in key for key in results[0]["saved_tp"])
def test_tp2_adapter_save_keeps_requested_embeddings():
results = _spawn("parity", 2, arch = "llama", save_kwargs = {"save_embedding_layers": True})
_check_parity(results, 2)
assert any("embed_tokens" in key for key in results[0]["saved_tp"])
@pytest.mark.parametrize("nested", [False, True])
def test_tp2_qlora_matches_one_process(nested):
results = _spawn("qlora", 2, nested = nested)
for res in results:
ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
logits, grads, losses, norms = res["tp"]
assert res["shard_err"] == 0.0
torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
assert set(grads) == set(ref_grads) and grads
for name in ref_grads:
torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)
assert norms == pytest.approx(ref_norms, rel = 1e-5)
@pytest.mark.parametrize("quantized", [False, True], ids = ["bf16_weights", "4bit_weights"])
def test_tp2_mlp_dropout_between_column_and_row_matches_one_process(quantized):
# The row projection's local in_features is intermediate_size / tp, also once a 4-bit weight is sharded.
results = _spawn("mlp_dropout", 2, quantized = quantized)
for res in results:
assert res["widths"] and set(res["widths"]) == {QLORA_SIZES["intermediate_size"] // 2}, res[
"widths"
]
ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
logits, grads, losses, norms = res["tp"]
torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
assert set(grads) == set(ref_grads) and grads
for name in ref_grads:
torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
assert losses == pytest.approx(ref_losses, rel = 1e-5, abs = 1e-6)
assert norms == pytest.approx(ref_norms, rel = 1e-5)
def test_tp4_gqa_matches_one_process():
overrides = {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 64}
_check_parity(_spawn("parity", 4, arch = "qwen3", overrides = overrides), 4)
def test_fused_lora_functions_read_local_shards():
results = _spawn("fused", 2)
if "skip" in results[0]:
pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
for res in results:
plain, fused = res[False], res[True]
assert plain["calls"] == [] and fused["calls"] == ["LoRA_QKV", "LoRA_W"], (
plain["calls"],
fused["calls"],
)
for key in ("O", "dX"):
# fp32 sums err by an ulp of their largest term, so a small element that cancels out of large ones
# (dX here peaks near 1e3, where one ulp is 1.2e-4) needs an atol at the tensor's scale.
atol = max(1e-5, 1e-6 * plain[key].abs().max().item())
torch.testing.assert_close(
fused[key], plain[key], rtol = 1e-5, atol = atol, msg = lambda m, key = key: f"{key}: {m}"
)
assert set(fused["grads"]) == set(plain["grads"]) and plain["grads"]
for name in plain["grads"]:
# The fused order sums in a different order than PEFT, so as for O / dX an element that cancels out of
# large terms needs an atol at the tensor's scale; a wrong shard would be off by O(1).
ref = plain["grads"][name]
torch.testing.assert_close(
fused["grads"][name],
ref,
rtol = 1e-4,
atol = max(1e-6, 1e-5 * ref.abs().max().item()),
msg = lambda m, name = name: f"{name}: {m}",
)
def test_apply_refuses_a_2d_mesh():
model = build(LLAMA)
class _Mesh:
ndim = 2
with pytest.raises(ValueError, match = "1-D tp mesh"):
tpn.apply_tensor_parallel(model, _Mesh())
def test_tp2_phi3_fused_qkv_under_a_packed_plan_matches_one_process():
# The tiny Phi3 is GQA (4 heads, 2 kv heads): qkv_proj parts are (q, kv, kv), not two packed halves.
_check_parity(_spawn("parity", 2, arch = "phi3", plan = PHI3_PACKED_PLAN), 2)
@pytest.mark.parametrize(
"lora_kwargs,save_kwargs",
[
({"bias": "all"}, None), # frozen bias-bearing adapter: every bias, trainable or not
({"bias": "lora_only"}, None),
({"modules_to_save": ["lm_head"]}, None),
({"modules_to_save": ["lm_head"]}, {"save_embedding_layers": True}),
],
)
def test_tp2_adapter_save_matches_peft_selection(lora_kwargs, save_kwargs):
results = _spawn(
"parity",
2,
arch = "llama_bias",
lora_kwargs = lora_kwargs,
save_kwargs = save_kwargs,
freeze_before_save = True,
fp64_ref_clip = "modules_to_save" in lora_kwargs,
)
_check_parity(results, 2)
saved = results[0]["saved_tp"]
if (
lora_kwargs.get("bias") == "all"
): # "lora_only" saves no base_layer bias on PEFT 0.18 (1 process too)
assert any("o_proj" in k and k.endswith("bias") for k in saved), sorted(saved)[:8]
if "modules_to_save" in lora_kwargs:
assert any("modules_to_save" in k or "lm_head" in k for k in saved), sorted(saved)[:8]
def test_tp2_tied_norm_weight_is_summed_once():
_check_parity(_spawn("parity", 2, arch = "qwen3", tie_norms = True), 2)
def test_tp2_entry_points_outside_the_regions_are_refused_or_gather():
results = _spawn("guards", 2)
if "skip" in results[0]:
pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
for res in results:
for applied, got in res.items():
assert got["forward_without_cache"] and got["clip_mesh_equal"], (applied, got)
for key, value in got.items():
if key.startswith("is_tp:"):
assert value is True, (applied, key)
elif key.startswith("fsdp2:"):
assert value and "does not compose with Unsloth tensor parallelism" in value, (
applied,
key,
value,
)
elif "for_inference" in key or key.startswith("generate:"):
assert value is None, (
applied,
key,
value,
) # gathered a replica (tp_replica.py)
elif key.startswith("replica_released:"):
assert value is True, (applied, key, value)
elif key.startswith("model_init:"):
assert value and "cannot be trained by the Trainer" in value, (
applied,
key,
value,
)
elif key.startswith("trainer:"):
# The Trainer takes a tensor parallel model with its default args (save_strategy "steps":
# per-rank checkpoints).
assert value is None, (applied, key, value)
elif key.startswith("cached:"):
assert value and "cached decoding" in value, (applied, key, value)
def test_save_refuses_a_tp_mesh_smaller_than_the_world():
for res in _spawn("save_subgroup", 2):
assert res["error"] and "data parallel > 1 is outside v1" in res["error"], res
@pytest.mark.parametrize(
"arch,world,overrides",
[
("llama", 2, None),
("llama_bias", 2, None), # colwise biases sliced on read, row biases whole
("phi3", 2, None), # fused qkv [q, kv, kv] and packed gate_up read as per-part slices
("gemma2", 2, None), # tied embeddings, softcapping
("qwen3", 4, {"num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 64}),
],
)
def test_sharded_loading_matches_full_load(arch, world, overrides):
skip_unsupported(arch)
with tempfile.TemporaryDirectory() as ckpt:
_save_checkpoint(arch, ckpt, overrides)
results = _spawn("sharded_load", world, arch = arch, ckpt = ckpt)
_check_parity(results, world)
for res in results:
assert res["diffs"] == [], res["diffs"][:6]
# The sharded load trains exactly like v1's full load + apply_tensor_parallel.
assert res["tp"][2] == res["v1"][2] and res["tp"][3] == res["v1"][3]
torch.testing.assert_close(res["tp"][0], res["v1"][0], rtol = 0, atol = 0)
assert res["unapplied"] and "call apply_tensor_parallel" in res["unapplied"], res[
"unapplied"
]
stats = res["stats"]
# Only this rank's slices of the sharded projections were read; replicated tensors were read whole.
assert stats["read_bytes_sliced"] * world == stats["sharded_numel"] * 4, stats
assert stats["read_bytes_full"] == stats["replicated_numel"] * 4, stats
assert stats["sharded_numel"] > 0 and stats["regions"] > 0, stats
full_local, sharded_local = res["local_numel"]
assert full_local == sharded_local
def test_sharded_loading_refuses_weight_dependent_lora_init():
with tempfile.TemporaryDirectory() as ckpt:
_save_checkpoint("llama", ckpt, None)
results = _spawn("sharded_lora_init", 2, ckpt = ckpt)
for res in results:
assert "depends on the base weight" in (res["pissa"] or ""), res
assert not res["pissa_tp"]
assert res["gaussian"] is None and res["gaussian_tp"], res
assert res["orthogonal"] is None and res["orthogonal_tp"], res
@pytest.mark.parametrize("init", ["orthogonal", "gaussian", True])
def test_sharded_loading_random_lora_init_matches_one_process(init):
"""Random LoRA inits read only the full in / out features, so the sharded factors are full size; tp rank 0's
broadcast makes per-rank seeds agree, and one AdamW step matches the unsharded run from the same seed."""
with tempfile.TemporaryDirectory() as ckpt:
_save_checkpoint("llama", ckpt, None)
results = _spawn(
"sharded_random_lora_init",
2,
ckpt = ckpt,
init = init,
sync_trainable = True,
seed_per_rank = True,
)
r0, r1 = results
assert r0["error"] is None and r1["error"] is None, (r0["error"], r1["error"])
assert set(r0["local"]) == set(r1["local"]) and r0["local"]
for res in results:
assert set(res["init_tp"]) == set(res["init_ref"])
for name, ref in res["init_ref"].items():
# Bit-identical to the one-process init from tp rank 0's seed: full-size factors, no shard-sized draw.
assert torch.equal(res["init_tp"][name], ref), name
if init == "orthogonal":
for name, A in res["init_ref"].items():
if ".lora_A." not in name:
continue
B = res["init_ref"][name.replace(".lora_A.", ".lora_B.")]
# Orthogonal init: both factors non-zero, B @ A == 0 (rows from complementary halves of one Q).
assert A.abs().sum() > 0 and B.abs().sum() > 0, name
torch.testing.assert_close(
B @ A, torch.zeros_like(B @ A), rtol = 0, atol = 1e-6, msg = name
)
ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
logits, grads, losses, norms = res["tp"]
torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
assert set(grads) == set(ref_grads) and grads
for name in ref_grads:
torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
assert losses == pytest.approx(ref_losses, rel = 1e-5)
assert norms == pytest.approx(ref_norms, rel = 1e-5)
for name, ref in res["step_ref"].items():
torch.testing.assert_close(res["step_tp"][name], ref, rtol = 1e-4, atol = 1e-6, msg = name)
# Every rank's local copy of a replicated factor is the same tensor after the broadcast.
for name in r0["local"]:
if r0["local"][name].shape == r1["local"][name].shape and name in r0["init_ref"]:
full = r0["init_ref"][name]
if r0["local"][name].shape == full.shape:
assert torch.equal(r0["local"][name], r1["local"][name]), name
def test_sharded_loading_refuses_quantized_loads():
from transformers import AutoModelForCausalLM
for key in ("load_in_4bit", "quantization_config"):
with pytest.raises(ValueError, match = "16-bit safetensors checkpoints only"):
tpn.load_tensor_parallel(
LLAMA, None, loader = AutoModelForCausalLM.from_pretrained, **{key: True}
)
# phi3 fused qkv / gate_up shards add reduction-order noise the optimizer compounds (step-3 grad
# norm 6.9e-5 / 7.7e-5 off one process on CI); the step-1 per-gradient check (rtol 1e-4, atol 1e-6)
# catches real tp errors like a missed all-reduce.
_TP_FSDP2_NORM_REL = {"phi3": 2e-4}
@pytest.mark.parametrize(
"arch,fit_layout,lora_kwargs",
[
("llama", True, None),
("llama", None, None), # automatic layout (frozen tp shards still replicated over dp)
("phi3", True, None), # packed / fused parts: saved in the 1-GPU order through FSDP2
(
"llama_bias",
True,
{"bias": "all"},
), # trainable column (Shard) and row (Replicate) biases
(
"llama",
True,
{"modules_to_save": ["lm_head"]},
), # a dp-only (1-D) trainable outside the tp regions
],
)
def test_tp2_x_fsdp2_dp2_matches_one_process(arch, fit_layout, lora_kwargs):
skip_unsupported(arch)
results = _spawn("tp_fsdp2", 4, arch = arch, fit_layout = fit_layout, lora_kwargs = lora_kwargs)
if "unsupported" in results[0]:
assert all("torch 2.7+" in (r["unsupported"] or "") for r in results), results
pytest.skip(
"torch < 2.7: tensor parallel x FSDP2 is refused (no fully_shard(ignored_params))"
)
for rank, res in enumerate(results):
assert set(res["grads"]) == set(res["ref_grads"]) and res["grads"]
for name, ref in res["ref_grads"].items():
torch.testing.assert_close(res["grads"][name], ref, rtol = 1e-4, atol = 1e-6, msg = name)
assert not res["mismatched"], res["mismatched"][:4]
assert res["losses"] == pytest.approx(res["ref_losses"], rel = 1e-5), (
res["losses"],
res["ref_losses"],
)
assert res["norms"] == pytest.approx(
res["ref_norms"], rel = _TP_FSDP2_NORM_REL.get(arch, 1e-5)
), (
res["norms"],
res["ref_norms"],
)
# CPU vector_norm sums fp32 squares sequentially: about 3e-5 relative on the 2M-element lm_head copy.
native_rel = 1e-4 if "modules_to_save" in (lora_kwargs or {}) else 1e-5
assert res["native_norms"][0] == pytest.approx(res["ref_norms"][0], rel = native_rel)
# FSDP2 shards the LoRA DTensors over dp on top of their tp placement.
assert all(
len(pl) == (1 if "lm_head" in n else 2) for n, pl in res["placements"].items()
), res["placements"]
assert "the data parallel dimension" not in (res["refusals"]["no_mesh"] or "")
assert "does not compose with Unsloth tensor parallelism" in (
res["refusals"]["no_mesh"] or ""
)
assert "must be the data parallel dimension" in (res["refusals"]["tp_mesh"] or ""), res[
"refusals"
]
assert "shard_frozen = True" in (res["refusals"]["shard_frozen"] or ""), res["refusals"]
saved_tp, saved_ref = results[0]["saved_tp"], results[0]["saved_ref"]
assert set(saved_tp) == set(saved_ref)
for key in saved_ref:
torch.testing.assert_close(
saved_tp[key],
saved_ref[key],
rtol = 1e-4,
atol = 1e-5,
msg = lambda m, key = key: f"{key}: {m}",
)
def test_tp_x_fsdp2_refuses_trainable_head_norms():
for res in _spawn("tp_fsdp2_trainable_norms", 4):
assert res["error"] and "trainable per-head norms" in res["error"], res
@pytest.mark.parametrize("compiled", [False, True])
@pytest.mark.parametrize("head_only", [False, True])
def test_tp2_x_fsdp2_dp2_saves_from_one_writer(head_only, compiled):
"""A frozen tp weight's mesh is its own tp group: the writer must not follow it (two writers,
two concurrent writes of the same optimizer.pt / scheduler.pt and two Hub pushes)."""
skip_unsupported("llama")
results = _spawn("tp_fsdp2_one_writer", 4, head_only = head_only, compiled = compiled)
if "unsupported" in results[0]:
pytest.skip("torch < 2.7: tensor parallel x FSDP2 is refused")
assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results
assert [r["writer"] for r in results] == [True, False, False, False], results
assert [r["full_keys"] > 0 for r in results] == [True, False, False, False], results
def test_tp_fsdp2_writer_keeps_pipeline_stages_separate():
results = _spawn("tp_fsdp2_writer_pipeline_slice", 4)
if "unsupported" in results[0]:
pytest.skip("torch < 2.7: no fully_shard(ignored_params)")
assert [r["writer_rank"] for r in results] == [0, 0, 2, 2], results
def test_trainer_splits_mixed_optimizer_groups_for_tp_only():
# modules_to_save lm_head: a plain trainable next to the LoRA DTensors (foreach / fused take one kind).
kw = {"modules_to_save": ["lm_head"]}
ref = _spawn("trainer", 1, dp = 1, tp = 1, lora_kwargs = kw)[0]
if "skip" in ref:
pytest.skip(f"import unsloth failed on this host: {ref['skip']}")
for rank, res in enumerate(_spawn("trainer", 2, dp = 1, tp = 2, lora_kwargs = kw)):
assert res["mixed_groups"] == 0, rank
assert "already built the scheduler" in (res["refusals"]["prebuilt_scheduler"] or ""), res[
"refusals"
]
assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5), (
rank,
res["losses"],
ref["losses"],
)
# transformers 5.x would clip mixed DTensor / plain gradients itself (_mixed_mesh_grad_norm). The
# reference norm is torch's CPU fp32 vector_norm over the plain lm_head gradient: ~2e-4 off.
assert res["grad_norms"] == pytest.approx(ref["grad_norms"], rel = 1e-3), (
rank,
res["grad_norms"],
ref["grad_norms"],
)
@pytest.mark.parametrize("dp,tp", [(1, 2), (2, 2)])
def test_trainer_trains_a_tensor_parallel_model_like_one_process(dp, tp):
ref = _spawn("trainer", 1, dp = dp, tp = 1)[0]
if "skip" in ref:
pytest.skip(f"import unsloth failed on this host: {ref['skip']}")
results = _spawn("trainer", dp * tp, dp = dp, tp = tp)
if "torch < 2.7" in str(results[0]["refusals"].get("no_shard_model", "")):
pytest.skip("torch < 2.7: tensor parallel x FSDP2 is refused")
assert len(ref["losses"]) == 3
for rank, res in enumerate(results):
assert "DistributedDataParallel" not in res["wrapped"], res["wrapped"]
assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5), (
rank,
res["losses"],
ref["losses"],
)
assert res["grad_norms"] == pytest.approx(ref["grad_norms"], rel = 1e-5), (
res["grad_norms"],
ref["grad_norms"],
)
assert set(res["state"]) == set(ref["state"]) and ref["state"]
for key, value in ref["state"].items():
torch.testing.assert_close(res["state"][key], value, rtol = 1e-4, atol = 1e-6, msg = key)
refusals = res["refusals"]
# Checkpoints are per rank, bitsandbytes AdamW steps local shards (or switches to torch's AdamW).
assert refusals["checkpoints"] is None, refusals
assert refusals["eval"] is None, refusals
assert refusals["optim_8bit"] is None, refusals
if dp > 1:
assert "covers 2 of 4 ranks" in (refusals["no_shard_model"] or ""), refusals
# The tp ranks of a replica read the same batches; dp replicas read different ones.
for rank, res in enumerate(results):
peer = results[rank ^ 1] # same dp replica (tp is the inner mesh dimension)
assert res["seen"] == peer["seen"]
if dp > 1:
assert res["seen"] != results[(rank + tp) % (dp * tp)]["seen"]
saved, saved_ref = results[0]["saved"], ref["saved"]
assert set(saved) == set(saved_ref)
for key in saved_ref:
torch.testing.assert_close(saved[key], saved_ref[key], rtol = 1e-4, atol = 1e-6, msg = key)
@pytest.mark.parametrize("strategy", ["batch_rebalance", "group_by_length"])
@pytest.mark.parametrize("dp", [1, 2])
def test_trainer_length_samplers_give_tp_peers_the_same_rows(dp, strategy):
results = _spawn("sampler", 2 * dp, dp = dp, strategy = strategy)
if "skip" in results[0]:
pytest.skip(results[0]["skip"])
for rank, res in enumerate(results):
assert res["seen"] and res["seen"] == results[rank ^ 1]["seen"], (
rank,
res,
results[rank ^ 1],
)
if dp > 1:
replicas = [{i for batch in results[r]["seen"] for i in batch} for r in (0, 2)]
assert not replicas[0] & replicas[1], replicas
@pytest.mark.parametrize("strategy", ["batch_rebalance", "group_by_length"])
@pytest.mark.parametrize("dp", [1, 2])
def test_trainer_length_samplers_resume_like_an_uninterrupted_run(dp, strategy):
results = _spawn("sampler_resume", 2 * dp, dp = dp, strategy = strategy)
if "skip" in results[0]:
pytest.skip(results[0]["skip"])
for rank, res in enumerate(results):
peer = results[rank ^ 1]
assert res["full"] == peer["full"], (rank, res["full"], peer["full"])
for k, seen in res["resumed"].items():
assert seen == peer["resumed"][k], (rank, k, seen, peer["resumed"][k])
assert seen == res["full"][2 * k :], (rank, k, seen, res["full"][2 * k :])
def test_a_rank_that_finished_is_not_failed_by_an_abort_in_teardown():
# The sampler-resume ranks finished and then died with -6 in interpreter teardown.
assert _spawn("teardown_abort", 2) == [{"rank": 0}, {"rank": 1}]