Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import multiprocessing
import os
if multiprocessing.parent_process() is not None or getattr(
multiprocessing.current_process(), "_inheriting", False
):
os.environ["CUDA_VISIBLE_DEVICES"] = "" # Gloo ranks must stay on CPU even on GPU hosts.
import datetime # noqa: E402
import importlib # noqa: E402
import sys # noqa: E402
import time # noqa: E402
import pytest # noqa: E402
import torch # noqa: E402
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_fsdp2_native_units import _load_native # noqa: E402
from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
def _reference_sync(
params,
dist,
mesh = None,
):
"""Pre-buffer-reuse implementation, retained as an independent reference."""
from unsloth.models.fsdp2_save import mesh_all_reduce
groups = {}
for p in params:
dtype = torch.float64 if p.dtype == torch.float64 else torch.float32
groups.setdefault((str(p.device), dtype), []).append(p)
for (device, dtype), group in groups.items():
sizes = [p.numel() for p in group]
total = sum(sizes)
flat = torch.zeros(total + len(group), dtype = dtype, device = group[0].device)
offset = 0
for i, (p, n) in enumerate(zip(group, sizes)):
if p.grad is not None:
flat[offset : offset + n].copy_(p.grad.detach().reshape(-1))
flat[total + i] = 1
offset += n
world = mesh_all_reduce(flat, mesh, dist)
flat[:total].div_(world)
used = flat[total:].tolist()
offset = 0
for i, (p, n) in enumerate(zip(group, sizes)):
if used[i] > 0:
value = flat[offset : offset + n].view(p.shape)
if p.grad is None:
p.grad = value.to(p.dtype).clone()
else:
p.grad.copy_(value)
offset += n
def _save():
_load_native()
return importlib.import_module("unsloth.models.fsdp2_save")
class _Reads:
"""Count device-to-host reads and scalar assignments that synchronize CUDA."""
def __init__(self):
self.n = 0
def __enter__(self):
self._saved = (torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__)
reads, (tolist0, item0, setitem0) = self, self._saved
def tolist(t, *a, **k):
reads.n += 1
return tolist0(t, *a, **k)
def item(t, *a, **k):
reads.n += 1
return item0(t, *a, **k)
def setitem(t, key, value):
if not isinstance(value, torch.Tensor):
reads.n += 1
return setitem0(t, key, value)
torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__ = tolist, item, setitem
return self
def __exit__(self, *exc):
torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__ = self._saved
def _make_params(spec):
return [torch.nn.Parameter(torch.zeros(shape, dtype = dtype)) for shape, dtype in spec]
def _grads(params):
return [None if p.grad is None else p.grad.clone() for p in params]
def _same(a, b):
if len(a) != len(b):
return False
for x, y in zip(a, b):
if (x is None) != (y is None):
return False
if x is not None and (x.dtype != y.dtype or x.shape != y.shape or not torch.equal(x, y)):
return False
return True
def _case_unit(rank):
import torch.distributed as dist
save = _save()
spec = [
((3, 4), torch.float32),
((5,), torch.float16),
((2, 2), torch.float64),
((7,), torch.float32),
((4,), torch.bfloat16),
((6,), torch.float16),
]
old, new = _make_params(spec), _make_params(spec)
buffers = {}
gen = torch.Generator().manual_seed(100 + rank)
res = {"mismatch": [], "reads_all_present": [], "reads_some_none": [], "alias": 0}
patterns = [
[0, 1, 2, 4, 5] if rank == 0 else [0, 1, 2],
[0, 1, 2, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 2] if rank == 1 else [0, 1, 2, 3, 4, 5],
]
for step, used in enumerate(patterns):
if step in (2, 4):
for p in old + new:
p.grad = None
for i in used:
g = (torch.randn(spec[i][0], generator = gen) * 1000).to(spec[i][1])
for p in (old[i], new[i]):
if p.grad is None:
p.grad = g.clone()
else:
p.grad.add_(g)
_reference_sync(old, dist)
everyone_all = all(p.grad is not None for p in new)
with _Reads() as reads:
save._sync_replicated_grads(new, dist, None, buffers)
(res["reads_all_present"] if everyone_all else res["reads_some_none"]).append(reads.n)
if not _same(_grads(old), _grads(new)):
res["mismatch"].append(step)
for p in new:
if p.grad is None:
continue
for _, flat in buffers.values():
lo, hi = flat.data_ptr(), flat.data_ptr() + flat.numel() * flat.element_size()
if lo <= p.grad.data_ptr() < hi:
res["alias"] += 1
res["n_buffers"] = len(buffers)
ids_before = {k: id(v[1]) for k, v in buffers.items()}
for p in old + new:
p.grad = torch.ones_like(p)
save._sync_replicated_grads(new, dist, None, buffers)
res["reused"] = {str(k): id(v[1]) == ids_before[k] for k, v in buffers.items()}
extra = _make_params([((9,), torch.float32)])
old2, new2 = old + _make_params([((9,), torch.float32)]), new + extra
for p in old2 + new2:
p.grad = (
(torch.arange(p.numel(), dtype = torch.float32) * (rank + 1)).view(p.shape).to(p.dtype)
)
_reference_sync(old2, dist)
save._sync_replicated_grads(new2, dist, None, buffers)
res["grown_same"] = _same(_grads(old2), _grads(new2))
res["grown_rebuilt"] = (
id(buffers[("cpu", torch.float32)][1]) != ids_before[("cpu", torch.float32)]
)
res["fp64_reused"] = (
id(buffers[("cpu", torch.float64)][1]) == ids_before[("cpu", torch.float64)]
)
old3, new3 = _make_params([((4,), torch.float64)]), _make_params([((4,), torch.float64)])
for p in old3 + new3:
p.grad = torch.full((4,), float(rank + 2), dtype = torch.float64)
before64 = id(buffers[("cpu", torch.float64)][1])
_reference_sync(old3, dist)
save._sync_replicated_grads(new3, dist, None, buffers)
res["resized_same"] = _same(_grads(old3), _grads(new3))
res["resized_rebuilt"] = id(buffers[("cpu", torch.float64)][1]) != before64
old4, new4 = _make_params(spec), _make_params(spec)
for p in old4 + new4:
p.grad = torch.full(p.shape, float(rank + 1)).to(p.dtype)
if rank == 0:
old4[1].grad = new4[1].grad = None
_reference_sync(old4, dist)
save._sync_replicated_grads(new4, dist)
res["nobuf_same"] = _same(_grads(old4), _grads(new4))
return res
class _Net(torch.nn.Module):
def __init__(self):
super().__init__()
torch.manual_seed(0)
self.body = torch.nn.Sequential(torch.nn.Linear(8, 16), torch.nn.Linear(16, 8))
self.norm = torch.nn.Parameter(torch.randn(8))
self.norm16 = torch.nn.Parameter(torch.randn(8).half())
self.branch = torch.nn.Parameter(torch.randn(8))
def forward(self, x, use_branch):
h = self.body(x) * self.norm + self.norm16.float()
if use_branch:
h = h * self.branch
return (h * h).mean()
def _build(save):
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
model = _Net()
mesh = init_device_mesh("cpu", (2,))
for layer in model.body:
fully_shard(layer, mesh = mesh)
fully_shard(model, mesh = mesh, ignored_params = {model.norm, model.norm16, model.branch})
names = save.reduce_unsharded_trainable_grads(model)
return model, names
GA = 4
def _train(save, native, rank, mode, impl):
import torch.distributed as dist
os.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode
saved = (save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce)
calls = []
def counting_all_reduce(*args, **kw):
calls[-1] += 1
return saved[2](*args, **kw)
save.mesh_all_reduce = counting_all_reduce
if impl == "old":
save._sync_replicated_grads = lambda params, d, mesh = None, *_: _reference_sync(
params, d, mesh
)
if impl in ("old", "nodefer", "oracle"):
save._fsdp_grad_sync_deferred = lambda *a: False
if impl == "oracle":
save._sync_replicated_grads = lambda *a, **k: None
try:
model, names = _build(save)
rep = [p for n, p in model.named_parameters() if n in set(names)]
opt = torch.optim.AdamW(model.parameters(), lr = 1e-2)
gen = torch.Generator().manual_seed(7 + rank)
out = []
for step in range(3):
for micro in range(GA):
x = torch.randn(4, 8, generator = gen)
use_branch = (micro + rank + step) % 3 == 0 and step != 0
calls.append(0)
if micro == GA - 1:
model(x, use_branch).backward()
else:
with native._no_grad_sync(model):
model(x, use_branch).backward()
if impl == "oracle":
_reference_sync(rep, dist)
grads = {}
for n, p in model.named_parameters():
g = p.grad
if g is not None and hasattr(g, "full_tensor"):
g = g.full_tensor()
grads[n] = None if g is None else g.detach().clone()
out.append(grads)
opt.step()
opt.zero_grad(set_to_none = True)
return names, out, calls
finally:
save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce = saved
def _diff(
a_steps,
b_steps,
skip = (),
):
bad, worst = [], 0.0
for step, (a, b) in enumerate(zip(a_steps, b_steps)):
for n in a:
if n in skip:
continue
x, y = a[n], b[n]
if (x is None) != (y is None):
bad.append((step, n, "none"))
worst = float("inf")
elif x is not None and (not torch.isfinite(x).all() or not torch.isfinite(y).all()):
bad.append((step, n, "nonfinite"))
worst = float("inf")
elif x is not None and not torch.equal(x, y):
bad.append((step, n))
worst = max(worst, (x.double() - y.double()).abs().max().item())
return bad, worst
def _case_modes(rank):
native = _load_native()
save = _save()
res = {}
for mode in ("skip", "defer", "each", "auto"):
runs = {
impl: _train(save, native, rank, mode, impl)
for impl in ("old", "nodefer", "new", "oracle")
}
names = runs["old"][0]
r = {"names": sorted(names), "branch_none_step0": runs["old"][1][0]["branch"] is None}
r["calls"] = {impl: v[2] for impl, v in runs.items()}
r["nodefer_vs_old"] = _diff(runs["old"][1], runs["nodefer"][1])[0]
r["new_vs_old"], r["new_vs_old_max"] = _diff(runs["old"][1], runs["new"][1])
# Wider fp16 sums change later updates, so only unaffected first-step gradients are bitwise.
r["new_vs_oracle"] = _diff(runs["oracle"][1][:1], runs["new"][1][:1], skip = ("norm16",))[0]
r["new_vs_oracle_max"] = _diff(runs["oracle"][1], runs["new"][1])[1]
res[mode] = r
return res
class _Fp16Rep(torch.nn.Module):
def __init__(self):
super().__init__()
self.body = torch.nn.Linear(4, 4)
self.rep = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
self.fail_forward = False
def forward(
self,
x,
scale,
use_rep = True,
):
loss = self.body(x).mean()
if self.fail_forward:
raise RuntimeError("injected forward failure")
return loss + (self.rep.float() * scale).sum() if use_rep else loss
def _case_fp16_overflow(rank):
"""Local 80000 overflows fp16 before averaging with the other rank to 60000."""
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
native = _load_native()
save = _save()
os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"
torch.manual_seed(0)
model = _Fp16Rep()
mesh = init_device_mesh("cpu", (2,))
fully_shard(model.body, mesh = mesh)
fully_shard(model, mesh = mesh, ignored_params = {model.rep})
names = save.reduce_unsharded_trainable_grads(model)
scale = 40000.0 if rank == 0 else 20000.0
x = torch.randn(2, 4)
with native._no_grad_sync(model):
model(x, scale).backward()
model(x, scale).backward()
return {"names": names, "grad": model.rep.grad.float().tolist(), "file": save.__file__}
def _case_fp16_edge(rank, mode):
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
native = _load_native()
save = _save()
os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"
torch.manual_seed(0)
model = _Fp16Rep()
if mode == "wide":
model.rep.grad_dtype = torch.float32
mesh = init_device_mesh("cpu", (2,))
fully_shard(model.body, mesh = mesh)
fully_shard(model, mesh = mesh, ignored_params = {model.rep})
save.reduce_unsharded_trainable_grads(model)
owner = model
if mode == "outer_recovery":
class Outer(torch.nn.Module):
def __init__(self, inner):
super().__init__()
self.inner = inner
def forward(self, *args, **kwargs):
return self.inner(*args, **kwargs)
owner = Outer(model)
fully_shard(owner, mesh = mesh)
save.reduce_unsharded_trainable_grads(owner)
x = torch.randn(2, 4)
with native._no_grad_sync(owner):
owner(x, 4.0).backward()
if mode == "wide":
owner(x, 2.0, use_rep = False).backward()
else:
model.fail_forward = True
with pytest.raises(RuntimeError, match = "injected forward failure"):
owner(x, 9.0)
owner.reset_iter_state()
owner.zero_grad(set_to_none = True)
model.fail_forward = False
owner(x, 2.0).backward()
state = model.__dict__["_unsloth_replicated_grad_state"]
return {
"grad": model.rep.grad.tolist(),
"dtype": str(model.rep.grad.dtype),
"pending": bool(state.get("pending")),
"queued": state["queued"],
}
def _worker(rank, world, port, out, case):
os.environ.update(
MASTER_ADDR = "127.0.0.1",
MASTER_PORT = str(port),
RANK = str(rank),
WORLD_SIZE = str(world),
LOCAL_RANK = str(rank),
LOCAL_WORLD_SIZE = str(world),
UNSLOTH_FSDP2 = "1",
)
import torch.distributed as dist
dist.init_process_group(
"gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 60)
)
try:
cases = {"unit": _case_unit, "modes": _case_modes, "fp16_overflow": _case_fp16_overflow}
res = _case_fp16_edge(rank, case[5:]) if case.startswith("edge_") else cases[case](rank)
out.put((rank, res))
except Exception as exc:
import traceback
out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
finally:
try:
dist.destroy_process_group()
except Exception:
pass
def _run(case, timeout = 240):
import torch.multiprocessing as mp
ctx = mp.get_context("spawn")
out = ctx.Queue()
procs = mp.start_processes(
_worker, args = (2, _free_port(), out, case), nprocs = 2, join = False, start_method = "spawn"
)
deadline = time.time() + timeout
while not procs.join(timeout = 5):
if time.time() > deadline:
for p in procs.processes:
p.kill()
pytest.fail(f"{case}: the ranks did not finish within {timeout} s (hung collective)")
results = {}
while not out.empty():
rank, r = out.get(timeout = 5)
results[rank] = r
assert sorted(results) == [0, 1], results
for rank, r in results.items():
print("REPL_SYNC_RESULT", case, rank, r)
assert "crash" not in r, r["crash"]
return results
def test_replicated_sync_bitwise_buffer_reuse_and_no_host_read():
for rank, r in _run("unit").items():
assert r["mismatch"] == [], (rank, r)
assert r["alias"] == 0, (rank, r)
assert r["reads_all_present"] == [0] * (3 if rank == 0 else 2), (rank, r)
assert r["reads_some_none"] == [1] * (2 if rank == 0 else 3), (rank, r)
assert r["n_buffers"] == 2 and all(r["reused"].values()), (rank, r)
assert r["grown_same"] and r["grown_rebuilt"] and r["fp64_reused"], (rank, r)
assert r["resized_same"] and r["resized_rebuilt"], (rank, r)
assert r["nobuf_same"], (rank, r)
def test_replicated_sync_follows_the_ga_sync_flag():
for rank, r in _run("modes", timeout = 400).items():
for mode, m in r.items():
assert m["names"] == ["branch", "norm", "norm16"], (rank, mode, m)
assert m["branch_none_step0"], (rank, mode, m)
assert m["nodefer_vs_old"] == [], (rank, mode, m)
assert m["calls"]["old"] == [1] * (3 * GA), (rank, mode, m)
deferred = mode != "each" # Auto without a fit layout defers synchronization.
want = ([0] * (GA - 1) + [1]) * 3 if deferred else [1] * (3 * GA)
assert m["calls"]["new"] == want, (rank, mode, m)
assert m["calls"]["oracle"] == ([0] * (GA - 1) + [1]) * 3, (rank, mode, m)
if deferred:
assert m["new_vs_oracle"] == [], (rank, mode, m)
assert m["new_vs_oracle_max"] < 1e-2, (rank, mode, m)
assert m["new_vs_old_max"] < 1e-2, (rank, mode, m)
else:
assert m["new_vs_old"] == [], (rank, mode, m)
def test_fp16_replica_local_sum_over_micro_batches_does_not_overflow():
for rank, r in _run("fp16_overflow").items():
assert r["names"] == ["rep"], (rank, r)
assert r["grad"] == [60000.0] * 4, (rank, r)
@pytest.mark.parametrize("mode", ["wide", "recovery", "outer_recovery"])
def test_fp16_replica_gradient_dtype_and_recovery(mode):
FSDPModule = _save()._fsdp_module_class()
if mode == "wide" and not hasattr(torch.Tensor, "grad_dtype"):
pytest.skip("torch has no explicit gradient dtype")
if mode != "wide" and not hasattr(FSDPModule, "reset_iter_state"):
pytest.skip("torch has no iteration recovery API")
for rank, r in _run("edge_" + mode).items():
assert r["grad"] == [4.0 if mode == "wide" else 2.0] * 4, (rank, mode, r)
assert r["dtype"] == ("torch.float32" if mode == "wide" else "torch.float16"), r
assert not r["pending"] and not r["queued"], r
@pytest.mark.parametrize(
"left,right", [(1.0, None), (None, 1.0), (1.0, float("nan")), (float("inf"), float("inf"))]
)
def test_gradient_comparison_rejects_missing_and_nonfinite(left, right):
a = None if left is None else torch.tensor([left])
b = None if right is None else torch.tensor([right])
bad, worst = _diff([{"rep": a}], [{"rep": b}])
assert bad and worst == float("inf")