Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO DDP gradient sync: copy each averaged bucket back with one _foreach_copy_ (#13268)
Daniel Hanf440d286e44c
diff --git a/tests/test_grpo_ddp_average_gradients_foreach.py b/tests/test_grpo_ddp_average_gradients_foreach.pynew file mode 100644index 00000000000..324c7b95c3f--- /dev/null+++ b/tests/test_grpo_ddp_average_gradients_foreach.py@@ -0,0 +1,114 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GRPO's manual DDP gradient average copies each bucket back with one fused copy.++``_unsloth_average_gradients`` is lifted from ``unsloth/models/rl.py`` with ``ast`` so the test stays+CPU-only; collectives are stubbed (a 2-rank all_reduce of identical grads is a SUM of two copies).+"""++import ast+from pathlib import Path+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+import torch.distributed as dist++SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"+++def _load():+ tree = ast.parse(SOURCE.read_text(encoding = "utf-8"))+ node = next(+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_average_gradients"+ )+ namespace = {"torch": torch}+ exec(compile(ast.Module([node], []), str(SOURCE), "exec"), namespace)+ return namespace["_unsloth_average_gradients"]+++def two_identical_ranks(monkeypatch):+ def all_reduce(+ tensor,+ op = None,+ group = None,+ ):+ if op is None or op == dist.ReduceOp.SUM:+ tensor.mul_(2)++ monkeypatch.setattr(dist, "get_world_size", lambda group = None: 2)+ monkeypatch.setattr(dist, "all_reduce", all_reduce)+++def _model(shapes, dtype = torch.float32):+ torch.manual_seed(0)+ module = torch.nn.Module()+ module.params = torch.nn.ParameterList(+ torch.nn.Parameter(torch.zeros(*s, dtype = dtype)) for s in shapes+ )+ for p in module.params:+ p.grad = torch.randn_like(p)+ return SimpleNamespace(process_group = None, module = module)++[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_bucket_copy_back_is_fused_and_exact(two_identical_ranks, monkeypatch, dtype):+ average = _load()+ # LoRA-like: many small A / B tensors that share a bucket, one non-contiguous gradient, and one that is+ # contiguous but strided (1, 1) because of a size-1 dim.+ ddp = _model([(4, 16), (16, 4), (4, 32), (32, 4), (8,), (1, 8)], dtype)+ ddp.module.params[1].grad = torch.randn(4, 16, dtype = dtype).t()+ ddp.module.params[5].grad = torch.randn(8, 1, dtype = dtype).t()+ assert ddp.module.params[5].grad.is_contiguous() and ddp.module.params[5].grad.stride() == (+ 1,+ 1,+ )+ before = [p.grad.clone() for p in ddp.module.params]+ calls = []+ real = torch._foreach_copy_++ def counting(dst, src, *args, **kwargs):+ calls.append(len(dst))+ assert all(d.stride() == s.stride() for d, s in zip(dst, src))+ return real(dst, src, *args, **kwargs)++ monkeypatch.setattr(torch, "_foreach_copy_", counting)+ average(ddp)+ assert calls == [len(before) - 2]+ for p, g in zip(ddp.module.params, before):+ # SUM of two identical ranks then div by 2 gives the gradient back bit for bit.+ assert torch.equal(p.grad, g)+++def test_oversized_gradient_still_reduced_in_place(two_identical_ranks):+ average = _load()+ ddp = _model([(64, 64), (3,)])+ before = [p.grad.clone() for p in ddp.module.params]+ average(ddp, bucket_bytes = 1)+ for p, g in zip(ddp.module.params, before):+ assert torch.equal(p.grad, g)++[email protected]("mode", ["missing", "raises"])+def test_per_tensor_fallback_when_foreach_copy_unavailable(two_identical_ranks, monkeypatch, mode):+ average = _load()+ ddp = _model([(4, 16), (16, 4), (8,)])+ ddp.module.params[1].grad = torch.randn(4, 16).t()+ before = [p.grad.clone() for p in ddp.module.params]+ if mode == "missing":+ monkeypatch.delattr(torch, "_foreach_copy_")+ else:++ def broken(dst, src, *args, **kwargs):+ dst[0].zero_() # a partial write before failing must not leak through+ raise RuntimeError("unsupported")++ monkeypatch.setattr(torch, "_foreach_copy_", broken)+ average(ddp)+ for p, g in zip(ddp.module.params, before):+ assert torch.equal(p.grad, g)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 17f7727d5f4..02ece8e4dc5 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2063,10 +2063,28 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):flat = torch.cat([g.reshape(-1) for g in chunk])dist.all_reduce(flat, group = group)flat.div_(world_size)- offset = 0- for g in chunk:- g.copy_(flat[offset : offset + g.numel()].view_as(g))- offset += g.numel()+ # One fused copy back: a per-tensor copy_ loop costs a launch per LoRA tensor (500+ for an 8B model).+ pieces = [+ x.view_as(g) for x, g in zip(torch.split(flat, [g.numel() for g in chunk]), chunk)+ ]+ # The fused route needs matching strides (one mismatch slows the whole list), so mismatched grads go alone.+ dst, src = [], []+ for g, x in zip(chunk, pieces):+ if g.stride() == x.stride():+ dst.append(g)+ src.append(x)+ else:+ g.copy_(x)+ foreach_copy = getattr(torch, "_foreach_copy_", None)+ try:+ if foreach_copy is None:+ raise NotImplementedError+ if dst:+ foreach_copy(dst, src)+ except (RuntimeError, NotImplementedError, TypeError):+ # Private op: fall back to the per-tensor copy if a torch build lacks or rejects it (idempotent).+ for g, x in zip(dst, src):+ g.copy_(x)start = stop