Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO DDP gradient sync: copy each averaged bucket back with one _foreach_copy_ (#13268)

Daniel Hanf440d286e44c
diff --git a/tests/test_grpo_ddp_average_gradients_foreach.py b/tests/test_grpo_ddp_average_gradients_foreach.py
new file mode 100644
index 00000000000..324c7b95c3f
--- /dev/null
+++ b/tests/test_grpo_ddp_average_gradients_foreach.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GRPO's manual DDP gradient average copies each bucket back with one fused copy.
+
+``_unsloth_average_gradients`` is lifted from ``unsloth/models/rl.py`` with ``ast`` so the test stays
+CPU-only; collectives are stubbed (a 2-rank all_reduce of identical grads is a SUM of two copies).
+"""
+
+import ast
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+import torch.distributed as dist
+
+SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"
+
+
+def _load():
+ tree = ast.parse(SOURCE.read_text(encoding = "utf-8"))
+ node = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_average_gradients"
+ )
+ namespace = {"torch": torch}
+ exec(compile(ast.Module([node], []), str(SOURCE), "exec"), namespace)
+ return namespace["_unsloth_average_gradients"]
+
+
+def two_identical_ranks(monkeypatch):
+ def all_reduce(
+ tensor,
+ op = None,
+ group = None,
+ ):
+ if op is None or op == dist.ReduceOp.SUM:
+ tensor.mul_(2)
+
+ monkeypatch.setattr(dist, "get_world_size", lambda group = None: 2)
+ monkeypatch.setattr(dist, "all_reduce", all_reduce)
+
+
+def _model(shapes, dtype = torch.float32):
+ torch.manual_seed(0)
+ module = torch.nn.Module()
+ module.params = torch.nn.ParameterList(
+ torch.nn.Parameter(torch.zeros(*s, dtype = dtype)) for s in shapes
+ )
+ for p in module.params:
+ p.grad = torch.randn_like(p)
+ return SimpleNamespace(process_group = None, module = module)
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_bucket_copy_back_is_fused_and_exact(two_identical_ranks, monkeypatch, dtype):
+ average = _load()
+ # LoRA-like: many small A / B tensors that share a bucket, one non-contiguous gradient, and one that is
+ # contiguous but strided (1, 1) because of a size-1 dim.
+ ddp = _model([(4, 16), (16, 4), (4, 32), (32, 4), (8,), (1, 8)], dtype)
+ ddp.module.params[1].grad = torch.randn(4, 16, dtype = dtype).t()
+ ddp.module.params[5].grad = torch.randn(8, 1, dtype = dtype).t()
+ assert ddp.module.params[5].grad.is_contiguous() and ddp.module.params[5].grad.stride() == (
+ 1,
+ 1,
+ )
+ before = [p.grad.clone() for p in ddp.module.params]
+ calls = []
+ real = torch._foreach_copy_
+
+ def counting(dst, src, *args, **kwargs):
+ calls.append(len(dst))
+ assert all(d.stride() == s.stride() for d, s in zip(dst, src))
+ return real(dst, src, *args, **kwargs)
+
+ monkeypatch.setattr(torch, "_foreach_copy_", counting)
+ average(ddp)
+ assert calls == [len(before) - 2]
+ for p, g in zip(ddp.module.params, before):
+ # SUM of two identical ranks then div by 2 gives the gradient back bit for bit.
+ assert torch.equal(p.grad, g)
+
+
+def test_oversized_gradient_still_reduced_in_place(two_identical_ranks):
+ average = _load()
+ ddp = _model([(64, 64), (3,)])
+ before = [p.grad.clone() for p in ddp.module.params]
+ average(ddp, bucket_bytes = 1)
+ for p, g in zip(ddp.module.params, before):
+ assert torch.equal(p.grad, g)
+
+
[email protected]("mode", ["missing", "raises"])
+def test_per_tensor_fallback_when_foreach_copy_unavailable(two_identical_ranks, monkeypatch, mode):
+ average = _load()
+ ddp = _model([(4, 16), (16, 4), (8,)])
+ ddp.module.params[1].grad = torch.randn(4, 16).t()
+ before = [p.grad.clone() for p in ddp.module.params]
+ if mode == "missing":
+ monkeypatch.delattr(torch, "_foreach_copy_")
+ else:
+
+ def broken(dst, src, *args, **kwargs):
+ dst[0].zero_() # a partial write before failing must not leak through
+ raise RuntimeError("unsupported")
+
+ monkeypatch.setattr(torch, "_foreach_copy_", broken)
+ average(ddp)
+ for p, g in zip(ddp.module.params, before):
+ assert torch.equal(p.grad, g)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 17f7727d5f4..02ece8e4dc5 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2063,10 +2063,28 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):
flat = torch.cat([g.reshape(-1) for g in chunk])
dist.all_reduce(flat, group = group)
flat.div_(world_size)
- offset = 0
- for g in chunk:
- g.copy_(flat[offset : offset + g.numel()].view_as(g))
- offset += g.numel()
+ # One fused copy back: a per-tensor copy_ loop costs a launch per LoRA tensor (500+ for an 8B model).
+ pieces = [
+ x.view_as(g) for x, g in zip(torch.split(flat, [g.numel() for g in chunk]), chunk)
+ ]
+ # The fused route needs matching strides (one mismatch slows the whole list), so mismatched grads go alone.
+ dst, src = [], []
+ for g, x in zip(chunk, pieces):
+ if g.stride() == x.stride():
+ dst.append(g)
+ src.append(x)
+ else:
+ g.copy_(x)
+ foreach_copy = getattr(torch, "_foreach_copy_", None)
+ try:
+ if foreach_copy is None:
+ raise NotImplementedError
+ if dst:
+ foreach_copy(dst, src)
+ except (RuntimeError, NotImplementedError, TypeError):
+ # Private op: fall back to the per-tensor copy if a torch build lacks or rejects it (idempotent).
+ for g, x in zip(dst, src):
+ g.copy_(x)
start = stop