Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

DDP: gradient_as_bucket_view for full finetuning, opt-in bf16 / fp16 gradient all-reduce (#14399)

Daniel Hana3395cf4bed1
diff --git a/tests/_ddp_comm_defaults_worker.py b/tests/_ddp_comm_defaults_worker.py
new file mode 100644
index 00000000000..0bd7a43c39d
--- /dev/null
+++ b/tests/_ddp_comm_defaults_worker.py
@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""One gloo rank for test_ddp_comm_defaults.py: a real accelerate Accelerator on CPU builds the DDP."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import types
+
+_DEFAULTS = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "models",
+ "_ddp_defaults.py",
+)
+
+
+def load_defaults():
+ # By path: importing the unsloth package needs a GPU.
+ spec = importlib.util.spec_from_file_location("_unsloth_ddp_defaults_under_test", _DEFAULTS)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def build(kind):
+ import torch
+
+ torch.manual_seed(0)
+ layers = [
+ torch.nn.Linear(8, 16),
+ torch.nn.GELU(),
+ torch.nn.Linear(16, 16),
+ torch.nn.GELU(),
+ torch.nn.Linear(16, 5),
+ ]
+ model = torch.nn.Sequential(*layers)
+ model.register_buffer("scale", torch.tensor(1.5), persistent = False)
+ if kind == "peft":
+ model.peft_config = {"default": object()}
+ return model
+
+
+def batch(rank, n = 6):
+ import torch
+ g = torch.Generator().manual_seed(1234 + rank)
+ return torch.randn(n, 8, generator = g), torch.randint(0, 5, (n,), generator = g)
+
+
+def loss_fn(model, x, y):
+ import torch
+ return torch.nn.functional.cross_entropy(model(x) * 1.0, y)
+
+
+def run_rank(rank, port, queue, cases):
+ import torch
+ import torch.distributed as dist
+
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ LOCAL_WORLD_SIZE = "2",
+ ACCELERATE_USE_CPU = "true",
+ )
+ torch.set_num_threads(1)
+ from accelerate import Accelerator
+ from accelerate.state import AcceleratorState, PartialState
+
+ out = {}
+ accelerator = Accelerator(cpu = True)
+ try:
+ defaults = load_defaults()
+ for name, kind, env, args in cases:
+ saved = {k: os.environ.get(k) for k in env}
+ os.environ.update(env)
+ try:
+ accelerator.ddp_handler = None
+ accelerator.prepare_model = types.MethodType(
+ type(accelerator).prepare_model, accelerator
+ )
+ trainer = types.SimpleNamespace(
+ accelerator = accelerator, args = types.SimpleNamespace(**args)
+ )
+ installed = defaults.install_unsloth_ddp_comm_defaults(trainer)
+ model = build(kind)
+ ddp = accelerator.prepare_model(model)
+ opt = torch.optim.SGD(ddp.parameters(), lr = 0.1)
+ x, y = batch(rank)
+ loss = loss_fn(ddp, x, y)
+ loss.backward()
+ grads = [p.grad.detach().clone() for p in ddp.module.parameters()]
+ storages = {p.grad.untyped_storage().data_ptr() for p in ddp.module.parameters()}
+ opt.step()
+ losses = [None, None]
+ dist.all_gather_object(losses, float(loss))
+ out[name] = dict(
+ installed = installed,
+ is_ddp = isinstance(ddp, torch.nn.parallel.DistributedDataParallel),
+ view = bool(ddp.gradient_as_bucket_view),
+ hooks = len(getattr(ddp, "_comm_hooks", []) or [])
+ or int(getattr(ddp, "_unsloth_test_hook", 0)),
+ n_grad_storages = len(storages),
+ losses = losses,
+ grads = [g.tolist() for g in grads],
+ params = [p.detach().tolist() for p in ddp.module.parameters()],
+ )
+ finally:
+ for k, v in saved.items():
+ if v is None:
+ os.environ.pop(k, None)
+ else:
+ os.environ[k] = v
+ queue.put((rank, out))
+ except BaseException:
+ import traceback
+ queue.put((rank, {"error": traceback.format_exc()}))
+ raise
+ finally:
+ dist.destroy_process_group()
+ AcceleratorState._reset_state(True)
+ PartialState._reset_state()
diff --git a/tests/test_ddp_comm_defaults.py b/tests/test_ddp_comm_defaults.py
new file mode 100644
index 00000000000..6e75f226318
--- /dev/null
+++ b/tests/test_ddp_comm_defaults.py
@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""DDP comm defaults on a real 2-rank gloo accelerate DDP must match one process on the joined batch."""
+
+from __future__ import annotations
+
+import ast
+import os
+import pathlib
+import socket
+import sys
+
+import pytest
+import torch
+
+pytest.importorskip("accelerate")
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # spawned ranks inherit sys.path
+import _ddp_comm_defaults_worker as worker # noqa: E402
+
+pytestmark = pytest.mark.skipif(sys.platform == "win32", reason = "two gloo ranks over TCP localhost")
+
+CASES = [
+ ("full", "full", {}, {}),
+ ("full_off", "full", {"UNSLOTH_DDP_GRADIENT_AS_BUCKET_VIEW": "0"}, {}),
+ ("peft", "peft", {}, {}),
+ ("bf16_hook", "full", {"UNSLOTH_DDP_COMM_HOOK": "bf16"}, {}),
+ ("fp16_hook", "peft", {"UNSLOTH_DDP_COMM_HOOK": "fp16"}, {}),
+]
+_UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
[email protected](scope = "module")
+def ranks():
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = worker.run_rank, args = (rank, port, queue, CASES)) for rank in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = dict(queue.get(timeout = 600) for _ in procs)
+ for rank, out in results.items():
+ assert "error" not in out, out["error"]
+ for p in procs:
+ p.join(120)
+ assert p.exitcode == 0
+ return results
+
+
+def _reference(kind):
+ """Single process, both ranks' samples: the mean of two equal-size rank means is the global mean."""
+ model = worker.build(kind)
+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)
+ xs, ys = zip(*(worker.batch(rank) for rank in range(2)))
+ loss = worker.loss_fn(model, torch.cat(xs), torch.cat(ys))
+ loss.backward()
+ grads = [p.grad.clone() for p in model.parameters()]
+ opt.step()
+ return grads, [p.detach().clone() for p in model.parameters()]
+
+
+def _assert_parity(
+ case,
+ kind,
+ exact = True,
+):
+ ref_grads, ref_params = _reference(kind)
+ tol = {} if exact else dict(rtol = 2e-2, atol = 2e-3)
+ for g, r in zip(case["grads"], ref_grads):
+ torch.testing.assert_close(
+ torch.tensor(g), r, **({"rtol": 1e-6, "atol": 1e-7} if exact else tol)
+ )
+ for p, r in zip(case["params"], ref_params):
+ torch.testing.assert_close(
+ torch.tensor(p), r, **({"rtol": 1e-6, "atol": 1e-7} if exact else tol)
+ )
+
+
+def test_full_finetune_gets_bucket_view(ranks):
+ for rank in (0, 1):
+ case = ranks[rank]["full"]
+ assert case["installed"] and case["is_ddp"]
+ assert case["view"] is True
+ assert case["n_grad_storages"] == 1
+ assert case["hooks"] == 0
+ _assert_parity(case, "full")
+
+
+def test_kill_switch_restores_torch_default_bitwise(ranks):
+ for rank in (0, 1):
+ on, off = ranks[rank]["full"], ranks[rank]["full_off"]
+ assert off["view"] is False
+ assert off["n_grad_storages"] > 1
+ assert on["losses"] == off["losses"]
+ assert on["grads"] == off["grads"]
+ assert on["params"] == off["params"]
+
+
+def test_lora_keeps_gradients_unaliased(ranks):
+ case = ranks[0]["peft"]
+ assert case["view"] is False
+ _assert_parity(case, "peft")
+
+
+def test_comm_hook_opt_in(ranks):
+ for name, kind in (("bf16_hook", "full"), ("fp16_hook", "peft")):
+ case = ranks[0][name]
+ assert case["hooks"] >= 1, name
+ _assert_parity(case, kind, exact = False)
+ assert ranks[0]["full"]["hooks"] == 0
+
+
+def test_trainer_init_installs_the_defaults():
+ tree = ast.parse(_UTILS.read_text(encoding = "utf-8"))
+ init = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_trainer_init"
+ ]
+ assert len(init) == 1
+ calls = [ast.unparse(n.func) for n in ast.walk(init[0]) if isinstance(n, ast.Call)]
+ assert "install_unsloth_ddp_comm_defaults" in calls
+ assert calls.index("_original_trainer_init") < calls.index("install_unsloth_ddp_comm_defaults")
diff --git a/unsloth/models/_ddp_defaults.py b/unsloth/models/_ddp_defaults.py
index ef228f4eeac..b4bb498641d 100644
--- a/unsloth/models/_ddp_defaults.py
+++ b/unsloth/models/_ddp_defaults.py
@@ -14,7 +14,10 @@ Without checkpointing True works (and tolerates an adapter a step never runs), s
import os
-__all__ = ["unsloth_default_ddp_find_unused_parameters"]
+__all__ = [
+ "unsloth_default_ddp_find_unused_parameters",
+ "install_unsloth_ddp_comm_defaults",
+]
_AUTO_FLAG = "_unsloth_auto_ddp_find_unused_parameters"
# LoRA tensors that are used whenever their base layer runs.
@@ -185,3 +188,95 @@ def unsloth_default_ddp_find_unused_parameters(args, model):
args.ddp_find_unused_parameters = False
object.__setattr__(args, _AUTO_FLAG, True)
return args.ddp_find_unused_parameters
+
+
+_COMM_HOOKS = ("bf16", "fp16")
+_COMM_PRINTED = set()
+
+
+def _note(key, message):
+ if key not in _COMM_PRINTED:
+ _COMM_PRINTED.add(key)
+ print(message)
+
+
+def _plain_ddp(accelerator):
+ """True when accelerate would wrap the model in torch DDP (not FSDP / DeepSpeed / Megatron / XLA)."""
+ distributed_type = str(getattr(getattr(accelerator, "distributed_type", None), "value", ""))
+ if not distributed_type.startswith("MULTI_"):
+ return False
+ state = getattr(accelerator, "state", None)
+ for plugin in ("fsdp_plugin", "deepspeed_plugin", "megatron_lm_plugin"):
+ if (
+ getattr(state, plugin, None) is not None
+ or getattr(accelerator, plugin, None) is not None
+ ):
+ return False
+ return True
+
+
+def _is_full_finetune(model):
+ if model is None or getattr(model, "peft_config", None) is not None:
+ return False
+ return any(p.requires_grad for p in model.parameters())
+
+
+def _prepare_ddp_handler(accelerator, model):
+ from accelerate.utils import DDPCommunicationHookType, DistributedDataParallelKwargs
+
+ handler = accelerator.ddp_handler
+ if handler is None:
+ handler = DistributedDataParallelKwargs()
+ if not isinstance(handler, DistributedDataParallelKwargs):
+ return
+ changed = False
+ # Full finetuning only: gradients become bucket views, one copy instead of two.
+ if (
+ os.environ.get("UNSLOTH_DDP_GRADIENT_AS_BUCKET_VIEW", "1") != "0"
+ and not handler.gradient_as_bucket_view
+ and _is_full_finetune(model)
+ ):
+ handler.gradient_as_bucket_view = True
+ changed = True
+ _note("view", "Unsloth: DDP gradient_as_bucket_view = True (full finetuning).")
+ # Opt-in: rounds gradients before the average.
+ hook = os.environ.get("UNSLOTH_DDP_COMM_HOOK", "").strip().lower()
+ if hook in _COMM_HOOKS and handler.comm_hook == DDPCommunicationHookType.NO:
+ handler.comm_hook = (
+ DDPCommunicationHookType.BF16 if hook == "bf16" else DDPCommunicationHookType.FP16
+ )
+ changed = True
+ _note("hook", f"Unsloth: DDP gradient all-reduce in {hook} (UNSLOTH_DDP_COMM_HOOK={hook}).")
+ elif hook and hook not in _COMM_HOOKS and hook not in ("0", "none", "off"):
+ _note("hook_bad", f"Unsloth: UNSLOTH_DDP_COMM_HOOK={hook} ignored (use bf16 or fp16).")
+ if changed:
+ accelerator.ddp_handler = handler
+
+
+def install_unsloth_ddp_comm_defaults(trainer):
+ """Wrap ``accelerator.prepare_model``: transformers 4.x replaces ``ddp_handler`` in ``_wrap_model``."""
+ accelerator = getattr(trainer, "accelerator", None)
+ try:
+ if accelerator is None or not _plain_ddp(accelerator):
+ return False
+ prepare_model = accelerator.prepare_model
+ if getattr(prepare_model, "_unsloth_ddp_comm_defaults", False):
+ return True
+
+ def _prepare_model_ddp_defaults(*args, **kwargs):
+ model = args[0] if args else kwargs.get("model")
+ # Native FSDP2 shards in Trainer.__init__; accelerate never builds DDP for it.
+ if getattr(model, "_unsloth_fsdp2_native", False) is not True:
+ try:
+ _prepare_ddp_handler(accelerator, model)
+ except Exception as e:
+ _note("err_pre", f"Unsloth: DDP defaults skipped ({e}).")
+ return prepare_model(*args, **kwargs)
+
+ _prepare_model_ddp_defaults._unsloth_ddp_comm_defaults = True
+ _prepare_model_ddp_defaults.__wrapped__ = prepare_model
+ accelerator.prepare_model = _prepare_model_ddp_defaults
+ return True
+ except Exception as e:
+ _note("err", f"Unsloth: DDP defaults skipped ({e}).")
+ return False
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 03d534edbbd..b86a56c7d80 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5613,6 +5613,9 @@ def patch_gradient_accumulation_fix(Trainer):
_replan_auto_offload_safely(self)
_block_swap_for_trainer_backend(self)
_enable_ddp_batched_grad_copy(self)
+ from ._ddp_defaults import install_unsloth_ddp_comm_defaults
+
+ install_unsloth_ddp_comm_defaults(self)
if os.environ.get("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0") != "0":
from ..optimizers.state_offload import install_optimizer_state_offload
install_optimizer_state_offload(self)