Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
DDP: gradient_as_bucket_view for full finetuning, opt-in bf16 / fp16 gradient all-reduce (#14399)
Daniel Hana3395cf4bed1
diff --git a/tests/_ddp_comm_defaults_worker.py b/tests/_ddp_comm_defaults_worker.pynew file mode 100644index 00000000000..0bd7a43c39d--- /dev/null+++ b/tests/_ddp_comm_defaults_worker.py@@ -0,0 +1,125 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""One gloo rank for test_ddp_comm_defaults.py: a real accelerate Accelerator on CPU builds the DDP."""++from __future__ import annotations++import importlib.util+import os+import types++_DEFAULTS = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "models",+ "_ddp_defaults.py",+)+++def load_defaults():+ # By path: importing the unsloth package needs a GPU.+ spec = importlib.util.spec_from_file_location("_unsloth_ddp_defaults_under_test", _DEFAULTS)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def build(kind):+ import torch++ torch.manual_seed(0)+ layers = [+ torch.nn.Linear(8, 16),+ torch.nn.GELU(),+ torch.nn.Linear(16, 16),+ torch.nn.GELU(),+ torch.nn.Linear(16, 5),+ ]+ model = torch.nn.Sequential(*layers)+ model.register_buffer("scale", torch.tensor(1.5), persistent = False)+ if kind == "peft":+ model.peft_config = {"default": object()}+ return model+++def batch(rank, n = 6):+ import torch+ g = torch.Generator().manual_seed(1234 + rank)+ return torch.randn(n, 8, generator = g), torch.randint(0, 5, (n,), generator = g)+++def loss_fn(model, x, y):+ import torch+ return torch.nn.functional.cross_entropy(model(x) * 1.0, y)+++def run_rank(rank, port, queue, cases):+ import torch+ import torch.distributed as dist++ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ LOCAL_WORLD_SIZE = "2",+ ACCELERATE_USE_CPU = "true",+ )+ torch.set_num_threads(1)+ from accelerate import Accelerator+ from accelerate.state import AcceleratorState, PartialState++ out = {}+ accelerator = Accelerator(cpu = True)+ try:+ defaults = load_defaults()+ for name, kind, env, args in cases:+ saved = {k: os.environ.get(k) for k in env}+ os.environ.update(env)+ try:+ accelerator.ddp_handler = None+ accelerator.prepare_model = types.MethodType(+ type(accelerator).prepare_model, accelerator+ )+ trainer = types.SimpleNamespace(+ accelerator = accelerator, args = types.SimpleNamespace(**args)+ )+ installed = defaults.install_unsloth_ddp_comm_defaults(trainer)+ model = build(kind)+ ddp = accelerator.prepare_model(model)+ opt = torch.optim.SGD(ddp.parameters(), lr = 0.1)+ x, y = batch(rank)+ loss = loss_fn(ddp, x, y)+ loss.backward()+ grads = [p.grad.detach().clone() for p in ddp.module.parameters()]+ storages = {p.grad.untyped_storage().data_ptr() for p in ddp.module.parameters()}+ opt.step()+ losses = [None, None]+ dist.all_gather_object(losses, float(loss))+ out[name] = dict(+ installed = installed,+ is_ddp = isinstance(ddp, torch.nn.parallel.DistributedDataParallel),+ view = bool(ddp.gradient_as_bucket_view),+ hooks = len(getattr(ddp, "_comm_hooks", []) or [])+ or int(getattr(ddp, "_unsloth_test_hook", 0)),+ n_grad_storages = len(storages),+ losses = losses,+ grads = [g.tolist() for g in grads],+ params = [p.detach().tolist() for p in ddp.module.parameters()],+ )+ finally:+ for k, v in saved.items():+ if v is None:+ os.environ.pop(k, None)+ else:+ os.environ[k] = v+ queue.put((rank, out))+ except BaseException:+ import traceback+ queue.put((rank, {"error": traceback.format_exc()}))+ raise+ finally:+ dist.destroy_process_group()+ AcceleratorState._reset_state(True)+ PartialState._reset_state()diff --git a/tests/test_ddp_comm_defaults.py b/tests/test_ddp_comm_defaults.pynew file mode 100644index 00000000000..6e75f226318--- /dev/null+++ b/tests/test_ddp_comm_defaults.py@@ -0,0 +1,132 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""DDP comm defaults on a real 2-rank gloo accelerate DDP must match one process on the joined batch."""++from __future__ import annotations++import ast+import os+import pathlib+import socket+import sys++import pytest+import torch++pytest.importorskip("accelerate")+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # spawned ranks inherit sys.path+import _ddp_comm_defaults_worker as worker # noqa: E402++pytestmark = pytest.mark.skipif(sys.platform == "win32", reason = "two gloo ranks over TCP localhost")++CASES = [+ ("full", "full", {}, {}),+ ("full_off", "full", {"UNSLOTH_DDP_GRADIENT_AS_BUCKET_VIEW": "0"}, {}),+ ("peft", "peft", {}, {}),+ ("bf16_hook", "full", {"UNSLOTH_DDP_COMM_HOOK": "bf16"}, {}),+ ("fp16_hook", "peft", {"UNSLOTH_DDP_COMM_HOOK": "fp16"}, {}),+]+_UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]++[email protected](scope = "module")+def ranks():+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = worker.run_rank, args = (rank, port, queue, CASES)) for rank in range(2)+ ]+ for p in procs:+ p.start()+ results = dict(queue.get(timeout = 600) for _ in procs)+ for rank, out in results.items():+ assert "error" not in out, out["error"]+ for p in procs:+ p.join(120)+ assert p.exitcode == 0+ return results+++def _reference(kind):+ """Single process, both ranks' samples: the mean of two equal-size rank means is the global mean."""+ model = worker.build(kind)+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)+ xs, ys = zip(*(worker.batch(rank) for rank in range(2)))+ loss = worker.loss_fn(model, torch.cat(xs), torch.cat(ys))+ loss.backward()+ grads = [p.grad.clone() for p in model.parameters()]+ opt.step()+ return grads, [p.detach().clone() for p in model.parameters()]+++def _assert_parity(+ case,+ kind,+ exact = True,+):+ ref_grads, ref_params = _reference(kind)+ tol = {} if exact else dict(rtol = 2e-2, atol = 2e-3)+ for g, r in zip(case["grads"], ref_grads):+ torch.testing.assert_close(+ torch.tensor(g), r, **({"rtol": 1e-6, "atol": 1e-7} if exact else tol)+ )+ for p, r in zip(case["params"], ref_params):+ torch.testing.assert_close(+ torch.tensor(p), r, **({"rtol": 1e-6, "atol": 1e-7} if exact else tol)+ )+++def test_full_finetune_gets_bucket_view(ranks):+ for rank in (0, 1):+ case = ranks[rank]["full"]+ assert case["installed"] and case["is_ddp"]+ assert case["view"] is True+ assert case["n_grad_storages"] == 1+ assert case["hooks"] == 0+ _assert_parity(case, "full")+++def test_kill_switch_restores_torch_default_bitwise(ranks):+ for rank in (0, 1):+ on, off = ranks[rank]["full"], ranks[rank]["full_off"]+ assert off["view"] is False+ assert off["n_grad_storages"] > 1+ assert on["losses"] == off["losses"]+ assert on["grads"] == off["grads"]+ assert on["params"] == off["params"]+++def test_lora_keeps_gradients_unaliased(ranks):+ case = ranks[0]["peft"]+ assert case["view"] is False+ _assert_parity(case, "peft")+++def test_comm_hook_opt_in(ranks):+ for name, kind in (("bf16_hook", "full"), ("fp16_hook", "peft")):+ case = ranks[0][name]+ assert case["hooks"] >= 1, name+ _assert_parity(case, kind, exact = False)+ assert ranks[0]["full"]["hooks"] == 0+++def test_trainer_init_installs_the_defaults():+ tree = ast.parse(_UTILS.read_text(encoding = "utf-8"))+ init = [+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_trainer_init"+ ]+ assert len(init) == 1+ calls = [ast.unparse(n.func) for n in ast.walk(init[0]) if isinstance(n, ast.Call)]+ assert "install_unsloth_ddp_comm_defaults" in calls+ assert calls.index("_original_trainer_init") < calls.index("install_unsloth_ddp_comm_defaults")diff --git a/unsloth/models/_ddp_defaults.py b/unsloth/models/_ddp_defaults.pyindex ef228f4eeac..b4bb498641d 100644--- a/unsloth/models/_ddp_defaults.py+++ b/unsloth/models/_ddp_defaults.py@@ -14,7 +14,10 @@ Without checkpointing True works (and tolerates an adapter a step never runs), simport os-__all__ = ["unsloth_default_ddp_find_unused_parameters"]+__all__ = [+ "unsloth_default_ddp_find_unused_parameters",+ "install_unsloth_ddp_comm_defaults",+]_AUTO_FLAG = "_unsloth_auto_ddp_find_unused_parameters"# LoRA tensors that are used whenever their base layer runs.@@ -185,3 +188,95 @@ def unsloth_default_ddp_find_unused_parameters(args, model):args.ddp_find_unused_parameters = Falseobject.__setattr__(args, _AUTO_FLAG, True)return args.ddp_find_unused_parameters+++_COMM_HOOKS = ("bf16", "fp16")+_COMM_PRINTED = set()+++def _note(key, message):+ if key not in _COMM_PRINTED:+ _COMM_PRINTED.add(key)+ print(message)+++def _plain_ddp(accelerator):+ """True when accelerate would wrap the model in torch DDP (not FSDP / DeepSpeed / Megatron / XLA)."""+ distributed_type = str(getattr(getattr(accelerator, "distributed_type", None), "value", ""))+ if not distributed_type.startswith("MULTI_"):+ return False+ state = getattr(accelerator, "state", None)+ for plugin in ("fsdp_plugin", "deepspeed_plugin", "megatron_lm_plugin"):+ if (+ getattr(state, plugin, None) is not None+ or getattr(accelerator, plugin, None) is not None+ ):+ return False+ return True+++def _is_full_finetune(model):+ if model is None or getattr(model, "peft_config", None) is not None:+ return False+ return any(p.requires_grad for p in model.parameters())+++def _prepare_ddp_handler(accelerator, model):+ from accelerate.utils import DDPCommunicationHookType, DistributedDataParallelKwargs++ handler = accelerator.ddp_handler+ if handler is None:+ handler = DistributedDataParallelKwargs()+ if not isinstance(handler, DistributedDataParallelKwargs):+ return+ changed = False+ # Full finetuning only: gradients become bucket views, one copy instead of two.+ if (+ os.environ.get("UNSLOTH_DDP_GRADIENT_AS_BUCKET_VIEW", "1") != "0"+ and not handler.gradient_as_bucket_view+ and _is_full_finetune(model)+ ):+ handler.gradient_as_bucket_view = True+ changed = True+ _note("view", "Unsloth: DDP gradient_as_bucket_view = True (full finetuning).")+ # Opt-in: rounds gradients before the average.+ hook = os.environ.get("UNSLOTH_DDP_COMM_HOOK", "").strip().lower()+ if hook in _COMM_HOOKS and handler.comm_hook == DDPCommunicationHookType.NO:+ handler.comm_hook = (+ DDPCommunicationHookType.BF16 if hook == "bf16" else DDPCommunicationHookType.FP16+ )+ changed = True+ _note("hook", f"Unsloth: DDP gradient all-reduce in {hook} (UNSLOTH_DDP_COMM_HOOK={hook}).")+ elif hook and hook not in _COMM_HOOKS and hook not in ("0", "none", "off"):+ _note("hook_bad", f"Unsloth: UNSLOTH_DDP_COMM_HOOK={hook} ignored (use bf16 or fp16).")+ if changed:+ accelerator.ddp_handler = handler+++def install_unsloth_ddp_comm_defaults(trainer):+ """Wrap ``accelerator.prepare_model``: transformers 4.x replaces ``ddp_handler`` in ``_wrap_model``."""+ accelerator = getattr(trainer, "accelerator", None)+ try:+ if accelerator is None or not _plain_ddp(accelerator):+ return False+ prepare_model = accelerator.prepare_model+ if getattr(prepare_model, "_unsloth_ddp_comm_defaults", False):+ return True++ def _prepare_model_ddp_defaults(*args, **kwargs):+ model = args[0] if args else kwargs.get("model")+ # Native FSDP2 shards in Trainer.__init__; accelerate never builds DDP for it.+ if getattr(model, "_unsloth_fsdp2_native", False) is not True:+ try:+ _prepare_ddp_handler(accelerator, model)+ except Exception as e:+ _note("err_pre", f"Unsloth: DDP defaults skipped ({e}).")+ return prepare_model(*args, **kwargs)++ _prepare_model_ddp_defaults._unsloth_ddp_comm_defaults = True+ _prepare_model_ddp_defaults.__wrapped__ = prepare_model+ accelerator.prepare_model = _prepare_model_ddp_defaults+ return True+ except Exception as e:+ _note("err", f"Unsloth: DDP defaults skipped ({e}).")+ return Falsediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 03d534edbbd..b86a56c7d80 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5613,6 +5613,9 @@ def patch_gradient_accumulation_fix(Trainer):_replan_auto_offload_safely(self)_block_swap_for_trainer_backend(self)_enable_ddp_batched_grad_copy(self)+ from ._ddp_defaults import install_unsloth_ddp_comm_defaults++ install_unsloth_ddp_comm_defaults(self)if os.environ.get("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0") != "0":from ..optimizers.state_offload import install_optimizer_state_offloadinstall_optimizer_state_offload(self)