Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix 4-bit VLM DDP training crash on ranks >= 1 (CheckpointError) (#13149)

Daniel Hane8f391a5d054
diff --git a/tests/test_ddp_rank_dispatch_hooks.py b/tests/test_ddp_rank_dispatch_hooks.py
new file mode 100644
index 00000000000..8369a56cef5
--- /dev/null
+++ b/tests/test_ddp_rank_dispatch_hooks.py
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""#3459: a DDP rank on its own card gets no dispatch hooks; one process on a non-default card (#5068) keeps them."""
+
+import types
+
+import pytest
+import torch
+
+pytest.importorskip("accelerate")
+
+
+def _run(monkeypatch, devices, distributed):
+ import accelerate
+ from unsloth.models import vision
+ from unsloth.models import loader_utils
+
+ calls = []
+ monkeypatch.setattr(accelerate, "dispatch_model", lambda model, **kw: calls.append(kw))
+ monkeypatch.setattr(vision, "is_distributed", lambda: distributed)
+ monkeypatch.setattr(vision, "_repair_dispatch_hooks", lambda model: 0)
+ monkeypatch.setattr(loader_utils, "no_placement_tensor_names", lambda model: set())
+ monkeypatch.setattr(
+ vision,
+ "_infer_device_map_from_loaded_model",
+ lambda model, skip = None: {f"layer{i}": d for i, d in enumerate(devices)},
+ )
+ params = [(f"layer{i}.weight", types.SimpleNamespace(device = d)) for i, d in enumerate(devices)]
+ model = types.SimpleNamespace(
+ is_loaded_in_4bit = True, hf_device_map = None, named_parameters = lambda: iter(params)
+ )
+ vision._attach_bnb_multidevice_hooks(model, True, False, False, False)
+ return calls
+
+
+def test_a_distributed_rank_on_its_own_card_gets_no_hooks(monkeypatch):
+ assert _run(monkeypatch, [torch.device("cuda", 1)] * 2, distributed = True) == []
+
+
+def test_a_single_process_on_a_non_default_card_keeps_its_hooks(monkeypatch):
+ assert len(_run(monkeypatch, [torch.device("cuda", 1)] * 2, distributed = False)) == 1
+
+
+def test_a_model_split_across_cards_keeps_its_hooks_under_a_distributed_launch(monkeypatch):
+ calls = _run(monkeypatch, [torch.device("cuda", 0), torch.device("cuda", 1)], distributed = True)
+ assert len(calls) == 1
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 069fd9c3b2b..01d2af1b443 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -555,7 +555,7 @@ def _hook_no_placement_ancestors(model):
def _attach_bnb_multidevice_hooks(
model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference
):
- """Attach accelerate AlignDevicesHook on a bnb model loaded across multiple devices (or a non-default device). No-op for single-GPU cuda:0, non-bnb, vLLM, or already-dispatched models."""
+ """Attach accelerate AlignDevicesHook on a bnb model loaded across multiple devices (or a non-default device). No-op for single-GPU cuda:0, a distributed rank on its own device, non-bnb, vLLM, or already-dispatched models."""
if fast_inference:
return
# Before the bnb gate: the rebuild happens whatever the quantization. Not under offload, where _embedding_dispatch_device READS this hook to place the ids.
@@ -604,6 +604,9 @@ def _attach_bnb_multidevice_hooks(
default_cuda = torch.device("cuda", 0)
if all_devs == {default_cuda}:
return
+ # A distributed rank holds its whole model on its own card, and the trainer already moves batches there. The hooks would only graph-break the compiled forward on ranks >= 1, so the non-reentrant checkpoint recompute stops matching the forward (#3459).
+ if len(all_devs) == 1 and is_distributed():
+ return
try:
from accelerate import dispatch_model