Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix 4-bit VLM DDP training crash on ranks >= 1 (CheckpointError) (#13149)
Daniel Hane8f391a5d054
diff --git a/tests/test_ddp_rank_dispatch_hooks.py b/tests/test_ddp_rank_dispatch_hooks.pynew file mode 100644index 00000000000..8369a56cef5--- /dev/null+++ b/tests/test_ddp_rank_dispatch_hooks.py@@ -0,0 +1,46 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""#3459: a DDP rank on its own card gets no dispatch hooks; one process on a non-default card (#5068) keeps them."""++import types++import pytest+import torch++pytest.importorskip("accelerate")+++def _run(monkeypatch, devices, distributed):+ import accelerate+ from unsloth.models import vision+ from unsloth.models import loader_utils++ calls = []+ monkeypatch.setattr(accelerate, "dispatch_model", lambda model, **kw: calls.append(kw))+ monkeypatch.setattr(vision, "is_distributed", lambda: distributed)+ monkeypatch.setattr(vision, "_repair_dispatch_hooks", lambda model: 0)+ monkeypatch.setattr(loader_utils, "no_placement_tensor_names", lambda model: set())+ monkeypatch.setattr(+ vision,+ "_infer_device_map_from_loaded_model",+ lambda model, skip = None: {f"layer{i}": d for i, d in enumerate(devices)},+ )+ params = [(f"layer{i}.weight", types.SimpleNamespace(device = d)) for i, d in enumerate(devices)]+ model = types.SimpleNamespace(+ is_loaded_in_4bit = True, hf_device_map = None, named_parameters = lambda: iter(params)+ )+ vision._attach_bnb_multidevice_hooks(model, True, False, False, False)+ return calls+++def test_a_distributed_rank_on_its_own_card_gets_no_hooks(monkeypatch):+ assert _run(monkeypatch, [torch.device("cuda", 1)] * 2, distributed = True) == []+++def test_a_single_process_on_a_non_default_card_keeps_its_hooks(monkeypatch):+ assert len(_run(monkeypatch, [torch.device("cuda", 1)] * 2, distributed = False)) == 1+++def test_a_model_split_across_cards_keeps_its_hooks_under_a_distributed_launch(monkeypatch):+ calls = _run(monkeypatch, [torch.device("cuda", 0), torch.device("cuda", 1)], distributed = True)+ assert len(calls) == 1diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 069fd9c3b2b..01d2af1b443 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -555,7 +555,7 @@ def _hook_no_placement_ancestors(model):def _attach_bnb_multidevice_hooks(model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference):- """Attach accelerate AlignDevicesHook on a bnb model loaded across multiple devices (or a non-default device). No-op for single-GPU cuda:0, non-bnb, vLLM, or already-dispatched models."""+ """Attach accelerate AlignDevicesHook on a bnb model loaded across multiple devices (or a non-default device). No-op for single-GPU cuda:0, a distributed rank on its own device, non-bnb, vLLM, or already-dispatched models."""if fast_inference:return# Before the bnb gate: the rebuild happens whatever the quantization. Not under offload, where _embedding_dispatch_device READS this hook to place the ids.@@ -604,6 +604,9 @@ def _attach_bnb_multidevice_hooks(default_cuda = torch.device("cuda", 0)if all_devs == {default_cuda}:return+ # A distributed rank holds its whole model on its own card, and the trainer already moves batches there. The hooks would only graph-break the compiled forward on ranks >= 1, so the non-reentrant checkpoint recompute stops matching the forward (#3459).+ if len(all_devs) == 1 and is_distributed():+ returntry:from accelerate import dispatch_model