Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

DeepSpeed ZeRO-2 / ZeRO-3: stage detection at load, MoE and fused LoRA leaves, precision resync, reentrant checkpointing (#14394)

Daniel Hanb098d77e8107
diff --git a/tests/test_deepspeed_zero3.py b/tests/test_deepspeed_zero3.py
index 71e558d2bfc..93053a89d13 100644
--- a/tests/test_deepspeed_zero3.py
+++ b/tests/test_deepspeed_zero3.py
@@ -51,3 +51,374 @@ def test_without_zero3_the_device_map_is_untouched(zero3):
zero3(False)
for device_map in (loader_utils.DEFAULT_DEVICE_MAP, "sequential", {"": "cuda:1"}, None):
assert loader_utils.deepspeed_zero3_device_map(device_map) is device_map
+
+
+_DS_ENV = (
+ "ACCELERATE_USE_DEEPSPEED",
+ "ACCELERATE_DEEPSPEED_ZERO_STAGE",
+ "ACCELERATE_DEEPSPEED_CONFIG_FILE",
+)
+
+
+def no_ds_env(monkeypatch):
+ for key in _DS_ENV:
+ monkeypatch.delenv(key, raising = False)
+ import transformers.integrations.deepspeed as hf_ds
+
+ monkeypatch.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", None, raising = False)
+ try:
+ from accelerate.state import AcceleratorState
+ monkeypatch.setitem(AcceleratorState._shared_state, "deepspeed_plugins", None)
+ except Exception:
+ pass
+ return monkeypatch
+
+
+def test_zero_stage_unset_without_deepspeed(no_ds_env):
+ assert loader_utils.deepspeed_zero_stage() is None
+ assert not loader_utils.deepspeed_zero3_requested()
+
+
[email protected]("stage", ["2", "3"])
+def test_zero_stage_from_accelerate_env(no_ds_env, stage):
+ no_ds_env.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ no_ds_env.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", stage)
+ assert loader_utils.deepspeed_zero_stage() == int(stage)
+ assert loader_utils.deepspeed_zero3_requested() == (stage == "3")
+
+
+def test_zero_stage_from_accelerate_config_file(no_ds_env, tmp_path):
+ import json
+
+ path = tmp_path / "ds.json"
+ path.write_text(json.dumps({"zero_optimization": {"stage": 3}}))
+ no_ds_env.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ no_ds_env.setenv("ACCELERATE_DEEPSPEED_CONFIG_FILE", str(path))
+ assert loader_utils.deepspeed_zero3_requested()
+ no_ds_env.delenv("ACCELERATE_USE_DEEPSPEED")
+ assert loader_utils.deepspeed_zero_stage() is None
+
+
+def test_zero_stage_from_training_arguments_config(no_ds_env):
+ import weakref
+ import transformers.integrations.deepspeed as hf_ds
+
+ class _Cfg:
+ config = {"zero_optimization": {"stage": 3}}
+
+ cfg = _Cfg()
+ no_ds_env.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", weakref.ref(cfg))
+ assert loader_utils.deepspeed_zero3_requested()
+
+
+def _tiny_qwen3_moe():
+ torch = pytest.importorskip("torch")
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+
+ config = Qwen3MoeConfig(
+ vocab_size = 64,
+ hidden_size = 16,
+ intermediate_size = 32,
+ moe_intermediate_size = 8,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 8,
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ )
+ with torch.device("meta"):
+ return Qwen3MoeForCausalLM(config)
+
+
+def fake_set_z3_leaf(monkeypatch):
+ import sys
+ import types
+
+ calls = []
+
+ def set_z3_leaf_modules(
+ model,
+ classes,
+ raise_if_not_found = True,
+ ):
+ calls.append(list(classes))
+ out = []
+ for module in model.modules():
+ if isinstance(module, tuple(classes)):
+ module._z3_leaf = True
+ out.append(module)
+ return out
+
+ utils = types.ModuleType("deepspeed.utils")
+ utils.set_z3_leaf_modules = set_z3_leaf_modules
+ ds = types.ModuleType("deepspeed")
+ ds.utils = utils
+ monkeypatch.setitem(sys.modules, "deepspeed", ds)
+ monkeypatch.setitem(sys.modules, "deepspeed.utils", utils)
+ return calls
+
+
+def test_moe_blocks_become_zero3_leaves(no_ds_env, fake_set_z3_leaf):
+ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)
+ model = _tiny_qwen3_moe()
+ names = loader_utils.set_deepspeed_moe_leaf_modules(model)
+ assert names == ("Qwen3MoeSparseMoeBlock",), names
+ blocks = [m for m in model.modules() if type(m).__name__ == "Qwen3MoeSparseMoeBlock"]
+ assert len(blocks) == 2 and all(getattr(m, "_z3_leaf", False) for m in blocks)
+ assert sum(bool(getattr(m, "_z3_leaf", False)) for m in model.modules()) == 2
+
+
+def test_moe_leaf_marking_is_zero3_only(no_ds_env, fake_set_z3_leaf):
+ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: False)
+ model = _tiny_qwen3_moe()
+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()
+ assert fake_set_z3_leaf == []
+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())
+
+
+def test_dense_model_has_no_moe_leaf(no_ds_env, fake_set_z3_leaf):
+ torch = pytest.importorskip("torch")
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)
+ with torch.device("meta"):
+ model = LlamaForCausalLM(
+ LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ )
+ )
+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()
+ assert fake_set_z3_leaf == []
+
+
+def _filled_args(
+ fp16,
+ bf16,
+ ds_fp16,
+ ds_bf16,
+ shared = True,
+):
+ import types
+
+ config = {
+ "fp16": {"enabled": ds_fp16},
+ "bf16": {"enabled": ds_bf16},
+ "zero_optimization": {"stage": 3},
+ }
+ plugin = types.SimpleNamespace(deepspeed_config = config)
+ hf = types.SimpleNamespace(config = config if shared else dict(config))
+ return types.SimpleNamespace(
+ fp16 = fp16,
+ bf16 = bf16,
+ fp16_full_eval = False,
+ bf16_full_eval = False,
+ deepspeed_plugin = plugin,
+ hf_deepspeed_config = hf,
+ ), config
+
+
+def test_forced_float32_disables_the_filled_deepspeed_fp16():
+ # T4 + Gemma3 / Qwen3-MoE: fp16 = True was filled in, then Unsloth switched to float32.
+ args, config = _filled_args(fp16 = False, bf16 = False, ds_fp16 = True, ds_bf16 = False)
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == ["fp16"]
+ assert config["fp16"]["enabled"] is False and config["bf16"]["enabled"] is False
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []
+
+
+def test_resync_clears_queued_precision_mismatches_and_dtype():
+ import inspect
+ import types
+
+ import torch
+
+ from transformers.integrations import deepspeed as hf_ds
+
+ assert 'self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}")' in (
+ inspect.getsource(hf_ds)
+ )
+ args, config = _filled_args(fp16 = False, bf16 = False, ds_fp16 = True, ds_bf16 = False)
+ args.hf_deepspeed_config = types.SimpleNamespace(
+ config = config,
+ mismatches = [
+ "- ds fp16.enabled=True vs hf fp16|fp16_full_eval=False",
+ "- ds train_micro_batch_size_per_gpu=4 vs hf per_device_train_batch_size=2",
+ ],
+ _dtype = torch.float16,
+ )
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == ["fp16"]
+ assert args.hf_deepspeed_config.mismatches == [
+ "- ds train_micro_batch_size_per_gpu=4 vs hf per_device_train_batch_size=2"
+ ]
+ assert args.hf_deepspeed_config._dtype is torch.float32
+
+
+def test_resync_dtype_follows_the_flags_when_sections_are_absent():
+ import types
+
+ import torch
+
+ config = {"zero_optimization": {"stage": 3}}
+ args = types.SimpleNamespace(
+ fp16 = False,
+ bf16 = True,
+ fp16_full_eval = False,
+ bf16_full_eval = False,
+ deepspeed_plugin = types.SimpleNamespace(deepspeed_config = config),
+ hf_deepspeed_config = types.SimpleNamespace(
+ config = config, mismatches = [], _dtype = torch.float32
+ ),
+ )
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []
+ assert args.hf_deepspeed_config._dtype is torch.bfloat16
+
+
+def test_auto_and_absent_sections_are_left_to_transformers():
+ import types
+
+ config = {"fp16": {"enabled": "auto"}}
+ args = types.SimpleNamespace(
+ fp16 = False,
+ bf16 = True,
+ deepspeed_plugin = types.SimpleNamespace(deepspeed_config = config),
+ hf_deepspeed_config = None,
+ )
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []
+ assert config == {"fp16": {"enabled": "auto"}}
+
+
+def test_no_deepspeed_config_is_a_no_op():
+ import types
+ args = types.SimpleNamespace(fp16 = True, bf16 = False)
+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []
+
+
+def test_rl_trainer_source_resyncs_deepspeed_after_the_precision_switch():
+ """The generated Unsloth*Trainer __init__ records the requested precision before its switches."""
+ import inspect
+ from unsloth.models import rl
+
+ source = inspect.getsource(rl)
+ first = source.index("_unsloth_precision_requested = tuple(")
+ switch = source.index("Switching to float32 training since model cannot work with float16")
+ sync = source.index("sync_deepspeed_mixed_precision(args)")
+ assert first < switch < sync
+
+
+ "distributed, fsdp2, stage, reentrant",
+ [
+ (False, False, None, True), # one GPU
+ (True, False, None, False), # DDP: non-reentrant (#3713)
+ (True, True, None, True), # FSDP2
+ (True, False, 3, True), # DeepSpeed ZeRO-3: non-reentrant recompute fails its checks
+ (True, False, 2, True), # DeepSpeed ZeRO-2: no DDP reducer either
+ ],
+)
+def test_vlm_gradient_checkpointing_reentrant(monkeypatch, distributed, fsdp2, stage, reentrant):
+ monkeypatch.setattr(loader_utils, "is_distributed", lambda: distributed)
+ monkeypatch.setattr(loader_utils, "fsdp2_requested", lambda: fsdp2)
+ monkeypatch.setattr(loader_utils, "deepspeed_zero_stage", lambda: stage)
+ assert loader_utils.vlm_gradient_checkpointing_reentrant() is reentrant
+
+
+def test_fused_lora_layers_become_zero3_leaves_only_under_zero3(monkeypatch):
+ torch = pytest.importorskip("torch")
+ layers = torch.nn.ModuleList([torch.nn.Linear(2, 2), torch.nn.Linear(2, 2)])
+ monkeypatch.setattr(loader_utils, "deepspeed_zero3_requested", lambda: False)
+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers) == 0
+ assert not any(getattr(m, "_z3_leaf", False) for m in layers)
+ monkeypatch.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)
+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers) == 2
+ assert all(m._z3_leaf for m in layers)
+ try:
+ from deepspeed.utils import z3_leaf_module
+ except Exception:
+ return
+ # The flag DeepSpeed's hook installer reads (deepspeed/utils/z3_leaf_module.py).
+ assert all(z3_leaf_module(m) for m in layers)
+
+
+def test_only_layers_with_installed_fused_kernels_are_leaves():
+ import types
+
+ torch = pytest.importorskip("torch")
+ from unsloth.models import llama
+
+ def fused_mlp(self, x):
+ return x
+
+ def layer(
+ qkv = None,
+ o = None,
+ mlp = False,
+ ):
+ m = torch.nn.Module()
+ m.self_attn = torch.nn.Module()
+ m.self_attn.apply_qkv = qkv or (lambda *a: None)
+ m.self_attn.apply_o = o or (lambda *a: None)
+ m.mlp = torch.nn.Module()
+ if mlp:
+ m.mlp.forward = types.MethodType(fused_mlp, m.mlp)
+ return m
+
+ # Qwen2 with a q_proj-only adapter: nothing fused, nothing gathered as a whole block.
+ plain = layer()
+ qkv = layer(qkv = llama.apply_lora_qkv)
+ o = layer(o = llama.apply_lora_o)
+ mlp = layer(mlp = True)
+ assert llama._fused_lora_layers([plain, qkv, o, mlp], fused_mlp) == [qkv, o, mlp]
+
+
+def test_trainer_hook_redoes_zero3_setup_for_a_model_loaded_before_deepspeed(
+ no_ds_env, fake_set_z3_leaf, capsys
+):
+ """Loaded and PEFT-patched before TrainingArguments(deepspeed=...): the trainer hook marks the leaves."""
+ import weakref
+ import transformers.integrations.deepspeed as hf_ds
+
+ model = _tiny_qwen3_moe()
+ layers = list(model.model.layers)
+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()
+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers[:1]) == 0
+ model._unsloth_vlm_gc_non_reentrant = True
+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())
+
+ class _Cfg:
+ config = {"zero_optimization": {"stage": 3}}
+
+ cfg = _Cfg()
+ no_ds_env.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", weakref.ref(cfg))
+ for _ in range(2):
+ loader_utils.apply_deepspeed_zero3_model_setup(model)
+ blocks = [m for m in model.modules() if type(m).__name__ == "Qwen3MoeSparseMoeBlock"]
+ assert len(blocks) == 2 and all(m._z3_leaf for m in blocks)
+ assert layers[0]._z3_leaf and not getattr(layers[1], "_z3_leaf", False)
+ assert sum(bool(getattr(m, "_z3_leaf", False)) for m in model.modules()) == 3
+ assert capsys.readouterr().out.count("non-reentrant gradient checkpointing") == 1
+
+
+def test_trainer_hook_is_a_no_op_without_zero3(no_ds_env, fake_set_z3_leaf):
+ model = _tiny_qwen3_moe()
+ loader_utils.set_deepspeed_fused_lora_leaf_modules(list(model.model.layers))
+ loader_utils.apply_deepspeed_zero3_model_setup(model)
+ loader_utils.apply_deepspeed_zero3_model_setup("not-a-model")
+ assert fake_set_z3_leaf == []
+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())
+
+
+def test_rl_trainer_source_redoes_zero3_setup():
+ import inspect
+ from unsloth.models import rl
+
+ source = inspect.getsource(rl)
+ assert source.index("sync_deepspeed_mixed_precision(args)") < source.index(
+ "apply_deepspeed_zero3_model_setup(model)"
+ )
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index e63fa142401..36eafe10b31 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -60,6 +60,8 @@ except ImportError: # unsloth_zoo predating the helper (unsloth-zoo#1751)
from .loader_utils import (
DEFAULT_DEVICE_MAP,
+ set_deepspeed_fused_lora_leaf_modules,
+ set_deepspeed_moe_leaf_modules,
OFFLOAD_EMBEDDING_AUTO,
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
@@ -3131,6 +3133,21 @@ def _decline_fused_lora_for_fsdp(model, reason = None) -> int:
return n
+def _fused_lora_layers(layers, mlp_function):
+ """Decoder layers with a fused LoRA kernel installed (QKV, O or MLP)."""
+ return [
+ layer
+ for layer in layers
+ if getattr(getattr(layer, "self_attn", None), "apply_qkv", None) is apply_lora_qkv
+ or getattr(getattr(layer, "self_attn", None), "apply_o", None) is apply_lora_o
+ or any(
+ getattr(getattr(getattr(layer, "mlp", None), name, None), "__func__", None)
+ is mlp_function
+ for name in ("forward", "_unsloth_forward")
+ )
+ ]
+
+
def _legacy_block_compile(model):
"""Per-block compile for FastLanguageModel LoRA. Off before torch 2.11 unless =1: untraced
fused LoRA (fast_lora.TRACE_LORA_FUNCTIONS) splits each block."""
@@ -4043,6 +4060,7 @@ class FastLlamaModel:
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
+ set_deepspeed_moe_leaf_modules(model)
mark_auto_offload_for_ddp(model, _ddp_auto_swap)
attach_offload_layers(
model,
@@ -5151,6 +5169,11 @@ class FastLlamaModel:
f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "
f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",
)
+ if n_qkv + n_o + n_mlp > 0:
+ # Only layers whose fused kernels were installed (the qwen2 fallback counts unfused QKV).
+ set_deepspeed_fused_lora_leaf_modules(
+ _fused_lora_layers(model.model.model.layers, _apply_lora_mlp)
+ )
patch_saving_functions(model)
# Patch cross entropy loss labels; fixes #10.
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index c6b267da4d0..9f29a8109cb 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -244,6 +244,159 @@ def deepspeed_zero3_device_map(device_map):
return None if is_automatic_device_map(device_map) else device_map
+def deepspeed_zero_stage():
+ """DeepSpeed ZeRO stage, else None. Unsloth loads before TrainingArguments, so `accelerate launch` env counts."""
+ try:
+ from transformers.integrations.deepspeed import deepspeed_config
+ config = deepspeed_config()
+ except Exception:
+ config = None
+ if config is None:
+ try:
+ from accelerate.state import AcceleratorState
+
+ plugins = AcceleratorState._shared_state.get("deepspeed_plugins", None) or {}
+ if not isinstance(plugins, dict):
+ plugins = {"": plugins}
+ for plugin in plugins.values():
+ if (
+ getattr(plugin, "selected", True)
+ and getattr(plugin, "zero_stage", None) is not None
+ ):
+ return int(plugin.zero_stage)
+ except Exception:
+ pass
+ if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() not in _TRUTHY:
+ return None
+ stage = str(os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", "")).strip()
+ if stage.isdigit():
+ return int(stage)
+ path = os.environ.get("ACCELERATE_DEEPSPEED_CONFIG_FILE", "")
+ if not path:
+ return None
+ try:
+ import json
+ with open(path, "r", encoding = "utf-8") as f:
+ config = json.load(f)
+ except Exception:
+ return None
+ try:
+ return int(config.get("zero_optimization", {}).get("stage"))
+ except Exception:
+ return None
+
+
+def deepspeed_zero3_requested():
+ return deepspeed_zero_stage() == 3
+
+
+def sync_deepspeed_mixed_precision(args):
+ """Resync an already-filled DeepSpeed config's fp16 / bf16 after Unsloth changed them, else
+ transformers stops with "ds fp16.enabled=True vs hf fp16|fp16_full_eval=False"."""
+ want = {
+ "fp16": bool(getattr(args, "fp16", False) or getattr(args, "fp16_full_eval", False)),
+ "bf16": bool(getattr(args, "bf16", False) or getattr(args, "bf16_full_eval", False)),
+ }
+ configs = []
+ for holder, attr in (
+ (getattr(args, "hf_deepspeed_config", None), "config"),
+ (getattr(args, "deepspeed_plugin", None), "deepspeed_config"),
+ ):
+ config = getattr(holder, attr, None) if holder is not None else None
+ if isinstance(config, dict) and all(config is not c for c in configs):
+ configs.append(config)
+ changed = []
+ for config in configs:
+ for key, enabled in want.items():
+ section = config.get(key)
+ if not isinstance(section, dict) or section.get("enabled", False) == "auto":
+ continue
+ if bool(section.get("enabled", False)) != enabled:
+ section["enabled"] = enabled
+ changed.append(key)
+ hf_config = getattr(args, "hf_deepspeed_config", None)
+ if hf_config is not None and isinstance(getattr(hf_config, "config", None), dict):
+ # trainer_config_process already queued mismatches for the old flags; the config now agrees.
+ mismatches = getattr(hf_config, "mismatches", None)
+ if isinstance(mismatches, list):
+ mismatches[:] = [
+ m
+ for m in mismatches
+ if not m.startswith(("- ds fp16.enabled=", "- ds bf16.enabled="))
+ ]
+ if hasattr(hf_config, "_dtype"):
+ # From the flags: accelerate adds absent fp16 / bf16 sections later without refreshing _dtype.
+ hf_config._dtype = (
+ torch.bfloat16 if want["bf16"] else torch.float16 if want["fp16"] else torch.float32
+ )
+ return changed
+
+
+def vlm_gradient_checkpointing_reentrant():
+ """Only DDP needs non-reentrant (#3713); under ZeRO-3 non-reentrant fails torch's saved-tensor check."""
+ return not is_distributed() or fsdp2_requested() or deepspeed_zero_stage() is not None
+
+
+def _is_moe_block(module):
+ children = dict(module.named_children())
+ return "experts" in children and ("gate" in children or "router" in children)
+
+
+def set_deepspeed_moe_leaf_modules(model):
+ """Mark MoE blocks ZeRO-3 leaves (before `deepspeed.initialize`): per-expert hooks give mismatched all-gathers across ranks."""
+ if model is None or not deepspeed_zero3_requested():
+ return ()
+ try:
+ from deepspeed.utils import set_z3_leaf_modules
+ except Exception:
+ return ()
+ classes = []
+ for module in model.modules():
+ if type(module) not in classes and _is_moe_block(module):
+ classes.append(type(module))
+ if not classes:
+ return ()
+ try:
+ set_z3_leaf_modules(model, classes)
+ except Exception as exc:
+ print(f"Unsloth: could not mark MoE blocks as DeepSpeed ZeRO-3 leaf modules ({exc}).")
+ return ()
+ return tuple(cls.__name__ for cls in classes)
+
+
+def set_deepspeed_fused_lora_leaf_modules(layers):
+ """Fused LoRA kernels read weights without calling their modules, so ZeRO-3's prefetch trace breaks unless layers are leaves."""
+ layers = list(layers)
+ for layer in layers:
+ layer._unsloth_fused_lora_layer = True
+ if not deepspeed_zero3_requested():
+ return 0
+ for layer in layers:
+ layer._z3_leaf = True
+ return len(layers)
+
+
+def apply_deepspeed_zero3_model_setup(model):
+ """Redo load-time ZeRO-3 setup at trainer time, for models loaded before DeepSpeed was visible."""
+ if not hasattr(model, "modules") or not deepspeed_zero3_requested():
+ return
+ try:
+ set_deepspeed_moe_leaf_modules(model)
+ set_deepspeed_fused_lora_leaf_modules(
+ m for m in model.modules() if getattr(m, "_unsloth_fused_lora_layer", False)
+ )
+ except Exception as exc:
+ print(f"Unsloth: could not apply the DeepSpeed ZeRO-3 model setup ({exc}).")
+ if getattr(model, "_unsloth_vlm_gc_non_reentrant", False) and not getattr(
+ model, "_unsloth_warned_zero3_gc", False
+ ):
+ model._unsloth_warned_zero3_gc = True
+ print(
+ "Unsloth: this model was loaded without DeepSpeed detected, so it uses non-reentrant gradient "
+ "checkpointing, which can fail under ZeRO-3. Load the model under the DeepSpeed launch instead."
+ )
+
+
UNSLOTH_DEVICE_MAP = "unsloth"
# Same planner, different answer when it declines: every veto path ends in "sequential", which fills the first device to its whole free budget, wrong for a caller that picked several cards.
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index ad4175dfa15..00c73a9086a 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2578,6 +2578,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
extra_args = ""
if "args" in call_args and "model" in call_args:
mixed_precision = (
+ "_unsloth_precision_requested = tuple(getattr(args, k, False) for k in ('fp16', 'bf16', 'fp16_full_eval', 'bf16_full_eval'))\n"
"use_bf16 = getattr(args, 'bf16', False)\n"
"if type(use_bf16) is not bool: use_bf16 = False\n"
"use_fp16 = getattr(args, 'fp16', False)\n"
@@ -2695,6 +2696,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" args.fp16_full_eval = args.fp16\n"
)
extra_args += eval_changes
+ if "args" in call_args and "model" in call_args:
+ # The switches above can contradict a DeepSpeed config already filled from the caller's flags.
+ extra_args += (
+ "if tuple(getattr(args, k, False) for k in ('fp16', 'bf16', 'fp16_full_eval', 'bf16_full_eval')) != _unsloth_precision_requested:\n"
+ " from unsloth.models.loader_utils import sync_deepspeed_mixed_precision\n"
+ " sync_deepspeed_mixed_precision(args)\n"
+ "if model is not None:\n"
+ " from unsloth.models.loader_utils import apply_deepspeed_zero3_model_setup\n"
+ " apply_deepspeed_zero3_model_setup(model)\n"
+ )
if "model" in call_args:
# No UNSLOTH_RETURN_LOGITS=1 for compute_metrics here: prediction_step forces it per eval call; here it leaked into training (#3071).
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 73e515ef359..4e68abc3772 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -142,6 +142,8 @@ from .remote_code_shims import apply_remote_code_shims
from .grouped_linear_lora import register_grouped_linear_lora
from .loader_utils import (
_bnb_bits_requested,
+ vlm_gradient_checkpointing_reentrant,
+ set_deepspeed_moe_leaf_modules,
fsdp2_requested,
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
@@ -5073,6 +5075,7 @@ class FastBaseModel:
float32_mixed_precision = None,
):
reject_alora(model)
+ set_deepspeed_moe_leaf_modules(model)
from ..import_fixes import fix_transformers_voxtral_realtime_inplace_audio_add
fix_transformers_voxtral_realtime_inplace_audio_add()
@@ -5088,12 +5091,15 @@ class FastBaseModel:
# VLMs can hit DDP "marked ready twice" with re-entrant checkpointing (#3713), so under DDP skip the offloaded/re-entrant checkpoint patch and default native checkpoint to non-reentrant, offloading via saved-tensor hooks instead.
# FSDP2 has no DDP reducer to mark twice: each unit reduce-scatters in its own post-backward hook,
# recompute included, so it keeps Unsloth's offloaded checkpointing (and unsloth_stream) as 1 GPU does.
- use_reentrant = not is_distributed() or fsdp2_requested()
+ use_reentrant = vlm_gradient_checkpointing_reentrant()
# DeepSeek-V4.1 layers consume tensors from earlier layers; reentrant GC's no_grad pass drops those grads.
_gc_model_type = (getattr(getattr(model, "config", None), "model_type", "") or "").lower()
_force_non_reentrant = _gc_model_type.startswith(_NON_REENTRANT_GC_MODEL_TYPES)
if _force_non_reentrant:
use_reentrant = False
+ model._unsloth_vlm_gc_non_reentrant = bool(
+ use_gradient_checkpointing and not use_reentrant and not _force_non_reentrant
+ )
if not use_reentrant:
unpatch_unsloth_gradient_checkpointing()
unpatch_unsloth_smart_gradient_checkpointing()