Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: keep the stock forward under DeepSpeed ZeRO-3 (#14396)

Daniel Han57c2a9485ccf
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index a3b09de69ad..fd82936b22f 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -895,3 +895,54 @@ def test_selective_dequant_off_never_reads_a_packed_expert_stack(monkeypatch):
M._experts_outside(layer)
assert reads == []
assert not getattr(layer.mlp.experts, "_unsloth_block_outside", False)
+
+
+def test_zero3_partitioned_training_keeps_the_stock_forward(monkeypatch):
+ import weakref
+
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(1)
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+
+ class FakeDsConfig:
+ def is_zero3(self):
+ return True
+
+ ds = types.ModuleType("transformers.integrations.deepspeed")
+ config = FakeDsConfig()
+ ds._hf_deepspeed_config_weak_ref = weakref.ref(config)
+ monkeypatch.setitem(sys.modules, ds.__name__, ds)
+
+ cls = _toy_class()
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([cls(), cls()])
+ assert M.patch_block_compile(model) == []
+
+ # ZeRO-3 configured after the load (TrainingArguments built later): the forward checks.
+ ds._hf_deepspeed_config_weak_ref = None
+ assert M.patch_block_compile(model) == [cls.__name__]
+ x = torch.randn(2, 4, 8, requires_grad = True)
+ ds._hf_deepspeed_config_weak_ref = weakref.ref(config)
+ model.layers[0](x).sum().backward()
+ assert calls == []
+ # Parameters partitioned by deepspeed.zero.Init (ds_id) without a live HF config; the first
+ # parameter can be an unpartitioned adapter.
+ ds._hf_deepspeed_config_weak_ref = None
+ monkeypatch.setitem(sys.modules, "deepspeed", types.ModuleType("deepspeed"))
+ params = list(model.layers[1].parameters())
+ assert len(params) > 1
+ params[-1].ds_id = 0
+ model.layers[1](x).sum().backward()
+ assert calls == []
+ model.layers[0](x).sum().backward()
+ assert calls
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index aaadba0dd87..a5c35e3449c 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -32,6 +32,22 @@ def _enabled():
) not in ("1", "partial")
+def _zero3_partitioned(layer = None):
+ # Compiled regions see ZeRO-3's empty placeholder, not the gathered weight (zoo's check).
+ module = sys.modules.get("transformers.integrations.deepspeed")
+ ref = getattr(module, "_hf_deepspeed_config_weak_ref", None)
+ config = ref() if ref is not None else None
+ try:
+ if config is not None and config.is_zero3():
+ return True
+ except Exception:
+ pass
+ if layer is not None and "deepspeed" in sys.modules:
+ # Any parameter: adapters added outside zero.Init sit next to partitioned base weights.
+ return any(hasattr(param, "ds_id") for param in layer.parameters())
+ return False
+
+
def _forced_float32(module):
# Zoo's float32-forced patches round to fp16 on purpose; Inductor drops those roundings.
# Stamped per layer at load: the env var only describes the latest load.
@@ -671,6 +687,7 @@ def _wrap(cls, compile_options):
or _has_cache(args, kwargs)
or not _enabled()
or _forced_float32(self)
+ or _zero3_partitioned(self)
):
return original(self, hidden_states, *args, **kwargs)
if not self.__dict__.get("_unsloth_block_moe_done", False):
@@ -829,7 +846,13 @@ def patch_block_compile(model):
else:
layer.__dict__["_unsloth_forced_float32"] = bool(forced)
forced = env if forced is None else forced
- if forced or not _enabled() or not torch.cuda.is_available() or not _torch_supported():
+ if (
+ forced
+ or not _enabled()
+ or not torch.cuda.is_available()
+ or not _torch_supported()
+ or _zero3_partitioned()
+ ):
return []
classes = _decoder_layer_classes(model)
if not classes: