Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: keep the stock forward under DeepSpeed ZeRO-3 (#14396)
Daniel Han57c2a9485ccf
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex a3b09de69ad..fd82936b22f 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -895,3 +895,54 @@ def test_selective_dequant_off_never_reads_a_packed_expert_stack(monkeypatch):M._experts_outside(layer)assert reads == []assert not getattr(layer.mlp.experts, "_unsloth_block_outside", False)+++def test_zero3_partitioned_training_keeps_the_stock_forward(monkeypatch):+ import weakref++ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(1)+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)++ class FakeDsConfig:+ def is_zero3(self):+ return True++ ds = types.ModuleType("transformers.integrations.deepspeed")+ config = FakeDsConfig()+ ds._hf_deepspeed_config_weak_ref = weakref.ref(config)+ monkeypatch.setitem(sys.modules, ds.__name__, ds)++ cls = _toy_class()+ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([cls(), cls()])+ assert M.patch_block_compile(model) == []++ # ZeRO-3 configured after the load (TrainingArguments built later): the forward checks.+ ds._hf_deepspeed_config_weak_ref = None+ assert M.patch_block_compile(model) == [cls.__name__]+ x = torch.randn(2, 4, 8, requires_grad = True)+ ds._hf_deepspeed_config_weak_ref = weakref.ref(config)+ model.layers[0](x).sum().backward()+ assert calls == []+ # Parameters partitioned by deepspeed.zero.Init (ds_id) without a live HF config; the first+ # parameter can be an unpartitioned adapter.+ ds._hf_deepspeed_config_weak_ref = None+ monkeypatch.setitem(sys.modules, "deepspeed", types.ModuleType("deepspeed"))+ params = list(model.layers[1].parameters())+ assert len(params) > 1+ params[-1].ds_id = 0+ model.layers[1](x).sum().backward()+ assert calls == []+ model.layers[0](x).sum().backward()+ assert callsdiff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex aaadba0dd87..a5c35e3449c 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -32,6 +32,22 @@ def _enabled():) not in ("1", "partial")+def _zero3_partitioned(layer = None):+ # Compiled regions see ZeRO-3's empty placeholder, not the gathered weight (zoo's check).+ module = sys.modules.get("transformers.integrations.deepspeed")+ ref = getattr(module, "_hf_deepspeed_config_weak_ref", None)+ config = ref() if ref is not None else None+ try:+ if config is not None and config.is_zero3():+ return True+ except Exception:+ pass+ if layer is not None and "deepspeed" in sys.modules:+ # Any parameter: adapters added outside zero.Init sit next to partitioned base weights.+ return any(hasattr(param, "ds_id") for param in layer.parameters())+ return False++def _forced_float32(module):# Zoo's float32-forced patches round to fp16 on purpose; Inductor drops those roundings.# Stamped per layer at load: the env var only describes the latest load.@@ -671,6 +687,7 @@ def _wrap(cls, compile_options):or _has_cache(args, kwargs)or not _enabled()or _forced_float32(self)+ or _zero3_partitioned(self)):return original(self, hidden_states, *args, **kwargs)if not self.__dict__.get("_unsloth_block_moe_done", False):@@ -829,7 +846,13 @@ def patch_block_compile(model):else:layer.__dict__["_unsloth_forced_float32"] = bool(forced)forced = env if forced is None else forced- if forced or not _enabled() or not torch.cuda.is_available() or not _torch_supported():+ if (+ forced+ or not _enabled()+ or not torch.cuda.is_available()+ or not _torch_supported()+ or _zero3_partitioned()+ ):return []classes = _decoder_layer_classes(model)if not classes: