Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: keep each compiled block whole under DDP (DeepSeek-OCR torchrun) (#15359)

Daniel Han839b06f34d36
diff --git a/tests/python/_block_compile_ddp_worker.py b/tests/python/_block_compile_ddp_worker.py
new file mode 100644
index 00000000000..ce5612eb5e8
--- /dev/null
+++ b/tests/python/_block_compile_ddp_worker.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""One gloo rank for test_block_compile_ddp_no_split.py. Loads _block_compile.py by file path: importing
+the unsloth package needs an accelerator."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+
+_BLOCK = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+ "unsloth",
+ "models",
+ "_block_compile.py",
+)
+
+
+def _load_block_compile():
+ spec = importlib.util.spec_from_file_location("_ddp_block_compile", _BLOCK)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def run_rank(
+ rank,
+ port,
+ queue,
+ recompute_miss = False,
+):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ CUDA_VISIBLE_DEVICES = "",
+ )
+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):
+ os.environ.pop(key, None)
+ try:
+ import torch
+ import torch.distributed as dist
+
+ torch.set_num_threads(1)
+ import transformers
+ from torch._dynamo.backends import distributed as dynamo_distributed
+
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ block = _load_block_compile()
+ splits = []
+ original_compile_fn = dynamo_distributed.DDPOptimizer.compile_fn
+
+ def counting_compile_fn(self, gm, example_inputs, **kwargs):
+ # A decoder block's graph reads the layer's own weights (L_self_modules_...).
+ if any(n.op == "placeholder" and n.name.startswith("l_self_") for n in gm.graph.nodes):
+ splits.append(len(gm.graph.nodes))
+ return original_compile_fn(self, gm, example_inputs, **kwargs)
+
+ dynamo_distributed.DDPOptimizer.compile_fn = counting_compile_fn
+
+ torch.manual_seed(0)
+ # Layers above DDP's 1 MB first bucket, so DDPOptimizer would split each block graph.
+ config = transformers.LlamaConfig(
+ vocab_size = 128,
+ hidden_size = 256,
+ intermediate_size = 1024,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ )
+ model = transformers.LlamaForCausalLM(config)
+ # What Unsloth arms for VLMs under DDP (vision.py): non-reentrant checkpointing.
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
+ model.train()
+ phase = [0]
+ if recompute_miss:
+ # A guard that differs between the forward and the checkpoint recompute (same branch, so
+ # numerics are unchanged): the recompute misses the forward's cache entry and compiles
+ # the block again, outside DDP.forward. This is the DeepSeek-OCR condition.
+ def hook(module, args):
+ if phase[0] >= 0:
+ return None
+ return (args[0] * 2,)
+
+ for layer in model.model.layers:
+ layer.mlp.register_forward_pre_hook(hook)
+ is_available = torch.cuda.is_available
+ torch.cuda.is_available = (
+ lambda: True
+ ) # patch_block_compile only installs on an accelerator
+ try:
+ wrapped = block.patch_block_compile(model)
+ finally:
+ torch.cuda.is_available = is_available
+ ddp = torch.nn.parallel.DistributedDataParallel(
+ model, find_unused_parameters = True, bucket_cap_mb = 1
+ )
+ losses = []
+ for step in range(2):
+ ids = torch.randint(
+ 1, 128, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)
+ )
+ phase[0] = 0
+ out = ddp(input_ids = ids, labels = ids)
+ phase[0] = 1
+ out.loss.backward()
+ losses.append(float(out.loss))
+ grad_sum = float(sum(p.grad.double().abs().sum() for p in model.parameters()))
+ compiled = [
+ bool(layer.__dict__.get("_unsloth_block_checked", 0)) for layer in model.model.layers
+ ]
+ queue.put(
+ (
+ rank,
+ dict(
+ wrapped = wrapped,
+ splits = splits,
+ losses = losses,
+ grad_sum = grad_sum,
+ compiled = compiled,
+ optimize_ddp = bool(torch._dynamo.config.optimize_ddp),
+ ),
+ )
+ )
+ dist.destroy_process_group()
+ except BaseException:
+ import traceback
+ queue.put((rank, {"error": traceback.format_exc()}))
+ raise
diff --git a/tests/python/test_block_compile_ddp_no_split.py b/tests/python/test_block_compile_ddp_no_split.py
new file mode 100644
index 00000000000..5b4907079c1
--- /dev/null
+++ b/tests/python/test_block_compile_ddp_no_split.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""A compiled decoder block called inside DDP.forward must not be split by dynamo's DDPOptimizer.
+
+Under DDP Unsloth checkpoints VLMs non-reentrantly. The recompute runs from backward, outside
+DDP.forward, where dynamo compiles without DDPOptimizer; a block graph split in the forward but
+compiled whole in the recompute saves a different number of tensors (DeepSeek-OCR on 2 T4s:
+"A different number of tensors was saved during the original forward and recomputation", 50 vs 49).
+The block is its own compiled region already, so its gradients still reach DDP block by block."""
+
+from __future__ import annotations
+
+import os
+import queue as queue_module
+import socket
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # spawned ranks inherit sys.path
+import _block_compile_ddp_worker as worker # noqa: E402
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("transformers")
+pytestmark = [
+ pytest.mark.skipif(sys.platform == "win32", reason = "two gloo ranks over TCP localhost"),
+ # Block compile installs only on torch 2.7+ (_torch_supported).
+ pytest.mark.skipif(
+ tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < (2, 7),
+ reason = "block compile needs torch 2.7+",
+ ),
+]
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _run(recompute_miss = False):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = worker.run_rank, args = (r, port, queue, recompute_miss)) for r in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = {}
+ try:
+ while len(results) < len(procs):
+ try:
+ rank, out = queue.get(timeout = 60 if results else 900)
+ except queue_module.Empty:
+ break
+ results[rank] = out
+ finally:
+ for p in procs:
+ p.join(5 if len(results) < len(procs) else 120)
+ if p.is_alive():
+ p.kill()
+ assert sorted(results) == [0, 1], results
+ for out in results.values():
+ assert "error" not in out, out["error"]
+ return results
+
+
+def test_compiled_block_is_not_split_by_ddp_optimizer():
+ results = _run()
+ for rank, out in results.items():
+ assert out["wrapped"] == ["LlamaDecoderLayer"], out
+ # The compiled region ran (not the stock forward) in every layer.
+ assert all(out["compiled"]), out
+ assert out["splits"] == [], (rank, out["splits"])
+ # Only scoped to the block call: the global setting is the user's.
+ assert out["optimize_ddp"] is True
+ assert len(out["losses"]) == 2 and out["grad_sum"] > 0
+ # DDP averaged the gradients: both ranks hold the same ones.
+ assert results[0]["grad_sum"] == pytest.approx(results[1]["grad_sum"], rel = 1e-6)
+
+
+def test_checkpoint_recompute_that_recompiles_matches_the_forward():
+ # On main: CheckpointError "A different number of tensors was saved" (27 vs 25 here).
+ results = _run(recompute_miss = True)
+ for out in results.values():
+ assert out["splits"] == [], out
+ assert len(out["losses"]) == 2 and out["grad_sum"] > 0
+ assert results[0]["grad_sum"] == pytest.approx(results[1]["grad_sum"], rel = 1e-6)
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 5ed54149dd8..72790c5d8f3 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -82,6 +82,37 @@ def _in_nonreentrant_checkpoint():
return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"
+def _ddp_optimizer_mode():
+ # optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"
+ # (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").
+ getter = getattr(torch._dynamo.utils, "get_optimize_ddp_mode", None) or getattr(
+ torch._dynamo.config, "_get_optimize_ddp_mode", None
+ )
+ if getter is not None:
+ return getter() == "ddp_optimizer"
+ return getattr(torch._dynamo.config, "optimize_ddp", False) is True
+
+
+try:
+ from torch.nn.parallel import DistributedDataParallel as _DDP
+except Exception:
+ _DDP = None
+
+
+def _no_ddp_graph_split():
+ """Inside DDP.forward dynamo hands a block's graph to DDPOptimizer, which splits it at gradient
+ bucket boundaries. A checkpoint recompute runs from backward, outside DDP.forward, so a block
+ graph compiled there is not split and saves a different number of tensors than the forward
+ (CheckpointError under DDP's non-reentrant checkpointing, DeepSeek-OCR). Each block is its own
+ region already, so its gradients still reach DDP's buckets block by block."""
+ try:
+ if _DDP is None or _DDP._get_active_ddp_module() is None or not _ddp_optimizer_mode():
+ return contextlib.nullcontext()
+ except Exception:
+ return contextlib.nullcontext()
+ return torch._dynamo.config.patch(optimize_ddp = False)
+
+
def _isolate_supported():
# torch 2.13+: per-call recompile_limit and isolate_recompiles (own cache bucket per region).
try:
@@ -851,7 +882,7 @@ def _wrap(
if varlen is not None:
kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}
# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.
- with capture_scratch():
+ with capture_scratch(), _no_ddp_graph_split():
if graphs and not graphs_off[0]:
shape = tuple(hidden_states.shape)
if shape not in graph_shapes: