Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: keep each compiled block whole under DDP (DeepSeek-OCR torchrun) (#15359)
Daniel Han839b06f34d36
diff --git a/tests/python/_block_compile_ddp_worker.py b/tests/python/_block_compile_ddp_worker.pynew file mode 100644index 00000000000..ce5612eb5e8--- /dev/null+++ b/tests/python/_block_compile_ddp_worker.py@@ -0,0 +1,133 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""One gloo rank for test_block_compile_ddp_no_split.py. Loads _block_compile.py by file path: importing+the unsloth package needs an accelerator."""++from __future__ import annotations++import importlib.util+import os+import sys++_BLOCK = os.path.join(+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),+ "unsloth",+ "models",+ "_block_compile.py",+)+++def _load_block_compile():+ spec = importlib.util.spec_from_file_location("_ddp_block_compile", _BLOCK)+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++def run_rank(+ rank,+ port,+ queue,+ recompute_miss = False,+):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ CUDA_VISIBLE_DEVICES = "",+ )+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):+ os.environ.pop(key, None)+ try:+ import torch+ import torch.distributed as dist++ torch.set_num_threads(1)+ import transformers+ from torch._dynamo.backends import distributed as dynamo_distributed++ dist.init_process_group("gloo", rank = rank, world_size = 2)+ block = _load_block_compile()+ splits = []+ original_compile_fn = dynamo_distributed.DDPOptimizer.compile_fn++ def counting_compile_fn(self, gm, example_inputs, **kwargs):+ # A decoder block's graph reads the layer's own weights (L_self_modules_...).+ if any(n.op == "placeholder" and n.name.startswith("l_self_") for n in gm.graph.nodes):+ splits.append(len(gm.graph.nodes))+ return original_compile_fn(self, gm, example_inputs, **kwargs)++ dynamo_distributed.DDPOptimizer.compile_fn = counting_compile_fn++ torch.manual_seed(0)+ # Layers above DDP's 1 MB first bucket, so DDPOptimizer would split each block graph.+ config = transformers.LlamaConfig(+ vocab_size = 128,+ hidden_size = 256,+ intermediate_size = 1024,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ )+ model = transformers.LlamaForCausalLM(config)+ # What Unsloth arms for VLMs under DDP (vision.py): non-reentrant checkpointing.+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})+ model.train()+ phase = [0]+ if recompute_miss:+ # A guard that differs between the forward and the checkpoint recompute (same branch, so+ # numerics are unchanged): the recompute misses the forward's cache entry and compiles+ # the block again, outside DDP.forward. This is the DeepSeek-OCR condition.+ def hook(module, args):+ if phase[0] >= 0:+ return None+ return (args[0] * 2,)++ for layer in model.model.layers:+ layer.mlp.register_forward_pre_hook(hook)+ is_available = torch.cuda.is_available+ torch.cuda.is_available = (+ lambda: True+ ) # patch_block_compile only installs on an accelerator+ try:+ wrapped = block.patch_block_compile(model)+ finally:+ torch.cuda.is_available = is_available+ ddp = torch.nn.parallel.DistributedDataParallel(+ model, find_unused_parameters = True, bucket_cap_mb = 1+ )+ losses = []+ for step in range(2):+ ids = torch.randint(+ 1, 128, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)+ )+ phase[0] = 0+ out = ddp(input_ids = ids, labels = ids)+ phase[0] = 1+ out.loss.backward()+ losses.append(float(out.loss))+ grad_sum = float(sum(p.grad.double().abs().sum() for p in model.parameters()))+ compiled = [+ bool(layer.__dict__.get("_unsloth_block_checked", 0)) for layer in model.model.layers+ ]+ queue.put(+ (+ rank,+ dict(+ wrapped = wrapped,+ splits = splits,+ losses = losses,+ grad_sum = grad_sum,+ compiled = compiled,+ optimize_ddp = bool(torch._dynamo.config.optimize_ddp),+ ),+ )+ )+ dist.destroy_process_group()+ except BaseException:+ import traceback+ queue.put((rank, {"error": traceback.format_exc()}))+ raisediff --git a/tests/python/test_block_compile_ddp_no_split.py b/tests/python/test_block_compile_ddp_no_split.pynew file mode 100644index 00000000000..5b4907079c1--- /dev/null+++ b/tests/python/test_block_compile_ddp_no_split.py@@ -0,0 +1,91 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""A compiled decoder block called inside DDP.forward must not be split by dynamo's DDPOptimizer.++Under DDP Unsloth checkpoints VLMs non-reentrantly. The recompute runs from backward, outside+DDP.forward, where dynamo compiles without DDPOptimizer; a block graph split in the forward but+compiled whole in the recompute saves a different number of tensors (DeepSeek-OCR on 2 T4s:+"A different number of tensors was saved during the original forward and recomputation", 50 vs 49).+The block is its own compiled region already, so its gradients still reach DDP block by block."""++from __future__ import annotations++import os+import queue as queue_module+import socket+import sys++import pytest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # spawned ranks inherit sys.path+import _block_compile_ddp_worker as worker # noqa: E402++torch = pytest.importorskip("torch")+pytest.importorskip("transformers")+pytestmark = [+ pytest.mark.skipif(sys.platform == "win32", reason = "two gloo ranks over TCP localhost"),+ # Block compile installs only on torch 2.7+ (_torch_supported).+ pytest.mark.skipif(+ tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < (2, 7),+ reason = "block compile needs torch 2.7+",+ ),+]+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _run(recompute_miss = False):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = worker.run_rank, args = (r, port, queue, recompute_miss)) for r in range(2)+ ]+ for p in procs:+ p.start()+ results = {}+ try:+ while len(results) < len(procs):+ try:+ rank, out = queue.get(timeout = 60 if results else 900)+ except queue_module.Empty:+ break+ results[rank] = out+ finally:+ for p in procs:+ p.join(5 if len(results) < len(procs) else 120)+ if p.is_alive():+ p.kill()+ assert sorted(results) == [0, 1], results+ for out in results.values():+ assert "error" not in out, out["error"]+ return results+++def test_compiled_block_is_not_split_by_ddp_optimizer():+ results = _run()+ for rank, out in results.items():+ assert out["wrapped"] == ["LlamaDecoderLayer"], out+ # The compiled region ran (not the stock forward) in every layer.+ assert all(out["compiled"]), out+ assert out["splits"] == [], (rank, out["splits"])+ # Only scoped to the block call: the global setting is the user's.+ assert out["optimize_ddp"] is True+ assert len(out["losses"]) == 2 and out["grad_sum"] > 0+ # DDP averaged the gradients: both ranks hold the same ones.+ assert results[0]["grad_sum"] == pytest.approx(results[1]["grad_sum"], rel = 1e-6)+++def test_checkpoint_recompute_that_recompiles_matches_the_forward():+ # On main: CheckpointError "A different number of tensors was saved" (27 vs 25 here).+ results = _run(recompute_miss = True)+ for out in results.values():+ assert out["splits"] == [], out+ assert len(out["losses"]) == 2 and out["grad_sum"] > 0+ assert results[0]["grad_sum"] == pytest.approx(results[1]["grad_sum"], rel = 1e-6)diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 5ed54149dd8..72790c5d8f3 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -82,6 +82,37 @@ def _in_nonreentrant_checkpoint():return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"+def _ddp_optimizer_mode():+ # optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"+ # (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").+ getter = getattr(torch._dynamo.utils, "get_optimize_ddp_mode", None) or getattr(+ torch._dynamo.config, "_get_optimize_ddp_mode", None+ )+ if getter is not None:+ return getter() == "ddp_optimizer"+ return getattr(torch._dynamo.config, "optimize_ddp", False) is True+++try:+ from torch.nn.parallel import DistributedDataParallel as _DDP+except Exception:+ _DDP = None+++def _no_ddp_graph_split():+ """Inside DDP.forward dynamo hands a block's graph to DDPOptimizer, which splits it at gradient+ bucket boundaries. A checkpoint recompute runs from backward, outside DDP.forward, so a block+ graph compiled there is not split and saves a different number of tensors than the forward+ (CheckpointError under DDP's non-reentrant checkpointing, DeepSeek-OCR). Each block is its own+ region already, so its gradients still reach DDP's buckets block by block."""+ try:+ if _DDP is None or _DDP._get_active_ddp_module() is None or not _ddp_optimizer_mode():+ return contextlib.nullcontext()+ except Exception:+ return contextlib.nullcontext()+ return torch._dynamo.config.patch(optimize_ddp = False)++def _isolate_supported():# torch 2.13+: per-call recompile_limit and isolate_recompiles (own cache bucket per region).try:@@ -851,7 +882,7 @@ def _wrap(if varlen is not None:kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.- with capture_scratch():+ with capture_scratch(), _no_ddp_graph_split():if graphs and not graphs_off[0]:shape = tuple(hidden_states.shape)if shape not in graph_shapes: