Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 block CUDA graphs: free the gathered weights of flex BlockMask blocks (#15756)
Daniel Han85676c3396c8
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.pyindex a98adb7da36..6f0081d44d9 100644--- a/tests/test_block_cudagraphs_clone.py+++ b/tests/test_block_cudagraphs_clone.py@@ -848,3 +848,38 @@ def test_flex_block_mask_calls_run_without_graphs(monkeypatch, mask):layer(x, attention_mask = value).sum().backward()assert x.grad is not Noneassert set(seen) == ({"plain"} if mask == "block_mask" else {"graphs"})+++def test_block_mask_calls_are_recorded_for_the_fsdp2_keep(monkeypatch):+ """The observer runs on the first BlockMask call and again once a non-BlockMask call regraphs."""+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)++ class FlexKeepDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ **kwargs,+ ):+ return self.proj(hidden_states)++ bc._wrap(FlexKeepDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = FlexKeepDecoderLayer().train()+ calls = []+ layer.__dict__["_unsloth_block_flex_observer"] = lambda m: calls.append(+ m.__dict__["_unsloth_block_flex_only"]+ )+ for mask in (BlockMask(), BlockMask(), None, BlockMask(), None):+ layer(torch.randn(2, 16, 8, requires_grad = True), attention_mask = mask).sum().backward()+ assert calls == [True, False]+ assert layer.__dict__["_unsloth_block_flex_only"] is False++ dense = FlexKeepDecoderLayer().train()+ dense.__dict__["_unsloth_block_flex_observer"] = lambda m: calls.append("dense")+ for mask in (None, BlockMask()):+ dense(torch.randn(2, 16, 8, requires_grad = True), attention_mask = mask).sum().backward()+ assert calls == [True, False] # graphed first: never dropped from the keep setdiff --git a/tests/test_fsdp2_native_block_cudagraphs_flex_gloo.py b/tests/test_fsdp2_native_block_cudagraphs_flex_gloo.pynew file mode 100644index 00000000000..d65868eb029--- /dev/null+++ b/tests/test_fsdp2_native_block_cudagraphs_flex_gloo.py@@ -0,0 +1,259 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_BLOCK_CUDAGRAPHS=1, native FSDP2 (gloo, world 2): flex BlockMask blocks free their weights."""++import importlib.util+import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_layout_agree_gloo import _init, _spawn # noqa: E402+from test_fsdp2_native_units import REPO, _load_native # noqa: E402++STEPS = 3+++def _load_block_compile():+ # by file path: a CPU-only `import unsloth` is refused+ name = "unsloth.models._block_compile"+ if name not in sys.modules:+ path = os.path.join(REPO, "unsloth", "models", "_block_compile.py")+ spec = importlib.util.spec_from_file_location(name, path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return sys.modules[name]+++def _mark_blocks(model, bc):+ """Stand-in for _block_compile's CUDA-only wrapper; `_test_no_mask` blocks count as non-BlockMask."""+ note = getattr(bc, "_note_block_mask_call", None)+ restore = {}+ for module in model.modules():+ cls = type(module)+ if cls.__name__.endswith("DecoderLayer") and cls not in restore:+ original = cls.forward++ def forward(+ self,+ *args,+ _original = original,+ **kwargs,+ ):+ seen = ((), {}) if self.__dict__.get("_test_no_mask") else (args[1:], kwargs)+ if note is not None:+ note(self, *seen)+ else:+ bc._flex_block_mask_call(*seen)+ return _original(self, *args, **kwargs)++ forward._unsloth_block_compiled = True+ restore[cls] = original+ cls.forward = forward+ return restore+++def _dense_flex_on_cpu():+ """CPU flex attention has no backward: keep the BlockMask but attend through SDPA."""+ from torch.nn.attention.flex_attention import create_mask+ from transformers.integrations.sdpa_attention import sdpa_attention_forward+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ def flex_attention_forward(module, query, key, value, attention_mask, **kwargs):+ if type(attention_mask).__name__ == "BlockMask":+ q, kv = query.shape[-2], key.shape[-2]+ attention_mask = create_mask(attention_mask.mask_mod, None, None, q, kv, query.device)+ kwargs.pop("is_causal", None)+ return sdpa_attention_forward(module, query, key, value, attention_mask, **kwargs)++ ALL_ATTENTION_FUNCTIONS["flex_attention"] = flex_attention_forward+++def _build(attn):+ import transformers++ config = transformers.Gemma3TextConfig(+ vocab_size = 256,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ sliding_window = 8,+ max_position_embeddings = 64,+ attn_implementation = attn,+ )+ torch.manual_seed(0)+ return transformers.Gemma3ForCausalLM(config).float()+++def _sizes(model):+ """{block index: [all-gather output storage sizes]} after the step."""+ from torch.distributed.fsdp import FSDPModule++ index = {id(m): i for i, layer in enumerate(model.model.layers) for m in layer.modules()}+ sizes = {i: [] for i in range(len(model.model.layers))}+ for module in model.modules():+ if not isinstance(module, FSDPModule):+ continue+ group = getattr(module._get_fsdp_state(), "_fsdp_param_group", None)+ for fsdp_param in getattr(group, "fsdp_params", None) or ():+ block = index.get(id(fsdp_param._module_info.module))+ if block is not None:+ for t in getattr(fsdp_param, "all_gather_outputs", None) or []:+ sizes[block].append(t.untyped_storage().size())+ return sizes+++def _run(+ native,+ bc,+ rank,+ attn,+ graphs,+ switch_to = None,+ no_mask = (),+ budget_blocks = None,+):+ os.environ["UNSLOTH_BLOCK_CUDAGRAPHS"] = "1" if graphs else "0"+ bc._COMPILE_OPTIONS["triton.cudagraphs"] = graphs # read at import, as _block_compile does+ model = _build(attn)+ for i in no_mask:+ model.model.layers[i].__dict__["_test_no_mask"] = True+ if budget_blocks is not None:+ block = sum(p.numel() * 4 for p in model.model.layers[0].parameters())+ os.environ["UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB"] = str(block * budget_blocks / 2**30)+ restore = _mark_blocks(model, bc)+ try:+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, fit_layout = False))+ model.train()+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)+ losses, sizes = [], []+ for step in range(STEPS):+ if switch_to is not None and step == STEPS - 1:+ model.config._attn_implementation = switch_to+ ids = torch.randint(+ 3, 256, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)+ )+ loss = model(input_ids = ids, labels = ids, use_cache = False).loss+ loss.backward()+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(loss.detach()))+ sizes.append(_sizes(model))+ off = [+ bool(layer.__dict__.get("_unsloth_block_graphs_off")) for layer in model.model.layers+ ]+ return {"losses": losses, "sizes": sizes, "graphs_off": off}+ finally:+ for cls, original in restore.items():+ cls.forward = original+ os.environ.pop("UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB", None)+++def _worker(rank, world, port, out, path):+ dist = _init(rank, world, port)+ try:+ native = _load_native()+ bc = _load_block_compile()+ _dense_flex_on_cpu()+ res = {+ "flex": _run(native, bc, rank, "flex_attention", True),+ "flex_no_graphs": _run(native, bc, rank, "flex_attention", False),+ "sdpa": _run(native, bc, rank, "sdpa", True),+ "flex_then_sdpa": _run(native, bc, rank, "flex_attention", True, switch_to = "sdpa"),+ # layer 0 called without a BlockMask; the budget holds one block, not two+ "mixed_budget": _run(+ native, bc, rank, "flex_attention", True, no_mask = (0,), budget_blocks = 1.5+ ),+ }+ torch.save(res, f"{path}.rank{rank}")+ out.put("ok")+ except Exception as exc:+ import traceback+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ os.environ.pop("UNSLOTH_BLOCK_CUDAGRAPHS", None)+ dist.destroy_process_group()++[email protected](scope = "module")+def ranks(tmp_path_factory):+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "Gemma3TextConfig"):+ pytest.skip("transformers without Gemma3")+ path = str(tmp_path_factory.mktemp("cudagraphs_flex") / "res.pt")+ payloads = _spawn(_worker, path)+ assert payloads == ["ok", "ok"], payloads+ res = [torch.load(f"{path}.rank{r}") for r in range(2)]+ for r, run in enumerate(res):+ for name, data in run.items():+ print("FLEX_KEEP", r, name, data["losses"], data["sizes"][-1], data["graphs_off"])+ return res+++def test_flex_block_weights_are_freed_under_block_cudagraphs(ranks):+ for res in ranks:+ for step in res["flex"]["sizes"]:+ for sizes in step.values():+ assert sizes and all(size == 0 for size in sizes), step+++def test_dense_block_weights_stay_kept_under_block_cudagraphs(ranks):+ for res in ranks:+ for sizes in res["sdpa"]["sizes"][-1].values():+ assert sizes and all(size > 0 for size in sizes), res["sdpa"]["sizes"]+++def test_a_call_without_block_mask_keeps_the_weights_again(ranks):+ # Sometimes flex, sometimes not: the graphed calls need one address, so the block keeps.+ for res in ranks:+ run = res["flex_then_sdpa"]+ assert all(s == 0 for sizes in run["sizes"][0].values() for s in sizes), run["sizes"]+ for sizes in run["sizes"][-1].values():+ assert sizes and all(size > 0 for size in sizes), run["sizes"]+++def test_flex_blocks_leave_the_keep_budget(ranks):+ # Two blocks over the budget; the flex one drops out, so the other keeps its weights and graphs.+ for res in ranks:+ run = res["mixed_budget"]+ last = run["sizes"][-1]+ assert last[0] and all(size > 0 for size in last[0]), run+ assert last[1] and all(size == 0 for size in last[1]), run+ assert run["graphs_off"][0] is False, run+++def test_losses_are_unchanged(ranks):+ for res in ranks:+ assert res["flex"]["losses"] == res["flex_no_graphs"]["losses"]+ assert len(set(res["flex"]["losses"])) == STEPS+++def test_flex_note_does_not_count_only_the_first_block(capsys):+ # Regression: the note once counted 1 block when every block takes a BlockMask.+ import types++ native = _load_native()+ blocks = [(torch.nn.Module(), [types.SimpleNamespace()], 10) for _ in range(3)]+ keeper = native._BlockWeightKeep(blocks, 1000)+ keeper.apply()+ for block, _, _ in blocks:+ block.__dict__["_unsloth_block_flex_only"] = True+ keeper.observe(block)+ out = capsys.readouterr().out+ assert out.count("flex attention blocks") == 1, out+ assert "1 flex attention blocks" not in out, out+ assert all(not getattr(p[0], "_unsloth_keep_unsharded", True) for _, p, _ in blocks)diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 19f6414e877..ae5ca2781fa 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -385,6 +385,20 @@ def _flex_block_mask_call(args, kwargs):return False+def _note_block_mask_call(layer, args, kwargs):+ """_flex_block_mask_call, also recording `_unsloth_block_flex_only`; the observer runs on each change."""+ flex = _flex_block_mask_call(args, kwargs)+ seen = layer.__dict__.get("_unsloth_block_flex_only")+ if seen is False or seen is flex:+ return flex+ # None -> True on a first BlockMask call; True -> False (sticky) on a call without one.+ layer.__dict__["_unsloth_block_flex_only"] = flex+ observer = layer.__dict__.get("_unsloth_block_flex_observer")+ if observer is not None and (flex or seen is True):+ observer(layer)+ return flex++def _ddp_optimizer_mode():# optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"# (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").@@ -1198,12 +1212,14 @@ def _wrap(_graphs_off("shapes", _SHAPES_NOTE.format(_CUDAGRAPH_MAX_SHAPES))else:graph_shapes.add(shape)+ # Before the graphs_off read: the observer may set or clear it for this block.+ flex = graphs and _note_block_mask_call(self, args, kwargs)if graphs and (_CUDAGRAPH_STATE["off"] is not Noneor self.__dict__.get("_unsloth_block_graphs_off", False)or _in_nonreentrant_checkpoint()or _graphs_unsafe_moe(self)- or _flex_block_mask_call(args, kwargs)+ or flex):if _CUDAGRAPH_STATE["off"] is not None and _CUDAGRAPH_STATE["release"] is None:_release_graph_pool()diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 6049bf6fd62..144299baf62 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1056,6 +1056,58 @@ def _gib(nbytes):return f"{nbytes / 2**30:.2f} GiB" if nbytes >= 2**30 / 10 else f"{nbytes / 2**20:.1f} MiB"+def _hold_unsharded(fsdp_param, hold):+ if hold:+ fsdp_param.free_unsharded_param = _skip_free_unsharded_param+ else:+ fsdp_param.__dict__.pop("free_unsharded_param", None)+ fsdp_param._unsloth_keep_unsharded = hold+++class _BlockWeightKeep:+ """Which compiled blocks keep gathered weights. Flex-only blocks (every call took a BlockMask) run+ ungraphed, so they hold nothing and leave the budget; one non-BlockMask call regraphs them."""++ def __init__(self, blocks, budget):+ self.blocks = blocks # [(block, [fsdp_param], nbytes)]+ self.budget = budget+ self.kept = None+ self.noted = False++ def apply(self):+ flex = [b.__dict__.get("_unsloth_block_flex_only") is True for b, _, _ in self.blocks]+ nbytes = sum(n for (_, _, n), f in zip(self.blocks, flex) if not f)+ keep = nbytes <= self.budget+ for (block, params, _), f in zip(self.blocks, flex):+ for fsdp_param in params:+ _hold_unsharded(fsdp_param, keep and not f)+ if not f:+ block.__dict__["_unsloth_block_graphs_off"] = not keep+ self.kept = keep+ return keep, nbytes, sum(flex)++ def observe(self, block):+ was = self.kept+ keep, nbytes, flex = self.apply()+ if flex and not self.noted:+ self.noted = True+ # First flex block, before the others ran: no count.+ _print_rank0(+ "Unsloth: native FSDP2 frees the gathered weights of flex attention blocks "+ "again: they run without block CUDA graphs."+ )+ if keep != was:+ _print_rank0(+ f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 now "+ + (+ f"keeps {_gib(nbytes)} per rank of gathered block weights (graphed blocks only)."+ if keep+ else f"would keep {_gib(nbytes)} per rank (budget {_gib(self.budget)}); "+ "blocks run without CUDA graphs."+ )+ )++def _keep_unsharded_for_block_cudagraphs(model):"""FSDP2 frees gathered params at reshard, moving their addresses so block CUDA graphs re-record everystep (2 x T4 Qwen3 LoRA: 6 records, 0 replays). Keep the storage; the next all-gather refills it in place.@@ -1068,7 +1120,7 @@ def _keep_unsharded_for_block_cudagraphs(model):if not blocks:return 0FSDPModule = _fsdp_module_class()- candidates, nbytes = [], 0+ per_block = {}for module in model.modules():if not isinstance(module, FSDPModule):continue@@ -1085,7 +1137,9 @@ def _keep_unsharded_for_block_cudagraphs(model):continueif not callable(getattr(type(fsdp_param), "free_unsharded_param", None)):return 0 # a torch without this FSDP2 internal: graphs re-record as before- candidates.append((fsdp_param, roots[owner]))+ block = roots[owner]+ entry = per_block.setdefault(id(block), [block, [], 0])+ entry[1].append(fsdp_param)try:# param_dtype / orig_dtype are set at FSDP2's lazy init (first forward), after this.dtype = (@@ -1093,31 +1147,31 @@ def _keep_unsharded_for_block_cudagraphs(model):or getattr(getattr(group, "mp_policy", None), "param_dtype", None)or fsdp_param.sharded_param.dtype)- nbytes += fsdp_param._orig_size.numel() * dtype.itemsize+ entry[2] += fsdp_param._orig_size.numel() * dtype.itemsizeexcept Exception:pass- if not candidates:+ if not per_block:return 0- budget = _block_cudagraphs_keep_budget()- if nbytes > budget:- layers = {id(block): block for _, block in candidates}- for block in layers.values():- block.__dict__["_unsloth_block_graphs_off"] = True+ keeper = _BlockWeightKeep(+ [tuple(e) for e in per_block.values()], _block_cudagraphs_keep_budget()+ )+ keep, nbytes, _ = keeper.apply()+ for block, _, _ in keeper.blocks:+ block.__dict__["_unsloth_block_flex_observer"] = keeper.observe+ count = sum(len(params) for _, params, _ in keeper.blocks)+ if not keep:_print_rank0(f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 would keep {_gib(nbytes)} "- f"per rank of gathered block weights allocated (budget {_gib(budget)}, "- f"UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB); {len(layers)} blocks run without CUDA graphs."+ f"per rank of gathered block weights allocated (budget {_gib(keeper.budget)}, "+ f"UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB); {len(keeper.blocks)} blocks run without CUDA graphs.")return 0- for fsdp_param, _ in candidates:- fsdp_param.free_unsharded_param = _skip_free_unsharded_param- fsdp_param._unsloth_keep_unsharded = True_print_rank0(f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 keeps the gathered weights of "- f"{len(candidates)} parameters allocated ({_gib(nbytes)} per rank) so the block CUDA graphs "+ f"{count} parameters allocated ({_gib(nbytes)} per rank) so the block CUDA graphs ""replay instead of re-recording every step.")- return len(candidates)+ return countdef _post_shard(