Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 block CUDA graphs: free the gathered weights of MoE blocks (#15720)
Daniel Han2e516f8ae477
diff --git a/tests/test_fsdp2_native_block_cudagraphs_moe_gloo.py b/tests/test_fsdp2_native_block_cudagraphs_moe_gloo.pynew file mode 100644index 00000000000..d0c10a65840--- /dev/null+++ b/tests/test_fsdp2_native_block_cudagraphs_moe_gloo.py@@ -0,0 +1,140 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_BLOCK_CUDAGRAPHS=1, native FSDP2 (gloo, world 2): MoE blocks run without graphs, so their+gathered weights must still be freed at reshard; dense blocks keep theirs."""++import importlib.util+import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_block_cudagraphs_gloo import _gathered, _mark_blocks # noqa: E402+from test_fsdp2_native_layout_agree_gloo import _init, _spawn # noqa: E402+from test_fsdp2_native_units import REPO, _load_native # noqa: E402++STEPS = 2+++def _load_block_compile():+ # by file path: a CPU-only `import unsloth` is refused+ name = "unsloth.models._block_compile"+ if name not in sys.modules:+ path = os.path.join(REPO, "unsloth", "models", "_block_compile.py")+ spec = importlib.util.spec_from_file_location(name, path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return sys.modules[name]+++def _build_moe():+ # layer 0 dense (mlp_only_layers), layer 1 MoE+ import transformers++ config = transformers.Qwen3MoeConfig(+ vocab_size = 256,+ hidden_size = 32,+ intermediate_size = 64,+ moe_intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ num_experts = 4,+ num_experts_per_tok = 2,+ mlp_only_layers = [0],+ max_position_embeddings = 64,+ )+ torch.manual_seed(0)+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+++def _block_index(model):+ out = {}+ for i, layer in enumerate(model.model.layers):+ for m in layer.modules():+ out[id(m)] = i+ return out+++def _worker(rank, world, port, out, path):+ dist = _init(rank, world, port)+ os.environ["UNSLOTH_BLOCK_CUDAGRAPHS"] = "1"+ try:+ native = _load_native()+ _load_block_compile()+ model = _build_moe()+ restore = _mark_blocks(model)+ try:+ index = _block_index(model)+ native.shard_model(+ model, native.FSDP2Config(param_dtype = torch.float32, fit_layout = False)+ )+ model.train()+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)+ for step in range(STEPS):+ ids = torch.randint(+ 3, 256, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)+ )+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()+ opt.step()+ opt.zero_grad(set_to_none = True)+ from torch.distributed.fsdp import FSDPModule++ sizes = {0: [], 1: []}+ for module in model.modules():+ if not isinstance(module, FSDPModule):+ continue+ group = getattr(module._get_fsdp_state(), "_fsdp_param_group", None)+ for fsdp_param in getattr(group, "fsdp_params", None) or ():+ block = index.get(id(fsdp_param._module_info.module))+ if block is None:+ continue+ for t in getattr(fsdp_param, "all_gather_outputs", None) or []:+ sizes[block].append(t.untyped_storage().size())+ assert _gathered(model)+ finally:+ for cls, original in restore.items():+ cls.forward = original+ torch.save(sizes, f"{path}.rank{rank}")+ out.put("ok")+ except Exception as exc:+ import traceback+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ os.environ.pop("UNSLOTH_BLOCK_CUDAGRAPHS", None)+ dist.destroy_process_group()++[email protected](scope = "module")+def ranks(tmp_path_factory):+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "Qwen3MoeConfig"):+ pytest.skip("transformers without Qwen3-MoE")+ path = str(tmp_path_factory.mktemp("cudagraphs_moe") / "res.pt")+ payloads = _spawn(_worker, path)+ assert payloads == ["ok", "ok"], payloads+ return [torch.load(f"{path}.rank{r}") for r in range(2)]+++def test_moe_block_weights_are_freed_under_block_cudagraphs(ranks):+ for sizes in ranks:+ assert sizes[1], "no FSDP-managed parameters in the MoE block"+ assert all(size == 0 for size in sizes[1]), sizes[1]+++def test_dense_block_weights_stay_kept_under_block_cudagraphs(ranks):+ for sizes in ranks:+ assert sizes[0], "no FSDP-managed parameters in the dense block"+ assert all(size > 0 for size in sizes[0]), sizes[0]diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 5ab502666a1..6049bf6fd62 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1010,9 +1010,20 @@ def _skip_free_unsharded_param():return None+def _block_never_graphed(module):+ """MoE blocks never record graphs; keeping their gathered weights would hold every expert unsharded."""+ block_compile = sys.modules.get(f"{__package__}._block_compile") if __package__ else None+ unsafe_moe = getattr(block_compile, "_graphs_unsafe_moe", None)+ if not callable(unsafe_moe):+ return False+ try:+ return bool(unsafe_moe(module))+ except Exception:+ return False++def _compiled_block_modules(model, roots = None):- """ids of every module inside a decoder block _block_compile wrapped (and did not turn off);- `roots` (a dict) collects id -> block module."""+ """ids of modules inside graph-capable blocks _block_compile wrapped; `roots` collects id -> block."""inside = set()for module in model.modules():if id(module) in inside or not getattr(@@ -1021,6 +1032,8 @@ def _compiled_block_modules(model, roots = None):continueif module.__dict__.get("_unsloth_block_mode", 1) == 0:continue+ if _block_never_graphed(module):+ continuefor m in module.modules():inside.add(id(m))if roots is not None: