Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 QLoRA: give the all-gathered 4-bit weight back its Params4bit class and 3D shape (#14204)
Daniel Han39efb789eb32
diff --git a/tests/test_fsdp2_qlora_moe_gloo.py b/tests/test_fsdp2_qlora_moe_gloo.pynew file mode 100644index 00000000000..09780ab8588--- /dev/null+++ b/tests/test_fsdp2_qlora_moe_gloo.py@@ -0,0 +1,319 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 + bitsandbytes 4-bit MoE experts: the all-gathered weight must look like the Params4bit it+was sharded from. The unsloth-zoo 4-bit MoE paths pick their kernel with `isinstance(w, Params4bit)`+(ModuleList experts: moe_grouped_modulelist `_expert_weight` / `_lin_compute_dtype`; transformers v5+3D experts: moe_utils_bnb4bit `_is_bnb4bit_param`) and read a 3D stack's logical shape from+`w._original_shape`. FSDP2 rebuilds the gathered weight as a plain nn.Parameter, so without the+fsdp2_qlora wrapper those paths fed packed bytes to a dense matmul ("contraction dimension of mat_a+and mat_b must match", "size mismatch").++GPU-free: real bitsandbytes 4-bit weights (CPU backend) sharded over 2 gloo ranks by fully_shard.+"""++import importlib.util+import os+import socket+import sys+from pathlib import Path++import pytest+import torch++_ROOT = Path(__file__).resolve().parents[1]+_MODULE_PATH = Path(+ os.environ.get("UNSLOTH_FSDP2_QLORA_PATH", _ROOT / "unsloth" / "models" / "fsdp2_qlora.py")+)++bnb = pytest.importorskip("bitsandbytes")+fully_shard_mod = pytest.importorskip("torch.distributed.fsdp")+if not hasattr(fully_shard_mod, "fully_shard"):+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)+if not isinstance(getattr(bnb.nn, "Params4bit", None), type):+ pytest.skip("bitsandbytes without a Params4bit class", allow_module_level = True)++H, I, E = 64, 128, 4+++def _load_module():+ spec = importlib.util.spec_from_file_location(+ "_unsloth_fsdp2_qlora_moe_under_test", _MODULE_PATH+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _stack_params4bit(shape, seed, module):+ """A 3D expert stack quantized as one tensor, as zoo `_make_expert_params4bit` builds it."""+ from bitsandbytes.functional import quantize_4bit+ from bitsandbytes.nn import Params4bit++ torch.manual_seed(seed)+ dense = torch.randn(shape) * 0.05+ packed, state = quantize_4bit(dense, blocksize = 64, quant_type = "nf4")+ param = torch.Tensor._make_subclass(Params4bit, packed, False)+ param.quant_state = state+ param.blocksize = 64+ param.compress_statistics = False+ param.quant_type = "nf4"+ param.quant_storage = torch.uint8+ param.bnb_quantized = True+ param.module = module+ param._original_shape = torch.Size(shape)+ return param+++def _linear4bit(in_features, out_features, seed):+ torch.manual_seed(seed)+ lin = bnb.nn.Linear4bit(+ in_features, out_features, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4"+ )+ lin.weight = bnb.nn.Params4bit(+ torch.randn(out_features, in_features) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ blocksize = 64,+ module = lin,+ )+ return lin.to("cpu")+++def _is_4bit(w):+ from bitsandbytes.nn import Params4bit+ return isinstance(w, Params4bit) and getattr(w, "quant_state", None) is not None+++def _dense(w, shape = None):+ """What the zoo 4-bit MoE readers do: dequantize a Params4bit to its logical shape, else use the+ tensor as a dense weight (which, for gathered packed bytes, is the crash this file guards)."""+ if _is_4bit(w):+ out = bnb.functional.dequantize_4bit(w.data, w.quant_state)+ shape = getattr(w, "_original_shape", None) or shape+ return (out.reshape(shape) if shape is not None else out).float()+ return w.float()+++class _Experts3D(torch.nn.Module):+ """transformers v5 layout: one gate_up_proj [E, 2I, H] and one down_proj [E, H, I]."""++ def __init__(self, seed):+ super().__init__()+ self.gate_up_proj = _stack_params4bit((E, 2 * I, H), seed, self)+ self.down_proj = _stack_params4bit((E, H, I), seed + 1, self)+ self.seen = []++ def forward(self, x, expert):+ gu_w, dn_w = self.gate_up_proj, self.down_proj+ self.seen.append(+ dict(+ is_4bit = _is_4bit(gu_w) and _is_4bit(dn_w),+ shape = tuple(getattr(gu_w, "_original_shape", ()) or ()),+ has_module = getattr(gu_w, "module", None) is self,+ gathered_mark = getattr(gu_w, "_unsloth_fsdp2_gathered", False),+ # Params4bit.__torch_function__ reads .module and the quant attributes on chunk.+ chunk = len(torch.chunk(gu_w, 2)),+ )+ )+ gate_up = _dense(gu_w)[expert]+ down = _dense(dn_w)[expert]+ g, u = (x @ gate_up.t()).chunk(2, dim = -1)+ return (torch.nn.functional.silu(g) * u) @ down.t()+++class _ExpertMLP(torch.nn.Module):+ def __init__(self, seed):+ super().__init__()+ self.gate_proj = _linear4bit(H, I, seed)+ self.up_proj = _linear4bit(H, I, seed + 1)+ self.down_proj = _linear4bit(I, H, seed + 2)+++class _ExpertsList(torch.nn.Module):+ """transformers 4.x layout: a ModuleList of Linear4bit experts, read by zoo's grouped MoE."""++ def __init__(self, seed):+ super().__init__()+ self.experts = torch.nn.ModuleList([_ExpertMLP(seed + 10 * e) for e in range(E)])+ self.seen = []++ def forward(self, x, expert):+ ex = self.experts[expert]+ ws = [ex.gate_proj.weight, ex.up_proj.weight, ex.down_proj.weight]+ self.seen.append(dict(is_4bit = all(_is_4bit(w) for w in ws)))+ g = x @ _dense(ws[0], (I, H)).t()+ u = x @ _dense(ws[1], (I, H)).t()+ return (torch.nn.functional.silu(g) * u) @ _dense(ws[2], (H, I)).t()+++class _Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.stack = _Experts3D(100)+ self.listed = _ExpertsList(200)+ torch.manual_seed(7)+ self.adapter = torch.nn.Linear(H, H, bias = False)++ def forward(self, x):+ x = self.adapter(x)+ out = 0+ for e in range(E):+ out = out + self.stack(x, e) + self.listed(x, e)+ return out+++def _worker(rank, world_size, port, case, queue):+ try:+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world_size),+ )+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ dist.init_process_group("gloo", rank = rank, world_size = world_size)+ m = _load_module()+ model, reference = _Model(), _Model()+ torch.manual_seed(1234 + rank)+ x = torch.randn(3, H)+ ref_x = x.clone().requires_grad_(True)+ expected = reference(ref_x)+ expected.sum().backward()++ info = m.prepare_fsdp2_qlora(+ model, torch.bfloat16 if case == "bf16" else None, float_storage = case == "bf16"+ )+ mesh = init_device_mesh("cpu", (world_size,))+ # torch < 2.12 cannot build a uint8 sharded Parameter without the guard.+ with m.nonfloat_param_guard(model):+ fully_shard(model.stack, mesh = mesh)+ fully_shard(model.listed, mesh = mesh)+ fully_shard(model, mesh = mesh)+ sharded = [+ n+ for n, p in model.named_parameters()+ if "adapter" not in n and p.to_local().numel() * world_size == p.full_tensor().numel()+ ]+ model.stack.seen.clear()+ model.listed.seen.clear()+ x = x.clone().requires_grad_(True)+ out = model(x)+ out.sum().backward()+ # Outside forward the module holds FSDP2's sharded DTensor again, whatever the gathered class.+ from torch.distributed.tensor import DTensor++ resharded = isinstance(model.stack.gate_up_proj, DTensor)+ queue.put(+ (+ rank,+ dict(+ params4bit = info["params4bit"],+ sharded = len(sharded),+ stack_seen = model.stack.seen,+ list_seen = model.listed.seen,+ max_diff = (out - expected).abs().max().item(),+ # FSDP2's backward hooks may sum the per-expert dX contributions in another order.+ grad_close = torch.allclose(x.grad, ref_x.grad, rtol = 1e-5, atol = 1e-6),+ adapter_grad = model.adapter.weight.grad is not None,+ resharded = resharded,+ ),+ )+ )+ dist.destroy_process_group()+ except BaseException as error:+ import traceback+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))+++def _run(case, world_size = 2):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = _worker, args = (r, world_size, port, case, queue))+ for r in range(world_size)+ ]+ for p in procs:+ p.start()+ results = {}+ for _ in procs:+ rank, result = queue.get(timeout = 240)+ results[rank] = result+ for p in procs:+ p.join(timeout = 60)+ return results+++_RUN_DIST = pytest.mark.skipif(+ sys.platform == "win32"+ or not torch.distributed.is_available()+ or not torch.distributed.is_gloo_available(),+ reason = "needs torch.distributed with gloo",+)+++@_RUN_DIST[email protected]("case", ["bf16", "uint8"])+def test_gathered_4bit_experts_keep_params4bit_class_and_shape(case):+ m = _load_module()+ if case == "bf16" and (m._missing_fsdp2_anchors() or m._fsdp2_checks_frozen_dtypes()):+ pytest.skip("this torch keeps 4-bit weights uint8 under FSDP2")+ if case == "uint8" and m._fsdp2_checks_frozen_dtypes():+ pytest.skip("torch < 2.8 leaves uint8 weights out of the FSDP2 units")+ results = _run(case)+ for rank, r in results.items():+ assert "error" not in r, r.get("error")+ assert r["params4bit"] == 2 + 3 * E, r+ assert r["sharded"] == r["params4bit"], r+ assert r["stack_seen"] and r["list_seen"], r+ for seen in r["stack_seen"]:+ assert seen["is_4bit"], r+ assert seen["shape"] == (E, 2 * I, H), r+ assert seen["has_module"], r+ # unsloth-zoo re-keys its NF4 pointer tables per call for these (moe_ready_epoch).+ assert seen["gathered_mark"], r+ assert seen["chunk"] == 2, r+ assert all(seen["is_4bit"] for seen in r["list_seen"]), r+ # Same packed bytes, same dequantization: bit-exact vs the unsharded model in forward.+ assert r["max_diff"] == 0.0, r+ assert r["grad_close"], r+ assert r["adapter_grad"], r+ assert r["resharded"], r+++def test_restore_param_class_leaves_other_tensors_alone():+ m = _load_module()+ from bitsandbytes.nn import Params4bit++ plain = torch.nn.Parameter(torch.zeros(4), requires_grad = False)+ ptr = plain.data_ptr()+ owner = torch.nn.Linear(1, 1)+ m._restore_param_class(plain, Params4bit, owner)+ assert type(plain) is Params4bit and plain.data_ptr() == ptr+ assert plain.module is owner+ # Not a plain Parameter (a DTensor or another subclass), or no recorded class: untouched.+ sub = torch.Tensor._make_subclass(Params4bit, torch.zeros(4), False)++ class Other(torch.nn.Parameter):+ pass++ other = Other(torch.zeros(4), requires_grad = False)+ m._restore_param_class(other, Params4bit)+ assert type(other) is Other+ m._restore_param_class(sub, Params4bit)+ assert type(sub) is Params4bit+ keep = torch.nn.Parameter(torch.zeros(4), requires_grad = False)+ m._restore_param_class(keep, None)+ assert type(keep) is torch.nn.Parameterdiff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.pyindex 6a65e96cea8..d971a7a4f71 100644--- a/unsloth/models/fsdp2_qlora.py+++ b/unsloth/models/fsdp2_qlora.py@@ -47,9 +47,15 @@ _QUANT_ATTRS = ("quant_type","quant_storage","bnb_quantized",+ # A transformers v5 3D expert stack (zoo moe_utils_bnb4bit) records its logical shape here.+ "_original_shape",)+# Marks an FSDP2 all-gathered 4-bit weight (read by unsloth-zoo's MoE pointer-table caches).+_GATHERED_MARK = "_unsloth_fsdp2_gathered"++def is_params4bit(tensor):return any(cls.__name__ == "Params4bit" for cls in type(tensor).__mro__)@@ -256,6 +262,31 @@ def _quant_meta(param):return {k: getattr(param, k) for k in _QUANT_ATTRS if hasattr(param, k)}+def _restore_param_class(+ unsharded,+ cls,+ module = None,+):+ """FSDP2 rebuilds the all-gathered weight as a plain nn.Parameter. Give it back the class it was+ sharded from (bitsandbytes Params4bit), so the `isinstance(weight, Params4bit)` checks of the+ unsloth-zoo 4-bit MoE paths (and of bitsandbytes) see what they see without FSDP2. Same object,+ same storage: only the Python type changes, plus the `module` Params4bit.__torch_function__ reads+ on torch.chunk / torch.split. A DTensor (HSDP / TP) or another subclass is left as it is."""+ if unsharded is None or cls is None or type(unsharded) is cls:+ return+ if type(unsharded) is not torch.nn.Parameter or not issubclass(cls, torch.nn.Parameter):+ return+ try:+ unsharded.__class__ = cls+ except TypeError:+ return+ if not hasattr(unsharded, "module"):+ try:+ unsharded.module = module+ except Exception:+ pass++def _attach_quant_meta(tensor, meta):if tensor is None:return@@ -364,6 +395,8 @@ def patch_fsdp2_params4bit():out = original_init_sharded_param(self, param, *args, **kwargs)if meta is not None:self._unsloth_quant_meta = meta+ self._unsloth_param_cls = type(param)+ self._unsloth_param_module = getattr(param, "module", None)_attach_quant_meta(getattr(self, "sharded_param", None), meta)return out@@ -377,6 +410,19 @@ def patch_fsdp2_params4bit():and getattr(unsharded, "quant_state", None) is not meta["quant_state"]):_attach_quant_meta(unsharded, meta)+ _restore_param_class(+ unsharded,+ getattr(self, "_unsloth_param_cls", None),+ getattr(self, "_unsloth_param_module", None),+ )+ if unsharded is not None and not getattr(unsharded, _GATHERED_MARK, False):+ # unsloth-zoo's 4-bit MoE paths cache NF4 pointer tables by weight address; FSDP2+ # frees and re-allocates this storage around every forward and backward, so they+ # re-key it per call instead (zoo moe_ready_epoch.FSDP2_GATHERED).+ try:+ setattr(unsharded, _GATHERED_MARK, True)+ except Exception:+ passreturn outdef init_dtype_attrs(self, mp_policy, *args, **kwargs):