Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: shard QLoRA 4-bit weights with fused LoRA kernels kept on (#13880)

Daniel Han086db552e645
diff --git a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
index d60bc6126a1..322fa0f29c4 100644
--- a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
+++ b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
@@ -2,7 +2,7 @@ import ast
from pathlib import Path
-_HELPERS = ("_packed_base", "_is_packed_state")
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
def _load_function(name):
@@ -15,7 +15,7 @@ def _load_function(name):
if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)
]
assert [f.name for f in funcs].count(name) == 1, (name, funcs)
- namespace = {"getattr": getattr, "_FP8_WEIGHT_DTYPES": ()}
+ namespace = {"getattr": getattr, "_FP8_WEIGHT_DTYPES": (), "_FSDP2_QLORA_PREPARED": True}
module = ast.Module(body = funcs, type_ignores = [])
ast.fix_missing_locations(module)
exec(compile(module, str(source), "exec"), namespace)
diff --git a/tests/python/test_get_lora_parameters_fp8_block_size.py b/tests/python/test_get_lora_parameters_fp8_block_size.py
index 0905c1eb10e..62d5dbc5512 100644
--- a/tests/python/test_get_lora_parameters_fp8_block_size.py
+++ b/tests/python/test_get_lora_parameters_fp8_block_size.py
@@ -2,7 +2,7 @@ import ast
from pathlib import Path
-_HELPERS = ("_packed_base", "_is_packed_state")
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
def _load_function(name):
@@ -15,7 +15,12 @@ def _load_function(name):
if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)
]
assert [f.name for f in funcs].count(name) == 1, (name, funcs)
- namespace = {"getattr": getattr, "hasattr": hasattr, "_FP8_WEIGHT_DTYPES": ()}
+ namespace = {
+ "getattr": getattr,
+ "hasattr": hasattr,
+ "_FP8_WEIGHT_DTYPES": (),
+ "_FSDP2_QLORA_PREPARED": True,
+ }
module = ast.Module(body = funcs, type_ignores = [])
ast.fix_missing_locations(module)
exec(compile(module, str(source), "exec"), namespace)
diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
new file mode 100644
index 00000000000..030e183ae86
--- /dev/null
+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 all-gathers a bitsandbytes 4-bit weight into a plain Parameter without `quant_state`. The fused
+LoRA kernels must then read the Linear4bit module's own QuantState, not treat the packed (N, 1) bytes as
+a dense weight."""
+
+import ast
+from pathlib import Path
+
+import torch
+
+_SOURCE = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
+
+
+def _load_function(name, fsdp2_qlora_prepared = True):
+ # Extract functions from kernels/utils.py without importing unsloth (which needs a GPU).
+ tree = ast.parse(_SOURCE.read_text(encoding = "utf-8"))
+ funcs = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)
+ ]
+ assert [f.name for f in funcs].count(name) == 1, (name, funcs)
+ namespace = {
+ "torch_Tensor": torch.Tensor,
+ "_FP8_WEIGHT_DTYPES": (),
+ "_FSDP2_QLORA_PREPARED": fsdp2_qlora_prepared,
+ }
+ module = ast.Module(body = funcs, type_ignores = [])
+ ast.fix_missing_locations(module)
+ exec(compile(module, str(_SOURCE), "exec"), namespace)
+ return namespace[name]
+
+
+class _Obj:
+ pass
+
+
+class Linear4bit(torch.nn.Module):
+ # Stands in for bitsandbytes.nn.Linear4bit, matched by class name.
+ pass
+
+
+class _LoRA:
+ def __init__(self, base_layer):
+ self.base_layer = base_layer
+ self.disable_adapters = False
+ self.merged = False
+ self.active_adapters = ["default"]
+ a, b = _Obj(), _Obj()
+ a.weight, b.weight = torch.zeros(4, 64), torch.zeros(32, 4)
+ self.lora_A = {"default": a}
+ self.lora_B = {"default": b}
+ self.scaling = {"default": 2.0}
+ self.lora_bias = {}
+
+
+def _fsdp2_unsharded_4bit(base_cls = Linear4bit):
+ base_layer = base_cls()
+ state = _Obj()
+ state.dtype = torch.float16
+ base_layer.quant_state = state
+ # The all-gathered packed bytes: (32 * 64 / 4, 1) in fp16 storage, a plain Parameter with no quant_state.
+ base_layer.weight = torch.nn.Parameter(
+ torch.zeros(512, 1, dtype = torch.float16), requires_grad = False
+ )
+ base_layer.bias = None
+ return _LoRA(base_layer), state
+
+
+def test_get_lora_parameters_falls_back_to_module_quant_state():
+ get_lora_parameters = _load_function("get_lora_parameters")
+ proj, state = _fsdp2_unsharded_4bit()
+ W, W_quant = get_lora_parameters(proj)[:2]
+ assert W is proj.base_layer.weight
+ assert W_quant is state
+
+
+def test_get_lora_parameters_bias_falls_back_to_module_quant_state():
+ get_lora_parameters_bias = _load_function("get_lora_parameters_bias")
+ proj, state = _fsdp2_unsharded_4bit()
+ assert get_lora_parameters_bias(proj)[1] is state
+
+
+def test_weight_quant_state_still_wins():
+ get_lora_parameters = _load_function("get_lora_parameters")
+ proj, state = _fsdp2_unsharded_4bit()
+ own = _Obj()
+ proj.base_layer.weight.quant_state = own
+ assert get_lora_parameters(proj)[1] is own
+
+
+def test_no_fallback_for_dense_or_non_bnb_weights():
+ get_lora_parameters = _load_function("get_lora_parameters")
+ # A dense weight is never paired with a stray module quant_state.
+ proj, _ = _fsdp2_unsharded_4bit()
+ proj.base_layer.weight = torch.nn.Parameter(torch.zeros(32, 64), requires_grad = False)
+ assert get_lora_parameters(proj)[1] is None
+ # Nor is a module that is not a bitsandbytes Linear4bit.
+ proj, _ = _fsdp2_unsharded_4bit(base_cls = torch.nn.Module)
+ assert get_lora_parameters(proj)[1] is None
+
+
+def test_dense_weight_never_takes_the_module_getattr_path():
+ # Every 16-bit LoRA forward (training and decode) asks for the quant state; a dense weight must
+ # not pay for nn.Module.__getattr__ raising and catching AttributeError on the base layer.
+ get_lora_parameters = _load_function("get_lora_parameters")
+ lookups = []
+
+ class _CountingLinear(torch.nn.Linear):
+ def __getattr__(self, name):
+ lookups.append(name)
+ return super().__getattr__(name)
+
+ base_layer = _CountingLinear(64, 32, bias = False)
+ proj = _LoRA(base_layer)
+ lookups.clear()
+ assert get_lora_parameters(proj)[1] is None
+ assert "quant_state" not in lookups, lookups
+
+
+def test_fallback_only_after_fsdp2_qlora_preparation():
+ # Without prepare_fsdp2_qlora no weight can lose its quant_state, so the default LoRA forward keeps
+ # the merge base's single getattr and never consults the base layer.
+ for name in ("get_lora_parameters", "get_lora_parameters_bias"):
+ fn = _load_function(name, fsdp2_qlora_prepared = False)
+ proj, _ = _fsdp2_unsharded_4bit()
+ assert fn(proj)[1] is None
diff --git a/tests/test_fsdp2_qlora.py b/tests/test_fsdp2_qlora.py
new file mode 100644
index 00000000000..79f13613c43
--- /dev/null
+++ b/tests/test_fsdp2_qlora.py
@@ -0,0 +1,557 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 + bitsandbytes 4-bit (QLoRA): GPU-free checks of unsloth/models/fsdp2_qlora.py.
+
+The module is loaded from its file (importing unsloth needs a GPU). The distributed tests shard real
+bitsandbytes Linear4bit layers (CPU backend) across 2 gloo ranks with FSDP2's fully_shard.
+"""
+
+import importlib.util
+import os
+import socket
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+_ROOT = Path(__file__).resolve().parents[1]
+_MODULE_PATH = Path(
+ os.environ.get("UNSLOTH_FSDP2_QLORA_PATH", _ROOT / "unsloth" / "models" / "fsdp2_qlora.py")
+)
+
+
+def _load_module():
+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_qlora_under_test", _MODULE_PATH)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+bnb = pytest.importorskip("bitsandbytes")
+fully_shard_mod = pytest.importorskip("torch.distributed.fsdp")
+if not hasattr(fully_shard_mod, "fully_shard"):
+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)
+
+
+def _linear4bit(in_features, out_features, storage, seed):
+ torch.manual_seed(seed)
+ lin = bnb.nn.Linear4bit(
+ in_features,
+ out_features,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ quant_type = "nf4",
+ quant_storage = storage,
+ )
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(out_features, in_features) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = storage,
+ blocksize = 64,
+ module = lin,
+ )
+ return lin.to("cpu")
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, storage, seed):
+ super().__init__()
+ self.up = _linear4bit(128, 256, storage, seed)
+ self.down = _linear4bit(256, 128, storage, seed + 1)
+ torch.manual_seed(seed + 2)
+ # Trainable fp32 adapter, like LoRA's A matrix.
+ self.adapter = torch.nn.Linear(128, 8, bias = False)
+ # Zero, so the sharded output can be compared bit for bit; its gradient is still nonzero.
+ torch.nn.init.zeros_(self.adapter.weight)
+
+ def forward(self, x):
+ h = self.down(torch.nn.functional.silu(self.up(x)))
+ return h + self.adapter(x.to(self.adapter.weight.dtype)).sum(-1, keepdim = True).to(h.dtype)
+
+
+class _Model(torch.nn.Module):
+ def __init__(self, storage):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_Block(storage, 10 * i) for i in range(2)])
+
+ def forward(self, x):
+ for layer in self.layers:
+ x = layer(x)
+ return x
+
+
+def test_quant_storage_dtype():
+ m = _load_module()
+ assert m.fsdp2_quant_storage_dtype(torch.float16) is torch.float16
+ assert m.fsdp2_quant_storage_dtype("bfloat16") is torch.bfloat16
+ assert m.fsdp2_quant_storage_dtype(torch.uint8) is None
+ assert m.fsdp2_quant_storage_dtype(None) is None
+
+
+def test_set_quant_storage_on_bnb_config():
+ m = _load_module()
+ from transformers import BitsAndBytesConfig
+
+ cfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_compute_dtype = torch.float16)
+ assert m.set_fsdp2_quant_storage(cfg, torch.float16)
+ assert cfg.bnb_4bit_quant_storage == torch.float16
+ # A caller's explicit float storage stays.
+ cfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_quant_storage = torch.float32)
+ assert not m.set_fsdp2_quant_storage(cfg, torch.bfloat16)
+ assert cfg.bnb_4bit_quant_storage == torch.float32
+ # 8-bit has no packed 4-bit storage.
+ cfg = BitsAndBytesConfig(load_in_8bit = True)
+ assert not m.set_fsdp2_quant_storage(cfg, torch.bfloat16)
+ assert not m.set_fsdp2_quant_storage(None, torch.bfloat16)
+
+
[email protected]("storage", [torch.bfloat16, torch.float16])
+def test_uint8_to_float_storage_is_a_view(storage):
+ m = _load_module()
+ lin = _linear4bit(128, 64, torch.uint8, 0)
+ weight = lin.weight
+ before = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)
+ raw = weight.data.clone()
+ model = torch.nn.Sequential(lin)
+ assert m.params4bit_to_float_storage(model, storage) == 1
+ assert lin.weight is weight
+ assert weight.dtype == storage and weight.quant_storage == storage
+ assert lin.quant_storage == storage
+ assert weight.shape == (raw.numel() // 2, 1)
+ assert torch.equal(weight.data.view(torch.uint8).reshape(-1), raw.reshape(-1))
+ after = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)
+ assert torch.equal(before, after)
+ x = torch.randn(3, 128, dtype = torch.bfloat16)
+ torch.testing.assert_close(lin(x), x @ before.t().to(x.dtype), rtol = 0, atol = 0)
+ # Idempotent; float storage is left alone.
+ assert m.params4bit_to_float_storage(model, storage) == 0
+ assert m.params4bit_to_float_storage(model, torch.uint8) == 0
+
+
+def _prepare_that_upcasts(accelerator, model):
+ # accelerate < 1.13 ends its FSDP2 wrap like this when mixed precision is on.
+ model = model.to(torch.float32)
+ return model
+
+
+def _prepare_that_keeps_dtypes(accelerator, model):
+ return model
+
+
+def test_float_storage_only_when_the_wrap_does_not_upcast():
+ m = _load_module()
+ assert m.float_storage_survives_wrap(_prepare_that_upcasts) is False
+ assert m.float_storage_survives_wrap(_prepare_that_keeps_dtypes) is True
+ # The installed accelerate's own function is read through any wrapper.
+ from accelerate.utils.fsdp_utils import fsdp2_prepare_model
+
+ assert m.float_storage_survives_wrap() is m.float_storage_survives_wrap(fsdp2_prepare_model)
+
+
+def test_upcasting_wrap_keeps_uint8_and_guards_parameter_creation():
+ m = _load_module()
+ lin = _linear4bit(128, 64, torch.uint8, 0)
+ model = torch.nn.Sequential(lin)
+ with m.fsdp2_qlora_wrap(model, _prepare_that_upcasts) as info:
+ assert info["params4bit"] == 1 and info["converted"] == 0
+ assert info["storage"] == ["uint8"]
+ # torch < 2.12 builds the sharded uint8 parameter with requires_grad = True first.
+ assert torch.nn.Parameter(torch.zeros(2, dtype = torch.uint8)).requires_grad is False
+ # What accelerate < 1.13 then does is harmless to uint8 bytes.
+ before = lin.weight.data.clone()
+ model.to(torch.float32)
+ assert lin.weight.dtype == torch.uint8 and torch.equal(lin.weight.data, before)
+ with pytest.raises(RuntimeError):
+ torch.nn.Parameter(torch.zeros(2, dtype = torch.uint8))
+
+
+def test_wrap_is_a_noop_without_4bit_weights():
+ m = _load_module()
+ model = torch.nn.Linear(4, 4)
+ with m.fsdp2_qlora_wrap(model, _prepare_that_keeps_dtypes) as info:
+ assert info is None
+ assert m.prepare_fsdp2_qlora(model)["params4bit"] == 0
+
+
+def test_anchor_check_refuses_unknown_fsdp_param():
+ m = _load_module()
+
+ class _Old:
+ def _init_sharded_param(self, tensor, device):
+ pass
+
+ def init_unsharded_param(self):
+ self._unsharded_param = None
+
+ missing = m._check_fsdp_param_anchors(_Old)
+ assert "FSDPParam.init_dtype_attrs" in missing
+
+ class _Renamed(_Old):
+ def init_dtype_attrs(self, mp_policy):
+ self.param_dtype = None
+
+ assert m._check_fsdp_param_anchors(_Renamed) == [
+ "FSDPParam._init_sharded_param(self, param, ...)"
+ ]
+
+
+def test_anchors_present_on_installed_torch():
+ m = _load_module()
+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam
+ assert m._check_fsdp_param_anchors(FSDPParam) == []
+
+
+def test_nonfloat_guard_restores_parameter_new():
+ m = _load_module()
+ before = torch.nn.Parameter.__dict__.get("__new__")
+ with m.nonfloat_param_guard():
+ p = torch.nn.Parameter(torch.zeros(4, dtype = torch.uint8))
+ assert p.requires_grad is False
+ assert torch.nn.Parameter(torch.zeros(4)).requires_grad is True
+ assert torch.nn.Parameter.__dict__.get("__new__") is before
+ with pytest.raises(RuntimeError):
+ torch.nn.Parameter(torch.zeros(4, dtype = torch.uint8))
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _worker(rank, world_size, port, case, queue):
+ try:
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world_size),
+ )
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard
+
+ dist.init_process_group("gloo", rank = rank, world_size = world_size)
+ m = _load_module()
+ storage = torch.uint8 if case.startswith("uint8") else torch.bfloat16
+ model = _Model(storage)
+ reference = _Model(storage)
+ if case.endswith("uniform"):
+ # Every all-gather input of a block in one float dtype: FSDP2's foreach-copy path.
+ for net in (model, reference):
+ for layer in net.layers:
+ layer.adapter.to(torch.bfloat16)
+ torch.manual_seed(1234)
+ x = torch.randn(4, 128, dtype = torch.bfloat16)
+ expected = reference(x).detach()
+
+ if case.startswith("unpatched"):
+ info = {"patched": False}
+ else:
+ info = m.prepare_fsdp2_qlora(
+ model, torch.bfloat16 if case == "uint8_converted" else None
+ )
+ if case == "uint8_converted":
+ assert info["converted"] == 4, info
+ # The dtype each all-gather moves its inputs in, as foreach_all_gather decided it.
+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives
+
+ gather_dtypes = []
+ decide = _fsdp_collectives._get_all_gather_input_metadatas
+
+ def _get_all_gather_input_metadatas(*args, **kwargs):
+ out = decide(*args, **kwargs)
+ gather_dtypes.append(str(out[2]))
+ return out
+
+ _fsdp_collectives._get_all_gather_input_metadatas = _get_all_gather_input_metadatas
+ packed_bytes = sum(
+ p.numel() * p.element_size() for p in model.parameters() if m.is_params4bit(p)
+ )
+ mesh = init_device_mesh("cpu", (world_size,))
+ if case.endswith("uniform"):
+ mp = MixedPrecisionPolicy()
+ else:
+ # fp16 all-gather: on bf16 storage this cast would scramble the packed bytes.
+ mp = MixedPrecisionPolicy(
+ param_dtype = torch.float16, reduce_dtype = torch.float32, cast_forward_inputs = False
+ )
+ for layer in model.layers:
+ fully_shard(layer, mesh = mesh, mp_policy = mp)
+ fully_shard(model, mesh = mesh, mp_policy = mp)
+
+ local_bytes = 0
+ for name, p in model.named_parameters():
+ if "adapter" in name:
+ continue
+ local = p.to_local()
+ local_bytes += local.numel() * local.element_size()
+ seen = {}
+
+ def hook(module, args):
+ w = module.weight
+ seen[id(module)] = (getattr(w, "quant_state", None) is not None, w.dtype)
+
+ for layer in model.layers:
+ layer.up.register_forward_pre_hook(hook)
+ layer.down.register_forward_pre_hook(hook)
+ out = model(x)
+ out.float().sum().backward()
+ grad_ok = all(p.grad is not None for n, p in model.named_parameters() if "adapter" in n)
+ sharded_has_qs = all(
+ getattr(p, "quant_state", None) is not None
+ for n, p in model.named_parameters()
+ if "adapter" not in n
+ )
+ result = dict(
+ info = {k: v for k, v in info.items() if k != "reason"},
+ max_diff = (out.float() - expected.float()).abs().max().item(),
+ packed_bytes = packed_bytes,
+ local_bytes = local_bytes,
+ weight_quant_state = [v[0] for v in seen.values()],
+ grad_ok = grad_ok,
+ sharded_has_qs = sharded_has_qs,
+ gather_dtypes = sorted(set(gather_dtypes)),
+ )
+ queue.put((rank, result))
+ dist.destroy_process_group()
+ except BaseException as error:
+ import traceback
+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
+
+
+def _old_torch_worker(rank, world_size, port, case, queue):
+ """torch < 2.8 (forced): uint8 weights must stay out of every FSDP2 unit, whose trainable float
+ adapter would otherwise share it with them (`FSDP expects uniform original parameter dtype`)."""
+ try:
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world_size),
+ )
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+ from torch.distributed.tensor import DTensor
+
+ dist.init_process_group("gloo", rank = rank, world_size = world_size)
+ m = _load_module()
+ m._fsdp2_checks_frozen_dtypes = lambda: True
+ model, reference = _Model(torch.uint8), _Model(torch.uint8)
+ torch.manual_seed(1234)
+ x = torch.randn(4, 128, dtype = torch.bfloat16)
+ expected = reference(x).detach()
+ mesh = init_device_mesh("cpu", (world_size,))
+ with m.fsdp2_qlora_wrap(model, _prepare_that_keeps_dtypes) as info:
+ for layer in model.layers:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ unit_dtypes = []
+ for module in model.modules():
+ state = getattr(module, "_get_fsdp_state", None)
+ group = state()._fsdp_param_group if state is not None else None
+ if group is not None:
+ unit_dtypes.append(
+ sorted({str(fp.sharded_param.dtype) for fp in group.fsdp_params})
+ )
+ out = model(x)
+ out.float().sum().backward()
+ queue.put(
+ (
+ rank,
+ dict(
+ storage = info["storage"],
+ unit_dtypes = unit_dtypes,
+ uint8_sharded = sum(
+ isinstance(p, DTensor)
+ for p in model.parameters()
+ if not p.is_floating_point()
+ ),
+ max_diff = (out.float() - expected.float()).abs().max().item(),
+ grad_ok = all(
+ p.grad is not None for n, p in model.named_parameters() if "adapter" in n
+ ),
+ ),
+ )
+ )
+ dist.destroy_process_group()
+ except BaseException as error:
+ import traceback
+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
+
+
+def _run(
+ case,
+ world_size = 2,
+ target = None,
+):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = target or _worker, args = (r, world_size, port, case, queue))
+ for r in range(world_size)
+ ]
+ for p in procs:
+ p.start()
+ results = {}
+ for _ in procs:
+ rank, result = queue.get(timeout = 240)
+ results[rank] = result
+ for p in procs:
+ p.join(timeout = 60)
+ return results
+
+
+_RUN_DIST = pytest.mark.skipif(
+ sys.platform == "win32"
+ or not torch.distributed.is_available()
+ or not torch.distributed.is_gloo_available(),
+ reason = "needs torch.distributed with gloo",
+)
+
+
+@_RUN_DIST
[email protected]("case", ["bf16", "uint8_converted", "bf16_uniform"])
+def test_fsdp2_shards_4bit_and_keeps_quant_state(case):
+ results = _run(case)
+ for rank, r in results.items():
+ assert "error" not in r, r.get("error")
+ assert r["info"]["patched"], r
+ # The packed 4-bit bytes are split across the 2 ranks, not replicated.
+ assert r["local_bytes"] * 2 == r["packed_bytes"], r
+ # FSDP2's all-gathered weight still carries bitsandbytes' QuantState in forward.
+ assert r["weight_quant_state"] and all(r["weight_quant_state"]), r
+ assert r["sharded_has_qs"], r
+ # Bit-exact vs the unsharded model: no fp16 cast touched the packed bytes.
+ assert r["max_diff"] == 0.0, r
+ assert r["grad_ok"], r
+ # Packed bytes always travel as bytes: CUDA's float foreach copy rewrites NaN bit patterns.
+ assert r["gather_dtypes"] == ["torch.uint8"], r
+
+
+@_RUN_DIST
+def test_unpatched_uniform_group_gathers_as_float():
+ # Control: without the patch a block whose inputs share bf16 is gathered as bf16.
+ results = _run("unpatched_uniform")
+ for rank, r in results.items():
+ assert "error" not in r, r.get("error")
+ assert "torch.bfloat16" in r["gather_dtypes"], r
+
+
+@_RUN_DIST
+def test_fsdp2_mixed_precision_corrupts_unpatched_bf16_storage():
+ # Control for the test above: without the patches, FSDP2 casts the packed bf16 bytes to fp16.
+ results = _run("unpatched")
+ for rank, r in results.items():
+ print(rank, r)
+ assert "error" not in r, r.get("error")
+ assert r["max_diff"] > 0.0, r
+
+
+def test_patch_is_idempotent_across_imports():
+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam
+
+ assert _load_module().patch_fsdp2_params4bit() == (True, None)
+ first = FSDPParam.init_dtype_attrs
+ # A second copy of the module (another import path) must not wrap the wrapper.
+ assert _load_module().patch_fsdp2_params4bit() == (True, None)
+ assert FSDPParam.init_dtype_attrs is first
+ assert getattr(first._unsloth_original, "_unsloth_original", None) is None
+
+
+def test_no_float_storage_when_fsdp2_checks_frozen_dtypes():
+ # torch < 2.8 asserts one dtype per unit, frozen weights included.
+ m = _load_module()
+ assert m._fsdp2_checks_frozen_dtypes() is False # installed torch >= 2.8
+ m._fsdp2_checks_frozen_dtypes = lambda: True
+ assert m.float_storage_survives_wrap(_prepare_that_keeps_dtypes) is False
+ from transformers import BitsAndBytesConfig
+
+ cfg = BitsAndBytesConfig(load_in_4bit = True)
+ assert not m.set_fsdp2_quant_storage(cfg, torch.float16)
+
+
+@_RUN_DIST
+def test_old_torch_leaves_uint8_weights_out_of_the_units():
+ results = _run("uint8_old_torch", target = _old_torch_worker)
+ for rank, r in results.items():
+ assert "error" not in r, r.get("error")
+ assert r["storage"] == ["uint8"], r
+ assert r["unit_dtypes"] and all(d == ["torch.float32"] for d in r["unit_dtypes"]), r
+ assert r["uint8_sharded"] == 0, r
+ assert r["max_diff"] == 0.0, r
+ assert r["grad_ok"], r
+
+
[email protected]("why", ["upcasting_wrap", "old_torch", "unpatched"])
+def test_caller_float_storage_falls_back_to_uint8_when_it_cannot_survive(monkeypatch, why):
+ m = _load_module()
+ lin = _linear4bit(128, 64, torch.bfloat16, 0)
+ weight = lin.weight
+ before = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)
+ raw = weight.data.view(torch.uint8).clone()
+ prepare = _prepare_that_keeps_dtypes
+ if why == "upcasting_wrap":
+ prepare = _prepare_that_upcasts
+ elif why == "old_torch":
+ monkeypatch.setattr(m, "_fsdp2_checks_frozen_dtypes", lambda: True)
+ else:
+ monkeypatch.setattr(m, "patch_fsdp2_params4bit", lambda: (False, "no anchors"))
+ monkeypatch.setattr(m, "_missing_fsdp2_anchors", lambda: ["x"])
+ model = torch.nn.Sequential(lin)
+ with m.fsdp2_qlora_wrap(model, prepare) as info:
+ assert info["reverted"] == 1 and info["storage"] == ["uint8"], info
+ model.to(torch.float32)
+ assert lin.weight is weight and weight.dtype == torch.uint8 and lin.quant_storage == torch.uint8
+ assert torch.equal(weight.data.reshape(-1), raw.reshape(-1))
+ assert torch.equal(bnb.functional.dequantize_4bit(weight.data, weight.quant_state), before)
+
+
+def test_float_storage_is_kept_when_it_survives():
+ m = _load_module()
+ lin = _linear4bit(128, 64, torch.bfloat16, 0)
+ with m.fsdp2_qlora_wrap(torch.nn.Sequential(lin), _prepare_that_keeps_dtypes) as info:
+ assert info["reverted"] == 0 and info["storage"] == ["bfloat16"], info
+
+
[email protected]("storage", [torch.bfloat16, torch.float32])
+def test_size_based_wrap_counts_packed_weights_as_uint8(storage):
+ m = _load_module()
+ from accelerate.utils import fsdp_utils
+ from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+
+ assert m.patch_size_based_wrap_policy()
+ uint8_block = _Block(torch.uint8, 0)
+ packed_block = _Block(torch.uint8, 0)
+ assert m.params4bit_to_float_storage(packed_block, storage) == 2
+ uint8_numel = sum(p.numel() for p in uint8_block.parameters())
+ plugin = type("Plugin", (), {})()
+ plugin.auto_wrap_policy = size_based_auto_wrap_policy
+ plugin.min_num_params = uint8_numel - 1
+ plugin.transformer_cls_names_to_wrap = None
+ policy = fsdp_utils.fsdp2_prepare_auto_wrap_policy(plugin, packed_block)
+ assert policy(uint8_block) and policy(packed_block)
+ original = fsdp_utils.fsdp2_prepare_auto_wrap_policy._unsloth_original
+ assert not original(plugin, packed_block)(packed_block)
+ # Only the counting changes: accelerate's FSDP2 size policy compares with `>`, so does this one.
+ for threshold in (uint8_numel - 1, uint8_numel):
+ plugin.min_num_params = threshold
+ assert fsdp_utils.fsdp2_prepare_auto_wrap_policy(plugin, uint8_block)(
+ uint8_block
+ ) == original(plugin, uint8_block)(uint8_block)
+
+
+def test_unquantized_params4bit_is_not_byte_viewed():
+ m = _load_module()
+ lin = bnb.nn.Linear4bit(128, 64, bias = False, quant_storage = torch.bfloat16)
+ dense = lin.weight.data.clone()
+ assert not lin.weight.bnb_quantized and lin.weight.is_floating_point()
+ assert m.params4bit_to_uint8_storage(torch.nn.Sequential(lin)) == (0, 0)
+ assert lin.weight.dtype == dense.dtype and torch.equal(lin.weight.data, dense)
diff --git a/tests/test_fsdp2_qlora_prepare_order.py b/tests/test_fsdp2_qlora_prepare_order.py
new file mode 100644
index 00000000000..ac263027c85
--- /dev/null
+++ b/tests/test_fsdp2_qlora_prepare_order.py
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The FSDP2 prepare hook falls back to uint8 4-bit storage before unifying frozen dtypes (torch < 2.8,
+forced here), so float-packed weights from an explicit bnb_4bit_quant_storage do not trip it."""
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+bnb = pytest.importorskip("bitsandbytes")
+pytest.importorskip("accelerate")
+
+from unsloth.models import fsdp2, fsdp2_qlora # noqa: E402
+
+
+def test_old_torch_float_packed_qlora_without_mixed_precision(monkeypatch):
+ import accelerate.accelerator as accelerator_module
+
+ torch.manual_seed(0)
+ lin = bnb.nn.Linear4bit(
+ 128,
+ 64,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ )
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(64, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ blocksize = 64,
+ module = lin,
+ )
+ model = torch.nn.Sequential(lin.to("cpu"), torch.nn.Linear(64, 8))
+ assert lin.weight.dtype == torch.bfloat16 and lin.weight.bnb_quantized
+
+ seen = {}
+
+ def stub(accelerator, model):
+ seen["dtypes"] = sorted({str(p.dtype) for p in model.parameters()})
+ return model
+
+ monkeypatch.setattr(accelerator_module, "fsdp2_prepare_model", stub)
+ monkeypatch.setattr(fsdp2, "_fsdp2_checks_frozen_dtypes", lambda: True)
+ monkeypatch.setattr(fsdp2_qlora, "_fsdp2_checks_frozen_dtypes", lambda: True)
+ assert fsdp2.install_fsdp2_prepare_hook()
+ plugin = types.SimpleNamespace(mixed_precision_policy = None)
+ accelerator = types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = "no"
+ )
+ accelerator_module.fsdp2_prepare_model(accelerator, model)
+ assert seen["dtypes"] == ["torch.float32", "torch.uint8"], seen
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 433f03515b4..685d6d14297 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -12,13 +12,14 @@
import sys
import torch
from unsloth_zoo.utils import Version
+from . import utils as _kernel_utils
from .utils import (
_has_active_dora_adapter,
_has_active_lora_bias,
_has_multiple_active_adapters,
addmm_,
fast_dequantize,
- QUANT_STATE,
+ _base_quant_state,
get_lora_parameters,
get_lora_parameters_bias,
matmul_lora,
@@ -749,7 +750,10 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:
lora_B = self.lora_B[active_adapter].weight
scaling = self.scaling[active_adapter]
W = self.base_layer.weight
- return _apply(LoRA_W, x, W, QUANT_STATE(W), lora_A, lora_B, scaling)
+ W_quant = getattr(W, "quant_state", None)
+ if W_quant is None and _kernel_utils._FSDP2_QLORA_PREPARED:
+ W_quant = _base_quant_state(self.base_layer, W)
+ return _apply(LoRA_W, x, W, W_quant, lora_A, lora_B, scaling)
pass
pass
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index f9a59a69d0e..d650ede50a3 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -327,6 +327,31 @@ def QUANT_STATE(W):
return getattr(W, "quant_state", None)
+# Set by models/fsdp2_qlora.prepare_fsdp2_qlora: only FSDP2-sharded 4-bit weights lose quant_state, so every
+# other LoRA forward keeps the single getattr(W, "quant_state") and never reaches _base_quant_state.
+_FSDP2_QLORA_PREPARED = False
+
+
+def _base_quant_state(base_layer, W):
+ """W's quant state, else its bitsandbytes Linear4bit's own copy of it. FSDP2 all-gathers the packed
+ bytes into a plain Parameter that drops `quant_state`; read without it, the packed (N, 1) buffer
+ would be taken for a dense weight."""
+ W_quant = getattr(W, "quant_state", None)
+ if W_quant is not None:
+ return W_quant
+ # Linear4bit keeps its copy as a plain attribute. getattr would go through nn.Module.__getattr__,
+ # which raises and catches for every dense weight: 10x this function's cost on each LoRA forward.
+ W_quant = base_layer.__dict__.get("quant_state", None)
+ if (
+ W_quant is None
+ or getattr(W, "ndim", 0) != 2
+ or W.shape[1] != 1
+ or not any(cls.__name__ == "Linear4bit" for cls in type(base_layer).__mro__)
+ ):
+ return None
+ return W_quant
+
+
# fp8 weight dtypes: a weight_scale / weight_scale_inv is only a quant state while the weight
# itself is still fp8. compressed-tensors layers expose an already-dequantized bf16 weight at
# forward time while keeping a weight_scale, and reading that as a quant state routes it into
@@ -432,13 +457,16 @@ def get_lora_parameters(proj):
if weight_fake_quantizer is not None:
W = weight_fake_quantizer(W)
- # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
- # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
W_quant = getattr(W, "quant_state", None)
- if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
- W_quant = getattr(base_layer, "weight_scale_inv", None)
- if W_quant is None:
- W_quant = getattr(base_layer, "weight_scale", None)
+ if W_quant is None:
+ if _FSDP2_QLORA_PREPARED:
+ W_quant = _base_quant_state(base_layer, W)
+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
+ if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
+ W_quant = getattr(base_layer, "weight_scale_inv", None)
+ if W_quant is None:
+ W_quant = getattr(base_layer, "weight_scale", None)
if getattr(base_layer, "quant_method", None) == "fp8":
W.block_size = getattr(base_layer, "block_size", [128, 128])
@@ -490,13 +518,16 @@ def get_lora_parameters_bias(proj):
packed = _packed_base(base_layer)
W = base_layer.weight if packed is None else packed[0]
- # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
- # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
W_quant = getattr(W, "quant_state", None)
- if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
- W_quant = getattr(base_layer, "weight_scale_inv", None)
- if W_quant is None:
- W_quant = getattr(base_layer, "weight_scale", None)
+ if W_quant is None:
+ if _FSDP2_QLORA_PREPARED:
+ W_quant = _base_quant_state(base_layer, W)
+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
+ if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
+ W_quant = getattr(base_layer, "weight_scale_inv", None)
+ if W_quant is None:
+ W_quant = getattr(base_layer, "weight_scale", None)
if getattr(base_layer, "quant_method", None) == "fp8":
W.block_size = getattr(base_layer, "block_size", [128, 128])
diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.py
index ab91d92ee17..f4a9cf7874e 100644
--- a/unsloth/models/fsdp2.py
+++ b/unsloth/models/fsdp2.py
@@ -3,7 +3,8 @@
"""What Unsloth adds around accelerate's `fsdp2_prepare_model` (which already wraps per block,
keeps the [norm, lm_head] tail gathered, groups tied embeddings). `UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1`
-skips the mixed precision fill and the last-block setting.
+skips the mixed precision fill and the last-block setting. bitsandbytes 4-bit (QLoRA) weights are
+sharded too, keeping their QuantState (`fsdp2_qlora.py`).
"""
import contextlib
@@ -424,10 +425,23 @@ def install_fsdp2_prepare_hook():
@functools.wraps(original)
def fsdp2_prepare_model(accelerator, model, *args, **kwargs):
- fill_fsdp2_mixed_precision(accelerator)
- unify_trainable_dtypes(accelerator, model)
- unify_frozen_dtypes(accelerator, model)
- model = original(accelerator, model, *args, **kwargs)
+ from .fsdp2_qlora import fsdp2_qlora_wrap
+
+ # The 4-bit fallback to uint8 storage first: float-packed weights would count as frozen
+ # 16-bit weights in the dtype unification.
+ with fsdp2_qlora_wrap(model, original) as qlora:
+ fill_fsdp2_mixed_precision(accelerator)
+ unify_trainable_dtypes(accelerator, model)
+ unify_frozen_dtypes(accelerator, model)
+ model = original(accelerator, model, *args, **kwargs)
+ if qlora is not None:
+ note = "" if qlora["patched"] else f" FSDPParam was not patched: {qlora['reason']}."
+ if qlora.get("reverted"):
+ note += f" {qlora['reverted']} float-packed weights kept as uint8, which this wrap preserves."
+ _print_rank0(
+ f"Unsloth: FSDP2 QLoRA: {qlora['params4bit']} 4-bit weights packed as "
+ f"{'/'.join(qlora['storage'])} ({qlora['converted']} repacked from uint8).{note}"
+ )
after_fsdp2_wrap(accelerator, model)
return model
diff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.py
new file mode 100644
index 00000000000..6a65e96cea8
--- /dev/null
+++ b/unsloth/models/fsdp2_qlora.py
@@ -0,0 +1,534 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""QLoRA (bitsandbytes 4-bit) under FSDP2.
+
+FSDP2 can only shard a frozen 4-bit weight whose packed bytes are stored in a float dtype: uint8 cannot
+become a sharded nn.Parameter before torch 2.12 and accelerate leaves it replicated on every rank. So:
+1. a fresh quantization stores the packed bytes as the compute dtype (`bnb_4bit_quant_storage`), and a
+ pre-quantized uint8 checkpoint is reinterpreted (a view, no copy) as the same dtype after loading;
+2. FSDPParam keeps the bitsandbytes QuantState on the sharded and the all-gathered parameter (FSDP2
+ rebuilds both as plain Parameters, which drops it), so the fused LoRA kernels and bnb's own forward
+ still read `weight.quant_state`;
+3. FSDP2 mixed precision never casts packed 4-bit bytes (or any other non-float parameter), and an
+ all-gather holding them moves bytes: when every input of a group shares one float dtype FSDP2
+ copies them with `torch._foreach_copy_`, which on CUDA rewrites float16 / bfloat16 NaN bit patterns
+ (about 1 in 60 packed float16 words) to the canonical NaN;
+4. `nonfloat_param_guard` lets torch < 2.12 shard what is still uint8; torch < 2.8, which wants one
+ dtype per unit frozen weights included, leaves uint8 weights out of the units (replicated);
+5. SIZE_BASED_WRAP counts float-packed weights at their uint8 numel, so wrap decisions do not move.
+Float storage is used only when 2 and 3 can be patched and the wrap does not upcast the whole model.
+Method wrappers (not source rewrites, as in axolotl's monkeypatch/fsdp2_qlora.py) hold on torch 2.6 .. 2.14.
+"""
+
+__all__ = [
+ "is_params4bit",
+ "float_storage_survives_wrap",
+ "fsdp2_quant_storage_dtype",
+ "set_fsdp2_quant_storage",
+ "params4bit_to_float_storage",
+ "params4bit_to_uint8_storage",
+ "patch_fsdp2_params4bit",
+ "nonfloat_param_guard",
+ "prepare_fsdp2_qlora",
+ "fsdp2_qlora_wrap",
+]
+
+import contextlib
+import inspect
+
+import torch
+
+_FLOAT_STORAGE = (torch.float16, torch.bfloat16, torch.float32)
+_QUANT_ATTRS = (
+ "quant_state",
+ "blocksize",
+ "compress_statistics",
+ "quant_type",
+ "quant_storage",
+ "bnb_quantized",
+)
+
+
+def is_params4bit(tensor):
+ return any(cls.__name__ == "Params4bit" for cls in type(tensor).__mro__)
+
+
+def float_storage_survives_wrap(prepare_fn = None):
+ """Whether 4-bit weights may be packed in a float dtype for FSDP2. False when this torch's FSDP2
+ cannot be patched (`patch_fsdp2_params4bit`), checks frozen dtypes per unit (torch < 2.8), or when accelerate's FSDP2 wrap ends with
+ `model.to(torch.float32)` (accelerate < 1.13 with mixed precision), which converts float-stored
+ packed bytes as if they were weights. uint8 storage is safe from both, so 4-bit weights then stay
+ uint8 (left replicated by accelerate, but correct)."""
+ if prepare_fn is None:
+ try:
+ from accelerate.utils.fsdp_utils import fsdp2_prepare_model as prepare_fn
+ except Exception:
+ prepare_fn = None
+ if _missing_fsdp2_anchors():
+ # Unpatched FSDP2 would scramble float-stored packed bytes (see 3 above).
+ return False
+ if _fsdp2_checks_frozen_dtypes():
+ # torch < 2.8 wants one dtype per unit, frozen weights included: packed bytes in the compute
+ # dtype next to float32 LoRA / upcast weights would fail it, while uint8 is left out by accelerate.
+ return False
+ if prepare_fn is None:
+ return True
+ prepare_fn = inspect.unwrap(prepare_fn)
+ try:
+ source = inspect.getsource(prepare_fn)
+ except (OSError, TypeError):
+ return False
+ return "model.to(torch.float32)" not in source
+
+
+def _fsdp2_checks_frozen_dtypes():
+ try:
+ from torch.distributed.fsdp._fully_shard._fsdp_param_group import FSDPParamGroup
+ return "requires_grad" not in inspect.getsource(FSDPParamGroup._init_mp_dtypes)
+ except Exception:
+ return True
+
+
+def fsdp2_quant_storage_dtype(dtype):
+ """The `bnb_4bit_quant_storage` FSDP2 can shard: the compute dtype, so the mixed precision cast
+ of the all-gather is a no-op on the packed bytes. None for a dtype bitsandbytes cannot store in."""
+ if isinstance(dtype, str):
+ dtype = getattr(torch, dtype, None)
+ return dtype if dtype in _FLOAT_STORAGE else None
+
+
+def set_fsdp2_quant_storage(quantization_config, dtype):
+ """Store the packed 4-bit bytes of a BitsAndBytesConfig in `dtype`. Returns True when changed.
+ A caller's explicit float storage is kept; only the uint8 default is replaced, and only when the
+ FSDP2 wrap will not upcast the whole model (`float_storage_survives_wrap`)."""
+ storage = fsdp2_quant_storage_dtype(dtype)
+ if quantization_config is None or storage is None or not float_storage_survives_wrap():
+ return False
+ if not getattr(quantization_config, "load_in_4bit", False):
+ return False
+ current = getattr(quantization_config, "bnb_4bit_quant_storage", None)
+ if isinstance(current, str):
+ current = getattr(torch, current, None)
+ if current is not None and current != torch.uint8:
+ return False
+ quantization_config.bnb_4bit_quant_storage = storage
+ return True
+
+
+def params4bit_to_float_storage(model, dtype):
+ """Reinterpret every uint8 Params4bit (a pre-quantized checkpoint) as `dtype` storage in place.
+ The bytes, the QuantState and the Parameter object are unchanged; only the element type of the
+ packed buffer differs, exactly what `bnb_4bit_quant_storage = dtype` would have produced.
+ Returns the number of weights converted."""
+ storage = fsdp2_quant_storage_dtype(dtype)
+ if storage is None:
+ return 0
+ itemsize = torch.empty((), dtype = storage).element_size()
+ n_converted = 0
+ for module in model.modules():
+ for name, param in list(module._parameters.items()):
+ if param is None or not is_params4bit(param) or param.dtype != torch.uint8:
+ continue
+ nbytes = param.numel()
+ if not param.is_contiguous() or nbytes % itemsize != 0 or param.device.type == "meta":
+ continue
+ param.data = param.data.reshape(-1).view(storage).reshape(-1, 1)
+ param.quant_storage = storage
+ if name == "weight" and hasattr(module, "quant_storage"):
+ # Linear4bit rebuilds a lost Params4bit with it (fix_4bit_weight_quant_state_from_module).
+ module.quant_storage = storage
+ n_converted += 1
+ return n_converted
+
+
+def params4bit_to_uint8_storage(model):
+ """Inverse of `params4bit_to_float_storage`: reinterpret float-packed Params4bit as the uint8
+ bitsandbytes stores by default. Returns (converted, left), left = those that could not be viewed."""
+ converted = left = 0
+ for module in model.modules():
+ for name, param in list(module._parameters.items()):
+ if param is None or not is_params4bit(param) or not param.is_floating_point():
+ continue
+ if (
+ not getattr(param, "bnb_quantized", False)
+ or getattr(param, "quant_state", None) is None
+ ):
+ # Still dense values bitsandbytes quantizes on the next device move, not packed bytes.
+ continue
+ if not param.is_contiguous() or param.device.type == "meta":
+ left += 1
+ continue
+ param.data = param.data.reshape(-1).view(torch.uint8).reshape(-1, 1)
+ param.quant_storage = torch.uint8
+ if name == "weight" and hasattr(module, "quant_storage"):
+ module.quant_storage = torch.uint8
+ converted += 1
+ return converted, left
+
+
+def _packed_numel(param):
+ if is_params4bit(param) and param.is_floating_point():
+ return param.numel() * param.element_size()
+ return param.numel()
+
+
+def patch_size_based_wrap_policy():
+ """accelerate's FSDP2 SIZE_BASED_WRAP sums `p.numel()`, which float packing divides by the storage
+ itemsize; count those weights as the uint8 bytes they hold so the same blocks get wrapped."""
+ try:
+ import accelerate.utils.fsdp_utils as fsdp_utils
+ from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+ except Exception:
+ return False
+ original = getattr(fsdp_utils, "fsdp2_prepare_auto_wrap_policy", None)
+ if original is None:
+ return False
+ if getattr(original, "_unsloth_original", None) is not None:
+ return True
+
+ def fsdp2_prepare_auto_wrap_policy(fsdp2_plugin, *args, **kwargs):
+ policy = original(fsdp2_plugin, *args, **kwargs)
+ fn = getattr(fsdp2_plugin, "auto_wrap_policy", None)
+ fn = getattr(fn, "func", fn)
+ min_num_params = getattr(fsdp2_plugin, "min_num_params", None)
+ if policy is None or fn is not size_based_auto_wrap_policy or min_num_params is None:
+ return policy
+
+ def size_policy(module):
+ return sum(_packed_numel(p) for p in module.parameters()) > min_num_params
+
+ return size_policy
+
+ fsdp2_prepare_auto_wrap_policy.__name__ = original.__name__
+ fsdp2_prepare_auto_wrap_policy.__doc__ = original.__doc__
+ fsdp2_prepare_auto_wrap_policy._unsloth_original = original
+ fsdp_utils.fsdp2_prepare_auto_wrap_policy = fsdp2_prepare_auto_wrap_policy
+ return True
+
+
+def nonfloat_frozen_unmanaged():
+ """While FSDP2 shards on torch < 2.8: leave frozen non-float parameters (uint8 Params4bit) out of
+ every unit, as accelerate >= 1.14 does with `ignored_params` on torch >= 2.7. torch < 2.8 asserts one
+ original dtype over all of a unit's parameters, so uint8 next to float LoRA weights cannot be sharded."""
+ modules = []
+ try:
+ from torch.distributed.fsdp._fully_shard import _fully_shard as _fully_shard_module
+ modules.append(_fully_shard_module)
+ except Exception:
+ pass
+ try:
+ from torch.distributed.fsdp._fully_shard import _fsdp_init as _fsdp_init_module
+ modules.append(_fsdp_init_module)
+ except Exception:
+ pass
+
+ patched = []
+ for module in modules:
+ original = getattr(module, "_get_managed_states", None)
+ if original is None:
+ continue
+
+ def _get_managed_states(
+ *args,
+ _original = original,
+ **kwargs,
+ ):
+ params, buffers = _original(*args, **kwargs)
+ params = [
+ p for p in params if p.requires_grad or p.is_floating_point() or p.is_complex()
+ ]
+ return params, buffers
+
+ module._get_managed_states = _get_managed_states
+ patched.append((module, original))
+ try:
+ yield len(patched)
+ finally:
+ for module, original in patched:
+ module._get_managed_states = original
+
+
+def _quant_meta(param):
+ if not is_params4bit(param) or getattr(param, "quant_state", None) is None:
+ return None
+ return {k: getattr(param, k) for k in _QUANT_ATTRS if hasattr(param, k)}
+
+
+def _attach_quant_meta(tensor, meta):
+ if tensor is None:
+ return
+ for k, v in meta.items():
+ try:
+ setattr(tensor, k, v)
+ except Exception:
+ pass
+
+
+def _check_fsdp_param_anchors(FSDPParam):
+ """The FSDPParam surface the wrappers rely on, in torch 2.6 .. 2.14. A string per missing anchor."""
+ missing = []
+ methods = {}
+ for name in ("_init_sharded_param", "init_unsharded_param", "init_dtype_attrs"):
+ method = getattr(FSDPParam, name, None)
+ if not callable(method):
+ missing.append(f"FSDPParam.{name}")
+ methods[name] = getattr(method, "_unsloth_original", method)
+ if missing:
+ return missing
+ try:
+ params = list(inspect.signature(methods["_init_sharded_param"]).parameters)
+ if len(params) < 2 or params[1] != "param":
+ missing.append("FSDPParam._init_sharded_param(self, param, ...)")
+ except (TypeError, ValueError):
+ missing.append("FSDPParam._init_sharded_param signature")
+ try:
+ if "self._unsharded_param" not in inspect.getsource(methods["init_unsharded_param"]):
+ missing.append("FSDPParam.init_unsharded_param sets self._unsharded_param")
+ if "self.param_dtype" not in inspect.getsource(methods["init_dtype_attrs"]):
+ missing.append("FSDPParam.init_dtype_attrs sets self.param_dtype")
+ except (OSError, TypeError):
+ missing.append("FSDPParam source")
+ return missing
+
+
+def _check_collectives_anchors(collectives):
+ """foreach_all_gather picks the collective dtype in `_get_all_gather_input_metadatas` (uint8 when
+ the inputs' dtypes differ) from what `_get_param_all_gather_inputs` returned, in torch 2.6 .. 2.14."""
+ missing = []
+ for name in (
+ "foreach_all_gather",
+ "_get_param_all_gather_inputs",
+ "_get_all_gather_input_metadatas",
+ ):
+ if not callable(getattr(collectives, name, None)):
+ missing.append(f"_fsdp_collectives.{name}")
+ if missing:
+ return missing
+ try:
+ source = inspect.getsource(collectives.foreach_all_gather)
+ except (OSError, TypeError):
+ return ["_fsdp_collectives source"]
+ for anchor in (
+ "_get_param_all_gather_inputs(fsdp_params)",
+ "_get_all_gather_input_metadatas(",
+ "if dtype == torch.uint8:",
+ ):
+ if anchor not in source:
+ missing.append(f"foreach_all_gather: {anchor}")
+ return missing
+
+
+def _missing_fsdp2_anchors():
+ try:
+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives
+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam
+ except Exception as error:
+ return [f"FSDP2 ({type(error).__name__})"]
+ return _check_fsdp_param_anchors(FSDPParam) + _check_collectives_anchors(_fsdp_collectives)
+
+
+def _is_packed_fsdp_param(fsdp_param):
+ return getattr(fsdp_param, "_unsloth_quant_meta", None) is not None
+
+
+# Set by _get_param_all_gather_inputs, read by the _get_all_gather_input_metadatas call right after it
+# in the same foreach_all_gather.
+_BYTE_ALL_GATHER = [False]
+_PATCH_STATE = {"patched": False, "reason": None}
+
+
+def patch_fsdp2_params4bit():
+ """Wrap FSDPParam (and the all-gather's dtype choice) so bitsandbytes 4-bit weights keep their
+ QuantState and their packed bytes through FSDP2. Idempotent. Returns (ok, reason): ok is False, with the missing anchors, when this
+ torch has no FSDP2 or a FSDPParam this was not written against; nothing is patched then."""
+ if _PATCH_STATE["patched"]:
+ return True, None
+ missing = _missing_fsdp2_anchors()
+ if missing:
+ return False, f"torch {torch.__version__} FSDP2 lacks " + ", ".join(missing)
+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives
+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam
+
+ if getattr(FSDPParam.init_dtype_attrs, "_unsloth_original", None) is not None:
+ _PATCH_STATE["patched"] = True
+ return True, None
+
+ original_init_sharded_param = FSDPParam._init_sharded_param
+ original_init_unsharded_param = FSDPParam.init_unsharded_param
+ original_init_dtype_attrs = FSDPParam.init_dtype_attrs
+
+ def _init_sharded_param(self, param, *args, **kwargs):
+ meta = _quant_meta(param)
+ out = original_init_sharded_param(self, param, *args, **kwargs)
+ if meta is not None:
+ self._unsloth_quant_meta = meta
+ _attach_quant_meta(getattr(self, "sharded_param", None), meta)
+ return out
+
+ def init_unsharded_param(self, *args, **kwargs):
+ out = original_init_unsharded_param(self, *args, **kwargs)
+ meta = getattr(self, "_unsloth_quant_meta", None)
+ if meta is not None:
+ unsharded = getattr(self, "_unsharded_param", None)
+ if (
+ unsharded is not None
+ and getattr(unsharded, "quant_state", None) is not meta["quant_state"]
+ ):
+ _attach_quant_meta(unsharded, meta)
+ return out
+
+ def init_dtype_attrs(self, mp_policy, *args, **kwargs):
+ out = original_init_dtype_attrs(self, mp_policy, *args, **kwargs)
+ if getattr(self, "param_dtype", None) is None:
+ return out
+ # A cast reinterprets nothing: it converts values, which destroys packed 4-bit bytes. Leave
+ # tensor subclasses with their own all-gather hooks (torchao NF4Tensor) to those hooks.
+ sharded = getattr(self, "sharded_param", None)
+ local = getattr(sharded, "_local_tensor", sharded)
+ packed = getattr(self, "_unsloth_quant_meta", None) is not None or (
+ sharded is not None and not sharded.is_floating_point()
+ )
+ if packed and not hasattr(local, "fsdp_pre_all_gather"):
+ self.param_dtype = None
+ return out
+
+ original_get_inputs = _fsdp_collectives._get_param_all_gather_inputs
+ original_get_metadatas = _fsdp_collectives._get_all_gather_input_metadatas
+
+ def _get_param_all_gather_inputs(fsdp_params, *args, **kwargs):
+ out = original_get_inputs(fsdp_params, *args, **kwargs)
+ _BYTE_ALL_GATHER[0] = any(_is_packed_fsdp_param(p) for p in fsdp_params)
+ return out
+
+ def _get_all_gather_input_metadatas(*args, **kwargs):
+ dtypes, numels, dtype = original_get_metadatas(*args, **kwargs)
+ if _BYTE_ALL_GATHER[0] and dtype != torch.uint8:
+ dtype = torch.uint8
+ _BYTE_ALL_GATHER[0] = False
+ return dtypes, numels, dtype
+
+ for fn, original in (
+ (_init_sharded_param, original_init_sharded_param),
+ (init_unsharded_param, original_init_unsharded_param),
+ (init_dtype_attrs, original_init_dtype_attrs),
+ (_get_param_all_gather_inputs, original_get_inputs),
+ (_get_all_gather_input_metadatas, original_get_metadatas),
+ ):
+ fn.__name__ = original.__name__
+ fn.__qualname__ = getattr(original, "__qualname__", original.__name__)
+ fn.__doc__ = original.__doc__
+ fn._unsloth_original = original
+
+ FSDPParam._init_sharded_param = _init_sharded_param
+ FSDPParam.init_unsharded_param = init_unsharded_param
+ FSDPParam.init_dtype_attrs = init_dtype_attrs
+ _fsdp_collectives._get_param_all_gather_inputs = _get_param_all_gather_inputs
+ _fsdp_collectives._get_all_gather_input_metadatas = _get_all_gather_input_metadatas
+ _PATCH_STATE["patched"] = True
+ return True, None
+
+
+def nonfloat_param_guard(model = None):
+ """While FSDP2 shards: freeze non-float parameters, and make `nn.Parameter(x)` default to
+ requires_grad = False for non-float `x`. torch < 2.12 builds the sharded parameter with the default
+ requires_grad = True before copying the flag over, which raises on uint8. Restored on exit."""
+ if model is not None:
+ for p in model.parameters():
+ if p.requires_grad and not (p.is_floating_point() or p.is_complex()):
+ p.requires_grad_(False)
+ Parameter = torch.nn.Parameter
+ original_new = Parameter.__dict__.get("__new__")
+
+ def __new__(
+ cls,
+ data = None,
+ requires_grad = True,
+ ):
+ if data is not None and not (data.is_floating_point() or data.is_complex()):
+ requires_grad = False
+ return original_new(cls, data, requires_grad)
+
+ Parameter.__new__ = __new__
+ try:
+ yield
+ finally:
+ if original_new is None:
+ del Parameter.__new__
+ else:
+ Parameter.__new__ = original_new
+
+
+def prepare_fsdp2_qlora(
+ model,
+ dtype = None,
+ float_storage = None,
+):
+ """Make a bitsandbytes 4-bit model shardable by FSDP2: float packed storage (unless
+ `float_storage` is False, default `float_storage_survives_wrap()`), QuantState kept through
+ sharding, no mixed precision cast of packed bytes. A no-op on a model without Params4bit.
+ Returns a dict (`params4bit`, `converted`, `patched`, `reason`) for logging and tests."""
+ info = {"params4bit": 0, "converted": 0, "patched": False, "reason": None}
+ params4bit = [p for p in model.parameters() if is_params4bit(p)]
+ info["params4bit"] = len(params4bit)
+ if not params4bit:
+ return info
+ try:
+ from ..kernels import utils as kernel_utils
+ except ImportError: # loaded on its own, without the Unsloth package (tests)
+ kernel_utils = None
+ if kernel_utils is not None:
+ # Sharded 4-bit weights can now reach the LoRA kernels without their quant_state.
+ kernel_utils._FSDP2_QLORA_PREPARED = True
+ if dtype is None:
+ dtype = next(
+ (
+ getattr(p.quant_state, "dtype", None)
+ for p in params4bit
+ if getattr(p, "quant_state", None) is not None
+ ),
+ None,
+ )
+ if float_storage is None:
+ float_storage = float_storage_survives_wrap()
+ if float_storage:
+ info["converted"] = params4bit_to_float_storage(model, dtype)
+ info["patched"], info["reason"] = patch_fsdp2_params4bit()
+ patch_size_based_wrap_policy()
+ return info
+
+
+def fsdp2_qlora_wrap(model, prepare_fn = None):
+ """Around an FSDP2 wrap (accelerate's `fsdp2_prepare_model` is `prepare_fn`): prepare the 4-bit
+ weights, and let uint8 ones still left shard on torch < 2.12. Yields the `prepare_fsdp2_qlora`
+ info, or None for a model without bitsandbytes 4-bit weights (nothing is touched then)."""
+ if not any(is_params4bit(p) for p in model.parameters()):
+ yield None
+ return
+ survives = float_storage_survives_wrap(prepare_fn)
+ info = prepare_fsdp2_qlora(model, float_storage = survives)
+ info["reverted"] = 0
+ if not survives or not info["patched"]:
+ # Float storage here comes only from the caller's own BitsAndBytesConfig. An unpatched FSDP2
+ # all-gather, accelerate < 1.13's `model.to(torch.float32)` or torch < 2.8's one dtype per unit
+ # would rewrite or refuse it; the same bytes as uint8 take the path uint8 loads take.
+ info["reverted"], left = params4bit_to_uint8_storage(model)
+ if left:
+ raise RuntimeError(
+ f"Unsloth: FSDP2 cannot keep {left} float-packed 4-bit weights intact here "
+ f"({info['reason'] or 'the wrap would convert them'}). Load them with the default uint8 "
+ "bnb_4bit_quant_storage, or train with DDP."
+ )
+ info["storage"] = sorted(
+ {str(p.dtype).replace("torch.", "") for p in model.parameters() if is_params4bit(p)}
+ )
+ nonfloat = any(not (p.is_floating_point() or p.is_complex()) for p in model.parameters())
+ with contextlib.ExitStack() as stack:
+ if nonfloat:
+ stack.enter_context(nonfloat_param_guard(model))
+ if _fsdp2_checks_frozen_dtypes():
+ stack.enter_context(nonfloat_frozen_unmanaged())
+ yield info
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 6fc7c4c4ed5..433d55148a6 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -67,6 +67,7 @@ from .loader_utils import (
_prepare_compressed_tensors_model,
fsdp_will_wrap,
fsdp2_keeps_fused_lora,
+ fsdp2_requested,
planner_class_mismatch_reason,
planner_model_class,
planner_config_overrides,
@@ -84,6 +85,7 @@ from .loader_utils import (
sync_load_when_quantizing,
warn_if_bitsandbytes_quantized_nothing,
)
+from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage
from ..utils.packing import (
get_packed_info_from_kwargs,
mask_packed_boundary_labels,
@@ -3509,6 +3511,10 @@ class FastLlamaModel:
bnb_4bit_compute_dtype = dtype,
llm_int8_skip_modules = llm_int8_skip_modules,
)
+ if fsdp2_requested():
+ # FSDP2 shards packed 4-bit bytes only in a float storage dtype; the compute dtype makes its
+ # mixed precision cast of them a no-op.
+ set_fsdp2_quant_storage(bnb_config, dtype)
# Pre-quantized checkpoints use the quantization_config baked into config.json, ignoring our
# runtime BitsAndBytesConfig, so merge our skip list into it to keep task heads like `score` in
# compute dtype (#5027).
@@ -3707,6 +3713,8 @@ class FastLlamaModel:
dtype = dtype,
)
_prepare_compressed_tensors_model(model)
+ if fsdp2_requested():
+ prepare_fsdp2_qlora(model, dtype)
model.fast_generate = make_fast_generate_wrapper(model.generate)
model.fast_generate_batches = None
else:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index b48c2f2785f..e83042a6bb3 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -142,6 +142,7 @@ from .remote_code_shims import apply_remote_code_shims
from .grouped_linear_lora import register_grouped_linear_lora
from .loader_utils import (
_bnb_bits_requested,
+ fsdp2_requested,
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
planner_config_overrides,
@@ -169,6 +170,7 @@ from .loader_utils import (
sync_load_when_quantizing,
warn_if_bitsandbytes_quantized_nothing,
)
+from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage
# `unsloth.save` imports `.models.loader_utils`, so binding a name out of it here at module
# scope closes a cycle and a cold `import unsloth.save` fails on the half-built module.
# `_gpu_init` hid that by importing `.models` first; the MLX branch never reaches it. So the
@@ -3665,6 +3667,10 @@ class FastBaseModel:
bnb_4bit_compute_dtype = bnb_compute_dtype,
llm_int8_skip_modules = _skip_modules,
)
+ if fsdp2_requested():
+ # FSDP2 shards packed 4-bit bytes only in a float storage dtype; the compute dtype makes its
+ # mixed precision cast of them a no-op.
+ set_fsdp2_quant_storage(bnb_config, bnb_compute_dtype)
elif load_in_8bit:
bnb_config = BitsAndBytesConfig(
load_in_8bit = True,
@@ -3923,6 +3929,8 @@ class FastBaseModel:
_prepare_compressed_tensors_model(model, full_finetuning = full_finetuning)
if full_finetuning:
_dequantize_bitsandbytes_for_full_finetuning(model, torch_dtype, model_name)
+ elif fsdp2_requested():
+ prepare_fsdp2_qlora(model, bnb_compute_dtype)
if load_in_16bit and not load_in_4bit and not load_in_8bit:
_dequantize_leftover_fp8_params(
model,