Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: shard QLoRA 4-bit weights with fused LoRA kernels kept on (#13880)
Daniel Han086db552e645
diff --git a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py b/tests/python/test_get_lora_parameters_bias_fp8_block_size.pyindex d60bc6126a1..322fa0f29c4 100644--- a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py+++ b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py@@ -2,7 +2,7 @@ import astfrom pathlib import Path-_HELPERS = ("_packed_base", "_is_packed_state")+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")def _load_function(name):@@ -15,7 +15,7 @@ def _load_function(name):if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)]assert [f.name for f in funcs].count(name) == 1, (name, funcs)- namespace = {"getattr": getattr, "_FP8_WEIGHT_DTYPES": ()}+ namespace = {"getattr": getattr, "_FP8_WEIGHT_DTYPES": (), "_FSDP2_QLORA_PREPARED": True}module = ast.Module(body = funcs, type_ignores = [])ast.fix_missing_locations(module)exec(compile(module, str(source), "exec"), namespace)diff --git a/tests/python/test_get_lora_parameters_fp8_block_size.py b/tests/python/test_get_lora_parameters_fp8_block_size.pyindex 0905c1eb10e..62d5dbc5512 100644--- a/tests/python/test_get_lora_parameters_fp8_block_size.py+++ b/tests/python/test_get_lora_parameters_fp8_block_size.py@@ -2,7 +2,7 @@ import astfrom pathlib import Path-_HELPERS = ("_packed_base", "_is_packed_state")+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")def _load_function(name):@@ -15,7 +15,12 @@ def _load_function(name):if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)]assert [f.name for f in funcs].count(name) == 1, (name, funcs)- namespace = {"getattr": getattr, "hasattr": hasattr, "_FP8_WEIGHT_DTYPES": ()}+ namespace = {+ "getattr": getattr,+ "hasattr": hasattr,+ "_FP8_WEIGHT_DTYPES": (),+ "_FSDP2_QLORA_PREPARED": True,+ }module = ast.Module(body = funcs, type_ignores = [])ast.fix_missing_locations(module)exec(compile(module, str(source), "exec"), namespace)diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.pynew file mode 100644index 00000000000..030e183ae86--- /dev/null+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py@@ -0,0 +1,130 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 all-gathers a bitsandbytes 4-bit weight into a plain Parameter without `quant_state`. The fused+LoRA kernels must then read the Linear4bit module's own QuantState, not treat the packed (N, 1) bytes as+a dense weight."""++import ast+from pathlib import Path++import torch++_SOURCE = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")+++def _load_function(name, fsdp2_qlora_prepared = True):+ # Extract functions from kernels/utils.py without importing unsloth (which needs a GPU).+ tree = ast.parse(_SOURCE.read_text(encoding = "utf-8"))+ funcs = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.FunctionDef) and node.name in (name, *_HELPERS)+ ]+ assert [f.name for f in funcs].count(name) == 1, (name, funcs)+ namespace = {+ "torch_Tensor": torch.Tensor,+ "_FP8_WEIGHT_DTYPES": (),+ "_FSDP2_QLORA_PREPARED": fsdp2_qlora_prepared,+ }+ module = ast.Module(body = funcs, type_ignores = [])+ ast.fix_missing_locations(module)+ exec(compile(module, str(_SOURCE), "exec"), namespace)+ return namespace[name]+++class _Obj:+ pass+++class Linear4bit(torch.nn.Module):+ # Stands in for bitsandbytes.nn.Linear4bit, matched by class name.+ pass+++class _LoRA:+ def __init__(self, base_layer):+ self.base_layer = base_layer+ self.disable_adapters = False+ self.merged = False+ self.active_adapters = ["default"]+ a, b = _Obj(), _Obj()+ a.weight, b.weight = torch.zeros(4, 64), torch.zeros(32, 4)+ self.lora_A = {"default": a}+ self.lora_B = {"default": b}+ self.scaling = {"default": 2.0}+ self.lora_bias = {}+++def _fsdp2_unsharded_4bit(base_cls = Linear4bit):+ base_layer = base_cls()+ state = _Obj()+ state.dtype = torch.float16+ base_layer.quant_state = state+ # The all-gathered packed bytes: (32 * 64 / 4, 1) in fp16 storage, a plain Parameter with no quant_state.+ base_layer.weight = torch.nn.Parameter(+ torch.zeros(512, 1, dtype = torch.float16), requires_grad = False+ )+ base_layer.bias = None+ return _LoRA(base_layer), state+++def test_get_lora_parameters_falls_back_to_module_quant_state():+ get_lora_parameters = _load_function("get_lora_parameters")+ proj, state = _fsdp2_unsharded_4bit()+ W, W_quant = get_lora_parameters(proj)[:2]+ assert W is proj.base_layer.weight+ assert W_quant is state+++def test_get_lora_parameters_bias_falls_back_to_module_quant_state():+ get_lora_parameters_bias = _load_function("get_lora_parameters_bias")+ proj, state = _fsdp2_unsharded_4bit()+ assert get_lora_parameters_bias(proj)[1] is state+++def test_weight_quant_state_still_wins():+ get_lora_parameters = _load_function("get_lora_parameters")+ proj, state = _fsdp2_unsharded_4bit()+ own = _Obj()+ proj.base_layer.weight.quant_state = own+ assert get_lora_parameters(proj)[1] is own+++def test_no_fallback_for_dense_or_non_bnb_weights():+ get_lora_parameters = _load_function("get_lora_parameters")+ # A dense weight is never paired with a stray module quant_state.+ proj, _ = _fsdp2_unsharded_4bit()+ proj.base_layer.weight = torch.nn.Parameter(torch.zeros(32, 64), requires_grad = False)+ assert get_lora_parameters(proj)[1] is None+ # Nor is a module that is not a bitsandbytes Linear4bit.+ proj, _ = _fsdp2_unsharded_4bit(base_cls = torch.nn.Module)+ assert get_lora_parameters(proj)[1] is None+++def test_dense_weight_never_takes_the_module_getattr_path():+ # Every 16-bit LoRA forward (training and decode) asks for the quant state; a dense weight must+ # not pay for nn.Module.__getattr__ raising and catching AttributeError on the base layer.+ get_lora_parameters = _load_function("get_lora_parameters")+ lookups = []++ class _CountingLinear(torch.nn.Linear):+ def __getattr__(self, name):+ lookups.append(name)+ return super().__getattr__(name)++ base_layer = _CountingLinear(64, 32, bias = False)+ proj = _LoRA(base_layer)+ lookups.clear()+ assert get_lora_parameters(proj)[1] is None+ assert "quant_state" not in lookups, lookups+++def test_fallback_only_after_fsdp2_qlora_preparation():+ # Without prepare_fsdp2_qlora no weight can lose its quant_state, so the default LoRA forward keeps+ # the merge base's single getattr and never consults the base layer.+ for name in ("get_lora_parameters", "get_lora_parameters_bias"):+ fn = _load_function(name, fsdp2_qlora_prepared = False)+ proj, _ = _fsdp2_unsharded_4bit()+ assert fn(proj)[1] is Nonediff --git a/tests/test_fsdp2_qlora.py b/tests/test_fsdp2_qlora.pynew file mode 100644index 00000000000..79f13613c43--- /dev/null+++ b/tests/test_fsdp2_qlora.py@@ -0,0 +1,557 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 + bitsandbytes 4-bit (QLoRA): GPU-free checks of unsloth/models/fsdp2_qlora.py.++The module is loaded from its file (importing unsloth needs a GPU). The distributed tests shard real+bitsandbytes Linear4bit layers (CPU backend) across 2 gloo ranks with FSDP2's fully_shard.+"""++import importlib.util+import os+import socket+import sys+from pathlib import Path++import pytest+import torch++_ROOT = Path(__file__).resolve().parents[1]+_MODULE_PATH = Path(+ os.environ.get("UNSLOTH_FSDP2_QLORA_PATH", _ROOT / "unsloth" / "models" / "fsdp2_qlora.py")+)+++def _load_module():+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_qlora_under_test", _MODULE_PATH)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++bnb = pytest.importorskip("bitsandbytes")+fully_shard_mod = pytest.importorskip("torch.distributed.fsdp")+if not hasattr(fully_shard_mod, "fully_shard"):+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)+++def _linear4bit(in_features, out_features, storage, seed):+ torch.manual_seed(seed)+ lin = bnb.nn.Linear4bit(+ in_features,+ out_features,+ bias = False,+ compute_dtype = torch.bfloat16,+ quant_type = "nf4",+ quant_storage = storage,+ )+ lin.weight = bnb.nn.Params4bit(+ torch.randn(out_features, in_features) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = storage,+ blocksize = 64,+ module = lin,+ )+ return lin.to("cpu")+++class _Block(torch.nn.Module):+ def __init__(self, storage, seed):+ super().__init__()+ self.up = _linear4bit(128, 256, storage, seed)+ self.down = _linear4bit(256, 128, storage, seed + 1)+ torch.manual_seed(seed + 2)+ # Trainable fp32 adapter, like LoRA's A matrix.+ self.adapter = torch.nn.Linear(128, 8, bias = False)+ # Zero, so the sharded output can be compared bit for bit; its gradient is still nonzero.+ torch.nn.init.zeros_(self.adapter.weight)++ def forward(self, x):+ h = self.down(torch.nn.functional.silu(self.up(x)))+ return h + self.adapter(x.to(self.adapter.weight.dtype)).sum(-1, keepdim = True).to(h.dtype)+++class _Model(torch.nn.Module):+ def __init__(self, storage):+ super().__init__()+ self.layers = torch.nn.ModuleList([_Block(storage, 10 * i) for i in range(2)])++ def forward(self, x):+ for layer in self.layers:+ x = layer(x)+ return x+++def test_quant_storage_dtype():+ m = _load_module()+ assert m.fsdp2_quant_storage_dtype(torch.float16) is torch.float16+ assert m.fsdp2_quant_storage_dtype("bfloat16") is torch.bfloat16+ assert m.fsdp2_quant_storage_dtype(torch.uint8) is None+ assert m.fsdp2_quant_storage_dtype(None) is None+++def test_set_quant_storage_on_bnb_config():+ m = _load_module()+ from transformers import BitsAndBytesConfig++ cfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_compute_dtype = torch.float16)+ assert m.set_fsdp2_quant_storage(cfg, torch.float16)+ assert cfg.bnb_4bit_quant_storage == torch.float16+ # A caller's explicit float storage stays.+ cfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_quant_storage = torch.float32)+ assert not m.set_fsdp2_quant_storage(cfg, torch.bfloat16)+ assert cfg.bnb_4bit_quant_storage == torch.float32+ # 8-bit has no packed 4-bit storage.+ cfg = BitsAndBytesConfig(load_in_8bit = True)+ assert not m.set_fsdp2_quant_storage(cfg, torch.bfloat16)+ assert not m.set_fsdp2_quant_storage(None, torch.bfloat16)++[email protected]("storage", [torch.bfloat16, torch.float16])+def test_uint8_to_float_storage_is_a_view(storage):+ m = _load_module()+ lin = _linear4bit(128, 64, torch.uint8, 0)+ weight = lin.weight+ before = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)+ raw = weight.data.clone()+ model = torch.nn.Sequential(lin)+ assert m.params4bit_to_float_storage(model, storage) == 1+ assert lin.weight is weight+ assert weight.dtype == storage and weight.quant_storage == storage+ assert lin.quant_storage == storage+ assert weight.shape == (raw.numel() // 2, 1)+ assert torch.equal(weight.data.view(torch.uint8).reshape(-1), raw.reshape(-1))+ after = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)+ assert torch.equal(before, after)+ x = torch.randn(3, 128, dtype = torch.bfloat16)+ torch.testing.assert_close(lin(x), x @ before.t().to(x.dtype), rtol = 0, atol = 0)+ # Idempotent; float storage is left alone.+ assert m.params4bit_to_float_storage(model, storage) == 0+ assert m.params4bit_to_float_storage(model, torch.uint8) == 0+++def _prepare_that_upcasts(accelerator, model):+ # accelerate < 1.13 ends its FSDP2 wrap like this when mixed precision is on.+ model = model.to(torch.float32)+ return model+++def _prepare_that_keeps_dtypes(accelerator, model):+ return model+++def test_float_storage_only_when_the_wrap_does_not_upcast():+ m = _load_module()+ assert m.float_storage_survives_wrap(_prepare_that_upcasts) is False+ assert m.float_storage_survives_wrap(_prepare_that_keeps_dtypes) is True+ # The installed accelerate's own function is read through any wrapper.+ from accelerate.utils.fsdp_utils import fsdp2_prepare_model++ assert m.float_storage_survives_wrap() is m.float_storage_survives_wrap(fsdp2_prepare_model)+++def test_upcasting_wrap_keeps_uint8_and_guards_parameter_creation():+ m = _load_module()+ lin = _linear4bit(128, 64, torch.uint8, 0)+ model = torch.nn.Sequential(lin)+ with m.fsdp2_qlora_wrap(model, _prepare_that_upcasts) as info:+ assert info["params4bit"] == 1 and info["converted"] == 0+ assert info["storage"] == ["uint8"]+ # torch < 2.12 builds the sharded uint8 parameter with requires_grad = True first.+ assert torch.nn.Parameter(torch.zeros(2, dtype = torch.uint8)).requires_grad is False+ # What accelerate < 1.13 then does is harmless to uint8 bytes.+ before = lin.weight.data.clone()+ model.to(torch.float32)+ assert lin.weight.dtype == torch.uint8 and torch.equal(lin.weight.data, before)+ with pytest.raises(RuntimeError):+ torch.nn.Parameter(torch.zeros(2, dtype = torch.uint8))+++def test_wrap_is_a_noop_without_4bit_weights():+ m = _load_module()+ model = torch.nn.Linear(4, 4)+ with m.fsdp2_qlora_wrap(model, _prepare_that_keeps_dtypes) as info:+ assert info is None+ assert m.prepare_fsdp2_qlora(model)["params4bit"] == 0+++def test_anchor_check_refuses_unknown_fsdp_param():+ m = _load_module()++ class _Old:+ def _init_sharded_param(self, tensor, device):+ pass++ def init_unsharded_param(self):+ self._unsharded_param = None++ missing = m._check_fsdp_param_anchors(_Old)+ assert "FSDPParam.init_dtype_attrs" in missing++ class _Renamed(_Old):+ def init_dtype_attrs(self, mp_policy):+ self.param_dtype = None++ assert m._check_fsdp_param_anchors(_Renamed) == [+ "FSDPParam._init_sharded_param(self, param, ...)"+ ]+++def test_anchors_present_on_installed_torch():+ m = _load_module()+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam+ assert m._check_fsdp_param_anchors(FSDPParam) == []+++def test_nonfloat_guard_restores_parameter_new():+ m = _load_module()+ before = torch.nn.Parameter.__dict__.get("__new__")+ with m.nonfloat_param_guard():+ p = torch.nn.Parameter(torch.zeros(4, dtype = torch.uint8))+ assert p.requires_grad is False+ assert torch.nn.Parameter(torch.zeros(4)).requires_grad is True+ assert torch.nn.Parameter.__dict__.get("__new__") is before+ with pytest.raises(RuntimeError):+ torch.nn.Parameter(torch.zeros(4, dtype = torch.uint8))+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _worker(rank, world_size, port, case, queue):+ try:+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world_size),+ )+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard++ dist.init_process_group("gloo", rank = rank, world_size = world_size)+ m = _load_module()+ storage = torch.uint8 if case.startswith("uint8") else torch.bfloat16+ model = _Model(storage)+ reference = _Model(storage)+ if case.endswith("uniform"):+ # Every all-gather input of a block in one float dtype: FSDP2's foreach-copy path.+ for net in (model, reference):+ for layer in net.layers:+ layer.adapter.to(torch.bfloat16)+ torch.manual_seed(1234)+ x = torch.randn(4, 128, dtype = torch.bfloat16)+ expected = reference(x).detach()++ if case.startswith("unpatched"):+ info = {"patched": False}+ else:+ info = m.prepare_fsdp2_qlora(+ model, torch.bfloat16 if case == "uint8_converted" else None+ )+ if case == "uint8_converted":+ assert info["converted"] == 4, info+ # The dtype each all-gather moves its inputs in, as foreach_all_gather decided it.+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives++ gather_dtypes = []+ decide = _fsdp_collectives._get_all_gather_input_metadatas++ def _get_all_gather_input_metadatas(*args, **kwargs):+ out = decide(*args, **kwargs)+ gather_dtypes.append(str(out[2]))+ return out++ _fsdp_collectives._get_all_gather_input_metadatas = _get_all_gather_input_metadatas+ packed_bytes = sum(+ p.numel() * p.element_size() for p in model.parameters() if m.is_params4bit(p)+ )+ mesh = init_device_mesh("cpu", (world_size,))+ if case.endswith("uniform"):+ mp = MixedPrecisionPolicy()+ else:+ # fp16 all-gather: on bf16 storage this cast would scramble the packed bytes.+ mp = MixedPrecisionPolicy(+ param_dtype = torch.float16, reduce_dtype = torch.float32, cast_forward_inputs = False+ )+ for layer in model.layers:+ fully_shard(layer, mesh = mesh, mp_policy = mp)+ fully_shard(model, mesh = mesh, mp_policy = mp)++ local_bytes = 0+ for name, p in model.named_parameters():+ if "adapter" in name:+ continue+ local = p.to_local()+ local_bytes += local.numel() * local.element_size()+ seen = {}++ def hook(module, args):+ w = module.weight+ seen[id(module)] = (getattr(w, "quant_state", None) is not None, w.dtype)++ for layer in model.layers:+ layer.up.register_forward_pre_hook(hook)+ layer.down.register_forward_pre_hook(hook)+ out = model(x)+ out.float().sum().backward()+ grad_ok = all(p.grad is not None for n, p in model.named_parameters() if "adapter" in n)+ sharded_has_qs = all(+ getattr(p, "quant_state", None) is not None+ for n, p in model.named_parameters()+ if "adapter" not in n+ )+ result = dict(+ info = {k: v for k, v in info.items() if k != "reason"},+ max_diff = (out.float() - expected.float()).abs().max().item(),+ packed_bytes = packed_bytes,+ local_bytes = local_bytes,+ weight_quant_state = [v[0] for v in seen.values()],+ grad_ok = grad_ok,+ sharded_has_qs = sharded_has_qs,+ gather_dtypes = sorted(set(gather_dtypes)),+ )+ queue.put((rank, result))+ dist.destroy_process_group()+ except BaseException as error:+ import traceback+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))+++def _old_torch_worker(rank, world_size, port, case, queue):+ """torch < 2.8 (forced): uint8 weights must stay out of every FSDP2 unit, whose trainable float+ adapter would otherwise share it with them (`FSDP expects uniform original parameter dtype`)."""+ try:+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world_size),+ )+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard+ from torch.distributed.tensor import DTensor++ dist.init_process_group("gloo", rank = rank, world_size = world_size)+ m = _load_module()+ m._fsdp2_checks_frozen_dtypes = lambda: True+ model, reference = _Model(torch.uint8), _Model(torch.uint8)+ torch.manual_seed(1234)+ x = torch.randn(4, 128, dtype = torch.bfloat16)+ expected = reference(x).detach()+ mesh = init_device_mesh("cpu", (world_size,))+ with m.fsdp2_qlora_wrap(model, _prepare_that_keeps_dtypes) as info:+ for layer in model.layers:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ unit_dtypes = []+ for module in model.modules():+ state = getattr(module, "_get_fsdp_state", None)+ group = state()._fsdp_param_group if state is not None else None+ if group is not None:+ unit_dtypes.append(+ sorted({str(fp.sharded_param.dtype) for fp in group.fsdp_params})+ )+ out = model(x)+ out.float().sum().backward()+ queue.put(+ (+ rank,+ dict(+ storage = info["storage"],+ unit_dtypes = unit_dtypes,+ uint8_sharded = sum(+ isinstance(p, DTensor)+ for p in model.parameters()+ if not p.is_floating_point()+ ),+ max_diff = (out.float() - expected.float()).abs().max().item(),+ grad_ok = all(+ p.grad is not None for n, p in model.named_parameters() if "adapter" in n+ ),+ ),+ )+ )+ dist.destroy_process_group()+ except BaseException as error:+ import traceback+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))+++def _run(+ case,+ world_size = 2,+ target = None,+):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = target or _worker, args = (r, world_size, port, case, queue))+ for r in range(world_size)+ ]+ for p in procs:+ p.start()+ results = {}+ for _ in procs:+ rank, result = queue.get(timeout = 240)+ results[rank] = result+ for p in procs:+ p.join(timeout = 60)+ return results+++_RUN_DIST = pytest.mark.skipif(+ sys.platform == "win32"+ or not torch.distributed.is_available()+ or not torch.distributed.is_gloo_available(),+ reason = "needs torch.distributed with gloo",+)+++@_RUN_DIST[email protected]("case", ["bf16", "uint8_converted", "bf16_uniform"])+def test_fsdp2_shards_4bit_and_keeps_quant_state(case):+ results = _run(case)+ for rank, r in results.items():+ assert "error" not in r, r.get("error")+ assert r["info"]["patched"], r+ # The packed 4-bit bytes are split across the 2 ranks, not replicated.+ assert r["local_bytes"] * 2 == r["packed_bytes"], r+ # FSDP2's all-gathered weight still carries bitsandbytes' QuantState in forward.+ assert r["weight_quant_state"] and all(r["weight_quant_state"]), r+ assert r["sharded_has_qs"], r+ # Bit-exact vs the unsharded model: no fp16 cast touched the packed bytes.+ assert r["max_diff"] == 0.0, r+ assert r["grad_ok"], r+ # Packed bytes always travel as bytes: CUDA's float foreach copy rewrites NaN bit patterns.+ assert r["gather_dtypes"] == ["torch.uint8"], r+++@_RUN_DIST+def test_unpatched_uniform_group_gathers_as_float():+ # Control: without the patch a block whose inputs share bf16 is gathered as bf16.+ results = _run("unpatched_uniform")+ for rank, r in results.items():+ assert "error" not in r, r.get("error")+ assert "torch.bfloat16" in r["gather_dtypes"], r+++@_RUN_DIST+def test_fsdp2_mixed_precision_corrupts_unpatched_bf16_storage():+ # Control for the test above: without the patches, FSDP2 casts the packed bf16 bytes to fp16.+ results = _run("unpatched")+ for rank, r in results.items():+ print(rank, r)+ assert "error" not in r, r.get("error")+ assert r["max_diff"] > 0.0, r+++def test_patch_is_idempotent_across_imports():+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam++ assert _load_module().patch_fsdp2_params4bit() == (True, None)+ first = FSDPParam.init_dtype_attrs+ # A second copy of the module (another import path) must not wrap the wrapper.+ assert _load_module().patch_fsdp2_params4bit() == (True, None)+ assert FSDPParam.init_dtype_attrs is first+ assert getattr(first._unsloth_original, "_unsloth_original", None) is None+++def test_no_float_storage_when_fsdp2_checks_frozen_dtypes():+ # torch < 2.8 asserts one dtype per unit, frozen weights included.+ m = _load_module()+ assert m._fsdp2_checks_frozen_dtypes() is False # installed torch >= 2.8+ m._fsdp2_checks_frozen_dtypes = lambda: True+ assert m.float_storage_survives_wrap(_prepare_that_keeps_dtypes) is False+ from transformers import BitsAndBytesConfig++ cfg = BitsAndBytesConfig(load_in_4bit = True)+ assert not m.set_fsdp2_quant_storage(cfg, torch.float16)+++@_RUN_DIST+def test_old_torch_leaves_uint8_weights_out_of_the_units():+ results = _run("uint8_old_torch", target = _old_torch_worker)+ for rank, r in results.items():+ assert "error" not in r, r.get("error")+ assert r["storage"] == ["uint8"], r+ assert r["unit_dtypes"] and all(d == ["torch.float32"] for d in r["unit_dtypes"]), r+ assert r["uint8_sharded"] == 0, r+ assert r["max_diff"] == 0.0, r+ assert r["grad_ok"], r++[email protected]("why", ["upcasting_wrap", "old_torch", "unpatched"])+def test_caller_float_storage_falls_back_to_uint8_when_it_cannot_survive(monkeypatch, why):+ m = _load_module()+ lin = _linear4bit(128, 64, torch.bfloat16, 0)+ weight = lin.weight+ before = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)+ raw = weight.data.view(torch.uint8).clone()+ prepare = _prepare_that_keeps_dtypes+ if why == "upcasting_wrap":+ prepare = _prepare_that_upcasts+ elif why == "old_torch":+ monkeypatch.setattr(m, "_fsdp2_checks_frozen_dtypes", lambda: True)+ else:+ monkeypatch.setattr(m, "patch_fsdp2_params4bit", lambda: (False, "no anchors"))+ monkeypatch.setattr(m, "_missing_fsdp2_anchors", lambda: ["x"])+ model = torch.nn.Sequential(lin)+ with m.fsdp2_qlora_wrap(model, prepare) as info:+ assert info["reverted"] == 1 and info["storage"] == ["uint8"], info+ model.to(torch.float32)+ assert lin.weight is weight and weight.dtype == torch.uint8 and lin.quant_storage == torch.uint8+ assert torch.equal(weight.data.reshape(-1), raw.reshape(-1))+ assert torch.equal(bnb.functional.dequantize_4bit(weight.data, weight.quant_state), before)+++def test_float_storage_is_kept_when_it_survives():+ m = _load_module()+ lin = _linear4bit(128, 64, torch.bfloat16, 0)+ with m.fsdp2_qlora_wrap(torch.nn.Sequential(lin), _prepare_that_keeps_dtypes) as info:+ assert info["reverted"] == 0 and info["storage"] == ["bfloat16"], info++[email protected]("storage", [torch.bfloat16, torch.float32])+def test_size_based_wrap_counts_packed_weights_as_uint8(storage):+ m = _load_module()+ from accelerate.utils import fsdp_utils+ from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy++ assert m.patch_size_based_wrap_policy()+ uint8_block = _Block(torch.uint8, 0)+ packed_block = _Block(torch.uint8, 0)+ assert m.params4bit_to_float_storage(packed_block, storage) == 2+ uint8_numel = sum(p.numel() for p in uint8_block.parameters())+ plugin = type("Plugin", (), {})()+ plugin.auto_wrap_policy = size_based_auto_wrap_policy+ plugin.min_num_params = uint8_numel - 1+ plugin.transformer_cls_names_to_wrap = None+ policy = fsdp_utils.fsdp2_prepare_auto_wrap_policy(plugin, packed_block)+ assert policy(uint8_block) and policy(packed_block)+ original = fsdp_utils.fsdp2_prepare_auto_wrap_policy._unsloth_original+ assert not original(plugin, packed_block)(packed_block)+ # Only the counting changes: accelerate's FSDP2 size policy compares with `>`, so does this one.+ for threshold in (uint8_numel - 1, uint8_numel):+ plugin.min_num_params = threshold+ assert fsdp_utils.fsdp2_prepare_auto_wrap_policy(plugin, uint8_block)(+ uint8_block+ ) == original(plugin, uint8_block)(uint8_block)+++def test_unquantized_params4bit_is_not_byte_viewed():+ m = _load_module()+ lin = bnb.nn.Linear4bit(128, 64, bias = False, quant_storage = torch.bfloat16)+ dense = lin.weight.data.clone()+ assert not lin.weight.bnb_quantized and lin.weight.is_floating_point()+ assert m.params4bit_to_uint8_storage(torch.nn.Sequential(lin)) == (0, 0)+ assert lin.weight.dtype == dense.dtype and torch.equal(lin.weight.data, dense)diff --git a/tests/test_fsdp2_qlora_prepare_order.py b/tests/test_fsdp2_qlora_prepare_order.pynew file mode 100644index 00000000000..ac263027c85--- /dev/null+++ b/tests/test_fsdp2_qlora_prepare_order.py@@ -0,0 +1,56 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The FSDP2 prepare hook falls back to uint8 4-bit storage before unifying frozen dtypes (torch < 2.8,+forced here), so float-packed weights from an explicit bnb_4bit_quant_storage do not trip it."""++import types++import pytest++torch = pytest.importorskip("torch")+bnb = pytest.importorskip("bitsandbytes")+pytest.importorskip("accelerate")++from unsloth.models import fsdp2, fsdp2_qlora # noqa: E402+++def test_old_torch_float_packed_qlora_without_mixed_precision(monkeypatch):+ import accelerate.accelerator as accelerator_module++ torch.manual_seed(0)+ lin = bnb.nn.Linear4bit(+ 128,+ 64,+ bias = False,+ compute_dtype = torch.bfloat16,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ )+ lin.weight = bnb.nn.Params4bit(+ torch.randn(64, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ blocksize = 64,+ module = lin,+ )+ model = torch.nn.Sequential(lin.to("cpu"), torch.nn.Linear(64, 8))+ assert lin.weight.dtype == torch.bfloat16 and lin.weight.bnb_quantized++ seen = {}++ def stub(accelerator, model):+ seen["dtypes"] = sorted({str(p.dtype) for p in model.parameters()})+ return model++ monkeypatch.setattr(accelerator_module, "fsdp2_prepare_model", stub)+ monkeypatch.setattr(fsdp2, "_fsdp2_checks_frozen_dtypes", lambda: True)+ monkeypatch.setattr(fsdp2_qlora, "_fsdp2_checks_frozen_dtypes", lambda: True)+ assert fsdp2.install_fsdp2_prepare_hook()+ plugin = types.SimpleNamespace(mixed_precision_policy = None)+ accelerator = types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = "no"+ )+ accelerator_module.fsdp2_prepare_model(accelerator, model)+ assert seen["dtypes"] == ["torch.float32", "torch.uint8"], seendiff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 433f03515b4..685d6d14297 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -12,13 +12,14 @@import sysimport torchfrom unsloth_zoo.utils import Version+from . import utils as _kernel_utilsfrom .utils import (_has_active_dora_adapter,_has_active_lora_bias,_has_multiple_active_adapters,addmm_,fast_dequantize,- QUANT_STATE,+ _base_quant_state,get_lora_parameters,get_lora_parameters_bias,matmul_lora,@@ -749,7 +750,10 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:lora_B = self.lora_B[active_adapter].weightscaling = self.scaling[active_adapter]W = self.base_layer.weight- return _apply(LoRA_W, x, W, QUANT_STATE(W), lora_A, lora_B, scaling)+ W_quant = getattr(W, "quant_state", None)+ if W_quant is None and _kernel_utils._FSDP2_QLORA_PREPARED:+ W_quant = _base_quant_state(self.base_layer, W)+ return _apply(LoRA_W, x, W, W_quant, lora_A, lora_B, scaling)passpassdiff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex f9a59a69d0e..d650ede50a3 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -327,6 +327,31 @@ def QUANT_STATE(W):return getattr(W, "quant_state", None)+# Set by models/fsdp2_qlora.prepare_fsdp2_qlora: only FSDP2-sharded 4-bit weights lose quant_state, so every+# other LoRA forward keeps the single getattr(W, "quant_state") and never reaches _base_quant_state.+_FSDP2_QLORA_PREPARED = False+++def _base_quant_state(base_layer, W):+ """W's quant state, else its bitsandbytes Linear4bit's own copy of it. FSDP2 all-gathers the packed+ bytes into a plain Parameter that drops `quant_state`; read without it, the packed (N, 1) buffer+ would be taken for a dense weight."""+ W_quant = getattr(W, "quant_state", None)+ if W_quant is not None:+ return W_quant+ # Linear4bit keeps its copy as a plain attribute. getattr would go through nn.Module.__getattr__,+ # which raises and catches for every dense weight: 10x this function's cost on each LoRA forward.+ W_quant = base_layer.__dict__.get("quant_state", None)+ if (+ W_quant is None+ or getattr(W, "ndim", 0) != 2+ or W.shape[1] != 1+ or not any(cls.__name__ == "Linear4bit" for cls in type(base_layer).__mro__)+ ):+ return None+ return W_quant++# fp8 weight dtypes: a weight_scale / weight_scale_inv is only a quant state while the weight# itself is still fp8. compressed-tensors layers expose an already-dequantized bf16 weight at# forward time while keeping a weight_scale, and reading that as a quant state routes it into@@ -432,13 +457,16 @@ def get_lora_parameters(proj):if weight_fake_quantizer is not None:W = weight_fake_quantizer(W)- # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed- # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.W_quant = getattr(W, "quant_state", None)- if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:- W_quant = getattr(base_layer, "weight_scale_inv", None)- if W_quant is None:- W_quant = getattr(base_layer, "weight_scale", None)+ if W_quant is None:+ if _FSDP2_QLORA_PREPARED:+ W_quant = _base_quant_state(base_layer, W)+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.+ if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:+ W_quant = getattr(base_layer, "weight_scale_inv", None)+ if W_quant is None:+ W_quant = getattr(base_layer, "weight_scale", None)if getattr(base_layer, "quant_method", None) == "fp8":W.block_size = getattr(base_layer, "block_size", [128, 128])@@ -490,13 +518,16 @@ def get_lora_parameters_bias(proj):packed = _packed_base(base_layer)W = base_layer.weight if packed is None else packed[0]- # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed- # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.W_quant = getattr(W, "quant_state", None)- if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:- W_quant = getattr(base_layer, "weight_scale_inv", None)- if W_quant is None:- W_quant = getattr(base_layer, "weight_scale", None)+ if W_quant is None:+ if _FSDP2_QLORA_PREPARED:+ W_quant = _base_quant_state(base_layer, W)+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.+ if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:+ W_quant = getattr(base_layer, "weight_scale_inv", None)+ if W_quant is None:+ W_quant = getattr(base_layer, "weight_scale", None)if getattr(base_layer, "quant_method", None) == "fp8":W.block_size = getattr(base_layer, "block_size", [128, 128])diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.pyindex ab91d92ee17..f4a9cf7874e 100644--- a/unsloth/models/fsdp2.py+++ b/unsloth/models/fsdp2.py@@ -3,7 +3,8 @@"""What Unsloth adds around accelerate's `fsdp2_prepare_model` (which already wraps per block,keeps the [norm, lm_head] tail gathered, groups tied embeddings). `UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1`-skips the mixed precision fill and the last-block setting.+skips the mixed precision fill and the last-block setting. bitsandbytes 4-bit (QLoRA) weights are+sharded too, keeping their QuantState (`fsdp2_qlora.py`)."""import contextlib@@ -424,10 +425,23 @@ def install_fsdp2_prepare_hook():@functools.wraps(original)def fsdp2_prepare_model(accelerator, model, *args, **kwargs):- fill_fsdp2_mixed_precision(accelerator)- unify_trainable_dtypes(accelerator, model)- unify_frozen_dtypes(accelerator, model)- model = original(accelerator, model, *args, **kwargs)+ from .fsdp2_qlora import fsdp2_qlora_wrap++ # The 4-bit fallback to uint8 storage first: float-packed weights would count as frozen+ # 16-bit weights in the dtype unification.+ with fsdp2_qlora_wrap(model, original) as qlora:+ fill_fsdp2_mixed_precision(accelerator)+ unify_trainable_dtypes(accelerator, model)+ unify_frozen_dtypes(accelerator, model)+ model = original(accelerator, model, *args, **kwargs)+ if qlora is not None:+ note = "" if qlora["patched"] else f" FSDPParam was not patched: {qlora['reason']}."+ if qlora.get("reverted"):+ note += f" {qlora['reverted']} float-packed weights kept as uint8, which this wrap preserves."+ _print_rank0(+ f"Unsloth: FSDP2 QLoRA: {qlora['params4bit']} 4-bit weights packed as "+ f"{'/'.join(qlora['storage'])} ({qlora['converted']} repacked from uint8).{note}"+ )after_fsdp2_wrap(accelerator, model)return modeldiff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.pynew file mode 100644index 00000000000..6a65e96cea8--- /dev/null+++ b/unsloth/models/fsdp2_qlora.py@@ -0,0 +1,534 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""QLoRA (bitsandbytes 4-bit) under FSDP2.++FSDP2 can only shard a frozen 4-bit weight whose packed bytes are stored in a float dtype: uint8 cannot+become a sharded nn.Parameter before torch 2.12 and accelerate leaves it replicated on every rank. So:+1. a fresh quantization stores the packed bytes as the compute dtype (`bnb_4bit_quant_storage`), and a+ pre-quantized uint8 checkpoint is reinterpreted (a view, no copy) as the same dtype after loading;+2. FSDPParam keeps the bitsandbytes QuantState on the sharded and the all-gathered parameter (FSDP2+ rebuilds both as plain Parameters, which drops it), so the fused LoRA kernels and bnb's own forward+ still read `weight.quant_state`;+3. FSDP2 mixed precision never casts packed 4-bit bytes (or any other non-float parameter), and an+ all-gather holding them moves bytes: when every input of a group shares one float dtype FSDP2+ copies them with `torch._foreach_copy_`, which on CUDA rewrites float16 / bfloat16 NaN bit patterns+ (about 1 in 60 packed float16 words) to the canonical NaN;+4. `nonfloat_param_guard` lets torch < 2.12 shard what is still uint8; torch < 2.8, which wants one+ dtype per unit frozen weights included, leaves uint8 weights out of the units (replicated);+5. SIZE_BASED_WRAP counts float-packed weights at their uint8 numel, so wrap decisions do not move.+Float storage is used only when 2 and 3 can be patched and the wrap does not upcast the whole model.+Method wrappers (not source rewrites, as in axolotl's monkeypatch/fsdp2_qlora.py) hold on torch 2.6 .. 2.14.+"""++__all__ = [+ "is_params4bit",+ "float_storage_survives_wrap",+ "fsdp2_quant_storage_dtype",+ "set_fsdp2_quant_storage",+ "params4bit_to_float_storage",+ "params4bit_to_uint8_storage",+ "patch_fsdp2_params4bit",+ "nonfloat_param_guard",+ "prepare_fsdp2_qlora",+ "fsdp2_qlora_wrap",+]++import contextlib+import inspect++import torch++_FLOAT_STORAGE = (torch.float16, torch.bfloat16, torch.float32)+_QUANT_ATTRS = (+ "quant_state",+ "blocksize",+ "compress_statistics",+ "quant_type",+ "quant_storage",+ "bnb_quantized",+)+++def is_params4bit(tensor):+ return any(cls.__name__ == "Params4bit" for cls in type(tensor).__mro__)+++def float_storage_survives_wrap(prepare_fn = None):+ """Whether 4-bit weights may be packed in a float dtype for FSDP2. False when this torch's FSDP2+ cannot be patched (`patch_fsdp2_params4bit`), checks frozen dtypes per unit (torch < 2.8), or when accelerate's FSDP2 wrap ends with+ `model.to(torch.float32)` (accelerate < 1.13 with mixed precision), which converts float-stored+ packed bytes as if they were weights. uint8 storage is safe from both, so 4-bit weights then stay+ uint8 (left replicated by accelerate, but correct)."""+ if prepare_fn is None:+ try:+ from accelerate.utils.fsdp_utils import fsdp2_prepare_model as prepare_fn+ except Exception:+ prepare_fn = None+ if _missing_fsdp2_anchors():+ # Unpatched FSDP2 would scramble float-stored packed bytes (see 3 above).+ return False+ if _fsdp2_checks_frozen_dtypes():+ # torch < 2.8 wants one dtype per unit, frozen weights included: packed bytes in the compute+ # dtype next to float32 LoRA / upcast weights would fail it, while uint8 is left out by accelerate.+ return False+ if prepare_fn is None:+ return True+ prepare_fn = inspect.unwrap(prepare_fn)+ try:+ source = inspect.getsource(prepare_fn)+ except (OSError, TypeError):+ return False+ return "model.to(torch.float32)" not in source+++def _fsdp2_checks_frozen_dtypes():+ try:+ from torch.distributed.fsdp._fully_shard._fsdp_param_group import FSDPParamGroup+ return "requires_grad" not in inspect.getsource(FSDPParamGroup._init_mp_dtypes)+ except Exception:+ return True+++def fsdp2_quant_storage_dtype(dtype):+ """The `bnb_4bit_quant_storage` FSDP2 can shard: the compute dtype, so the mixed precision cast+ of the all-gather is a no-op on the packed bytes. None for a dtype bitsandbytes cannot store in."""+ if isinstance(dtype, str):+ dtype = getattr(torch, dtype, None)+ return dtype if dtype in _FLOAT_STORAGE else None+++def set_fsdp2_quant_storage(quantization_config, dtype):+ """Store the packed 4-bit bytes of a BitsAndBytesConfig in `dtype`. Returns True when changed.+ A caller's explicit float storage is kept; only the uint8 default is replaced, and only when the+ FSDP2 wrap will not upcast the whole model (`float_storage_survives_wrap`)."""+ storage = fsdp2_quant_storage_dtype(dtype)+ if quantization_config is None or storage is None or not float_storage_survives_wrap():+ return False+ if not getattr(quantization_config, "load_in_4bit", False):+ return False+ current = getattr(quantization_config, "bnb_4bit_quant_storage", None)+ if isinstance(current, str):+ current = getattr(torch, current, None)+ if current is not None and current != torch.uint8:+ return False+ quantization_config.bnb_4bit_quant_storage = storage+ return True+++def params4bit_to_float_storage(model, dtype):+ """Reinterpret every uint8 Params4bit (a pre-quantized checkpoint) as `dtype` storage in place.+ The bytes, the QuantState and the Parameter object are unchanged; only the element type of the+ packed buffer differs, exactly what `bnb_4bit_quant_storage = dtype` would have produced.+ Returns the number of weights converted."""+ storage = fsdp2_quant_storage_dtype(dtype)+ if storage is None:+ return 0+ itemsize = torch.empty((), dtype = storage).element_size()+ n_converted = 0+ for module in model.modules():+ for name, param in list(module._parameters.items()):+ if param is None or not is_params4bit(param) or param.dtype != torch.uint8:+ continue+ nbytes = param.numel()+ if not param.is_contiguous() or nbytes % itemsize != 0 or param.device.type == "meta":+ continue+ param.data = param.data.reshape(-1).view(storage).reshape(-1, 1)+ param.quant_storage = storage+ if name == "weight" and hasattr(module, "quant_storage"):+ # Linear4bit rebuilds a lost Params4bit with it (fix_4bit_weight_quant_state_from_module).+ module.quant_storage = storage+ n_converted += 1+ return n_converted+++def params4bit_to_uint8_storage(model):+ """Inverse of `params4bit_to_float_storage`: reinterpret float-packed Params4bit as the uint8+ bitsandbytes stores by default. Returns (converted, left), left = those that could not be viewed."""+ converted = left = 0+ for module in model.modules():+ for name, param in list(module._parameters.items()):+ if param is None or not is_params4bit(param) or not param.is_floating_point():+ continue+ if (+ not getattr(param, "bnb_quantized", False)+ or getattr(param, "quant_state", None) is None+ ):+ # Still dense values bitsandbytes quantizes on the next device move, not packed bytes.+ continue+ if not param.is_contiguous() or param.device.type == "meta":+ left += 1+ continue+ param.data = param.data.reshape(-1).view(torch.uint8).reshape(-1, 1)+ param.quant_storage = torch.uint8+ if name == "weight" and hasattr(module, "quant_storage"):+ module.quant_storage = torch.uint8+ converted += 1+ return converted, left+++def _packed_numel(param):+ if is_params4bit(param) and param.is_floating_point():+ return param.numel() * param.element_size()+ return param.numel()+++def patch_size_based_wrap_policy():+ """accelerate's FSDP2 SIZE_BASED_WRAP sums `p.numel()`, which float packing divides by the storage+ itemsize; count those weights as the uint8 bytes they hold so the same blocks get wrapped."""+ try:+ import accelerate.utils.fsdp_utils as fsdp_utils+ from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy+ except Exception:+ return False+ original = getattr(fsdp_utils, "fsdp2_prepare_auto_wrap_policy", None)+ if original is None:+ return False+ if getattr(original, "_unsloth_original", None) is not None:+ return True++ def fsdp2_prepare_auto_wrap_policy(fsdp2_plugin, *args, **kwargs):+ policy = original(fsdp2_plugin, *args, **kwargs)+ fn = getattr(fsdp2_plugin, "auto_wrap_policy", None)+ fn = getattr(fn, "func", fn)+ min_num_params = getattr(fsdp2_plugin, "min_num_params", None)+ if policy is None or fn is not size_based_auto_wrap_policy or min_num_params is None:+ return policy++ def size_policy(module):+ return sum(_packed_numel(p) for p in module.parameters()) > min_num_params++ return size_policy++ fsdp2_prepare_auto_wrap_policy.__name__ = original.__name__+ fsdp2_prepare_auto_wrap_policy.__doc__ = original.__doc__+ fsdp2_prepare_auto_wrap_policy._unsloth_original = original+ fsdp_utils.fsdp2_prepare_auto_wrap_policy = fsdp2_prepare_auto_wrap_policy+ return True+++def nonfloat_frozen_unmanaged():+ """While FSDP2 shards on torch < 2.8: leave frozen non-float parameters (uint8 Params4bit) out of+ every unit, as accelerate >= 1.14 does with `ignored_params` on torch >= 2.7. torch < 2.8 asserts one+ original dtype over all of a unit's parameters, so uint8 next to float LoRA weights cannot be sharded."""+ modules = []+ try:+ from torch.distributed.fsdp._fully_shard import _fully_shard as _fully_shard_module+ modules.append(_fully_shard_module)+ except Exception:+ pass+ try:+ from torch.distributed.fsdp._fully_shard import _fsdp_init as _fsdp_init_module+ modules.append(_fsdp_init_module)+ except Exception:+ pass++ patched = []+ for module in modules:+ original = getattr(module, "_get_managed_states", None)+ if original is None:+ continue++ def _get_managed_states(+ *args,+ _original = original,+ **kwargs,+ ):+ params, buffers = _original(*args, **kwargs)+ params = [+ p for p in params if p.requires_grad or p.is_floating_point() or p.is_complex()+ ]+ return params, buffers++ module._get_managed_states = _get_managed_states+ patched.append((module, original))+ try:+ yield len(patched)+ finally:+ for module, original in patched:+ module._get_managed_states = original+++def _quant_meta(param):+ if not is_params4bit(param) or getattr(param, "quant_state", None) is None:+ return None+ return {k: getattr(param, k) for k in _QUANT_ATTRS if hasattr(param, k)}+++def _attach_quant_meta(tensor, meta):+ if tensor is None:+ return+ for k, v in meta.items():+ try:+ setattr(tensor, k, v)+ except Exception:+ pass+++def _check_fsdp_param_anchors(FSDPParam):+ """The FSDPParam surface the wrappers rely on, in torch 2.6 .. 2.14. A string per missing anchor."""+ missing = []+ methods = {}+ for name in ("_init_sharded_param", "init_unsharded_param", "init_dtype_attrs"):+ method = getattr(FSDPParam, name, None)+ if not callable(method):+ missing.append(f"FSDPParam.{name}")+ methods[name] = getattr(method, "_unsloth_original", method)+ if missing:+ return missing+ try:+ params = list(inspect.signature(methods["_init_sharded_param"]).parameters)+ if len(params) < 2 or params[1] != "param":+ missing.append("FSDPParam._init_sharded_param(self, param, ...)")+ except (TypeError, ValueError):+ missing.append("FSDPParam._init_sharded_param signature")+ try:+ if "self._unsharded_param" not in inspect.getsource(methods["init_unsharded_param"]):+ missing.append("FSDPParam.init_unsharded_param sets self._unsharded_param")+ if "self.param_dtype" not in inspect.getsource(methods["init_dtype_attrs"]):+ missing.append("FSDPParam.init_dtype_attrs sets self.param_dtype")+ except (OSError, TypeError):+ missing.append("FSDPParam source")+ return missing+++def _check_collectives_anchors(collectives):+ """foreach_all_gather picks the collective dtype in `_get_all_gather_input_metadatas` (uint8 when+ the inputs' dtypes differ) from what `_get_param_all_gather_inputs` returned, in torch 2.6 .. 2.14."""+ missing = []+ for name in (+ "foreach_all_gather",+ "_get_param_all_gather_inputs",+ "_get_all_gather_input_metadatas",+ ):+ if not callable(getattr(collectives, name, None)):+ missing.append(f"_fsdp_collectives.{name}")+ if missing:+ return missing+ try:+ source = inspect.getsource(collectives.foreach_all_gather)+ except (OSError, TypeError):+ return ["_fsdp_collectives source"]+ for anchor in (+ "_get_param_all_gather_inputs(fsdp_params)",+ "_get_all_gather_input_metadatas(",+ "if dtype == torch.uint8:",+ ):+ if anchor not in source:+ missing.append(f"foreach_all_gather: {anchor}")+ return missing+++def _missing_fsdp2_anchors():+ try:+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam+ except Exception as error:+ return [f"FSDP2 ({type(error).__name__})"]+ return _check_fsdp_param_anchors(FSDPParam) + _check_collectives_anchors(_fsdp_collectives)+++def _is_packed_fsdp_param(fsdp_param):+ return getattr(fsdp_param, "_unsloth_quant_meta", None) is not None+++# Set by _get_param_all_gather_inputs, read by the _get_all_gather_input_metadatas call right after it+# in the same foreach_all_gather.+_BYTE_ALL_GATHER = [False]+_PATCH_STATE = {"patched": False, "reason": None}+++def patch_fsdp2_params4bit():+ """Wrap FSDPParam (and the all-gather's dtype choice) so bitsandbytes 4-bit weights keep their+ QuantState and their packed bytes through FSDP2. Idempotent. Returns (ok, reason): ok is False, with the missing anchors, when this+ torch has no FSDP2 or a FSDPParam this was not written against; nothing is patched then."""+ if _PATCH_STATE["patched"]:+ return True, None+ missing = _missing_fsdp2_anchors()+ if missing:+ return False, f"torch {torch.__version__} FSDP2 lacks " + ", ".join(missing)+ from torch.distributed.fsdp._fully_shard import _fsdp_collectives+ from torch.distributed.fsdp._fully_shard._fsdp_param import FSDPParam++ if getattr(FSDPParam.init_dtype_attrs, "_unsloth_original", None) is not None:+ _PATCH_STATE["patched"] = True+ return True, None++ original_init_sharded_param = FSDPParam._init_sharded_param+ original_init_unsharded_param = FSDPParam.init_unsharded_param+ original_init_dtype_attrs = FSDPParam.init_dtype_attrs++ def _init_sharded_param(self, param, *args, **kwargs):+ meta = _quant_meta(param)+ out = original_init_sharded_param(self, param, *args, **kwargs)+ if meta is not None:+ self._unsloth_quant_meta = meta+ _attach_quant_meta(getattr(self, "sharded_param", None), meta)+ return out++ def init_unsharded_param(self, *args, **kwargs):+ out = original_init_unsharded_param(self, *args, **kwargs)+ meta = getattr(self, "_unsloth_quant_meta", None)+ if meta is not None:+ unsharded = getattr(self, "_unsharded_param", None)+ if (+ unsharded is not None+ and getattr(unsharded, "quant_state", None) is not meta["quant_state"]+ ):+ _attach_quant_meta(unsharded, meta)+ return out++ def init_dtype_attrs(self, mp_policy, *args, **kwargs):+ out = original_init_dtype_attrs(self, mp_policy, *args, **kwargs)+ if getattr(self, "param_dtype", None) is None:+ return out+ # A cast reinterprets nothing: it converts values, which destroys packed 4-bit bytes. Leave+ # tensor subclasses with their own all-gather hooks (torchao NF4Tensor) to those hooks.+ sharded = getattr(self, "sharded_param", None)+ local = getattr(sharded, "_local_tensor", sharded)+ packed = getattr(self, "_unsloth_quant_meta", None) is not None or (+ sharded is not None and not sharded.is_floating_point()+ )+ if packed and not hasattr(local, "fsdp_pre_all_gather"):+ self.param_dtype = None+ return out++ original_get_inputs = _fsdp_collectives._get_param_all_gather_inputs+ original_get_metadatas = _fsdp_collectives._get_all_gather_input_metadatas++ def _get_param_all_gather_inputs(fsdp_params, *args, **kwargs):+ out = original_get_inputs(fsdp_params, *args, **kwargs)+ _BYTE_ALL_GATHER[0] = any(_is_packed_fsdp_param(p) for p in fsdp_params)+ return out++ def _get_all_gather_input_metadatas(*args, **kwargs):+ dtypes, numels, dtype = original_get_metadatas(*args, **kwargs)+ if _BYTE_ALL_GATHER[0] and dtype != torch.uint8:+ dtype = torch.uint8+ _BYTE_ALL_GATHER[0] = False+ return dtypes, numels, dtype++ for fn, original in (+ (_init_sharded_param, original_init_sharded_param),+ (init_unsharded_param, original_init_unsharded_param),+ (init_dtype_attrs, original_init_dtype_attrs),+ (_get_param_all_gather_inputs, original_get_inputs),+ (_get_all_gather_input_metadatas, original_get_metadatas),+ ):+ fn.__name__ = original.__name__+ fn.__qualname__ = getattr(original, "__qualname__", original.__name__)+ fn.__doc__ = original.__doc__+ fn._unsloth_original = original++ FSDPParam._init_sharded_param = _init_sharded_param+ FSDPParam.init_unsharded_param = init_unsharded_param+ FSDPParam.init_dtype_attrs = init_dtype_attrs+ _fsdp_collectives._get_param_all_gather_inputs = _get_param_all_gather_inputs+ _fsdp_collectives._get_all_gather_input_metadatas = _get_all_gather_input_metadatas+ _PATCH_STATE["patched"] = True+ return True, None+++def nonfloat_param_guard(model = None):+ """While FSDP2 shards: freeze non-float parameters, and make `nn.Parameter(x)` default to+ requires_grad = False for non-float `x`. torch < 2.12 builds the sharded parameter with the default+ requires_grad = True before copying the flag over, which raises on uint8. Restored on exit."""+ if model is not None:+ for p in model.parameters():+ if p.requires_grad and not (p.is_floating_point() or p.is_complex()):+ p.requires_grad_(False)+ Parameter = torch.nn.Parameter+ original_new = Parameter.__dict__.get("__new__")++ def __new__(+ cls,+ data = None,+ requires_grad = True,+ ):+ if data is not None and not (data.is_floating_point() or data.is_complex()):+ requires_grad = False+ return original_new(cls, data, requires_grad)++ Parameter.__new__ = __new__+ try:+ yield+ finally:+ if original_new is None:+ del Parameter.__new__+ else:+ Parameter.__new__ = original_new+++def prepare_fsdp2_qlora(+ model,+ dtype = None,+ float_storage = None,+):+ """Make a bitsandbytes 4-bit model shardable by FSDP2: float packed storage (unless+ `float_storage` is False, default `float_storage_survives_wrap()`), QuantState kept through+ sharding, no mixed precision cast of packed bytes. A no-op on a model without Params4bit.+ Returns a dict (`params4bit`, `converted`, `patched`, `reason`) for logging and tests."""+ info = {"params4bit": 0, "converted": 0, "patched": False, "reason": None}+ params4bit = [p for p in model.parameters() if is_params4bit(p)]+ info["params4bit"] = len(params4bit)+ if not params4bit:+ return info+ try:+ from ..kernels import utils as kernel_utils+ except ImportError: # loaded on its own, without the Unsloth package (tests)+ kernel_utils = None+ if kernel_utils is not None:+ # Sharded 4-bit weights can now reach the LoRA kernels without their quant_state.+ kernel_utils._FSDP2_QLORA_PREPARED = True+ if dtype is None:+ dtype = next(+ (+ getattr(p.quant_state, "dtype", None)+ for p in params4bit+ if getattr(p, "quant_state", None) is not None+ ),+ None,+ )+ if float_storage is None:+ float_storage = float_storage_survives_wrap()+ if float_storage:+ info["converted"] = params4bit_to_float_storage(model, dtype)+ info["patched"], info["reason"] = patch_fsdp2_params4bit()+ patch_size_based_wrap_policy()+ return info+++def fsdp2_qlora_wrap(model, prepare_fn = None):+ """Around an FSDP2 wrap (accelerate's `fsdp2_prepare_model` is `prepare_fn`): prepare the 4-bit+ weights, and let uint8 ones still left shard on torch < 2.12. Yields the `prepare_fsdp2_qlora`+ info, or None for a model without bitsandbytes 4-bit weights (nothing is touched then)."""+ if not any(is_params4bit(p) for p in model.parameters()):+ yield None+ return+ survives = float_storage_survives_wrap(prepare_fn)+ info = prepare_fsdp2_qlora(model, float_storage = survives)+ info["reverted"] = 0+ if not survives or not info["patched"]:+ # Float storage here comes only from the caller's own BitsAndBytesConfig. An unpatched FSDP2+ # all-gather, accelerate < 1.13's `model.to(torch.float32)` or torch < 2.8's one dtype per unit+ # would rewrite or refuse it; the same bytes as uint8 take the path uint8 loads take.+ info["reverted"], left = params4bit_to_uint8_storage(model)+ if left:+ raise RuntimeError(+ f"Unsloth: FSDP2 cannot keep {left} float-packed 4-bit weights intact here "+ f"({info['reason'] or 'the wrap would convert them'}). Load them with the default uint8 "+ "bnb_4bit_quant_storage, or train with DDP."+ )+ info["storage"] = sorted(+ {str(p.dtype).replace("torch.", "") for p in model.parameters() if is_params4bit(p)}+ )+ nonfloat = any(not (p.is_floating_point() or p.is_complex()) for p in model.parameters())+ with contextlib.ExitStack() as stack:+ if nonfloat:+ stack.enter_context(nonfloat_param_guard(model))+ if _fsdp2_checks_frozen_dtypes():+ stack.enter_context(nonfloat_frozen_unmanaged())+ yield infodiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 6fc7c4c4ed5..433d55148a6 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -67,6 +67,7 @@ from .loader_utils import (_prepare_compressed_tensors_model,fsdp_will_wrap,fsdp2_keeps_fused_lora,+ fsdp2_requested,planner_class_mismatch_reason,planner_model_class,planner_config_overrides,@@ -84,6 +85,7 @@ from .loader_utils import (sync_load_when_quantizing,warn_if_bitsandbytes_quantized_nothing,)+from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storagefrom ..utils.packing import (get_packed_info_from_kwargs,mask_packed_boundary_labels,@@ -3509,6 +3511,10 @@ class FastLlamaModel:bnb_4bit_compute_dtype = dtype,llm_int8_skip_modules = llm_int8_skip_modules,)+ if fsdp2_requested():+ # FSDP2 shards packed 4-bit bytes only in a float storage dtype; the compute dtype makes its+ # mixed precision cast of them a no-op.+ set_fsdp2_quant_storage(bnb_config, dtype)# Pre-quantized checkpoints use the quantization_config baked into config.json, ignoring our# runtime BitsAndBytesConfig, so merge our skip list into it to keep task heads like `score` in# compute dtype (#5027).@@ -3707,6 +3713,8 @@ class FastLlamaModel:dtype = dtype,)_prepare_compressed_tensors_model(model)+ if fsdp2_requested():+ prepare_fsdp2_qlora(model, dtype)model.fast_generate = make_fast_generate_wrapper(model.generate)model.fast_generate_batches = Noneelse:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex b48c2f2785f..e83042a6bb3 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -142,6 +142,7 @@ from .remote_code_shims import apply_remote_code_shimsfrom .grouped_linear_lora import register_grouped_linear_lorafrom .loader_utils import (_bnb_bits_requested,+ fsdp2_requested,DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,planner_config_overrides,@@ -169,6 +170,7 @@ from .loader_utils import (sync_load_when_quantizing,warn_if_bitsandbytes_quantized_nothing,)+from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage# `unsloth.save` imports `.models.loader_utils`, so binding a name out of it here at module# scope closes a cycle and a cold `import unsloth.save` fails on the half-built module.# `_gpu_init` hid that by importing `.models` first; the MLX branch never reaches it. So the@@ -3665,6 +3667,10 @@ class FastBaseModel:bnb_4bit_compute_dtype = bnb_compute_dtype,llm_int8_skip_modules = _skip_modules,)+ if fsdp2_requested():+ # FSDP2 shards packed 4-bit bytes only in a float storage dtype; the compute dtype makes its+ # mixed precision cast of them a no-op.+ set_fsdp2_quant_storage(bnb_config, bnb_compute_dtype)elif load_in_8bit:bnb_config = BitsAndBytesConfig(load_in_8bit = True,@@ -3923,6 +3929,8 @@ class FastBaseModel:_prepare_compressed_tensors_model(model, full_finetuning = full_finetuning)if full_finetuning:_dequantize_bitsandbytes_for_full_finetuning(model, torch_dtype, model_name)+ elif fsdp2_requested():+ prepare_fsdp2_qlora(model, bnb_compute_dtype)if load_in_16bit and not load_in_4bit and not load_in_8bit:_dequantize_leftover_fp8_params(model,