Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 QLoRA: keep packed 4-bit bytes out of the loader dtype pass, fix merged_4bit save (#14165)

Daniel Han37745d5ad1f7
diff --git a/tests/test_fsdp2_qlora_dtype_pass.py b/tests/test_fsdp2_qlora_dtype_pass.py
new file mode 100644
index 00000000000..ce24e480f08
--- /dev/null
+++ b/tests/test_fsdp2_qlora_dtype_pass.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 packs 4-bit weights in a float dtype. The vision loader's dtype pass (unsloth_zoo's
+patch_model_and_tokenizer: `module.to(module._pre_set_compute_dtype)`, set on gpt-oss's 4-bit
+down_projs when loading in float16) would then cast the packed bytes as if they were weights.
+`packed_4bit_as_uint8` shows them as uint8 for that pass, as without FSDP2."""
+
+import ast
+import importlib.util
+from pathlib import Path
+
+import pytest
+import torch
+
+bnb = pytest.importorskip("bitsandbytes")
+
+_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _load_module():
+ path = _ROOT / "unsloth" / "models" / "fsdp2_qlora.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_qlora_dtype_pass", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _float_packed_linear4bit(storage):
+ torch.manual_seed(0)
+ lin = bnb.nn.Linear4bit(
+ 128, 64, bias = True, compute_dtype = torch.float16, quant_type = "nf4", quant_storage = storage
+ )
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(64, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = storage,
+ blocksize = 64,
+ module = lin,
+ )
+ lin = lin.to("cpu")
+ lin.bias = torch.nn.Parameter(torch.ones(64, dtype = torch.float16))
+ assert lin.weight.dtype == storage and lin.weight.bnb_quantized
+ return lin
+
+
+def _dtype_pass(model):
+ # unsloth_zoo.patching_utils.patch_model_and_tokenizer, "Upcast or downcast if explicitly set".
+ for module in model.modules():
+ if hasattr(module, "_pre_set_compute_dtype"):
+ module.to(module._pre_set_compute_dtype)
+
+
[email protected]("storage", [torch.float16, torch.bfloat16])
+def test_packed_bytes_survive_the_pre_set_compute_dtype_pass(storage):
+ fsdp2_qlora = _load_module()
+ lin = _float_packed_linear4bit(storage)
+ lin._pre_set_compute_dtype = torch.float32
+ model = torch.nn.Sequential(lin)
+ weight = lin.weight
+ before = weight.data.clone()
+ with fsdp2_qlora.packed_4bit_as_uint8(model) as n:
+ assert n == 1
+ _dtype_pass(model)
+ assert lin.weight is weight
+ assert weight.dtype == storage and weight.quant_storage == storage
+ assert torch.equal(weight.data.view(torch.uint8), before.view(torch.uint8))
+ # The rest of the module is still cast, as without FSDP2.
+ assert lin.bias.dtype == torch.float32
+
+
+def test_disabled_and_uint8_are_untouched():
+ fsdp2_qlora = _load_module()
+ lin = _float_packed_linear4bit(torch.bfloat16)
+ with fsdp2_qlora.packed_4bit_as_uint8(lin, enabled = False) as n:
+ assert n == 0 and lin.weight.dtype == torch.bfloat16
+ lin8 = _float_packed_linear4bit(torch.uint8)
+ with fsdp2_qlora.packed_4bit_as_uint8(lin8) as n:
+ assert n == 0 and lin8.weight.dtype == torch.uint8
+
+
+def test_without_it_the_pass_casts_packed_bytes():
+ # The failure the helper prevents: the packed float16 words are converted to float32 values.
+ lin = _float_packed_linear4bit(torch.float16)
+ lin._pre_set_compute_dtype = torch.float32
+ _dtype_pass(torch.nn.Sequential(lin))
+ assert lin.weight.dtype == torch.float32
+
+
+def test_vision_loader_wraps_patch_model_and_tokenizer():
+ tree = ast.parse((_ROOT / "unsloth" / "models" / "vision.py").read_text(encoding = "utf-8"))
+ wrapped = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.With):
+ continue
+ ctx = [
+ item.context_expr.func.id
+ for item in node.items
+ if isinstance(item.context_expr, ast.Call)
+ and isinstance(item.context_expr.func, ast.Name)
+ ]
+ if "packed_4bit_as_uint8" not in ctx:
+ continue
+ for inner in ast.walk(node):
+ if (
+ isinstance(inner, ast.Call)
+ and isinstance(inner.func, ast.Name)
+ and inner.func.id == "patch_model_and_tokenizer"
+ ):
+ wrapped.append(inner)
+ assert len(wrapped) == 1
diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.py
index 18f5ba7e89e..4640c4f18c6 100644
--- a/tests/test_fsdp2_save_args.py
+++ b/tests/test_fsdp2_save_args.py
@@ -96,3 +96,63 @@ def test_peft_saves_gather_the_frozen_tensors_peft_writes():
_forwarding, model, ("out", True, None, False), {}
)
assert arguments["save_embedding_layers"] is False
+
+
+ "method,offload",
+ [
+ ("merged_16bit", True),
+ ("lora", True),
+ ("merged_4bit", False),
+ ("merged_4bit_forced", False),
+ ],
+)
+def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method, offload):
+ # PEFT's 4-bit merge dequantizes on the packed weights' GPU; LoRA offloaded to the CPU next to
+ # it fails with "Expected all tensors to be on the same device".
+ import contextlib
+
+ seen = {}
+
+ @contextlib.contextmanager
+ def _gathered(model, **kwargs):
+ seen.update(kwargs)
+ yield False
+
+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _gathered)
+ save = unsloth_save.unsloth_generic_save_pretrained_merged
+ for call in (dict(save_method = method), {}):
+ seen.clear()
+ positional = () if call else (method,)
+ assert save(_Model(), "out", None, *positional, **call) is None
+ got = seen.get("offload_to_cpu", True)
+ assert (got is True) if offload else callable(got)
+
+
+def test_merged_4bit_keeps_only_merge_operands_on_the_gpu():
+ # Embeddings and LoRA on unquantized layers still go to the CPU; only the adapters of layers
+ # with a packed 4-bit base stay with it.
+ class Quant(torch.nn.Parameter):
+ pass
+
+ def lora_layer(quantized):
+ layer = torch.nn.Module()
+ layer.base_layer = torch.nn.Linear(4, 4)
+ if quantized:
+ layer.base_layer.weight.quant_state = object()
+ layer.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})
+ layer.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 4, bias = False)})
+ return layer
+
+ model = torch.nn.Module()
+ model.embed = torch.nn.Embedding(8, 4)
+ model.q4 = lora_layer(True)
+ model.plain = lora_layer(False)
+ offload = fsdp2_save.offload_all_but_4bit_merge_operands(model)
+ assert not offload("q4.lora_A.default.weight", model.q4.lora_A["default"].weight)
+ assert not offload("q4.lora_B.default.weight", model.q4.lora_B["default"].weight)
+ assert offload("embed.weight", model.embed.weight)
+ assert offload("plain.lora_A.default.weight", model.plain.lora_A["default"].weight)
+ assert offload("plain.base_layer.weight", model.plain.base_layer.weight)
diff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.py
index a32cd98790a..c24ea0a3214 100644
--- a/unsloth/models/fsdp2_qlora.py
+++ b/unsloth/models/fsdp2_qlora.py
@@ -173,6 +173,42 @@ def params4bit_to_uint8_storage(model):
return converted, left
+def packed_4bit_as_uint8(model, enabled = True):
+ """Show float-packed 4-bit weights as the uint8 bitsandbytes stores by default for the duration.
+ Loader dtype passes (`module.to(module._pre_set_compute_dtype)`, forced float32) cast every
+ floating parameter of a module, so packed bytes held in float16 / bfloat16 would be converted as
+ if they were weights; as uint8 they are skipped, as they are without FSDP2. Restored afterwards."""
+ packed = []
+ if enabled:
+ for module in model.modules():
+ for name, param in module._parameters.items():
+ if (
+ param is None
+ or not is_params4bit(param)
+ or not param.is_floating_point()
+ or not getattr(param, "bnb_quantized", False)
+ or getattr(param, "quant_state", None) is None
+ or not param.is_contiguous()
+ or param.device.type == "meta"
+ ):
+ continue
+ packed.append((module, name, param.dtype))
+ for module, name, _ in packed:
+ param = module._parameters[name]
+ param.data = param.data.reshape(-1).view(torch.uint8).reshape(-1, 1)
+ param.quant_storage = torch.uint8
+ try:
+ yield len(packed)
+ finally:
+ for module, name, dtype in packed:
+ param = module._parameters.get(name)
+ if param is None or param.dtype != torch.uint8:
+ continue
+ param.data = param.data.reshape(-1).view(dtype).reshape(-1, 1)
+ param.quant_storage = dtype
+
+
def _packed_numel(param):
if is_params4bit(param) and param.is_floating_point():
return param.numel() * param.element_size()
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 6b7cfca52de..7229a8803f8 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -163,6 +163,7 @@ def fsdp2_gathered(
enter; rank 0 sees whole parameters (and yields True), the other ranks yield False and should
not write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),
plus those `include(name, param)` selects (frozen ones the save still writes).
+ `offload_to_cpu` may be a predicate `(name, param) -> bool`.
Packed 4-bit weights stay on their device: the merge dequantizes them there."""
if _ACTIVE:
yield _ACTIVE[-1]
@@ -206,7 +207,8 @@ def fsdp2_gathered(
del full
continue
module = owners[id(param)][0][0]
- if offload_to_cpu and getattr(param, "quant_state", None) is None:
+ offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpu
+ if offload and getattr(param, "quant_state", None) is None:
full = full.cpu()
replaced[id(param)] = (param, _rebuild(param, full.detach(), module))
if writer:
@@ -227,6 +229,28 @@ def fsdp2_gathered(
_wait_for_writer(dist, writer)
+def offload_all_but_4bit_merge_operands(model):
+ """`offload_to_cpu` for a merged 4-bit save: PEFT merges a LoRA layer where its packed 4-bit base
+ weight stays (its GPU), so only the adapter weights of those layers stay there too. Everything
+ else (embeddings, unquantized towers, LoRA on unquantized layers with their base) goes to the CPU."""
+ keep = set()
+ for module in model.modules() if isinstance(model, torch.nn.Module) else ():
+ weight = getattr(getattr(module, "base_layer", None), "weight", None)
+ if weight is None or getattr(weight, "quant_state", None) is None:
+ continue
+ for attr in (
+ "lora_A",
+ "lora_B",
+ "lora_embedding_A",
+ "lora_embedding_B",
+ "lora_magnitude_vector",
+ ):
+ sub = getattr(module, attr, None)
+ if isinstance(sub, torch.nn.Module):
+ keep.update(id(param) for param in sub.parameters())
+ return lambda name, param: id(param) not in keep
+
+
def fsdp2_gather_active():
return bool(_ACTIVE)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index fe6c0cb4d40..c7bfed88d5e 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -170,7 +170,7 @@ from .loader_utils import (
sync_load_when_quantizing,
warn_if_bitsandbytes_quantized_nothing,
)
-from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage
+from .fsdp2_qlora import packed_4bit_as_uint8, prepare_fsdp2_qlora, set_fsdp2_quant_storage
# `unsloth.save` imports `.models.loader_utils`, so binding a name out of it here at module
# scope closes a cycle and a cold `import unsloth.save` fails on the half-built module.
# `_gpu_init` hid that by importing `.models` first; the MLX branch never reaches it. So the
@@ -4362,14 +4362,16 @@ class FastBaseModel:
patch_medias_processor = None
if patch_medias_processor is not None:
patch_medias_processor(tokenizer)
- model, tokenizer = patch_model_and_tokenizer(
- model,
- tokenizer,
- downcast_rope = False,
- fix_embeddings = False,
- do_forced_float32 = do_forced_float32,
- correct_dtype = correct_dtype,
- )
+ # FSDP2 packs 4-bit weights in a float dtype, which the dtype passes in here would cast.
+ with packed_4bit_as_uint8(model, enabled = fsdp2_requested()):
+ model, tokenizer = patch_model_and_tokenizer(
+ model,
+ tokenizer,
+ downcast_rope = False,
+ fix_embeddings = False,
+ do_forced_float32 = do_forced_float32,
+ correct_dtype = correct_dtype,
+ )
try:
model, tokenizer = patch_tokenizer(model, tokenizer)
diff --git a/unsloth/save.py b/unsloth/save.py
index 6d5b62e8f94..6e8aa5310f3 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6228,9 +6228,10 @@ def _fsdp2_peft_saved_frozen(model, arguments):
return include
-def _fsdp2_collective_save(trainable_only):
+def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):
"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others
- return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough.
+ return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;
+ `offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.
A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""
def decorate(fn):
@@ -6251,7 +6252,10 @@ def _fsdp2_collective_save(trainable_only):
+ ("trainable" if only else "sharded")
+ " weights to rank 0, which writes. Every rank must make this call."
)
- with fsdp2_gathered(self, trainable_only = only, include = include) as writer:
+ offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)
+ with fsdp2_gathered(
+ self, trainable_only = only, include = include, offload_to_cpu = offload
+ ) as writer:
if writer is False:
return None
return fn(self, *args, **kwargs)
@@ -6273,9 +6277,23 @@ def _merged_save_method(args, kwargs):
return args[2] if len(args) > 2 else "merged_16bit"
+def _fsdp2_merged_offload(model, args, kwargs):
+ if _merged_save_method(args, kwargs) not in (
+ "merged_4bit",
+ "merged_4bit_forced",
+ "forced_merged_4bit",
+ ):
+ return True
+ from .models.fsdp2_save import offload_all_but_4bit_merge_operands
+ return offload_all_but_4bit_merge_operands(model)
+
+
@_fsdp2_collective_save(
lambda self, args, kwargs: isinstance(self, PeftModel)
- and _is_adapter_save_method(_merged_save_method(args, kwargs))
+ and _is_adapter_save_method(_merged_save_method(args, kwargs)),
+ # merged_4bit merges with PEFT where the packed 4-bit weights stay (on the GPU), so those layers'
+ # LoRA weights stay on that device too; every other gathered weight still goes to the CPU.
+ offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),
)
def unsloth_generic_save_pretrained_merged(
self,