Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 QLoRA: keep 4-bit merge operands on one device in push_to_hub_merged (#14724)

Daniel Hanebdb28d64dfc
diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.py
index 5e1f9c45612..2a8580c0334 100644
--- a/tests/test_fsdp2_save_args.py
+++ b/tests/test_fsdp2_save_args.py
@@ -125,6 +125,24 @@ def test_peft_embedding_membership_survives_a_reshard():
assert not any("q_proj.base_layer" in n for n in names), names
+class _QLoraModel(torch.nn.Module):
+ """One LoRA layer over a packed 4-bit base, one over a plain Linear."""
+
+ def __init__(self):
+ super().__init__()
+ for name, quantized in (("q4", True), ("plain", False)):
+ layer = torch.nn.Module()
+ layer.base_layer = torch.nn.Linear(4, 4)
+ if quantized:
+ layer.base_layer.weight.quant_state = object()
+ layer.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})
+ layer.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 4, bias = False)})
+ setattr(self, name, layer)
+
+
+ "entry", ["unsloth_generic_save_pretrained_merged", "unsloth_generic_push_to_hub_merged"]
+)
@pytest.mark.parametrize(
"method,offload",
[
@@ -132,11 +150,11 @@ def test_peft_embedding_membership_survives_a_reshard():
("lora", True),
("merged_4bit", False),
("merged_4bit_forced", False),
+ ("forced_merged_4bit", False),
],
)
-def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method, offload):
- # PEFT's 4-bit merge dequantizes on the packed weights' GPU; LoRA offloaded to the CPU next to
- # it fails with "Expected all tensors to be on the same device".
+def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, entry, method, offload):
+ # PEFT's 4-bit merge runs on the packed weight's GPU: CPU-offloaded LoRA fails there (save and Hub).
import contextlib
seen = {}
@@ -149,13 +167,23 @@ def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method,
monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])
monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)
monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _gathered)
- save = unsloth_save.unsloth_generic_save_pretrained_merged
+ save = getattr(unsloth_save, entry)
+ model = _QLoraModel()
for call in (dict(save_method = method), {}):
seen.clear()
positional = () if call else (method,)
- assert save(_Model(), "out", None, *positional, **call) is None
+ assert save(model, "out", None, *positional, **call) is None
got = seen.get("offload_to_cpu", True)
- assert (got is True) if offload else callable(got)
+ if offload:
+ assert got is True
+ continue
+ assert callable(got)
+ params = dict(model.named_parameters())
+ decide = lambda name: got(name, params[name])
+ assert not decide("q4.lora_A.default.weight")
+ assert not decide("q4.lora_B.default.weight")
+ assert decide("plain.lora_A.default.weight")
+ assert decide("q4.base_layer.bias")
def test_merged_4bit_keeps_only_merge_operands_on_the_gpu():
diff --git a/unsloth/save.py b/unsloth/save.py
index 830de413e8e..727a3f6144e 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6644,7 +6644,8 @@ def unsloth_generic_save_pretrained_merged(
# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.
@_fsdp2_collective_save(
lambda self, args, kwargs: isinstance(self, PeftModel)
- and _is_adapter_save_method(_merged_save_method(args, kwargs))
+ and _is_adapter_save_method(_merged_save_method(args, kwargs)),
+ offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),
)
def unsloth_generic_push_to_hub_merged(
self,