Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 QLoRA: keep 4-bit merge operands on one device in push_to_hub_merged (#14724)
Daniel Hanebdb28d64dfc
diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.pyindex 5e1f9c45612..2a8580c0334 100644--- a/tests/test_fsdp2_save_args.py+++ b/tests/test_fsdp2_save_args.py@@ -125,6 +125,24 @@ def test_peft_embedding_membership_survives_a_reshard():assert not any("q_proj.base_layer" in n for n in names), names+class _QLoraModel(torch.nn.Module):+ """One LoRA layer over a packed 4-bit base, one over a plain Linear."""++ def __init__(self):+ super().__init__()+ for name, quantized in (("q4", True), ("plain", False)):+ layer = torch.nn.Module()+ layer.base_layer = torch.nn.Linear(4, 4)+ if quantized:+ layer.base_layer.weight.quant_state = object()+ layer.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})+ layer.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 4, bias = False)})+ setattr(self, name, layer)+++ "entry", ["unsloth_generic_save_pretrained_merged", "unsloth_generic_push_to_hub_merged"]+)@pytest.mark.parametrize("method,offload",[@@ -132,11 +150,11 @@ def test_peft_embedding_membership_survives_a_reshard():("lora", True),("merged_4bit", False),("merged_4bit_forced", False),+ ("forced_merged_4bit", False),],)-def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method, offload):- # PEFT's 4-bit merge dequantizes on the packed weights' GPU; LoRA offloaded to the CPU next to- # it fails with "Expected all tensors to be on the same device".+def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, entry, method, offload):+ # PEFT's 4-bit merge runs on the packed weight's GPU: CPU-offloaded LoRA fails there (save and Hub).import contextlibseen = {}@@ -149,13 +167,23 @@ def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method,monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _gathered)- save = unsloth_save.unsloth_generic_save_pretrained_merged+ save = getattr(unsloth_save, entry)+ model = _QLoraModel()for call in (dict(save_method = method), {}):seen.clear()positional = () if call else (method,)- assert save(_Model(), "out", None, *positional, **call) is None+ assert save(model, "out", None, *positional, **call) is Nonegot = seen.get("offload_to_cpu", True)- assert (got is True) if offload else callable(got)+ if offload:+ assert got is True+ continue+ assert callable(got)+ params = dict(model.named_parameters())+ decide = lambda name: got(name, params[name])+ assert not decide("q4.lora_A.default.weight")+ assert not decide("q4.lora_B.default.weight")+ assert decide("plain.lora_A.default.weight")+ assert decide("q4.base_layer.bias")def test_merged_4bit_keeps_only_merge_operands_on_the_gpu():diff --git a/unsloth/save.py b/unsloth/save.pyindex 830de413e8e..727a3f6144e 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6644,7 +6644,8 @@ def unsloth_generic_save_pretrained_merged(# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel)- and _is_adapter_save_method(_merged_save_method(args, kwargs))+ and _is_adapter_save_method(_merged_save_method(args, kwargs)),+ offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),)def unsloth_generic_push_to_hub_merged(self,