Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 QLoRA merged saves: stage the packed 4-bit base on the CPU and merge one layer at a time (#14823)

Daniel Hanbb166e3cc5d8
diff --git a/tests/test_fsdp2_merged_save_memory_cuda.py b/tests/test_fsdp2_merged_save_memory_cuda.py
new file mode 100644
index 00000000000..4de09d86fcd
--- /dev/null
+++ b/tests/test_fsdp2_merged_save_memory_cuda.py
@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Writer-GPU memory of an FSDP2 merged save of a sharded NF4 QLoRA (unsloth/models/fsdp2_save.py),
+two NCCL ranks with a GPU each, or, with one visible GPU, one NCCL rank whose shards FSDP2 keeps on
+the CPU (CPUOffloadPolicy), so every gathered tensor is a real allocation on the writer's GPU there
+too. `python tests/test_fsdp2_merged_save_memory_cuda.py --out result.json` runs the second form.
+
+The previous placement gathered every packed 4-bit weight onto rank 0's GPU and kept them there
+until the write, so a merged save needed the whole quantized base on one card above its shards. The
+merged saves now stage each packed weight on the CPU right after gathering it; merged_4bit brings
+one layer back at a time to merge it. The peak above the sharded footprint must stay about one layer
+(not the whole base), and the merged bytes must equal the previous placement's.
+The module is loaded from its file (importing unsloth is not needed)."""
+
+import importlib.util
+import os
+import warnings
+from pathlib import Path
+
+import pytest
+import torch
+
+_ROOT = Path(__file__).resolve().parents[1]
+N_LAYERS = 64
+DIM = 2048
+RANK = 8
+
+if __name__ != "__main__":
+ if not torch.cuda.is_available() or not torch.distributed.is_nccl_available():
+ pytest.skip("needs a CUDA GPU with NCCL", allow_module_level = True)
+ pytest.importorskip("bitsandbytes")
+ pytest.importorskip("peft")
+ fsdp_mod = pytest.importorskip("torch.distributed.fsdp")
+ if not hasattr(fsdp_mod, "fully_shard"):
+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)
+
+
+def _load(name):
+ path = _ROOT / "unsloth" / "models" / f"{name}.py"
+ spec = importlib.util.spec_from_file_location(f"_unsloth_{name}_under_test", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _model(device):
+ import bitsandbytes as bnb
+ from peft import LoraConfig, get_peft_model
+
+ class Block(torch.nn.Module):
+ def __init__(self, seed):
+ super().__init__()
+ torch.manual_seed(seed)
+ self.q = bnb.nn.Linear4bit(
+ DIM, DIM, bias = False, compute_dtype = torch.bfloat16,
+ quant_type = "nf4", quant_storage = torch.bfloat16,
+ )
+ self.q.weight = bnb.nn.Params4bit(
+ torch.randn(DIM, DIM) * 0.02, requires_grad = False, quant_type = "nf4",
+ quant_storage = torch.bfloat16, blocksize = 64, module = self.q,
+ )
+
+ def forward(self, x):
+ return x + self.q(x)
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([Block(i + 1) for i in range(N_LAYERS)])
+
+ model = Model().to(device) # quantizes on the GPU
+ model.is_loaded_in_4bit = True
+ torch.manual_seed(0)
+ config = LoraConfig(r = RANK, lora_alpha = 16, target_modules = ["q"], init_lora_weights = False)
+ return get_peft_model(model, config)
+
+
+def _sharded(device, world_size, q):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import CPUOffloadPolicy, fully_shard
+
+ model = _model(device)
+ q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh("cuda", (world_size,))
+ # One rank: a shard is the whole tensor and gathering it aliases it, so keep the shards on the
+ # CPU and every gather allocates on the GPU, as a gather of two GPU shards does.
+ kwargs = {} if world_size > 1 else {"offload_policy": CPUOffloadPolicy()}
+ for layer in model.base_model.model.layers:
+ fully_shard(layer, mesh = mesh, **kwargs)
+ fully_shard(model, mesh = mesh, **kwargs)
+ return model
+
+
+def _worker(rank, world_size, port, queue):
+ try:
+ torch.cuda.set_device(rank if torch.cuda.device_count() >= world_size else 0)
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port),
+ RANK = str(rank), WORLD_SIZE = str(world_size),
+ )
+ import torch.distributed as dist
+
+ dist.init_process_group("nccl", rank = rank, world_size = world_size)
+ warnings.filterwarnings("ignore")
+ m, q = _load("fsdp2_save"), _load("fsdp2_qlora")
+ device = torch.device("cuda", torch.cuda.current_device())
+ packed_layer = DIM * DIM // 2
+ out = {
+ "packed_layer": packed_layer,
+ "packed_total": packed_layer * N_LAYERS,
+ # fp32 lora_A + lora_B of every layer: the merged_4bit gather keeps them on the GPU.
+ "lora_total": N_LAYERS * 2 * RANK * DIM * 4,
+ }
+ for method in ("merged_4bit", "merged_16bit"):
+ for mode in ("head", "streamed"):
+ model = _sharded(device, world_size, q)
+ torch.cuda.synchronize()
+ torch.cuda.empty_cache()
+ base = torch.cuda.memory_allocated(device)
+ torch.cuda.reset_peak_memory_stats(device)
+ if mode == "streamed":
+ options = m.merged_save_gather_options(model, method)
+ elif method == "merged_4bit":
+ options = dict(offload_to_cpu = m.offload_all_but_4bit_merge_operands(model))
+ else:
+ options = dict(offload_to_cpu = True)
+ written = None
+ with m.fsdp2_gathered(model, **options) as writer:
+ if writer and method == "merged_4bit":
+ if mode == "streamed":
+ m.merge_4bit_lora_layer_by_layer(model)
+ merged = model.base_model.merge_and_unload()
+ written = {
+ k: v.detach().reshape(-1).view(torch.uint8).cpu().clone()
+ for k, v in merged.state_dict().items()
+ if isinstance(v, torch.Tensor)
+ }
+ elif writer:
+ # merged_16bit reads the LoRA weights (the base comes from the 16-bit files).
+ written = {
+ k: v.detach().cpu().clone()
+ for k, v in model.state_dict().items()
+ if "lora_" in k
+ }
+ torch.cuda.synchronize()
+ peak = torch.cuda.max_memory_allocated(device) - base
+ out[(method, mode)] = dict(peak_above_shards = peak, written = written)
+ del model
+ dist.destroy_process_group()
+ if rank == 0:
+ for method in ("merged_4bit", "merged_16bit"):
+ a, b = out[(method, "head")]["written"], out[(method, "streamed")]["written"]
+ out[(method, "same_bytes")] = sorted(a) == sorted(b) and all(
+ torch.equal(a[k], b[k]) for k in a
+ )
+ for mode in ("head", "streamed"):
+ out[(method, mode)].pop("written")
+ queue.put((rank, out))
+ except BaseException as error:
+ import traceback
+
+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
+
+
+def test_merged_saves_hold_one_packed_layer_on_the_writer_gpu():
+ import torch.multiprocessing as mp
+ from tests._rendezvous_port import free_port
+
+ world_size = 2 if torch.cuda.device_count() >= 2 else 1
+ ctx = mp.get_context("spawn")
+ queue = ctx.Queue()
+ port = free_port()
+ procs = [ctx.Process(target = _worker, args = (r, world_size, port, queue)) for r in range(world_size)]
+ for p in procs:
+ p.start()
+ results = dict(queue.get(timeout = 600) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ _check(results)
+
+
+def _check(results):
+ for r in results.values():
+ assert "error" not in r, r.get("error")
+ r0 = results[0]
+ layer, total = r0["packed_layer"], r0["packed_total"]
+ print({k: v for k, v in r0.items()})
+ for method in ("merged_4bit", "merged_16bit"):
+ assert r0[(method, "same_bytes")], method
+ head = r0[(method, "head")]["peak_above_shards"]
+ streamed = r0[(method, "streamed")]["peak_above_shards"]
+ # The previous placement holds the whole gathered packed base.
+ assert head >= total, (method, head, total)
+ # Streaming holds one gathered layer, and merged_4bit one layer's merge temporaries on top (bf16
+ # dequant, fp32 LoRA delta, the requantize copy: a few dozen packed layers' worth, whatever the
+ # depth) next to the adapters, never the base.
+ assert r0[("merged_16bit", "streamed")]["peak_above_shards"] <= 2 * layer, r0
+ merged_4bit = r0[("merged_4bit", "streamed")]["peak_above_shards"]
+ assert merged_4bit <= 40 * layer + r0["lora_total"], r0
+ assert merged_4bit < total, r0
+
+
+if __name__ == "__main__":
+ import argparse
+ import json
+ import queue as queue_module
+ import sys
+
+ sys.path.insert(0, str(_ROOT))
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--out", required = True)
+ args = parser.parse_args()
+ from tests._rendezvous_port import free_port
+
+ results = queue_module.Queue()
+ _worker(0, 1, free_port(), results)
+ rank, result = results.get()
+ printable = {str(k): v for k, v in result.items()}
+ with open(args.out, "w") as f:
+ json.dump(printable, f, indent = 1)
+ print(json.dumps(printable))
+ if rank == 0:
+ _check({0: result})
+ print("CHECK PASSED")
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.py
index 1277c1a8f75..f3ada289115 100644
--- a/tests/test_fsdp2_save_coverage.py
+++ b/tests/test_fsdp2_save_coverage.py
@@ -95,9 +95,10 @@ def test_refusal_passes_through_without_fsdp2(monkeypatch):
assert export(object(), 1) == ("ran", 1)
-def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):
- """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, so the gather may put
- them on the CPU; a merged save dequantizes them where they are and keeps them there."""
+def test_full_and_merged_saves_stage_4bit_on_the_cpu(monkeypatch):
+ """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, merged_16bit reads its
+ base from the 16-bit checkpoint, and merged_4bit brings one packed layer at a time back to its
+ device to merge, so every one of them gathers the packed bytes onto the CPU."""
seen = []
@contextlib.contextmanager
@@ -119,4 +120,93 @@ def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):
is None
)
assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None
- assert seen == [True, True, False, False, False], seen
+ assert seen == [True, True, True, True, True], seen
+
+
+def test_streamed_merge_failure_clears_every_merged_layer(monkeypatch):
+ # A later layer's merge raising must not leave earlier layers claiming a merge the gather undoes.
+ import torch
+
+ class _Layer(torch.nn.Module):
+ def __init__(self, fail):
+ super().__init__()
+ self.base_layer = torch.nn.Linear(2, 2, bias = False)
+ self.merged_adapters = []
+ self.fail = fail
+
+ def merge(self):
+ if self.fail:
+ raise RuntimeError("merge failed")
+ self.merged_adapters.append("default")
+
+ model = torch.nn.Sequential(_Layer(False), _Layer(False), _Layer(True))
+ staged = {id(layer.base_layer.weight): "cpu" for layer in model}
+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])
+ monkeypatch.setattr(fsdp2_save, "_restage_4bit", lambda weight, device: weight)
+ with pytest.raises(RuntimeError, match = "merge failed"):
+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)
+ assert [layer.merged_adapters for layer in model] == [[], [], []]
+
+
+def test_streamed_merge_restage_failure_clears_merge_flags(monkeypatch):
+ # A failing restage after a successful merge must not leave the layer claiming the merge.
+ import torch
+
+ class _Layer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.base_layer = torch.nn.Linear(2, 2, bias = False)
+ self.base_layer.weight.quant_state = object()
+ self.merged_adapters = []
+
+ def merge(self):
+ self.merged_adapters.append("default")
+
+ model = torch.nn.Sequential(_Layer(), _Layer())
+ staged = {id(layer.base_layer.weight): "cuda" for layer in model}
+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])
+ calls = []
+
+ def restage(weight, device):
+ calls.append(device)
+ if device == "cpu" and len(calls) == 4:
+ raise RuntimeError("host out of memory")
+ return weight
+
+ monkeypatch.setattr(fsdp2_save, "_restage_4bit", restage)
+ with pytest.raises(RuntimeError, match = "host out of memory"):
+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)
+ assert [layer.merged_adapters for layer in model] == [[], []]
+
+
+def test_streamed_merge_gpu_restage_failure_clears_earlier_merge_flags(monkeypatch):
+ # The GPU restage of a later layer failing must also clear the layers merged before it.
+ import torch
+
+ class _Layer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.base_layer = torch.nn.Linear(2, 2, bias = False)
+ self.base_layer.weight.quant_state = object()
+ self.merged_adapters = []
+
+ def merge(self):
+ self.merged_adapters.append("default")
+
+ model = torch.nn.Sequential(_Layer(), _Layer())
+ staged = {id(layer.base_layer.weight): "cuda" for layer in model}
+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])
+ calls = []
+
+ def restage(weight, device):
+ calls.append(device)
+ if device == "cuda" and len(calls) == 3:
+ raise RuntimeError("CUDA out of memory")
+ return weight
+
+ monkeypatch.setattr(fsdp2_save, "_restage_4bit", restage)
+ second = model[1].base_layer.weight
+ with pytest.raises(RuntimeError, match = "CUDA out of memory"):
+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)
+ assert [layer.merged_adapters for layer in model] == [[], []]
+ assert model[1].base_layer._parameters["weight"] is second
diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.py
index 204e7138115..9967d30c373 100644
--- a/tests/test_fsdp2_save_gloo.py
+++ b/tests/test_fsdp2_save_gloo.py
@@ -297,6 +297,137 @@ def _case_merge_offload(rank, world_size, m):
return out
+class _QLoraBlock(torch.nn.Module):
+ def __init__(self, seed):
+ super().__init__()
+ bnb = pytest.importorskip("bitsandbytes")
+ torch.manual_seed(seed)
+ self.q = bnb.nn.Linear4bit(
+ 128, 128, bias = False, compute_dtype = torch.bfloat16,
+ quant_type = "nf4", quant_storage = torch.bfloat16,
+ )
+ self.q.weight = bnb.nn.Params4bit(
+ torch.randn(128, 128) * 0.05, requires_grad = False, quant_type = "nf4",
+ quant_storage = torch.bfloat16, blocksize = 64, module = self.q,
+ )
+ self.o = torch.nn.Linear(128, 128, bias = False).requires_grad_(False)
+
+ def forward(self, x):
+ return x + self.o(self.q(x))
+
+
+class _QLoraModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_QLoraBlock(7 * i + 1) for i in range(N_LAYERS)])
+
+ def forward(self, x):
+ for layer in self.layers:
+ x = layer(x)
+ return x
+
+
+def _qlora_peft(seed = 5):
+ from peft import LoraConfig, get_peft_model
+
+ model = _QLoraModel().to("cpu") # quantizes the Params4bit
+ model.is_loaded_in_4bit = True # what PEFT reads to give Linear4bit its 4-bit LoRA layer
+ torch.manual_seed(seed)
+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q", "o"], init_lora_weights = False)
+ return get_peft_model(model, config)
+
+
+def _packed_bytes(sd):
+ return {
+ k: v.detach().reshape(-1).view(torch.uint8).clone() if isinstance(v, torch.Tensor) else v
+ for k, v in sd.items()
+ }
+
+
+def _case_merged_4bit_streamed(rank, world_size, m):
+ """merged_4bit of a sharded NF4 QLoRA, as save_pretrained_merged runs it: the gather with the
+ merged save's options, the streamed layer by layer merge, then PEFT's merge_and_unload and the
+ state dict the export writes. `head` is #14739's placement (every packed weight stays on its
+ device, PEFT merges them all there); `streamed` stages them on the CPU and brings one back at a
+ time. Both must write the bytes an unsharded merge writes."""
+ import warnings
+
+ q = _load("fsdp2_qlora")
+ from peft.tuners.lora import bnb as peft_bnb
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ warnings.filterwarnings("ignore")
+ reference = _packed_bytes(_qlora_peft().base_model.merge_and_unload().state_dict())
+ out = {}
+ original_cpu = torch.Tensor.cpu
+ original_merge = peft_bnb.Linear4bit.merge
+ original_restage = getattr(m, "_restage_4bit", None)
+ for mode in ("head", "streamed"):
+ model = _qlora_peft()
+ q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh("cpu", (world_size,))
+ for layer in model.base_model.model.layers:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ if mode == "head":
+ options = dict(offload_to_cpu = m.offload_all_but_4bit_merge_operands(model))
+ else:
+ options = m.merged_save_gather_options(model, "merged_4bit")
+ events = []
+ moved = []
+
+ def recording_cpu(self, *args, **kwargs):
+ moved.append(tuple(self.shape))
+ return original_cpu(self, *args, **kwargs)
+
+ def recording_merge(self, *args, **kwargs):
+ events.append("merge")
+ return original_merge(self, *args, **kwargs)
+
+ def recording_restage(weight, device):
+ events.append("restage")
+ return original_restage(weight, device)
+
+ torch.Tensor.cpu = recording_cpu
+ peft_bnb.Linear4bit.merge = recording_merge
+ if original_restage is not None:
+ m._restage_4bit = recording_restage
+ try:
+ with m.fsdp2_gathered(model, **options) as writer:
+ torch.Tensor.cpu = original_cpu
+ packed_moved = sum(len(s) == 2 and s[-1] == 1 for s in moved)
+ if writer:
+ staged = (
+ m.merge_4bit_lora_layer_by_layer(model) if mode == "streamed" else []
+ )
+ on_cpu_before_unload = [
+ layer.q.base_layer.weight.quant_state.absmax.device.type
+ for layer in model.base_model.model.layers
+ ]
+ events.append("unload")
+ merged = model.base_model.merge_and_unload()
+ sd = _packed_bytes(merged.state_dict())
+ out[mode] = dict(
+ packed_moved = packed_moved,
+ streamed_layers = len(staged),
+ events = events,
+ states_cpu = on_cpu_before_unload,
+ same_keys = sorted(sd) == sorted(reference),
+ same_bytes = all(
+ torch.equal(sd[k], reference[k]) if isinstance(sd[k], torch.Tensor)
+ else sd[k] == reference[k]
+ for k in reference
+ ),
+ )
+ finally:
+ torch.Tensor.cpu = original_cpu
+ peft_bnb.Linear4bit.merge = original_merge
+ if original_restage is not None:
+ m._restage_4bit = original_restage
+ return out
+
+
def _case_replicated(rank, world_size, m, hooked):
import torch.distributed as dist
@@ -805,6 +936,8 @@ def _worker(rank, world_size, port, case, queue):
result = _case_merge_offload(rank, world_size, m)
elif case == "gather_4bit":
result = _case_gather_4bit(rank, world_size, m)
+ elif case == "merged_4bit_streamed":
+ result = _case_merged_4bit_streamed(rank, world_size, m)
elif case == "gather_4bit_offload":
result = _case_gather_4bit_offload(rank, world_size, m)
elif case == "dcp_quant_state":
@@ -1015,3 +1148,27 @@ def test_full_dcp_broadcast_load_restores_quant_state_over_nccl():
for broadcast in (False, True):
assert r[f"legacy_{broadcast}"] is True, r
assert r[f"mismatch_{broadcast}"] == "ValueError", r
+
+
+def test_merged_4bit_save_streams_the_packed_base_one_layer_at_a_time():
+ """merged_4bit of a sharded NF4 QLoRA kept every gathered packed weight on rank 0's accelerator
+ until PEFT merged them all, so the whole quantized base had to fit on one GPU next to its shards.
+ The gather now stages each packed weight on the CPU right after gathering it, and the merge
+ brings one layer back at a time (restage, merge, restage), writing the same bytes as an unsharded
+ merge and as the previous placement."""
+ pytest.importorskip("bitsandbytes")
+ pytest.importorskip("peft")
+ r0 = _run("merged_4bit_streamed")[0]
+ head, streamed = r0["head"], r0["streamed"]
+ for r in (head, streamed):
+ assert r["same_keys"] and r["same_bytes"], r
+ assert head["packed_moved"] == 0, head
+ # Every packed weight left the gathering device as soon as it was gathered.
+ assert streamed["packed_moved"] == N_LAYERS, streamed
+ assert streamed["streamed_layers"] == N_LAYERS, streamed
+ assert all(device == "cpu" for device in streamed["states_cpu"]), streamed
+ # One layer on its device at a time: each merge is bracketed by its own two restages, and PEFT's
+ # unload merges nothing more.
+ assert streamed["events"] == ["restage", "merge", "restage"] * N_LAYERS + ["unload"] + [
+ "merge"
+ ] * N_LAYERS, streamed["events"]
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 3556b0a57b6..a1cafa42bc8 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -27,6 +27,9 @@ import torch
# Writer flag of the gather in progress: a nested save (save_pretrained_merged -> save_pretrained,
# or on a submodule) must neither gather again nor reshard what rank 0 swapped in.
_ACTIVE = []
+# Per gather in progress: id of each packed 4-bit weight staged on the CPU -> the device it was
+# gathered on, where `merge_4bit_lora_layer_by_layer` merges it.
+_STAGED_4BIT = []
def _fsdp_module_class():
@@ -132,18 +135,31 @@ def _gather_full(param, DTensor):
return gathered.view(param.dtype)
-def _quant_state_on_cpu(quant_state):
- """A CPU copy of a bitsandbytes QuantState. `QuantState.to` moves in place (its nested state
- too), and the live one must stay where the sharded weight computes with it."""
+def _quant_state_on_cpu(quant_state, device = "cpu"):
+ """A copy of a bitsandbytes QuantState on `device` (the CPU by default). `QuantState.to` moves in
+ place (its nested state too), and the live one must stay where the sharded weight computes with it."""
if not hasattr(quant_state, "to"):
return quant_state
state = copy.copy(quant_state)
if getattr(state, "state2", None) is not None:
state.state2 = copy.copy(state.state2)
- state.to("cpu")
+ state.to(device)
return state
+def _restage_4bit(weight, device):
+ """The packed Params4bit `weight` on `device`, with its own copy of the QuantState there. Every
+ other attribute is carried over as is, which is what PEFT's 4-bit merge rebuilds the weight from."""
+ from bitsandbytes.nn import Params4bit
+
+ moved = torch.Tensor._make_subclass(Params4bit, weight.data.to(device), False)
+ moved.__dict__.update(weight.__dict__)
+ quant_state = getattr(weight, "quant_state", None)
+ if quant_state is not None:
+ moved.quant_state = _quant_state_on_cpu(quant_state, device)
+ return moved
+
+
def _rebuild(param, full, module):
"""A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""
quant_state = getattr(param, "quant_state", None)
@@ -194,8 +210,9 @@ def fsdp2_gathered(
plus those `include(name, param)` selects (frozen ones the save still writes).
`offload_to_cpu` may be a predicate `(name, param) -> bool`.
Packed 4-bit weights stay on their device (a merge dequantizes them there) unless `offload_4bit`:
- a save that only writes the packed bytes then gets them, with a CPU copy of their QuantState, on
- the CPU, so rank 0 never holds the whole quantized base on its accelerator."""
+ they then go, with a CPU copy of their QuantState, to the CPU right after each one is gathered, so
+ rank 0 never holds the whole quantized base on its accelerator. A merge brings them back one layer
+ at a time (`merge_4bit_lora_layer_by_layer`)."""
if _ACTIVE:
yield _ACTIVE[-1]
return
@@ -218,6 +235,7 @@ def fsdp2_gathered(
if param is not None:
owners.setdefault(id(param), []).append((module, name))
replaced = {}
+ staged = {}
seen = set()
pushed = False
try:
@@ -240,25 +258,32 @@ def fsdp2_gathered(
module = owners[id(param)][0][0]
offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpu
quant_state = getattr(param, "quant_state", None)
+ home = None
if offload and quant_state is None:
full = full.cpu()
elif offload and offload_4bit:
+ home = full.device
full = full.cpu()
quant_state = _quant_state_on_cpu(quant_state)
rebuilt = _rebuild(param, full.detach(), module)
if quant_state is not None and rebuilt.quant_state is not quant_state:
rebuilt.quant_state = quant_state
+ if home is not None:
+ staged[id(rebuilt)] = home
replaced[id(param)] = (param, rebuilt)
if writer:
for key, (old, new) in replaced.items():
for module, name in owners[key]:
module._parameters[name] = new
_ACTIVE.append(writer)
+ _STAGED_4BIT.append(staged)
pushed = True
yield writer
finally:
if pushed:
_ACTIVE.pop()
+ _STAGED_4BIT.pop()
+ staged.clear()
for key, (old, new) in replaced.items():
for module, name in owners[key]:
module._parameters[name] = old
@@ -301,6 +326,89 @@ def offload_all_but_4bit_merge_operands(model):
return lambda name, param: not name.startswith(prefixes)
+_MERGED_4BIT_METHODS = ("merged_4bit", "merged_4bit_forced", "forced_merged_4bit")
+
+
+def _merges_onto_dequantized_4bit(model):
+ """unsloth_zoo folds a merged_16bit LoRA onto dequant(W4) for a few architectures (falcon_h1),
+ dequantizing the live packed weight where it sits."""
+ try:
+ from unsloth_zoo.saving_utils import (
+ _model_type_needs_dequant_merge_base,
+ find_lora_base_model,
+ )
+ except Exception:
+ config = getattr(model, "config", None)
+ return (getattr(config, "model_type", "") or "").lower() == "falcon_h1"
+ try:
+ model = find_lora_base_model(model)
+ except Exception:
+ pass
+ return bool(_model_type_needs_dequant_merge_base(model))
+
+
+def merged_save_gather_options(model, save_method):
+ """`fsdp2_gathered` options for save_pretrained_merged / push_to_hub_merged.
+
+ merged_16bit reads its base weights from the 16-bit checkpoint and only the LoRA weights from the
+ model, so the packed 4-bit base goes to the CPU like everything else.
+ merged_4bit stages the packed base on the CPU too, keeps the adapters of 4-bit layers on their
+ device, and `merge_4bit_lora_layer_by_layer` brings one packed layer at a time back there to merge.
+ Either way rank 0's accelerator holds one gathered layer above its shards, not the whole base.
+ A merged_16bit that dequantizes the live 4-bit base (`_merges_onto_dequantized_4bit`) and other
+ export methods keep the previous placement."""
+ if save_method in _MERGED_4BIT_METHODS:
+ return dict(offload_to_cpu = offload_all_but_4bit_merge_operands(model), offload_4bit = True)
+ if save_method == "merged_16bit" and not _merges_onto_dequantized_4bit(model):
+ return dict(offload_to_cpu = True, offload_4bit = True)
+ return dict(offload_to_cpu = True)
+
+
+def merge_4bit_lora_layer_by_layer(model):
+ """Writer, inside `fsdp2_gathered` with packed 4-bit weights staged on the CPU: merges each LoRA
+ layer over such a base with PEFT's own `merge`, one layer at a time, on the device that weight
+ was gathered on (where the merge ran before, so the requantized bytes are the same), then stages
+ the merged weight back on the CPU. PEFT's `merge_and_unload` afterwards finds those layers merged
+ and only unloads them. Returns the merged layers (see `unmerge_flags`); [] outside such a gather."""
+ staged = _STAGED_4BIT[-1] if _STAGED_4BIT else None
+ if not staged or not isinstance(model, torch.nn.Module):
+ return []
+ merged = []
+ for module in model.modules():
+ base_layer = getattr(module, "base_layer", None)
+ if not isinstance(base_layer, torch.nn.Module) or not callable(getattr(module, "merge", None)):
+ continue
+ weight = base_layer._parameters.get("weight", None)
+ home = staged.get(id(weight)) if weight is not None else None
+ if home is None:
+ continue
+ before = list(getattr(module, "merged_adapters", None) or ())
+ try:
+ base_layer._parameters["weight"] = _restage_4bit(weight, home)
+ module.merge()
+ result = base_layer._parameters.get("weight", None)
+ if result is not None and getattr(result, "quant_state", None) is not None:
+ result = _restage_4bit(result, "cpu")
+ base_layer._parameters["weight"] = result
+ except BaseException:
+ base_layer._parameters["weight"] = weight
+ # Layers merged earlier in this loop get their unmerged shards back from the gather too.
+ if list(getattr(module, "merged_adapters", None) or ()) != before:
+ merged.append((module, before))
+ unmerge_flags(merged)
+ raise
+ if list(getattr(module, "merged_adapters", None) or ()) != before:
+ merged.append((module, before))
+ return merged
+
+
+def unmerge_flags(merged):
+ """Undo `merge_4bit_lora_layer_by_layer`'s bookkeeping when the export fails before it unloads:
+ the gather puts the sharded (unmerged) base weights back, so the layers must not claim a merge."""
+ for module, before in merged:
+ module.merged_adapters[:] = before
+
+
def fsdp2_gather_active():
return bool(_ACTIVE)
diff --git a/unsloth/save.py b/unsloth/save.py
index 9c60b7668c3..d4e25cacd3d 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6270,6 +6270,17 @@ def _cast_state_dict_16bit(model, state_dict, target_dtype):
}
+@contextmanager
+def _unmerge_flags_on_error(streamed):
+ try:
+ yield
+ except BaseException:
+ if streamed:
+ from .models.fsdp2_save import unmerge_flags
+ unmerge_flags(streamed)
+ raise
+
+
@_normalize_tied_weights_keys_for_save
@torch.inference_mode
def unsloth_generic_save(
@@ -6510,7 +6521,14 @@ def unsloth_generic_save(
# merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.
in_place = save_method in ("merged_4bit", "forced_merged_4bit")
- with nullcontext() if in_place else lora_relative_to_original_base(model):
+ # Under FSDP2 the gathered packed 4-bit base waits on the CPU: merge it one layer at a time on
+ # its GPU first, so the merge below only unloads those layers.
+ streamed = []
+ if in_place and getattr(model.config, "quantization_config", None):
+ from .models.fsdp2_save import merge_4bit_lora_layer_by_layer
+ streamed = merge_4bit_lora_layer_by_layer(model)
+ relative = nullcontext() if in_place else lora_relative_to_original_base(model)
+ with relative, _unmerge_flags_on_error(streamed):
merge_and_overwrite_lora(
_modelscope_base_model_name
if os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" and not in_place
@@ -6602,12 +6620,14 @@ def _fsdp2_collective_save(
trainable_only,
offload_to_cpu = None,
offload_4bit = False,
+ gather_options = None,
):
"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others
return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;
`offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.
`offload_4bit`: the save only writes packed 4-bit weights (never merges them), so they go to
- the CPU too.
+ the CPU too. `gather_options(self, args, kwargs)`, when given, returns the `offload_to_cpu` /
+ `offload_4bit` options for `fsdp2_gathered` instead (merged saves).
A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""
def decorate(fn):
@@ -6629,12 +6649,14 @@ def _fsdp2_collective_save(
+ " weights to rank 0, which writes. Every rank must make this call."
)
offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)
+ options = dict(offload_to_cpu = offload, offload_4bit = offload_4bit)
+ if gather_options is not None and not only:
+ options.update(gather_options(self, args, kwargs))
with fsdp2_gathered(
self,
trainable_only = only,
include = include,
- offload_to_cpu = offload,
- offload_4bit = offload_4bit,
+ **options,
) as writer:
if writer is False:
return None
@@ -6668,12 +6690,19 @@ def _fsdp2_merged_offload(model, args, kwargs):
return offload_all_but_4bit_merge_operands(model)
+def _fsdp2_merged_gather_options(model, args, kwargs):
+ from .models.fsdp2_save import merged_save_gather_options
+ return merged_save_gather_options(model, _merged_save_method(args, kwargs))
+
+
@_fsdp2_collective_save(
lambda self, args, kwargs: isinstance(self, PeftModel)
and _is_adapter_save_method(_merged_save_method(args, kwargs)),
- # merged_4bit merges with PEFT where the packed 4-bit weights stay (on the GPU), so those layers'
- # LoRA weights stay on that device too; every other gathered weight still goes to the CPU.
+ # Packed 4-bit weights are staged on the CPU as they are gathered. merged_4bit merges with PEFT
+ # one layer at a time on the GPU the packed weight came from, so those layers' LoRA weights stay
+ # on that device; every other gathered weight goes to the CPU.
offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),
+ gather_options = _fsdp2_merged_gather_options,
)
def unsloth_generic_save_pretrained_merged(
self,
@@ -6838,6 +6867,7 @@ def unsloth_generic_save_pretrained_merged(
lambda self, args, kwargs: isinstance(self, PeftModel)
and _is_adapter_save_method(_merged_save_method(args, kwargs)),
offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),
+ gather_options = _fsdp2_merged_gather_options,
)
def unsloth_generic_push_to_hub_merged(
self,