Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 QLoRA merged saves: stage the packed 4-bit base on the CPU and merge one layer at a time (#14823)
Daniel Hanbb166e3cc5d8
diff --git a/tests/test_fsdp2_merged_save_memory_cuda.py b/tests/test_fsdp2_merged_save_memory_cuda.pynew file mode 100644index 00000000000..4de09d86fcd--- /dev/null+++ b/tests/test_fsdp2_merged_save_memory_cuda.py@@ -0,0 +1,225 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Writer-GPU memory of an FSDP2 merged save of a sharded NF4 QLoRA (unsloth/models/fsdp2_save.py),+two NCCL ranks with a GPU each, or, with one visible GPU, one NCCL rank whose shards FSDP2 keeps on+the CPU (CPUOffloadPolicy), so every gathered tensor is a real allocation on the writer's GPU there+too. `python tests/test_fsdp2_merged_save_memory_cuda.py --out result.json` runs the second form.++The previous placement gathered every packed 4-bit weight onto rank 0's GPU and kept them there+until the write, so a merged save needed the whole quantized base on one card above its shards. The+merged saves now stage each packed weight on the CPU right after gathering it; merged_4bit brings+one layer back at a time to merge it. The peak above the sharded footprint must stay about one layer+(not the whole base), and the merged bytes must equal the previous placement's.+The module is loaded from its file (importing unsloth is not needed)."""++import importlib.util+import os+import warnings+from pathlib import Path++import pytest+import torch++_ROOT = Path(__file__).resolve().parents[1]+N_LAYERS = 64+DIM = 2048+RANK = 8++if __name__ != "__main__":+ if not torch.cuda.is_available() or not torch.distributed.is_nccl_available():+ pytest.skip("needs a CUDA GPU with NCCL", allow_module_level = True)+ pytest.importorskip("bitsandbytes")+ pytest.importorskip("peft")+ fsdp_mod = pytest.importorskip("torch.distributed.fsdp")+ if not hasattr(fsdp_mod, "fully_shard"):+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)+++def _load(name):+ path = _ROOT / "unsloth" / "models" / f"{name}.py"+ spec = importlib.util.spec_from_file_location(f"_unsloth_{name}_under_test", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _model(device):+ import bitsandbytes as bnb+ from peft import LoraConfig, get_peft_model++ class Block(torch.nn.Module):+ def __init__(self, seed):+ super().__init__()+ torch.manual_seed(seed)+ self.q = bnb.nn.Linear4bit(+ DIM, DIM, bias = False, compute_dtype = torch.bfloat16,+ quant_type = "nf4", quant_storage = torch.bfloat16,+ )+ self.q.weight = bnb.nn.Params4bit(+ torch.randn(DIM, DIM) * 0.02, requires_grad = False, quant_type = "nf4",+ quant_storage = torch.bfloat16, blocksize = 64, module = self.q,+ )++ def forward(self, x):+ return x + self.q(x)++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([Block(i + 1) for i in range(N_LAYERS)])++ model = Model().to(device) # quantizes on the GPU+ model.is_loaded_in_4bit = True+ torch.manual_seed(0)+ config = LoraConfig(r = RANK, lora_alpha = 16, target_modules = ["q"], init_lora_weights = False)+ return get_peft_model(model, config)+++def _sharded(device, world_size, q):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import CPUOffloadPolicy, fully_shard++ model = _model(device)+ q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh("cuda", (world_size,))+ # One rank: a shard is the whole tensor and gathering it aliases it, so keep the shards on the+ # CPU and every gather allocates on the GPU, as a gather of two GPU shards does.+ kwargs = {} if world_size > 1 else {"offload_policy": CPUOffloadPolicy()}+ for layer in model.base_model.model.layers:+ fully_shard(layer, mesh = mesh, **kwargs)+ fully_shard(model, mesh = mesh, **kwargs)+ return model+++def _worker(rank, world_size, port, queue):+ try:+ torch.cuda.set_device(rank if torch.cuda.device_count() >= world_size else 0)+ os.environ.update(+ MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port),+ RANK = str(rank), WORLD_SIZE = str(world_size),+ )+ import torch.distributed as dist++ dist.init_process_group("nccl", rank = rank, world_size = world_size)+ warnings.filterwarnings("ignore")+ m, q = _load("fsdp2_save"), _load("fsdp2_qlora")+ device = torch.device("cuda", torch.cuda.current_device())+ packed_layer = DIM * DIM // 2+ out = {+ "packed_layer": packed_layer,+ "packed_total": packed_layer * N_LAYERS,+ # fp32 lora_A + lora_B of every layer: the merged_4bit gather keeps them on the GPU.+ "lora_total": N_LAYERS * 2 * RANK * DIM * 4,+ }+ for method in ("merged_4bit", "merged_16bit"):+ for mode in ("head", "streamed"):+ model = _sharded(device, world_size, q)+ torch.cuda.synchronize()+ torch.cuda.empty_cache()+ base = torch.cuda.memory_allocated(device)+ torch.cuda.reset_peak_memory_stats(device)+ if mode == "streamed":+ options = m.merged_save_gather_options(model, method)+ elif method == "merged_4bit":+ options = dict(offload_to_cpu = m.offload_all_but_4bit_merge_operands(model))+ else:+ options = dict(offload_to_cpu = True)+ written = None+ with m.fsdp2_gathered(model, **options) as writer:+ if writer and method == "merged_4bit":+ if mode == "streamed":+ m.merge_4bit_lora_layer_by_layer(model)+ merged = model.base_model.merge_and_unload()+ written = {+ k: v.detach().reshape(-1).view(torch.uint8).cpu().clone()+ for k, v in merged.state_dict().items()+ if isinstance(v, torch.Tensor)+ }+ elif writer:+ # merged_16bit reads the LoRA weights (the base comes from the 16-bit files).+ written = {+ k: v.detach().cpu().clone()+ for k, v in model.state_dict().items()+ if "lora_" in k+ }+ torch.cuda.synchronize()+ peak = torch.cuda.max_memory_allocated(device) - base+ out[(method, mode)] = dict(peak_above_shards = peak, written = written)+ del model+ dist.destroy_process_group()+ if rank == 0:+ for method in ("merged_4bit", "merged_16bit"):+ a, b = out[(method, "head")]["written"], out[(method, "streamed")]["written"]+ out[(method, "same_bytes")] = sorted(a) == sorted(b) and all(+ torch.equal(a[k], b[k]) for k in a+ )+ for mode in ("head", "streamed"):+ out[(method, mode)].pop("written")+ queue.put((rank, out))+ except BaseException as error:+ import traceback++ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))+++def test_merged_saves_hold_one_packed_layer_on_the_writer_gpu():+ import torch.multiprocessing as mp+ from tests._rendezvous_port import free_port++ world_size = 2 if torch.cuda.device_count() >= 2 else 1+ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = free_port()+ procs = [ctx.Process(target = _worker, args = (r, world_size, port, queue)) for r in range(world_size)]+ for p in procs:+ p.start()+ results = dict(queue.get(timeout = 600) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ _check(results)+++def _check(results):+ for r in results.values():+ assert "error" not in r, r.get("error")+ r0 = results[0]+ layer, total = r0["packed_layer"], r0["packed_total"]+ print({k: v for k, v in r0.items()})+ for method in ("merged_4bit", "merged_16bit"):+ assert r0[(method, "same_bytes")], method+ head = r0[(method, "head")]["peak_above_shards"]+ streamed = r0[(method, "streamed")]["peak_above_shards"]+ # The previous placement holds the whole gathered packed base.+ assert head >= total, (method, head, total)+ # Streaming holds one gathered layer, and merged_4bit one layer's merge temporaries on top (bf16+ # dequant, fp32 LoRA delta, the requantize copy: a few dozen packed layers' worth, whatever the+ # depth) next to the adapters, never the base.+ assert r0[("merged_16bit", "streamed")]["peak_above_shards"] <= 2 * layer, r0+ merged_4bit = r0[("merged_4bit", "streamed")]["peak_above_shards"]+ assert merged_4bit <= 40 * layer + r0["lora_total"], r0+ assert merged_4bit < total, r0+++if __name__ == "__main__":+ import argparse+ import json+ import queue as queue_module+ import sys++ sys.path.insert(0, str(_ROOT))+ parser = argparse.ArgumentParser()+ parser.add_argument("--out", required = True)+ args = parser.parse_args()+ from tests._rendezvous_port import free_port++ results = queue_module.Queue()+ _worker(0, 1, free_port(), results)+ rank, result = results.get()+ printable = {str(k): v for k, v in result.items()}+ with open(args.out, "w") as f:+ json.dump(printable, f, indent = 1)+ print(json.dumps(printable))+ if rank == 0:+ _check({0: result})+ print("CHECK PASSED")diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.pyindex 1277c1a8f75..f3ada289115 100644--- a/tests/test_fsdp2_save_coverage.py+++ b/tests/test_fsdp2_save_coverage.py@@ -95,9 +95,10 @@ def test_refusal_passes_through_without_fsdp2(monkeypatch):assert export(object(), 1) == ("ran", 1)-def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):- """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, so the gather may put- them on the CPU; a merged save dequantizes them where they are and keeps them there."""+def test_full_and_merged_saves_stage_4bit_on_the_cpu(monkeypatch):+ """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, merged_16bit reads its+ base from the 16-bit checkpoint, and merged_4bit brings one packed layer at a time back to its+ device to merge, so every one of them gathers the packed bytes onto the CPU."""seen = []@contextlib.contextmanager@@ -119,4 +120,93 @@ def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):is None)assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None- assert seen == [True, True, False, False, False], seen+ assert seen == [True, True, True, True, True], seen+++def test_streamed_merge_failure_clears_every_merged_layer(monkeypatch):+ # A later layer's merge raising must not leave earlier layers claiming a merge the gather undoes.+ import torch++ class _Layer(torch.nn.Module):+ def __init__(self, fail):+ super().__init__()+ self.base_layer = torch.nn.Linear(2, 2, bias = False)+ self.merged_adapters = []+ self.fail = fail++ def merge(self):+ if self.fail:+ raise RuntimeError("merge failed")+ self.merged_adapters.append("default")++ model = torch.nn.Sequential(_Layer(False), _Layer(False), _Layer(True))+ staged = {id(layer.base_layer.weight): "cpu" for layer in model}+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])+ monkeypatch.setattr(fsdp2_save, "_restage_4bit", lambda weight, device: weight)+ with pytest.raises(RuntimeError, match = "merge failed"):+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)+ assert [layer.merged_adapters for layer in model] == [[], [], []]+++def test_streamed_merge_restage_failure_clears_merge_flags(monkeypatch):+ # A failing restage after a successful merge must not leave the layer claiming the merge.+ import torch++ class _Layer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.base_layer = torch.nn.Linear(2, 2, bias = False)+ self.base_layer.weight.quant_state = object()+ self.merged_adapters = []++ def merge(self):+ self.merged_adapters.append("default")++ model = torch.nn.Sequential(_Layer(), _Layer())+ staged = {id(layer.base_layer.weight): "cuda" for layer in model}+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])+ calls = []++ def restage(weight, device):+ calls.append(device)+ if device == "cpu" and len(calls) == 4:+ raise RuntimeError("host out of memory")+ return weight++ monkeypatch.setattr(fsdp2_save, "_restage_4bit", restage)+ with pytest.raises(RuntimeError, match = "host out of memory"):+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)+ assert [layer.merged_adapters for layer in model] == [[], []]+++def test_streamed_merge_gpu_restage_failure_clears_earlier_merge_flags(monkeypatch):+ # The GPU restage of a later layer failing must also clear the layers merged before it.+ import torch++ class _Layer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.base_layer = torch.nn.Linear(2, 2, bias = False)+ self.base_layer.weight.quant_state = object()+ self.merged_adapters = []++ def merge(self):+ self.merged_adapters.append("default")++ model = torch.nn.Sequential(_Layer(), _Layer())+ staged = {id(layer.base_layer.weight): "cuda" for layer in model}+ monkeypatch.setattr(fsdp2_save, "_STAGED_4BIT", [staged])+ calls = []++ def restage(weight, device):+ calls.append(device)+ if device == "cuda" and len(calls) == 3:+ raise RuntimeError("CUDA out of memory")+ return weight++ monkeypatch.setattr(fsdp2_save, "_restage_4bit", restage)+ second = model[1].base_layer.weight+ with pytest.raises(RuntimeError, match = "CUDA out of memory"):+ fsdp2_save.merge_4bit_lora_layer_by_layer(model)+ assert [layer.merged_adapters for layer in model] == [[], []]+ assert model[1].base_layer._parameters["weight"] is seconddiff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.pyindex 204e7138115..9967d30c373 100644--- a/tests/test_fsdp2_save_gloo.py+++ b/tests/test_fsdp2_save_gloo.py@@ -297,6 +297,137 @@ def _case_merge_offload(rank, world_size, m):return out+class _QLoraBlock(torch.nn.Module):+ def __init__(self, seed):+ super().__init__()+ bnb = pytest.importorskip("bitsandbytes")+ torch.manual_seed(seed)+ self.q = bnb.nn.Linear4bit(+ 128, 128, bias = False, compute_dtype = torch.bfloat16,+ quant_type = "nf4", quant_storage = torch.bfloat16,+ )+ self.q.weight = bnb.nn.Params4bit(+ torch.randn(128, 128) * 0.05, requires_grad = False, quant_type = "nf4",+ quant_storage = torch.bfloat16, blocksize = 64, module = self.q,+ )+ self.o = torch.nn.Linear(128, 128, bias = False).requires_grad_(False)++ def forward(self, x):+ return x + self.o(self.q(x))+++class _QLoraModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_QLoraBlock(7 * i + 1) for i in range(N_LAYERS)])++ def forward(self, x):+ for layer in self.layers:+ x = layer(x)+ return x+++def _qlora_peft(seed = 5):+ from peft import LoraConfig, get_peft_model++ model = _QLoraModel().to("cpu") # quantizes the Params4bit+ model.is_loaded_in_4bit = True # what PEFT reads to give Linear4bit its 4-bit LoRA layer+ torch.manual_seed(seed)+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q", "o"], init_lora_weights = False)+ return get_peft_model(model, config)+++def _packed_bytes(sd):+ return {+ k: v.detach().reshape(-1).view(torch.uint8).clone() if isinstance(v, torch.Tensor) else v+ for k, v in sd.items()+ }+++def _case_merged_4bit_streamed(rank, world_size, m):+ """merged_4bit of a sharded NF4 QLoRA, as save_pretrained_merged runs it: the gather with the+ merged save's options, the streamed layer by layer merge, then PEFT's merge_and_unload and the+ state dict the export writes. `head` is #14739's placement (every packed weight stays on its+ device, PEFT merges them all there); `streamed` stages them on the CPU and brings one back at a+ time. Both must write the bytes an unsharded merge writes."""+ import warnings++ q = _load("fsdp2_qlora")+ from peft.tuners.lora import bnb as peft_bnb+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ warnings.filterwarnings("ignore")+ reference = _packed_bytes(_qlora_peft().base_model.merge_and_unload().state_dict())+ out = {}+ original_cpu = torch.Tensor.cpu+ original_merge = peft_bnb.Linear4bit.merge+ original_restage = getattr(m, "_restage_4bit", None)+ for mode in ("head", "streamed"):+ model = _qlora_peft()+ q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh("cpu", (world_size,))+ for layer in model.base_model.model.layers:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ if mode == "head":+ options = dict(offload_to_cpu = m.offload_all_but_4bit_merge_operands(model))+ else:+ options = m.merged_save_gather_options(model, "merged_4bit")+ events = []+ moved = []++ def recording_cpu(self, *args, **kwargs):+ moved.append(tuple(self.shape))+ return original_cpu(self, *args, **kwargs)++ def recording_merge(self, *args, **kwargs):+ events.append("merge")+ return original_merge(self, *args, **kwargs)++ def recording_restage(weight, device):+ events.append("restage")+ return original_restage(weight, device)++ torch.Tensor.cpu = recording_cpu+ peft_bnb.Linear4bit.merge = recording_merge+ if original_restage is not None:+ m._restage_4bit = recording_restage+ try:+ with m.fsdp2_gathered(model, **options) as writer:+ torch.Tensor.cpu = original_cpu+ packed_moved = sum(len(s) == 2 and s[-1] == 1 for s in moved)+ if writer:+ staged = (+ m.merge_4bit_lora_layer_by_layer(model) if mode == "streamed" else []+ )+ on_cpu_before_unload = [+ layer.q.base_layer.weight.quant_state.absmax.device.type+ for layer in model.base_model.model.layers+ ]+ events.append("unload")+ merged = model.base_model.merge_and_unload()+ sd = _packed_bytes(merged.state_dict())+ out[mode] = dict(+ packed_moved = packed_moved,+ streamed_layers = len(staged),+ events = events,+ states_cpu = on_cpu_before_unload,+ same_keys = sorted(sd) == sorted(reference),+ same_bytes = all(+ torch.equal(sd[k], reference[k]) if isinstance(sd[k], torch.Tensor)+ else sd[k] == reference[k]+ for k in reference+ ),+ )+ finally:+ torch.Tensor.cpu = original_cpu+ peft_bnb.Linear4bit.merge = original_merge+ if original_restage is not None:+ m._restage_4bit = original_restage+ return out++def _case_replicated(rank, world_size, m, hooked):import torch.distributed as dist@@ -805,6 +936,8 @@ def _worker(rank, world_size, port, case, queue):result = _case_merge_offload(rank, world_size, m)elif case == "gather_4bit":result = _case_gather_4bit(rank, world_size, m)+ elif case == "merged_4bit_streamed":+ result = _case_merged_4bit_streamed(rank, world_size, m)elif case == "gather_4bit_offload":result = _case_gather_4bit_offload(rank, world_size, m)elif case == "dcp_quant_state":@@ -1015,3 +1148,27 @@ def test_full_dcp_broadcast_load_restores_quant_state_over_nccl():for broadcast in (False, True):assert r[f"legacy_{broadcast}"] is True, rassert r[f"mismatch_{broadcast}"] == "ValueError", r+++def test_merged_4bit_save_streams_the_packed_base_one_layer_at_a_time():+ """merged_4bit of a sharded NF4 QLoRA kept every gathered packed weight on rank 0's accelerator+ until PEFT merged them all, so the whole quantized base had to fit on one GPU next to its shards.+ The gather now stages each packed weight on the CPU right after gathering it, and the merge+ brings one layer back at a time (restage, merge, restage), writing the same bytes as an unsharded+ merge and as the previous placement."""+ pytest.importorskip("bitsandbytes")+ pytest.importorskip("peft")+ r0 = _run("merged_4bit_streamed")[0]+ head, streamed = r0["head"], r0["streamed"]+ for r in (head, streamed):+ assert r["same_keys"] and r["same_bytes"], r+ assert head["packed_moved"] == 0, head+ # Every packed weight left the gathering device as soon as it was gathered.+ assert streamed["packed_moved"] == N_LAYERS, streamed+ assert streamed["streamed_layers"] == N_LAYERS, streamed+ assert all(device == "cpu" for device in streamed["states_cpu"]), streamed+ # One layer on its device at a time: each merge is bracketed by its own two restages, and PEFT's+ # unload merges nothing more.+ assert streamed["events"] == ["restage", "merge", "restage"] * N_LAYERS + ["unload"] + [+ "merge"+ ] * N_LAYERS, streamed["events"]diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 3556b0a57b6..a1cafa42bc8 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -27,6 +27,9 @@ import torch# Writer flag of the gather in progress: a nested save (save_pretrained_merged -> save_pretrained,# or on a submodule) must neither gather again nor reshard what rank 0 swapped in._ACTIVE = []+# Per gather in progress: id of each packed 4-bit weight staged on the CPU -> the device it was+# gathered on, where `merge_4bit_lora_layer_by_layer` merges it.+_STAGED_4BIT = []def _fsdp_module_class():@@ -132,18 +135,31 @@ def _gather_full(param, DTensor):return gathered.view(param.dtype)-def _quant_state_on_cpu(quant_state):- """A CPU copy of a bitsandbytes QuantState. `QuantState.to` moves in place (its nested state- too), and the live one must stay where the sharded weight computes with it."""+def _quant_state_on_cpu(quant_state, device = "cpu"):+ """A copy of a bitsandbytes QuantState on `device` (the CPU by default). `QuantState.to` moves in+ place (its nested state too), and the live one must stay where the sharded weight computes with it."""if not hasattr(quant_state, "to"):return quant_statestate = copy.copy(quant_state)if getattr(state, "state2", None) is not None:state.state2 = copy.copy(state.state2)- state.to("cpu")+ state.to(device)return state+def _restage_4bit(weight, device):+ """The packed Params4bit `weight` on `device`, with its own copy of the QuantState there. Every+ other attribute is carried over as is, which is what PEFT's 4-bit merge rebuilds the weight from."""+ from bitsandbytes.nn import Params4bit++ moved = torch.Tensor._make_subclass(Params4bit, weight.data.to(device), False)+ moved.__dict__.update(weight.__dict__)+ quant_state = getattr(weight, "quant_state", None)+ if quant_state is not None:+ moved.quant_state = _quant_state_on_cpu(quant_state, device)+ return moved++def _rebuild(param, full, module):"""A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""quant_state = getattr(param, "quant_state", None)@@ -194,8 +210,9 @@ def fsdp2_gathered(plus those `include(name, param)` selects (frozen ones the save still writes).`offload_to_cpu` may be a predicate `(name, param) -> bool`.Packed 4-bit weights stay on their device (a merge dequantizes them there) unless `offload_4bit`:- a save that only writes the packed bytes then gets them, with a CPU copy of their QuantState, on- the CPU, so rank 0 never holds the whole quantized base on its accelerator."""+ they then go, with a CPU copy of their QuantState, to the CPU right after each one is gathered, so+ rank 0 never holds the whole quantized base on its accelerator. A merge brings them back one layer+ at a time (`merge_4bit_lora_layer_by_layer`)."""if _ACTIVE:yield _ACTIVE[-1]return@@ -218,6 +235,7 @@ def fsdp2_gathered(if param is not None:owners.setdefault(id(param), []).append((module, name))replaced = {}+ staged = {}seen = set()pushed = Falsetry:@@ -240,25 +258,32 @@ def fsdp2_gathered(module = owners[id(param)][0][0]offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpuquant_state = getattr(param, "quant_state", None)+ home = Noneif offload and quant_state is None:full = full.cpu()elif offload and offload_4bit:+ home = full.devicefull = full.cpu()quant_state = _quant_state_on_cpu(quant_state)rebuilt = _rebuild(param, full.detach(), module)if quant_state is not None and rebuilt.quant_state is not quant_state:rebuilt.quant_state = quant_state+ if home is not None:+ staged[id(rebuilt)] = homereplaced[id(param)] = (param, rebuilt)if writer:for key, (old, new) in replaced.items():for module, name in owners[key]:module._parameters[name] = new_ACTIVE.append(writer)+ _STAGED_4BIT.append(staged)pushed = Trueyield writerfinally:if pushed:_ACTIVE.pop()+ _STAGED_4BIT.pop()+ staged.clear()for key, (old, new) in replaced.items():for module, name in owners[key]:module._parameters[name] = old@@ -301,6 +326,89 @@ def offload_all_but_4bit_merge_operands(model):return lambda name, param: not name.startswith(prefixes)+_MERGED_4BIT_METHODS = ("merged_4bit", "merged_4bit_forced", "forced_merged_4bit")+++def _merges_onto_dequantized_4bit(model):+ """unsloth_zoo folds a merged_16bit LoRA onto dequant(W4) for a few architectures (falcon_h1),+ dequantizing the live packed weight where it sits."""+ try:+ from unsloth_zoo.saving_utils import (+ _model_type_needs_dequant_merge_base,+ find_lora_base_model,+ )+ except Exception:+ config = getattr(model, "config", None)+ return (getattr(config, "model_type", "") or "").lower() == "falcon_h1"+ try:+ model = find_lora_base_model(model)+ except Exception:+ pass+ return bool(_model_type_needs_dequant_merge_base(model))+++def merged_save_gather_options(model, save_method):+ """`fsdp2_gathered` options for save_pretrained_merged / push_to_hub_merged.++ merged_16bit reads its base weights from the 16-bit checkpoint and only the LoRA weights from the+ model, so the packed 4-bit base goes to the CPU like everything else.+ merged_4bit stages the packed base on the CPU too, keeps the adapters of 4-bit layers on their+ device, and `merge_4bit_lora_layer_by_layer` brings one packed layer at a time back there to merge.+ Either way rank 0's accelerator holds one gathered layer above its shards, not the whole base.+ A merged_16bit that dequantizes the live 4-bit base (`_merges_onto_dequantized_4bit`) and other+ export methods keep the previous placement."""+ if save_method in _MERGED_4BIT_METHODS:+ return dict(offload_to_cpu = offload_all_but_4bit_merge_operands(model), offload_4bit = True)+ if save_method == "merged_16bit" and not _merges_onto_dequantized_4bit(model):+ return dict(offload_to_cpu = True, offload_4bit = True)+ return dict(offload_to_cpu = True)+++def merge_4bit_lora_layer_by_layer(model):+ """Writer, inside `fsdp2_gathered` with packed 4-bit weights staged on the CPU: merges each LoRA+ layer over such a base with PEFT's own `merge`, one layer at a time, on the device that weight+ was gathered on (where the merge ran before, so the requantized bytes are the same), then stages+ the merged weight back on the CPU. PEFT's `merge_and_unload` afterwards finds those layers merged+ and only unloads them. Returns the merged layers (see `unmerge_flags`); [] outside such a gather."""+ staged = _STAGED_4BIT[-1] if _STAGED_4BIT else None+ if not staged or not isinstance(model, torch.nn.Module):+ return []+ merged = []+ for module in model.modules():+ base_layer = getattr(module, "base_layer", None)+ if not isinstance(base_layer, torch.nn.Module) or not callable(getattr(module, "merge", None)):+ continue+ weight = base_layer._parameters.get("weight", None)+ home = staged.get(id(weight)) if weight is not None else None+ if home is None:+ continue+ before = list(getattr(module, "merged_adapters", None) or ())+ try:+ base_layer._parameters["weight"] = _restage_4bit(weight, home)+ module.merge()+ result = base_layer._parameters.get("weight", None)+ if result is not None and getattr(result, "quant_state", None) is not None:+ result = _restage_4bit(result, "cpu")+ base_layer._parameters["weight"] = result+ except BaseException:+ base_layer._parameters["weight"] = weight+ # Layers merged earlier in this loop get their unmerged shards back from the gather too.+ if list(getattr(module, "merged_adapters", None) or ()) != before:+ merged.append((module, before))+ unmerge_flags(merged)+ raise+ if list(getattr(module, "merged_adapters", None) or ()) != before:+ merged.append((module, before))+ return merged+++def unmerge_flags(merged):+ """Undo `merge_4bit_lora_layer_by_layer`'s bookkeeping when the export fails before it unloads:+ the gather puts the sharded (unmerged) base weights back, so the layers must not claim a merge."""+ for module, before in merged:+ module.merged_adapters[:] = before++def fsdp2_gather_active():return bool(_ACTIVE)diff --git a/unsloth/save.py b/unsloth/save.pyindex 9c60b7668c3..d4e25cacd3d 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6270,6 +6270,17 @@ def _cast_state_dict_16bit(model, state_dict, target_dtype):}+@contextmanager+def _unmerge_flags_on_error(streamed):+ try:+ yield+ except BaseException:+ if streamed:+ from .models.fsdp2_save import unmerge_flags+ unmerge_flags(streamed)+ raise++@_normalize_tied_weights_keys_for_save@torch.inference_modedef unsloth_generic_save(@@ -6510,7 +6521,14 @@ def unsloth_generic_save(# merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.in_place = save_method in ("merged_4bit", "forced_merged_4bit")- with nullcontext() if in_place else lora_relative_to_original_base(model):+ # Under FSDP2 the gathered packed 4-bit base waits on the CPU: merge it one layer at a time on+ # its GPU first, so the merge below only unloads those layers.+ streamed = []+ if in_place and getattr(model.config, "quantization_config", None):+ from .models.fsdp2_save import merge_4bit_lora_layer_by_layer+ streamed = merge_4bit_lora_layer_by_layer(model)+ relative = nullcontext() if in_place else lora_relative_to_original_base(model)+ with relative, _unmerge_flags_on_error(streamed):merge_and_overwrite_lora(_modelscope_base_model_nameif os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" and not in_place@@ -6602,12 +6620,14 @@ def _fsdp2_collective_save(trainable_only,offload_to_cpu = None,offload_4bit = False,+ gather_options = None,):"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the othersreturn None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;`offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.`offload_4bit`: the save only writes packed 4-bit weights (never merges them), so they go to- the CPU too.+ the CPU too. `gather_options(self, args, kwargs)`, when given, returns the `offload_to_cpu` /+ `offload_4bit` options for `fsdp2_gathered` instead (merged saves).A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""def decorate(fn):@@ -6629,12 +6649,14 @@ def _fsdp2_collective_save(+ " weights to rank 0, which writes. Every rank must make this call.")offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)+ options = dict(offload_to_cpu = offload, offload_4bit = offload_4bit)+ if gather_options is not None and not only:+ options.update(gather_options(self, args, kwargs))with fsdp2_gathered(self,trainable_only = only,include = include,- offload_to_cpu = offload,- offload_4bit = offload_4bit,+ **options,) as writer:if writer is False:return None@@ -6668,12 +6690,19 @@ def _fsdp2_merged_offload(model, args, kwargs):return offload_all_but_4bit_merge_operands(model)+def _fsdp2_merged_gather_options(model, args, kwargs):+ from .models.fsdp2_save import merged_save_gather_options+ return merged_save_gather_options(model, _merged_save_method(args, kwargs))++@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel)and _is_adapter_save_method(_merged_save_method(args, kwargs)),- # merged_4bit merges with PEFT where the packed 4-bit weights stay (on the GPU), so those layers'- # LoRA weights stay on that device too; every other gathered weight still goes to the CPU.+ # Packed 4-bit weights are staged on the CPU as they are gathered. merged_4bit merges with PEFT+ # one layer at a time on the GPU the packed weight came from, so those layers' LoRA weights stay+ # on that device; every other gathered weight goes to the CPU.offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),+ gather_options = _fsdp2_merged_gather_options,)def unsloth_generic_save_pretrained_merged(self,@@ -6838,6 +6867,7 @@ def unsloth_generic_save_pretrained_merged(lambda self, args, kwargs: isinstance(self, PeftModel)and _is_adapter_save_method(_merged_save_method(args, kwargs)),offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),+ gather_options = _fsdp2_merged_gather_options,)def unsloth_generic_push_to_hub_merged(self,