Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 NF4 saves: stage packed bytes on the CPU, restore QuantState on full DCP load (#14739)

Daniel Han59cd08bfb4ee
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.py
index 25a4056c815..1277c1a8f75 100644
--- a/tests/test_fsdp2_save_coverage.py
+++ b/tests/test_fsdp2_save_coverage.py
@@ -93,3 +93,30 @@ def test_refusal_passes_through_without_fsdp2(monkeypatch):
monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])
export = unsloth_save._fsdp2_refuses_save("save_pretrained_gguf")(lambda self, x: ("ran", x))
assert export(object(), 1) == ("ran", 1)
+
+
+def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):
+ """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, so the gather may put
+ them on the CPU; a merged save dequantizes them where they are and keeps them there."""
+ seen = []
+
+ @contextlib.contextmanager
+ def gathered(model, **kwargs):
+ seen.append(kwargs.get("offload_4bit", False))
+ yield False
+
+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)
+ model = unsloth_save.patch_saving_functions(_tiny_llama())
+ assert model.save_pretrained("out") is None
+ assert model.push_to_hub("user/repo") is None
+ assert unsloth_save.unsloth_generic_save_pretrained_merged(model, "out", None) is None
+ assert (
+ unsloth_save.unsloth_generic_save_pretrained_merged(
+ model, "out", None, save_method = "merged_4bit"
+ )
+ is None
+ )
+ assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None
+ assert seen == [True, True, False, False, False], seen
diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.py
index 2255ca22be2..7d35883189e 100644
--- a/tests/test_fsdp2_save_gloo.py
+++ b/tests/test_fsdp2_save_gloo.py
@@ -471,6 +471,269 @@ def _case_dcp_bnb(rank, world_size, m):
return out
+def _nf4_model(nested = False, device = "cpu"):
+ bnb = pytest.importorskip("bitsandbytes")
+
+ def lin(seed):
+ torch.manual_seed(seed)
+ layer = bnb.nn.Linear4bit(
+ 128,
+ 96,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ compress_statistics = nested,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ )
+ layer.weight = bnb.nn.Params4bit(
+ torch.randn(96, 128) * 0.05,
+ requires_grad = False,
+ compress_statistics = nested,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ blocksize = 64,
+ module = layer,
+ )
+ return layer.to(device)
+
+ return torch.nn.Sequential(lin(0), lin(1))
+
+
+def _case_gather_4bit_offload(rank, world_size, m):
+ """A save that only writes the packed 4-bit bytes (`offload_4bit`) gets them and a copy of their
+ QuantState on the CPU; the merge path (default) keeps them where they are. On CPU ranks `.cpu()`
+ is a no-op, so the calls are recorded."""
+ q = _load("fsdp2_qlora")
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ model = _nf4_model()
+ reference = {k: v.clone() for k, v in model.state_dict().items()}
+ q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh("cpu", (world_size,))
+ for layer in model:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ live = [layer.weight.quant_state for layer in model]
+ live_absmax = [s.absmax for s in live]
+ original_cpu = torch.Tensor.cpu
+ moved = []
+
+ def recording_cpu(self, *args, **kwargs):
+ moved.append((tuple(self.shape), self.dtype))
+ return original_cpu(self, *args, **kwargs)
+
+ out = {}
+ for offload_4bit in (False, True):
+ moved.clear()
+ torch.Tensor.cpu = recording_cpu
+ try:
+ kwargs = {"offload_4bit": True} if offload_4bit else {}
+ with m.fsdp2_gathered(model, **kwargs) as writer:
+ torch.Tensor.cpu = original_cpu
+ if writer:
+ states = [layer.weight.quant_state for layer in model]
+ sd = model.state_dict()
+ out[offload_4bit] = dict(
+ # The packed bytes are the only (n, 1) tensors gathered here.
+ packed_moved = sum(len(shape) == 2 and shape[-1] == 1 for shape, _ in moved),
+ own_state = [s is not l for s, l in zip(states, live)],
+ state_cpu = [s.absmax.device.type == "cpu" for s in states],
+ same_keys = sorted(sd) == sorted(reference),
+ same_bytes = all(
+ torch.equal(
+ sd[k].reshape(-1).view(torch.uint8),
+ reference[k].reshape(-1).view(torch.uint8),
+ )
+ if isinstance(sd[k], torch.Tensor)
+ else sd[k] == reference[k]
+ for k in reference
+ ),
+ )
+ finally:
+ torch.Tensor.cpu = original_cpu
+ # The live QuantState is untouched (same object, same tensors).
+ out["live_kept"] = all(
+ layer.weight.quant_state is s and s.absmax is a
+ for layer, s, a in zip(model, live, live_absmax)
+ )
+ return out
+
+
+def _case_dcp_quant_state(rank, world_size, m):
+ """A full DCP load restores the QuantState it saved, not just the packed bytes, and refuses one
+ that does not fit the model."""
+ q = _load("fsdp2_qlora")
+ from torch.distributed.checkpoint.state_dict import StateDictOptions
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+ import torch.distributed.checkpoint.state_dict as dcp
+
+ out = {}
+ for nested in (False, True):
+ model = _nf4_model(nested)
+ q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh("cpu", (world_size,))
+ for layer in model:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ m.install_dcp_bnb_state_dict()
+ save_opts = StateDictOptions(full_state_dict = True) # every rank gets the full dict
+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = save_opts).items()}
+ states = [layer.weight.quant_state for layer in model]
+ before = [s.absmax.clone() for s in states]
+ results = {}
+ for broadcast in (False, True):
+ for s in states:
+ s.absmax.mul_(2) # a requantized / differently initialized base
+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = broadcast)
+ payload = saved if (rank == 0 or not broadcast) else {}
+ dcp.set_model_state_dict(model, payload, options = options)
+ results[broadcast] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))
+ out[f"restored_nested_{nested}"] = results
+ bad = dict(saved)
+ key = next(k for k in bad if k.endswith("weight.absmax"))
+ bad[key] = bad[key][:-1].clone()
+ try:
+ dcp.set_model_state_dict(
+ model, bad, options = StateDictOptions(full_state_dict = True)
+ )
+ out[f"mismatch_nested_{nested}"] = "loaded"
+ except ValueError as error:
+ out[f"mismatch_nested_{nested}"] = "ValueError" if "4-bit" in str(error) else str(error)
+ return out
+
+
+def _sharded_nf4(world_size, m, device = "cpu"):
+ q = _load("fsdp2_qlora")
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ model = _nf4_model(device = device)
+ q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh(device.split(":")[0], (world_size,))
+ for layer in model:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ m.install_dcp_bnb_state_dict()
+ return model
+
+
+def _legacy_and_mismatched_loads(rank, model, saved, out):
+ """Layer 0 saved without its QuantState entries (a checkpoint from before they were saved): with
+ the live packed bytes it loads, keeps the live QuantState and dequantizes to the live weights;
+ with other packed bytes it raises. Both with and without broadcast_from_rank0."""
+ from bitsandbytes.functional import QuantState, dequantize_4bit
+ from torch.distributed.checkpoint.state_dict import StateDictOptions
+ import torch.distributed.checkpoint.state_dict as dcp
+
+ def dequantized(state_dict):
+ stats = {k[len("0.weight.") :]: v for k, v in state_dict.items() if k.startswith("0.weight.")}
+ state = QuantState.from_dict(stats, device = "cpu")
+ packed = state_dict["0.weight"].cpu().reshape(-1, 1)
+ state.absmax, state.code = state.absmax.cpu(), state.code.cpu()
+ return dequantize_4bit(packed, state).float()
+
+ reference = dequantized({k: v.cpu() for k, v in saved.items()})
+ live_absmax = model[0].weight.quant_state.absmax.clone()
+ legacy = {k: v for k, v in saved.items() if not k.startswith("0.weight.")}
+ mismatched = dict(legacy)
+ mismatched["0.weight"] = saved["1.weight"].clone() # same shape, other bytes
+ full = StateDictOptions(full_state_dict = True) # every rank gets the full dict
+ for broadcast in (False, True):
+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = broadcast)
+ try:
+ dcp.set_model_state_dict(model, legacy if (rank == 0 or not broadcast) else {}, options = options)
+ after = dcp.get_model_state_dict(model, options = full)
+ out[f"legacy_{broadcast}"] = (
+ torch.equal(model[0].weight.quant_state.absmax, live_absmax)
+ and torch.equal(dequantized(after), reference)
+ )
+ except Exception as error:
+ out[f"legacy_{broadcast}"] = f"{type(error).__name__}: {error}"
+ try:
+ dcp.set_model_state_dict(
+ model, mismatched if (rank == 0 or not broadcast) else {}, options = options
+ )
+ out[f"mismatch_{broadcast}"] = "loaded"
+ except ValueError as error:
+ out[f"mismatch_{broadcast}"] = "ValueError" if "`0.weight`" in str(error) else str(error)
+
+
+def _case_dcp_quant_state_missing(rank, world_size, m):
+ """A full DCP state dict holding a packed 4-bit weight but none of its QuantState entries loads
+ only when the bytes are the live ones (older checkpoints of a frozen base); other bytes would be
+ paired with the live scales and must not load. One without the packed weight still loads.
+ The broadcast_from_rank0 presence check must not send a CPU tensor through the process group
+ (NCCL rejects those): with tensor broadcasts of CPU tensors refused, the restore still works."""
+ from torch.distributed.checkpoint.state_dict import StateDictOptions
+ import torch.distributed as dist
+ import torch.distributed.checkpoint.state_dict as dcp
+
+ model = _sharded_nf4(world_size, m)
+ full = StateDictOptions(full_state_dict = True)
+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = full).items()}
+ out = {}
+ _legacy_and_mismatched_loads(rank, model, saved, out)
+ stripped = {k: v for k, v in saved.items() if not k.startswith("0.weight.")}
+ # No packed weight and no QuantState for layer 0: nothing of it is loaded, so nothing to refuse.
+ no_weight = {k: v for k, v in stripped.items() if k != "0.weight"}
+ try:
+ dcp.set_model_state_dict(
+ model, no_weight, options = StateDictOptions(full_state_dict = True, strict = False)
+ )
+ out["no_weight"] = "loaded"
+ except Exception as error:
+ out["no_weight"] = f"{type(error).__name__}: {error}"
+
+ # NCCL-like group: a tensor collective on a CPU tensor fails. Only the restore runs under it.
+ states = [layer.weight.quant_state for layer in model]
+ before = [s.absmax.clone() for s in states]
+ for s in states:
+ s.absmax.mul_(2)
+ original = dist.broadcast
+
+ def nccl_like_broadcast(tensor, *args, **kwargs):
+ if tensor.device.type == "cpu":
+ raise RuntimeError("No backend type associated with device type cpu")
+ return original(tensor, *args, **kwargs)
+
+ dist.broadcast = nccl_like_broadcast
+ try:
+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True)
+ m._restore_quant_state(model, saved if rank == 0 else {}, options)
+ out["nccl_like_flag"] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))
+ except RuntimeError as error:
+ out["nccl_like_flag"] = str(error)
+ finally:
+ dist.broadcast = original
+ return out
+
+
+def _case_nccl_quant_state(rank, world_size, m):
+ """Real NCCL group on two GPUs: a full DCP load with broadcast_from_rank0 restores the saved
+ QuantState on every rank and refuses a packed weight saved without it."""
+ from torch.distributed.checkpoint.state_dict import StateDictOptions
+ import torch.distributed.checkpoint.state_dict as dcp
+
+ model = _sharded_nf4(world_size, m, device = f"cuda:{rank}")
+ full = StateDictOptions(full_state_dict = True) # every rank gets the full dict
+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = full).items()}
+ states = [layer.weight.quant_state for layer in model]
+ before = [s.absmax.clone() for s in states]
+ for s in states:
+ s.absmax.mul_(2)
+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True)
+ out = {"backend": torch.distributed.get_backend()}
+ try:
+ dcp.set_model_state_dict(model, saved if rank == 0 else {}, options = options)
+ out["restored"] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))
+ except Exception as error:
+ out["restored"] = f"{type(error).__name__}: {error}"
+ _legacy_and_mismatched_loads(rank, model, saved, out)
+ return out
+
+
def _case_fp16_reduce(rank, world_size, m):
"""An fp16 trainable parameter FSDP2 does not manage: its gradient mean must not overflow."""
@@ -501,6 +764,9 @@ def _case_slow_writer(rank, world_size, m):
def _worker(rank, world_size, port, case, queue):
try:
+ nccl = case.startswith("nccl")
+ if nccl:
+ torch.cuda.set_device(rank)
os.environ.update(
MASTER_ADDR = "127.0.0.1",
MASTER_PORT = str(port),
@@ -513,7 +779,7 @@ def _worker(rank, world_size, port, case, queue):
timeout = datetime.timedelta(seconds = 4) if case == "slow_writer" else None
dist.init_process_group(
- "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})
+ "nccl" if nccl else "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})
)
m = _load("fsdp2_save")
if case == "slow_writer":
@@ -526,6 +792,14 @@ def _worker(rank, world_size, port, case, queue):
result = _case_merge_offload(rank, world_size, m)
elif case == "gather_4bit":
result = _case_gather_4bit(rank, world_size, m)
+ elif case == "gather_4bit_offload":
+ result = _case_gather_4bit_offload(rank, world_size, m)
+ elif case == "dcp_quant_state":
+ result = _case_dcp_quant_state(rank, world_size, m)
+ elif case == "dcp_quant_state_missing":
+ result = _case_dcp_quant_state_missing(rank, world_size, m)
+ elif case == "nccl_quant_state":
+ result = _case_nccl_quant_state(rank, world_size, m)
elif case.startswith("replicated"):
result = _case_replicated(rank, world_size, m, hooked = case.endswith("hooked"))
else:
@@ -537,7 +811,7 @@ def _worker(rank, world_size, port, case, queue):
queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
-def _run(case, world_size = 2):
+def _run(case, world_size = 2, cpu_only = True):
import torch.multiprocessing as mp
ctx = mp.get_context("spawn")
@@ -549,7 +823,8 @@ def _run(case, world_size = 2):
]
# CPU only: a spawned child inherits this environment before it imports torch.
saved = os.environ.get("CUDA_VISIBLE_DEVICES")
- os.environ["CUDA_VISIBLE_DEVICES"] = ""
+ if cpu_only:
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
try:
for p in procs:
p.start()
@@ -665,3 +940,60 @@ def test_merge_offload_keeps_4bit_adapters_after_an_eval_forward():
expected[f"layers.{i}.{part}.default.weight"] = i == 1 # layer 1 is not 4-bit
expected[f"layers.{i}.base_layer.weight"] = True
assert decisions == expected, decisions
+
+
+def test_packed_4bit_save_stages_bytes_and_quant_state_on_the_cpu():
+ """A full save of a sharded NF4 model kept every gathered packed weight on rank 0's accelerator,
+ so the whole quantized base had to fit on one GPU. Saves that only write the bytes now get them
+ on the CPU, with a CPU copy of the QuantState; the merge path is unchanged and the written state
+ dict is byte for byte the same."""
+ pytest.importorskip("bitsandbytes")
+ r0 = _run("gather_4bit_offload")[0]
+ kept, staged = r0[False], r0[True]
+ assert kept["packed_moved"] == 0 and kept["own_state"] == [False, False], kept
+ assert staged["packed_moved"] == 2, staged
+ assert staged["own_state"] == [True, True] and all(staged["state_cpu"]), staged
+ for r in (kept, staged):
+ assert r["same_keys"] and r["same_bytes"], r
+ assert r0["live_kept"], r0
+
+
+def test_full_dcp_load_restores_the_saved_quant_state():
+ """A full DCP load dropped the saved QuantState entries, so the packed bytes were silently paired
+ with whatever scales the live model had."""
+ pytest.importorskip("bitsandbytes")
+ results = _run("dcp_quant_state")
+ for r in results.values():
+ for nested in (False, True):
+ assert r[f"restored_nested_{nested}"] == {False: True, True: True}, r
+ assert r[f"mismatch_nested_{nested}"] == "ValueError", r
+
+
+def test_full_dcp_load_refuses_a_packed_4bit_weight_without_its_quant_state():
+ """A packed 4-bit weight whose QuantState entries the checkpoint lacks was loaded silently and
+ dequantized with the live model's scales, also when its bytes were not the live ones; an older
+ checkpoint of the same frozen base (same bytes) must keep loading; the broadcast_from_rank0 presence check sent a CPU
+ tensor through the process group, which an NCCL group cannot carry."""
+ pytest.importorskip("bitsandbytes")
+ for r in _run("dcp_quant_state_missing").values():
+ for broadcast in (False, True):
+ assert r[f"legacy_{broadcast}"] is True, r
+ assert r[f"mismatch_{broadcast}"] == "ValueError", r
+ assert r["no_weight"] == "loaded", r
+ assert r["nccl_like_flag"] is True, r
+
+
+ not torch.cuda.is_available()
+ or torch.cuda.device_count() < 2
+ or not torch.distributed.is_nccl_available(),
+ reason = "needs two CUDA GPUs and NCCL",
+)
+def test_full_dcp_broadcast_load_restores_quant_state_over_nccl():
+ pytest.importorskip("bitsandbytes")
+ for r in _run("nccl_quant_state", cpu_only = False).values():
+ assert r["backend"] == "nccl", r
+ assert r["restored"] is True, r
+ for broadcast in (False, True):
+ assert r[f"legacy_{broadcast}"] is True, r
+ assert r[f"mismatch_{broadcast}"] == "ValueError", r
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 50790dd42b9..fe187aa11a0 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -16,8 +16,10 @@ __all__ = [
]
import contextlib
+import copy
import datetime
import functools
+import logging
import os
import torch
@@ -130,6 +132,18 @@ def _gather_full(param, DTensor):
return gathered.view(param.dtype)
+def _quant_state_on_cpu(quant_state):
+ """A CPU copy of a bitsandbytes QuantState. `QuantState.to` moves in place (its nested state
+ too), and the live one must stay where the sharded weight computes with it."""
+ if not hasattr(quant_state, "to"):
+ return quant_state
+ state = copy.copy(quant_state)
+ if getattr(state, "state2", None) is not None:
+ state.state2 = copy.copy(state.state2)
+ state.to("cpu")
+ return state
+
+
def _rebuild(param, full, module):
"""A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""
quant_state = getattr(param, "quant_state", None)
@@ -172,13 +186,16 @@ def fsdp2_gathered(
trainable_only = False,
offload_to_cpu = True,
include = None,
+ offload_4bit = False,
):
"""Yields None for a model FSDP2 does not shard (nothing happens). Otherwise every rank must
enter; rank 0 sees whole parameters (and yields True), the other ranks yield False and should
not write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),
plus those `include(name, param)` selects (frozen ones the save still writes).
`offload_to_cpu` may be a predicate `(name, param) -> bool`.
- Packed 4-bit weights stay on their device: the merge dequantizes them there."""
+ Packed 4-bit weights stay on their device (a merge dequantizes them there) unless `offload_4bit`:
+ a save that only writes the packed bytes then gets them, with a CPU copy of their QuantState, on
+ the CPU, so rank 0 never holds the whole quantized base on its accelerator."""
if _ACTIVE:
yield _ACTIVE[-1]
return
@@ -222,9 +239,16 @@ def fsdp2_gathered(
continue
module = owners[id(param)][0][0]
offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpu
- if offload and getattr(param, "quant_state", None) is None:
+ quant_state = getattr(param, "quant_state", None)
+ if offload and quant_state is None:
full = full.cpu()
- replaced[id(param)] = (param, _rebuild(param, full.detach(), module))
+ elif offload and offload_4bit:
+ full = full.cpu()
+ quant_state = _quant_state_on_cpu(quant_state)
+ rebuilt = _rebuild(param, full.detach(), module)
+ if quant_state is not None and rebuilt.quant_state is not quant_state:
+ rebuilt.quant_state = quant_state
+ replaced[id(param)] = (param, rebuilt)
if writer:
for key, (old, new) in replaced.items():
for module, name in owners[key]:
@@ -459,9 +483,136 @@ def _add_quant_state_entries(model, state_dict, options):
state_dict[prefix + "weight." + key] = value.cpu() if cpu else value
+_LEGACY_4BIT_NOTICE = [False]
+logger = logging.getLogger(__name__)
+
+
+def _as_bytes(tensor):
+ return tensor.detach().contiguous().reshape(-1).view(torch.uint8)
+
+
+def _same_packed_bytes(live, loaded, broadcast_dist):
+ """Whether the packed 4-bit bytes being loaded equal the live (possibly sharded) weight's. Every
+ rank gathers the live weight (a collective); with `broadcast_dist` only rank 0 holds `loaded`,
+ decides, and sends the verdict, so all ranks agree."""
+ DTensor = _dtensor_class()
+ if DTensor is not None and isinstance(live, DTensor):
+ live = _gather_full(live, DTensor)
+ if DTensor is not None and isinstance(loaded, DTensor):
+ loaded = loaded.full_tensor()
+ same = False
+ if isinstance(loaded, torch.Tensor):
+ a, b = _as_bytes(live), _as_bytes(loaded)
+ same = a.numel() == b.numel() and torch.equal(a.cpu(), b.cpu())
+ if broadcast_dist is not None:
+ verdict = [same]
+ broadcast_dist.broadcast_object_list(verdict, src = 0)
+ same = bool(verdict[0])
+ return same
+
+
+def _restore_quant_state(model, state_dict, options):
+ """Loading a full DCP state dict: the saved QuantState entries (`_add_quant_state_entries`) are
+ copied into each live QuantState in place, so the packed bytes DCP loads are dequantized with the
+ scales they were saved with. A saved QuantState that does not fit the live one (other shape,
+ blocksize, quant type or nesting) raises instead of pairing the bytes with the wrong scales. With
+ `broadcast_from_rank0` only rank 0 holds the entries and sends them to the others."""
+ entries = _linear4bit_entries(model)
+ if not entries:
+ return
+ dist = _dist()
+ broadcast = dist is not None and bool(getattr(options, "broadcast_from_rank0", False))
+ if not isinstance(state_dict, dict):
+ state_dict = {}
+ if broadcast:
+ # An object broadcast, not a tensor one: a CPU tensor cannot go through an NCCL group, and
+ # broadcast_object_list stages on whatever device the group's backend needs.
+ has = [any(
+ k == prefix + "weight" or k.startswith(prefix + "weight.")
+ for prefix, _ in entries
+ for k in state_dict
+ )]
+ dist.broadcast_object_list(has, src = 0)
+ if not has[0]:
+ return
+ from bitsandbytes.functional import QuantState
+
+ for prefix, module in entries:
+ live = module.weight.quant_state
+ head = prefix + "weight."
+ saved = {
+ k[len(head) :]: v
+ for k, v in state_dict.items()
+ if k.startswith(head) and isinstance(v, torch.Tensor)
+ }
+ loads_weight = head[:-1] in state_dict
+ if broadcast:
+ payload = [(loads_weight, {k: v.detach().cpu() for k, v in saved.items()})]
+ dist.broadcast_object_list(payload, src = 0)
+ loads_weight, saved = payload[0]
+ if not saved:
+ if loads_weight:
+ # A checkpoint written before QuantState was saved (a frozen QLoRA base): its packed
+ # bytes are the live ones, so the live scales are theirs. Other bytes would be
+ # dequantized with the wrong scales.
+ same = _same_packed_bytes(
+ module.weight, state_dict.get(head[:-1]), dist if broadcast else None
+ )
+ if not same:
+ raise ValueError(
+ f"Unsloth: the checkpoint holds the 4-bit weight `{head[:-1]}` but none of "
+ "its quantization state (`weight.absmax`, `weight.quant_state.*`), and its "
+ "packed bytes differ from the model's, so they cannot be dequantized "
+ "correctly. Save the full state dict again with Unsloth, or load it without "
+ "the frozen 4-bit base weights."
+ )
+ if not _LEGACY_4BIT_NOTICE[0]:
+ _LEGACY_4BIT_NOTICE[0] = True
+ logger.info(
+ "Unsloth: the checkpoint has 4-bit weights without their quantization state "
+ "(saved by an older Unsloth); their packed bytes match the model's, so its "
+ "quantization state is kept."
+ )
+ continue
+ expected = set(live.as_dict(packed = True))
+ if set(saved) != expected:
+ raise ValueError(
+ f"Unsloth: the checkpoint's 4-bit quantization state for `{head[:-1]}` has entries "
+ f"{sorted(saved)}, the model's has {sorted(expected)}."
+ )
+ loaded = QuantState.from_dict(saved, device = live.absmax.device)
+ pairs = [("absmax", loaded.absmax, live.absmax), ("code", loaded.code, live.code)]
+ if live.nested:
+ pairs += [
+ ("offset", loaded.offset, live.offset),
+ ("state2.absmax", loaded.state2.absmax, live.state2.absmax),
+ ("state2.code", loaded.state2.code, live.state2.code),
+ ]
+ same = (
+ loaded.quant_type == live.quant_type
+ and loaded.blocksize == live.blocksize
+ and loaded.nested == live.nested
+ and tuple(loaded.shape) == tuple(live.shape)
+ and loaded.dtype == live.dtype
+ and (not live.nested or loaded.state2.blocksize == live.state2.blocksize)
+ and all(
+ isinstance(a, torch.Tensor) and a.shape == b.shape and a.dtype == b.dtype
+ for _, a, b in pairs
+ )
+ )
+ if not same:
+ raise ValueError(
+ f"Unsloth: the checkpoint's 4-bit quantization state for `{head[:-1]}` does not "
+ "match the model's (shape, blocksize, quant type or nesting differ)."
+ )
+ with torch.no_grad():
+ for _, a, b in pairs:
+ b.copy_(a.to(b.device))
+
+
def _without_quant_state_entries(name, args, kwargs):
- """Loading: drop the entries `_add_quant_state_entries` wrote (DCP would reject them as unexpected
- keys); the QuantState is rebuilt identically when the 4-bit model is loaded again."""
+ """Loading: the entries `_add_quant_state_entries` wrote are restored into the live QuantState
+ (`_restore_quant_state`) and then dropped, as DCP would reject them as unexpected keys."""
model = args[0] if args else kwargs.get("model")
if not isinstance(model, torch.nn.Module):
return args, kwargs
@@ -472,6 +623,12 @@ def _without_quant_state_entries(name, args, kwargs):
)
if not drop:
return args, kwargs
+ if name == "set_model_state_dict" and len(args) > 1:
+ loading = args[1]
+ else:
+ loading = kwargs.get("model_state_dict")
+ if name in ("set_model_state_dict", "set_state_dict"):
+ _restore_quant_state(model, loading, kwargs.get("options", None))
def strip(state_dict):
if not isinstance(state_dict, dict) or not drop.intersection(state_dict):
diff --git a/unsloth/save.py b/unsloth/save.py
index 727a3f6144e..5f9d92761cd 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6416,10 +6416,12 @@ def _fsdp2_peft_saved_frozen(model, arguments):
return include
-def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):
+def _fsdp2_collective_save(trainable_only, offload_to_cpu = None, offload_4bit = False):
"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others
return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;
`offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.
+ `offload_4bit`: the save only writes packed 4-bit weights (never merges them), so they go to
+ the CPU too.
A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""
def decorate(fn):
@@ -6442,7 +6444,11 @@ def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):
)
offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)
with fsdp2_gathered(
- self, trainable_only = only, include = include, offload_to_cpu = offload
+ self,
+ trainable_only = only,
+ include = include,
+ offload_to_cpu = offload,
+ offload_4bit = offload_4bit,
) as writer:
if writer is False:
return None
@@ -8411,10 +8417,13 @@ def patch_saving_functions(model, vision = False):
# Under FSDP2 every rank calls push_to_hub: the shards are gathered and rank 0 uploads. Inside,
# the wrapped save_pretrained sees the gather already active and writes directly.
globals()["unsloth_push_to_hub"] = _fsdp2_collective_save(
- lambda self, args, kwargs: isinstance(self, PeftModel)
+ lambda self, args, kwargs: isinstance(self, PeftModel),
+ offload_4bit = True,
)(globals()["unsloth_push_to_hub"])
- @_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))
+ @_fsdp2_collective_save(
+ lambda self, args, kwargs: isinstance(self, PeftModel), offload_4bit = True
+ )
def unsloth_model_save_pretrained(self, *args, **kwargs):
"""`safe_serialization = None` means the safetensors default, not a pickle.