Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 NF4 saves: stage packed bytes on the CPU, restore QuantState on full DCP load (#14739)
Daniel Han59cd08bfb4ee
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.pyindex 25a4056c815..1277c1a8f75 100644--- a/tests/test_fsdp2_save_coverage.py+++ b/tests/test_fsdp2_save_coverage.py@@ -93,3 +93,30 @@ def test_refusal_passes_through_without_fsdp2(monkeypatch):monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])export = unsloth_save._fsdp2_refuses_save("save_pretrained_gguf")(lambda self, x: ("ran", x))assert export(object(), 1) == ("ran", 1)+++def test_only_saves_that_write_packed_bytes_stage_4bit_on_the_cpu(monkeypatch):+ """save_pretrained / push_to_hub write the packed 4-bit bytes as they are, so the gather may put+ them on the CPU; a merged save dequantizes them where they are and keeps them there."""+ seen = []++ @contextlib.contextmanager+ def gathered(model, **kwargs):+ seen.append(kwargs.get("offload_4bit", False))+ yield False++ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)+ model = unsloth_save.patch_saving_functions(_tiny_llama())+ assert model.save_pretrained("out") is None+ assert model.push_to_hub("user/repo") is None+ assert unsloth_save.unsloth_generic_save_pretrained_merged(model, "out", None) is None+ assert (+ unsloth_save.unsloth_generic_save_pretrained_merged(+ model, "out", None, save_method = "merged_4bit"+ )+ is None+ )+ assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None+ assert seen == [True, True, False, False, False], seendiff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.pyindex 2255ca22be2..7d35883189e 100644--- a/tests/test_fsdp2_save_gloo.py+++ b/tests/test_fsdp2_save_gloo.py@@ -471,6 +471,269 @@ def _case_dcp_bnb(rank, world_size, m):return out+def _nf4_model(nested = False, device = "cpu"):+ bnb = pytest.importorskip("bitsandbytes")++ def lin(seed):+ torch.manual_seed(seed)+ layer = bnb.nn.Linear4bit(+ 128,+ 96,+ bias = False,+ compute_dtype = torch.bfloat16,+ compress_statistics = nested,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ )+ layer.weight = bnb.nn.Params4bit(+ torch.randn(96, 128) * 0.05,+ requires_grad = False,+ compress_statistics = nested,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ blocksize = 64,+ module = layer,+ )+ return layer.to(device)++ return torch.nn.Sequential(lin(0), lin(1))+++def _case_gather_4bit_offload(rank, world_size, m):+ """A save that only writes the packed 4-bit bytes (`offload_4bit`) gets them and a copy of their+ QuantState on the CPU; the merge path (default) keeps them where they are. On CPU ranks `.cpu()`+ is a no-op, so the calls are recorded."""+ q = _load("fsdp2_qlora")+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ model = _nf4_model()+ reference = {k: v.clone() for k, v in model.state_dict().items()}+ q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh("cpu", (world_size,))+ for layer in model:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ live = [layer.weight.quant_state for layer in model]+ live_absmax = [s.absmax for s in live]+ original_cpu = torch.Tensor.cpu+ moved = []++ def recording_cpu(self, *args, **kwargs):+ moved.append((tuple(self.shape), self.dtype))+ return original_cpu(self, *args, **kwargs)++ out = {}+ for offload_4bit in (False, True):+ moved.clear()+ torch.Tensor.cpu = recording_cpu+ try:+ kwargs = {"offload_4bit": True} if offload_4bit else {}+ with m.fsdp2_gathered(model, **kwargs) as writer:+ torch.Tensor.cpu = original_cpu+ if writer:+ states = [layer.weight.quant_state for layer in model]+ sd = model.state_dict()+ out[offload_4bit] = dict(+ # The packed bytes are the only (n, 1) tensors gathered here.+ packed_moved = sum(len(shape) == 2 and shape[-1] == 1 for shape, _ in moved),+ own_state = [s is not l for s, l in zip(states, live)],+ state_cpu = [s.absmax.device.type == "cpu" for s in states],+ same_keys = sorted(sd) == sorted(reference),+ same_bytes = all(+ torch.equal(+ sd[k].reshape(-1).view(torch.uint8),+ reference[k].reshape(-1).view(torch.uint8),+ )+ if isinstance(sd[k], torch.Tensor)+ else sd[k] == reference[k]+ for k in reference+ ),+ )+ finally:+ torch.Tensor.cpu = original_cpu+ # The live QuantState is untouched (same object, same tensors).+ out["live_kept"] = all(+ layer.weight.quant_state is s and s.absmax is a+ for layer, s, a in zip(model, live, live_absmax)+ )+ return out+++def _case_dcp_quant_state(rank, world_size, m):+ """A full DCP load restores the QuantState it saved, not just the packed bytes, and refuses one+ that does not fit the model."""+ q = _load("fsdp2_qlora")+ from torch.distributed.checkpoint.state_dict import StateDictOptions+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard+ import torch.distributed.checkpoint.state_dict as dcp++ out = {}+ for nested in (False, True):+ model = _nf4_model(nested)+ q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh("cpu", (world_size,))+ for layer in model:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ m.install_dcp_bnb_state_dict()+ save_opts = StateDictOptions(full_state_dict = True) # every rank gets the full dict+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = save_opts).items()}+ states = [layer.weight.quant_state for layer in model]+ before = [s.absmax.clone() for s in states]+ results = {}+ for broadcast in (False, True):+ for s in states:+ s.absmax.mul_(2) # a requantized / differently initialized base+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = broadcast)+ payload = saved if (rank == 0 or not broadcast) else {}+ dcp.set_model_state_dict(model, payload, options = options)+ results[broadcast] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))+ out[f"restored_nested_{nested}"] = results+ bad = dict(saved)+ key = next(k for k in bad if k.endswith("weight.absmax"))+ bad[key] = bad[key][:-1].clone()+ try:+ dcp.set_model_state_dict(+ model, bad, options = StateDictOptions(full_state_dict = True)+ )+ out[f"mismatch_nested_{nested}"] = "loaded"+ except ValueError as error:+ out[f"mismatch_nested_{nested}"] = "ValueError" if "4-bit" in str(error) else str(error)+ return out+++def _sharded_nf4(world_size, m, device = "cpu"):+ q = _load("fsdp2_qlora")+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ model = _nf4_model(device = device)+ q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh(device.split(":")[0], (world_size,))+ for layer in model:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ m.install_dcp_bnb_state_dict()+ return model+++def _legacy_and_mismatched_loads(rank, model, saved, out):+ """Layer 0 saved without its QuantState entries (a checkpoint from before they were saved): with+ the live packed bytes it loads, keeps the live QuantState and dequantizes to the live weights;+ with other packed bytes it raises. Both with and without broadcast_from_rank0."""+ from bitsandbytes.functional import QuantState, dequantize_4bit+ from torch.distributed.checkpoint.state_dict import StateDictOptions+ import torch.distributed.checkpoint.state_dict as dcp++ def dequantized(state_dict):+ stats = {k[len("0.weight.") :]: v for k, v in state_dict.items() if k.startswith("0.weight.")}+ state = QuantState.from_dict(stats, device = "cpu")+ packed = state_dict["0.weight"].cpu().reshape(-1, 1)+ state.absmax, state.code = state.absmax.cpu(), state.code.cpu()+ return dequantize_4bit(packed, state).float()++ reference = dequantized({k: v.cpu() for k, v in saved.items()})+ live_absmax = model[0].weight.quant_state.absmax.clone()+ legacy = {k: v for k, v in saved.items() if not k.startswith("0.weight.")}+ mismatched = dict(legacy)+ mismatched["0.weight"] = saved["1.weight"].clone() # same shape, other bytes+ full = StateDictOptions(full_state_dict = True) # every rank gets the full dict+ for broadcast in (False, True):+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = broadcast)+ try:+ dcp.set_model_state_dict(model, legacy if (rank == 0 or not broadcast) else {}, options = options)+ after = dcp.get_model_state_dict(model, options = full)+ out[f"legacy_{broadcast}"] = (+ torch.equal(model[0].weight.quant_state.absmax, live_absmax)+ and torch.equal(dequantized(after), reference)+ )+ except Exception as error:+ out[f"legacy_{broadcast}"] = f"{type(error).__name__}: {error}"+ try:+ dcp.set_model_state_dict(+ model, mismatched if (rank == 0 or not broadcast) else {}, options = options+ )+ out[f"mismatch_{broadcast}"] = "loaded"+ except ValueError as error:+ out[f"mismatch_{broadcast}"] = "ValueError" if "`0.weight`" in str(error) else str(error)+++def _case_dcp_quant_state_missing(rank, world_size, m):+ """A full DCP state dict holding a packed 4-bit weight but none of its QuantState entries loads+ only when the bytes are the live ones (older checkpoints of a frozen base); other bytes would be+ paired with the live scales and must not load. One without the packed weight still loads.+ The broadcast_from_rank0 presence check must not send a CPU tensor through the process group+ (NCCL rejects those): with tensor broadcasts of CPU tensors refused, the restore still works."""+ from torch.distributed.checkpoint.state_dict import StateDictOptions+ import torch.distributed as dist+ import torch.distributed.checkpoint.state_dict as dcp++ model = _sharded_nf4(world_size, m)+ full = StateDictOptions(full_state_dict = True)+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = full).items()}+ out = {}+ _legacy_and_mismatched_loads(rank, model, saved, out)+ stripped = {k: v for k, v in saved.items() if not k.startswith("0.weight.")}+ # No packed weight and no QuantState for layer 0: nothing of it is loaded, so nothing to refuse.+ no_weight = {k: v for k, v in stripped.items() if k != "0.weight"}+ try:+ dcp.set_model_state_dict(+ model, no_weight, options = StateDictOptions(full_state_dict = True, strict = False)+ )+ out["no_weight"] = "loaded"+ except Exception as error:+ out["no_weight"] = f"{type(error).__name__}: {error}"++ # NCCL-like group: a tensor collective on a CPU tensor fails. Only the restore runs under it.+ states = [layer.weight.quant_state for layer in model]+ before = [s.absmax.clone() for s in states]+ for s in states:+ s.absmax.mul_(2)+ original = dist.broadcast++ def nccl_like_broadcast(tensor, *args, **kwargs):+ if tensor.device.type == "cpu":+ raise RuntimeError("No backend type associated with device type cpu")+ return original(tensor, *args, **kwargs)++ dist.broadcast = nccl_like_broadcast+ try:+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True)+ m._restore_quant_state(model, saved if rank == 0 else {}, options)+ out["nccl_like_flag"] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))+ except RuntimeError as error:+ out["nccl_like_flag"] = str(error)+ finally:+ dist.broadcast = original+ return out+++def _case_nccl_quant_state(rank, world_size, m):+ """Real NCCL group on two GPUs: a full DCP load with broadcast_from_rank0 restores the saved+ QuantState on every rank and refuses a packed weight saved without it."""+ from torch.distributed.checkpoint.state_dict import StateDictOptions+ import torch.distributed.checkpoint.state_dict as dcp++ model = _sharded_nf4(world_size, m, device = f"cuda:{rank}")+ full = StateDictOptions(full_state_dict = True) # every rank gets the full dict+ saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = full).items()}+ states = [layer.weight.quant_state for layer in model]+ before = [s.absmax.clone() for s in states]+ for s in states:+ s.absmax.mul_(2)+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True)+ out = {"backend": torch.distributed.get_backend()}+ try:+ dcp.set_model_state_dict(model, saved if rank == 0 else {}, options = options)+ out["restored"] = all(torch.equal(s.absmax, b) for s, b in zip(states, before))+ except Exception as error:+ out["restored"] = f"{type(error).__name__}: {error}"+ _legacy_and_mismatched_loads(rank, model, saved, out)+ return out++def _case_fp16_reduce(rank, world_size, m):"""An fp16 trainable parameter FSDP2 does not manage: its gradient mean must not overflow."""@@ -501,6 +764,9 @@ def _case_slow_writer(rank, world_size, m):def _worker(rank, world_size, port, case, queue):try:+ nccl = case.startswith("nccl")+ if nccl:+ torch.cuda.set_device(rank)os.environ.update(MASTER_ADDR = "127.0.0.1",MASTER_PORT = str(port),@@ -513,7 +779,7 @@ def _worker(rank, world_size, port, case, queue):timeout = datetime.timedelta(seconds = 4) if case == "slow_writer" else Nonedist.init_process_group(- "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})+ "nccl" if nccl else "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {}))m = _load("fsdp2_save")if case == "slow_writer":@@ -526,6 +792,14 @@ def _worker(rank, world_size, port, case, queue):result = _case_merge_offload(rank, world_size, m)elif case == "gather_4bit":result = _case_gather_4bit(rank, world_size, m)+ elif case == "gather_4bit_offload":+ result = _case_gather_4bit_offload(rank, world_size, m)+ elif case == "dcp_quant_state":+ result = _case_dcp_quant_state(rank, world_size, m)+ elif case == "dcp_quant_state_missing":+ result = _case_dcp_quant_state_missing(rank, world_size, m)+ elif case == "nccl_quant_state":+ result = _case_nccl_quant_state(rank, world_size, m)elif case.startswith("replicated"):result = _case_replicated(rank, world_size, m, hooked = case.endswith("hooked"))else:@@ -537,7 +811,7 @@ def _worker(rank, world_size, port, case, queue):queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))-def _run(case, world_size = 2):+def _run(case, world_size = 2, cpu_only = True):import torch.multiprocessing as mpctx = mp.get_context("spawn")@@ -549,7 +823,8 @@ def _run(case, world_size = 2):]# CPU only: a spawned child inherits this environment before it imports torch.saved = os.environ.get("CUDA_VISIBLE_DEVICES")- os.environ["CUDA_VISIBLE_DEVICES"] = ""+ if cpu_only:+ os.environ["CUDA_VISIBLE_DEVICES"] = ""try:for p in procs:p.start()@@ -665,3 +940,60 @@ def test_merge_offload_keeps_4bit_adapters_after_an_eval_forward():expected[f"layers.{i}.{part}.default.weight"] = i == 1 # layer 1 is not 4-bitexpected[f"layers.{i}.base_layer.weight"] = Trueassert decisions == expected, decisions+++def test_packed_4bit_save_stages_bytes_and_quant_state_on_the_cpu():+ """A full save of a sharded NF4 model kept every gathered packed weight on rank 0's accelerator,+ so the whole quantized base had to fit on one GPU. Saves that only write the bytes now get them+ on the CPU, with a CPU copy of the QuantState; the merge path is unchanged and the written state+ dict is byte for byte the same."""+ pytest.importorskip("bitsandbytes")+ r0 = _run("gather_4bit_offload")[0]+ kept, staged = r0[False], r0[True]+ assert kept["packed_moved"] == 0 and kept["own_state"] == [False, False], kept+ assert staged["packed_moved"] == 2, staged+ assert staged["own_state"] == [True, True] and all(staged["state_cpu"]), staged+ for r in (kept, staged):+ assert r["same_keys"] and r["same_bytes"], r+ assert r0["live_kept"], r0+++def test_full_dcp_load_restores_the_saved_quant_state():+ """A full DCP load dropped the saved QuantState entries, so the packed bytes were silently paired+ with whatever scales the live model had."""+ pytest.importorskip("bitsandbytes")+ results = _run("dcp_quant_state")+ for r in results.values():+ for nested in (False, True):+ assert r[f"restored_nested_{nested}"] == {False: True, True: True}, r+ assert r[f"mismatch_nested_{nested}"] == "ValueError", r+++def test_full_dcp_load_refuses_a_packed_4bit_weight_without_its_quant_state():+ """A packed 4-bit weight whose QuantState entries the checkpoint lacks was loaded silently and+ dequantized with the live model's scales, also when its bytes were not the live ones; an older+ checkpoint of the same frozen base (same bytes) must keep loading; the broadcast_from_rank0 presence check sent a CPU+ tensor through the process group, which an NCCL group cannot carry."""+ pytest.importorskip("bitsandbytes")+ for r in _run("dcp_quant_state_missing").values():+ for broadcast in (False, True):+ assert r[f"legacy_{broadcast}"] is True, r+ assert r[f"mismatch_{broadcast}"] == "ValueError", r+ assert r["no_weight"] == "loaded", r+ assert r["nccl_like_flag"] is True, r+++ not torch.cuda.is_available()+ or torch.cuda.device_count() < 2+ or not torch.distributed.is_nccl_available(),+ reason = "needs two CUDA GPUs and NCCL",+)+def test_full_dcp_broadcast_load_restores_quant_state_over_nccl():+ pytest.importorskip("bitsandbytes")+ for r in _run("nccl_quant_state", cpu_only = False).values():+ assert r["backend"] == "nccl", r+ assert r["restored"] is True, r+ for broadcast in (False, True):+ assert r[f"legacy_{broadcast}"] is True, r+ assert r[f"mismatch_{broadcast}"] == "ValueError", rdiff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 50790dd42b9..fe187aa11a0 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -16,8 +16,10 @@ __all__ = []import contextlib+import copyimport datetimeimport functools+import loggingimport osimport torch@@ -130,6 +132,18 @@ def _gather_full(param, DTensor):return gathered.view(param.dtype)+def _quant_state_on_cpu(quant_state):+ """A CPU copy of a bitsandbytes QuantState. `QuantState.to` moves in place (its nested state+ too), and the live one must stay where the sharded weight computes with it."""+ if not hasattr(quant_state, "to"):+ return quant_state+ state = copy.copy(quant_state)+ if getattr(state, "state2", None) is not None:+ state.state2 = copy.copy(state.state2)+ state.to("cpu")+ return state++def _rebuild(param, full, module):"""A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""quant_state = getattr(param, "quant_state", None)@@ -172,13 +186,16 @@ def fsdp2_gathered(trainable_only = False,offload_to_cpu = True,include = None,+ offload_4bit = False,):"""Yields None for a model FSDP2 does not shard (nothing happens). Otherwise every rank mustenter; rank 0 sees whole parameters (and yields True), the other ranks yield False and shouldnot write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),plus those `include(name, param)` selects (frozen ones the save still writes).`offload_to_cpu` may be a predicate `(name, param) -> bool`.- Packed 4-bit weights stay on their device: the merge dequantizes them there."""+ Packed 4-bit weights stay on their device (a merge dequantizes them there) unless `offload_4bit`:+ a save that only writes the packed bytes then gets them, with a CPU copy of their QuantState, on+ the CPU, so rank 0 never holds the whole quantized base on its accelerator."""if _ACTIVE:yield _ACTIVE[-1]return@@ -222,9 +239,16 @@ def fsdp2_gathered(continuemodule = owners[id(param)][0][0]offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpu- if offload and getattr(param, "quant_state", None) is None:+ quant_state = getattr(param, "quant_state", None)+ if offload and quant_state is None:full = full.cpu()- replaced[id(param)] = (param, _rebuild(param, full.detach(), module))+ elif offload and offload_4bit:+ full = full.cpu()+ quant_state = _quant_state_on_cpu(quant_state)+ rebuilt = _rebuild(param, full.detach(), module)+ if quant_state is not None and rebuilt.quant_state is not quant_state:+ rebuilt.quant_state = quant_state+ replaced[id(param)] = (param, rebuilt)if writer:for key, (old, new) in replaced.items():for module, name in owners[key]:@@ -459,9 +483,136 @@ def _add_quant_state_entries(model, state_dict, options):state_dict[prefix + "weight." + key] = value.cpu() if cpu else value+_LEGACY_4BIT_NOTICE = [False]+logger = logging.getLogger(__name__)+++def _as_bytes(tensor):+ return tensor.detach().contiguous().reshape(-1).view(torch.uint8)+++def _same_packed_bytes(live, loaded, broadcast_dist):+ """Whether the packed 4-bit bytes being loaded equal the live (possibly sharded) weight's. Every+ rank gathers the live weight (a collective); with `broadcast_dist` only rank 0 holds `loaded`,+ decides, and sends the verdict, so all ranks agree."""+ DTensor = _dtensor_class()+ if DTensor is not None and isinstance(live, DTensor):+ live = _gather_full(live, DTensor)+ if DTensor is not None and isinstance(loaded, DTensor):+ loaded = loaded.full_tensor()+ same = False+ if isinstance(loaded, torch.Tensor):+ a, b = _as_bytes(live), _as_bytes(loaded)+ same = a.numel() == b.numel() and torch.equal(a.cpu(), b.cpu())+ if broadcast_dist is not None:+ verdict = [same]+ broadcast_dist.broadcast_object_list(verdict, src = 0)+ same = bool(verdict[0])+ return same+++def _restore_quant_state(model, state_dict, options):+ """Loading a full DCP state dict: the saved QuantState entries (`_add_quant_state_entries`) are+ copied into each live QuantState in place, so the packed bytes DCP loads are dequantized with the+ scales they were saved with. A saved QuantState that does not fit the live one (other shape,+ blocksize, quant type or nesting) raises instead of pairing the bytes with the wrong scales. With+ `broadcast_from_rank0` only rank 0 holds the entries and sends them to the others."""+ entries = _linear4bit_entries(model)+ if not entries:+ return+ dist = _dist()+ broadcast = dist is not None and bool(getattr(options, "broadcast_from_rank0", False))+ if not isinstance(state_dict, dict):+ state_dict = {}+ if broadcast:+ # An object broadcast, not a tensor one: a CPU tensor cannot go through an NCCL group, and+ # broadcast_object_list stages on whatever device the group's backend needs.+ has = [any(+ k == prefix + "weight" or k.startswith(prefix + "weight.")+ for prefix, _ in entries+ for k in state_dict+ )]+ dist.broadcast_object_list(has, src = 0)+ if not has[0]:+ return+ from bitsandbytes.functional import QuantState++ for prefix, module in entries:+ live = module.weight.quant_state+ head = prefix + "weight."+ saved = {+ k[len(head) :]: v+ for k, v in state_dict.items()+ if k.startswith(head) and isinstance(v, torch.Tensor)+ }+ loads_weight = head[:-1] in state_dict+ if broadcast:+ payload = [(loads_weight, {k: v.detach().cpu() for k, v in saved.items()})]+ dist.broadcast_object_list(payload, src = 0)+ loads_weight, saved = payload[0]+ if not saved:+ if loads_weight:+ # A checkpoint written before QuantState was saved (a frozen QLoRA base): its packed+ # bytes are the live ones, so the live scales are theirs. Other bytes would be+ # dequantized with the wrong scales.+ same = _same_packed_bytes(+ module.weight, state_dict.get(head[:-1]), dist if broadcast else None+ )+ if not same:+ raise ValueError(+ f"Unsloth: the checkpoint holds the 4-bit weight `{head[:-1]}` but none of "+ "its quantization state (`weight.absmax`, `weight.quant_state.*`), and its "+ "packed bytes differ from the model's, so they cannot be dequantized "+ "correctly. Save the full state dict again with Unsloth, or load it without "+ "the frozen 4-bit base weights."+ )+ if not _LEGACY_4BIT_NOTICE[0]:+ _LEGACY_4BIT_NOTICE[0] = True+ logger.info(+ "Unsloth: the checkpoint has 4-bit weights without their quantization state "+ "(saved by an older Unsloth); their packed bytes match the model's, so its "+ "quantization state is kept."+ )+ continue+ expected = set(live.as_dict(packed = True))+ if set(saved) != expected:+ raise ValueError(+ f"Unsloth: the checkpoint's 4-bit quantization state for `{head[:-1]}` has entries "+ f"{sorted(saved)}, the model's has {sorted(expected)}."+ )+ loaded = QuantState.from_dict(saved, device = live.absmax.device)+ pairs = [("absmax", loaded.absmax, live.absmax), ("code", loaded.code, live.code)]+ if live.nested:+ pairs += [+ ("offset", loaded.offset, live.offset),+ ("state2.absmax", loaded.state2.absmax, live.state2.absmax),+ ("state2.code", loaded.state2.code, live.state2.code),+ ]+ same = (+ loaded.quant_type == live.quant_type+ and loaded.blocksize == live.blocksize+ and loaded.nested == live.nested+ and tuple(loaded.shape) == tuple(live.shape)+ and loaded.dtype == live.dtype+ and (not live.nested or loaded.state2.blocksize == live.state2.blocksize)+ and all(+ isinstance(a, torch.Tensor) and a.shape == b.shape and a.dtype == b.dtype+ for _, a, b in pairs+ )+ )+ if not same:+ raise ValueError(+ f"Unsloth: the checkpoint's 4-bit quantization state for `{head[:-1]}` does not "+ "match the model's (shape, blocksize, quant type or nesting differ)."+ )+ with torch.no_grad():+ for _, a, b in pairs:+ b.copy_(a.to(b.device))++def _without_quant_state_entries(name, args, kwargs):- """Loading: drop the entries `_add_quant_state_entries` wrote (DCP would reject them as unexpected- keys); the QuantState is rebuilt identically when the 4-bit model is loaded again."""+ """Loading: the entries `_add_quant_state_entries` wrote are restored into the live QuantState+ (`_restore_quant_state`) and then dropped, as DCP would reject them as unexpected keys."""model = args[0] if args else kwargs.get("model")if not isinstance(model, torch.nn.Module):return args, kwargs@@ -472,6 +623,12 @@ def _without_quant_state_entries(name, args, kwargs):)if not drop:return args, kwargs+ if name == "set_model_state_dict" and len(args) > 1:+ loading = args[1]+ else:+ loading = kwargs.get("model_state_dict")+ if name in ("set_model_state_dict", "set_state_dict"):+ _restore_quant_state(model, loading, kwargs.get("options", None))def strip(state_dict):if not isinstance(state_dict, dict) or not drop.intersection(state_dict):diff --git a/unsloth/save.py b/unsloth/save.pyindex 727a3f6144e..5f9d92761cd 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6416,10 +6416,12 @@ def _fsdp2_peft_saved_frozen(model, arguments):return include-def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):+def _fsdp2_collective_save(trainable_only, offload_to_cpu = None, offload_4bit = False):"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the othersreturn None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;`offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.+ `offload_4bit`: the save only writes packed 4-bit weights (never merges them), so they go to+ the CPU too.A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""def decorate(fn):@@ -6442,7 +6444,11 @@ def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):)offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)with fsdp2_gathered(- self, trainable_only = only, include = include, offload_to_cpu = offload+ self,+ trainable_only = only,+ include = include,+ offload_to_cpu = offload,+ offload_4bit = offload_4bit,) as writer:if writer is False:return None@@ -8411,10 +8417,13 @@ def patch_saving_functions(model, vision = False):# Under FSDP2 every rank calls push_to_hub: the shards are gathered and rank 0 uploads. Inside,# the wrapped save_pretrained sees the gather already active and writes directly.globals()["unsloth_push_to_hub"] = _fsdp2_collective_save(- lambda self, args, kwargs: isinstance(self, PeftModel)+ lambda self, args, kwargs: isinstance(self, PeftModel),+ offload_4bit = True,)(globals()["unsloth_push_to_hub"])- @_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))+ @_fsdp2_collective_save(+ lambda self, args, kwargs: isinstance(self, PeftModel), offload_4bit = True+ )def unsloth_model_save_pretrained(self, *args, **kwargs):"""`safe_serialization = None` means the safetensors default, not a pickle.