Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: gather for push_to_hub, refuse GGUF / torchao exports, size offloaded state and 4-bit counts per shard (#14205)

Daniel Hanfbe440f114d0
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.py
new file mode 100644
index 00000000000..25a4056c815
--- /dev/null
+++ b/tests/test_fsdp2_save_coverage.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Which exports run under FSDP2 (unsloth/save.py). save_pretrained and save_pretrained_merged
+already gather the shards to rank 0; push_to_hub_merged and push_to_hub did not, so every rank
+merged and uploaded its own shards. They now gather like the saves. GGUF and torchao exports
+rebuild or quantize the whole model in-process and are refused under FSDP2 with a way to get the
+same file. No GPU work: FSDP2 is faked by fsdp2_units, the gather by a recording context manager."""
+
+import contextlib
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("peft")
+
+from unsloth import save as unsloth_save # noqa: E402
+from unsloth.models import fsdp2_save # noqa: E402
+
+
+def fake_fsdp2(monkeypatch):
+ calls = []
+
+ @contextlib.contextmanager
+ def gathered(
+ model,
+ trainable_only = False,
+ **kwargs,
+ ):
+ calls.append(trainable_only)
+ yield False # a non-writer rank: the export must not run here
+
+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)
+ return calls
+
+
+def _tiny_llama():
+ from transformers import LlamaConfig, LlamaForCausalLM
+ cfg = LlamaConfig(
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ vocab_size = 32,
+ )
+ return LlamaForCausalLM(cfg)
+
+
+def test_push_to_hub_merged_gathers_like_save_pretrained_merged(fake_fsdp2):
+ model = _tiny_llama()
+ # Not rank 0: gathered, nothing merged or uploaded, None returned.
+ assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None
+ assert (
+ unsloth_save.unsloth_generic_push_to_hub_merged(
+ model, "user/repo", None, save_method = "lora"
+ )
+ is None
+ )
+ # A plain (non-PEFT) model gathers everything, as save_pretrained_merged does.
+ assert fake_fsdp2 == [False, False]
+
+
+def test_push_to_hub_gathers(fake_fsdp2):
+ model = unsloth_save.patch_saving_functions(_tiny_llama())
+ assert model.push_to_hub("user/repo") is None
+ assert fake_fsdp2 == [False]
+
+
+ "method, call",
+ [
+ ("save_pretrained_gguf", lambda m: unsloth_save.unsloth_save_pretrained_gguf(m, "out")),
+ ("push_to_hub_gguf", lambda m: unsloth_save.unsloth_push_to_hub_gguf(m, "user/repo")),
+ (
+ "save_pretrained_torchao",
+ lambda m: unsloth_save.unsloth_save_pretrained_torchao(m, "out"),
+ ),
+ ],
+)
+def test_gguf_and_torchao_are_refused_under_fsdp2(fake_fsdp2, method, call):
+ with pytest.raises(
+ RuntimeError, match = rf"`{method}` does not support a model sharded by FSDP2"
+ ):
+ call(_tiny_llama())
+ assert fake_fsdp2 == []
+
+
+def test_refusal_passes_through_without_fsdp2(monkeypatch):
+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])
+ export = unsloth_save._fsdp2_refuses_save("save_pretrained_gguf")(lambda self, x: ("ran", x))
+ assert export(object(), 1) == ("ran", 1)
diff --git a/tests/test_optimizer_state_offload_dtensor_chunks.py b/tests/test_optimizer_state_offload_dtensor_chunks.py
new file mode 100644
index 00000000000..8e5e2734d12
--- /dev/null
+++ b/tests/test_optimizer_state_offload_dtensor_chunks.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_OFFLOAD_OPTIMIZER_STATE chunked step under FSDP2: a DTensor's numel() is the global
+one, but each rank holds (and moves to the device) only its local shard of the optimizer state.
+Sizing chunks by the global numel overcounted by world_size, so the step ran in world_size times
+more chunks than chunk_bytes allows. A fake 2-rank process group on CPU, no GPU."""
+
+import importlib.util
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+import torch
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_PATH = os.path.join(_ROOT, "unsloth", "optimizers", "state_offload.py")
+
+_SCRIPT = textwrap.dedent(
+ """
+ import importlib.util, sys, torch
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.tensor import DTensor, Shard
+ from torch.testing._internal.distributed.fake_pg import FakeStore
+
+ dist.init_process_group("fake", store = FakeStore(), rank = 0, world_size = 2)
+ spec = importlib.util.spec_from_file_location("_unsloth_state_offload", sys.argv[1])
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ mesh = init_device_mesh("cpu", (2,))
+ n = 1000 # global numel per parameter; this rank holds n // 2
+
+ def shard(local):
+ return DTensor.from_local(local, mesh, [Shard(0)], run_check = False)
+
+ params = [torch.nn.Parameter(shard(torch.randn(n // 2))) for _ in range(2)]
+ assert params[0].numel() == n and params[0]._local_tensor.numel() == n // 2
+ optimizer = torch.optim.AdamW(params)
+ # Never stepped: 2 fp32 moments of the local shard = 4n bytes per parameter on this rank.
+ unstepped = module._StateOffloader(optimizer, chunk_bytes = 9 * n)._chunks(params)
+ for p in params:
+ optimizer.state[p] = {"exp_avg": shard(torch.zeros(n // 2)), "exp_avg_sq": shard(torch.zeros(n // 2))}
+ stepped = module._StateOffloader(optimizer, chunk_bytes = 9 * n)._chunks(params)
+ print("CHUNKS", len(unstepped), len(stepped))
+ dist.destroy_process_group()
+ """
+)
+
+
+def test_dtensor_state_is_sized_by_its_local_shard(tmp_path):
+ pytest.importorskip("torch.distributed.tensor")
+ try:
+ from torch.testing._internal.distributed.fake_pg import FakeStore # noqa: F401
+ except Exception:
+ pytest.skip("torch without the fake process group")
+ script = tmp_path / "chunks.py"
+ script.write_text(_SCRIPT)
+ out = subprocess.run(
+ [sys.executable, str(script), _PATH], capture_output = True, text = True, timeout = 300
+ )
+ assert out.returncode == 0, out.stderr[-3000:]
+ line = [l for l in out.stdout.splitlines() if l.startswith("CHUNKS")][-1]
+ # 2 parameters x 4n local bytes = 8n fit one 9n chunk; at the global numel each would be 8n.
+ assert line.split()[1:] == ["1", "1"], line
diff --git a/tests/test_quant_param_count_float_storage.py b/tests/test_quant_param_count_float_storage.py
new file mode 100644
index 00000000000..270b2a3f800
--- /dev/null
+++ b/tests/test_quant_param_count_float_storage.py
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""extract_quant_model_param_count (the printed parameter count of a quantized model) counts a
+bitsandbytes 4-bit weight as two values per packed byte. FSDP2 QLoRA stores those bytes in a float
+dtype (bnb_4bit_quant_storage = bfloat16 / float16 / float32), where one element is 2 or 4 bytes,
+so counting two values per element printed about half (or a quarter) of the real count.
+
+The function is read from unsloth/models/_utils.py without importing unsloth (which needs a GPU)."""
+
+import ast
+from pathlib import Path
+
+import pytest
+import torch
+
+bnb = pytest.importorskip("bitsandbytes")
+
+_UTILS = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"
+
+
+def _load():
+ tree = ast.parse(_UTILS.read_text(encoding = "utf-8"))
+ fn = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "extract_quant_model_param_count"
+ )
+ namespace = {}
+ exec(compile(ast.Module(body = [fn], type_ignores = []), str(_UTILS), "exec"), namespace)
+ return namespace["extract_quant_model_param_count"]
+
+
+def _model(storage):
+ lin = bnb.nn.Linear4bit(64, 32, bias = True, quant_type = "nf4")
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(32, 64), requires_grad = False, quant_type = "nf4", blocksize = 64, module = lin
+ )
+ lin = lin.to("cpu") # quantizes
+ w = lin.weight
+ if storage is not torch.uint8:
+ # What unsloth's fsdp2_qlora.params4bit_to_float_storage does: same bytes, float elements.
+ w.data = w.data.reshape(-1).view(storage).reshape(-1, 1)
+ w.quant_storage = storage
+ return torch.nn.Sequential(lin)
+
+
[email protected]("storage", [torch.uint8, torch.bfloat16, torch.float16, torch.float32])
+def test_float_packed_4bit_weights_count_every_value(storage):
+ count = _load()(_model(storage))
+ assert count == 32 * 64 + 32
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
+def test_unquantized_params4bit_counts_its_dense_values(dtype):
+ # Before bitsandbytes' first quantizing move a Params4bit holds dense values (bnb_quantized
+ # False): one element is one weight, whatever its dtype.
+ lin = bnb.nn.Linear4bit(64, 32, bias = False, quant_type = "nf4")
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(32, 64, dtype = dtype), requires_grad = False, quant_type = "nf4", module = lin
+ )
+ assert lin.weight.bnb_quantized is False
+ assert _load()(torch.nn.Sequential(lin)) == 32 * 64
+
+
[email protected]("storage", [torch.bfloat16, torch.float32])
+def test_fsdp2_sharded_4bit_weight_counts_every_value(storage):
+ # FSDP2 replaces the Params4bit with a DTensor-like parameter that keeps quant_state and the
+ # other quant attributes (fsdp2_qlora), not the Params4bit class.
+ packed = 32 * 64 // 2 // torch.empty((), dtype = storage).element_size()
+ p = torch.nn.Parameter(torch.zeros(packed, 1, dtype = storage), requires_grad = False)
+ p.quant_state = _model(torch.uint8)[0].weight.quant_state
+ p.bnb_quantized = True
+ module = torch.nn.Module()
+ module.weight = p
+ assert _load()(module) == 32 * 64
+
+
+def test_non_bitsandbytes_quant_state_is_not_counted_as_4bit():
+ # compressed-tensors packed int32 weights (2, 4 or 8 bit) carry their own quant_state: the
+ # bitsandbytes two-per-byte count does not apply to them.
+ p = torch.nn.Parameter(torch.zeros(32, 8, dtype = torch.int32), requires_grad = False)
+ p.quant_state = object()
+ module = torch.nn.Module()
+ module.weight = p
+ assert _load()(module) == 32 * 8
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 2d20604c749..9bc93477aae 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -3658,8 +3658,18 @@ def extract_quant_model_param_count(model):
"""Quantized model param count, from the difference in param class."""
count: int = 0
for name, p in model.named_parameters():
- if p.__class__.__name__ == "Params4bit":
- count += 2 * p.numel()
+ # Under FSDP2 the parameter is a DTensor that carries the Params4bit quant attributes
+ # (fsdp2_qlora), so its class name alone does not identify it. Only a bitsandbytes quant
+ # state counts: compressed-tensors packed weights also carry one, at 2, 4 or 8 bits.
+ quant_state = getattr(p, "quant_state", None)
+ is_4bit = p.__class__.__name__ == "Params4bit" or (
+ quant_state is not None and type(quant_state).__module__.split(".")[0] == "bitsandbytes"
+ )
+ if is_4bit and getattr(p, "bnb_quantized", True) is not False:
+ # Two 4-bit weights per packed byte. FSDP2 QLoRA stores the bytes in a float dtype
+ # (bnb_4bit_quant_storage), where one element holds element_size() bytes. Before its
+ # first quantizing move a Params4bit still holds dense values: counted as they are.
+ count += 2 * p.numel() * p.element_size()
else:
count += p.numel()
return count
diff --git a/unsloth/optimizers/state_offload.py b/unsloth/optimizers/state_offload.py
index 9611a91f952..0711dd0f5b0 100644
--- a/unsloth/optimizers/state_offload.py
+++ b/unsloth/optimizers/state_offload.py
@@ -96,6 +96,18 @@ def _held_tensors(value):
return [t for _, t in _tensor_attrs(value)]
+def _local(tensor):
+ """This rank's shard of a DTensor (FSDP2), whose numel() is the global one; else the tensor."""
+ if DTensor is not None and isinstance(tensor, DTensor):
+ return tensor._local_tensor
+ return tensor
+
+
+def _local_nbytes(tensor):
+ local = _local(tensor)
+ return local.numel() * local.element_size()
+
+
def _source_device(local, param, key, device_of):
if local.device.type != "cpu":
return local.device
@@ -251,7 +263,7 @@ class _StateOffloader:
for p in params:
held = [t for v in state.get(p, {}).values() for t in _held_tensors(v)]
if held:
- nbytes = sum(v.numel() * v.element_size() for v in held)
+ nbytes = sum(_local_nbytes(v) for v in held)
else:
# Not stepped yet: two moments, a third with amsgrad or centered RMSprop with
# momentum (square_avg, momentum_buffer, grad_avg), at least fp32 each.
@@ -263,7 +275,7 @@ class _StateOffloader:
or "ademamix" in type(self.optimizer).__name__.lower()
)
moments = 3 if third else 2
- nbytes = moments * max(p.element_size(), 4) * p.numel()
+ nbytes = moments * max(p.element_size(), 4) * _local(p).numel()
if chunk and size + nbytes > self.chunk_bytes:
chunks.append(chunk)
chunk, size = [], 0
diff --git a/unsloth/save.py b/unsloth/save.py
index f53d243a980..6d5b62e8f94 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -4159,6 +4159,30 @@ def _assert_export_target_is_not_base_with_lora_layers(self):
)
+def _fsdp2_refuses_save(method):
+ """Exports that rebuild or quantize the whole model in this process (GGUF, torchao) cannot run on a
+ model FSDP2 shards: each rank would export its own shards, all into the same place. Raise on every
+ rank (so none waits on another) and say how to get the same file. Passed through without FSDP2."""
+
+ def decorate(fn):
+ @functools.wraps(fn)
+ def wrapper(self, *args, **kwargs):
+ from .models.fsdp2_save import fsdp2_units, fsdp2_gather_active
+ if fsdp2_units(self) and not fsdp2_gather_active():
+ raise RuntimeError(
+ f"Unsloth: `{method}` does not support a model sharded by FSDP2. Save the merged "
+ "model on every rank with `model.save_pretrained_merged(save_directory, tokenizer, "
+ 'save_method = "merged_16bit")`, then in a single process load that folder with '
+ f"`FastLanguageModel.from_pretrained` and call `{method}` there."
+ )
+ return fn(self, *args, **kwargs)
+
+ return wrapper
+
+ return decorate
+
+
+@_fsdp2_refuses_save("save_pretrained_gguf")
@_normalize_tied_weights_keys_for_save
def unsloth_save_pretrained_gguf(
self,
@@ -4810,6 +4834,7 @@ def _free_merge_if_disk_is_tight(
return freed
+@_fsdp2_refuses_save("push_to_hub_gguf")
def unsloth_push_to_hub_gguf(
self,
repo_id: str,
@@ -6397,6 +6422,11 @@ def unsloth_generic_save_pretrained_merged(
gc.collect()
+# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.
+@_fsdp2_collective_save(
+ lambda self, args, kwargs: isinstance(self, PeftModel)
+ and _is_adapter_save_method(_merged_save_method(args, kwargs))
+)
def unsloth_generic_push_to_hub_merged(
self,
repo_id: str,
@@ -7511,6 +7541,7 @@ def _unsloth_save_torchao(
clean_gpu_cache()
+@_fsdp2_refuses_save("save_pretrained_torchao")
def unsloth_save_pretrained_torchao(
self,
save_directory: Union[str, os.PathLike],
@@ -8049,6 +8080,11 @@ def patch_saving_functions(model, vision = False):
pass
'''
exec(push_to_hub_text, globals())
+ # Under FSDP2 every rank calls push_to_hub: the shards are gathered and rank 0 uploads. Inside,
+ # the wrapped save_pretrained sees the gather already active and writes directly.
+ globals()["unsloth_push_to_hub"] = _fsdp2_collective_save(
+ lambda self, args, kwargs: isinstance(self, PeftModel)
+ )(globals()["unsloth_push_to_hub"])
@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))
def unsloth_model_save_pretrained(self, *args, **kwargs):