Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: gather for push_to_hub, refuse GGUF / torchao exports, size offloaded state and 4-bit counts per shard (#14205)
Daniel Hanfbe440f114d0
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.pynew file mode 100644index 00000000000..25a4056c815--- /dev/null+++ b/tests/test_fsdp2_save_coverage.py@@ -0,0 +1,95 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Which exports run under FSDP2 (unsloth/save.py). save_pretrained and save_pretrained_merged+already gather the shards to rank 0; push_to_hub_merged and push_to_hub did not, so every rank+merged and uploaded its own shards. They now gather like the saves. GGUF and torchao exports+rebuild or quantize the whole model in-process and are refused under FSDP2 with a way to get the+same file. No GPU work: FSDP2 is faked by fsdp2_units, the gather by a recording context manager."""++import contextlib++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("peft")++from unsloth import save as unsloth_save # noqa: E402+from unsloth.models import fsdp2_save # noqa: E402+++def fake_fsdp2(monkeypatch):+ calls = []++ @contextlib.contextmanager+ def gathered(+ model,+ trainable_only = False,+ **kwargs,+ ):+ calls.append(trainable_only)+ yield False # a non-writer rank: the export must not run here++ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)+ return calls+++def _tiny_llama():+ from transformers import LlamaConfig, LlamaForCausalLM+ cfg = LlamaConfig(+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ vocab_size = 32,+ )+ return LlamaForCausalLM(cfg)+++def test_push_to_hub_merged_gathers_like_save_pretrained_merged(fake_fsdp2):+ model = _tiny_llama()+ # Not rank 0: gathered, nothing merged or uploaded, None returned.+ assert unsloth_save.unsloth_generic_push_to_hub_merged(model, "user/repo", None) is None+ assert (+ unsloth_save.unsloth_generic_push_to_hub_merged(+ model, "user/repo", None, save_method = "lora"+ )+ is None+ )+ # A plain (non-PEFT) model gathers everything, as save_pretrained_merged does.+ assert fake_fsdp2 == [False, False]+++def test_push_to_hub_gathers(fake_fsdp2):+ model = unsloth_save.patch_saving_functions(_tiny_llama())+ assert model.push_to_hub("user/repo") is None+ assert fake_fsdp2 == [False]+++ "method, call",+ [+ ("save_pretrained_gguf", lambda m: unsloth_save.unsloth_save_pretrained_gguf(m, "out")),+ ("push_to_hub_gguf", lambda m: unsloth_save.unsloth_push_to_hub_gguf(m, "user/repo")),+ (+ "save_pretrained_torchao",+ lambda m: unsloth_save.unsloth_save_pretrained_torchao(m, "out"),+ ),+ ],+)+def test_gguf_and_torchao_are_refused_under_fsdp2(fake_fsdp2, method, call):+ with pytest.raises(+ RuntimeError, match = rf"`{method}` does not support a model sharded by FSDP2"+ ):+ call(_tiny_llama())+ assert fake_fsdp2 == []+++def test_refusal_passes_through_without_fsdp2(monkeypatch):+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])+ export = unsloth_save._fsdp2_refuses_save("save_pretrained_gguf")(lambda self, x: ("ran", x))+ assert export(object(), 1) == ("ran", 1)diff --git a/tests/test_optimizer_state_offload_dtensor_chunks.py b/tests/test_optimizer_state_offload_dtensor_chunks.pynew file mode 100644index 00000000000..8e5e2734d12--- /dev/null+++ b/tests/test_optimizer_state_offload_dtensor_chunks.py@@ -0,0 +1,67 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_OFFLOAD_OPTIMIZER_STATE chunked step under FSDP2: a DTensor's numel() is the global+one, but each rank holds (and moves to the device) only its local shard of the optimizer state.+Sizing chunks by the global numel overcounted by world_size, so the step ran in world_size times+more chunks than chunk_bytes allows. A fake 2-rank process group on CPU, no GPU."""++import importlib.util+import os+import subprocess+import sys+import textwrap++import pytest+import torch++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_PATH = os.path.join(_ROOT, "unsloth", "optimizers", "state_offload.py")++_SCRIPT = textwrap.dedent(+ """+ import importlib.util, sys, torch+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import DTensor, Shard+ from torch.testing._internal.distributed.fake_pg import FakeStore++ dist.init_process_group("fake", store = FakeStore(), rank = 0, world_size = 2)+ spec = importlib.util.spec_from_file_location("_unsloth_state_offload", sys.argv[1])+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ mesh = init_device_mesh("cpu", (2,))+ n = 1000 # global numel per parameter; this rank holds n // 2++ def shard(local):+ return DTensor.from_local(local, mesh, [Shard(0)], run_check = False)++ params = [torch.nn.Parameter(shard(torch.randn(n // 2))) for _ in range(2)]+ assert params[0].numel() == n and params[0]._local_tensor.numel() == n // 2+ optimizer = torch.optim.AdamW(params)+ # Never stepped: 2 fp32 moments of the local shard = 4n bytes per parameter on this rank.+ unstepped = module._StateOffloader(optimizer, chunk_bytes = 9 * n)._chunks(params)+ for p in params:+ optimizer.state[p] = {"exp_avg": shard(torch.zeros(n // 2)), "exp_avg_sq": shard(torch.zeros(n // 2))}+ stepped = module._StateOffloader(optimizer, chunk_bytes = 9 * n)._chunks(params)+ print("CHUNKS", len(unstepped), len(stepped))+ dist.destroy_process_group()+ """+)+++def test_dtensor_state_is_sized_by_its_local_shard(tmp_path):+ pytest.importorskip("torch.distributed.tensor")+ try:+ from torch.testing._internal.distributed.fake_pg import FakeStore # noqa: F401+ except Exception:+ pytest.skip("torch without the fake process group")+ script = tmp_path / "chunks.py"+ script.write_text(_SCRIPT)+ out = subprocess.run(+ [sys.executable, str(script), _PATH], capture_output = True, text = True, timeout = 300+ )+ assert out.returncode == 0, out.stderr[-3000:]+ line = [l for l in out.stdout.splitlines() if l.startswith("CHUNKS")][-1]+ # 2 parameters x 4n local bytes = 8n fit one 9n chunk; at the global numel each would be 8n.+ assert line.split()[1:] == ["1", "1"], linediff --git a/tests/test_quant_param_count_float_storage.py b/tests/test_quant_param_count_float_storage.pynew file mode 100644index 00000000000..270b2a3f800--- /dev/null+++ b/tests/test_quant_param_count_float_storage.py@@ -0,0 +1,86 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""extract_quant_model_param_count (the printed parameter count of a quantized model) counts a+bitsandbytes 4-bit weight as two values per packed byte. FSDP2 QLoRA stores those bytes in a float+dtype (bnb_4bit_quant_storage = bfloat16 / float16 / float32), where one element is 2 or 4 bytes,+so counting two values per element printed about half (or a quarter) of the real count.++The function is read from unsloth/models/_utils.py without importing unsloth (which needs a GPU)."""++import ast+from pathlib import Path++import pytest+import torch++bnb = pytest.importorskip("bitsandbytes")++_UTILS = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"+++def _load():+ tree = ast.parse(_UTILS.read_text(encoding = "utf-8"))+ fn = next(+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "extract_quant_model_param_count"+ )+ namespace = {}+ exec(compile(ast.Module(body = [fn], type_ignores = []), str(_UTILS), "exec"), namespace)+ return namespace["extract_quant_model_param_count"]+++def _model(storage):+ lin = bnb.nn.Linear4bit(64, 32, bias = True, quant_type = "nf4")+ lin.weight = bnb.nn.Params4bit(+ torch.randn(32, 64), requires_grad = False, quant_type = "nf4", blocksize = 64, module = lin+ )+ lin = lin.to("cpu") # quantizes+ w = lin.weight+ if storage is not torch.uint8:+ # What unsloth's fsdp2_qlora.params4bit_to_float_storage does: same bytes, float elements.+ w.data = w.data.reshape(-1).view(storage).reshape(-1, 1)+ w.quant_storage = storage+ return torch.nn.Sequential(lin)++[email protected]("storage", [torch.uint8, torch.bfloat16, torch.float16, torch.float32])+def test_float_packed_4bit_weights_count_every_value(storage):+ count = _load()(_model(storage))+ assert count == 32 * 64 + 32++[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])+def test_unquantized_params4bit_counts_its_dense_values(dtype):+ # Before bitsandbytes' first quantizing move a Params4bit holds dense values (bnb_quantized+ # False): one element is one weight, whatever its dtype.+ lin = bnb.nn.Linear4bit(64, 32, bias = False, quant_type = "nf4")+ lin.weight = bnb.nn.Params4bit(+ torch.randn(32, 64, dtype = dtype), requires_grad = False, quant_type = "nf4", module = lin+ )+ assert lin.weight.bnb_quantized is False+ assert _load()(torch.nn.Sequential(lin)) == 32 * 64++[email protected]("storage", [torch.bfloat16, torch.float32])+def test_fsdp2_sharded_4bit_weight_counts_every_value(storage):+ # FSDP2 replaces the Params4bit with a DTensor-like parameter that keeps quant_state and the+ # other quant attributes (fsdp2_qlora), not the Params4bit class.+ packed = 32 * 64 // 2 // torch.empty((), dtype = storage).element_size()+ p = torch.nn.Parameter(torch.zeros(packed, 1, dtype = storage), requires_grad = False)+ p.quant_state = _model(torch.uint8)[0].weight.quant_state+ p.bnb_quantized = True+ module = torch.nn.Module()+ module.weight = p+ assert _load()(module) == 32 * 64+++def test_non_bitsandbytes_quant_state_is_not_counted_as_4bit():+ # compressed-tensors packed int32 weights (2, 4 or 8 bit) carry their own quant_state: the+ # bitsandbytes two-per-byte count does not apply to them.+ p = torch.nn.Parameter(torch.zeros(32, 8, dtype = torch.int32), requires_grad = False)+ p.quant_state = object()+ module = torch.nn.Module()+ module.weight = p+ assert _load()(module) == 32 * 8diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 2d20604c749..9bc93477aae 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -3658,8 +3658,18 @@ def extract_quant_model_param_count(model):"""Quantized model param count, from the difference in param class."""count: int = 0for name, p in model.named_parameters():- if p.__class__.__name__ == "Params4bit":- count += 2 * p.numel()+ # Under FSDP2 the parameter is a DTensor that carries the Params4bit quant attributes+ # (fsdp2_qlora), so its class name alone does not identify it. Only a bitsandbytes quant+ # state counts: compressed-tensors packed weights also carry one, at 2, 4 or 8 bits.+ quant_state = getattr(p, "quant_state", None)+ is_4bit = p.__class__.__name__ == "Params4bit" or (+ quant_state is not None and type(quant_state).__module__.split(".")[0] == "bitsandbytes"+ )+ if is_4bit and getattr(p, "bnb_quantized", True) is not False:+ # Two 4-bit weights per packed byte. FSDP2 QLoRA stores the bytes in a float dtype+ # (bnb_4bit_quant_storage), where one element holds element_size() bytes. Before its+ # first quantizing move a Params4bit still holds dense values: counted as they are.+ count += 2 * p.numel() * p.element_size()else:count += p.numel()return countdiff --git a/unsloth/optimizers/state_offload.py b/unsloth/optimizers/state_offload.pyindex 9611a91f952..0711dd0f5b0 100644--- a/unsloth/optimizers/state_offload.py+++ b/unsloth/optimizers/state_offload.py@@ -96,6 +96,18 @@ def _held_tensors(value):return [t for _, t in _tensor_attrs(value)]+def _local(tensor):+ """This rank's shard of a DTensor (FSDP2), whose numel() is the global one; else the tensor."""+ if DTensor is not None and isinstance(tensor, DTensor):+ return tensor._local_tensor+ return tensor+++def _local_nbytes(tensor):+ local = _local(tensor)+ return local.numel() * local.element_size()++def _source_device(local, param, key, device_of):if local.device.type != "cpu":return local.device@@ -251,7 +263,7 @@ class _StateOffloader:for p in params:held = [t for v in state.get(p, {}).values() for t in _held_tensors(v)]if held:- nbytes = sum(v.numel() * v.element_size() for v in held)+ nbytes = sum(_local_nbytes(v) for v in held)else:# Not stepped yet: two moments, a third with amsgrad or centered RMSprop with# momentum (square_avg, momentum_buffer, grad_avg), at least fp32 each.@@ -263,7 +275,7 @@ class _StateOffloader:or "ademamix" in type(self.optimizer).__name__.lower())moments = 3 if third else 2- nbytes = moments * max(p.element_size(), 4) * p.numel()+ nbytes = moments * max(p.element_size(), 4) * _local(p).numel()if chunk and size + nbytes > self.chunk_bytes:chunks.append(chunk)chunk, size = [], 0diff --git a/unsloth/save.py b/unsloth/save.pyindex f53d243a980..6d5b62e8f94 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -4159,6 +4159,30 @@ def _assert_export_target_is_not_base_with_lora_layers(self):)+def _fsdp2_refuses_save(method):+ """Exports that rebuild or quantize the whole model in this process (GGUF, torchao) cannot run on a+ model FSDP2 shards: each rank would export its own shards, all into the same place. Raise on every+ rank (so none waits on another) and say how to get the same file. Passed through without FSDP2."""++ def decorate(fn):+ @functools.wraps(fn)+ def wrapper(self, *args, **kwargs):+ from .models.fsdp2_save import fsdp2_units, fsdp2_gather_active+ if fsdp2_units(self) and not fsdp2_gather_active():+ raise RuntimeError(+ f"Unsloth: `{method}` does not support a model sharded by FSDP2. Save the merged "+ "model on every rank with `model.save_pretrained_merged(save_directory, tokenizer, "+ 'save_method = "merged_16bit")`, then in a single process load that folder with '+ f"`FastLanguageModel.from_pretrained` and call `{method}` there."+ )+ return fn(self, *args, **kwargs)++ return wrapper++ return decorate+++@_fsdp2_refuses_save("save_pretrained_gguf")@_normalize_tied_weights_keys_for_savedef unsloth_save_pretrained_gguf(self,@@ -4810,6 +4834,7 @@ def _free_merge_if_disk_is_tight(return freed+@_fsdp2_refuses_save("push_to_hub_gguf")def unsloth_push_to_hub_gguf(self,repo_id: str,@@ -6397,6 +6422,11 @@ def unsloth_generic_save_pretrained_merged(gc.collect()+# Like save_pretrained_merged: every rank calls it, rank 0 gathers, merges and uploads.+@_fsdp2_collective_save(+ lambda self, args, kwargs: isinstance(self, PeftModel)+ and _is_adapter_save_method(_merged_save_method(args, kwargs))+)def unsloth_generic_push_to_hub_merged(self,repo_id: str,@@ -7511,6 +7541,7 @@ def _unsloth_save_torchao(clean_gpu_cache()+@_fsdp2_refuses_save("save_pretrained_torchao")def unsloth_save_pretrained_torchao(self,save_directory: Union[str, os.PathLike],@@ -8049,6 +8080,11 @@ def patch_saving_functions(model, vision = False):pass'''exec(push_to_hub_text, globals())+ # Under FSDP2 every rank calls push_to_hub: the shards are gathered and rank 0 uploads. Inside,+ # the wrapped save_pretrained sees the gather already active and writes directly.+ globals()["unsloth_push_to_hub"] = _fsdp2_collective_save(+ lambda self, args, kwargs: isinstance(self, PeftModel)+ )(globals()["unsloth_push_to_hub"])@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))def unsloth_model_save_pretrained(self, *args, **kwargs):