Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 QAT export: count the converted copy in the writer's host RAM check (#16166)

Daniel Han242f14b1d2eb
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.py
index 19b6a3ccaf6..c1486c64c7e 100644
--- a/tests/test_fsdp2_save_coverage.py
+++ b/tests/test_fsdp2_save_coverage.py
@@ -174,6 +174,37 @@ def test_qat_torchao_export_converts_where_one_gpu_would(monkeypatch):
assert seen == [{"offload_to_cpu": "gpu"}, dict(offload_to_cpu = True, offload_4bit = True)]
+def test_host_staged_qat_convert_counts_the_converted_copy(monkeypatch):
+ gib = 1024**3
+ gathers = []
+
+ @contextlib.contextmanager
+ def gathered(model, **kwargs):
+ gathers.append(kwargs)
+ yield True
+
+ monkeypatch.delenv("UNSLOTH_FSDP2_EXPORT_SKIP_RAM_CHECK", raising = False)
+ monkeypatch.setattr(fsdp2_save, "_dist", lambda: None)
+ monkeypatch.setattr(fsdp2_save, "mesh_writer_rank", lambda model: 0)
+ monkeypatch.setattr(fsdp2_save, "_gathered_host_bytes", lambda *a, **k: 10 * gib)
+ monkeypatch.setattr(fsdp2_save, "_host_available_bytes", lambda: 13 * gib)
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)
+ model = torch.nn.Linear(4, 4)
+ options = fsdp2_save.gather_on_accelerator_if_it_fits(model)
+ assert options["offload_to_cpu"] is True
+ with pytest.raises(RuntimeError, match = "converted copy"):
+ fsdp2_save.fsdp2_export_on_writer(
+ model, lambda: "exported", "save_pretrained_torchao", **options
+ )
+ assert gathers == []
+ assert (
+ fsdp2_save.fsdp2_export_on_writer(
+ model, lambda: "exported", "save_pretrained_gguf", offload_to_cpu = True
+ )
+ == "exported"
+ )
+
+
def test_rank0_export_passes_through_without_fsdp2(monkeypatch):
monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])
export = unsloth_save._fsdp2_export_on_rank0("save_pretrained_gguf")(lambda self, x: ("ran", x))
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index c8032a27511..545f6713ee4 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -634,7 +634,7 @@ def gather_on_accelerator_if_it_fits(model):
(torchao's QAT convert). On one GPU they sit on the GPU, so the writer keeps the gathered weights
on its accelerator when they fit there (with 1 GiB or 10% of the card spare), and the checkpoint
is the one a single GPU writes. Otherwise they go to the CPU as for any other export."""
- on_cpu = dict(offload_to_cpu = True, offload_4bit = True)
+ on_cpu = dict(offload_to_cpu = True, offload_4bit = True, converts_in_place = True)
if not isinstance(model, torch.nn.Module):
return on_cpu
DTensor = _dtensor_class()
@@ -672,10 +672,14 @@ def gather_on_accelerator_if_it_fits(model):
# Not on the CPU: CUDA computes torchao's `scale = max / 7.5` as max * (1 / 7.5), the CPU
# divides, and for fp32 weights the two differ in the last bit of most scales.
return dict(on_cpu, convert_on = device)
- return dict(offload_to_cpu = False, offload_4bit = False)
+ return dict(offload_to_cpu = False, offload_4bit = False, converts_in_place = True)
-def _host_ram_refusal(method, need):
+def _host_ram_refusal(
+ method,
+ need,
+ what = "gathered weights",
+):
"""None when `need` gathered bytes fit this host's available RAM, else the refusal message."""
if os.environ.get("UNSLOTH_FSDP2_EXPORT_SKIP_RAM_CHECK", "0") == "1":
return None
@@ -688,7 +692,7 @@ def _host_ram_refusal(method, need):
return (
f"Unsloth: `{method}` under FSDP2 gathers the whole model into the host RAM of the rank that "
f"exports: that needs about {(need + headroom) / _GIB:.1f} GiB ({need / _GIB:.1f} GiB of "
- f"gathered weights + {headroom / _GIB:.1f} GiB headroom), but only {available / _GIB:.1f} GiB "
+ f"{what} + {headroom / _GIB:.1f} GiB headroom), but only {available / _GIB:.1f} GiB "
"is available. Save the adapter with `model.save_pretrained(save_directory)` (it gathers only "
"the LoRA weights), then in a single process load that folder with "
f"`FastLanguageModel.from_pretrained` and call `{method}` there. Set "
@@ -780,6 +784,7 @@ def fsdp2_export_on_writer(
offload_to_cpu = True,
offload_4bit = False,
convert_on = None,
+ converts_in_place = False,
):
"""An export that rebuilds or quantizes the whole model in one process (GGUF, torchao, OpenVINO),
on a model FSDP2 shards. Every rank must call it. The writer (global rank 0, or the lowest rank
@@ -794,7 +799,13 @@ def fsdp2_export_on_writer(
refusal = None
if writer:
need = _gathered_host_bytes(model, trainable_only, include, offload_to_cpu, offload_4bit)
- refusal = _host_ram_refusal(method, need)
+ what = "gathered weights"
+ if converts_in_place:
+ need, what = (
+ _accelerator_convert_bytes(need),
+ "gathered weights and their converted copy",
+ )
+ refusal = _host_ram_refusal(method, need, what)
if dist is not None:
refusal = _share_from_writer(dist, writer, writer_rank, "fit", refusal)
if refusal: