Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 QAT export: count the converted copy in the writer's host RAM check (#16166)
Daniel Han242f14b1d2eb
diff --git a/tests/test_fsdp2_save_coverage.py b/tests/test_fsdp2_save_coverage.pyindex 19b6a3ccaf6..c1486c64c7e 100644--- a/tests/test_fsdp2_save_coverage.py+++ b/tests/test_fsdp2_save_coverage.py@@ -174,6 +174,37 @@ def test_qat_torchao_export_converts_where_one_gpu_would(monkeypatch):assert seen == [{"offload_to_cpu": "gpu"}, dict(offload_to_cpu = True, offload_4bit = True)]+def test_host_staged_qat_convert_counts_the_converted_copy(monkeypatch):+ gib = 1024**3+ gathers = []++ @contextlib.contextmanager+ def gathered(model, **kwargs):+ gathers.append(kwargs)+ yield True++ monkeypatch.delenv("UNSLOTH_FSDP2_EXPORT_SKIP_RAM_CHECK", raising = False)+ monkeypatch.setattr(fsdp2_save, "_dist", lambda: None)+ monkeypatch.setattr(fsdp2_save, "mesh_writer_rank", lambda model: 0)+ monkeypatch.setattr(fsdp2_save, "_gathered_host_bytes", lambda *a, **k: 10 * gib)+ monkeypatch.setattr(fsdp2_save, "_host_available_bytes", lambda: 13 * gib)+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", gathered)+ model = torch.nn.Linear(4, 4)+ options = fsdp2_save.gather_on_accelerator_if_it_fits(model)+ assert options["offload_to_cpu"] is True+ with pytest.raises(RuntimeError, match = "converted copy"):+ fsdp2_save.fsdp2_export_on_writer(+ model, lambda: "exported", "save_pretrained_torchao", **options+ )+ assert gathers == []+ assert (+ fsdp2_save.fsdp2_export_on_writer(+ model, lambda: "exported", "save_pretrained_gguf", offload_to_cpu = True+ )+ == "exported"+ )++def test_rank0_export_passes_through_without_fsdp2(monkeypatch):monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [])export = unsloth_save._fsdp2_export_on_rank0("save_pretrained_gguf")(lambda self, x: ("ran", x))diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex c8032a27511..545f6713ee4 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -634,7 +634,7 @@ def gather_on_accelerator_if_it_fits(model):(torchao's QAT convert). On one GPU they sit on the GPU, so the writer keeps the gathered weightson its accelerator when they fit there (with 1 GiB or 10% of the card spare), and the checkpointis the one a single GPU writes. Otherwise they go to the CPU as for any other export."""- on_cpu = dict(offload_to_cpu = True, offload_4bit = True)+ on_cpu = dict(offload_to_cpu = True, offload_4bit = True, converts_in_place = True)if not isinstance(model, torch.nn.Module):return on_cpuDTensor = _dtensor_class()@@ -672,10 +672,14 @@ def gather_on_accelerator_if_it_fits(model):# Not on the CPU: CUDA computes torchao's `scale = max / 7.5` as max * (1 / 7.5), the CPU# divides, and for fp32 weights the two differ in the last bit of most scales.return dict(on_cpu, convert_on = device)- return dict(offload_to_cpu = False, offload_4bit = False)+ return dict(offload_to_cpu = False, offload_4bit = False, converts_in_place = True)-def _host_ram_refusal(method, need):+def _host_ram_refusal(+ method,+ need,+ what = "gathered weights",+):"""None when `need` gathered bytes fit this host's available RAM, else the refusal message."""if os.environ.get("UNSLOTH_FSDP2_EXPORT_SKIP_RAM_CHECK", "0") == "1":return None@@ -688,7 +692,7 @@ def _host_ram_refusal(method, need):return (f"Unsloth: `{method}` under FSDP2 gathers the whole model into the host RAM of the rank that "f"exports: that needs about {(need + headroom) / _GIB:.1f} GiB ({need / _GIB:.1f} GiB of "- f"gathered weights + {headroom / _GIB:.1f} GiB headroom), but only {available / _GIB:.1f} GiB "+ f"{what} + {headroom / _GIB:.1f} GiB headroom), but only {available / _GIB:.1f} GiB ""is available. Save the adapter with `model.save_pretrained(save_directory)` (it gathers only ""the LoRA weights), then in a single process load that folder with "f"`FastLanguageModel.from_pretrained` and call `{method}` there. Set "@@ -780,6 +784,7 @@ def fsdp2_export_on_writer(offload_to_cpu = True,offload_4bit = False,convert_on = None,+ converts_in_place = False,):"""An export that rebuilds or quantizes the whole model in one process (GGUF, torchao, OpenVINO),on a model FSDP2 shards. Every rank must call it. The writer (global rank 0, or the lowest rank@@ -794,7 +799,13 @@ def fsdp2_export_on_writer(refusal = Noneif writer:need = _gathered_host_bytes(model, trainable_only, include, offload_to_cpu, offload_4bit)- refusal = _host_ram_refusal(method, need)+ what = "gathered weights"+ if converts_in_place:+ need, what = (+ _accelerator_convert_bytes(need),+ "gathered weights and their converted copy",+ )+ refusal = _host_ram_refusal(method, need, what)if dist is not None:refusal = _share_from_writer(dist, writer, writer_rank, "fit", refusal)if refusal: