Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add dtype option to save_pretrained_merged for float32 merges (#13823)

Daniel Hanf5f3c4360614
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.py
index 849c76e5838..063c0c64ec0 100644
--- a/tests/saving/test_gguf_disk_preflight.py
+++ b/tests/saving/test_gguf_disk_preflight.py
@@ -445,6 +445,11 @@ class TestMergeSizing:
# Not the GGUF estimate, and no headroom: `_FakeModel` has no adapter, so no merge_and_overwrite_lora.
assert sized == [_merge_preflight_ask(10 * GB, 0)]
+ def test_a_float32_merge_is_four_bytes_per_parameter(self, sized):
+ """unsloth#5186: `dtype = float32` doubles the checkpoint, so Kaggle must budget for it."""
+ S._preflight_merge_disk(_FakeModel(), "model", "merged_16bit", widen_to_float32 = True)
+ assert sized == [_merge_preflight_ask(20 * GB, 0)]
+
@pytest.mark.parametrize("save_method", ["merged 16bit", "MERGED_16BIT", " merged-16bit "])
def test_supported_spellings_are_measured_too(self, sized, save_method):
"""`unsloth_save_model` normalizes spaces, so these are the same export."""
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index 81bcb50c94c..9f50ae1a825 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import ast
import contextlib
+import json
import os
import sys
import types
@@ -192,6 +193,8 @@ _GENERIC_SAVE_REAL_HELPERS = (
"_is_adapter_save_method",
"_honours_safe_serialization",
"_refuse_unsaveable_text_core",
+ "_requests_float32",
+ "_stamp_saved_config_float32",
)
_GENERIC_SAVE_STUBBED_HELPERS = frozenset(
{
@@ -221,6 +224,7 @@ def _routing_environment(monkeypatch, model):
*_GENERIC_SAVE_REAL_HELPERS,
"unsloth_generic_save",
os = os,
+ json = json,
PeftModel = _PeftModel,
PreTrainedTokenizerBase = type("Tokenizer", (), {}),
ProcessorMixin = type("Processor", (), {}),
@@ -241,6 +245,7 @@ def _routing_environment(monkeypatch, model):
torch = types.SimpleNamespace(
bfloat16 = "bfloat16",
float16 = "float16",
+ float32 = "float32",
save = lambda *args, **kwargs: None,
cuda = types.SimpleNamespace(is_bf16_supported = lambda: False),
),
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index 3499565f192..0fc749c3b53 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -146,6 +146,7 @@ def saving(monkeypatch, tmp_path):
env = dict(
os = os,
+ sys = sys,
gc = gc,
Path = Path,
HfApi = Api,
@@ -156,6 +157,7 @@ def saving(monkeypatch, tmp_path):
save = lambda: None,
float16 = "float16",
bfloat16 = "bfloat16",
+ float32 = "float32",
cuda = SimpleNamespace(is_bf16_supported = lambda: False),
),
get_token = lambda: "cached-fixture",
@@ -185,6 +187,7 @@ def saving(monkeypatch, tmp_path):
"_honours_safe_serialization",
"_assert_export_target_is_not_base_with_lora_layers",
"_refuse_unsaveable_text_core",
+ "_requests_float32",
}
nodes = []
for node in tree.body:
@@ -542,6 +545,8 @@ def test_default_full_finetune_push_uploads_16bit_safetensors(monkeypatch):
operation.path_in_repo: (
load_file(operation.path_or_fileobj)
if operation.path_in_repo.endswith(".safetensors")
+ else json.loads(Path(operation.path_or_fileobj).read_text())
+ if operation.path_in_repo == "config.json"
else None
)
for operation in kwargs["operations"]
@@ -571,6 +576,17 @@ def test_default_full_finetune_push_uploads_16bit_safetensors(monkeypatch):
assert {tensor.dtype for tensor in commits[0]["model.safetensors"].values()} == {target}
assert next(model.parameters()).dtype == torch.float32
+ # unsloth#5186: `dtype = float32` writes float32 tensors and config from a 16-bit model.
+ commits.clear()
+ model = model.to(torch.bfloat16)
+ save.unsloth_generic_push_to_hub_merged(
+ model, "owner/model", token = "fixture", dtype = torch.float32
+ )
+ assert next(model.parameters()).dtype == torch.bfloat16
+ assert {tensor.dtype for tensor in commits[0]["model.safetensors"].values()} == {torch.float32}
+ config = commits[0]["config.json"]
+ assert config.get("dtype", config.get("torch_dtype")) == "float32"
+
def test_the_push_names_the_destination_and_not_the_staging_folder(saving, capsys):
"""Name the requested repository, not the temp staging folder, which is deleted moments later."""
@@ -667,3 +683,29 @@ def test_a_pickle_request_this_transformers_cannot_honour_is_reported(saving):
assert push(HonoursIt(), safe_serialization = False) == []
assert push(Patched(), safe_serialization = False) == []
assert push(NoSafeSerialization()) == []
+
+
+def test_merged_dtype_reaches_the_streaming_merger(saving):
+ """unsloth#5186: `dtype` reaches the zoo merge as `output_dtype`; the default stays None."""
+ env, records, _ = saving
+ env["unsloth_generic_push_to_hub_merged"](PeftModel(), "owner/model", token = "fixture")
+ assert records["merges"][-1]["output_dtype"] is None
+
+ sys.modules["unsloth_zoo.saving_utils"]._upcast_shard_to_float32 = lambda *args: None
+ env["unsloth_generic_push_to_hub_merged"](
+ PeftModel(), "owner/model", token = "fixture", dtype = "float32"
+ )
+ assert records["merges"][-1]["output_dtype"] == "float32"
+
+
+def test_merged_dtype_refuses_an_older_zoo(saving):
+ """An older unsloth_zoo ignores `output_dtype` and writes the base dtype, so refuse."""
+ env, records, _ = saving
+ prewarmed = []
+ env["_prewarm_base_model_hub_cache"] = lambda *args, **kwargs: prewarmed.append(args)
+ with pytest.raises(RuntimeError, match = "newer unsloth_zoo"):
+ env["unsloth_generic_push_to_hub_merged"](
+ PeftModel(), "owner/model", token = "fixture", dtype = "float32"
+ )
+ # Refused before the base model is downloaded into the Hub cache.
+ assert records["merges"] == [] and prewarmed == []
diff --git a/unsloth/save.py b/unsloth/save.py
index 3a4daf34e87..c9badc77c76 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -3226,6 +3226,38 @@ def _merge_writer_disposition(model, save_method):
return (True, True) if takes_generic_fallback else (False, False)
+def _requests_float32(dtype):
+ if isinstance(dtype, str):
+ return dtype.strip().lower().removeprefix("torch.") in ("float32", "fp32", "f32")
+ return dtype is torch.float32
+
+
+def _stamp_saved_config_float32(save_directory):
+ # save_pretrained writes the resident model's 16-bit dtype, and dtype = "auto" would then
+ # cast the float32 tensors back down on load.
+ path = os.path.join(save_directory, "config.json")
+ if not os.path.exists(path):
+ return
+ with open(path, "r", encoding = "utf-8") as f:
+ data = json.load(f)
+
+ def _walk(obj):
+ if isinstance(obj, dict):
+ for key in ("torch_dtype", "dtype"):
+ if obj.get(key) in ("bfloat16", "float16"):
+ obj[key] = "float32"
+ for value in obj.values():
+ _walk(value)
+ elif isinstance(obj, list):
+ for value in obj:
+ _walk(value)
+
+ _walk(data)
+ with open(path, "w", encoding = "utf-8") as f:
+ json.dump(data, f, indent = 2, sort_keys = True)
+ f.write("\n")
+
+
def _preflight_merge_disk(
model,
save_directory,
@@ -3235,6 +3267,7 @@ def _preflight_merge_disk(
forwards_state_dict = False,
writes_model_verbatim = False,
writer_runs_merge_guard = False,
+ widen_to_float32 = False,
):
"""Kaggle only: send a merge that cannot fit in /kaggle/working to /tmp. Never raises. A merge short of space on an ordinary machine is already handled by unsloth_zoo's own guard, which knows about shard streaming and the push_to_hub fallbacks; this exists so the ~20GB Kaggle working directory stops being the ceiling when a terabyte of overlay is mounted beside it. Skipped when pushing to the hub, where `save_directory` is a repo id and rewriting it would push to the wrong repository. `forwards_state_dict`: the writer hands a supplied `state_dict` to `save_pretrained` for a 16-bit save, casting floating entries to two bytes, as `unsloth_generic_save` does; `unsloth_save_model`'s merge path instead rebuilds the dict from the merged layers and drops what it was given. `writes_model_verbatim`: the writer copies what it holds with no cast at all, which is `unsloth_save_model`'s generic architecture fallback, so the checkpoint is the dict's own bytes at its own dtypes, or the resident parameters' bytes when no dict was supplied. `writer_runs_merge_guard`: the writer is `unsloth_generic_save`, whose PEFT branch is the only caller of `merge_and_overwrite_lora` here and so the only writer bringing its `free * 0.95` guard, deliberately separate from the two SIZING flags since a compressed export is cast to two bytes and sized accordingly yet reserves nothing unless there is an adapter to merge. `_merge_writer_disposition` decides the first two for the public entrypoint."""
if push_to_hub:
@@ -3266,6 +3299,9 @@ def _preflight_merge_disk(
need = _cast_16bit_state_dict_bytes(supplied_dict)
else:
need = model_16bit_bytes(model)
+ # `dtype = float32` on merged_16bit writes four bytes per parameter, not two.
+ if widen_to_float32 and method == "merged_16bit":
+ need *= 2
if need <= 0:
return save_directory
# What the torchao staging directory costs on whatever filesystem tempfile resolves to; zero for every other export, which stages nothing.
@@ -5606,6 +5642,7 @@ def unsloth_generic_save(
temporary_location: str = "_unsloth_temporary_saved_buffers",
maximum_memory_usage: float = 0.9,
datasets: Optional[List[str]] = None,
+ dtype = None,
):
if save_method == "merged_4bit":
raise RuntimeError(
@@ -5658,11 +5695,18 @@ def unsloth_generic_save(
if ("16bit" in save_method or is_qwen3_5_vlm) and state_dict is None:
state_dict = model.state_dict()
if "16bit" in save_method:
- _target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
- state_dict = {
- k: v.to(dtype = _target_dtype) if v.is_floating_point() else v
- for k, v in state_dict.items()
- }
+ if _requests_float32(dtype):
+ # On the CPU: a float32 copy of a 16-bit model would double its GPU footprint.
+ state_dict = {
+ k: v.to(device = "cpu", dtype = torch.float32) if v.is_floating_point() else v
+ for k, v in state_dict.items()
+ }
+ else:
+ _target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+ state_dict = {
+ k: v.to(dtype = _target_dtype) if v.is_floating_point() else v
+ for k, v in state_dict.items()
+ }
if is_qwen3_5_vlm:
state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)
if state_dict is not None:
@@ -5676,6 +5720,8 @@ def unsloth_generic_save(
print(f"Unsloth: Saving full fine-tuned model to '{save_directory}' ...")
model.save_pretrained(save_directory, **_save_kwargs)
+ if "16bit" in save_method and _requests_float32(dtype):
+ _stamp_saved_config_float32(save_directory)
# Guarded: an older zoo must not raise once the weights are already on disk.
try:
from unsloth_zoo.saving_utils import reconcile_mtp_config
@@ -5733,9 +5779,18 @@ def unsloth_generic_save(
)
else:
raise_if_merging_mistral_format_view(model, save_method)
- _prewarm_base_model_hub_cache(model, save_method = save_method, token = token)
from unsloth_zoo.saving_utils import merge_and_overwrite_lora
+ # An older unsloth_zoo accepts `output_dtype` but still writes the base dtype.
+ if dtype is not None and not hasattr(
+ sys.modules.get("unsloth_zoo.saving_utils"), "_upcast_shard_to_float32"
+ ):
+ raise RuntimeError(
+ "Unsloth: `dtype` needs a newer unsloth_zoo. Update via "
+ "`pip install --upgrade --no-cache-dir unsloth_zoo`."
+ )
+ _prewarm_base_model_hub_cache(model, save_method = save_method, token = token)
+
# merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.
in_place = save_method in ("merged_4bit", "forced_merged_4bit")
with nullcontext() if in_place else lora_relative_to_original_base(model):
@@ -5750,7 +5805,7 @@ def unsloth_generic_save(
private = private,
token = token,
save_method = save_method,
- output_dtype = None,
+ output_dtype = dtype,
low_disk_space_usage = True,
use_temp_file = False,
)
@@ -5790,6 +5845,7 @@ def unsloth_generic_save_pretrained_merged(
num_calibration_samples: int = 512,
max_seq_length: int = 2048,
install_missing_dependencies: bool = True,
+ dtype = None,
):
"""
Same as .push_to_hub(...) except 4bit weights are auto
@@ -5809,6 +5865,10 @@ def unsloth_generic_save_pretrained_merged(
`nvfp4` needs calibration data (defaults to ultrachat; override with `calibration_dataset`).
Pass ``install_missing_dependencies=False`` to never auto-install llm-compressor.
+ `dtype` (merged_16bit only): None keeps the base checkpoint's dtype (bfloat16 / float16).
+ `torch.float32` (or "float32") writes float32 safetensors without rounding the merged
+ LoRA delta back to 16 bits; shards are streamed, so peak RAM stays about one tensor.
+
`safe_serialization` defaults to safetensors. `None` is stronger than the default `True`: on a host
with at most two physical CPUs the default downgrades to a pickle, since safetensors is
roughly 10x slower there, while `None` pins safetensors through that fallback. `False`
@@ -5832,6 +5892,7 @@ def unsloth_generic_save_pretrained_merged(
forwards_state_dict = True,
# And it is the writer that runs merge_and_overwrite_lora when there IS one, which no other entrypoint here does.
writer_runs_merge_guard = True,
+ widen_to_float32 = _requests_float32(dtype),
)
_compressed = _normalize_compressed_method(save_method)
@@ -5929,6 +5990,7 @@ def unsloth_generic_push_to_hub_merged(
num_calibration_samples: int = 512,
max_seq_length: int = 2048,
install_missing_dependencies: bool = True,
+ dtype = None,
):
"""
Same as .push_to_hub(...) except 4bit weights are auto
@@ -5945,6 +6007,10 @@ def unsloth_generic_push_to_hub_merged(
4. FP8 / FP4 compressed export for vLLM: `fp8`, `mxfp4`, `nvfp4`, `mxfp8`.
Pass ``install_missing_dependencies=False`` to never auto-install llm-compressor.
+ `dtype` (merged_16bit only): None keeps the base checkpoint's dtype (bfloat16 / float16).
+ `torch.float32` (or "float32") writes float32 safetensors without rounding the merged
+ LoRA delta back to 16 bits; shards are streamed, so peak RAM stays about one tensor.
+
`safe_serialization` defaults to safetensors. `None` is stronger than the default `True`: on a host
with at most two physical CPUs the default downgrades to a pickle, since safetensors is
roughly 10x slower there, while `None` pins safetensors through that fallback. `False`