Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add dtype option to save_pretrained_merged for float32 merges (#13823)
Daniel Hanf5f3c4360614
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.pyindex 849c76e5838..063c0c64ec0 100644--- a/tests/saving/test_gguf_disk_preflight.py+++ b/tests/saving/test_gguf_disk_preflight.py@@ -445,6 +445,11 @@ class TestMergeSizing:# Not the GGUF estimate, and no headroom: `_FakeModel` has no adapter, so no merge_and_overwrite_lora.assert sized == [_merge_preflight_ask(10 * GB, 0)]+ def test_a_float32_merge_is_four_bytes_per_parameter(self, sized):+ """unsloth#5186: `dtype = float32` doubles the checkpoint, so Kaggle must budget for it."""+ S._preflight_merge_disk(_FakeModel(), "model", "merged_16bit", widen_to_float32 = True)+ assert sized == [_merge_preflight_ask(20 * GB, 0)]+@pytest.mark.parametrize("save_method", ["merged 16bit", "MERGED_16BIT", " merged-16bit "])def test_supported_spellings_are_measured_too(self, sized, save_method):"""`unsloth_save_model` normalizes spaces, so these are the same export."""diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex 81bcb50c94c..9f50ae1a825 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport astimport contextlib+import jsonimport osimport sysimport types@@ -192,6 +193,8 @@ _GENERIC_SAVE_REAL_HELPERS = ("_is_adapter_save_method","_honours_safe_serialization","_refuse_unsaveable_text_core",+ "_requests_float32",+ "_stamp_saved_config_float32",)_GENERIC_SAVE_STUBBED_HELPERS = frozenset({@@ -221,6 +224,7 @@ def _routing_environment(monkeypatch, model):*_GENERIC_SAVE_REAL_HELPERS,"unsloth_generic_save",os = os,+ json = json,PeftModel = _PeftModel,PreTrainedTokenizerBase = type("Tokenizer", (), {}),ProcessorMixin = type("Processor", (), {}),@@ -241,6 +245,7 @@ def _routing_environment(monkeypatch, model):torch = types.SimpleNamespace(bfloat16 = "bfloat16",float16 = "float16",+ float32 = "float32",save = lambda *args, **kwargs: None,cuda = types.SimpleNamespace(is_bf16_supported = lambda: False),),diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex 3499565f192..0fc749c3b53 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -146,6 +146,7 @@ def saving(monkeypatch, tmp_path):env = dict(os = os,+ sys = sys,gc = gc,Path = Path,HfApi = Api,@@ -156,6 +157,7 @@ def saving(monkeypatch, tmp_path):save = lambda: None,float16 = "float16",bfloat16 = "bfloat16",+ float32 = "float32",cuda = SimpleNamespace(is_bf16_supported = lambda: False),),get_token = lambda: "cached-fixture",@@ -185,6 +187,7 @@ def saving(monkeypatch, tmp_path):"_honours_safe_serialization","_assert_export_target_is_not_base_with_lora_layers","_refuse_unsaveable_text_core",+ "_requests_float32",}nodes = []for node in tree.body:@@ -542,6 +545,8 @@ def test_default_full_finetune_push_uploads_16bit_safetensors(monkeypatch):operation.path_in_repo: (load_file(operation.path_or_fileobj)if operation.path_in_repo.endswith(".safetensors")+ else json.loads(Path(operation.path_or_fileobj).read_text())+ if operation.path_in_repo == "config.json"else None)for operation in kwargs["operations"]@@ -571,6 +576,17 @@ def test_default_full_finetune_push_uploads_16bit_safetensors(monkeypatch):assert {tensor.dtype for tensor in commits[0]["model.safetensors"].values()} == {target}assert next(model.parameters()).dtype == torch.float32+ # unsloth#5186: `dtype = float32` writes float32 tensors and config from a 16-bit model.+ commits.clear()+ model = model.to(torch.bfloat16)+ save.unsloth_generic_push_to_hub_merged(+ model, "owner/model", token = "fixture", dtype = torch.float32+ )+ assert next(model.parameters()).dtype == torch.bfloat16+ assert {tensor.dtype for tensor in commits[0]["model.safetensors"].values()} == {torch.float32}+ config = commits[0]["config.json"]+ assert config.get("dtype", config.get("torch_dtype")) == "float32"+def test_the_push_names_the_destination_and_not_the_staging_folder(saving, capsys):"""Name the requested repository, not the temp staging folder, which is deleted moments later."""@@ -667,3 +683,29 @@ def test_a_pickle_request_this_transformers_cannot_honour_is_reported(saving):assert push(HonoursIt(), safe_serialization = False) == []assert push(Patched(), safe_serialization = False) == []assert push(NoSafeSerialization()) == []+++def test_merged_dtype_reaches_the_streaming_merger(saving):+ """unsloth#5186: `dtype` reaches the zoo merge as `output_dtype`; the default stays None."""+ env, records, _ = saving+ env["unsloth_generic_push_to_hub_merged"](PeftModel(), "owner/model", token = "fixture")+ assert records["merges"][-1]["output_dtype"] is None++ sys.modules["unsloth_zoo.saving_utils"]._upcast_shard_to_float32 = lambda *args: None+ env["unsloth_generic_push_to_hub_merged"](+ PeftModel(), "owner/model", token = "fixture", dtype = "float32"+ )+ assert records["merges"][-1]["output_dtype"] == "float32"+++def test_merged_dtype_refuses_an_older_zoo(saving):+ """An older unsloth_zoo ignores `output_dtype` and writes the base dtype, so refuse."""+ env, records, _ = saving+ prewarmed = []+ env["_prewarm_base_model_hub_cache"] = lambda *args, **kwargs: prewarmed.append(args)+ with pytest.raises(RuntimeError, match = "newer unsloth_zoo"):+ env["unsloth_generic_push_to_hub_merged"](+ PeftModel(), "owner/model", token = "fixture", dtype = "float32"+ )+ # Refused before the base model is downloaded into the Hub cache.+ assert records["merges"] == [] and prewarmed == []diff --git a/unsloth/save.py b/unsloth/save.pyindex 3a4daf34e87..c9badc77c76 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -3226,6 +3226,38 @@ def _merge_writer_disposition(model, save_method):return (True, True) if takes_generic_fallback else (False, False)+def _requests_float32(dtype):+ if isinstance(dtype, str):+ return dtype.strip().lower().removeprefix("torch.") in ("float32", "fp32", "f32")+ return dtype is torch.float32+++def _stamp_saved_config_float32(save_directory):+ # save_pretrained writes the resident model's 16-bit dtype, and dtype = "auto" would then+ # cast the float32 tensors back down on load.+ path = os.path.join(save_directory, "config.json")+ if not os.path.exists(path):+ return+ with open(path, "r", encoding = "utf-8") as f:+ data = json.load(f)++ def _walk(obj):+ if isinstance(obj, dict):+ for key in ("torch_dtype", "dtype"):+ if obj.get(key) in ("bfloat16", "float16"):+ obj[key] = "float32"+ for value in obj.values():+ _walk(value)+ elif isinstance(obj, list):+ for value in obj:+ _walk(value)++ _walk(data)+ with open(path, "w", encoding = "utf-8") as f:+ json.dump(data, f, indent = 2, sort_keys = True)+ f.write("\n")++def _preflight_merge_disk(model,save_directory,@@ -3235,6 +3267,7 @@ def _preflight_merge_disk(forwards_state_dict = False,writes_model_verbatim = False,writer_runs_merge_guard = False,+ widen_to_float32 = False,):"""Kaggle only: send a merge that cannot fit in /kaggle/working to /tmp. Never raises. A merge short of space on an ordinary machine is already handled by unsloth_zoo's own guard, which knows about shard streaming and the push_to_hub fallbacks; this exists so the ~20GB Kaggle working directory stops being the ceiling when a terabyte of overlay is mounted beside it. Skipped when pushing to the hub, where `save_directory` is a repo id and rewriting it would push to the wrong repository. `forwards_state_dict`: the writer hands a supplied `state_dict` to `save_pretrained` for a 16-bit save, casting floating entries to two bytes, as `unsloth_generic_save` does; `unsloth_save_model`'s merge path instead rebuilds the dict from the merged layers and drops what it was given. `writes_model_verbatim`: the writer copies what it holds with no cast at all, which is `unsloth_save_model`'s generic architecture fallback, so the checkpoint is the dict's own bytes at its own dtypes, or the resident parameters' bytes when no dict was supplied. `writer_runs_merge_guard`: the writer is `unsloth_generic_save`, whose PEFT branch is the only caller of `merge_and_overwrite_lora` here and so the only writer bringing its `free * 0.95` guard, deliberately separate from the two SIZING flags since a compressed export is cast to two bytes and sized accordingly yet reserves nothing unless there is an adapter to merge. `_merge_writer_disposition` decides the first two for the public entrypoint."""if push_to_hub:@@ -3266,6 +3299,9 @@ def _preflight_merge_disk(need = _cast_16bit_state_dict_bytes(supplied_dict)else:need = model_16bit_bytes(model)+ # `dtype = float32` on merged_16bit writes four bytes per parameter, not two.+ if widen_to_float32 and method == "merged_16bit":+ need *= 2if need <= 0:return save_directory# What the torchao staging directory costs on whatever filesystem tempfile resolves to; zero for every other export, which stages nothing.@@ -5606,6 +5642,7 @@ def unsloth_generic_save(temporary_location: str = "_unsloth_temporary_saved_buffers",maximum_memory_usage: float = 0.9,datasets: Optional[List[str]] = None,+ dtype = None,):if save_method == "merged_4bit":raise RuntimeError(@@ -5658,11 +5695,18 @@ def unsloth_generic_save(if ("16bit" in save_method or is_qwen3_5_vlm) and state_dict is None:state_dict = model.state_dict()if "16bit" in save_method:- _target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16- state_dict = {- k: v.to(dtype = _target_dtype) if v.is_floating_point() else v- for k, v in state_dict.items()- }+ if _requests_float32(dtype):+ # On the CPU: a float32 copy of a 16-bit model would double its GPU footprint.+ state_dict = {+ k: v.to(device = "cpu", dtype = torch.float32) if v.is_floating_point() else v+ for k, v in state_dict.items()+ }+ else:+ _target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16+ state_dict = {+ k: v.to(dtype = _target_dtype) if v.is_floating_point() else v+ for k, v in state_dict.items()+ }if is_qwen3_5_vlm:state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)if state_dict is not None:@@ -5676,6 +5720,8 @@ def unsloth_generic_save(print(f"Unsloth: Saving full fine-tuned model to '{save_directory}' ...")model.save_pretrained(save_directory, **_save_kwargs)+ if "16bit" in save_method and _requests_float32(dtype):+ _stamp_saved_config_float32(save_directory)# Guarded: an older zoo must not raise once the weights are already on disk.try:from unsloth_zoo.saving_utils import reconcile_mtp_config@@ -5733,9 +5779,18 @@ def unsloth_generic_save()else:raise_if_merging_mistral_format_view(model, save_method)- _prewarm_base_model_hub_cache(model, save_method = save_method, token = token)from unsloth_zoo.saving_utils import merge_and_overwrite_lora+ # An older unsloth_zoo accepts `output_dtype` but still writes the base dtype.+ if dtype is not None and not hasattr(+ sys.modules.get("unsloth_zoo.saving_utils"), "_upcast_shard_to_float32"+ ):+ raise RuntimeError(+ "Unsloth: `dtype` needs a newer unsloth_zoo. Update via "+ "`pip install --upgrade --no-cache-dir unsloth_zoo`."+ )+ _prewarm_base_model_hub_cache(model, save_method = save_method, token = token)+# merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.in_place = save_method in ("merged_4bit", "forced_merged_4bit")with nullcontext() if in_place else lora_relative_to_original_base(model):@@ -5750,7 +5805,7 @@ def unsloth_generic_save(private = private,token = token,save_method = save_method,- output_dtype = None,+ output_dtype = dtype,low_disk_space_usage = True,use_temp_file = False,)@@ -5790,6 +5845,7 @@ def unsloth_generic_save_pretrained_merged(num_calibration_samples: int = 512,max_seq_length: int = 2048,install_missing_dependencies: bool = True,+ dtype = None,):"""Same as .push_to_hub(...) except 4bit weights are auto@@ -5809,6 +5865,10 @@ def unsloth_generic_save_pretrained_merged(`nvfp4` needs calibration data (defaults to ultrachat; override with `calibration_dataset`).Pass ``install_missing_dependencies=False`` to never auto-install llm-compressor.+ `dtype` (merged_16bit only): None keeps the base checkpoint's dtype (bfloat16 / float16).+ `torch.float32` (or "float32") writes float32 safetensors without rounding the merged+ LoRA delta back to 16 bits; shards are streamed, so peak RAM stays about one tensor.+`safe_serialization` defaults to safetensors. `None` is stronger than the default `True`: on a hostwith at most two physical CPUs the default downgrades to a pickle, since safetensors isroughly 10x slower there, while `None` pins safetensors through that fallback. `False`@@ -5832,6 +5892,7 @@ def unsloth_generic_save_pretrained_merged(forwards_state_dict = True,# And it is the writer that runs merge_and_overwrite_lora when there IS one, which no other entrypoint here does.writer_runs_merge_guard = True,+ widen_to_float32 = _requests_float32(dtype),)_compressed = _normalize_compressed_method(save_method)@@ -5929,6 +5990,7 @@ def unsloth_generic_push_to_hub_merged(num_calibration_samples: int = 512,max_seq_length: int = 2048,install_missing_dependencies: bool = True,+ dtype = None,):"""Same as .push_to_hub(...) except 4bit weights are auto@@ -5945,6 +6007,10 @@ def unsloth_generic_push_to_hub_merged(4. FP8 / FP4 compressed export for vLLM: `fp8`, `mxfp4`, `nvfp4`, `mxfp8`.Pass ``install_missing_dependencies=False`` to never auto-install llm-compressor.+ `dtype` (merged_16bit only): None keeps the base checkpoint's dtype (bfloat16 / float16).+ `torch.float32` (or "float32") writes float32 safetensors without rounding the merged+ LoRA delta back to 16 bits; shards are streamed, so peak RAM stays about one tensor.+`safe_serialization` defaults to safetensors. `None` is stronger than the default `True`: on a hostwith at most two physical CPUs the default downgrades to a pickle, since safetensors isroughly 10x slower there, while `None` pins safetensors through that fallback. `False`