Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep fp32-only tensors on 16-bit merged saves (#13819)

Daniel Han699f44630a00
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.py
index 063c0c64ec0..4816839e5b8 100644
--- a/tests/saving/test_gguf_disk_preflight.py
+++ b/tests/saving/test_gguf_disk_preflight.py
@@ -1500,7 +1500,8 @@ class TestASuppliedDictIsWhatASixteenBitSaveWrites:
generic = inspect.getsource(S.unsloth_generic_save)
assert '("16bit" in save_method or is_qwen3_5_vlm) and state_dict is None' in generic
- assert "v.to(dtype = _target_dtype) if v.is_floating_point() else v" in generic
+ assert "state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)" in generic
+ # The other writer overwrites the caller's dictionary with its own.
assert "state_dict = OrderedDict()" in inspect.getsource(S.unsloth_save_model)
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index 9f50ae1a825..f401126abce 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -193,6 +193,7 @@ _GENERIC_SAVE_REAL_HELPERS = (
"_is_adapter_save_method",
"_honours_safe_serialization",
"_refuse_unsaveable_text_core",
+ "_cast_state_dict_16bit",
"_requests_float32",
"_stamp_saved_config_float32",
)
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index 0fc749c3b53..d3381bd32e9 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -187,6 +187,7 @@ def saving(monkeypatch, tmp_path):
"_honours_safe_serialization",
"_assert_export_target_is_not_base_with_lora_layers",
"_refuse_unsaveable_text_core",
+ "_cast_state_dict_16bit",
"_requests_float32",
}
nodes = []
diff --git a/tests/test_save_keep_fp32_modules.py b/tests/test_save_keep_fp32_modules.py
new file mode 100644
index 00000000000..23ac8458a54
--- /dev/null
+++ b/tests/test_save_keep_fp32_modules.py
@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A 16-bit save of a non-PEFT model must keep what transformers loads in fp32
+(`_keep_in_fp32_modules_strict`, e.g. the MoE router's `e_score_correction_bias`): a bf16 copy
+moves the expert-selection scores by up to 0.03 and changes which experts fire. Sliced with
+`ast`: importing `unsloth.save` needs a GPU."""
+
+import ast
+import re
+from pathlib import Path
+
+import pytest
+import torch
+
+SAVE_PY = Path(__file__).resolve().parents[1] / "unsloth" / "save.py"
+
+
[email protected](scope = "module")
+def helpers():
+ tree = ast.parse(SAVE_PY.read_text(encoding = "utf-8"))
+ wanted = {"_cast_state_dict_16bit"}
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in wanted]
+ assert {n.name for n in nodes} == wanted
+ namespace = {"re": re, "torch": torch}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), "<save>", "exec"), namespace)
+ return namespace
+
+
+def _tiny_glm_lite():
+ pytest.importorskip("transformers.models.glm4_moe_lite") # transformers >= 5.3
+ from transformers import AutoConfig, AutoModelForCausalLM
+
+ config = AutoConfig.for_model(
+ "glm4_moe_lite",
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ moe_intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ n_routed_experts = 4,
+ n_shared_experts = 1,
+ num_experts_per_tok = 2,
+ first_k_dense_replace = 1,
+ kv_lora_rank = 16,
+ q_lora_rank = 16,
+ qk_rope_head_dim = 8,
+ qk_nope_head_dim = 8,
+ v_head_dim = 8,
+ n_group = 1,
+ topk_group = 1,
+ )
+ return AutoModelForCausalLM.from_config(config, dtype = torch.bfloat16)
+
+
[email protected]("target", [torch.bfloat16, torch.float16])
+def test_router_bias_stays_fp32(helpers, target):
+ model = _tiny_glm_lite()
+ state_dict = model.state_dict()
+ biases = [k for k in state_dict if k.endswith("e_score_correction_bias")]
+ assert biases and all(state_dict[k].dtype == torch.float32 for k in biases)
+ with torch.no_grad():
+ for k in biases:
+ state_dict[k].copy_(torch.linspace(-1, 1, state_dict[k].numel()) / 3)
+ out = helpers["_cast_state_dict_16bit"](model, state_dict, target)
+ for k in biases:
+ assert out[k].dtype == torch.float32 and torch.equal(out[k], state_dict[k])
+ others = [k for k, v in out.items() if v.is_floating_point() and k not in biases]
+ assert others and all(out[k].dtype == target for k in others)
+
+
[email protected]("target", [torch.bfloat16, torch.float16])
+def test_strict_list_from_a_submodule_is_honoured(helpers, target):
+ """Same rule on every transformers version: a strict name declared on a nested submodule."""
+
+ class Router(torch.nn.Module):
+ _keep_in_fp32_modules_strict = ["e_score_correction_bias"]
+
+ def __init__(self):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.randn(4, 8, dtype = torch.bfloat16))
+ self.register_buffer("e_score_correction_bias", torch.linspace(-1, 1, 4) / 3)
+
+ model = torch.nn.Sequential(torch.nn.Linear(8, 8).to(torch.bfloat16), Router())
+ state_dict = model.state_dict()
+ out = helpers["_cast_state_dict_16bit"](model, state_dict, target)
+ assert out["1.e_score_correction_bias"].dtype == torch.float32
+ assert torch.equal(out["1.e_score_correction_bias"], state_dict["1.e_score_correction_bias"])
+ assert out["1.weight"].dtype == target and out["0.weight"].dtype == target
+
+
+def test_models_without_fp32_modules_cast_everything(helpers):
+ module = torch.nn.Linear(4, 4)
+ out = helpers["_cast_state_dict_16bit"](module, module.state_dict(), torch.bfloat16)
+ assert all(v.dtype == torch.bfloat16 for v in out.values())
+
+
+def test_non_strict_list_only_for_fp16(helpers):
+ class Model(torch.nn.Module):
+ _keep_in_fp32_modules = ["wo"]
+
+ def __init__(self):
+ super().__init__()
+ self.wo = torch.nn.Linear(2, 2)
+ self.wi = torch.nn.Linear(2, 2)
+ self.word_embeddings = torch.nn.Embedding(3, 2)
+
+ model = Model()
+ bf16 = helpers["_cast_state_dict_16bit"](model, model.state_dict(), torch.bfloat16)
+ fp16 = helpers["_cast_state_dict_16bit"](model, model.state_dict(), torch.float16)
+ assert bf16["wo.weight"].dtype == torch.bfloat16
+ assert fp16["wo.weight"].dtype == torch.float32 and fp16["wi.weight"].dtype == torch.float16
+ # A name is a whole path component: "wo" does not keep "word_embeddings" in fp32.
+ assert fp16["word_embeddings.weight"].dtype == torch.float16
+
+
+def test_generic_save_casts_through_the_helper():
+ tree = ast.parse(SAVE_PY.read_text(encoding = "utf-8"))
+ fn = next(
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "unsloth_generic_save"
+ )
+ calls = {
+ c.func.id for c in ast.walk(fn) if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
+ }
+ assert "_cast_state_dict_16bit" in calls
+ decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in fn.decorator_list]
+ assert decorators == ["_normalize_tied_weights_keys_for_save", "inference_mode"]
diff --git a/unsloth/save.py b/unsloth/save.py
index c9badc77c76..c282bf21509 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -5616,6 +5616,28 @@ def _refuse_unsaveable_text_core(model, save_method):
)
+def _cast_state_dict_16bit(model, state_dict, target_dtype):
+ """Cast to `target_dtype`, except what transformers keeps in fp32 there (`_keep_in_fp32_modules[_strict]`)."""
+ names = set()
+ for module in model.modules() if hasattr(model, "modules") else ():
+ names.update(getattr(module, "_keep_in_fp32_modules_strict", None) or ())
+ if target_dtype == torch.float16:
+ names.update(getattr(module, "_keep_in_fp32_modules", None) or ())
+ # Whole dotted path components, as transformers matches them: "wo" is not "word_embeddings".
+ keep = (
+ re.compile("|".join(rf"(^|\.){re.escape(name)}($|\.)" for name in sorted(names)))
+ if names
+ else None
+ )
+ return {
+ k: v.to(dtype = target_dtype)
+ if v.is_floating_point()
+ and not (keep is not None and v.dtype == torch.float32 and keep.search(k))
+ else v
+ for k, v in state_dict.items()
+ }
+
+
@_normalize_tied_weights_keys_for_save
@torch.inference_mode
def unsloth_generic_save(
@@ -5703,10 +5725,7 @@ def unsloth_generic_save(
}
else:
_target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
- state_dict = {
- k: v.to(dtype = _target_dtype) if v.is_floating_point() else v
- for k, v in state_dict.items()
- }
+ state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)
if is_qwen3_5_vlm:
state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)
if state_dict is not None: