Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep fp32-only tensors on 16-bit merged saves (#13819)
Daniel Han699f44630a00
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.pyindex 063c0c64ec0..4816839e5b8 100644--- a/tests/saving/test_gguf_disk_preflight.py+++ b/tests/saving/test_gguf_disk_preflight.py@@ -1500,7 +1500,8 @@ class TestASuppliedDictIsWhatASixteenBitSaveWrites:generic = inspect.getsource(S.unsloth_generic_save)assert '("16bit" in save_method or is_qwen3_5_vlm) and state_dict is None' in generic- assert "v.to(dtype = _target_dtype) if v.is_floating_point() else v" in generic+ assert "state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)" in generic+ # The other writer overwrites the caller's dictionary with its own.assert "state_dict = OrderedDict()" in inspect.getsource(S.unsloth_save_model)diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex 9f50ae1a825..f401126abce 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -193,6 +193,7 @@ _GENERIC_SAVE_REAL_HELPERS = ("_is_adapter_save_method","_honours_safe_serialization","_refuse_unsaveable_text_core",+ "_cast_state_dict_16bit","_requests_float32","_stamp_saved_config_float32",)diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex 0fc749c3b53..d3381bd32e9 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -187,6 +187,7 @@ def saving(monkeypatch, tmp_path):"_honours_safe_serialization","_assert_export_target_is_not_base_with_lora_layers","_refuse_unsaveable_text_core",+ "_cast_state_dict_16bit","_requests_float32",}nodes = []diff --git a/tests/test_save_keep_fp32_modules.py b/tests/test_save_keep_fp32_modules.pynew file mode 100644index 00000000000..23ac8458a54--- /dev/null+++ b/tests/test_save_keep_fp32_modules.py@@ -0,0 +1,129 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A 16-bit save of a non-PEFT model must keep what transformers loads in fp32+(`_keep_in_fp32_modules_strict`, e.g. the MoE router's `e_score_correction_bias`): a bf16 copy+moves the expert-selection scores by up to 0.03 and changes which experts fire. Sliced with+`ast`: importing `unsloth.save` needs a GPU."""++import ast+import re+from pathlib import Path++import pytest+import torch++SAVE_PY = Path(__file__).resolve().parents[1] / "unsloth" / "save.py"++[email protected](scope = "module")+def helpers():+ tree = ast.parse(SAVE_PY.read_text(encoding = "utf-8"))+ wanted = {"_cast_state_dict_16bit"}+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in wanted]+ assert {n.name for n in nodes} == wanted+ namespace = {"re": re, "torch": torch}+ exec(compile(ast.Module(body = nodes, type_ignores = []), "<save>", "exec"), namespace)+ return namespace+++def _tiny_glm_lite():+ pytest.importorskip("transformers.models.glm4_moe_lite") # transformers >= 5.3+ from transformers import AutoConfig, AutoModelForCausalLM++ config = AutoConfig.for_model(+ "glm4_moe_lite",+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ moe_intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ n_routed_experts = 4,+ n_shared_experts = 1,+ num_experts_per_tok = 2,+ first_k_dense_replace = 1,+ kv_lora_rank = 16,+ q_lora_rank = 16,+ qk_rope_head_dim = 8,+ qk_nope_head_dim = 8,+ v_head_dim = 8,+ n_group = 1,+ topk_group = 1,+ )+ return AutoModelForCausalLM.from_config(config, dtype = torch.bfloat16)++[email protected]("target", [torch.bfloat16, torch.float16])+def test_router_bias_stays_fp32(helpers, target):+ model = _tiny_glm_lite()+ state_dict = model.state_dict()+ biases = [k for k in state_dict if k.endswith("e_score_correction_bias")]+ assert biases and all(state_dict[k].dtype == torch.float32 for k in biases)+ with torch.no_grad():+ for k in biases:+ state_dict[k].copy_(torch.linspace(-1, 1, state_dict[k].numel()) / 3)+ out = helpers["_cast_state_dict_16bit"](model, state_dict, target)+ for k in biases:+ assert out[k].dtype == torch.float32 and torch.equal(out[k], state_dict[k])+ others = [k for k, v in out.items() if v.is_floating_point() and k not in biases]+ assert others and all(out[k].dtype == target for k in others)++[email protected]("target", [torch.bfloat16, torch.float16])+def test_strict_list_from_a_submodule_is_honoured(helpers, target):+ """Same rule on every transformers version: a strict name declared on a nested submodule."""++ class Router(torch.nn.Module):+ _keep_in_fp32_modules_strict = ["e_score_correction_bias"]++ def __init__(self):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.randn(4, 8, dtype = torch.bfloat16))+ self.register_buffer("e_score_correction_bias", torch.linspace(-1, 1, 4) / 3)++ model = torch.nn.Sequential(torch.nn.Linear(8, 8).to(torch.bfloat16), Router())+ state_dict = model.state_dict()+ out = helpers["_cast_state_dict_16bit"](model, state_dict, target)+ assert out["1.e_score_correction_bias"].dtype == torch.float32+ assert torch.equal(out["1.e_score_correction_bias"], state_dict["1.e_score_correction_bias"])+ assert out["1.weight"].dtype == target and out["0.weight"].dtype == target+++def test_models_without_fp32_modules_cast_everything(helpers):+ module = torch.nn.Linear(4, 4)+ out = helpers["_cast_state_dict_16bit"](module, module.state_dict(), torch.bfloat16)+ assert all(v.dtype == torch.bfloat16 for v in out.values())+++def test_non_strict_list_only_for_fp16(helpers):+ class Model(torch.nn.Module):+ _keep_in_fp32_modules = ["wo"]++ def __init__(self):+ super().__init__()+ self.wo = torch.nn.Linear(2, 2)+ self.wi = torch.nn.Linear(2, 2)+ self.word_embeddings = torch.nn.Embedding(3, 2)++ model = Model()+ bf16 = helpers["_cast_state_dict_16bit"](model, model.state_dict(), torch.bfloat16)+ fp16 = helpers["_cast_state_dict_16bit"](model, model.state_dict(), torch.float16)+ assert bf16["wo.weight"].dtype == torch.bfloat16+ assert fp16["wo.weight"].dtype == torch.float32 and fp16["wi.weight"].dtype == torch.float16+ # A name is a whole path component: "wo" does not keep "word_embeddings" in fp32.+ assert fp16["word_embeddings.weight"].dtype == torch.float16+++def test_generic_save_casts_through_the_helper():+ tree = ast.parse(SAVE_PY.read_text(encoding = "utf-8"))+ fn = next(+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "unsloth_generic_save"+ )+ calls = {+ c.func.id for c in ast.walk(fn) if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)+ }+ assert "_cast_state_dict_16bit" in calls+ decorators = [getattr(d, "id", getattr(d, "attr", None)) for d in fn.decorator_list]+ assert decorators == ["_normalize_tied_weights_keys_for_save", "inference_mode"]diff --git a/unsloth/save.py b/unsloth/save.pyindex c9badc77c76..c282bf21509 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -5616,6 +5616,28 @@ def _refuse_unsaveable_text_core(model, save_method):)+def _cast_state_dict_16bit(model, state_dict, target_dtype):+ """Cast to `target_dtype`, except what transformers keeps in fp32 there (`_keep_in_fp32_modules[_strict]`)."""+ names = set()+ for module in model.modules() if hasattr(model, "modules") else ():+ names.update(getattr(module, "_keep_in_fp32_modules_strict", None) or ())+ if target_dtype == torch.float16:+ names.update(getattr(module, "_keep_in_fp32_modules", None) or ())+ # Whole dotted path components, as transformers matches them: "wo" is not "word_embeddings".+ keep = (+ re.compile("|".join(rf"(^|\.){re.escape(name)}($|\.)" for name in sorted(names)))+ if names+ else None+ )+ return {+ k: v.to(dtype = target_dtype)+ if v.is_floating_point()+ and not (keep is not None and v.dtype == torch.float32 and keep.search(k))+ else v+ for k, v in state_dict.items()+ }++@_normalize_tied_weights_keys_for_save@torch.inference_modedef unsloth_generic_save(@@ -5703,10 +5725,7 @@ def unsloth_generic_save(}else:_target_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16- state_dict = {- k: v.to(dtype = _target_dtype) if v.is_floating_point() else v- for k, v in state_dict.items()- }+ state_dict = _cast_state_dict_16bit(model, state_dict, _target_dtype)if is_qwen3_5_vlm:state_dict = _qwen3_5_vlm_state_dict_for_save(state_dict)if state_dict is not None: