Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cast fp16 leftovers to the requested dtype after a text_only pre-quantized load (#11692)

Daniel Han23ab9ed2e7ae
diff --git a/tests/test_text_only_prequantized_dtype.py b/tests/test_text_only_prequantized_dtype.py
new file mode 100644
index 00000000000..92ea7484481
--- /dev/null
+++ b/tests/test_text_only_prequantized_dtype.py
@@ -0,0 +1,257 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Text-only loads of pre-quantized VLM repos must end up in the requested dtype (CPU only)."""
+
+import ast
+import os
+import re
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+nn = torch.nn
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+UTILS = os.path.join(ROOT, "unsloth", "models", "_utils.py")
+VISION = os.path.join(ROOT, "unsloth", "models", "vision.py")
+HELPER = "_cast_text_only_prequantized_params"
+
+
+def _load_helper():
+ src = open(UTILS, encoding = "utf-8").read()
+ for node in ast.parse(src).body:
+ if isinstance(node, ast.FunctionDef) and node.name == HELPER:
+ ns = {"torch": torch, "re": re}
+ exec(ast.get_source_segment(src, node), ns)
+ return ns[HELPER]
+ raise AssertionError(f"{HELPER} not found in unsloth/models/_utils.py")
+
+
+class _QuantStorage(nn.Parameter):
+ """Stands in for bnb Params4bit."""
+
+
+class _TinyDecoder(nn.Module):
+ def __init__(self, tie = False):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(16, 8).half()
+ self.in_proj_qkv = nn.Linear(8, 8, bias = False).half()
+ self.A_log = nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ self.norm = nn.LayerNorm(8)
+ self.quantized = nn.Linear(8, 8, bias = False)
+ self.quantized.weight = _QuantStorage(
+ torch.zeros(32, 1, dtype = torch.float16), requires_grad = False
+ )
+ self.lm_head = nn.Linear(8, 16, bias = False).to(torch.bfloat16)
+ if tie:
+ self.lm_head.weight = self.embed_tokens.weight
+ self.hf_quantizer = types.SimpleNamespace(pre_quantized = True)
+
+ def forward(self, ids):
+ h = self.in_proj_qkv(self.embed_tokens(ids))
+ return self.lm_head(h)
+
+
+def test_prequantized_float16_leftovers_follow_requested_dtype():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ ids = torch.tensor([[1, 2, 3]])
+ with pytest.raises(RuntimeError):
+ model(ids)
+ n = cast(model, torch.bfloat16)
+ assert n == 3
+ assert model.embed_tokens.weight.dtype == torch.bfloat16
+ assert model.in_proj_qkv.weight.dtype == torch.bfloat16
+ assert model.A_log.dtype == torch.bfloat16
+ assert model.norm.weight.dtype == torch.float32
+ assert model.quantized.weight.dtype == torch.float16
+ assert isinstance(model.quantized.weight, _QuantStorage)
+ with torch.no_grad():
+ assert model(ids).dtype == torch.bfloat16
+
+
+def test_float16_request_is_a_no_op_for_a_float16_checkpoint():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.lm_head.to(torch.float16)
+ before = {k: (v.dtype, v.data_ptr()) for k, v in model.named_parameters()}
+ assert cast(model, torch.float16) == 0
+ assert before == {k: (v.dtype, v.data_ptr()) for k, v in model.named_parameters()}
+
+
+def test_bfloat16_checkpoint_on_a_float16_device_is_cast_down():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.embed_tokens.to(torch.bfloat16)
+ cast(model, torch.float16)
+ assert model.embed_tokens.weight.dtype == torch.float16
+ assert model.lm_head.weight.dtype == torch.float16
+
+
+def test_tied_weights_stay_tied():
+ cast = _load_helper()
+ model = _TinyDecoder(tie = True)
+ cast(model, torch.bfloat16)
+ assert model.lm_head.weight is model.embed_tokens.weight
+ assert model.embed_tokens.weight.dtype == torch.bfloat16
+
+
+def test_not_prequantized_or_unknown_dtype_is_left_alone():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.hf_quantizer = None
+ assert cast(model, torch.bfloat16) == 0
+ assert model.embed_tokens.weight.dtype == torch.float16
+ model.hf_quantizer = types.SimpleNamespace(pre_quantized = False)
+ assert cast(model, torch.bfloat16) == 0
+ model.hf_quantizer = types.SimpleNamespace(pre_quantized = True)
+ assert cast(model, None) == 0
+ assert cast(model, "auto") == 0
+ assert model.embed_tokens.weight.dtype == torch.float16
+
+
+def test_keep_in_fp32_plan_is_honoured():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model._get_dtype_plan = lambda dtype: {"A_log": torch.float32}
+ cast(model, torch.bfloat16)
+ assert model.A_log.dtype == torch.float32
+ assert model.embed_tokens.weight.dtype == torch.bfloat16
+
+
+def test_real_bnb_params4bit_is_untouched():
+ bnb = pytest.importorskip("bitsandbytes")
+ cast = _load_helper()
+ model = _TinyDecoder()
+ storage = torch.zeros(32, 1, dtype = torch.bfloat16)
+ model.quantized.weight = bnb.nn.Params4bit(storage, requires_grad = False)
+ cast(model, torch.float16)
+ assert type(model.quantized.weight) is bnb.nn.Params4bit
+ assert model.quantized.weight.dtype == torch.bfloat16
+
+
+def _fast_base_from_pretrained():
+ tree = ast.parse(open(VISION, encoding = "utf-8").read())
+ for node in tree.body:
+ if isinstance(node, ast.ClassDef) and node.name == "FastBaseModel":
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef) and item.name == "from_pretrained":
+ return item
+ raise AssertionError("FastBaseModel.from_pretrained not found")
+
+
+def test_vision_loader_casts_right_after_the_text_only_load():
+ fn = _fast_base_from_pretrained()
+ load_line = None
+ for node in ast.walk(fn):
+ if (
+ isinstance(node, ast.Assign)
+ and isinstance(node.value, ast.Call)
+ and isinstance(node.value.func, ast.Attribute)
+ and node.value.func.attr == "from_pretrained"
+ and isinstance(node.value.func.value, ast.Name)
+ and node.value.func.value.id == "auto_model"
+ ):
+ load_line = node.lineno
+ assert load_line is not None
+ calls = []
+ for node in ast.walk(fn):
+ if not isinstance(node, ast.If):
+ continue
+ test_names = {n.id for n in ast.walk(node.test) if isinstance(n, ast.Name)}
+ if "text_only_decoder" not in test_names:
+ continue
+ for call in ast.walk(node):
+ if (
+ isinstance(call, ast.Call)
+ and isinstance(call.func, ast.Name)
+ and call.func.id == HELPER
+ ):
+ args = [a.id for a in call.args if isinstance(a, ast.Name)]
+ calls.append((call.lineno, args))
+ assert calls, f"{HELPER} is not called under a text_only_decoder guard"
+ line, args = min(calls)
+ assert line > load_line
+ assert args == ["model", "torch_dtype"]
+ src_lines = open(VISION, encoding = "utf-8").read().splitlines()
+ tail = "\n".join(src_lines[load_line:line])
+ assert "_attach_bnb_multidevice_hooks(" not in tail
+ assert 'embed_tokens.to("cpu")' not in tail
+
+
+ "requested, resolved",
+ [(torch.bfloat16, torch.float16), (torch.float16, torch.bfloat16)],
+ ids = ["awq_bf16_to_fp16", "fbgemm_fp8_fp16_to_bf16"],
+)
+def test_quantizer_resolved_dtype_wins_over_the_request(requested, resolved):
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.embed_tokens.to(torch.bfloat16 if resolved == torch.float16 else torch.float16)
+ model.lm_head.to(resolved)
+ model.config = types.SimpleNamespace(dtype = resolved)
+ cast(model, requested)
+ assert model.embed_tokens.weight.dtype == resolved
+ assert model.in_proj_qkv.weight.dtype == resolved
+ assert model.lm_head.weight.dtype == resolved
+ assert model.quantized.weight.dtype == torch.float16
+
+
[email protected]("offload", ["cpu", "disk"])
+def test_offloaded_leftovers_materialize_in_the_requested_dtype(offload, tmp_path):
+ accelerate = pytest.importorskip("accelerate")
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.quantized = nn.Identity()
+ model.lm_head.to(torch.float32)
+ model.norm.to(torch.float16)
+ ids = torch.tensor([[1, 2, 3]])
+ if offload == "cpu":
+ accelerate.cpu_offload(model, execution_device = torch.device("cpu"))
+ else:
+ accelerate.disk_offload(
+ model, offload_dir = str(tmp_path), execution_device = torch.device("cpu")
+ )
+ assert model.embed_tokens.weight.device.type == "meta"
+ assert model.embed_tokens.weight.dtype == torch.float16
+ cast(model, torch.float32)
+ assert model.embed_tokens.weight.dtype == torch.float32
+ assert model.in_proj_qkv.weight.dtype == torch.float32
+ with torch.no_grad():
+ out = model(ids)
+ assert out.dtype == torch.float32
+
+
+class _EetqLikeLinear(nn.Module):
+ """Like transformers' EetqLinear: int8 weight, float16 weight_scales / bias."""
+
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.zeros(8, 8, dtype = torch.int8), requires_grad = False)
+ self.weight_scales = nn.Parameter(torch.ones(8, dtype = torch.float16))
+ self.bias = nn.Parameter(torch.zeros(8, dtype = torch.float16))
+
+
+class _Fp8LikeLinear(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.zeros(8, 8).to(torch.float8_e4m3fn), requires_grad = False)
+ self.bias = nn.Parameter(torch.zeros(8, dtype = torch.float16))
+
+
+def test_quantized_modules_keep_their_own_float_parameters():
+ cast = _load_helper()
+ model = _TinyDecoder()
+ model.eetq = _EetqLikeLinear()
+ model.fp8 = _Fp8LikeLinear()
+ model.quantized.bias = nn.Parameter(torch.zeros(32, dtype = torch.float16))
+ cast(model, torch.bfloat16)
+ assert model.eetq.weight_scales.dtype == torch.float16
+ assert model.eetq.bias.dtype == torch.float16
+ assert model.eetq.weight.dtype == torch.int8
+ assert model.fp8.bias.dtype == torch.float16
+ assert model.quantized.bias.dtype == torch.float16
+ assert model.embed_tokens.weight.dtype == torch.bfloat16
+ assert model.A_log.dtype == torch.bfloat16
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 8c7351e92cd..ed9e6d4dd2e 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1206,6 +1206,49 @@ def _apply_text_only_key_mapping(kwargs, parent_config, text_config):
kwargs["key_mapping"] = {**mapping, **user_mapping} if user_mapping else mapping
+def _cast_text_only_prequantized_params(model, dtype):
+ # transformers>=5 keeps the checkpoint dtype for key_mapping-renamed keys on pre-quantized loads.
+ if dtype not in (torch.float16, torch.bfloat16, torch.float32):
+ return 0
+ quantizer = getattr(model, "hf_quantizer", None)
+ if quantizer is None or not getattr(quantizer, "pre_quantized", False):
+ return 0
+ # Quantizer may override the request (AWQ, FBGEMM FP8); config.dtype holds the result.
+ resolved = getattr(getattr(model, "config", None), "dtype", None)
+ if resolved in (torch.float16, torch.bfloat16, torch.float32):
+ dtype = resolved
+ keep_fp32 = []
+ get_dtype_plan = getattr(model, "_get_dtype_plan", None)
+ if callable(get_dtype_plan):
+ try:
+ keep_fp32 = [k for k, v in get_dtype_plan(dtype).items() if v == torch.float32]
+ except Exception:
+ keep_fp32 = []
+
+ def _is_quantized_storage(t):
+ if type(t) is not torch.nn.Parameter or type(t.data) is not torch.Tensor:
+ return True
+ return not t.dtype.is_floating_point or t.dtype.itemsize == 1
+
+ n_cast = 0
+ for module_name, module in model.named_modules():
+ own = list(module.named_parameters(recurse = False))
+ # Quantized modules' fp16 scales / bias are the kernel's contract (EETQ), not leftovers.
+ if any(_is_quantized_storage(t) for _, t in own):
+ continue
+ for param_name, param in own:
+ name = f"{module_name}.{param_name}" if module_name else param_name
+ # accelerate casts offloaded weights to the meta placeholder's dtype, so recasting it suffices.
+ if param.dtype not in (torch.float16, torch.bfloat16):
+ continue
+ target = torch.float32 if any(re.search(k, name) for k in keep_fp32) else dtype
+ if param.dtype == target:
+ continue
+ param.data = param.data.to(target)
+ n_cast += 1
+ return n_cast
+
+
def resolve_attention_implementation(
model_class,
config,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index ee8146ee50f..e9821b73e5b 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -79,6 +79,7 @@ from ._utils import (
_config_get,
_is_flash_attention_requested,
_apply_text_only_key_mapping,
+ _cast_text_only_prequantized_params,
_select_moe_detection_targets,
set_task_config_attr,
)
@@ -1755,6 +1756,9 @@ class FastBaseModel:
trust_remote_code = trust_remote_code,
**kwargs,
)
+ if text_only_decoder:
+ # Must run before offload / hooks capture the weights.
+ _cast_text_only_prequantized_params(model, torch_dtype)
# transformers 5 leaves remote code's non-persistent buffers (RoPE inv_freq, decay slopes) uninitialised.
restore_remote_code_non_persistent_buffers(model)
# Must precede _attach_bnb_multidevice_hooks: it returns early while offload_embedding is True.