Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cast fp16 leftovers to the requested dtype after a text_only pre-quantized load (#11692)
Daniel Han23ab9ed2e7ae
diff --git a/tests/test_text_only_prequantized_dtype.py b/tests/test_text_only_prequantized_dtype.pynew file mode 100644index 00000000000..92ea7484481--- /dev/null+++ b/tests/test_text_only_prequantized_dtype.py@@ -0,0 +1,257 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Text-only loads of pre-quantized VLM repos must end up in the requested dtype (CPU only)."""++import ast+import os+import re+import types++import pytest++torch = pytest.importorskip("torch")+nn = torch.nn++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+UTILS = os.path.join(ROOT, "unsloth", "models", "_utils.py")+VISION = os.path.join(ROOT, "unsloth", "models", "vision.py")+HELPER = "_cast_text_only_prequantized_params"+++def _load_helper():+ src = open(UTILS, encoding = "utf-8").read()+ for node in ast.parse(src).body:+ if isinstance(node, ast.FunctionDef) and node.name == HELPER:+ ns = {"torch": torch, "re": re}+ exec(ast.get_source_segment(src, node), ns)+ return ns[HELPER]+ raise AssertionError(f"{HELPER} not found in unsloth/models/_utils.py")+++class _QuantStorage(nn.Parameter):+ """Stands in for bnb Params4bit."""+++class _TinyDecoder(nn.Module):+ def __init__(self, tie = False):+ super().__init__()+ self.embed_tokens = nn.Embedding(16, 8).half()+ self.in_proj_qkv = nn.Linear(8, 8, bias = False).half()+ self.A_log = nn.Parameter(torch.zeros(4, dtype = torch.float16))+ self.norm = nn.LayerNorm(8)+ self.quantized = nn.Linear(8, 8, bias = False)+ self.quantized.weight = _QuantStorage(+ torch.zeros(32, 1, dtype = torch.float16), requires_grad = False+ )+ self.lm_head = nn.Linear(8, 16, bias = False).to(torch.bfloat16)+ if tie:+ self.lm_head.weight = self.embed_tokens.weight+ self.hf_quantizer = types.SimpleNamespace(pre_quantized = True)++ def forward(self, ids):+ h = self.in_proj_qkv(self.embed_tokens(ids))+ return self.lm_head(h)+++def test_prequantized_float16_leftovers_follow_requested_dtype():+ cast = _load_helper()+ model = _TinyDecoder()+ ids = torch.tensor([[1, 2, 3]])+ with pytest.raises(RuntimeError):+ model(ids)+ n = cast(model, torch.bfloat16)+ assert n == 3+ assert model.embed_tokens.weight.dtype == torch.bfloat16+ assert model.in_proj_qkv.weight.dtype == torch.bfloat16+ assert model.A_log.dtype == torch.bfloat16+ assert model.norm.weight.dtype == torch.float32+ assert model.quantized.weight.dtype == torch.float16+ assert isinstance(model.quantized.weight, _QuantStorage)+ with torch.no_grad():+ assert model(ids).dtype == torch.bfloat16+++def test_float16_request_is_a_no_op_for_a_float16_checkpoint():+ cast = _load_helper()+ model = _TinyDecoder()+ model.lm_head.to(torch.float16)+ before = {k: (v.dtype, v.data_ptr()) for k, v in model.named_parameters()}+ assert cast(model, torch.float16) == 0+ assert before == {k: (v.dtype, v.data_ptr()) for k, v in model.named_parameters()}+++def test_bfloat16_checkpoint_on_a_float16_device_is_cast_down():+ cast = _load_helper()+ model = _TinyDecoder()+ model.embed_tokens.to(torch.bfloat16)+ cast(model, torch.float16)+ assert model.embed_tokens.weight.dtype == torch.float16+ assert model.lm_head.weight.dtype == torch.float16+++def test_tied_weights_stay_tied():+ cast = _load_helper()+ model = _TinyDecoder(tie = True)+ cast(model, torch.bfloat16)+ assert model.lm_head.weight is model.embed_tokens.weight+ assert model.embed_tokens.weight.dtype == torch.bfloat16+++def test_not_prequantized_or_unknown_dtype_is_left_alone():+ cast = _load_helper()+ model = _TinyDecoder()+ model.hf_quantizer = None+ assert cast(model, torch.bfloat16) == 0+ assert model.embed_tokens.weight.dtype == torch.float16+ model.hf_quantizer = types.SimpleNamespace(pre_quantized = False)+ assert cast(model, torch.bfloat16) == 0+ model.hf_quantizer = types.SimpleNamespace(pre_quantized = True)+ assert cast(model, None) == 0+ assert cast(model, "auto") == 0+ assert model.embed_tokens.weight.dtype == torch.float16+++def test_keep_in_fp32_plan_is_honoured():+ cast = _load_helper()+ model = _TinyDecoder()+ model._get_dtype_plan = lambda dtype: {"A_log": torch.float32}+ cast(model, torch.bfloat16)+ assert model.A_log.dtype == torch.float32+ assert model.embed_tokens.weight.dtype == torch.bfloat16+++def test_real_bnb_params4bit_is_untouched():+ bnb = pytest.importorskip("bitsandbytes")+ cast = _load_helper()+ model = _TinyDecoder()+ storage = torch.zeros(32, 1, dtype = torch.bfloat16)+ model.quantized.weight = bnb.nn.Params4bit(storage, requires_grad = False)+ cast(model, torch.float16)+ assert type(model.quantized.weight) is bnb.nn.Params4bit+ assert model.quantized.weight.dtype == torch.bfloat16+++def _fast_base_from_pretrained():+ tree = ast.parse(open(VISION, encoding = "utf-8").read())+ for node in tree.body:+ if isinstance(node, ast.ClassDef) and node.name == "FastBaseModel":+ for item in node.body:+ if isinstance(item, ast.FunctionDef) and item.name == "from_pretrained":+ return item+ raise AssertionError("FastBaseModel.from_pretrained not found")+++def test_vision_loader_casts_right_after_the_text_only_load():+ fn = _fast_base_from_pretrained()+ load_line = None+ for node in ast.walk(fn):+ if (+ isinstance(node, ast.Assign)+ and isinstance(node.value, ast.Call)+ and isinstance(node.value.func, ast.Attribute)+ and node.value.func.attr == "from_pretrained"+ and isinstance(node.value.func.value, ast.Name)+ and node.value.func.value.id == "auto_model"+ ):+ load_line = node.lineno+ assert load_line is not None+ calls = []+ for node in ast.walk(fn):+ if not isinstance(node, ast.If):+ continue+ test_names = {n.id for n in ast.walk(node.test) if isinstance(n, ast.Name)}+ if "text_only_decoder" not in test_names:+ continue+ for call in ast.walk(node):+ if (+ isinstance(call, ast.Call)+ and isinstance(call.func, ast.Name)+ and call.func.id == HELPER+ ):+ args = [a.id for a in call.args if isinstance(a, ast.Name)]+ calls.append((call.lineno, args))+ assert calls, f"{HELPER} is not called under a text_only_decoder guard"+ line, args = min(calls)+ assert line > load_line+ assert args == ["model", "torch_dtype"]+ src_lines = open(VISION, encoding = "utf-8").read().splitlines()+ tail = "\n".join(src_lines[load_line:line])+ assert "_attach_bnb_multidevice_hooks(" not in tail+ assert 'embed_tokens.to("cpu")' not in tail+++ "requested, resolved",+ [(torch.bfloat16, torch.float16), (torch.float16, torch.bfloat16)],+ ids = ["awq_bf16_to_fp16", "fbgemm_fp8_fp16_to_bf16"],+)+def test_quantizer_resolved_dtype_wins_over_the_request(requested, resolved):+ cast = _load_helper()+ model = _TinyDecoder()+ model.embed_tokens.to(torch.bfloat16 if resolved == torch.float16 else torch.float16)+ model.lm_head.to(resolved)+ model.config = types.SimpleNamespace(dtype = resolved)+ cast(model, requested)+ assert model.embed_tokens.weight.dtype == resolved+ assert model.in_proj_qkv.weight.dtype == resolved+ assert model.lm_head.weight.dtype == resolved+ assert model.quantized.weight.dtype == torch.float16++[email protected]("offload", ["cpu", "disk"])+def test_offloaded_leftovers_materialize_in_the_requested_dtype(offload, tmp_path):+ accelerate = pytest.importorskip("accelerate")+ cast = _load_helper()+ model = _TinyDecoder()+ model.quantized = nn.Identity()+ model.lm_head.to(torch.float32)+ model.norm.to(torch.float16)+ ids = torch.tensor([[1, 2, 3]])+ if offload == "cpu":+ accelerate.cpu_offload(model, execution_device = torch.device("cpu"))+ else:+ accelerate.disk_offload(+ model, offload_dir = str(tmp_path), execution_device = torch.device("cpu")+ )+ assert model.embed_tokens.weight.device.type == "meta"+ assert model.embed_tokens.weight.dtype == torch.float16+ cast(model, torch.float32)+ assert model.embed_tokens.weight.dtype == torch.float32+ assert model.in_proj_qkv.weight.dtype == torch.float32+ with torch.no_grad():+ out = model(ids)+ assert out.dtype == torch.float32+++class _EetqLikeLinear(nn.Module):+ """Like transformers' EetqLinear: int8 weight, float16 weight_scales / bias."""++ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.zeros(8, 8, dtype = torch.int8), requires_grad = False)+ self.weight_scales = nn.Parameter(torch.ones(8, dtype = torch.float16))+ self.bias = nn.Parameter(torch.zeros(8, dtype = torch.float16))+++class _Fp8LikeLinear(nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.zeros(8, 8).to(torch.float8_e4m3fn), requires_grad = False)+ self.bias = nn.Parameter(torch.zeros(8, dtype = torch.float16))+++def test_quantized_modules_keep_their_own_float_parameters():+ cast = _load_helper()+ model = _TinyDecoder()+ model.eetq = _EetqLikeLinear()+ model.fp8 = _Fp8LikeLinear()+ model.quantized.bias = nn.Parameter(torch.zeros(32, dtype = torch.float16))+ cast(model, torch.bfloat16)+ assert model.eetq.weight_scales.dtype == torch.float16+ assert model.eetq.bias.dtype == torch.float16+ assert model.eetq.weight.dtype == torch.int8+ assert model.fp8.bias.dtype == torch.float16+ assert model.quantized.bias.dtype == torch.float16+ assert model.embed_tokens.weight.dtype == torch.bfloat16+ assert model.A_log.dtype == torch.bfloat16diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 8c7351e92cd..ed9e6d4dd2e 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1206,6 +1206,49 @@ def _apply_text_only_key_mapping(kwargs, parent_config, text_config):kwargs["key_mapping"] = {**mapping, **user_mapping} if user_mapping else mapping+def _cast_text_only_prequantized_params(model, dtype):+ # transformers>=5 keeps the checkpoint dtype for key_mapping-renamed keys on pre-quantized loads.+ if dtype not in (torch.float16, torch.bfloat16, torch.float32):+ return 0+ quantizer = getattr(model, "hf_quantizer", None)+ if quantizer is None or not getattr(quantizer, "pre_quantized", False):+ return 0+ # Quantizer may override the request (AWQ, FBGEMM FP8); config.dtype holds the result.+ resolved = getattr(getattr(model, "config", None), "dtype", None)+ if resolved in (torch.float16, torch.bfloat16, torch.float32):+ dtype = resolved+ keep_fp32 = []+ get_dtype_plan = getattr(model, "_get_dtype_plan", None)+ if callable(get_dtype_plan):+ try:+ keep_fp32 = [k for k, v in get_dtype_plan(dtype).items() if v == torch.float32]+ except Exception:+ keep_fp32 = []++ def _is_quantized_storage(t):+ if type(t) is not torch.nn.Parameter or type(t.data) is not torch.Tensor:+ return True+ return not t.dtype.is_floating_point or t.dtype.itemsize == 1++ n_cast = 0+ for module_name, module in model.named_modules():+ own = list(module.named_parameters(recurse = False))+ # Quantized modules' fp16 scales / bias are the kernel's contract (EETQ), not leftovers.+ if any(_is_quantized_storage(t) for _, t in own):+ continue+ for param_name, param in own:+ name = f"{module_name}.{param_name}" if module_name else param_name+ # accelerate casts offloaded weights to the meta placeholder's dtype, so recasting it suffices.+ if param.dtype not in (torch.float16, torch.bfloat16):+ continue+ target = torch.float32 if any(re.search(k, name) for k in keep_fp32) else dtype+ if param.dtype == target:+ continue+ param.data = param.data.to(target)+ n_cast += 1+ return n_cast++def resolve_attention_implementation(model_class,config,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex ee8146ee50f..e9821b73e5b 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -79,6 +79,7 @@ from ._utils import (_config_get,_is_flash_attention_requested,_apply_text_only_key_mapping,+ _cast_text_only_prequantized_params,_select_moe_detection_targets,set_task_config_attr,)@@ -1755,6 +1756,9 @@ class FastBaseModel:trust_remote_code = trust_remote_code,**kwargs,)+ if text_only_decoder:+ # Must run before offload / hooks capture the weights.+ _cast_text_only_prequantized_params(model, torch_dtype)# transformers 5 leaves remote code's non-persistent buffers (RoPE inv_freq, decay slopes) uninitialised.restore_remote_code_non_persistent_buffers(model)# Must precede _attach_bnb_multidevice_hooks: it returns early while offload_embedding is True.