Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix LM-head weight fake quantization in fused CE (#14006)
cherry77-cloud5f8a2ec7f76d
diff --git a/tests/test_qat_lm_head_fused_ce.py b/tests/test_qat_lm_head_fused_ce.pynew file mode 100644index 00000000000..06c3641f9a3--- /dev/null+++ b/tests/test_qat_lm_head_fused_ce.py@@ -0,0 +1,138 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Fused CE reads lm_head.weight directly, so it must apply the same TorchAO QAT fake quantization as+lm_head.forward(): the activation fake quantizer on the hidden states and the weight fake quantizer on+the weight. NVFP4 / MX QAT linears fake-quantize inside forward() and must take the unfused path."""++import inspect++import pytest+import torch++pytest.importorskip("torchao.quantization.qat")++import unsloth # noqa: F401+import unsloth.models.llama as llama+import unsloth.models.mistral as mistral+from unsloth.models._utils import TorchAOConfig, _prepare_model_for_qat++DIM = 128+VOCAB = 96+++class _Head(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lm_head = torch.nn.Linear(DIM, VOCAB, bias = False)+++def _base_config(scheme):+ from torchao.quantization import (+ Int4WeightOnlyConfig,+ Int8DynamicActivationIntxWeightConfig,+ IntxWeightOnlyConfig,+ )+ from torchao.quantization.granularity import PerAxis, PerGroup++ if scheme == "int8":+ return IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0))+ if scheme == "int4":+ return Int4WeightOnlyConfig(group_size = DIM)+ if scheme == "int8-int4":+ return Int8DynamicActivationIntxWeightConfig(+ weight_dtype = torch.int4, weight_granularity = PerGroup(32)+ )+ mx = pytest.importorskip("torchao.prototype.mx_formats")+ if scheme == "nvfp4":+ return mx.NVFP4DynamicActivationNVFP4WeightConfig()+ return mx.MXDynamicActivationMXWeightConfig(+ activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn+ )+++def _qat_head(scheme):+ torch.manual_seed(0)+ model = _Head()+ _prepare_model_for_qat(+ model,+ TorchAOConfig(qat_scheme = None, base_config_and_filter_fns = [(_base_config(scheme), None)]),+ )+ return model.lm_head++[email protected]("scheme", ["int8", "int4", "int8-int4"])+def test_fused_inputs_match_lm_head_forward(scheme):+ lm_head = _qat_head(scheme)+ x = torch.randn(2, 5, DIM)+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)+ expected = lm_head(x)+ # Without the fake quantizers the plain weight gives different logits, so the check can fail.+ assert not torch.equal(torch.nn.functional.linear(x, lm_head.weight), expected)+ torch.testing.assert_close(+ torch.nn.functional.linear(hidden_states, weight), expected, rtol = 0, atol = 0+ )+ assert not llama.lm_head_needs_forward(lm_head)+++def test_activation_fake_quant_sees_head_dtype():+ # A trained lm_head is upcast to fp32 while fp16 hidden states arrive; forward() sees them cast.+ lm_head = _qat_head("int8-int4")+ x = torch.randn(2, 5, DIM).half()+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)+ assert hidden_states.dtype == torch.float32+ torch.testing.assert_close(+ torch.nn.functional.linear(hidden_states, weight), lm_head(x.float()), rtol = 0, atol = 0+ )+++def test_plain_lm_head_is_untouched():+ lm_head = torch.nn.Linear(DIM, VOCAB, bias = False)+ x = torch.randn(2, 5, DIM)+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)+ assert hidden_states is x and weight is lm_head.weight+ assert not llama.lm_head_needs_forward(lm_head)++[email protected]("scheme", ["nvfp4", "mx"])+def test_forward_fake_quant_lm_head_takes_unfused_path(scheme):+ lm_head = _qat_head(scheme)+ assert llama.lm_head_needs_forward(lm_head)++[email protected]("scheme", ["nvfp4", "mx"])+def test_forward_fake_quant_lm_head_in_modules_to_save_takes_unfused_path(scheme):+ peft = pytest.importorskip("peft")++ class _Model(_Head):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(DIM, DIM, bias = False)++ torch.manual_seed(0)+ model = peft.get_peft_model(+ _Model(), peft.LoraConfig(r = 16, target_modules = ["proj"], modules_to_save = ["lm_head"])+ )+ only_head_copy = (+ lambda m, fqn: isinstance(m, torch.nn.Linear) and "lm_head.modules_to_save" in fqn+ )+ _prepare_model_for_qat(+ model,+ TorchAOConfig(+ qat_scheme = None, base_config_and_filter_fns = [(_base_config(scheme), only_head_copy)]+ ),+ )+ lm_head = model.base_model.model.lm_head+ assert type(lm_head).__name__ == "ModulesToSaveWrapper"+ assert llama.lm_head_needs_forward(lm_head)+++ "forward",+ [llama.CausalLM_fast_forward, mistral.MistralForCausalLM_fast_forward],+ ids = ["llama", "mistral"],+)+def test_fused_ce_branches_fake_quantize_lm_head(forward):+ source = "".join(inspect.getsource(forward).split())+ assert "fake_quantize_lm_head_inputs(self.lm_head," in source+ assert "lm_head_needs_forward(self.lm_head)" in sourcediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 193cb1eee09..49ce312f5b5 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -224,6 +224,27 @@ def original_apply_qkv(self, X):return Q, K, V+def fake_quantize_lm_head_inputs(lm_head_module, hidden_states, weight):+ # Fused CE bypasses lm_head.forward(), where TorchAO QAT fake-quantizes both the input and the weight.+ activation_fake_quantizer = getattr(lm_head_module, "activation_fake_quantizer", None)+ if activation_fake_quantizer is not None:+ # The unfused path feeds lm_head hidden_states.to(weight dtype): a trained fp32 head on fp16 differs.+ hidden_states = activation_fake_quantizer(hidden_states.to(weight.dtype))+ weight_fake_quantizer = getattr(lm_head_module, "weight_fake_quantizer", None)+ if weight_fake_quantizer is not None:+ weight = weight_fake_quantizer(weight)+ return hidden_states, weight+++def lm_head_needs_forward(lm_head_module):+ # NVFP4 / MX QAT linears fake-quantize inside forward() with no quantizer children: fused CE cannot mirror them.+ # modules() also reaches the copy inside a PEFT modules_to_save wrapper, which isinstance alone misses.+ forward_fake_quant_linears = _fast_lora._forward_fake_quant_linears()+ return bool(forward_fake_quant_linears) and any(+ isinstance(module, forward_fake_quant_linears) for module in lm_head_module.modules()+ )++def original_apply_o(self, X):O = self.o_proj(X)return O@@ -1875,7 +1896,9 @@ def CausalLM_fast_forward(fast_forward_inference):elif num_logits_to_keep != 0:logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(dtype))else:- RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"+ RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or (+ lm_head_needs_forward(self.lm_head)+ )# Below 1024 normal Unsloth uses less VRAM.if bsz * q_len <= 1024 and not RETURN_LOGITS:# unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.@@ -1891,6 +1914,10 @@ def CausalLM_fast_forward(fast_forward_inference):resolve_logit_transforms(self.config))+ hidden_states, lm_head = fake_quantize_lm_head_inputs(+ self.lm_head, hidden_states, lm_head+ )+if self.config.model_type == "falcon_h1" and logit_scale_multiply:# Via the resolver, so a nullable lm_head_multiplier reads as "off".hidden_states = hidden_states * logit_scale_multiplydiff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 9147dd53d2c..89334513380 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -36,6 +36,8 @@ from .llama import (apply_logit_transforms,resolve_logit_scaling,resolve_logit_transforms,+ fake_quantize_lm_head_inputs,+ lm_head_needs_forward,_cache_as_legacy_tuple,_cached_prefill_defaults,_run_backbone,@@ -340,7 +342,9 @@ def MistralForCausalLM_fast_forward(elif num_logits_to_keep != 0:logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(lm_head.dtype))else:- RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"+ RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or (+ lm_head_needs_forward(self.lm_head)+ )# Below 1024 normal Unsloth uses less VRAM.if bsz * q_len <= 1024 and not RETURN_LOGITS:# unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.@@ -362,6 +366,10 @@ def MistralForCausalLM_fast_forward(kwargs.get("packed_seq_lengths"),)+ hidden_states, lm_head = fake_quantize_lm_head_inputs(+ self.lm_head, hidden_states, lm_head+ )+loss = unsloth_fused_ce_loss(trainer = None,hidden_states = hidden_states,