Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix LM-head weight fake quantization in fused CE (#14006)

cherry77-cloud5f8a2ec7f76d
diff --git a/tests/test_qat_lm_head_fused_ce.py b/tests/test_qat_lm_head_fused_ce.py
new file mode 100644
index 00000000000..06c3641f9a3
--- /dev/null
+++ b/tests/test_qat_lm_head_fused_ce.py
@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Fused CE reads lm_head.weight directly, so it must apply the same TorchAO QAT fake quantization as
+lm_head.forward(): the activation fake quantizer on the hidden states and the weight fake quantizer on
+the weight. NVFP4 / MX QAT linears fake-quantize inside forward() and must take the unfused path."""
+
+import inspect
+
+import pytest
+import torch
+
+pytest.importorskip("torchao.quantization.qat")
+
+import unsloth # noqa: F401
+import unsloth.models.llama as llama
+import unsloth.models.mistral as mistral
+from unsloth.models._utils import TorchAOConfig, _prepare_model_for_qat
+
+DIM = 128
+VOCAB = 96
+
+
+class _Head(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lm_head = torch.nn.Linear(DIM, VOCAB, bias = False)
+
+
+def _base_config(scheme):
+ from torchao.quantization import (
+ Int4WeightOnlyConfig,
+ Int8DynamicActivationIntxWeightConfig,
+ IntxWeightOnlyConfig,
+ )
+ from torchao.quantization.granularity import PerAxis, PerGroup
+
+ if scheme == "int8":
+ return IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0))
+ if scheme == "int4":
+ return Int4WeightOnlyConfig(group_size = DIM)
+ if scheme == "int8-int4":
+ return Int8DynamicActivationIntxWeightConfig(
+ weight_dtype = torch.int4, weight_granularity = PerGroup(32)
+ )
+ mx = pytest.importorskip("torchao.prototype.mx_formats")
+ if scheme == "nvfp4":
+ return mx.NVFP4DynamicActivationNVFP4WeightConfig()
+ return mx.MXDynamicActivationMXWeightConfig(
+ activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn
+ )
+
+
+def _qat_head(scheme):
+ torch.manual_seed(0)
+ model = _Head()
+ _prepare_model_for_qat(
+ model,
+ TorchAOConfig(qat_scheme = None, base_config_and_filter_fns = [(_base_config(scheme), None)]),
+ )
+ return model.lm_head
+
+
[email protected]("scheme", ["int8", "int4", "int8-int4"])
+def test_fused_inputs_match_lm_head_forward(scheme):
+ lm_head = _qat_head(scheme)
+ x = torch.randn(2, 5, DIM)
+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)
+ expected = lm_head(x)
+ # Without the fake quantizers the plain weight gives different logits, so the check can fail.
+ assert not torch.equal(torch.nn.functional.linear(x, lm_head.weight), expected)
+ torch.testing.assert_close(
+ torch.nn.functional.linear(hidden_states, weight), expected, rtol = 0, atol = 0
+ )
+ assert not llama.lm_head_needs_forward(lm_head)
+
+
+def test_activation_fake_quant_sees_head_dtype():
+ # A trained lm_head is upcast to fp32 while fp16 hidden states arrive; forward() sees them cast.
+ lm_head = _qat_head("int8-int4")
+ x = torch.randn(2, 5, DIM).half()
+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)
+ assert hidden_states.dtype == torch.float32
+ torch.testing.assert_close(
+ torch.nn.functional.linear(hidden_states, weight), lm_head(x.float()), rtol = 0, atol = 0
+ )
+
+
+def test_plain_lm_head_is_untouched():
+ lm_head = torch.nn.Linear(DIM, VOCAB, bias = False)
+ x = torch.randn(2, 5, DIM)
+ hidden_states, weight = llama.fake_quantize_lm_head_inputs(lm_head, x, lm_head.weight)
+ assert hidden_states is x and weight is lm_head.weight
+ assert not llama.lm_head_needs_forward(lm_head)
+
+
[email protected]("scheme", ["nvfp4", "mx"])
+def test_forward_fake_quant_lm_head_takes_unfused_path(scheme):
+ lm_head = _qat_head(scheme)
+ assert llama.lm_head_needs_forward(lm_head)
+
+
[email protected]("scheme", ["nvfp4", "mx"])
+def test_forward_fake_quant_lm_head_in_modules_to_save_takes_unfused_path(scheme):
+ peft = pytest.importorskip("peft")
+
+ class _Model(_Head):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(DIM, DIM, bias = False)
+
+ torch.manual_seed(0)
+ model = peft.get_peft_model(
+ _Model(), peft.LoraConfig(r = 16, target_modules = ["proj"], modules_to_save = ["lm_head"])
+ )
+ only_head_copy = (
+ lambda m, fqn: isinstance(m, torch.nn.Linear) and "lm_head.modules_to_save" in fqn
+ )
+ _prepare_model_for_qat(
+ model,
+ TorchAOConfig(
+ qat_scheme = None, base_config_and_filter_fns = [(_base_config(scheme), only_head_copy)]
+ ),
+ )
+ lm_head = model.base_model.model.lm_head
+ assert type(lm_head).__name__ == "ModulesToSaveWrapper"
+ assert llama.lm_head_needs_forward(lm_head)
+
+
+ "forward",
+ [llama.CausalLM_fast_forward, mistral.MistralForCausalLM_fast_forward],
+ ids = ["llama", "mistral"],
+)
+def test_fused_ce_branches_fake_quantize_lm_head(forward):
+ source = "".join(inspect.getsource(forward).split())
+ assert "fake_quantize_lm_head_inputs(self.lm_head," in source
+ assert "lm_head_needs_forward(self.lm_head)" in source
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 193cb1eee09..49ce312f5b5 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -224,6 +224,27 @@ def original_apply_qkv(self, X):
return Q, K, V
+def fake_quantize_lm_head_inputs(lm_head_module, hidden_states, weight):
+ # Fused CE bypasses lm_head.forward(), where TorchAO QAT fake-quantizes both the input and the weight.
+ activation_fake_quantizer = getattr(lm_head_module, "activation_fake_quantizer", None)
+ if activation_fake_quantizer is not None:
+ # The unfused path feeds lm_head hidden_states.to(weight dtype): a trained fp32 head on fp16 differs.
+ hidden_states = activation_fake_quantizer(hidden_states.to(weight.dtype))
+ weight_fake_quantizer = getattr(lm_head_module, "weight_fake_quantizer", None)
+ if weight_fake_quantizer is not None:
+ weight = weight_fake_quantizer(weight)
+ return hidden_states, weight
+
+
+def lm_head_needs_forward(lm_head_module):
+ # NVFP4 / MX QAT linears fake-quantize inside forward() with no quantizer children: fused CE cannot mirror them.
+ # modules() also reaches the copy inside a PEFT modules_to_save wrapper, which isinstance alone misses.
+ forward_fake_quant_linears = _fast_lora._forward_fake_quant_linears()
+ return bool(forward_fake_quant_linears) and any(
+ isinstance(module, forward_fake_quant_linears) for module in lm_head_module.modules()
+ )
+
+
def original_apply_o(self, X):
O = self.o_proj(X)
return O
@@ -1875,7 +1896,9 @@ def CausalLM_fast_forward(fast_forward_inference):
elif num_logits_to_keep != 0:
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(dtype))
else:
- RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"
+ RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or (
+ lm_head_needs_forward(self.lm_head)
+ )
# Below 1024 normal Unsloth uses less VRAM.
if bsz * q_len <= 1024 and not RETURN_LOGITS:
# unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.
@@ -1891,6 +1914,10 @@ def CausalLM_fast_forward(fast_forward_inference):
resolve_logit_transforms(self.config)
)
+ hidden_states, lm_head = fake_quantize_lm_head_inputs(
+ self.lm_head, hidden_states, lm_head
+ )
+
if self.config.model_type == "falcon_h1" and logit_scale_multiply:
# Via the resolver, so a nullable lm_head_multiplier reads as "off".
hidden_states = hidden_states * logit_scale_multiply
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 9147dd53d2c..89334513380 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -36,6 +36,8 @@ from .llama import (
apply_logit_transforms,
resolve_logit_scaling,
resolve_logit_transforms,
+ fake_quantize_lm_head_inputs,
+ lm_head_needs_forward,
_cache_as_legacy_tuple,
_cached_prefill_defaults,
_run_backbone,
@@ -340,7 +342,9 @@ def MistralForCausalLM_fast_forward(
elif num_logits_to_keep != 0:
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(lm_head.dtype))
else:
- RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"
+ RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or (
+ lm_head_needs_forward(self.lm_head)
+ )
# Below 1024 normal Unsloth uses less VRAM.
if bsz * q_len <= 1024 and not RETURN_LOGITS:
# unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.
@@ -362,6 +366,10 @@ def MistralForCausalLM_fast_forward(
kwargs.get("packed_seq_lengths"),
)
+ hidden_states, lm_head = fake_quantize_lm_head_inputs(
+ self.lm_head, hidden_states, lm_head
+ )
+
loss = unsloth_fused_ce_loss(
trainer = None,
hidden_states = hidden_states,