Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix lfq loss (#2527)

Wenhua Cheng071829b02097
diff --git a/auto_round/algorithms/quantization/sign_round/lfq_loss.py b/auto_round/algorithms/quantization/sign_round/lfq_loss.py
new file mode 100644
index 000000000..3290f63c3
--- /dev/null
+++ b/auto_round/algorithms/quantization/sign_round/lfq_loss.py
@@ -0,0 +1,140 @@
+# Copyright (c) 2024-2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Memory-efficient soft cross-entropy for the LFQ (last-block) loss.
+
+The LFQ objective (Eq. 3 of the paper) minimizes the cross-entropy between the
+FP model's token distribution ``σ(X·W_FP·W_Head)`` and the quantized model's
+distribution ``σ(X·W_q·W_Head)``. A naive implementation materializes two full
+``[batch*seq, vocab]`` float32 tensors (softmax and log_softmax) plus the
+autograd graph that produces them -- prohibitive for large-vocab models.
+
+This module reproduces the essence of Unsloth's fused cross-entropy trick, but
+for a *soft* target distribution instead of hard integer labels:
+
+* logsumexp reformulation so ``log_softmax`` is never materialized::
+
+ softCE_i = logsumexp(q_i) - Σ_j softmax(fp_i)_j · q_{i,j} (Σ_j p=1)
+
+* analytic gradient (no autograd graph through softmax)::
+
+ dL/dq_{i,j} = softmax(q_i)_j - softmax(fp_i)_j
+
+* row chunking: only ``chunk_rows × vocab`` float32 elements are live at a time;
+ softmax is recomputed in the backward pass (compute-for-memory tradeoff).
+
+Unlike Unsloth's Triton kernel this is pure PyTorch, so it runs on CPU / CUDA /
+HPU without a Triton dependency.
+"""
+
+from __future__ import annotations
+
+import torch
+
+__all__ = ["DEFAULT_SOFT_CE_CHUNK_ELEMS", "soft_cross_entropy"]
+
+# Target number of (rows × vocab) float32 elements processed per chunk.
+# 2**23 ≈ 8.4M elems ≈ 32 MB per temporary fp32 tensor.
+DEFAULT_SOFT_CE_CHUNK_ELEMS = 1 << 23
+
+
+def _rows_per_chunk(vocab_size: int, chunk_elems: int) -> int:
+ return max(1, chunk_elems // max(1, vocab_size))
+
+
+class _SoftCrossEntropy(torch.autograd.Function):
+ """Chunked soft cross-entropy with an analytic, memory-light backward."""
+
+ @staticmethod
+ def forward(ctx, q_logits, fp_logits, valid_mask, chunk_elems):
+ # q_logits / fp_logits: [N, V]; valid_mask: [N] (float, 1.0 keep / 0.0 drop)
+ n_rows, vocab_size = q_logits.shape
+ rows = _rows_per_chunk(vocab_size, chunk_elems)
+
+ denom = valid_mask.sum().clamp_min(1.0)
+ total = q_logits.new_zeros((), dtype=torch.float32)
+
+ for start in range(0, n_rows, rows):
+ end = min(start + rows, n_rows)
+ m = valid_mask[start:end]
+ # No host-side ``if m.sum() == 0`` skip here: it would force a
+ # device-to-host sync on every chunk (dozens per forward for large
+ # vocabs). Invalid rows are already zeroed by multiplying with ``m``,
+ # and ``denom`` guards the all-invalid case, so the math stays correct.
+ q_c = q_logits[start:end].float()
+ lse_q = torch.logsumexp(q_c, dim=-1)
+ p_fp = torch.softmax(fp_logits[start:end].float(), dim=-1)
+ dot = (p_fp * q_c).sum(dim=-1)
+ ce = (lse_q - dot) * m
+ total = total + ce.sum()
+
+ loss = total / denom
+
+ ctx.save_for_backward(q_logits, fp_logits, valid_mask, denom)
+ ctx.chunk_elems = chunk_elems
+ return loss
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ q_logits, fp_logits, valid_mask, denom = ctx.saved_tensors
+ n_rows, vocab_size = q_logits.shape
+ rows = _rows_per_chunk(vocab_size, ctx.chunk_elems)
+
+ # scale = dLoss / denom, broadcast per valid row.
+ scale = (grad_output / denom).to(torch.float32)
+ dq = torch.empty_like(q_logits)
+
+ for start in range(0, n_rows, rows):
+ end = min(start + rows, n_rows)
+ m = valid_mask[start:end].unsqueeze(-1)
+ # dL/dq = softmax(q) - softmax(fp), recomputed to avoid storing it.
+ grad = torch.softmax(q_logits[start:end].float(), dim=-1)
+ grad -= torch.softmax(fp_logits[start:end].float(), dim=-1)
+ grad *= m * scale
+ dq[start:end] = grad.to(dq.dtype)
+
+ return dq, None, None, None
+
+
+def soft_cross_entropy(
+ q_logits: torch.Tensor,
+ fp_logits: torch.Tensor,
+ valid_mask: torch.Tensor = None,
+ chunk_elems: int = DEFAULT_SOFT_CE_CHUNK_ELEMS,
+) -> torch.Tensor:
+ """Mean soft cross-entropy ``-Σ_j σ(fp)_j log σ(q)_j`` over valid positions.
+
+ Gradients flow only through ``q_logits``; ``fp_logits`` is treated as a
+ detached soft target.
+
+ Args:
+ q_logits: Quantized logits, shape ``[..., vocab]``.
+ fp_logits: FP logits (soft target), same shape as ``q_logits``.
+ valid_mask: Optional boolean/0-1 mask over positions (all leading dims
+ except vocab). Positions with 0 are excluded from the loss.
+ chunk_elems: Target ``rows × vocab`` float32 elements per chunk.
+
+ Returns:
+ Scalar loss tensor.
+ """
+ vocab_size = q_logits.shape[-1]
+ q2d = q_logits.reshape(-1, vocab_size)
+ fp2d = fp_logits.reshape(-1, vocab_size).detach()
+
+ if valid_mask is None:
+ mask = q2d.new_ones(q2d.shape[0], dtype=torch.float32)
+ else:
+ mask = valid_mask.reshape(-1).to(device=q2d.device, dtype=torch.float32)
+
+ return _SoftCrossEntropy.apply(q2d, fp2d, mask, chunk_elems)
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 7bf06c853..914a12686 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -270,49 +270,81 @@ class SignRoundQuantizer(BaseQuantizer):
name_lower = block_name.lower()
return not any(kw in name_lower for kw in self._NON_TEXT_BLOCK_KEYWORDS)
- def lfq_loss(self, hidden_state: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
- """Compute LM cross-entropy loss from the last block's hidden states.
+ def lfq_loss(
+ self,
+ q_hidden_state: torch.Tensor,
+ fp_hidden_state: torch.Tensor,
+ input_ids: torch.Tensor = None,
+ ) -> torch.Tensor:
+ """Compute the LFQ loss between the FP and quantized token distributions.
+
+ Implements Eq. 3 of the LFQ paper: the cross-entropy between the FP
+ model's logit distribution and the quantized model's logit distribution,
- Applies every post-block module (final norm, optional projection, …) in
- order, then runs lm_head and computes next-token prediction loss.
- Positions marked with ``-100`` in *input_ids* are excluded from the loss.
+ L_CE = -(1/L) Σ_{i,j} σ(X·W_FP·W_Head)_{i,j} · log σ(X·W_q·W_Head)_{i,j}
+
+ where ``σ`` is the softmax over the vocabulary dimension. Both logits are
+ obtained by applying every post-block module (final norm, optional
+ projection, …) in order followed by ``lm_head``. The FP distribution is
+ the (detached) soft target; gradients flow only through the quantized
+ branch. This is a distribution-matching objective, *not* a hard-label
+ next-token cross-entropy.
Args:
- hidden_state: Last block output, shape ``[batch, seq_len, hidden]``.
- input_ids: Token-ID labels with ``-100`` for ignored positions,
- shape ``[batch, seq_len]``.
+ q_hidden_state: Quantized last-block output ``X·W_q``, shape
+ ``[batch, seq_len, hidden]``.
+ fp_hidden_state: FP last-block output ``X·W_FP`` used to build the
+ soft target distribution, same shape as above.
+ input_ids: Optional token-ID labels with ``-100`` for ignored
+ (e.g. padding) positions, shape ``[batch, seq_len]``.
+ When provided, those positions are excluded from the
+ loss; token values themselves are not used as targets.
Returns:
- Scalar cross-entropy loss tensor.
+ Scalar LFQ loss tensor.
"""
+ from .lfq_loss import soft_cross_entropy
+
self._init_lm_components()
- device = hidden_state.device
+ device = q_hidden_state.device
for module in self._post_block_modules:
module.to(device)
- hidden_state = module(hidden_state)
-
self._lm_head.to(device)
- logits = self._lm_head(hidden_state)
- if hasattr(self.model, "loss_function"):
- loss = self.model.loss_function(
- logits=logits,
- labels=input_ids.to(device),
- vocab_size=self.model.config.vocab_size,
- )
- else:
- import torch.nn.functional as F
-
- # Standard causal-LM shift: predict token t+1 from hidden state t.
- shift_logits = logits[..., :-1, :].contiguous()
- shift_labels = input_ids[..., 1:].contiguous().to(device)
- loss = F.cross_entropy(
- shift_logits.view(-1, shift_logits.size(-1)),
- shift_labels.view(-1),
- ignore_index=-100,
- )
- return loss
+ def _to_logits(hidden: torch.Tensor) -> torch.Tensor:
+ for module in self._post_block_modules:
+ hidden = module(hidden)
+ return self._lm_head(hidden)
+
+ # FP logits form the (detached) soft target distribution σ(X·W_FP·W_Head).
+ with torch.no_grad():
+ fp_logits = _to_logits(fp_hidden_state.to(device))
+
+ # Quantized logits: gradients flow through this branch only.
+ q_logits = _to_logits(q_hidden_state)
+
+ # Position mask for the soft-CE objective.
+ #
+ # IMPORTANT: this is a *deliberate same-position* mask — no next-token
+ # shift is applied. LFQ (Eq. 3) matches the FP and quantized output
+ # distributions σ(X·W_FP·W_Head) vs σ(X·W_q·W_Head) at each position; it
+ # is a per-position distillation objective, not next-token prediction,
+ # so logits and the target distribution are aligned position-for-position
+ # (unlike the legacy hard-label CE path that used shift_logits/labels).
+ #
+ # ``input_ids`` already carries the calibrator's -100 ignore markers
+ # (pad tokens, trailing repeated tokens, and the last position of each
+ # sample). Those positions are simply dropped here; the token *values*
+ # are never used as targets.
+ valid_mask = None
+ if input_ids is not None:
+ valid_mask = input_ids.to(device) != -100
+
+ # Memory-efficient soft cross-entropy (Unsloth-style logsumexp + analytic
+ # gradient + row chunking): never materializes a full float32 log_softmax
+ # tensor nor an autograd graph through softmax.
+ return soft_cross_entropy(q_logits, fp_logits, valid_mask)
def quantize_block(
self,
@@ -530,7 +562,11 @@ class SignRoundQuantizer(BaseQuantizer):
and input_ids is not None
and self._is_text_decoder_block(block_ctx.block_name)
):
- loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))
+ loss = self.lfq_loss(
+ pred_output,
+ ref_output,
+ torch.cat([input_ids[i] for i in indices], dim=0),
+ )
else:
loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
num_elm = 1 if num_elm <= 0 else num_elm
diff --git a/test/unit/test_cpu/algorithms/test_lfq_loss.py b/test/unit/test_cpu/algorithms/test_lfq_loss.py
new file mode 100644
index 000000000..a61b77470
--- /dev/null
+++ b/test/unit/test_cpu/algorithms/test_lfq_loss.py
@@ -0,0 +1,138 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CPU tests for the LFQ memory-efficient soft cross-entropy.
+
+The custom ``torch.autograd.Function`` behind ``soft_cross_entropy`` relies on
+an analytic backward, row chunking, and valid-row normalization, so these tests
+pin both the forward value and the gradient against a straightforward reference
+implementation across the tricky cases:
+
+* ``chunk_elems < vocab_size`` (forces the chunk loop to iterate),
+* masked rows (some positions ignored),
+* the all-masked case (no valid positions),
+* 3-D ``[batch, seq, vocab]`` inputs that get flattened internally.
+"""
+
+import pytest
+import torch
+
+from auto_round.algorithms.quantization.sign_round.lfq_loss import soft_cross_entropy
+
+
+def _reference_soft_ce(q_logits, fp_logits, valid_mask=None):
+ """Naive, readable soft cross-entropy used as the ground truth."""
+ vocab = q_logits.shape[-1]
+ q2d = q_logits.reshape(-1, vocab)
+ fp2d = fp_logits.reshape(-1, vocab).detach()
+
+ log_q = torch.log_softmax(q2d.double(), dim=-1)
+ p_fp = torch.softmax(fp2d.double(), dim=-1)
+ ce = -(p_fp * log_q).sum(dim=-1) # [N]
+
+ if valid_mask is None:
+ return ce.mean()
+ m = valid_mask.reshape(-1).to(ce.dtype)
+ return (ce * m).sum() / m.sum().clamp_min(1.0)
+
+
+def _assert_matches(q, fp, mask, chunk_elems):
+ """Compare loss + grad of the chunked impl against the reference."""
+ q_test = q.detach().clone().requires_grad_(True)
+ loss = soft_cross_entropy(q_test, fp, mask, chunk_elems=chunk_elems)
+ loss.backward()
+
+ q_ref = q.detach().clone().requires_grad_(True)
+ ref = _reference_soft_ce(q_ref, fp, mask)
+ ref.backward()
+
+ assert torch.allclose(loss.double(), ref, atol=1e-6, rtol=1e-5), f"loss mismatch: {loss.item()} vs {ref.item()}"
+ assert torch.allclose(
+ q_test.grad.double(), q_ref.grad, atol=1e-6, rtol=1e-5
+ ), f"grad max diff: {(q_test.grad.double() - q_ref.grad).abs().max().item()}"
+
+
[email protected]("chunk_elems", [1 << 23, 60, 20]) # full, 3-row, 1-row chunks
+def test_soft_ce_unmasked_chunking(chunk_elems):
+ torch.manual_seed(0)
+ n, v = 11, 20
+ q = torch.randn(n, v, dtype=torch.float64)
+ fp = torch.randn(n, v, dtype=torch.float64)
+ _assert_matches(q, fp, None, chunk_elems)
+
+
[email protected]("chunk_elems", [1 << 23, 20])
+def test_soft_ce_masked_rows(chunk_elems):
+ torch.manual_seed(1)
+ n, v = 9, 20
+ q = torch.randn(n, v, dtype=torch.float64)
+ fp = torch.randn(n, v, dtype=torch.float64)
+ mask = torch.tensor([1, 1, 0, 1, 0, 0, 1, 1, 0], dtype=torch.bool)
+ _assert_matches(q, fp, mask, chunk_elems)
+
+
+def test_soft_ce_all_masked():
+ """All positions ignored → finite (zero) loss and zero gradients, no NaN."""
+ torch.manual_seed(2)
+ n, v = 5, 16
+ q = torch.randn(n, v, dtype=torch.float64, requires_grad=True)
+ fp = torch.randn(n, v, dtype=torch.float64)
+ mask = torch.zeros(n, dtype=torch.bool)
+
+ loss = soft_cross_entropy(q, fp, mask, chunk_elems=40)
+ loss.backward()
+
+ assert torch.isfinite(loss), "loss must be finite when all rows are masked"
+ assert float(loss) == pytest.approx(0.0, abs=1e-9)
+ assert torch.allclose(q.grad, torch.zeros_like(q.grad))
+
+
+def test_soft_ce_3d_inputs_flattened():
+ """[batch, seq, vocab] inputs and a [batch, seq] mask are handled."""
+ torch.manual_seed(3)
+ b, s, v = 2, 4, 18
+ q = torch.randn(b, s, v, dtype=torch.float64)
+ fp = torch.randn(b, s, v, dtype=torch.float64)
+ mask = torch.ones(b, s, dtype=torch.bool)
+ mask[0, -1] = False # ignore one position
+ _assert_matches(q, fp, mask, chunk_elems=v * 2)
+
+
+def test_soft_ce_fp_target_is_detached():
+ """Gradients must not flow into the FP soft target."""
+ torch.manual_seed(4)
+ n, v = 6, 20
+ q = torch.randn(n, v, dtype=torch.float64, requires_grad=True)
+ fp = torch.randn(n, v, dtype=torch.float64, requires_grad=True)
+
+ loss = soft_cross_entropy(q, fp, None, chunk_elems=40)
+ loss.backward()
+
+ assert q.grad is not None
+ assert fp.grad is None, "FP target must be detached (no gradient)"
+
+
+def test_soft_ce_matches_hard_onehot_reference():
+ """With one-hot FP targets, soft CE collapses to standard CE vs those labels."""
+ torch.manual_seed(5)
+ n, v = 7, 20
+ q = torch.randn(n, v, dtype=torch.float64, requires_grad=True)
+ labels = torch.randint(0, v, (n,))
+ # Sharp FP logits → near one-hot soft target at ``labels``.
+ fp = torch.full((n, v), -30.0, dtype=torch.float64)
+ fp[torch.arange(n), labels] = 30.0
+
+ loss = soft_cross_entropy(q, fp, None, chunk_elems=1 << 23)
+ ref = torch.nn.functional.cross_entropy(q.double(), labels, reduction="mean")
+ assert torch.allclose(loss.double(), ref.double(), atol=1e-5, rtol=1e-4)