Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine alg and support lfq (#2082)

Wenhua Chengf1af646b5081
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index bf399f814..ce2d27d97 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -77,6 +77,7 @@ class BlockContext:
is_mllm: bool = False # fail-fast gate for algorithms that don't support MLLM
is_diffusion: bool = False # fail-fast gate for algorithms that don't support diffusion
pbar: Any = None
+ block_cnt: int = 0 # total number of blocks being quantized in this run
# ---------------------------------------------------------------------------
@@ -274,7 +275,7 @@ class AlgorithmComposer:
handles.extend(self.block_quantizer.register_qinput_forward_hooks(block))
return handles
- def _attach_act_max_for_outside_layer(self, layer: "torch.nn.Module", fp_input, q_input) -> None:
+ def _attach_act_max_for_outside_layer(self, layer: "torch.nn.Module", fp_inputs, q_inputs) -> None:
"""Compute and attach act_max for an outside-block layer from cached inputs.
Mirrors the hook-based act_max collection done for in-block layers, but
@@ -282,14 +283,14 @@ class AlgorithmComposer:
Args:
layer: The layer module to attach act_max to.
- fp_input: List of FP input tensors collected during calibration.
- q_input: Optional list of quantized input tensors; used instead of
- ``fp_input`` when provided.
+ fp_inputs: List of FP input tensors collected during calibration.
+ q_inputs: Optional list of quantized input tensors; used instead of
+ ``fp_inputs`` when provided.
"""
from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
- target_input = q_input or fp_input
+ target_input = q_inputs or fp_inputs
act_group_size = getattr(layer, "act_group_size")
if act_group_size is None:
act_group_size = layer.group_size
@@ -340,7 +341,7 @@ class AlgorithmComposer:
input_others,
block_ctx: BlockContext,
q_inputs=None,
- valid_token_mask=None,
+ input_ids=None,
**kwargs,
) -> tuple:
"""Run the full per-block algorithm pipeline: calibration → quantization → collection.
@@ -358,7 +359,7 @@ class AlgorithmComposer:
input_ids: Full-precision (FP) cached inputs.
input_others: Auxiliary kwargs (attention_mask, position_ids, …).
ctx: Per-block lifecycle context (:class:`BlockContext`).
- q_input: Quantized-input tensors from the previous block, or ``None``.
+ q_inputs: Quantized-input tensors from the previous block, or ``None``.
valid_token_mask: Optional mask for per-token loss weighting.
Returns:
@@ -441,7 +442,7 @@ class AlgorithmComposer:
reference_output,
q_inputs,
block_ctx,
- valid_token_mask=valid_token_mask,
+ input_ids=input_ids,
)
# ── Step 5: post_quantize_block ─────────────────────────────────────────
@@ -461,10 +462,10 @@ class AlgorithmComposer:
def compress_layer_outside_block(
self,
layer: "torch.nn.Module",
- fp_input=None,
- q_input=None,
- valid_token_mask=None,
+ fp_inputs=None,
+ q_inputs=None,
disable_opt_rtn=None, # TODO wenhuach rename this to search_init_scale
+ input_ids=None,
) -> None:
"""Quantize a single layer that lives outside transformer blocks.
@@ -475,14 +476,14 @@ class AlgorithmComposer:
Args:
layer: The layer module to quantize. Must have a ``global_name``
attribute.
- fp_input: Per-sample FP activations for calibration, or ``None``
+ fp_inputs: Per-sample FP activations for calibration, or ``None``
to fall back to zero-shot RTN.
- q_input: Optional quantized activations from a previous stage.
+ q_inputs: Optional quantized activations from a previous stage.
valid_token_mask: Per-sample token masks for loss weighting.
disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config.
"""
# Attach act_max for static activation quantization when inputs are available.
- if fp_input is not None:
+ if fp_inputs is not None:
from auto_round.compressors.utils import is_nv_fp
act_data_type = getattr(layer, "act_data_type")
@@ -490,7 +491,7 @@ class AlgorithmComposer:
act_data_type = "fp"
act_dynamic = getattr(layer, "act_dynamic", True)
if is_nv_fp(act_data_type) or not act_dynamic:
- self._attach_act_max_for_outside_layer(layer, fp_input, q_input)
+ self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)
# Infrastructure: move layer to the tuning device before handing off to the quantizer.
device = getattr(layer, "tuning_device", device_manager.device) # TODO this should be handled by compressor
@@ -498,10 +499,10 @@ class AlgorithmComposer:
return self.block_quantizer.quantize_layer_outside_block(
layer,
- fp_input=fp_input,
- q_input=q_input,
- valid_token_mask=valid_token_mask,
+ fp_inputs=fp_inputs,
+ q_inputs=q_inputs,
disable_opt_rtn=disable_opt_rtn,
+ input_ids=input_ids,
)
# ── Convenience act-calib helpers ────────────────────────────────────────
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index a977b43e3..c83d805d5 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -153,7 +153,7 @@ class BaseQuantizer(BaseAlgorithm):
fp_outputs: "list[torch.Tensor]",
q_inputs: "list[torch.Tensor] | None",
block_ctx: "BlockContext",
- valid_token_mask: "list[torch.Tensor] | None" = None,
+ input_ids: "list[torch.Tensor] | None" = None,
**kwargs,
) -> dict:
"""Apply the quantization algorithm to a prepared block.
@@ -161,15 +161,16 @@ class BaseQuantizer(BaseAlgorithm):
all infrastructure work (device placement, data collection, act-max hook
registration, DDP setup) has been completed.
Args:
- block: The transformer block module to quantize.
- fp_inputs: FP calibration inputs (list[Tensor] or dict for diffusion).
- input_others: Auxiliary kwargs (attention_mask, position_ids, …).
- fp_outputs: FP reference outputs used as quantization targets.
- q_inputs: Quantized inputs from the previous block, or ``None``.
- block_ctx: Per-block pipeline context (:class:`BlockContext`).
- valid_token_mask: Per-sample boolean masks ``[1, seq_len]``; ``None`` if
- no padding masking is needed.
- **kwargs: Reserved for future parameters.
+ block: The transformer block module to quantize.
+ fp_inputs: FP calibration inputs (list[Tensor] or dict for diffusion).
+ input_others: Auxiliary kwargs (attention_mask, position_ids, …).
+ fp_outputs: FP reference outputs used as quantization targets.
+ q_inputs: Quantized inputs from the previous block, or ``None``.
+ block_ctx: Per-block pipeline context (:class:`BlockContext`).
+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per
+ sample); used to derive the valid-token loss mask.
+ ``None`` for diffusion or zero-shot paths.
+ **kwargs: Reserved for future parameters.
Returns:
dict: Best quantization parameters found, or ``{}`` if not applicable.
Raises:
@@ -180,20 +181,34 @@ class BaseQuantizer(BaseAlgorithm):
def quantize_layer_outside_block(
self,
layer: "torch.nn.Module",
- fp_input: "list[torch.Tensor] | None" = None,
- q_input: "list[torch.Tensor] | None" = None,
+ fp_inputs: "list[torch.Tensor] | None" = None,
+ q_inputs: "list[torch.Tensor] | None" = None,
disable_opt_rtn: "bool | None" = None,
- valid_token_mask: "list[torch.Tensor] | None" = None,
+ input_ids: "list[torch.Tensor] | None" = None,
) -> None:
+ """Quantize a single layer outside a transformer block using RTN fallback.
+ Args:
+ layer: The layer module to quantize. Must have a
+ ``global_name`` attribute for model re-insertion.
+ fp_inputs: Optional FP calibration inputs; unused in base RTN.
+ q_inputs: Optional quantized activations; unused in base RTN.
+ disable_opt_rtn: ``True`` skips optimized-RTN scale/zp search.
+ ``None`` defers to ``self.config.disable_opt_rtn``.
+ input_ids: Raw token IDs from the tokenizer; used to derive
+ the valid-token loss mask. ``None`` for RTN fallback.
+ """
+ self._quantize_layer_via_rtn(layer, disable_opt_rtn=disable_opt_rtn)
"""Quantize a single layer outside a transformer block using RTN fallback.
Args:
layer: The layer module to quantize. Must have a
``global_name`` attribute for model re-insertion.
- fp_input: Optional FP calibration inputs; unused in base RTN.
- q_input: Optional quantized activations; unused in base RTN.
+ fp_inputs: Optional FP calibration inputs; unused in base RTN.
+ q_inputs: Optional quantized activations; unused in base RTN.
disable_opt_rtn: ``True`` skips optimized-RTN scale/zp search.
``None`` defers to ``self.config.disable_opt_rtn``.
valid_token_mask: Per-sample masks; unused in base RTN.
+ input_ids: Original FP calibration inputs, same as ``fp_inputs``
+ when ``q_inputs`` is ``None``; unused in base RTN.
"""
self._quantize_layer_via_rtn(layer, disable_opt_rtn=disable_opt_rtn)
@@ -252,14 +267,39 @@ class BaseQuantizer(BaseAlgorithm):
raise
set_module(self.model, layer_name, layer)
+ def _compute_valid_token_mask(self, input_ids: list) -> "list | None":
+ """Derive a per-sample valid-token loss mask from raw token IDs.
+
+ ``LLMCalibrator.calib`` already marks every position that should be
+ excluded from the loss with ``-100`` (PyTorch's standard ignore index):
+ pad tokens, trailing repeated tokens, and the last position of each
+ sample. This method therefore only needs a single comparison.
+
+ Args:
+ input_ids: List of ``[1, seq_len]`` integer tensors (one per sample),
+ with ignored positions already set to ``-100``.
+
+ Returns:
+ List of ``[1, seq_len]`` ``torch.long`` masks (1 = valid, 0 = ignore),
+ or ``None`` when every position is valid (all-ones → fast path).
+ """
+ if not input_ids:
+ return None
+
+ masks = [(ids != -100).to(torch.long) for ids in input_ids]
+ # All-ones → masking is a no-op; use the faster unmasked loss path.
+ if all(m.all().item() for m in masks):
+ return None
+ return masks
+
def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):
"""Place a block on the correct device(s) for quantization.
Default: move to primary device.
- Returns ``(block, card_0_in_high_risk, loss_device)``.
+ Returns the block after device placement.
Subclasses override for multi-GPU tensor-parallel dispatch.
"""
block = block.to(device_manager.device)
- return block, False, device_manager.device
+ return block
# ── Lifecycle hooks ───────────────────────────────────────────────────────
def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:
diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.py
index 95525b1e2..0bff58eb8 100644
--- a/auto_round/algorithms/quantization/rtn/quantizer.py
+++ b/auto_round/algorithms/quantization/rtn/quantizer.py
@@ -38,7 +38,7 @@ class RTNQuantizer(BaseQuantizer):
fp_outputs,
q_inputs,
block_ctx,
- valid_token_mask=None,
+ input_ids=None,
**kwargs,
) -> dict:
"""Apply zero-shot RTN quantization to a block.
@@ -54,10 +54,7 @@ class RTNQuantizer(BaseQuantizer):
q_inputs: Quantized inputs from the previous block, or ``None`` when
cascaded quantized-input is disabled.
block_ctx: Per-block pipeline context (BlockContext).
- valid_token_mask: Per-sample boolean/int masks of shape
- ``[1, seq_len]`` indicating valid (non-padding) token positions.
- ``1`` means valid, ``0`` means padding. ``None`` if no masking
- is needed (e.g. standard string datasets without padding).
+ input_ids: Raw token IDs from the tokenizer (unused in RTN).
**kwargs: Reserved for forward-compatibility with future parameters.
Returns:
@@ -124,7 +121,7 @@ class OptimizedRTNQuantizer(RTNQuantizer):
fp_outputs,
q_inputs,
block_ctx,
- valid_token_mask=None,
+ input_ids=None,
**kwargs,
):
"""Apply imatrix-informed RTN quantization to a block.
@@ -140,11 +137,7 @@ class OptimizedRTNQuantizer(RTNQuantizer):
q_inputs: Quantized inputs from the previous block, or ``None`` when
cascaded quantized-input is disabled.
block_ctx: Per-block pipeline context (BlockContext).
- valid_token_mask: Per-sample boolean/int masks of shape
- ``[1, seq_len]`` indicating valid (non-padding) token positions.
- ``1`` means valid, ``0`` means padding. ``None`` if no masking
- is needed (e.g. standard string datasets without padding).
- Currently unused in imatrix-RTN; reserved for future use.
+ input_ids: Raw token IDs from the tokenizer (unused in RTN).
**kwargs: Reserved for forward-compatibility with future parameters.
"""
# Normalize imatrix and quantize layers
diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.py
index 4c883a130..823b10e05 100644
--- a/auto_round/algorithms/quantization/sign_round/config.py
+++ b/auto_round/algorithms/quantization/sign_round/config.py
@@ -38,6 +38,7 @@ class SignRoundConfig(QuantizationConfig):
enable_quanted_input: bool = True,
optimizer: str | None = None, # TODO later wenhuach delete this
enable_adam: bool = False, # TODO later wenhuach delete this
+ enable_lfq: bool = False,
**kwargs,
) -> None:
"""Initialize a SignRound configuration.
@@ -89,9 +90,6 @@ class SignRoundConfig(QuantizationConfig):
self.momentum = momentum
self.enable_alg_ext = enable_alg_ext
- # Some helpers
- self.infer_bs_coeff = 1
-
self.enable_minmax_tuning = enable_minmax_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning
if self.enable_norm_bias_tuning:
@@ -101,6 +99,9 @@ class SignRoundConfig(QuantizationConfig):
self.enable_quanted_input = enable_quanted_input
self.optimizer = optimizer
self.enable_adam = enable_adam
+ self.enable_lfq = enable_lfq
+ if self.enable_lfq:
+ logger.warning("the `enable_lfq` feature is experimental and currently has limited model support.")
def finalize_scheme(self) -> None:
"""Resolve lr/minmax_lr once `bits` is known (low-bit schemes use a higher lr)."""
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 7487fdfa9..0de9fffcb 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -60,6 +60,7 @@ class SignRoundQuantizer(BaseQuantizer):
self.not_use_best_mse = config.not_use_best_mse
self.enable_quanted_input = config.enable_quanted_input
self.dynamic_max_gap = config.dynamic_max_gap
+ self.enable_lfq = config.enable_lfq
self.optimizer = self._get_optimizer(optimizer=config.optimizer)
self.wrapper_block = wrapper_block
@@ -67,7 +68,8 @@ class SignRoundQuantizer(BaseQuantizer):
def dispatch_block(self, block, input_ids, input_others):
"""Multi-GPU aware block dispatch for SignRound tuning.
- Stores card_0_in_high_risk and loss_device on self for use in quantize_block.
+ Stores _card_0_in_high_risk and _loss_device on self for use in quantize_block.
+ Returns the block after device placement.
"""
from auto_round.utils import is_auto_device_mapping
@@ -99,7 +101,7 @@ class SignRoundQuantizer(BaseQuantizer):
self._card_0_in_high_risk = card_0_in_high_risk
self._loss_device = loss_device
- return block, card_0_in_high_risk, loss_device
+ return block
def _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:
current_tensors = [tensor[i] for i in indices]
@@ -116,6 +118,7 @@ class SignRoundQuantizer(BaseQuantizer):
loss_func: Callable,
device: Union[str, torch.device] = "cpu",
valid_token_mask: Optional[torch.Tensor] = None,
+ input_ids=None,
):
autocast_ctx = (
nullcontext()
@@ -149,6 +152,148 @@ class SignRoundQuantizer(BaseQuantizer):
else:
return inputs.shape[self.calibration_context.batch_dim]
+ def _init_lm_components(self) -> None:
+ """Lazily locate and cache ``_post_block_modules`` and ``_lm_head`` for LFQ loss.
+
+ ``_post_block_modules`` is an ordered list of modules that must be applied
+ between the last transformer-block output and the lm_head projection.
+ Most architectures have a single final norm; OPT additionally has an
+ optional ``project_out`` projection.
+
+ Supported without manual configuration:
+ LLaMA / Qwen / Gemma / Mistral / InternLM / Phi-3 — ``model.model.norm``
+ OPT — ``model.model.decoder.{final_layer_norm, project_out}`` (both optional)
+ GPT-2 / Falcon / Bloom — ``model.transformer.ln_f``
+ GPT-NeoX / Pythia — ``model.gpt_neox.final_layer_norm``
+ Phi / Phi-2 — ``model.model.final_layernorm``
+ MPT — ``model.transformer.norm_f``
+ ChatGLM — ``model.transformer.encoder.final_layernorm``
+ RWKV — ``model.rwkv.ln_out``
+
+ Raises ``AttributeError`` if no lm_head equivalent can be found.
+ """
+ if hasattr(self, "_lm_head"):
+ return
+
+ model = self.model
+
+ # ── lm_head ──────────────────────────────────────────────────────────
+ for name in ("lm_head", "embed_out", "output", "head"):
+ if hasattr(model, name):
+ self._lm_head = getattr(model, name)
+ break
+ else:
+ raise AttributeError(
+ f"Cannot locate lm_head in {type(model).__name__}. " "Checked: lm_head, embed_out, output, head."
+ )
+
+ # ── post-block processing (ordered list applied before lm_head) ───────
+ # OPT: decoder has both an optional final_layer_norm *and* an optional
+ # project_out that maps ffn_dim → word_embed_proj_dim. Detect by probing
+ # for the characteristic project_out attribute (may be None).
+ try:
+ decoder = model.model.decoder
+ _ = decoder.project_out # raises AttributeError if not an OPT decoder
+ self._post_block_modules = [m for m in (decoder.final_layer_norm, decoder.project_out) if m is not None]
+ return
+ except AttributeError:
+ pass
+
+ # All other architectures: single optional final norm.
+ norm_getters = [
+ lambda: model.model.norm, # LLaMA / Qwen / Gemma / Mistral / InternLM / Phi-3
+ lambda: model.transformer.ln_f, # GPT-2 / Falcon / Bloom
+ lambda: model.gpt_neox.final_layer_norm, # GPT-NeoX / Pythia
+ lambda: model.model.final_layernorm, # Phi / Phi-2
+ lambda: model.transformer.norm_f, # MPT
+ lambda: model.transformer.encoder.final_layernorm, # ChatGLM
+ lambda: model.rwkv.ln_out, # RWKV
+ ]
+ self._post_block_modules = []
+ for getter in norm_getters:
+ try:
+ norm = getter()
+ if norm is not None:
+ self._post_block_modules = [norm]
+ break
+ except AttributeError:
+ continue
+
+ # Keywords that identify non-text (visual / audio / multimodal) blocks.
+ # LFQ loss is only meaningful for pure language-model decoder blocks.
+ _NON_TEXT_BLOCK_KEYWORDS = frozenset(
+ {
+ "vis",
+ "vision",
+ "visual",
+ "image",
+ "img",
+ "audio",
+ "video",
+ "patch",
+ "pixel",
+ "clip",
+ "vit",
+ "perceiver",
+ "resampler",
+ "connector",
+ "projector",
+ }
+ )
+
+ def _is_text_decoder_block(self, block_name: str) -> bool:
+ """Return ``True`` if *block_name* refers to a text-decoder block.
+
+ Blocks whose names contain any of the non-text keywords (vision, audio,
+ image, …) are considered multimodal and excluded from LFQ loss.
+ """
+ name_lower = block_name.lower()
+ return not any(kw in name_lower for kw in self._NON_TEXT_BLOCK_KEYWORDS)
+
+ def lfq_loss(self, hidden_state: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
+ """Compute LM cross-entropy loss from the last block's hidden states.
+
+ Applies every post-block module (final norm, optional projection, …) in
+ order, then runs lm_head and computes next-token prediction loss.
+ Positions marked with ``-100`` in *input_ids* are excluded from the loss.
+
+ Args:
+ hidden_state: Last block output, shape ``[batch, seq_len, hidden]``.
+ input_ids: Token-ID labels with ``-100`` for ignored positions,
+ shape ``[batch, seq_len]``.
+
+ Returns:
+ Scalar cross-entropy loss tensor.
+ """
+ self._init_lm_components()
+ device = hidden_state.device
+
+ for module in self._post_block_modules:
+ module.to(device)
+ hidden_state = module(hidden_state)
+
+ self._lm_head.to(device)
+ logits = self._lm_head(hidden_state)
+
+ if hasattr(self.model, "loss_function"):
+ loss = self.model.loss_function(
+ logits=logits,
+ labels=input_ids.to(device),
+ vocab_size=self.model.config.vocab_size,
+ )
+ else:
+ import torch.nn.functional as F
+
+ # Standard causal-LM shift: predict token t+1 from hidden state t.
+ shift_logits = logits[..., :-1, :].contiguous()
+ shift_labels = input_ids[..., 1:].contiguous().to(device)
+ loss = F.cross_entropy(
+ shift_logits.view(-1, shift_logits.size(-1)),
+ shift_labels.view(-1),
+ ignore_index=-100,
+ )
+ return loss
+
def quantize_block(
self,
block,
@@ -157,7 +302,7 @@ class SignRoundQuantizer(BaseQuantizer):
fp_outputs,
q_inputs,
block_ctx,
- valid_token_mask=None,
+ input_ids=None,
**kwargs,
) -> dict:
"""Apply the AutoRound optimization algorithm to a block.
@@ -177,11 +322,10 @@ class SignRoundQuantizer(BaseQuantizer):
q_inputs: Quantized inputs from the previous block, or ``None`` when
cascaded quantized-input is disabled.
block_ctx: Per-block pipeline context (BlockContext).
- valid_token_mask: Per-sample boolean/int masks of shape
- ``[1, seq_len]`` indicating valid (non-padding) token positions.
- ``1`` means valid, ``0`` means padding. ``None`` if no masking
- is needed (e.g. standard string datasets without padding).
- When provided, the loss is computed only over valid token positions.
+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per
+ sample). Used to derive the valid-token loss mask once (result
+ cached on ``self._cached_valid_token_mask`` for reuse across
+ all blocks). ``None`` disables loss masking.
**kwargs: Reserved for forward-compatibility with future parameters.
Returns:
@@ -193,6 +337,14 @@ class SignRoundQuantizer(BaseQuantizer):
card_0_in_high_risk = getattr(self, "_card_0_in_high_risk", False)
mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk
+ valid_token_mask = None
+ # Derive valid_token_mask from raw token IDs when not supplied by caller.
+ # Result is cached on self so it is computed only once across all blocks.
+ if input_ids is not None:
+ if not hasattr(self, "_cached_valid_token_mask"):
+ self._cached_valid_token_mask = self._compute_valid_token_mask(input_ids)
+ valid_token_mask = self._cached_valid_token_mask
+
# Use quantized inputs if available and enabled
active_inputs = q_inputs if (q_inputs is not None and self.enable_quanted_input) else fp_inputs
nsamples = len(active_inputs) if isinstance(active_inputs, list) else self._count_samples(active_inputs)
@@ -265,6 +417,7 @@ class SignRoundQuantizer(BaseQuantizer):
batch_size = self.calibration_context.batch_size
global_batch_size = batch_size * self.gradient_accumulate_steps
global_batch_size = min(nsamples, global_batch_size)
+ # Compute num_elm once before the loop (used to normalise the accumulated loss).
# We assume the block input and output shape is same
if self.gradient_accumulate_steps != 1 and not valid_token_mask:
whole_indices = torch.arange(global_batch_size)
@@ -303,7 +456,15 @@ class SignRoundQuantizer(BaseQuantizer):
pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)
if loss_device is not None:
pred_output = pred_output.to(loss_device)
- loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
+ if (
+ block_ctx.block_index == block_ctx.block_cnt - 1
+ and self.enable_lfq
+ and input_ids is not None
+ and self._is_text_decoder_block(block_ctx.block_name)
+ ):
+ loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))
+ else:
+ loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
num_elm = 1 if num_elm <= 0 else num_elm
total_loss += loss.item() / num_elm
@@ -366,37 +527,35 @@ class SignRoundQuantizer(BaseQuantizer):
def quantize_layer_outside_block(
self,
layer: "torch.nn.Module",
- fp_input: Optional[list[torch.Tensor]] = None,
- q_input: Optional[list[torch.Tensor]] = None,
- valid_token_mask: Optional[list[torch.Tensor]] = None,
+ fp_inputs: Optional[list[torch.Tensor]] = None,
+ q_inputs: Optional[list[torch.Tensor]] = None,
disable_opt_rtn: Optional[bool] = None,
+ input_ids: Optional[list[torch.Tensor]] = None,
):
"""Quantize a single layer that lives outside a transformer block.
- When ``fp_input`` is provided the layer is tuned with the sign-gradient
+ When ``fp_inputs`` is provided the layer is tuned with the sign-gradient
descent optimizer (same loss loop as block-level quantization). When
- ``fp_input`` is ``None`` the method falls back to zero-shot RTN.
+ ``fp_inputs`` is ``None`` the method falls back to zero-shot RTN.
Args:
layer: The layer module to quantize. Must have a ``global_name``
attribute for model re-insertion and logging.
- fp_input: Per-sample FP activations fed into this layer, used as
+ fp_inputs: Per-sample FP activations fed into this layer, used as
calibration inputs during optimization. ``None`` triggers RTN
fallback.
- q_input: Per-sample quantized activations from the previous stage,
- used instead of ``fp_input`` during the forward pass when
+ q_inputs: Per-sample quantized activations from the previous stage,
+ used instead of ``fp_inputs`` during the forward pass when
cascaded quantized-input is enabled. ``None`` means use
- ``fp_input`` for both reference and tuning forward.
- valid_token_mask: Per-sample boolean/int masks of shape
- ``[1, seq_len]`` indicating valid (non-padding) token positions.
- ``1`` means valid, ``0`` means padding. ``None`` if no masking
- is needed. Forwarded to the loss computation to weight out
- padding tokens.
+ ``fp_inputs`` for both reference and tuning forward.
disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config.
+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per
+ sample); used to derive the valid-token loss mask via
+ ``_compute_valid_token_mask``. ``None`` disables loss masking.
"""
layer_name = layer.global_name
- if fp_input is None:
+ if fp_inputs is None:
logger.info(f"using rtn to quantize {layer_name}")
self._quantize_layer_via_rtn(
layer,
@@ -406,13 +565,21 @@ class SignRoundQuantizer(BaseQuantizer):
)
return
+ # Derive valid_token_mask from raw token IDs when not supplied by caller.
+ # Reuse the cached mask if already computed by a previous block.
+ valid_token_mask = None
+ if input_ids is not None:
+ if not hasattr(self, "_cached_valid_token_mask"):
+ self._cached_valid_token_mask = self._compute_valid_token_mask(input_ids)
+ valid_token_mask = self._cached_valid_token_mask
+
logger.info(f"quantizing layer {layer_name}")
# Layer is already on the correct device (placed by the caller / AlgorithmComposer).
device = layer.weight.device if hasattr(layer, "weight") else device_manager.device
- for i in range(len(fp_input)):
- fp_input[i] = fp_input[i].to(layer.weight.dtype)
- if q_input is not None:
- q_input[i] = q_input[i].to(layer.weight.dtype)
+ for i in range(len(fp_inputs)):
+ fp_inputs[i] = fp_inputs[i].to(layer.weight.dtype)
+ if q_inputs is not None:
+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
wrapper_linear = WrapperLinear(
layer,
@@ -449,7 +616,7 @@ class SignRoundQuantizer(BaseQuantizer):
)
else:
lr_schedule = copy.deepcopy(self.lr_scheduler)
- nsamples = len(fp_input)
+ nsamples = len(fp_inputs)
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
best_params = None
@@ -469,30 +636,31 @@ class SignRoundQuantizer(BaseQuantizer):
batch_size = 1 # Force to low gpu
global_batch_size = gradient_accumulate_steps
global_batch_size = min(nsamples, global_batch_size)
- if gradient_accumulate_steps != 1 and not valid_token_mask:
- whole_indices = torch.arange(global_batch_size).tolist()
- if q_input is not None:
- num_elm = self._count_layer_input_elements(q_input, whole_indices)
+ # Compute num_elm once before the loop.
+ if gradient_accumulate_steps != 1:
+ whole_indices = list(range(global_batch_size))
+ if valid_token_mask:
+ num_elm = self._get_non_zero_cnt(valid_token_mask, whole_indices)
+ elif q_inputs is not None:
+ num_elm = self._count_layer_input_elements(q_inputs, whole_indices)
else:
- num_elm = self._count_layer_input_elements(fp_input, whole_indices)
+ num_elm = self._count_layer_input_elements(fp_inputs, whole_indices)
index_sampler = IndexSampler(nsamples, global_batch_size)
for i in range(self.iters):
total_loss = 0
global_indices = index_sampler.next_batch()
- if valid_token_mask:
- num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)
for batch_start in range(0, len(global_indices), batch_size):
indices = global_indices[batch_start : batch_start + batch_size]
- if q_input is not None:
- current_input = [q_input[i] for i in indices]
+ if q_inputs is not None:
+ current_input = [q_inputs[i] for i in indices]
current_input = torch.cat(current_input, dim=0).to(device)
- org_input = [fp_input[i] for i in indices]
+ org_input = [fp_inputs[i] for i in indices]
org_input = torch.cat(org_input, dim=0).to(device)
else:
- current_input = [fp_input[i] for i in indices]
+ current_input = [fp_inputs[i] for i in indices]
current_input = torch.cat(current_input, dim=0).to(device)
org_input = current_input
with torch.no_grad():
@@ -553,6 +721,12 @@ class SignRoundQuantizer(BaseQuantizer):
dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
logger.info(dump_info)
+ def finalize_run(self) -> None:
+ """Clear per-run caches (``_cached_valid_token_mask``, LFQ components)."""
+ for attr in ("_cached_valid_token_mask", "_lm_head", "_post_block_modules"):
+ if hasattr(self, attr):
+ delattr(self, attr)
+
def _get_optimizer(self, optimizer: Any):
"""Returns the specified optimizer. In SignRound, we fix the optimizer.
diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.py
index c4fba7557..9bd45ae2d 100644
--- a/auto_round/calibration/llm.py
+++ b/auto_round/calibration/llm.py
@@ -337,6 +337,28 @@ class LLMCalibrator(Calibrator):
input_ids = data_new["input_ids"]
if input_ids.shape[-1] < self.seqlen:
continue
+ # Cache raw token IDs for quantize_block. Positions that should be
+ # excluded from the loss are marked with -100 (PyTorch's standard
+ # ignore index):
+ # • actual pad tokens (matched via pad_token_id when available, or
+ # detected as trailing repeated tokens otherwise)
+ # • the last token of every sample (no next-token target)
+ if "input_ids" not in self.inputs:
+ self.inputs["input_ids"] = []
+ ids_to_cache = input_ids.clone()
+ if self.tokenizer is not None and getattr(self.tokenizer, "pad_token_id", None) is not None:
+ ids_to_cache[ids_to_cache == self.tokenizer.pad_token_id] = -100
+ else:
+ # Heuristic: trailing repeated tokens are treated as padding.
+ for b in range(ids_to_cache.shape[0]):
+ last_token = ids_to_cache[b, -1]
+ j = ids_to_cache.shape[1] - 2
+ while j >= 0 and ids_to_cache[b, j] == last_token:
+ ids_to_cache[b, j] = -100
+ j -= 1
+ # Always exclude the last position (no supervision target).
+ ids_to_cache[:, -1] = -100
+ self.inputs["input_ids"].extend(list(torch.split(ids_to_cache.to("cpu"), 1, dim=0)))
if need_attention_mask:
if (
isinstance(data_new, dict)
@@ -374,9 +396,6 @@ class LLMCalibrator(Calibrator):
# Force the last token in each sequence to be masked out so the mask is
# never "all ones".
new_attention_mask[:, -1] = 0
- if "valid_token_mask" not in self.inputs:
- self.inputs["valid_token_mask"] = []
- self.inputs["valid_token_mask"].extend(list(torch.split(new_attention_mask, 1, dim=0)))
else:
new_attention_mask = None
diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.py
index cb18306e5..8291d1d82 100644
--- a/auto_round/cli/algorithms.py
+++ b/auto_round/cli/algorithms.py
@@ -334,6 +334,12 @@ class AutoRound(AlgorithmHandler):
action=argparse.BooleanOptionalAction,
help="Use the Adam-based SignRound variant.",
)
+ group.add_argument(
+ "--enable_lfq",
+ default=False,
+ action=argparse.BooleanOptionalAction,
+ help="Enable last-block LM cross-entropy (LFQ) loss for the final transformer block (experimental).",
+ )
def build(self, args, common_kwargs: dict[str, Any]):
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
@@ -351,6 +357,7 @@ class AutoRound(AlgorithmHandler):
not_use_best_mse=getattr(args, "not_use_best_mse", False),
enable_quanted_input=getattr(args, "enable_quanted_input", True),
enable_adam=getattr(args, "enable_adam", False),
+ enable_lfq=getattr(args, "enable_lfq", False),
**common_kwargs,
)
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index 6e0e5d12b..30bad9d2a 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -177,7 +177,7 @@ class CompressionOrchestrator(BaseOrchestrator):
nblocks: int = 1,
pbar: tqdm | None = None,
input_others_extra_blocks: dict | None = None,
- valid_token_mask: list[torch.Tensor] | None = None,
+ token_ids: list[torch.Tensor] | None = None,
):
"""Quantize and dequantize the weights of the specified blocks in the model.
@@ -229,7 +229,7 @@ class CompressionOrchestrator(BaseOrchestrator):
materialize_model_(m)
convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, device_manager.device)
- m, _, _ = self.alg_composer.dispatch_block(m, input_ids, input_others)
+ m = self.alg_composer.dispatch_block(m, input_ids, input_others)
# ── Pipeline lifecycle: per-block setup ───────────────────────────
from auto_round.algorithms.composer import BlockContext
@@ -250,6 +250,7 @@ class CompressionOrchestrator(BaseOrchestrator):
is_mllm=self.model_context.is_mllm,
is_diffusion=self.model_context.is_diffusion,
pbar=pbar,
+ block_cnt=(len(block_names) + nblocks - 1) // nblocks,
)
# ── Run block pipeline (calibration → quantization → collection) ──
@@ -258,8 +259,8 @@ class CompressionOrchestrator(BaseOrchestrator):
input_ids,
input_others,
block_ctx=ctx,
- q_input=q_input,
- valid_token_mask=valid_token_mask,
+ q_inputs=q_input,
+ input_ids=token_ids,
)
# ── Infrastructure: memory management ─────────────────────────────
@@ -513,8 +514,8 @@ class CompressionOrchestrator(BaseOrchestrator):
to_cache_layer_names,
last_cache_name=_last_cache_name,
)
- # Whether the token is pad token or not. For signround, the pad token should not be taken into account in loss
- valid_token_mask = all_inputs.pop("valid_token_mask", None)
+ # Raw token IDs from the tokenizer, cached during calibration for use in quantize_block.
+ input_ids_cache = all_inputs.pop("input_ids", None)
self.inputs = all_inputs
all_q_inputs = None
@@ -580,7 +581,7 @@ class CompressionOrchestrator(BaseOrchestrator):
nblocks=self.nblocks,
pbar=pbar,
input_others_extra_blocks=all_inputs,
- valid_token_mask=valid_token_mask,
+ token_ids=input_ids_cache,
)
if self.compress_context.is_immediate_packing and len(self.formats) != 1:
raise ValueError(
@@ -594,7 +595,7 @@ class CompressionOrchestrator(BaseOrchestrator):
pbar.close()
if self.compress_context.low_cpu_mem_usage:
self._offloader.reload(self.model_context.model)
- self._quantize_layers_outside_blocks(layer_names, all_inputs, valid_token_mask=valid_token_mask)
+ self._quantize_layers_outside_blocks(layer_names, all_inputs, token_ids=input_ids_cache)
convert_module_to_hp_if_necessary(
self.model_context.model, self.model_context.amp_dtype, device_manager.device, to_cpu=True
@@ -651,7 +652,10 @@ class CompressionOrchestrator(BaseOrchestrator):
# module.to("meta")
def _quantize_layers_outside_blocks(
- self, layer_names: list, layer_inputs: dict, valid_token_mask: list[torch.Tensor] | None = None
+ self,
+ layer_names: list,
+ layer_inputs: dict,
+ token_ids: list[torch.Tensor] | None = None,
) -> None:
"""Quantizes specified layers based on inputs and configuration.
@@ -691,7 +695,7 @@ class CompressionOrchestrator(BaseOrchestrator):
self.alg_composer.compress_layer_outside_block(
get_module(self.model, layer_name),
disable_opt_rtn=getattr(self, "disable_opt_rtn", False),
- valid_token_mask=valid_token_mask, # TODO wenhuach has not filter out loss
+ input_ids=token_ids,
)
layer_names.remove(layer_name)
if self.compress_context.is_immediate_packing:
@@ -732,7 +736,10 @@ class CompressionOrchestrator(BaseOrchestrator):
q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
q_layer_input = to_device(q_layer_input, self.compress_context.cache_device)
self.alg_composer.compress_layer_outside_block(
- get_module(self.model, layer_name), fp_input=layer_input, q_input=q_layer_input
+ get_module(self.model, layer_name),
+ fp_inputs=layer_input,
+ q_inputs=q_layer_input,
+ input_ids=token_ids,
)
if self.compress_context.is_immediate_packing:
immediate_pack(layer_name, self.layer_config)
@@ -847,7 +854,7 @@ class CompressionOrchestrator(BaseOrchestrator):
block_name=getattr(block, "global_name", ""),
block_index=0,
)
- self.alg_composer.compress_block(block, None, {}, block_ctx=ctx, q_inputs=None, valid_token_mask=None)
+ self.alg_composer.compress_block(block, None, {}, block_ctx=ctx, q_inputs=None)
mv_module_from_gpu(block)
return None, None
@@ -940,7 +947,7 @@ class CompressionOrchestrator(BaseOrchestrator):
input_ids,
input_others,
block_ctx=ctx,
- q_input=q_input,
+ q_inputs=q_input,
)
# ── Cleanup ───────────────────────────────────────────────────────────
diff --git a/test/test_cpu/core/test_llmc_quantize_block.py b/test/test_cpu/core/test_llmc_quantize_block.py
index ec6a8db40..450ad0f57 100644
--- a/test/test_cpu/core/test_llmc_quantize_block.py
+++ b/test/test_cpu/core/test_llmc_quantize_block.py
@@ -17,9 +17,7 @@ class _QuantizerStub:
def _get_block_outputs(self, block, input_ids, input_others, bs, save_output=True):
return torch.ones(1, 1)
- def quantize_block(
- self, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx=None, valid_token_mask=None, **kwargs
- ):
+ def quantize_block(self, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx=None, **kwargs):
return None