PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine alg and support lfq (#2082)
Wenhua Chengf1af646b5081
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex bf399f814..ce2d27d97 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -77,6 +77,7 @@ class BlockContext:is_mllm: bool = False # fail-fast gate for algorithms that don't support MLLMis_diffusion: bool = False # fail-fast gate for algorithms that don't support diffusionpbar: Any = None+ block_cnt: int = 0 # total number of blocks being quantized in this run# ---------------------------------------------------------------------------@@ -274,7 +275,7 @@ class AlgorithmComposer:handles.extend(self.block_quantizer.register_qinput_forward_hooks(block))return handles- def _attach_act_max_for_outside_layer(self, layer: "torch.nn.Module", fp_input, q_input) -> None:+ def _attach_act_max_for_outside_layer(self, layer: "torch.nn.Module", fp_inputs, q_inputs) -> None:"""Compute and attach act_max for an outside-block layer from cached inputs.Mirrors the hook-based act_max collection done for in-block layers, but@@ -282,14 +283,14 @@ class AlgorithmComposer:Args:layer: The layer module to attach act_max to.- fp_input: List of FP input tensors collected during calibration.- q_input: Optional list of quantized input tensors; used instead of- ``fp_input`` when provided.+ fp_inputs: List of FP input tensors collected during calibration.+ q_inputs: Optional list of quantized input tensors; used instead of+ ``fp_inputs`` when provided."""from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size- target_input = q_input or fp_input+ target_input = q_inputs or fp_inputsact_group_size = getattr(layer, "act_group_size")if act_group_size is None:act_group_size = layer.group_size@@ -340,7 +341,7 @@ class AlgorithmComposer:input_others,block_ctx: BlockContext,q_inputs=None,- valid_token_mask=None,+ input_ids=None,**kwargs,) -> tuple:"""Run the full per-block algorithm pipeline: calibration → quantization → collection.@@ -358,7 +359,7 @@ class AlgorithmComposer:input_ids: Full-precision (FP) cached inputs.input_others: Auxiliary kwargs (attention_mask, position_ids, …).ctx: Per-block lifecycle context (:class:`BlockContext`).- q_input: Quantized-input tensors from the previous block, or ``None``.+ q_inputs: Quantized-input tensors from the previous block, or ``None``.valid_token_mask: Optional mask for per-token loss weighting.Returns:@@ -441,7 +442,7 @@ class AlgorithmComposer:reference_output,q_inputs,block_ctx,- valid_token_mask=valid_token_mask,+ input_ids=input_ids,)# ── Step 5: post_quantize_block ─────────────────────────────────────────@@ -461,10 +462,10 @@ class AlgorithmComposer:def compress_layer_outside_block(self,layer: "torch.nn.Module",- fp_input=None,- q_input=None,- valid_token_mask=None,+ fp_inputs=None,+ q_inputs=None,disable_opt_rtn=None, # TODO wenhuach rename this to search_init_scale+ input_ids=None,) -> None:"""Quantize a single layer that lives outside transformer blocks.@@ -475,14 +476,14 @@ class AlgorithmComposer:Args:layer: The layer module to quantize. Must have a ``global_name``attribute.- fp_input: Per-sample FP activations for calibration, or ``None``+ fp_inputs: Per-sample FP activations for calibration, or ``None``to fall back to zero-shot RTN.- q_input: Optional quantized activations from a previous stage.+ q_inputs: Optional quantized activations from a previous stage.valid_token_mask: Per-sample token masks for loss weighting.disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config."""# Attach act_max for static activation quantization when inputs are available.- if fp_input is not None:+ if fp_inputs is not None:from auto_round.compressors.utils import is_nv_fpact_data_type = getattr(layer, "act_data_type")@@ -490,7 +491,7 @@ class AlgorithmComposer:act_data_type = "fp"act_dynamic = getattr(layer, "act_dynamic", True)if is_nv_fp(act_data_type) or not act_dynamic:- self._attach_act_max_for_outside_layer(layer, fp_input, q_input)+ self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)# Infrastructure: move layer to the tuning device before handing off to the quantizer.device = getattr(layer, "tuning_device", device_manager.device) # TODO this should be handled by compressor@@ -498,10 +499,10 @@ class AlgorithmComposer:return self.block_quantizer.quantize_layer_outside_block(layer,- fp_input=fp_input,- q_input=q_input,- valid_token_mask=valid_token_mask,+ fp_inputs=fp_inputs,+ q_inputs=q_inputs,disable_opt_rtn=disable_opt_rtn,+ input_ids=input_ids,)# ── Convenience act-calib helpers ────────────────────────────────────────diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex a977b43e3..c83d805d5 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -153,7 +153,7 @@ class BaseQuantizer(BaseAlgorithm):fp_outputs: "list[torch.Tensor]",q_inputs: "list[torch.Tensor] | None",block_ctx: "BlockContext",- valid_token_mask: "list[torch.Tensor] | None" = None,+ input_ids: "list[torch.Tensor] | None" = None,**kwargs,) -> dict:"""Apply the quantization algorithm to a prepared block.@@ -161,15 +161,16 @@ class BaseQuantizer(BaseAlgorithm):all infrastructure work (device placement, data collection, act-max hookregistration, DDP setup) has been completed.Args:- block: The transformer block module to quantize.- fp_inputs: FP calibration inputs (list[Tensor] or dict for diffusion).- input_others: Auxiliary kwargs (attention_mask, position_ids, …).- fp_outputs: FP reference outputs used as quantization targets.- q_inputs: Quantized inputs from the previous block, or ``None``.- block_ctx: Per-block pipeline context (:class:`BlockContext`).- valid_token_mask: Per-sample boolean masks ``[1, seq_len]``; ``None`` if- no padding masking is needed.- **kwargs: Reserved for future parameters.+ block: The transformer block module to quantize.+ fp_inputs: FP calibration inputs (list[Tensor] or dict for diffusion).+ input_others: Auxiliary kwargs (attention_mask, position_ids, …).+ fp_outputs: FP reference outputs used as quantization targets.+ q_inputs: Quantized inputs from the previous block, or ``None``.+ block_ctx: Per-block pipeline context (:class:`BlockContext`).+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per+ sample); used to derive the valid-token loss mask.+ ``None`` for diffusion or zero-shot paths.+ **kwargs: Reserved for future parameters.Returns:dict: Best quantization parameters found, or ``{}`` if not applicable.Raises:@@ -180,20 +181,34 @@ class BaseQuantizer(BaseAlgorithm):def quantize_layer_outside_block(self,layer: "torch.nn.Module",- fp_input: "list[torch.Tensor] | None" = None,- q_input: "list[torch.Tensor] | None" = None,+ fp_inputs: "list[torch.Tensor] | None" = None,+ q_inputs: "list[torch.Tensor] | None" = None,disable_opt_rtn: "bool | None" = None,- valid_token_mask: "list[torch.Tensor] | None" = None,+ input_ids: "list[torch.Tensor] | None" = None,) -> None:+ """Quantize a single layer outside a transformer block using RTN fallback.+ Args:+ layer: The layer module to quantize. Must have a+ ``global_name`` attribute for model re-insertion.+ fp_inputs: Optional FP calibration inputs; unused in base RTN.+ q_inputs: Optional quantized activations; unused in base RTN.+ disable_opt_rtn: ``True`` skips optimized-RTN scale/zp search.+ ``None`` defers to ``self.config.disable_opt_rtn``.+ input_ids: Raw token IDs from the tokenizer; used to derive+ the valid-token loss mask. ``None`` for RTN fallback.+ """+ self._quantize_layer_via_rtn(layer, disable_opt_rtn=disable_opt_rtn)"""Quantize a single layer outside a transformer block using RTN fallback.Args:layer: The layer module to quantize. Must have a``global_name`` attribute for model re-insertion.- fp_input: Optional FP calibration inputs; unused in base RTN.- q_input: Optional quantized activations; unused in base RTN.+ fp_inputs: Optional FP calibration inputs; unused in base RTN.+ q_inputs: Optional quantized activations; unused in base RTN.disable_opt_rtn: ``True`` skips optimized-RTN scale/zp search.``None`` defers to ``self.config.disable_opt_rtn``.valid_token_mask: Per-sample masks; unused in base RTN.+ input_ids: Original FP calibration inputs, same as ``fp_inputs``+ when ``q_inputs`` is ``None``; unused in base RTN."""self._quantize_layer_via_rtn(layer, disable_opt_rtn=disable_opt_rtn)@@ -252,14 +267,39 @@ class BaseQuantizer(BaseAlgorithm):raiseset_module(self.model, layer_name, layer)+ def _compute_valid_token_mask(self, input_ids: list) -> "list | None":+ """Derive a per-sample valid-token loss mask from raw token IDs.++ ``LLMCalibrator.calib`` already marks every position that should be+ excluded from the loss with ``-100`` (PyTorch's standard ignore index):+ pad tokens, trailing repeated tokens, and the last position of each+ sample. This method therefore only needs a single comparison.++ Args:+ input_ids: List of ``[1, seq_len]`` integer tensors (one per sample),+ with ignored positions already set to ``-100``.++ Returns:+ List of ``[1, seq_len]`` ``torch.long`` masks (1 = valid, 0 = ignore),+ or ``None`` when every position is valid (all-ones → fast path).+ """+ if not input_ids:+ return None++ masks = [(ids != -100).to(torch.long) for ids in input_ids]+ # All-ones → masking is a no-op; use the faster unmasked loss path.+ if all(m.all().item() for m in masks):+ return None+ return masks+def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):"""Place a block on the correct device(s) for quantization.Default: move to primary device.- Returns ``(block, card_0_in_high_risk, loss_device)``.+ Returns the block after device placement.Subclasses override for multi-GPU tensor-parallel dispatch."""block = block.to(device_manager.device)- return block, False, device_manager.device+ return block# ── Lifecycle hooks ───────────────────────────────────────────────────────def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.pyindex 95525b1e2..0bff58eb8 100644--- a/auto_round/algorithms/quantization/rtn/quantizer.py+++ b/auto_round/algorithms/quantization/rtn/quantizer.py@@ -38,7 +38,7 @@ class RTNQuantizer(BaseQuantizer):fp_outputs,q_inputs,block_ctx,- valid_token_mask=None,+ input_ids=None,**kwargs,) -> dict:"""Apply zero-shot RTN quantization to a block.@@ -54,10 +54,7 @@ class RTNQuantizer(BaseQuantizer):q_inputs: Quantized inputs from the previous block, or ``None`` whencascaded quantized-input is disabled.block_ctx: Per-block pipeline context (BlockContext).- valid_token_mask: Per-sample boolean/int masks of shape- ``[1, seq_len]`` indicating valid (non-padding) token positions.- ``1`` means valid, ``0`` means padding. ``None`` if no masking- is needed (e.g. standard string datasets without padding).+ input_ids: Raw token IDs from the tokenizer (unused in RTN).**kwargs: Reserved for forward-compatibility with future parameters.Returns:@@ -124,7 +121,7 @@ class OptimizedRTNQuantizer(RTNQuantizer):fp_outputs,q_inputs,block_ctx,- valid_token_mask=None,+ input_ids=None,**kwargs,):"""Apply imatrix-informed RTN quantization to a block.@@ -140,11 +137,7 @@ class OptimizedRTNQuantizer(RTNQuantizer):q_inputs: Quantized inputs from the previous block, or ``None`` whencascaded quantized-input is disabled.block_ctx: Per-block pipeline context (BlockContext).- valid_token_mask: Per-sample boolean/int masks of shape- ``[1, seq_len]`` indicating valid (non-padding) token positions.- ``1`` means valid, ``0`` means padding. ``None`` if no masking- is needed (e.g. standard string datasets without padding).- Currently unused in imatrix-RTN; reserved for future use.+ input_ids: Raw token IDs from the tokenizer (unused in RTN).**kwargs: Reserved for forward-compatibility with future parameters."""# Normalize imatrix and quantize layersdiff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.pyindex 4c883a130..823b10e05 100644--- a/auto_round/algorithms/quantization/sign_round/config.py+++ b/auto_round/algorithms/quantization/sign_round/config.py@@ -38,6 +38,7 @@ class SignRoundConfig(QuantizationConfig):enable_quanted_input: bool = True,optimizer: str | None = None, # TODO later wenhuach delete thisenable_adam: bool = False, # TODO later wenhuach delete this+ enable_lfq: bool = False,**kwargs,) -> None:"""Initialize a SignRound configuration.@@ -89,9 +90,6 @@ class SignRoundConfig(QuantizationConfig):self.momentum = momentumself.enable_alg_ext = enable_alg_ext- # Some helpers- self.infer_bs_coeff = 1-self.enable_minmax_tuning = enable_minmax_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuningif self.enable_norm_bias_tuning:@@ -101,6 +99,9 @@ class SignRoundConfig(QuantizationConfig):self.enable_quanted_input = enable_quanted_inputself.optimizer = optimizerself.enable_adam = enable_adam+ self.enable_lfq = enable_lfq+ if self.enable_lfq:+ logger.warning("the `enable_lfq` feature is experimental and currently has limited model support.")def finalize_scheme(self) -> None:"""Resolve lr/minmax_lr once `bits` is known (low-bit schemes use a higher lr)."""diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 7487fdfa9..0de9fffcb 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -60,6 +60,7 @@ class SignRoundQuantizer(BaseQuantizer):self.not_use_best_mse = config.not_use_best_mseself.enable_quanted_input = config.enable_quanted_inputself.dynamic_max_gap = config.dynamic_max_gap+ self.enable_lfq = config.enable_lfqself.optimizer = self._get_optimizer(optimizer=config.optimizer)self.wrapper_block = wrapper_block@@ -67,7 +68,8 @@ class SignRoundQuantizer(BaseQuantizer):def dispatch_block(self, block, input_ids, input_others):"""Multi-GPU aware block dispatch for SignRound tuning.- Stores card_0_in_high_risk and loss_device on self for use in quantize_block.+ Stores _card_0_in_high_risk and _loss_device on self for use in quantize_block.+ Returns the block after device placement."""from auto_round.utils import is_auto_device_mapping@@ -99,7 +101,7 @@ class SignRoundQuantizer(BaseQuantizer):self._card_0_in_high_risk = card_0_in_high_riskself._loss_device = loss_device- return block, card_0_in_high_risk, loss_device+ return blockdef _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:current_tensors = [tensor[i] for i in indices]@@ -116,6 +118,7 @@ class SignRoundQuantizer(BaseQuantizer):loss_func: Callable,device: Union[str, torch.device] = "cpu",valid_token_mask: Optional[torch.Tensor] = None,+ input_ids=None,):autocast_ctx = (nullcontext()@@ -149,6 +152,148 @@ class SignRoundQuantizer(BaseQuantizer):else:return inputs.shape[self.calibration_context.batch_dim]+ def _init_lm_components(self) -> None:+ """Lazily locate and cache ``_post_block_modules`` and ``_lm_head`` for LFQ loss.++ ``_post_block_modules`` is an ordered list of modules that must be applied+ between the last transformer-block output and the lm_head projection.+ Most architectures have a single final norm; OPT additionally has an+ optional ``project_out`` projection.++ Supported without manual configuration:+ LLaMA / Qwen / Gemma / Mistral / InternLM / Phi-3 — ``model.model.norm``+ OPT — ``model.model.decoder.{final_layer_norm, project_out}`` (both optional)+ GPT-2 / Falcon / Bloom — ``model.transformer.ln_f``+ GPT-NeoX / Pythia — ``model.gpt_neox.final_layer_norm``+ Phi / Phi-2 — ``model.model.final_layernorm``+ MPT — ``model.transformer.norm_f``+ ChatGLM — ``model.transformer.encoder.final_layernorm``+ RWKV — ``model.rwkv.ln_out``++ Raises ``AttributeError`` if no lm_head equivalent can be found.+ """+ if hasattr(self, "_lm_head"):+ return++ model = self.model++ # ── lm_head ──────────────────────────────────────────────────────────+ for name in ("lm_head", "embed_out", "output", "head"):+ if hasattr(model, name):+ self._lm_head = getattr(model, name)+ break+ else:+ raise AttributeError(+ f"Cannot locate lm_head in {type(model).__name__}. " "Checked: lm_head, embed_out, output, head."+ )++ # ── post-block processing (ordered list applied before lm_head) ───────+ # OPT: decoder has both an optional final_layer_norm *and* an optional+ # project_out that maps ffn_dim → word_embed_proj_dim. Detect by probing+ # for the characteristic project_out attribute (may be None).+ try:+ decoder = model.model.decoder+ _ = decoder.project_out # raises AttributeError if not an OPT decoder+ self._post_block_modules = [m for m in (decoder.final_layer_norm, decoder.project_out) if m is not None]+ return+ except AttributeError:+ pass++ # All other architectures: single optional final norm.+ norm_getters = [+ lambda: model.model.norm, # LLaMA / Qwen / Gemma / Mistral / InternLM / Phi-3+ lambda: model.transformer.ln_f, # GPT-2 / Falcon / Bloom+ lambda: model.gpt_neox.final_layer_norm, # GPT-NeoX / Pythia+ lambda: model.model.final_layernorm, # Phi / Phi-2+ lambda: model.transformer.norm_f, # MPT+ lambda: model.transformer.encoder.final_layernorm, # ChatGLM+ lambda: model.rwkv.ln_out, # RWKV+ ]+ self._post_block_modules = []+ for getter in norm_getters:+ try:+ norm = getter()+ if norm is not None:+ self._post_block_modules = [norm]+ break+ except AttributeError:+ continue++ # Keywords that identify non-text (visual / audio / multimodal) blocks.+ # LFQ loss is only meaningful for pure language-model decoder blocks.+ _NON_TEXT_BLOCK_KEYWORDS = frozenset(+ {+ "vis",+ "vision",+ "visual",+ "image",+ "img",+ "audio",+ "video",+ "patch",+ "pixel",+ "clip",+ "vit",+ "perceiver",+ "resampler",+ "connector",+ "projector",+ }+ )++ def _is_text_decoder_block(self, block_name: str) -> bool:+ """Return ``True`` if *block_name* refers to a text-decoder block.++ Blocks whose names contain any of the non-text keywords (vision, audio,+ image, …) are considered multimodal and excluded from LFQ loss.+ """+ name_lower = block_name.lower()+ return not any(kw in name_lower for kw in self._NON_TEXT_BLOCK_KEYWORDS)++ def lfq_loss(self, hidden_state: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:+ """Compute LM cross-entropy loss from the last block's hidden states.++ Applies every post-block module (final norm, optional projection, …) in+ order, then runs lm_head and computes next-token prediction loss.+ Positions marked with ``-100`` in *input_ids* are excluded from the loss.++ Args:+ hidden_state: Last block output, shape ``[batch, seq_len, hidden]``.+ input_ids: Token-ID labels with ``-100`` for ignored positions,+ shape ``[batch, seq_len]``.++ Returns:+ Scalar cross-entropy loss tensor.+ """+ self._init_lm_components()+ device = hidden_state.device++ for module in self._post_block_modules:+ module.to(device)+ hidden_state = module(hidden_state)++ self._lm_head.to(device)+ logits = self._lm_head(hidden_state)++ if hasattr(self.model, "loss_function"):+ loss = self.model.loss_function(+ logits=logits,+ labels=input_ids.to(device),+ vocab_size=self.model.config.vocab_size,+ )+ else:+ import torch.nn.functional as F++ # Standard causal-LM shift: predict token t+1 from hidden state t.+ shift_logits = logits[..., :-1, :].contiguous()+ shift_labels = input_ids[..., 1:].contiguous().to(device)+ loss = F.cross_entropy(+ shift_logits.view(-1, shift_logits.size(-1)),+ shift_labels.view(-1),+ ignore_index=-100,+ )+ return loss+def quantize_block(self,block,@@ -157,7 +302,7 @@ class SignRoundQuantizer(BaseQuantizer):fp_outputs,q_inputs,block_ctx,- valid_token_mask=None,+ input_ids=None,**kwargs,) -> dict:"""Apply the AutoRound optimization algorithm to a block.@@ -177,11 +322,10 @@ class SignRoundQuantizer(BaseQuantizer):q_inputs: Quantized inputs from the previous block, or ``None`` whencascaded quantized-input is disabled.block_ctx: Per-block pipeline context (BlockContext).- valid_token_mask: Per-sample boolean/int masks of shape- ``[1, seq_len]`` indicating valid (non-padding) token positions.- ``1`` means valid, ``0`` means padding. ``None`` if no masking- is needed (e.g. standard string datasets without padding).- When provided, the loss is computed only over valid token positions.+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per+ sample). Used to derive the valid-token loss mask once (result+ cached on ``self._cached_valid_token_mask`` for reuse across+ all blocks). ``None`` disables loss masking.**kwargs: Reserved for forward-compatibility with future parameters.Returns:@@ -193,6 +337,14 @@ class SignRoundQuantizer(BaseQuantizer):card_0_in_high_risk = getattr(self, "_card_0_in_high_risk", False)mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk+ valid_token_mask = None+ # Derive valid_token_mask from raw token IDs when not supplied by caller.+ # Result is cached on self so it is computed only once across all blocks.+ if input_ids is not None:+ if not hasattr(self, "_cached_valid_token_mask"):+ self._cached_valid_token_mask = self._compute_valid_token_mask(input_ids)+ valid_token_mask = self._cached_valid_token_mask+# Use quantized inputs if available and enabledactive_inputs = q_inputs if (q_inputs is not None and self.enable_quanted_input) else fp_inputsnsamples = len(active_inputs) if isinstance(active_inputs, list) else self._count_samples(active_inputs)@@ -265,6 +417,7 @@ class SignRoundQuantizer(BaseQuantizer):batch_size = self.calibration_context.batch_sizeglobal_batch_size = batch_size * self.gradient_accumulate_stepsglobal_batch_size = min(nsamples, global_batch_size)+ # Compute num_elm once before the loop (used to normalise the accumulated loss).# We assume the block input and output shape is sameif self.gradient_accumulate_steps != 1 and not valid_token_mask:whole_indices = torch.arange(global_batch_size)@@ -303,7 +456,15 @@ class SignRoundQuantizer(BaseQuantizer):pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)if loss_device is not None:pred_output = pred_output.to(loss_device)- loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)+ if (+ block_ctx.block_index == block_ctx.block_cnt - 1+ and self.enable_lfq+ and input_ids is not None+ and self._is_text_decoder_block(block_ctx.block_name)+ ):+ loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))+ else:+ loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)num_elm = 1 if num_elm <= 0 else num_elmtotal_loss += loss.item() / num_elm@@ -366,37 +527,35 @@ class SignRoundQuantizer(BaseQuantizer):def quantize_layer_outside_block(self,layer: "torch.nn.Module",- fp_input: Optional[list[torch.Tensor]] = None,- q_input: Optional[list[torch.Tensor]] = None,- valid_token_mask: Optional[list[torch.Tensor]] = None,+ fp_inputs: Optional[list[torch.Tensor]] = None,+ q_inputs: Optional[list[torch.Tensor]] = None,disable_opt_rtn: Optional[bool] = None,+ input_ids: Optional[list[torch.Tensor]] = None,):"""Quantize a single layer that lives outside a transformer block.- When ``fp_input`` is provided the layer is tuned with the sign-gradient+ When ``fp_inputs`` is provided the layer is tuned with the sign-gradientdescent optimizer (same loss loop as block-level quantization). When- ``fp_input`` is ``None`` the method falls back to zero-shot RTN.+ ``fp_inputs`` is ``None`` the method falls back to zero-shot RTN.Args:layer: The layer module to quantize. Must have a ``global_name``attribute for model re-insertion and logging.- fp_input: Per-sample FP activations fed into this layer, used as+ fp_inputs: Per-sample FP activations fed into this layer, used ascalibration inputs during optimization. ``None`` triggers RTNfallback.- q_input: Per-sample quantized activations from the previous stage,- used instead of ``fp_input`` during the forward pass when+ q_inputs: Per-sample quantized activations from the previous stage,+ used instead of ``fp_inputs`` during the forward pass whencascaded quantized-input is enabled. ``None`` means use- ``fp_input`` for both reference and tuning forward.- valid_token_mask: Per-sample boolean/int masks of shape- ``[1, seq_len]`` indicating valid (non-padding) token positions.- ``1`` means valid, ``0`` means padding. ``None`` if no masking- is needed. Forwarded to the loss computation to weight out- padding tokens.+ ``fp_inputs`` for both reference and tuning forward.disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config.+ input_ids: Raw token IDs from the tokenizer (``[1, seq_len]`` per+ sample); used to derive the valid-token loss mask via+ ``_compute_valid_token_mask``. ``None`` disables loss masking."""layer_name = layer.global_name- if fp_input is None:+ if fp_inputs is None:logger.info(f"using rtn to quantize {layer_name}")self._quantize_layer_via_rtn(layer,@@ -406,13 +565,21 @@ class SignRoundQuantizer(BaseQuantizer):)return+ # Derive valid_token_mask from raw token IDs when not supplied by caller.+ # Reuse the cached mask if already computed by a previous block.+ valid_token_mask = None+ if input_ids is not None:+ if not hasattr(self, "_cached_valid_token_mask"):+ self._cached_valid_token_mask = self._compute_valid_token_mask(input_ids)+ valid_token_mask = self._cached_valid_token_mask+logger.info(f"quantizing layer {layer_name}")# Layer is already on the correct device (placed by the caller / AlgorithmComposer).device = layer.weight.device if hasattr(layer, "weight") else device_manager.device- for i in range(len(fp_input)):- fp_input[i] = fp_input[i].to(layer.weight.dtype)- if q_input is not None:- q_input[i] = q_input[i].to(layer.weight.dtype)+ for i in range(len(fp_inputs)):+ fp_inputs[i] = fp_inputs[i].to(layer.weight.dtype)+ if q_inputs is not None:+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)wrapper_linear = WrapperLinear(layer,@@ -449,7 +616,7 @@ class SignRoundQuantizer(BaseQuantizer):)else:lr_schedule = copy.deepcopy(self.lr_scheduler)- nsamples = len(fp_input)+ nsamples = len(fp_inputs)last_best_iter = 0best_loss = torch.finfo(torch.float).maxbest_params = None@@ -469,30 +636,31 @@ class SignRoundQuantizer(BaseQuantizer):batch_size = 1 # Force to low gpuglobal_batch_size = gradient_accumulate_stepsglobal_batch_size = min(nsamples, global_batch_size)- if gradient_accumulate_steps != 1 and not valid_token_mask:- whole_indices = torch.arange(global_batch_size).tolist()- if q_input is not None:- num_elm = self._count_layer_input_elements(q_input, whole_indices)+ # Compute num_elm once before the loop.+ if gradient_accumulate_steps != 1:+ whole_indices = list(range(global_batch_size))+ if valid_token_mask:+ num_elm = self._get_non_zero_cnt(valid_token_mask, whole_indices)+ elif q_inputs is not None:+ num_elm = self._count_layer_input_elements(q_inputs, whole_indices)else:- num_elm = self._count_layer_input_elements(fp_input, whole_indices)+ num_elm = self._count_layer_input_elements(fp_inputs, whole_indices)index_sampler = IndexSampler(nsamples, global_batch_size)for i in range(self.iters):total_loss = 0global_indices = index_sampler.next_batch()- if valid_token_mask:- num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)for batch_start in range(0, len(global_indices), batch_size):indices = global_indices[batch_start : batch_start + batch_size]- if q_input is not None:- current_input = [q_input[i] for i in indices]+ if q_inputs is not None:+ current_input = [q_inputs[i] for i in indices]current_input = torch.cat(current_input, dim=0).to(device)- org_input = [fp_input[i] for i in indices]+ org_input = [fp_inputs[i] for i in indices]org_input = torch.cat(org_input, dim=0).to(device)else:- current_input = [fp_input[i] for i in indices]+ current_input = [fp_inputs[i] for i in indices]current_input = torch.cat(current_input, dim=0).to(device)org_input = current_inputwith torch.no_grad():@@ -553,6 +721,12 @@ class SignRoundQuantizer(BaseQuantizer):dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"logger.info(dump_info)+ def finalize_run(self) -> None:+ """Clear per-run caches (``_cached_valid_token_mask``, LFQ components)."""+ for attr in ("_cached_valid_token_mask", "_lm_head", "_post_block_modules"):+ if hasattr(self, attr):+ delattr(self, attr)+def _get_optimizer(self, optimizer: Any):"""Returns the specified optimizer. In SignRound, we fix the optimizer.diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.pyindex c4fba7557..9bd45ae2d 100644--- a/auto_round/calibration/llm.py+++ b/auto_round/calibration/llm.py@@ -337,6 +337,28 @@ class LLMCalibrator(Calibrator):input_ids = data_new["input_ids"]if input_ids.shape[-1] < self.seqlen:continue+ # Cache raw token IDs for quantize_block. Positions that should be+ # excluded from the loss are marked with -100 (PyTorch's standard+ # ignore index):+ # • actual pad tokens (matched via pad_token_id when available, or+ # detected as trailing repeated tokens otherwise)+ # • the last token of every sample (no next-token target)+ if "input_ids" not in self.inputs:+ self.inputs["input_ids"] = []+ ids_to_cache = input_ids.clone()+ if self.tokenizer is not None and getattr(self.tokenizer, "pad_token_id", None) is not None:+ ids_to_cache[ids_to_cache == self.tokenizer.pad_token_id] = -100+ else:+ # Heuristic: trailing repeated tokens are treated as padding.+ for b in range(ids_to_cache.shape[0]):+ last_token = ids_to_cache[b, -1]+ j = ids_to_cache.shape[1] - 2+ while j >= 0 and ids_to_cache[b, j] == last_token:+ ids_to_cache[b, j] = -100+ j -= 1+ # Always exclude the last position (no supervision target).+ ids_to_cache[:, -1] = -100+ self.inputs["input_ids"].extend(list(torch.split(ids_to_cache.to("cpu"), 1, dim=0)))if need_attention_mask:if (isinstance(data_new, dict)@@ -374,9 +396,6 @@ class LLMCalibrator(Calibrator):# Force the last token in each sequence to be masked out so the mask is# never "all ones".new_attention_mask[:, -1] = 0- if "valid_token_mask" not in self.inputs:- self.inputs["valid_token_mask"] = []- self.inputs["valid_token_mask"].extend(list(torch.split(new_attention_mask, 1, dim=0)))else:new_attention_mask = Nonediff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.pyindex cb18306e5..8291d1d82 100644--- a/auto_round/cli/algorithms.py+++ b/auto_round/cli/algorithms.py@@ -334,6 +334,12 @@ class AutoRound(AlgorithmHandler):action=argparse.BooleanOptionalAction,help="Use the Adam-based SignRound variant.",)+ group.add_argument(+ "--enable_lfq",+ default=False,+ action=argparse.BooleanOptionalAction,+ help="Enable last-block LM cross-entropy (LFQ) loss for the final transformer block (experimental).",+ )def build(self, args, common_kwargs: dict[str, Any]):from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig@@ -351,6 +357,7 @@ class AutoRound(AlgorithmHandler):not_use_best_mse=getattr(args, "not_use_best_mse", False),enable_quanted_input=getattr(args, "enable_quanted_input", True),enable_adam=getattr(args, "enable_adam", False),+ enable_lfq=getattr(args, "enable_lfq", False),**common_kwargs,)diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex 6e0e5d12b..30bad9d2a 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -177,7 +177,7 @@ class CompressionOrchestrator(BaseOrchestrator):nblocks: int = 1,pbar: tqdm | None = None,input_others_extra_blocks: dict | None = None,- valid_token_mask: list[torch.Tensor] | None = None,+ token_ids: list[torch.Tensor] | None = None,):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -229,7 +229,7 @@ class CompressionOrchestrator(BaseOrchestrator):materialize_model_(m)convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, device_manager.device)- m, _, _ = self.alg_composer.dispatch_block(m, input_ids, input_others)+ m = self.alg_composer.dispatch_block(m, input_ids, input_others)# ── Pipeline lifecycle: per-block setup ───────────────────────────from auto_round.algorithms.composer import BlockContext@@ -250,6 +250,7 @@ class CompressionOrchestrator(BaseOrchestrator):is_mllm=self.model_context.is_mllm,is_diffusion=self.model_context.is_diffusion,pbar=pbar,+ block_cnt=(len(block_names) + nblocks - 1) // nblocks,)# ── Run block pipeline (calibration → quantization → collection) ──@@ -258,8 +259,8 @@ class CompressionOrchestrator(BaseOrchestrator):input_ids,input_others,block_ctx=ctx,- q_input=q_input,- valid_token_mask=valid_token_mask,+ q_inputs=q_input,+ input_ids=token_ids,)# ── Infrastructure: memory management ─────────────────────────────@@ -513,8 +514,8 @@ class CompressionOrchestrator(BaseOrchestrator):to_cache_layer_names,last_cache_name=_last_cache_name,)- # Whether the token is pad token or not. For signround, the pad token should not be taken into account in loss- valid_token_mask = all_inputs.pop("valid_token_mask", None)+ # Raw token IDs from the tokenizer, cached during calibration for use in quantize_block.+ input_ids_cache = all_inputs.pop("input_ids", None)self.inputs = all_inputsall_q_inputs = None@@ -580,7 +581,7 @@ class CompressionOrchestrator(BaseOrchestrator):nblocks=self.nblocks,pbar=pbar,input_others_extra_blocks=all_inputs,- valid_token_mask=valid_token_mask,+ token_ids=input_ids_cache,)if self.compress_context.is_immediate_packing and len(self.formats) != 1:raise ValueError(@@ -594,7 +595,7 @@ class CompressionOrchestrator(BaseOrchestrator):pbar.close()if self.compress_context.low_cpu_mem_usage:self._offloader.reload(self.model_context.model)- self._quantize_layers_outside_blocks(layer_names, all_inputs, valid_token_mask=valid_token_mask)+ self._quantize_layers_outside_blocks(layer_names, all_inputs, token_ids=input_ids_cache)convert_module_to_hp_if_necessary(self.model_context.model, self.model_context.amp_dtype, device_manager.device, to_cpu=True@@ -651,7 +652,10 @@ class CompressionOrchestrator(BaseOrchestrator):# module.to("meta")def _quantize_layers_outside_blocks(- self, layer_names: list, layer_inputs: dict, valid_token_mask: list[torch.Tensor] | None = None+ self,+ layer_names: list,+ layer_inputs: dict,+ token_ids: list[torch.Tensor] | None = None,) -> None:"""Quantizes specified layers based on inputs and configuration.@@ -691,7 +695,7 @@ class CompressionOrchestrator(BaseOrchestrator):self.alg_composer.compress_layer_outside_block(get_module(self.model, layer_name),disable_opt_rtn=getattr(self, "disable_opt_rtn", False),- valid_token_mask=valid_token_mask, # TODO wenhuach has not filter out loss+ input_ids=token_ids,)layer_names.remove(layer_name)if self.compress_context.is_immediate_packing:@@ -732,7 +736,10 @@ class CompressionOrchestrator(BaseOrchestrator):q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else Noneq_layer_input = to_device(q_layer_input, self.compress_context.cache_device)self.alg_composer.compress_layer_outside_block(- get_module(self.model, layer_name), fp_input=layer_input, q_input=q_layer_input+ get_module(self.model, layer_name),+ fp_inputs=layer_input,+ q_inputs=q_layer_input,+ input_ids=token_ids,)if self.compress_context.is_immediate_packing:immediate_pack(layer_name, self.layer_config)@@ -847,7 +854,7 @@ class CompressionOrchestrator(BaseOrchestrator):block_name=getattr(block, "global_name", ""),block_index=0,)- self.alg_composer.compress_block(block, None, {}, block_ctx=ctx, q_inputs=None, valid_token_mask=None)+ self.alg_composer.compress_block(block, None, {}, block_ctx=ctx, q_inputs=None)mv_module_from_gpu(block)return None, None@@ -940,7 +947,7 @@ class CompressionOrchestrator(BaseOrchestrator):input_ids,input_others,block_ctx=ctx,- q_input=q_input,+ q_inputs=q_input,)# ── Cleanup ───────────────────────────────────────────────────────────diff --git a/test/test_cpu/core/test_llmc_quantize_block.py b/test/test_cpu/core/test_llmc_quantize_block.pyindex ec6a8db40..450ad0f57 100644--- a/test/test_cpu/core/test_llmc_quantize_block.py+++ b/test/test_cpu/core/test_llmc_quantize_block.py@@ -17,9 +17,7 @@ class _QuantizerStub:def _get_block_outputs(self, block, input_ids, input_others, bs, save_output=True):return torch.ones(1, 1)- def quantize_block(- self, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx=None, valid_token_mask=None, **kwargs- ):+ def quantize_block(self, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx=None, **kwargs):return None