Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

22.7 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import traceback
from typing import TYPE_CHECKING, Any

import torch

if TYPE_CHECKING:
    from auto_round.algorithms.composer import AlgorithmComposer, BlockContext

from auto_round.algorithms.base import BaseAlgorithm
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.logger import logger
from auto_round.utils import (
    check_to_quantized,
    clear_memory,
    convert_module_to_hp_if_necessary,
    set_module,
)
from auto_round.utils.device_manager import device_manager
from auto_round.wrapper import WrapperLinear


# TODO wenhuach annotate this class and functions clearly with details
class BaseQuantizer(BaseAlgorithm):
    """Base class for terminal weight-compression algorithms in a QuantizationPipeline.
    Developers adding a new quantization algorithm should inherit from this
    class and override at minimum :meth:`quantize_block`.
    Lifecycle hooks to override as needed:
        - :meth:`prepare_run`                  – model-level setup (once before all blocks)
        - :meth:`register_fp_input_forward_hooks` – register act-calib hooks
        - :meth:`quantize_block`               – **must override**: quantize a single block
        - :meth:`quantize_layer_outside_block` – quantize layers outside blocks
        - :meth:`finalize_run`                 – model-level teardown (once after all blocks)
    A quantizer whose calibration hooks write statistics on modules declares
    them in :attr:`calibration_stats`.
    """

    #: Statistics this quantizer's calibration hooks write on modules, mapped
    #: to their shard reduction ("sum" | "max"). Under --data_parallel
    #: the hook-carrying calibration forwards shard across the block mirrors
    #: and the engine folds these attributes back into the home block.
    #: ``None`` = undeclared: if the quantizer registers calibration hooks,
    #: those passes stay serial. A quantizer without hooks needs no declaration.
    calibration_stats: dict[str, str] | None = None

    def __init__(self, config: QuantizationConfig) -> None:
        super().__init__(config)
        # Whether to feed quantized-block outputs as inputs to the next block.
        # Subclasses that support cascaded quantized-input (e.g. SignRoundQuantizer)
        # override this from their config.  Defaults to False for zero-shot algorithms
        # (RTN) where activations are not used during weight optimization.
        self.enable_quanted_input = getattr(config, "enable_quanted_input", False)

    def can_compile_block_forward(self):
        return True

    # ── Calibration hook registration ─────────────────────────────────────────
    def register_fp_input_forward_hooks(self, block: torch.nn.Module) -> list:
        """Register hooks that fire during the reference (FP-input) block forward.
        Subclasses override to add statistics collection hooks (e.g. imatrix).
        Returns a list of hook handles that the caller must remove when done.
        Note: act_max hooks are registered by the Compressor, not by the quantizer.
        """
        return []

    def register_qinput_forward_hooks(self, block: torch.nn.Module) -> list:
        """Register hooks that fire during the quantized-input block forward.
        Used when act-calib policy requires collecting stats from quantized
        activations. Returns a list of hook handles that the caller must remove.
        Note: act_max hooks are registered by the Compressor, not by the quantizer.
        """
        return []

    # ── Embedding quantization ────────────────────────────────────────────────
    @torch.inference_mode()
    def quantize_embedding_layer(self) -> bool:
        """Quantize all embedding layers marked for quantization in the model.
        Iterates modules, applies the appropriate quantization function based on
        bits / group_size / dtype, and writes quantized weights + scale/zp back
        onto the module.
        Note:
            Most schemes do **not** quantize embeddings. Currently only GGUF
            formats require this. Subclasses rarely need to override.
        Returns:
            bool: ``True`` if at least one embedding layer was quantized.
        """
        is_quantized = False
        for name, module in self.model_context.model.named_modules():
            if not isinstance(module, torch.nn.Embedding):
                continue
            if not check_to_quantized(module):
                continue
            is_quantized = True
            bits = getattr(module, "bits", None)
            group_size = getattr(module, "group_size", None)
            sym = getattr(module, "sym", None)
            data_type = getattr(module, "data_type", None)
            super_bits = getattr(module, "super_bits", None)
            super_group_size = getattr(module, "super_group_size", None)
            scale_dtype = self.scale_dtype
            quant_dtype = data_type
            if quant_dtype not in QUANT_FUNC_WITH_DTYPE:
                quant_dtype = f"{quant_dtype}_{'sym' if sym else 'asym'}"
            if not hasattr(self, "iters") or self.iters <= 0:  # pylint: disable=E1101
                tmp_dtype = "rtn_" + quant_dtype
                if tmp_dtype in QUANT_FUNC_WITH_DTYPE:
                    quant_dtype = tmp_dtype
            quant_func = QUANT_FUNC_WITH_DTYPE[quant_dtype]
            # float32 is used in RTN scale search; avoids caching a bf16 copy.
            weight_dtype = torch.float32 if super_group_size is not None else module.weight.dtype
            quant_kwargs = {
                "bits": bits,
                "group_size": group_size,
                "super_bits": super_bits,
                "super_group_size": super_group_size,
                "scale_dtype": scale_dtype,
            }
            try:
                weight, scale, zp = quant_func(
                    module.weight.to(dtype=weight_dtype, device=device_manager.device),
                    **quant_kwargs,
                )
            except torch.OutOfMemoryError:
                cuda_error_msg = traceback.format_exc()
                logger.error(cuda_error_msg)
                logger.warning("falling back to CPU")
                weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)
            module.weight.data.copy_(weight.cpu())
            for param_name, val in zip(["scale", "zp"], [scale, zp]):
                if isinstance(val, dict):
                    for k, v in val.items():
                        setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
                elif isinstance(val, torch.Tensor):
                    setattr(module, param_name, val.cpu())
                else:
                    setattr(module, param_name, val)
            del weight, scale, zp
            clear_memory()
        return is_quantized

    # ── Abstract quantization interface ───────────────────────────────────────
    def parallel_constraints(self) -> dict:
        """Declare eligibility facts for the single-process data-parallel lane.

        The parallel engine consumes the declaration; algorithm internals stay
        quantizer internals: ``enable_lfq`` reports LFQ layers whose handling is
        single-device. A gradient scaler needs no declaration: every rank keeps
        its own, and the gradients are averaged before ``scaler.step``, so the
        ranks see the same inf/nan checks and keep the same scale.
        """
        return {"enable_lfq": bool(getattr(self, "enable_lfq", False))}

    def quantize_block(
        self,
        block: "torch.nn.Module",
        fp_inputs: "list[torch.Tensor] | dict",
        input_others: dict,
        fp_outputs: "list[torch.Tensor]",
        q_inputs: "list[torch.Tensor] | None",
        block_ctx: "BlockContext",
        input_ids: "list[torch.Tensor] | None" = None,
        **kwargs,
    ) -> dict:
        """Apply the quantization algorithm to a prepared block.
        This is the **pure-algorithm** entry point called by the Compressor after
        all infrastructure work (device placement, data collection, act-max hook
        registration, DDP setup) has been completed.
        Args:
            block:        The transformer block module to quantize.
            fp_inputs:    FP calibration inputs (list[Tensor] or dict for diffusion).
            input_others: Auxiliary kwargs (attention_mask, position_ids, …).
            fp_outputs:   FP reference outputs used as quantization targets.
            q_inputs:     Quantized inputs from the previous block, or ``None``.
            block_ctx:    Per-block pipeline context (:class:`BlockContext`).
            input_ids:    Raw token IDs from the tokenizer (``[1, seq_len]`` per
                          sample); used to derive the valid-token loss mask.
                          ``None`` for diffusion or zero-shot paths.
            **kwargs:     Reserved for future parameters.
        Returns:
            dict: Best quantization parameters found, or ``{}`` if not applicable.
                Leave common tuning wrappers installed: the composer loads these
                parameters, collects outputs and unwraps/restores FP weights.
                Algorithms that directly modify weights need no wrapper handling.
        Raises:
            NotImplementedError: Subclasses must override this method.
        """
        raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizer")

    def quantize_layer_outside_block(
        self,
        layer: "torch.nn.Module",
        fp_inputs: "list[torch.Tensor] | None" = None,
        q_inputs: "list[torch.Tensor] | None" = None,
        disable_opt_rtn: "bool | None" = None,
        input_ids: "list[torch.Tensor] | None" = None,
    ) -> None:
        """Quantize a single layer outside a transformer block using RTN fallback.
        Args:
            layer:           The layer module to quantize.  Must have a
                             ``global_name`` attribute for model re-insertion.
            fp_inputs:       Optional FP calibration inputs; unused in base RTN.
            q_inputs:        Optional quantized activations; unused in base RTN.
            disable_opt_rtn: ``True`` skips optimized-RTN scale/zp search.
                             ``None`` defers to ``self.config.disable_opt_rtn``.
            input_ids:       Raw token IDs from the tokenizer; used to derive
                             the valid-token loss mask. ``None`` for RTN fallback.
        """
        self._quantize_layer_via_rtn(layer, disable_opt_rtn=disable_opt_rtn)

    @torch.no_grad()
    def _quantize_layer_via_rtn(
        self, layer: "torch.nn.Module", disable_opt_rtn: "bool | None" = None, defer_search: bool = False
    ) -> "torch.nn.Module":
        """Quantize one layer with RTN (with optional optimized scale/zp search)."""
        layer_name = layer.global_name
        layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, device_manager.device)
        set_module(self.model, layer_name, layer)
        tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else device_manager.device
        try:
            if disable_opt_rtn is None:
                disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))
            if (
                not disable_opt_rtn
                and getattr(self.config, "orig_disable_opt_rtn", None) is None
                and self.model_context.is_moe_model
                and "expert" in layer.global_name
                and "shared_expert" not in layer.global_name
                and self.config.super_bits is None
            ):
                disable_opt_rtn = True
                logger.warning_once(
                    "MoE layer detected: optimized RTN is disabled for efficiency. "
                    "Use `--enable_opt_rtn` to force-enable it for MoE layers."
                )
            layer = layer.to(tuning_device)
            layer = WrapperLinear(
                layer,
                device=tuning_device,
                enable_minmax_tuning=False,
                enable_norm_bias_tuning=False,
                enable_round_tuning=False,
                enable_torch_compile=self.compress_context.enable_torch_compile,
                disable_opt_rtn=disable_opt_rtn,
                enable_neuqi=getattr(self.config, "enable_neuqi", False),
                iters=0,
            )
            if defer_search:
                # staged for the batched zero-shot search driver: the wrapper
                # (with its resolved quant func) is returned without running
                # the search; the caller finishes via unwrapper({}) or the
                # batched call + _apply_qdq + set_module
                return layer
            layer = layer.unwrapper({})
        except torch.OutOfMemoryError:
            from auto_round.algorithms.quantization.search_dispatch import dump_oom_tensor_census_

            dump_oom_tensor_census_("rtn layer search")
            cuda_error_msg = traceback.format_exc()
            layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer
            logger.error(cuda_error_msg)
            logger.warning("falling back to CPU.")
            layer.to("cpu")
            layer = WrapperLinear(
                layer,
                enable_minmax_tuning=False,
                enable_norm_bias_tuning=False,
                enable_round_tuning=False,
                enable_torch_compile=self.compress_context.enable_torch_compile,
                disable_opt_rtn=disable_opt_rtn,
                enable_neuqi=getattr(self.config, "enable_neuqi", False),
                iters=0,
            )
            layer = layer.unwrapper({})
        set_module(self.model, layer_name, layer)

    def _compute_valid_token_mask(self, input_ids: list) -> "list | None":
        """Derive a per-sample valid-token loss mask from raw token IDs.

        ``LLMCalibrator.calib`` already marks every position that should be
        excluded from the loss with ``-100`` (PyTorch's standard ignore index):
        pad tokens, trailing repeated tokens, and the last position of each
        sample.  This method therefore only needs a single comparison.

        Args:
            input_ids: List of ``[1, seq_len]`` integer tensors (one per sample),
                with ignored positions already set to ``-100``.

        Returns:
            List of ``[1, seq_len]`` ``torch.long`` masks (1 = valid, 0 = ignore),
            or ``None`` when every position is valid (all-ones → fast path).
        """
        if not input_ids:
            return None

        masks = [(ids != -100).to(torch.long) for ids in input_ids]
        # All-ones → masking is a no-op; use the faster unmasked loss path.
        if all(m.all().item() for m in masks):
            return None
        return masks

    # ── Tune-loop memory policy hooks (override to customize) ─────────────────

    def maybe_adapt_moe_implementation(self, block: "torch.nn.Module", tensors: Any, batch_size: int) -> None:
        """Switch the MoE experts implementation to the per-expert loop when the
        grouped path's working set is not expected to fit next to the tuning
        state (``AR_MOE_EXPERTS_IMPL=auto``; explicit values are honored as-is).

        Runs once per run; no-op for blocks without grouped expert stacks.
        The decision reads only the block's wrapper layout, the batch size, and
        the fleet's free memory, so algorithms rarely need to override this.
        """
        from auto_round.algorithms.quantization.tune_memory import _maybe_auto_linear_loop_for_tuning

        _maybe_auto_linear_loop_for_tuning(
            block,
            tensors,
            batch_size,
            getattr(self, "iters", 0),
            getattr(getattr(self, "model_context", None), "config", None),
            self.model,
        )

    def pull_tuning_pool(
        self,
        pool: Any,
        target_device: str,
        block: "torch.nn.Module",
        batch_size: int,
        label: str,
        charge_activation: bool = True,
    ) -> Any:
        """Bulk-move a calibration pool onto the device that reads it every
        iteration when its footprint fits next to the working set; a declined
        pull keeps the per-batch gather.

        Bulk pulls are a capacity optimization only: when the user explicitly
        parked memory on the host (``low_gpu_mem_usage`` or
        ``calibration_data_device="cpu"``) the pool stays where that policy
        put it, however much room the target device has. Under data-parallel
        thread ranks the engine already placed each rank's shard on its device,
        so the pool stays as it is.
        """
        from auto_round.algorithms.parallel.spmd import current_rank

        if current_rank() is not None:
            return pool
        cc = self.compress_context
        if (
            getattr(cc, "low_gpu_mem_usage", False)
            or str(getattr(cc, "calibration_data_device", "auto") or "auto").lower() == "cpu"
        ):
            return pool
        from auto_round.algorithms.quantization.tune_memory import _pull_pool_if_fits

        config = getattr(getattr(self, "model_context", None), "config", None)
        return _pull_pool_if_fits(
            pool,
            str(target_device),
            block,
            batch_size,
            getattr(self, "iters", 0),
            label,
            charge_activation=charge_activation,
            config=config,
        )

    def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):
        """Place a block on the correct device(s) for quantization.
        Default: move to primary device.
        Returns the block after device placement.
        Subclasses override for multi-GPU tensor-parallel dispatch.
        """
        block = block.to(device_manager.device)
        return block

    def finalize_block(self, block: "torch.nn.Module", block_ctx: "BlockContext | None" = None) -> None:
        """Post-tuning, quantizer-agnostic finalization for a compressed block.

        Invoked by the composer right after :meth:`quantize_block` returns, so
        format-specific post-processing stays out of the per-algorithm tuning
        loop. The default generates MoE experts' ``act_max`` automatically for
        NVFP static-activation schemes (``WrapperWALayer``); it is a no-op for
        schemes/blocks that do not need it. Subclasses may override to add their
        own finalization.
        """
        from auto_round.schemes import is_nvfp4_e5m3_gs
        from auto_round.utils import set_amax_for_all_moe_layers

        if self.config.is_act_nv_fp or is_nvfp4_e5m3_gs(self.config.act_data_type or ""):
            # enable moe experts act_max automatic generation for WrapperWALayer
            set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")

    # ── Lifecycle hooks ───────────────────────────────────────────────────────
    #: Algorithms that tune the whole model jointly (a single forward to the LM
    #: head) instead of block-by-block set this to ``True``. The composer then
    #: calls :meth:`quantize_model` once and skips the per-block loop.
    whole_model_tuning: bool = False

    #: The composer preserves original FP weights across all preceding stages
    #: when a consumer needs them (e.g. to construct a full-precision teacher).
    #: Algorithms never manage preservation flags or stage cleanup themselves.
    requires_original_weights: bool = False

    def get_calibration_token_ids(self, token_ids, calibration):
        """Resolve this algorithm's sample count; override for custom sampling.

        Reuse the common cached samples when nsamples is unset, take a prefix
        when fewer are needed, or collect raw tokens without block forwards.
        """
        target = self.config.nsamples
        if token_ids is not None:
            if target is None or target == len(token_ids):
                return token_ids
            if target < len(token_ids):
                logger.info("Algorithm stage uses the first %d of %d calibration samples.", target, len(token_ids))
                return list(token_ids)[:target]
        if target is None:
            target = calibration.nsamples
        collect = getattr(calibration, "collect_token_ids", None)
        if callable(collect):
            logger.info("Collecting %d calibration samples for the algorithm stage.", target)
            collected = collect(target)
            if collected is not None and len(collected) > 0:
                return collected
        logger.warning("Cannot collect %d calibration token ids; falling back to the cached samples.", target)
        return token_ids

    def quantize_model(self, model, token_ids, device=None) -> dict:
        """Quantize the entire model jointly (whole-network tuning).

        Only invoked when :attr:`whole_model_tuning` is ``True``. The default
        implementation is intentionally absent; whole-model algorithms (e.g. QAD)
        override it to run their own full-model forward/optimize loop.

        Args:
            model: The model to quantize (all quantizable layers tuned jointly).
            token_ids: Per-sample calibration token-id tensors.
            device: Target compute device.

        Returns:
            Best tuning parameters, with common wrappers left installed for
            composer finalization, just like quantize_block. In-place algorithms
            may return an empty dict.
        """
        raise NotImplementedError(
            f"{type(self).__name__} sets whole_model_tuning=True but does not implement quantize_model()."
        )

    def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:
        """Model-level preparation (called once before block iteration starts)."""
        return

    def finalize_run(self) -> None:
        """Model-level teardown (called once after all blocks are processed).
        Must be idempotent — the Compressor calls this inside a ``try/finally``.
        """
        return