Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

45.8 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Algorithm Fusion Pipeline abstraction.

This module defines the core data structures and utilities for composing
pre-processing algorithms (AWQ smooth, SmoothQuant, Rotation…) with a
block-quantization algorithm (RTN, SignRound/AutoRound…) into a single
shared-calibration bundle.

Design invariants (see AWQ_REFACTOR_PLAN.md §0.0 and §3.0):
- ``AlgorithmComposer`` is the *first-class abstraction*; it is NOT just
  AWQ's helper.
- All block-wise scheduling in ``Compressor`` operates against
  ``AlgorithmComposer``, never against a concrete ``AWQTransform``.
- Single-algorithm use is expressed as
  ``AlgorithmComposer(preprocessors=[], block_quantizer=q)``, which is
  semantically identical to the current direct-quantizer path.
"""

from __future__ import annotations

from contextlib import nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any

import torch

from auto_round.algorithms.block_runner import BlockForwardRunner
from auto_round.algorithms.config_resolver import (
    get_algorithm_class,
    resolve_shared_config_values,
    split_quantization_configs,
)
from auto_round.algorithms.parallel.data_parallel import (
    ENGINE_CALIBRATION_STATS,
    calibration_merge_stats,
    parallel_state,
)
from auto_round.algorithms.utils import _has_nvfp4_layer
from auto_round.logger import logger
from auto_round.utils import clear_memory
from auto_round.utils.device_manager import device_manager
from auto_round.wrapper import preserve_original_weights, store_warm_start_params_and_unwrap, unwrapper_block

if TYPE_CHECKING:  # avoid circular imports at runtime
    from auto_round.algorithms.quantization.base import BaseQuantizer
    from auto_round.algorithms.quantization.config import QuantizationConfig
    from auto_round.algorithms.transforms.base import BasePreprocessor
    from auto_round.compressors import BaseOrchestrator


# ---------------------------------------------------------------------------
# Context dataclasses
# ---------------------------------------------------------------------------
@dataclass
class BlockContext:
    """Per-block context threaded through the lifecycle hooks.

    Passed to lifecycle methods like ``block_forward_hooks``,
    ``pre_quantize_block``, ``post_quantize_block``, etc.

    ``block_names`` preserves the *scheduling group* (which may contain more
    than one block when ``nblocks > 1``).  Pre-processing algorithms that
    only support single-block operation (e.g. AWQ Phase-1) must check
    ``len(block_names) == 1`` in ``prepare_block_group`` and raise
    ``ValueError`` with a user-readable message.
    """

    model: torch.nn.Module
    block_names: list[str]  # scheduling group; len > 1 when nblocks > 1
    block_name: str  # = block_names[0] for single-block; descriptive label for multi
    block_index: int  # 0-based index within the current all_blocks group
    bs: int = 1
    is_mllm: bool = False  # fail-fast gate for algorithms that don't support MLLM
    is_diffusion: bool = False  # fail-fast gate for algorithms that don't support diffusion
    pbar: Any = None
    block_cnt: int = 0  # total number of blocks being quantized in this run
    # Ordered names of every block in the current all_blocks group (used e.g. by the
    # LFQ loss to forward a non-final block's output through the remaining FP blocks).
    all_block_names: list[str] | None = None


# ---------------------------------------------------------------------------
# AlgorithmComposer
# ---------------------------------------------------------------------------
def _release_pool_inplace(obj: Any) -> None:
    """Release a calibration pool's tensors by mutating its containers in place.

    The pool object is shared with the orchestrator (which holds it as the
    block's ``input_ids``), so dropping a local reference frees nothing. The
    containers are the same objects on both sides; setting their entries to
    None drops the last strong references to the tensors (hooks are removed
    and the tuning loop reads the q pool, so nothing re-reads this pool).
    Immutable containers (tuples) are skipped: their tensors stay alive until
    the block-end release, matching the pre-existing behavior.
    """
    if isinstance(obj, list):
        for i in range(len(obj)):
            obj[i] = None
    elif isinstance(obj, dict):
        for v in obj.values():
            _release_pool_inplace(v)
    return obj


class AlgorithmComposer:
    """An ordered composition of preprocessors and quantizer stages, built from
    a list of algorithm config objects and an optional compressor.

    The ``preprocessors`` list is order-sensitive: preprocessors are applied in
    the listed order (e.g. ``[Rotation, AWQ]``).  The block quantizer is always
    the terminal weight-compression step, regardless of its position in the
    input config list. Quantizers execute in their relative config order;
    ``block_quantizer`` exposes the active stage (the terminal stage when idle).

    Usage::

        composer = AlgorithmComposer(configs, compressor=self)
    """

    def __init__(self, configs: list, orchestrator: BaseOrchestrator = None) -> None:
        """Build the pipeline from a list of algorithm config instances.

        Resolution rules:

        1. If no ``QuantizationConfig`` with a ``BaseQuantizer`` is found in
           *configs*, a default :class:`RTNConfig` is appended automatically.
        2. ``BasePreprocessor`` instances go into ``preprocessors`` (in order).
        3. ``BaseQuantizer`` instances form an ordered stage sequence.
        4. Every stage validates its capabilities before execution.
        5. If *compressor* is provided, every member is bound to it and
           ``block_forward`` / quantization metadata are extracted from it.

        Args:
            configs:    List of algorithm config objects (``QuantizationConfig``
                        subclasses such as :class:`RTNConfig`, :class:`SignRoundConfig`,
                        :class:`AWQConfig`, …).
            compressor: The :class:`~auto_round.compressors.base.BaseCompressor`
                        instance driving this pipeline.  When supplied, members are
                        bound to it and ``block_forward`` is taken from it.
        """
        from auto_round.algorithms.quantization.base import BaseQuantizer
        from auto_round.algorithms.quantization.config import QuantizationConfig
        from auto_round.algorithms.transforms.base import BasePreprocessor, BaseRotationConfig

        self._orchestrator = orchestrator
        configs = list(configs)

        # Rotation configs travel in the same config list but are not block
        # quantizers / preprocessors (they are ``BaseRotationConfig``, not
        # ``QuantizationConfig``). Capture them here and wrap each in a
        # ``RotationPreprocessor`` member below so the orchestrator stays
        # rotation-agnostic and every rotation owns its own lifecycle.
        #
        # Whether rotation runs per-block (layer-wise) is a field on each
        # rotation config (``BaseRotationConfig.layerwise``).
        self._rotation_configs = [c for c in configs if isinstance(c, BaseRotationConfig)]

        _, block_quantizer_configs = split_quantization_configs(configs)
        if not block_quantizer_configs:
            from auto_round.algorithms.quantization.rtn.config import RTNConfig

            configs = configs + [RTNConfig()]

        # Sequential quantizers own independent schedules/behavior, but operate
        # on the same quantization scheme. Do not reconcile their local settings.
        configs = resolve_shared_config_values(configs, reconcile_non_scheme=len(block_quantizer_configs) <= 1)

        preprocessors: list = []
        block_quantizers: list = []

        for cfg in configs:
            if not isinstance(cfg, QuantizationConfig):
                continue
            from auto_round.algorithms.registry import normalize_algorithm_config

            cfg = normalize_algorithm_config(cfg)
            alg_cls = get_algorithm_class(cfg)
            if alg_cls is None:
                raise ValueError(f"Unknown algorithm config type {type(cfg).__name__!r}.")
            q = alg_cls(cfg)
            if orchestrator is not None:
                q.bind(orchestrator)
            if isinstance(q, BasePreprocessor):
                preprocessors.append(q)
            elif isinstance(q, BaseQuantizer):
                block_quantizers.append(q)
            else:
                raise TypeError(
                    f"Algorithm class {type(q).__name__} must inherit either " "BasePreprocessor or BaseQuantizer."
                )

        if len(block_quantizers) == 0:
            raise ValueError("No block quantizer found in configs.")
        self.quantizers = tuple(block_quantizers)
        self._preserve_weights = False
        self._preserve_stage_weights = []
        needs_original_weights = False
        for quantizer in reversed(self.quantizers):
            self._preserve_stage_weights.append(needs_original_weights)
            if quantizer.config.nsamples is not None and not quantizer.whole_model_tuning:
                raise NotImplementedError(
                    f"{type(quantizer.config).__name__}.nsamples overrides are only supported for whole-model tuning. "
                    "Use the common calibration nsamples argument for this stage."
                )
            needs_original_weights = needs_original_weights or quantizer.requires_original_weights
        self._preserve_stage_weights.reverse()

        seen = set()
        for pre in preprocessors:
            name = type(pre).__name__
            if name in seen:
                raise ValueError(
                    f"Duplicate preprocessor {name} in AlgorithmComposer. "
                    "Repeated instances of the same preprocessor are not supported yet."
                )
            seen.add(name)

        self.preprocessors = preprocessors

        can_compile_block_forward = False
        self.block_quantizer = block_quantizers[-1]
        self.block_forward = None
        if orchestrator is not None:
            user_torch_compile = bool(
                getattr(getattr(orchestrator, "compress_context", None), "enable_torch_compile", False)
            )
            rotation_configs = getattr(orchestrator, "rotation_configs", ())
            blockers = []
            for quantizer in self.quantizers:
                if not quantizer.can_compile_block_forward():
                    blockers.append(type(quantizer).__name__)
            for component in [*self.preprocessors, *rotation_configs]:
                if not getattr(component, "can_compile_block_forward", lambda: True)():
                    blockers.append(type(component).__name__)
            can_compile_block_forward = user_torch_compile and not blockers
            if user_torch_compile and blockers:
                logger.info(
                    "Block-forward torch.compile is disabled because %s is incompatible.",
                    ", ".join(blockers),
                )

            if _has_nvfp4_layer(orchestrator):
                can_compile_block_forward = False
                logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")

            # Bind compressor-level infrastructure (set before _build_quantizer is called).
            self.block_forward = (
                BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)
                if orchestrator is not None
                else None
            )
            # A little tricky
            for quantizer in self.quantizers:
                quantizer.bind_block_forward_runner(self.block_forward)
        self.scheme = getattr(orchestrator, "scheme_context", None)

        # Rotation is modelled as ordinary pipeline members (see
        # ``RotationPreprocessor``) that own their entire lifecycle. They are
        # built from the rotation configs captured above and bound like every
        # other member so ``finalize_run`` can reach the live model.
        from auto_round.algorithms.transforms.member import RotationPreprocessor

        self._rotation_members = [RotationPreprocessor(cfg) for cfg in self._rotation_configs]
        if orchestrator is not None:
            for rotation_member in self._rotation_members:
                rotation_member.bind(orchestrator)
        # Guards the one-shot model-level rotation stage (apply_model_transforms).
        self._rotation_prepared: bool = False

    def compress_model(self, token_ids, run_blocks) -> bool:
        """Run ordered block/model tuning stages using the bound compressor.

        Returns True when the terminal stage handles the whole model (including
        outside-block layers). Preprocessors/rotations run only in the first
        block stage. The composer owns weight preservation and cleanup; algorithms
        only implement block/model tuning and optionally customize sampling.
        """
        orchestrator = self._orchestrator
        if orchestrator is None:
            raise RuntimeError("compress_model requires a bound compressor.")
        streaming_export = (
            orchestrator.compress_context.is_immediate_saving or orchestrator.compress_context.is_immediate_packing
        )
        if streaming_export and (len(self.quantizers) > 1 or self.block_quantizer.whole_model_tuning):
            raise NotImplementedError("Multi-stage/whole-model tuning does not support immediate saving/packing.")
        from auto_round import envs

        if envs.AR_RESUME_DIR and len(self.quantizers) > 1:
            raise NotImplementedError("Resuming a multi-stage quantization run is not supported.")
        if self.quantizers[0].whole_model_tuning and (self.preprocessors or self.has_layerwise_rotation):
            raise NotImplementedError("Block preprocessors/layer-wise rotation require a leading block-tuning stage.")
        terminal = self.block_quantizer
        preprocessors, rotations = self.preprocessors, self._rotation_members
        try:
            self.prepare_run()
            for index, (quantizer, preserve) in enumerate(zip(self.quantizers, self._preserve_stage_weights)):
                self.block_quantizer = quantizer
                self._preserve_weights = preserve
                if index:
                    self.preprocessors, self._rotation_members = [], []
                logger.info(
                    "Starting algorithm stage %d/%d: %s.", index + 1, len(self.quantizers), type(quantizer).__name__
                )
                if quantizer.whole_model_tuning:
                    orchestrator._reload_model_for_tuning()
                    runner = getattr(self, "block_forward", None)
                    if runner is not None:
                        runner.pool_placement = None
                    stage_token_ids = quantizer.get_calibration_token_ids(token_ids, orchestrator.calibration)
                    self._run_tuning(
                        orchestrator.model_context.model,
                        lambda: quantizer.quantize_model(
                            orchestrator.model_context.model, stage_token_ids, device=device_manager.device
                        ),
                    )
                else:
                    run_blocks(index > 0)
            return terminal.whole_model_tuning
        finally:
            self.block_quantizer = terminal
            self._preserve_weights = False
            self.preprocessors, self._rotation_members = preprocessors, rotations
            self.finalize_run()

    def _run_tuning(self, module, tune, collect_output=None):
        """Common block/model tuning finalization, including failure cleanup."""
        preserve = getattr(self, "_preserve_weights", False)
        with preserve_original_weights(module) if preserve else nullcontext():
            try:
                best_params = tune() or {}
                if preserve:
                    store_warm_start_params_and_unwrap(module, best_params, before_unwrap=collect_output)
                else:
                    unwrapper_block(module, best_params)
            except BaseException:
                # Do not bake partially tuned parameters when optimization or
                # output collection fails. Remove any remaining common wrappers.
                from auto_round.utils import set_module

                for name, wrapped in list(module.named_modules()):
                    if hasattr(wrapped, "orig_layer"):
                        set_module(module, name, wrapped.orig_layer)
                raise

    # ── Internal hook helpers (act_max calibration) ───────────────────────────

    def _register_act_max_hooks(self, block: torch.nn.Module) -> list:
        """Register per-module act_max tracking hooks for static activation quantization.

        Returns a list of hook handles that the caller must remove when done.
        """
        from auto_round.data_type.utils import register_act_max_hooks

        return register_act_max_hooks(block, getattr(self.block_quantizer.config, "act_data_type", None))

    def _get_fp_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
        """Register FP-input act_max + quantizer forward hooks.

        Returns ``(handles, merge_stats)``: the shard-merge map of the
        statistics these hooks write (``None``: the pass stays serial, see
        ``calibration_merge_stats``).
        """
        if not self.need_quanted_input():
            # If having q_input, act_max will be collected in q_input forward hook,
            # no need to collect in fp_input forward hook
            handles = self._register_act_max_hooks(block)
        else:
            handles = []
        quantizer_hooks = self.block_quantizer.register_fp_input_forward_hooks(block)
        handles.extend(quantizer_hooks)
        return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)

    def _get_q_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
        """Register Q-input act_max + quantizer forward hooks; returns
        ``(handles, merge_stats)`` like ``_get_fp_act_hooks``."""
        handles = self._register_act_max_hooks(block)
        quantizer_hooks = self.block_quantizer.register_qinput_forward_hooks(block)
        handles.extend(quantizer_hooks)
        return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)

    def _attach_act_max_for_outside_layer(self, layer: torch.nn.Module, fp_inputs, q_inputs) -> None:
        """Compute and attach act_max for an outside-block layer from cached inputs.

        Mirrors the hook-based act_max collection done for in-block layers, but
        iterates over already-cached tensors directly instead of running a forward pass.

        Args:
            layer: The layer module to attach act_max to.
            fp_inputs: List of FP input tensors collected during calibration.
            q_inputs: Optional list of quantized input tensors; used instead of
                ``fp_inputs`` when provided.
        """
        from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
        from auto_round.schemes import needs_nvfp_global_scale as is_nv_fp

        target_input = q_inputs or fp_inputs
        act_group_size = layer.act_group_size
        if act_group_size is None:
            act_group_size = layer.group_size
        act_data_type = layer.act_data_type
        if act_data_type is None:
            act_data_type = layer.data_type
        is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False

        for inp in target_input:
            if isinstance(inp, (tuple, list)):
                inp = inp[0]
            if inp.numel() == 0:
                continue
            inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)
            act_max = torch.max(torch.abs(inp), dim=-1).values

            if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:
                layer.act_max = act_max
                if is_act_nv_fp_flag:
                    max_val = act_max.max()
                    layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
                continue

            act_max = act_max.to(layer.act_max.device)
            if is_act_nv_fp_flag:
                max_val = torch.max(act_max.max(), layer.act_max.max())
                layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
            else:
                layer.act_max = torch.max(act_max, layer.act_max)

    def need_quanted_input(self):
        for preprocessor in self.preprocessors:
            if getattr(preprocessor, "enable_quanted_input", False):
                return True
        return bool(getattr(self.block_quantizer, "enable_quanted_input", False))

    def compress_embedding_layer(self):
        return self.block_quantizer.quantize_embedding_layer()

    def _collection_context(self, block, fp_inputs):
        """TuneParallelContext for the collection phase (P3): mirror devices
        for sharding the no-grad collection forwards, with the same engagement
        resolver as the tune so the gates can never diverge."""
        from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext

        return TuneParallelContext.for_collection(self, block, fp_inputs)

    def _collect_forward_timed(self, *args, **kwargs):
        """_collect_forward with wall accumulation into self.last_collect_wall."""
        import time as _ctime

        _t0 = _ctime.perf_counter()
        try:
            return self._collect_forward(*args, **kwargs)
        finally:
            self.last_collect_wall = getattr(self, "last_collect_wall", 0.0) + (_ctime.perf_counter() - _t0)

    def _collect_forward(
        self,
        block,
        inputs,
        input_others,
        out_dev=None,
        allow_shard: bool = True,
        hook_pass: bool = False,
        merge_stats=ENGINE_CALIBRATION_STATS,
    ):
        """Collection forward: sharded across DDP mirrors when eligible.

        ``out_dev`` optionally overrides the runner's output cache device.
        ``allow_shard=False`` forces the serial path. ``merge_stats`` is the
        shard-merge map of the statistics the pass's hooks write (from
        ``_get_fp_act_hooks`` / ``_get_q_act_hooks``); ``None`` keeps the
        pass serial (undeclared quantizer statistics).

        ``hook_pass=True`` may cap the concurrent shards (env-driven; default
        no cap): forward hooks
        force dynamo graph breaks, leaving the compiled runner as
        python-bound eager sections that GIL-convoy under many threads.
        """
        if self._coll_ctx is None:
            from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext

            self._coll_ctx = TuneParallelContext()
        return self._coll_ctx.collect_forward(
            self.block_forward,
            block,
            inputs,
            input_others,
            out_dev,
            allow_shard=allow_shard,
            hook_pass=hook_pass,
            merge_stats=merge_stats,
        )

    # ── Per-block pipeline orchestration ─────────────────────────────────────

    def compress_block(
        self,
        block,
        fp_inputs,
        input_others,
        block_ctx: BlockContext,
        q_inputs=None,
        input_ids=None,
        reference_output=None,
        **kwargs,
    ) -> tuple:
        """Run the full per-block algorithm pipeline: calibration → quantization → collection.

        Orchestrates preprocessors, quantizer calibration, and block quantization in
        the canonical order.  Infrastructure concerns (device management, memory cleanup,
        q_input variable reassignment) remain the caller's responsibility.

        The interface mirrors :meth:`~auto_round.algorithms.quantization.base.BaseQuantizer.quantize_block`
        but covers the entire multi-step pipeline including preprocessor calibration and
        output collection.

        Args:
            block: The transformer block to process.
            input_ids: Full-precision (FP) cached inputs.
            input_others: Auxiliary kwargs (attention_mask, position_ids, …).
            ctx: Per-block lifecycle context (:class:`BlockContext`).
            q_inputs: Quantized-input tensors from the previous block, or ``None``.
            valid_token_mask: Optional mask for per-token loss weighting.

        Returns:
            ``(new_q_input, reference_output)``:

            - *new_q_input*: block output after quantization (``None`` when
              ``enable_quanted_input`` is ``False``).
            - *reference_output*: FP reference output collected before optimization.
        """
        self.last_collect_wall = 0.0
        block_forward_fn = self.block_forward

        self._coll_ctx = self._collection_context(block, fp_inputs)
        # distributed calibration pool: each DDP device owns its sample
        # shard (matching the tune shards), so shard-local reads stay
        # device-local; serial consumers use device-safe cats
        self._coll_ctx.distribute_pools(fp_inputs, q_inputs)

        # ── Step 0: Layer-wise rotation (before any reference/calibration) ────
        # Each rotation member rotates this block's weights and installs online
        # hooks so all downstream calibration and reference collection operate on
        # the rotated block. No-op unless a member prepared layer-wise rotation.
        for rotation_member in self._rotation_members:
            rotation_member.on_block_ready(block, block_ctx)

        # Routed-dispatch shape recorders: fire-once pre-hooks on MoE containers
        # capture (top_k, routed_rows) from the first NATURAL dispatch during
        # the forwards below, so the iters>0 activation budget can fall back to
        # pure arg shapes when config spellings are unknown (arch-agnostic).
        try:
            from auto_round.algorithms.quantization.tune_memory import _ensure_routed_shape_recorders_

            _ensure_routed_shape_recorders_(block)
        except Exception as e:  # pragma: no cover - diagnostics only
            logger.debug("[routed-shape] recorder setup skipped (%s)", e)

        # ── Step 1: Preprocessor calibration (e.g. AWQ activation stats) ──────
        with torch.no_grad():
            pre_hooks = []
            for pre in self.preprocessors:
                hooks = pre.register_fp_input_forward_hooks(block)
                if not hooks:
                    continue
                pre_inputs, pre_others = pre.select_calibration_samples(fp_inputs, input_others)
                if pre_inputs is fp_inputs and pre_others is input_others:
                    pre_hooks.extend(hooks)
                    continue
                block_forward_fn(block, pre_inputs, pre_others)
                for h in hooks:
                    h.remove()
            if pre_hooks:
                block_forward_fn(block, fp_inputs, input_others)
            for h in pre_hooks:
                h.remove()

            pre_q_hooks = []
            for pre in self.preprocessors:
                if hasattr(pre, "register_qinput_forward_hooks"):
                    pre_q_hooks.extend(pre.register_qinput_forward_hooks(block))
            if pre_q_hooks:
                block_forward_fn(block, q_inputs if q_inputs is not None else fp_inputs, input_others)
            for h in pre_q_hooks:
                h.remove()

        # ── Step 2: pre_quantize_block (stats consolidation + weight transforms) ──
        for pre in self.preprocessors:
            pre.pre_quantize_block(block_ctx)

        act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True
        data_type = self.scheme.data_type if self.scheme else "int"
        group_size = self.scheme.group_size if self.scheme else -1
        act_data_type = self.scheme.act_data_type if self.scheme else data_type

        # A reference supplied by an external caller may have been collected
        # before a weight-changing preprocessor (for example AWQ smoothing or
        # hard clipping). Such a reference is stale and must not be used as the
        # quantizer target or propagated as the next block's FP input.
        reference_invalidated = any(pre.invalidates_reference_output(block_ctx) for pre in self.preprocessors)

        reference_next_input = None
        # ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────
        if fp_inputs is not None:
            with torch.no_grad():
                quant_hooks, merge_stats = self._get_fp_act_hooks(block)
                if reference_output is None or reference_invalidated:
                    reference_output = self._collect_forward_timed(
                        block,
                        fp_inputs,
                        input_others,
                        # the declared statistics (act_max, the quantizer's
                        # calibration_stats) are folded from the mirrors, so
                        # hook-carrying passes may shard
                        hook_pass=bool(quant_hooks),
                        merge_stats=merge_stats,
                    )
                    reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_output
                else:
                    # No forward happened in this step, so last_output_dict may
                    # belong to an earlier AWQ subset/calibration replay.
                    reference_next_input = reference_output
                for h in quant_hooks:
                    h.remove()

                if self.block_quantizer.enable_quanted_input:
                    q_hooks, q_merge_stats = self._get_q_act_hooks(block)
                    if q_hooks:
                        self._collect_forward_timed(
                            block,
                            q_inputs if q_inputs is not None else fp_inputs,
                            input_others,
                            hook_pass=bool(q_hooks),
                            merge_stats=q_merge_stats,
                        )
                        for h in q_hooks:
                            h.remove()

            # ── Step 3.5: MoE scale alignment + global scale update ─────────────────
            # Must run after calibration hooks (act_max collected) and before quantize_block.
            if act_data_type is not None or not act_dynamic:
                from auto_round.schemes import needs_nvfp_global_scale
                from auto_round.utils import set_amax_for_all_moe_layers

                if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
                    set_amax_for_all_moe_layers(block, attr_name="act_max")

        # SVDQuant may have replaced the original linears during pre-quantize.
        # Update NVFP4 scales after that transformation even on the zero-shot
        # path, where fp_inputs is None and the calibration block above is skipped.
        if act_data_type is not None or not act_dynamic:
            from auto_round.data_type.utils import update_block_global_scale_if_needed

            update_block_global_scale_if_needed(block, data_type, group_size)

        if q_inputs is not None and fp_inputs is not q_inputs:
            # Release the fp pool at its last use (before tuning): the q pool is
            # the tuning input and the fp pool is already consumed into
            # reference_output. The release is IN PLACE -- the orchestrator
            # holds the same container object as its ``input_ids``, so merely
            # dropping this local reference freed nothing and the pool stayed
            # resident through the whole tune loop. Freed VRAM here directly
            # widens the tune-loop's local-pull decision (hot pools on the
            # compute device when they fit).
            _release_pool_inplace(fp_inputs)
            fp_inputs = None
            clear_memory()
        else:
            # lanes without a separate q-input pool (first block; quanted-input
            # disabled) still get the pre-tuning cache clear the base code had
            clear_memory()
        # ── Step 3.6: mirror-pool handoff ──────────────────────────────────────
        # The collection passes may have left a per-block mirror pool alive
        # (built once, reused across the collection forwards). At iters=0 the
        # RTN/OptRTN quantize runs on the home block, the mirrors are synced
        # to it afterwards and the cascade forwards run on them; at iters>0
        # the tune ranks tune the mirrors (spmd.quantize_block_on_ranks).
        from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
        from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer

        # pool readers: the RTN family (its mirrors are synced to the
        # quantized home block for the sharded cascade) and the SignRound
        # family (its tune ranks run on the mirrors). NOTE: this ensure
        # is only the SAFETY NET for when no collection pass was
        # shard-eligible -- ref-collect sharding builds the pool lazily for
        # EVERY lane and is untouched by this gate.
        _pool_consumer = isinstance(self.block_quantizer, (RTNQuantizer, SignRoundQuantizer))
        if _pool_consumer and getattr(self._coll_ctx, "devices", None) and len(self._coll_ctx.devices) >= 2:
            self._coll_ctx.ensure_pool(block)
        _pool = getattr(self._coll_ctx, "pool", None)
        if _pool is not None:
            # the tune ranks take the resident mirrors (no second weight copy);
            # the RTN family leaves them to _settle_block_pool. A pool nobody
            # consumed is released before the post-quantize steps below.
            _state = parallel_state(self.block_quantizer, create=True)
            _state.pool = _pool
            _state.pool_used = False
        # ── Step 4: quantize_block ──────────────────────────────────────────────
        # When quantized input is available from the previous block, use it;
        # otherwise fall back to the FP input.
        effective_input = q_inputs if q_inputs is not None else fp_inputs
        warm_start_outputs = []
        collect_warm_start_output = None
        if getattr(self, "_preserve_weights", False) and self.block_quantizer.enable_quanted_input:

            def collect_warm_start_output(wrapped_block):
                output = block_forward_fn(wrapped_block, effective_input, input_others)
                warm_start_outputs.append(getattr(block_forward_fn, "last_output_dict", None) or output)

        def tune_block():
            # data-parallel tune: the unchanged quantize_block runs once per
            # rank (one thread per device); otherwise the serial call
            from auto_round.algorithms.parallel.spmd import quantize_block_on_ranks

            engaged, rank0_params = quantize_block_on_ranks(
                self.block_quantizer,
                block,
                effective_input,
                input_others,
                reference_output,
                q_inputs,
                block_ctx,
                input_ids=input_ids,
            )
            if engaged:
                return rank0_params
            return self.block_quantizer.quantize_block(
                block, effective_input, input_others, reference_output, q_inputs, block_ctx, input_ids=input_ids
            )

        try:
            self._run_tuning(block, tune_block, collect_warm_start_output)

            # Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept
            # out of the per-algorithm tuning loop.
            self.block_quantizer.finalize_block(block, block_ctx)

            # the pool is block-scoped and belongs to the shared tune state:
            # settle it (sync the mirrors for the RTN family, keep or release
            # it) before the post-quantize steps and the Step-6 cascade
            _pool = self._settle_block_pool(block, _pool)

            # ── Step 5: post_quantize_block ─────────────────────────────────────
            for pre in self.preprocessors:
                pre.post_quantize_block(block_ctx)

            # ── Step 6: Collect quantized-block outputs for the next block ──────
            if warm_start_outputs:
                # Warm-up has restored FP weights by now; use the output collected
                # while the wrappers still held the best quantization parameters.
                new_q_input = warm_start_outputs[0]
            elif self.block_quantizer.enable_quanted_input:
                with torch.no_grad():
                    new_q_input = self._collect_forward_timed(block, effective_input, input_others)
                    new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input
            else:
                new_q_input = None
        finally:
            # the pool is block-scoped: drop it even on failures so the next
            # block's pool starts clean
            _state = parallel_state(self.block_quantizer)
            if _state is not None:
                _state.pool = None
            self._coll_ctx.release_pool()

        return new_q_input, reference_next_input

    def _settle_block_pool(self, block, pool):
        """Pool disposition after quantize_block; returns the pool kept for the Step-6 cascade.

        - zero-shot RTN/OptRTN quantized the HOME block through main's search
          lane (which spreads its searches over idle devices itself) -> sync
          the quantized home state onto the mirrors and keep the pool, so the
          cascade still runs sharded;
        - the tune ranks took the mirrors (they end the block in their own
          tuned state while home was unwrapped with the BEST params) -> drop
          the pool; the cascade runs on the canonical home block;
        - unconsumed (declined lane / vanished plan) -> release.
        """
        if pool is None:
            return None
        from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
        from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer

        if isinstance(self.block_quantizer, RTNQuantizer):
            sync_mirrors_to_home(self.block_quantizer, block)
        _state = parallel_state(self.block_quantizer)
        if _state is None or _state.pool is None or not _state.pool_used:
            self._coll_ctx.release_pool()
            return None
        return pool

    def compress_layer_outside_block(
        self,
        layer: torch.nn.Module,
        fp_inputs=None,
        q_inputs=None,
        disable_opt_rtn=None,  # TODO wenhuach rename this to search_init_scale
        input_ids=None,
    ) -> None:
        """Quantize a single layer that lives outside transformer blocks.

        Mirrors :meth:`compress_block` for the outside-block case: attaches
        act_max calibration when static activation quantization is required,
        then delegates to the block quantizer.

        Args:
            layer: The layer module to quantize. Must have a ``global_name``
                attribute.
            fp_inputs: Per-sample FP activations for calibration, or ``None``
                to fall back to zero-shot RTN.
            q_inputs: Optional quantized activations from a previous stage.
            valid_token_mask: Per-sample token masks for loss weighting.
            disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config.
        """
        # Attach act_max for static activation quantization when inputs are available.
        if fp_inputs is not None:
            from auto_round.schemes import needs_nvfp_global_scale

            act_data_type = layer.act_data_type
            if act_data_type is None:
                act_data_type = "fp"
            act_dynamic = getattr(layer, "act_dynamic", True)
            if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
                self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)

        # Infrastructure: move layer to the tuning device before handing off to the quantizer.
        device = getattr(layer, "tuning_device", device_manager.device)  # TODO this should be handled by compressor
        layer = layer.to(device)

        return self.block_quantizer.quantize_layer_outside_block(
            layer,
            fp_inputs=fp_inputs,
            q_inputs=q_inputs,
            disable_opt_rtn=disable_opt_rtn,
            input_ids=input_ids,
        )

    # ── Convenience act-calib helpers ────────────────────────────────────────

    def members(self) -> list:
        """Return preprocessors, rotations, then all ordered quantizer stages.

        Run-level preparation and cleanup cover every member exactly once.
        Per-block work and dispatch use only the currently active quantizer.
        """
        return list(self.preprocessors) + list(self._rotation_members) + list(self.quantizers)

    def dispatch_block(self, block: torch.nn.Module, input_ids, input_others: dict):
        """Dispatch block to device(s) via the pipeline's algorithms.

        Iterates active members; if exactly one overrides the default dispatch_block,
        it is called. If multiple override, warns and uses the first one only.
        If none override, uses the block_quantizer's default (simple .to(device)).
        """
        from auto_round.algorithms.quantization.base import BaseQuantizer

        overriders = []
        seen_ids = set()
        for member in [*self.preprocessors, *self._rotation_members, self.block_quantizer]:
            if not hasattr(member, "dispatch_block"):
                continue
            # Check if the member overrides the base default
            if type(member).dispatch_block is not BaseQuantizer.dispatch_block:
                # Custom pipelines may expose the same member through more than
                # one role; dispatch it only once.
                if id(member) in seen_ids:
                    continue
                seen_ids.add(id(member))
                overriders.append(member)

        if len(overriders) > 1:
            names = [type(m).__name__ for m in overriders]
            logger.warning(
                f"Multiple pipeline members override dispatch_block: {names}. "
                f"Only {names[0]} will be used; others are ignored."
            )

        if overriders:
            return overriders[0].dispatch_block(block, input_ids, input_others)
        return self.block_quantizer.dispatch_block(block, input_ids, input_others)

    def prepare_run(self, composer: AlgorithmComposer = None):
        for alg in self.members():
            alg.prepare_run(composer=self)

    def finalize_run(self):
        # Rotation members are part of ``members()`` and finalize themselves
        # (layer-wise teardown lives in ``RotationPreprocessor.finalize_run``).
        for alg in self.members():
            alg.finalize_run()

    # ------------------------------------------------------------------
    # Rotation lifecycle (delegated to rotation members)
    # ------------------------------------------------------------------
    #
    # Rotation is a model-level pre-quantisation transform. Full-model rotation
    # must run *before* calibration data is cached, which is earlier than the
    # per-member ``prepare_run`` stage; layer-wise rotation instead prepares its
    # matrices here and rotates each block from within ``compress_block``. All
    # algorithm-specific decisions live in the rotation members; the composer
    # only drives the single generic entry point :meth:`apply_model_transforms`.

    def _resolve_rotation_data_type(self) -> str:
        """Best-effort resolution of the quantization data_type for rotation dispatch."""
        if self.scheme is not None and getattr(self.scheme, "data_type", None):
            return self.scheme.data_type
        if self.block_quantizer is not None:
            return getattr(self.block_quantizer.config, "data_type", "mx_fp")
        return "mx_fp"

    def apply_model_transforms(self, model: torch.nn.Module) -> torch.nn.Module:
        """Apply model-level pre-quantisation transforms (rotation) to *model*.

        Generic entry point invoked once by the orchestrator before calibration
        caching / the block loop. Each rotation member either rotates the whole
        model immediately (full-model mode) or prepares its rotation matrices and
        defers per-block work to :meth:`compress_block` (layer-wise mode).
        Idempotent — repeated calls are a no-op.

        Returns:
            The (possibly mutated) model.
        """
        if self._rotation_prepared:
            return model
        if not self._rotation_members:
            self._rotation_prepared = True
            return model

        data_type = self._resolve_rotation_data_type()
        logger.info("Applying Hadamard transform to the model.")
        for rotation_member in self._rotation_members:
            # The member honours its own config (``layerwise`` field).
            model = rotation_member.rotate_model(model, data_type=data_type)

        self._rotation_prepared = True
        return model

    @property
    def has_layerwise_rotation(self) -> bool:
        """Whether any rotation member is driving per-block (layer-wise) rotation."""
        return any(rotation_member.is_layerwise_active for rotation_member in self._rotation_members)