PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Algorithm Fusion Pipeline abstraction.
This module defines the core data structures and utilities for composing
pre-processing algorithms (AWQ smooth, SmoothQuant, Rotation…) with a
block-quantization algorithm (RTN, SignRound/AutoRound…) into a single
shared-calibration bundle.
Design invariants (see AWQ_REFACTOR_PLAN.md §0.0 and §3.0):
- ``AlgorithmComposer`` is the *first-class abstraction*; it is NOT just
AWQ's helper.
- All block-wise scheduling in ``Compressor`` operates against
``AlgorithmComposer``, never against a concrete ``AWQTransform``.
- Single-algorithm use is expressed as
``AlgorithmComposer(preprocessors=[], block_quantizer=q)``, which is
semantically identical to the current direct-quantizer path.
"""
from __future__ import annotations
from contextlib import nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from auto_round.algorithms.block_runner import BlockForwardRunner
from auto_round.algorithms.config_resolver import (
get_algorithm_class,
resolve_shared_config_values,
split_quantization_configs,
)
from auto_round.algorithms.parallel.data_parallel import (
ENGINE_CALIBRATION_STATS,
calibration_merge_stats,
parallel_state,
)
from auto_round.algorithms.utils import _has_nvfp4_layer
from auto_round.logger import logger
from auto_round.utils import clear_memory
from auto_round.utils.device_manager import device_manager
from auto_round.wrapper import preserve_original_weights, store_warm_start_params_and_unwrap, unwrapper_block
if TYPE_CHECKING: # avoid circular imports at runtime
from auto_round.algorithms.quantization.base import BaseQuantizer
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.transforms.base import BasePreprocessor
from auto_round.compressors import BaseOrchestrator
# ---------------------------------------------------------------------------
# Context dataclasses
# ---------------------------------------------------------------------------
@dataclass
class BlockContext:
"""Per-block context threaded through the lifecycle hooks.
Passed to lifecycle methods like ``block_forward_hooks``,
``pre_quantize_block``, ``post_quantize_block``, etc.
``block_names`` preserves the *scheduling group* (which may contain more
than one block when ``nblocks > 1``). Pre-processing algorithms that
only support single-block operation (e.g. AWQ Phase-1) must check
``len(block_names) == 1`` in ``prepare_block_group`` and raise
``ValueError`` with a user-readable message.
"""
model: torch.nn.Module
block_names: list[str] # scheduling group; len > 1 when nblocks > 1
block_name: str # = block_names[0] for single-block; descriptive label for multi
block_index: int # 0-based index within the current all_blocks group
bs: int = 1
is_mllm: bool = False # fail-fast gate for algorithms that don't support MLLM
is_diffusion: bool = False # fail-fast gate for algorithms that don't support diffusion
pbar: Any = None
block_cnt: int = 0 # total number of blocks being quantized in this run
# Ordered names of every block in the current all_blocks group (used e.g. by the
# LFQ loss to forward a non-final block's output through the remaining FP blocks).
all_block_names: list[str] | None = None
# ---------------------------------------------------------------------------
# AlgorithmComposer
# ---------------------------------------------------------------------------
def _release_pool_inplace(obj: Any) -> None:
"""Release a calibration pool's tensors by mutating its containers in place.
The pool object is shared with the orchestrator (which holds it as the
block's ``input_ids``), so dropping a local reference frees nothing. The
containers are the same objects on both sides; setting their entries to
None drops the last strong references to the tensors (hooks are removed
and the tuning loop reads the q pool, so nothing re-reads this pool).
Immutable containers (tuples) are skipped: their tensors stay alive until
the block-end release, matching the pre-existing behavior.
"""
if isinstance(obj, list):
for i in range(len(obj)):
obj[i] = None
elif isinstance(obj, dict):
for v in obj.values():
_release_pool_inplace(v)
return obj
class AlgorithmComposer:
"""An ordered composition of preprocessors and quantizer stages, built from
a list of algorithm config objects and an optional compressor.
The ``preprocessors`` list is order-sensitive: preprocessors are applied in
the listed order (e.g. ``[Rotation, AWQ]``). The block quantizer is always
the terminal weight-compression step, regardless of its position in the
input config list. Quantizers execute in their relative config order;
``block_quantizer`` exposes the active stage (the terminal stage when idle).
Usage::
composer = AlgorithmComposer(configs, compressor=self)
"""
def __init__(self, configs: list, orchestrator: BaseOrchestrator = None) -> None:
"""Build the pipeline from a list of algorithm config instances.
Resolution rules:
1. If no ``QuantizationConfig`` with a ``BaseQuantizer`` is found in
*configs*, a default :class:`RTNConfig` is appended automatically.
2. ``BasePreprocessor`` instances go into ``preprocessors`` (in order).
3. ``BaseQuantizer`` instances form an ordered stage sequence.
4. Every stage validates its capabilities before execution.
5. If *compressor* is provided, every member is bound to it and
``block_forward`` / quantization metadata are extracted from it.
Args:
configs: List of algorithm config objects (``QuantizationConfig``
subclasses such as :class:`RTNConfig`, :class:`SignRoundConfig`,
:class:`AWQConfig`, …).
compressor: The :class:`~auto_round.compressors.base.BaseCompressor`
instance driving this pipeline. When supplied, members are
bound to it and ``block_forward`` is taken from it.
"""
from auto_round.algorithms.quantization.base import BaseQuantizer
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.transforms.base import BasePreprocessor, BaseRotationConfig
self._orchestrator = orchestrator
configs = list(configs)
# Rotation configs travel in the same config list but are not block
# quantizers / preprocessors (they are ``BaseRotationConfig``, not
# ``QuantizationConfig``). Capture them here and wrap each in a
# ``RotationPreprocessor`` member below so the orchestrator stays
# rotation-agnostic and every rotation owns its own lifecycle.
#
# Whether rotation runs per-block (layer-wise) is a field on each
# rotation config (``BaseRotationConfig.layerwise``).
self._rotation_configs = [c for c in configs if isinstance(c, BaseRotationConfig)]
_, block_quantizer_configs = split_quantization_configs(configs)
if not block_quantizer_configs:
from auto_round.algorithms.quantization.rtn.config import RTNConfig
configs = configs + [RTNConfig()]
# Sequential quantizers own independent schedules/behavior, but operate
# on the same quantization scheme. Do not reconcile their local settings.
configs = resolve_shared_config_values(configs, reconcile_non_scheme=len(block_quantizer_configs) <= 1)
preprocessors: list = []
block_quantizers: list = []
for cfg in configs:
if not isinstance(cfg, QuantizationConfig):
continue
from auto_round.algorithms.registry import normalize_algorithm_config
cfg = normalize_algorithm_config(cfg)
alg_cls = get_algorithm_class(cfg)
if alg_cls is None:
raise ValueError(f"Unknown algorithm config type {type(cfg).__name__!r}.")
q = alg_cls(cfg)
if orchestrator is not None:
q.bind(orchestrator)
if isinstance(q, BasePreprocessor):
preprocessors.append(q)
elif isinstance(q, BaseQuantizer):
block_quantizers.append(q)
else:
raise TypeError(
f"Algorithm class {type(q).__name__} must inherit either " "BasePreprocessor or BaseQuantizer."
)
if len(block_quantizers) == 0:
raise ValueError("No block quantizer found in configs.")
self.quantizers = tuple(block_quantizers)
self._preserve_weights = False
self._preserve_stage_weights = []
needs_original_weights = False
for quantizer in reversed(self.quantizers):
self._preserve_stage_weights.append(needs_original_weights)
if quantizer.config.nsamples is not None and not quantizer.whole_model_tuning:
raise NotImplementedError(
f"{type(quantizer.config).__name__}.nsamples overrides are only supported for whole-model tuning. "
"Use the common calibration nsamples argument for this stage."
)
needs_original_weights = needs_original_weights or quantizer.requires_original_weights
self._preserve_stage_weights.reverse()
seen = set()
for pre in preprocessors:
name = type(pre).__name__
if name in seen:
raise ValueError(
f"Duplicate preprocessor {name} in AlgorithmComposer. "
"Repeated instances of the same preprocessor are not supported yet."
)
seen.add(name)
self.preprocessors = preprocessors
can_compile_block_forward = False
self.block_quantizer = block_quantizers[-1]
self.block_forward = None
if orchestrator is not None:
user_torch_compile = bool(
getattr(getattr(orchestrator, "compress_context", None), "enable_torch_compile", False)
)
rotation_configs = getattr(orchestrator, "rotation_configs", ())
blockers = []
for quantizer in self.quantizers:
if not quantizer.can_compile_block_forward():
blockers.append(type(quantizer).__name__)
for component in [*self.preprocessors, *rotation_configs]:
if not getattr(component, "can_compile_block_forward", lambda: True)():
blockers.append(type(component).__name__)
can_compile_block_forward = user_torch_compile and not blockers
if user_torch_compile and blockers:
logger.info(
"Block-forward torch.compile is disabled because %s is incompatible.",
", ".join(blockers),
)
if _has_nvfp4_layer(orchestrator):
can_compile_block_forward = False
logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")
# Bind compressor-level infrastructure (set before _build_quantizer is called).
self.block_forward = (
BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)
if orchestrator is not None
else None
)
# A little tricky
for quantizer in self.quantizers:
quantizer.bind_block_forward_runner(self.block_forward)
self.scheme = getattr(orchestrator, "scheme_context", None)
# Rotation is modelled as ordinary pipeline members (see
# ``RotationPreprocessor``) that own their entire lifecycle. They are
# built from the rotation configs captured above and bound like every
# other member so ``finalize_run`` can reach the live model.
from auto_round.algorithms.transforms.member import RotationPreprocessor
self._rotation_members = [RotationPreprocessor(cfg) for cfg in self._rotation_configs]
if orchestrator is not None:
for rotation_member in self._rotation_members:
rotation_member.bind(orchestrator)
# Guards the one-shot model-level rotation stage (apply_model_transforms).
self._rotation_prepared: bool = False
def compress_model(self, token_ids, run_blocks) -> bool:
"""Run ordered block/model tuning stages using the bound compressor.
Returns True when the terminal stage handles the whole model (including
outside-block layers). Preprocessors/rotations run only in the first
block stage. The composer owns weight preservation and cleanup; algorithms
only implement block/model tuning and optionally customize sampling.
"""
orchestrator = self._orchestrator
if orchestrator is None:
raise RuntimeError("compress_model requires a bound compressor.")
streaming_export = (
orchestrator.compress_context.is_immediate_saving or orchestrator.compress_context.is_immediate_packing
)
if streaming_export and (len(self.quantizers) > 1 or self.block_quantizer.whole_model_tuning):
raise NotImplementedError("Multi-stage/whole-model tuning does not support immediate saving/packing.")
from auto_round import envs
if envs.AR_RESUME_DIR and len(self.quantizers) > 1:
raise NotImplementedError("Resuming a multi-stage quantization run is not supported.")
if self.quantizers[0].whole_model_tuning and (self.preprocessors or self.has_layerwise_rotation):
raise NotImplementedError("Block preprocessors/layer-wise rotation require a leading block-tuning stage.")
terminal = self.block_quantizer
preprocessors, rotations = self.preprocessors, self._rotation_members
try:
self.prepare_run()
for index, (quantizer, preserve) in enumerate(zip(self.quantizers, self._preserve_stage_weights)):
self.block_quantizer = quantizer
self._preserve_weights = preserve
if index:
self.preprocessors, self._rotation_members = [], []
logger.info(
"Starting algorithm stage %d/%d: %s.", index + 1, len(self.quantizers), type(quantizer).__name__
)
if quantizer.whole_model_tuning:
orchestrator._reload_model_for_tuning()
runner = getattr(self, "block_forward", None)
if runner is not None:
runner.pool_placement = None
stage_token_ids = quantizer.get_calibration_token_ids(token_ids, orchestrator.calibration)
self._run_tuning(
orchestrator.model_context.model,
lambda: quantizer.quantize_model(
orchestrator.model_context.model, stage_token_ids, device=device_manager.device
),
)
else:
run_blocks(index > 0)
return terminal.whole_model_tuning
finally:
self.block_quantizer = terminal
self._preserve_weights = False
self.preprocessors, self._rotation_members = preprocessors, rotations
self.finalize_run()
def _run_tuning(self, module, tune, collect_output=None):
"""Common block/model tuning finalization, including failure cleanup."""
preserve = getattr(self, "_preserve_weights", False)
with preserve_original_weights(module) if preserve else nullcontext():
try:
best_params = tune() or {}
if preserve:
store_warm_start_params_and_unwrap(module, best_params, before_unwrap=collect_output)
else:
unwrapper_block(module, best_params)
except BaseException:
# Do not bake partially tuned parameters when optimization or
# output collection fails. Remove any remaining common wrappers.
from auto_round.utils import set_module
for name, wrapped in list(module.named_modules()):
if hasattr(wrapped, "orig_layer"):
set_module(module, name, wrapped.orig_layer)
raise
# ── Internal hook helpers (act_max calibration) ───────────────────────────
def _register_act_max_hooks(self, block: torch.nn.Module) -> list:
"""Register per-module act_max tracking hooks for static activation quantization.
Returns a list of hook handles that the caller must remove when done.
"""
from auto_round.data_type.utils import register_act_max_hooks
return register_act_max_hooks(block, getattr(self.block_quantizer.config, "act_data_type", None))
def _get_fp_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
"""Register FP-input act_max + quantizer forward hooks.
Returns ``(handles, merge_stats)``: the shard-merge map of the
statistics these hooks write (``None``: the pass stays serial, see
``calibration_merge_stats``).
"""
if not self.need_quanted_input():
# If having q_input, act_max will be collected in q_input forward hook,
# no need to collect in fp_input forward hook
handles = self._register_act_max_hooks(block)
else:
handles = []
quantizer_hooks = self.block_quantizer.register_fp_input_forward_hooks(block)
handles.extend(quantizer_hooks)
return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)
def _get_q_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
"""Register Q-input act_max + quantizer forward hooks; returns
``(handles, merge_stats)`` like ``_get_fp_act_hooks``."""
handles = self._register_act_max_hooks(block)
quantizer_hooks = self.block_quantizer.register_qinput_forward_hooks(block)
handles.extend(quantizer_hooks)
return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)
def _attach_act_max_for_outside_layer(self, layer: torch.nn.Module, fp_inputs, q_inputs) -> None:
"""Compute and attach act_max for an outside-block layer from cached inputs.
Mirrors the hook-based act_max collection done for in-block layers, but
iterates over already-cached tensors directly instead of running a forward pass.
Args:
layer: The layer module to attach act_max to.
fp_inputs: List of FP input tensors collected during calibration.
q_inputs: Optional list of quantized input tensors; used instead of
``fp_inputs`` when provided.
"""
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.schemes import needs_nvfp_global_scale as is_nv_fp
target_input = q_inputs or fp_inputs
act_group_size = layer.act_group_size
if act_group_size is None:
act_group_size = layer.group_size
act_data_type = layer.act_data_type
if act_data_type is None:
act_data_type = layer.data_type
is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False
for inp in target_input:
if isinstance(inp, (tuple, list)):
inp = inp[0]
if inp.numel() == 0:
continue
inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)
act_max = torch.max(torch.abs(inp), dim=-1).values
if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:
layer.act_max = act_max
if is_act_nv_fp_flag:
max_val = act_max.max()
layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
continue
act_max = act_max.to(layer.act_max.device)
if is_act_nv_fp_flag:
max_val = torch.max(act_max.max(), layer.act_max.max())
layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
else:
layer.act_max = torch.max(act_max, layer.act_max)
def need_quanted_input(self):
for preprocessor in self.preprocessors:
if getattr(preprocessor, "enable_quanted_input", False):
return True
return bool(getattr(self.block_quantizer, "enable_quanted_input", False))
def compress_embedding_layer(self):
return self.block_quantizer.quantize_embedding_layer()
def _collection_context(self, block, fp_inputs):
"""TuneParallelContext for the collection phase (P3): mirror devices
for sharding the no-grad collection forwards, with the same engagement
resolver as the tune so the gates can never diverge."""
from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
return TuneParallelContext.for_collection(self, block, fp_inputs)
def _collect_forward_timed(self, *args, **kwargs):
"""_collect_forward with wall accumulation into self.last_collect_wall."""
import time as _ctime
_t0 = _ctime.perf_counter()
try:
return self._collect_forward(*args, **kwargs)
finally:
self.last_collect_wall = getattr(self, "last_collect_wall", 0.0) + (_ctime.perf_counter() - _t0)
def _collect_forward(
self,
block,
inputs,
input_others,
out_dev=None,
allow_shard: bool = True,
hook_pass: bool = False,
merge_stats=ENGINE_CALIBRATION_STATS,
):
"""Collection forward: sharded across DDP mirrors when eligible.
``out_dev`` optionally overrides the runner's output cache device.
``allow_shard=False`` forces the serial path. ``merge_stats`` is the
shard-merge map of the statistics the pass's hooks write (from
``_get_fp_act_hooks`` / ``_get_q_act_hooks``); ``None`` keeps the
pass serial (undeclared quantizer statistics).
``hook_pass=True`` may cap the concurrent shards (env-driven; default
no cap): forward hooks
force dynamo graph breaks, leaving the compiled runner as
python-bound eager sections that GIL-convoy under many threads.
"""
if self._coll_ctx is None:
from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
self._coll_ctx = TuneParallelContext()
return self._coll_ctx.collect_forward(
self.block_forward,
block,
inputs,
input_others,
out_dev,
allow_shard=allow_shard,
hook_pass=hook_pass,
merge_stats=merge_stats,
)
# ── Per-block pipeline orchestration ─────────────────────────────────────
def compress_block(
self,
block,
fp_inputs,
input_others,
block_ctx: BlockContext,
q_inputs=None,
input_ids=None,
reference_output=None,
**kwargs,
) -> tuple:
"""Run the full per-block algorithm pipeline: calibration → quantization → collection.
Orchestrates preprocessors, quantizer calibration, and block quantization in
the canonical order. Infrastructure concerns (device management, memory cleanup,
q_input variable reassignment) remain the caller's responsibility.
The interface mirrors :meth:`~auto_round.algorithms.quantization.base.BaseQuantizer.quantize_block`
but covers the entire multi-step pipeline including preprocessor calibration and
output collection.
Args:
block: The transformer block to process.
input_ids: Full-precision (FP) cached inputs.
input_others: Auxiliary kwargs (attention_mask, position_ids, …).
ctx: Per-block lifecycle context (:class:`BlockContext`).
q_inputs: Quantized-input tensors from the previous block, or ``None``.
valid_token_mask: Optional mask for per-token loss weighting.
Returns:
``(new_q_input, reference_output)``:
- *new_q_input*: block output after quantization (``None`` when
``enable_quanted_input`` is ``False``).
- *reference_output*: FP reference output collected before optimization.
"""
self.last_collect_wall = 0.0
block_forward_fn = self.block_forward
self._coll_ctx = self._collection_context(block, fp_inputs)
# distributed calibration pool: each DDP device owns its sample
# shard (matching the tune shards), so shard-local reads stay
# device-local; serial consumers use device-safe cats
self._coll_ctx.distribute_pools(fp_inputs, q_inputs)
# ── Step 0: Layer-wise rotation (before any reference/calibration) ────
# Each rotation member rotates this block's weights and installs online
# hooks so all downstream calibration and reference collection operate on
# the rotated block. No-op unless a member prepared layer-wise rotation.
for rotation_member in self._rotation_members:
rotation_member.on_block_ready(block, block_ctx)
# Routed-dispatch shape recorders: fire-once pre-hooks on MoE containers
# capture (top_k, routed_rows) from the first NATURAL dispatch during
# the forwards below, so the iters>0 activation budget can fall back to
# pure arg shapes when config spellings are unknown (arch-agnostic).
try:
from auto_round.algorithms.quantization.tune_memory import _ensure_routed_shape_recorders_
_ensure_routed_shape_recorders_(block)
except Exception as e: # pragma: no cover - diagnostics only
logger.debug("[routed-shape] recorder setup skipped (%s)", e)
# ── Step 1: Preprocessor calibration (e.g. AWQ activation stats) ──────
with torch.no_grad():
pre_hooks = []
for pre in self.preprocessors:
hooks = pre.register_fp_input_forward_hooks(block)
if not hooks:
continue
pre_inputs, pre_others = pre.select_calibration_samples(fp_inputs, input_others)
if pre_inputs is fp_inputs and pre_others is input_others:
pre_hooks.extend(hooks)
continue
block_forward_fn(block, pre_inputs, pre_others)
for h in hooks:
h.remove()
if pre_hooks:
block_forward_fn(block, fp_inputs, input_others)
for h in pre_hooks:
h.remove()
pre_q_hooks = []
for pre in self.preprocessors:
if hasattr(pre, "register_qinput_forward_hooks"):
pre_q_hooks.extend(pre.register_qinput_forward_hooks(block))
if pre_q_hooks:
block_forward_fn(block, q_inputs if q_inputs is not None else fp_inputs, input_others)
for h in pre_q_hooks:
h.remove()
# ── Step 2: pre_quantize_block (stats consolidation + weight transforms) ──
for pre in self.preprocessors:
pre.pre_quantize_block(block_ctx)
act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True
data_type = self.scheme.data_type if self.scheme else "int"
group_size = self.scheme.group_size if self.scheme else -1
act_data_type = self.scheme.act_data_type if self.scheme else data_type
# A reference supplied by an external caller may have been collected
# before a weight-changing preprocessor (for example AWQ smoothing or
# hard clipping). Such a reference is stale and must not be used as the
# quantizer target or propagated as the next block's FP input.
reference_invalidated = any(pre.invalidates_reference_output(block_ctx) for pre in self.preprocessors)
reference_next_input = None
# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────
if fp_inputs is not None:
with torch.no_grad():
quant_hooks, merge_stats = self._get_fp_act_hooks(block)
if reference_output is None or reference_invalidated:
reference_output = self._collect_forward_timed(
block,
fp_inputs,
input_others,
# the declared statistics (act_max, the quantizer's
# calibration_stats) are folded from the mirrors, so
# hook-carrying passes may shard
hook_pass=bool(quant_hooks),
merge_stats=merge_stats,
)
reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_output
else:
# No forward happened in this step, so last_output_dict may
# belong to an earlier AWQ subset/calibration replay.
reference_next_input = reference_output
for h in quant_hooks:
h.remove()
if self.block_quantizer.enable_quanted_input:
q_hooks, q_merge_stats = self._get_q_act_hooks(block)
if q_hooks:
self._collect_forward_timed(
block,
q_inputs if q_inputs is not None else fp_inputs,
input_others,
hook_pass=bool(q_hooks),
merge_stats=q_merge_stats,
)
for h in q_hooks:
h.remove()
# ── Step 3.5: MoE scale alignment + global scale update ─────────────────
# Must run after calibration hooks (act_max collected) and before quantize_block.
if act_data_type is not None or not act_dynamic:
from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import set_amax_for_all_moe_layers
if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
# SVDQuant may have replaced the original linears during pre-quantize.
# Update NVFP4 scales after that transformation even on the zero-shot
# path, where fp_inputs is None and the calibration block above is skipped.
if act_data_type is not None or not act_dynamic:
from auto_round.data_type.utils import update_block_global_scale_if_needed
update_block_global_scale_if_needed(block, data_type, group_size)
if q_inputs is not None and fp_inputs is not q_inputs:
# Release the fp pool at its last use (before tuning): the q pool is
# the tuning input and the fp pool is already consumed into
# reference_output. The release is IN PLACE -- the orchestrator
# holds the same container object as its ``input_ids``, so merely
# dropping this local reference freed nothing and the pool stayed
# resident through the whole tune loop. Freed VRAM here directly
# widens the tune-loop's local-pull decision (hot pools on the
# compute device when they fit).
_release_pool_inplace(fp_inputs)
fp_inputs = None
clear_memory()
else:
# lanes without a separate q-input pool (first block; quanted-input
# disabled) still get the pre-tuning cache clear the base code had
clear_memory()
# ── Step 3.6: mirror-pool handoff ──────────────────────────────────────
# The collection passes may have left a per-block mirror pool alive
# (built once, reused across the collection forwards). At iters=0 the
# RTN/OptRTN quantize runs on the home block, the mirrors are synced
# to it afterwards and the cascade forwards run on them; at iters>0
# the tune ranks tune the mirrors (spmd.quantize_block_on_ranks).
from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
# pool readers: the RTN family (its mirrors are synced to the
# quantized home block for the sharded cascade) and the SignRound
# family (its tune ranks run on the mirrors). NOTE: this ensure
# is only the SAFETY NET for when no collection pass was
# shard-eligible -- ref-collect sharding builds the pool lazily for
# EVERY lane and is untouched by this gate.
_pool_consumer = isinstance(self.block_quantizer, (RTNQuantizer, SignRoundQuantizer))
if _pool_consumer and getattr(self._coll_ctx, "devices", None) and len(self._coll_ctx.devices) >= 2:
self._coll_ctx.ensure_pool(block)
_pool = getattr(self._coll_ctx, "pool", None)
if _pool is not None:
# the tune ranks take the resident mirrors (no second weight copy);
# the RTN family leaves them to _settle_block_pool. A pool nobody
# consumed is released before the post-quantize steps below.
_state = parallel_state(self.block_quantizer, create=True)
_state.pool = _pool
_state.pool_used = False
# ── Step 4: quantize_block ──────────────────────────────────────────────
# When quantized input is available from the previous block, use it;
# otherwise fall back to the FP input.
effective_input = q_inputs if q_inputs is not None else fp_inputs
warm_start_outputs = []
collect_warm_start_output = None
if getattr(self, "_preserve_weights", False) and self.block_quantizer.enable_quanted_input:
def collect_warm_start_output(wrapped_block):
output = block_forward_fn(wrapped_block, effective_input, input_others)
warm_start_outputs.append(getattr(block_forward_fn, "last_output_dict", None) or output)
def tune_block():
# data-parallel tune: the unchanged quantize_block runs once per
# rank (one thread per device); otherwise the serial call
from auto_round.algorithms.parallel.spmd import quantize_block_on_ranks
engaged, rank0_params = quantize_block_on_ranks(
self.block_quantizer,
block,
effective_input,
input_others,
reference_output,
q_inputs,
block_ctx,
input_ids=input_ids,
)
if engaged:
return rank0_params
return self.block_quantizer.quantize_block(
block, effective_input, input_others, reference_output, q_inputs, block_ctx, input_ids=input_ids
)
try:
self._run_tuning(block, tune_block, collect_warm_start_output)
# Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept
# out of the per-algorithm tuning loop.
self.block_quantizer.finalize_block(block, block_ctx)
# the pool is block-scoped and belongs to the shared tune state:
# settle it (sync the mirrors for the RTN family, keep or release
# it) before the post-quantize steps and the Step-6 cascade
_pool = self._settle_block_pool(block, _pool)
# ── Step 5: post_quantize_block ─────────────────────────────────────
for pre in self.preprocessors:
pre.post_quantize_block(block_ctx)
# ── Step 6: Collect quantized-block outputs for the next block ──────
if warm_start_outputs:
# Warm-up has restored FP weights by now; use the output collected
# while the wrappers still held the best quantization parameters.
new_q_input = warm_start_outputs[0]
elif self.block_quantizer.enable_quanted_input:
with torch.no_grad():
new_q_input = self._collect_forward_timed(block, effective_input, input_others)
new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input
else:
new_q_input = None
finally:
# the pool is block-scoped: drop it even on failures so the next
# block's pool starts clean
_state = parallel_state(self.block_quantizer)
if _state is not None:
_state.pool = None
self._coll_ctx.release_pool()
return new_q_input, reference_next_input
def _settle_block_pool(self, block, pool):
"""Pool disposition after quantize_block; returns the pool kept for the Step-6 cascade.
- zero-shot RTN/OptRTN quantized the HOME block through main's search
lane (which spreads its searches over idle devices itself) -> sync
the quantized home state onto the mirrors and keep the pool, so the
cascade still runs sharded;
- the tune ranks took the mirrors (they end the block in their own
tuned state while home was unwrapped with the BEST params) -> drop
the pool; the cascade runs on the canonical home block;
- unconsumed (declined lane / vanished plan) -> release.
"""
if pool is None:
return None
from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
if isinstance(self.block_quantizer, RTNQuantizer):
sync_mirrors_to_home(self.block_quantizer, block)
_state = parallel_state(self.block_quantizer)
if _state is None or _state.pool is None or not _state.pool_used:
self._coll_ctx.release_pool()
return None
return pool
def compress_layer_outside_block(
self,
layer: torch.nn.Module,
fp_inputs=None,
q_inputs=None,
disable_opt_rtn=None, # TODO wenhuach rename this to search_init_scale
input_ids=None,
) -> None:
"""Quantize a single layer that lives outside transformer blocks.
Mirrors :meth:`compress_block` for the outside-block case: attaches
act_max calibration when static activation quantization is required,
then delegates to the block quantizer.
Args:
layer: The layer module to quantize. Must have a ``global_name``
attribute.
fp_inputs: Per-sample FP activations for calibration, or ``None``
to fall back to zero-shot RTN.
q_inputs: Optional quantized activations from a previous stage.
valid_token_mask: Per-sample token masks for loss weighting.
disable_opt_rtn: Override optimized-RTN; ``None`` defers to quantizer config.
"""
# Attach act_max for static activation quantization when inputs are available.
if fp_inputs is not None:
from auto_round.schemes import needs_nvfp_global_scale
act_data_type = layer.act_data_type
if act_data_type is None:
act_data_type = "fp"
act_dynamic = getattr(layer, "act_dynamic", True)
if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)
# Infrastructure: move layer to the tuning device before handing off to the quantizer.
device = getattr(layer, "tuning_device", device_manager.device) # TODO this should be handled by compressor
layer = layer.to(device)
return self.block_quantizer.quantize_layer_outside_block(
layer,
fp_inputs=fp_inputs,
q_inputs=q_inputs,
disable_opt_rtn=disable_opt_rtn,
input_ids=input_ids,
)
# ── Convenience act-calib helpers ────────────────────────────────────────
def members(self) -> list:
"""Return preprocessors, rotations, then all ordered quantizer stages.
Run-level preparation and cleanup cover every member exactly once.
Per-block work and dispatch use only the currently active quantizer.
"""
return list(self.preprocessors) + list(self._rotation_members) + list(self.quantizers)
def dispatch_block(self, block: torch.nn.Module, input_ids, input_others: dict):
"""Dispatch block to device(s) via the pipeline's algorithms.
Iterates active members; if exactly one overrides the default dispatch_block,
it is called. If multiple override, warns and uses the first one only.
If none override, uses the block_quantizer's default (simple .to(device)).
"""
from auto_round.algorithms.quantization.base import BaseQuantizer
overriders = []
seen_ids = set()
for member in [*self.preprocessors, *self._rotation_members, self.block_quantizer]:
if not hasattr(member, "dispatch_block"):
continue
# Check if the member overrides the base default
if type(member).dispatch_block is not BaseQuantizer.dispatch_block:
# Custom pipelines may expose the same member through more than
# one role; dispatch it only once.
if id(member) in seen_ids:
continue
seen_ids.add(id(member))
overriders.append(member)
if len(overriders) > 1:
names = [type(m).__name__ for m in overriders]
logger.warning(
f"Multiple pipeline members override dispatch_block: {names}. "
f"Only {names[0]} will be used; others are ignored."
)
if overriders:
return overriders[0].dispatch_block(block, input_ids, input_others)
return self.block_quantizer.dispatch_block(block, input_ids, input_others)
def prepare_run(self, composer: AlgorithmComposer = None):
for alg in self.members():
alg.prepare_run(composer=self)
def finalize_run(self):
# Rotation members are part of ``members()`` and finalize themselves
# (layer-wise teardown lives in ``RotationPreprocessor.finalize_run``).
for alg in self.members():
alg.finalize_run()
# ------------------------------------------------------------------
# Rotation lifecycle (delegated to rotation members)
# ------------------------------------------------------------------
#
# Rotation is a model-level pre-quantisation transform. Full-model rotation
# must run *before* calibration data is cached, which is earlier than the
# per-member ``prepare_run`` stage; layer-wise rotation instead prepares its
# matrices here and rotates each block from within ``compress_block``. All
# algorithm-specific decisions live in the rotation members; the composer
# only drives the single generic entry point :meth:`apply_model_transforms`.
def _resolve_rotation_data_type(self) -> str:
"""Best-effort resolution of the quantization data_type for rotation dispatch."""
if self.scheme is not None and getattr(self.scheme, "data_type", None):
return self.scheme.data_type
if self.block_quantizer is not None:
return getattr(self.block_quantizer.config, "data_type", "mx_fp")
return "mx_fp"
def apply_model_transforms(self, model: torch.nn.Module) -> torch.nn.Module:
"""Apply model-level pre-quantisation transforms (rotation) to *model*.
Generic entry point invoked once by the orchestrator before calibration
caching / the block loop. Each rotation member either rotates the whole
model immediately (full-model mode) or prepares its rotation matrices and
defers per-block work to :meth:`compress_block` (layer-wise mode).
Idempotent — repeated calls are a no-op.
Returns:
The (possibly mutated) model.
"""
if self._rotation_prepared:
return model
if not self._rotation_members:
self._rotation_prepared = True
return model
data_type = self._resolve_rotation_data_type()
logger.info("Applying Hadamard transform to the model.")
for rotation_member in self._rotation_members:
# The member honours its own config (``layerwise`` field).
model = rotation_member.rotate_model(model, data_type=data_type)
self._rotation_prepared = True
return model
@property
def has_layerwise_rotation(self) -> bool:
"""Whether any rotation member is driving per-block (layer-wise) rotation."""
return any(rotation_member.is_layerwise_active for rotation_member in self._rotation_members)