Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add blockwise rotation. (#2087)

lkk9b746ff95859
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 26bdffd6d..4ea01e8d8 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -122,10 +122,20 @@ class AlgorithmComposer:
"""
from auto_round.algorithms.quantization.base import BaseQuantizer
from auto_round.algorithms.quantization.config import QuantizationConfig
- from auto_round.algorithms.transforms.base import BasePreprocessor
+ from auto_round.algorithms.transforms.base import BasePreprocessor, BaseRotationConfig
configs = list(configs)
+ # Rotation configs travel in the same config list but are not block
+ # quantizers / preprocessors (they are ``BaseRotationConfig``, not
+ # ``QuantizationConfig``). Capture them here and wrap each in a
+ # ``RotationPreprocessor`` member below so the orchestrator stays
+ # rotation-agnostic and every rotation owns its own lifecycle.
+ #
+ # Whether rotation runs per-block (layer-wise) is a field on each
+ # rotation config (``BaseRotationConfig.layerwise``).
+ self._rotation_configs = [c for c in configs if isinstance(c, BaseRotationConfig)]
+
_, block_quantizer_configs = split_quantization_configs(configs)
if not block_quantizer_configs:
from auto_round.algorithms.quantization.rtn.config import RTNConfig
@@ -212,6 +222,19 @@ class AlgorithmComposer:
self.block_quantizer.bind_block_forward_runner(self.block_forward)
self.scheme = getattr(orchestrator, "scheme_context", None)
+ # Rotation is modelled as ordinary pipeline members (see
+ # ``RotationPreprocessor``) that own their entire lifecycle. They are
+ # built from the rotation configs captured above and bound like every
+ # other member so ``finalize_run`` can reach the live model.
+ from auto_round.algorithms.transforms.member import RotationPreprocessor
+
+ self._rotation_members = [RotationPreprocessor(cfg) for cfg in self._rotation_configs]
+ if orchestrator is not None:
+ for rotation_member in self._rotation_members:
+ rotation_member.bind(orchestrator)
+ # Guards the one-shot model-level rotation stage (apply_model_transforms).
+ self._rotation_prepared: bool = False
+
# ── Internal hook helpers (act_max calibration) ───────────────────────────
def _register_act_max_hooks(self, block: "torch.nn.Module") -> list:
@@ -353,6 +376,13 @@ class AlgorithmComposer:
"""
block_forward_fn = self.block_forward
+ # ── Step 0: Layer-wise rotation (before any reference/calibration) ────
+ # Each rotation member rotates this block's weights and installs online
+ # hooks so all downstream calibration and reference collection operate on
+ # the rotated block. No-op unless a member prepared layer-wise rotation.
+ for rotation_member in self._rotation_members:
+ rotation_member.on_block_ready(block, block_ctx)
+
# ── Step 1: Preprocessor calibration (e.g. AWQ activation stats) ──────
with torch.no_grad():
pre_hooks = []
@@ -504,13 +534,15 @@ class AlgorithmComposer:
# ── Convenience act-calib helpers ────────────────────────────────────────
def members(self) -> list:
- """Return the canonical execution order: preprocessors, then block quantizer.
+ """Return the canonical execution order: preprocessors, rotations, then block quantizer.
Preprocessor order follows the input config list. The block quantizer
is always returned last, so its position in that input list has no
- effect on pipeline execution.
+ effect on pipeline execution. Rotation members sit between the
+ preprocessors and the block quantizer so ``prepare_run`` / ``finalize_run``
+ cover them like any other member.
"""
- return list(self.preprocessors) + [self.block_quantizer]
+ return list(self.preprocessors) + list(self._rotation_members) + [self.block_quantizer]
def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):
"""Dispatch block to device(s) via the pipeline's algorithms.
@@ -545,5 +577,58 @@ class AlgorithmComposer:
alg.prepare_run(composer=self)
def finalize_run(self):
+ # Rotation members are part of ``members()`` and finalize themselves
+ # (layer-wise teardown lives in ``RotationPreprocessor.finalize_run``).
for alg in self.members():
alg.finalize_run()
+
+ # ------------------------------------------------------------------
+ # Rotation lifecycle (delegated to rotation members)
+ # ------------------------------------------------------------------
+ #
+ # Rotation is a model-level pre-quantisation transform. Full-model rotation
+ # must run *before* calibration data is cached, which is earlier than the
+ # per-member ``prepare_run`` stage; layer-wise rotation instead prepares its
+ # matrices here and rotates each block from within ``compress_block``. All
+ # algorithm-specific decisions live in the rotation members; the composer
+ # only drives the single generic entry point :meth:`apply_model_transforms`.
+
+ def _resolve_rotation_data_type(self) -> str:
+ """Best-effort resolution of the quantization data_type for rotation dispatch."""
+ if self.scheme is not None and getattr(self.scheme, "data_type", None):
+ return self.scheme.data_type
+ if self.block_quantizer is not None:
+ return getattr(self.block_quantizer.config, "data_type", "mx_fp")
+ return "mx_fp"
+
+ def apply_model_transforms(self, model: "torch.nn.Module") -> "torch.nn.Module":
+ """Apply model-level pre-quantisation transforms (rotation) to *model*.
+
+ Generic entry point invoked once by the orchestrator before calibration
+ caching / the block loop. Each rotation member either rotates the whole
+ model immediately (full-model mode) or prepares its rotation matrices and
+ defers per-block work to :meth:`compress_block` (layer-wise mode).
+ Idempotent — repeated calls are a no-op.
+
+ Returns:
+ The (possibly mutated) model.
+ """
+ if self._rotation_prepared:
+ return model
+ if not self._rotation_members:
+ self._rotation_prepared = True
+ return model
+
+ data_type = self._resolve_rotation_data_type()
+ logger.info("Applying Hadamard transform to the model.")
+ for rotation_member in self._rotation_members:
+ # The member honours its own config (``layerwise`` field).
+ model = rotation_member.rotate_model(model, data_type=data_type)
+
+ self._rotation_prepared = True
+ return model
+
+ @property
+ def has_layerwise_rotation(self) -> bool:
+ """Whether any rotation member is driving per-block (layer-wise) rotation."""
+ return any(rotation_member.is_layerwise_active for rotation_member in self._rotation_members)
diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.py
index 46fcbb370..d83935c55 100644
--- a/auto_round/algorithms/transforms/base.py
+++ b/auto_round/algorithms/transforms/base.py
@@ -67,6 +67,13 @@ class BaseRotationConfig:
#: Human-readable algorithm name, must be unique across all subclasses.
algorithm: str = "base"
+ #: Apply rotation per decoder block (layer-wise), in lock-step with
+ #: AutoRound's block-wise quantization, instead of rotating the whole model
+ #: up-front. Only honoured by rotation algorithms whose
+ #: :attr:`BaseRotation.supports_layerwise` is ``True`` (SpinQuant/QuaRot);
+ #: unsupported algorithms transparently fall back to full-model rotation.
+ layerwise: bool = False
+
def can_compile_block_forward(self) -> bool:
"""Whether block replay can be compiled after applying this rotation."""
return True
@@ -117,6 +124,89 @@ class BaseRotation(ABC):
The transformed model.
"""
+ # ------------------------------------------------------------------
+ # Layer-wise rotation interface (optional)
+ # ------------------------------------------------------------------
+
+ @property
+ def supports_layerwise(self) -> bool:
+ """Whether this rotation algorithm supports layer-wise execution.
+
+ When ``True``, the composer can call :meth:`prepare_layerwise`
+ during ``apply_model_transforms`` and then :meth:`rotate_layer`
+ per-block inside the block-quantization loop — avoiding the need to
+ load the entire model onto GPU at once.
+ """
+ return False
+
+ def prepare_layerwise(
+ self,
+ model: torch.nn.Module,
+ data_type: str = "mx_fp",
+ **kwargs: Any,
+ ) -> "BaseRotation":
+ """Prepare for layer-wise rotation without modifying model weights.
+
+ Called once when the rotation config has ``layerwise=True``.
+ Implementations should initialise rotation matrices (as model
+ buffers) and any other lightweight state, but must **not** modify
+ model weights or register hooks yet.
+
+ Args:
+ model: The model to prepare (not modified).
+ data_type: Quantization data type.
+ **kwargs: Algorithm-specific arguments (e.g. ``dataloader``).
+
+ Returns:
+ ``self``, so the composer can later call :meth:`rotate_layer`.
+
+ Raises:
+ NotImplementedError: If the algorithm does not support
+ layer-wise rotation.
+ """
+ raise NotImplementedError(
+ f"{self.__class__.__name__} does not support layer-wise rotation. " f"Use full-model rotation instead."
+ )
+
+ def rotate_layer(
+ self,
+ layer: torch.nn.Module,
+ layer_idx: int,
+ **kwargs: Any,
+ ) -> None:
+ """Apply rotation to a single decoder layer.
+
+ Called per-block in the block-quantization loop, after the block
+ is materialised and placed on the target device, **before**
+ reference-output collection.
+
+ Implementations should:
+ - Rotate weights of target modules inside *layer*.
+ - Register any online hooks (R1/R3/R4) needed at inference time.
+ - Be idempotent — calling twice on the same layer should be safe.
+
+ Args:
+ layer: A single decoder layer, already on the target device.
+ layer_idx: Zero-based index of this layer in the model.
+ **kwargs: Algorithm-specific arguments.
+
+ Raises:
+ NotImplementedError: If the algorithm does not support
+ layer-wise rotation.
+ """
+ raise NotImplementedError(f"{self.__class__.__name__} does not support layer-wise rotation.")
+
+ def finalize_layerwise(self, model: torch.nn.Module) -> None:
+ """Post-loop cleanup after all layers have been rotated.
+
+ Called once after the block-quantization loop completes.
+ Default implementation is a no-op.
+
+ Args:
+ model: The fully-rotated model.
+ """
+ pass
+
# ------------------------------------------------------------------
# Factory
# ------------------------------------------------------------------
diff --git a/auto_round/algorithms/transforms/hadamard/apply.py b/auto_round/algorithms/transforms/hadamard/apply.py
index 73706083d..9e7d03570 100644
--- a/auto_round/algorithms/transforms/hadamard/apply.py
+++ b/auto_round/algorithms/transforms/hadamard/apply.py
@@ -162,6 +162,84 @@ class HadamardRotation(BaseRotation, SerializerMixin):
setattr(model, "_rotation_config", cfg)
return model
+ # ------------------------------------------------------------------
+ # Layer-wise (block-wise) rotation interface
+ # ------------------------------------------------------------------
+ #
+ # The per-Linear ``transform`` backend fuses an independent Hadamard into
+ # each Linear's weight/input; there is no cross-layer (residual-stream)
+ # coupling, so applying it one decoder layer at a time is mathematically
+ # identical to the full-model pass. That makes block-wise rotation a trivial
+ # scoping of :meth:`apply_to_model` down to a single layer's sub-modules.
+ #
+ # The ``inplace`` / QuaRot backend rotates the shared residual stream and
+ # therefore *couples* consecutive layers; it cannot be decomposed per-block,
+ # so :attr:`supports_layerwise` reports ``False`` for it (clean fall back to
+ # full-model rotation) and :meth:`prepare_layerwise` refuses it explicitly.
+
+ @property
+ def supports_layerwise(self) -> bool:
+ """Hadamard supports layer-wise rotation for the per-Linear backend.
+
+ The QuaRot ``inplace`` backend couples layers through the residual
+ stream and cannot run per-block, so it reports ``False`` (the composer
+ then falls back to full-model rotation).
+ """
+ cfg = self.config
+ backend = getattr(cfg, "backend", "auto")
+ hadamard_type = getattr(cfg, "hadamard_type", "") or ""
+ if backend == "inplace" or "inplace" in hadamard_type:
+ return False
+ return True
+
+ def prepare_layerwise(
+ self,
+ model: torch.nn.Module,
+ data_type: str = "mx_fp",
+ location: str = "weight",
+ **kwargs: Any,
+ ) -> "HadamardRotation":
+ """Prepare for per-block Hadamard rotation without touching weights.
+
+ The per-Linear Hadamard needs no global pre-computation — matrices are
+ built lazily inside :meth:`rotate_layer`. This only validates that the
+ resolved backend is the per-Linear ``transform`` path and records the
+ config on *model* (so serialization still works, since the model-level
+ :meth:`apply_to_model` — which normally sets ``_rotation_config`` — is
+ skipped in layer-wise mode).
+ """
+ from auto_round.algorithms.transforms.hadamard.dispatcher import resolve_hadamard_backend
+
+ backend = resolve_hadamard_backend(self.config, data_type)
+ if backend != "transform":
+ raise NotImplementedError(
+ "Layer-wise Hadamard rotation only supports the per-Linear 'transform' backend "
+ f"(MXFP4 / NVFP4). Resolved backend={backend!r} for data_type={data_type!r}. "
+ "Use full-model rotation (layerwise=False), or the inplace/QuaRot rotation path."
+ )
+ self._layerwise_location = location
+ self._layerwise_data_type = data_type
+ setattr(model, "_rotation_config", self.config)
+ return self
+
+ def rotate_layer(
+ self,
+ layer: torch.nn.Module,
+ layer_idx: int,
+ **kwargs: Any,
+ ) -> None:
+ """Apply the per-Linear Hadamard to every target module inside *layer*."""
+ cfg = self.config
+ location = getattr(self, "_layerwise_location", "weight")
+ data_type = getattr(self, "_layerwise_data_type", "mx_fp")
+ target_types = (torch.nn.Linear, QModuleBase)
+ for name, module in layer.named_modules():
+ if not isinstance(module, target_types):
+ continue
+ if "lm_head" in name:
+ continue
+ _apply_to_module(layer, module, cfg, location, data_type)
+
# ------------------------------------------------------------------
# SerializerMixin — Save side
# ------------------------------------------------------------------
diff --git a/auto_round/algorithms/transforms/hadamard/config.py b/auto_round/algorithms/transforms/hadamard/config.py
index ed72ad1c4..f2f2bf85a 100644
--- a/auto_round/algorithms/transforms/hadamard/config.py
+++ b/auto_round/algorithms/transforms/hadamard/config.py
@@ -77,6 +77,11 @@ class RotationConfig(AlgorithmConfig, BaseModel, BaseRotationConfig):
backend: str = Field(default="auto")
block_size: Optional[int] = Field(default=None)
hadamard_type: str = Field(default="hadamard")
+ # Apply the Hadamard rotation per decoder block, in lock-step with block-wise
+ # quantization, instead of rotating the whole model up-front. Honoured by the
+ # per-Linear transform backend; unsupported cases fall back to full-model
+ # rotation (see ``RotationPreprocessor.rotate_model``).
+ layerwise: bool = Field(default=False)
# ---- inplace-only ----
fuse_online_to_weight: Optional[bool] = Field(default=None)
diff --git a/auto_round/algorithms/transforms/member.py b/auto_round/algorithms/transforms/member.py
new file mode 100644
index 000000000..448e98ff3
--- /dev/null
+++ b/auto_round/algorithms/transforms/member.py
@@ -0,0 +1,178 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rotation as a first-class pipeline member.
+
+``RotationPreprocessor`` wraps a :class:`~auto_round.algorithms.transforms.base.BaseRotation`
+and plugs it into the ordinary :class:`~auto_round.algorithms.composer.AlgorithmComposer`
+member lifecycle. It owns the *entire* rotation lifecycle so the composer stays
+rotation-agnostic:
+
+* :meth:`rotate_model` – model-level (Phase 4.5) full-model rotation, or
+ layer-wise preparation (R matrices only).
+* :meth:`on_block_ready` – per-block rotation for layer-wise mode, run as the
+ first step of ``compress_block``. Handles ``nblocks``
+ block fusion (``WrapperMultiblock``) so each fused
+ decoder layer receives the correct ``layer_idx``.
+* :meth:`finalize_run` – layer-wise teardown, invoked automatically via the
+ composer's ``members()`` finalize loop.
+
+The composer only ever calls these generic entry points in a thin loop; all
+algorithm-specific decisions (does this rotation support layer-wise? which
+sub-modules to rotate?) live here / in the concrete ``BaseRotation``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from auto_round.algorithms.transforms.base import BasePreprocessor
+from auto_round.utils import logger
+
+if TYPE_CHECKING: # pragma: no cover
+ import torch
+
+ from auto_round.algorithms.composer import BlockContext
+ from auto_round.algorithms.transforms.base import BaseRotation
+
+
+class RotationPreprocessor(BasePreprocessor):
+ """Pipeline member that owns a single rotation transform's lifecycle.
+
+ Instances are built by the composer from the rotation configs found in the
+ algorithm config list and are added both to ``members()`` (so ``prepare_run``
+ / ``finalize_run`` cover them) and to the composer's dedicated rotation list
+ (so the model-level and per-block entry points can reach them).
+ """
+
+ def __init__(self, config: Any) -> None:
+ super().__init__(config)
+ self._rotation: "BaseRotation | None" = None
+ # Set once :meth:`rotate_model` decides layer-wise preparation succeeded.
+ self._layerwise_active: bool = False
+
+ # ------------------------------------------------------------------
+ # Lazy rotation construction
+ # ------------------------------------------------------------------
+ @property
+ def rotation(self) -> "BaseRotation":
+ """The concrete :class:`BaseRotation` for this member (built lazily)."""
+ if self._rotation is None:
+ from auto_round.algorithms.transforms import normalize_rotation_config
+ from auto_round.algorithms.transforms.base import BaseRotation
+
+ normalised = normalize_rotation_config(self.config)
+ if normalised is None:
+ raise ValueError(f"Rotation member received an empty config: {self.config!r}.")
+ self._rotation = BaseRotation.from_config(normalised)
+ return self._rotation
+
+ @property
+ def is_layerwise(self) -> bool:
+ """Whether the config *requests* per-block rotation (intent only; see
+ :attr:`is_layerwise_active` for whether it actually took effect)."""
+ return bool(getattr(self.config, "layerwise", False))
+
+ @property
+ def is_layerwise_active(self) -> bool:
+ """Whether this member prepared (and will drive) per-block rotation."""
+ return self._layerwise_active
+
+ # ------------------------------------------------------------------
+ # Model-level entry (Phase 4.5)
+ # ------------------------------------------------------------------
+ def rotate_model(
+ self,
+ model: "torch.nn.Module",
+ data_type: str = "mx_fp",
+ ) -> "torch.nn.Module":
+ """Rotate *model* up-front, or prepare layer-wise rotation matrices.
+
+ Whether to rotate per-block is taken solely from the rotation config's
+ ``layerwise`` field (via :attr:`is_layerwise`).
+
+ For full-model rotation the model is rotated immediately and returned.
+ For layer-wise rotation (when the underlying algorithm supports it) only
+ the rotation matrices are initialised; per-block work is deferred to
+ :meth:`on_block_ready`. When layer-wise is requested but unsupported, the
+ member transparently falls back to full-model rotation.
+
+ Returns:
+ The (possibly mutated) model.
+ """
+ rotation = self.rotation
+ if self.is_layerwise:
+ if rotation.supports_layerwise:
+ logger.info(
+ "[Rotation] Layer-wise mode: preparing R matrices only " "(rotation deferred to per-block hook)."
+ )
+ rotation.prepare_layerwise(model, data_type=data_type)
+ self._layerwise_active = True
+ return model
+ logger.warning(
+ f"[Rotation] {rotation.__class__.__name__} does not support "
+ "layer-wise mode. Falling back to full-model rotation."
+ )
+
+ return rotation.apply_to_model(model, data_type=data_type)
+
+ # ------------------------------------------------------------------
+ # Per-block entry (compress_block step 0)
+ # ------------------------------------------------------------------
+ def on_block_ready(self, block: "torch.nn.Module", ctx: "BlockContext") -> None:
+ """Rotate the block about to be quantized (layer-wise mode only).
+
+ No-op unless :meth:`rotate_model` prepared layer-wise rotation. Iterates
+ the (possibly fused) decoder layers inside *block* and rotates each with
+ its global ``layer_idx``.
+ """
+ if not self._layerwise_active:
+ return
+
+ for layer, layer_idx in self._iter_layers(block, ctx.block_index):
+ self.rotation.rotate_layer(layer, layer_idx=layer_idx)
+
+ # ------------------------------------------------------------------
+ # Teardown (members() finalize loop)
+ # ------------------------------------------------------------------
+ def finalize_run(self) -> None:
+ """Finalize layer-wise rotation once all blocks have been processed."""
+ if not self._layerwise_active:
+ return
+ model = self.model_context.model if self.model_context is not None else None
+ if model is None:
+ return
+ self.rotation.finalize_layerwise(model)
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+ @staticmethod
+ def _iter_layers(block: "torch.nn.Module", block_index: int):
+ """Yield ``(decoder_layer, layer_idx)`` for every layer inside *block*.
+
+ ``block_index`` is the global index of the *first* layer in this block.
+ When ``nblocks > 1`` several consecutive decoder layers are fused into a
+ single :class:`~auto_round.wrapper.WrapperMultiblock`; each fused layer
+ ``j`` maps to global index ``block_index + j``. A plain (non-fused) block
+ maps directly to ``block_index``.
+ """
+ from auto_round.wrapper import WrapperMultiblock
+
+ if isinstance(block, WrapperMultiblock):
+ sub_layers = list(block.layers)
+ else:
+ sub_layers = [block]
+
+ for j, layer in enumerate(sub_layers):
+ yield layer, block_index + j
diff --git a/auto_round/algorithms/transforms/spinquant/apply.py b/auto_round/algorithms/transforms/spinquant/apply.py
index 108820bb5..f3c0e0c22 100644
--- a/auto_round/algorithms/transforms/spinquant/apply.py
+++ b/auto_round/algorithms/transforms/spinquant/apply.py
@@ -106,6 +106,67 @@ class SpinQuantRotation(BaseRotation, SerializerMixin):
preprocessor = SpinQuantPreprocessor(model, self.config)
return preprocessor.preprocess(dataloader)
+ # ------------------------------------------------------------------
+ # Layer-wise rotation interface (block-wise quantization)
+ # ------------------------------------------------------------------
+
+ @property
+ def supports_layerwise(self) -> bool:
+ """SpinQuant supports layer-wise rotation (Online R1 + R2/R3/R4)."""
+ return True
+
+ def prepare_layerwise(
+ self,
+ model: torch.nn.Module,
+ data_type: str = "mx_fp",
+ **kwargs: Any,
+ ) -> "SpinQuantRotation":
+ """Prepare for layer-wise rotation: init R matrices only.
+
+ Creates a :class:`SpinQuantPreprocessor`, calls its
+ :meth:`~SpinQuantPreprocessor.prepare` method (which initialises
+ rotation matrices without modifying weights), and stores the
+ preprocessor for later :meth:`rotate_layer` calls.
+
+ Args:
+ model: The model to prepare (weights not modified).
+ data_type: Quantization data type (informational).
+ **kwargs: ``dataloader`` forwarded when ``trainable_rotation=True``.
+
+ Returns:
+ ``self``, for chaining.
+ """
+ from auto_round.algorithms.transforms.spinquant.preprocessor import (
+ SpinQuantPreprocessor,
+ )
+
+ dataloader = kwargs.get("dataloader")
+ self._preprocessor = SpinQuantPreprocessor(model, self.config)
+ self._preprocessor.prepare(dataloader)
+ return self
+
+ def rotate_layer(
+ self,
+ layer: torch.nn.Module,
+ layer_idx: int,
+ **kwargs: Any,
+ ) -> None:
+ """Apply rotation to a single decoder layer.
+
+ Delegates to :meth:`SpinQuantPreprocessor.rotate_layer`.
+ """
+ if not hasattr(self, "_preprocessor"):
+ raise RuntimeError(
+ "prepare_layerwise() must be called before rotate_layer(). "
+ "The preprocessor has not been initialised."
+ )
+ self._preprocessor.rotate_layer(layer, layer_idx)
+
+ def finalize_layerwise(self, model: torch.nn.Module) -> None:
+ """Cleanup after all layers have been rotated."""
+ if hasattr(self, "_preprocessor"):
+ self._preprocessor.finalize()
+
# ------------------------------------------------------------------
# SerializerMixin — Save side
# ------------------------------------------------------------------
diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.py
index f13df42be..0c41271b6 100644
--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py
+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py
@@ -333,6 +333,271 @@ class SpinQuantPreprocessor:
logger.info("[SpinQuant] Preprocessing complete!")
return self.model
+ # ------------------------------------------------------------------
+ # Layer-wise API (for block-lifecycle / block-wise quantization)
+ # ------------------------------------------------------------------
+
+ def prepare(self, dataloader: Optional[Any] = None) -> None:
+ """Global preparation for layer-wise (block-wise) rotation.
+
+ Performs all lightweight, non-destructive steps:
+ - Validates dimensions
+ - Initialises rotation matrices (stored as model buffers)
+ - Optionally trains rotation matrices (requires full model + dataloader)
+ - Pre-computes rotation state shared across layers
+
+ Does **NOT** modify model weights or register hooks — that is done
+ per-layer by :meth:`rotate_layer`.
+
+ This method is the layer-wise counterpart of :meth:`preprocess`.
+
+ Args:
+ dataloader: Required only when ``trainable_rotation=True``.
+
+ Raises:
+ ValueError: If ``online_r1_rotation=False`` while R1 is enabled
+ (offline R1 is incompatible with layer-wise mode because it
+ rotates the inter-layer hidden-state space via embed_tokens /
+ lm_head, which breaks pre-cached block inputs).
+ """
+ # Layer-wise rotation only works with Online R1 (offline R1 rotates the
+ # shared residual stream, which is incompatible with pre-cached block
+ # inputs).
+ if self.config.r1 and not self.config.online_r1_rotation:
+ raise ValueError(
+ "[SpinQuant] Layer-wise rotation requires online_r1_rotation=True. "
+ "Offline R1 changes inter-layer hidden state space, which is "
+ "incompatible with pre-cached block inputs. Use full-model "
+ "rotation (config `layerwise=False`) for offline R1."
+ )
+
+ logger.info("[SpinQuant] Preparing for layer-wise rotation...")
+ logger.info(
+ f"[SpinQuant] Model architecture: hidden_size={self.hidden_size}, "
+ f"head_dim={self.head_dim}, intermediate_size={self.intermediate_size}"
+ )
+ logger.info(
+ f"[SpinQuant] Rotation config: R1={self.config.r1}, R2={self.config.r2}, "
+ f"R3={self.config.r3}, R4={self.config.r4}, "
+ f"online_r1={self.config.online_r1_rotation}"
+ )
+
+ # Step 1: Validate dimensions
+ self._validate_dimensions()
+
+ # Step 2: Initialise rotation matrices (lightweight — only R matrix buffers)
+ logger.info("[SpinQuant] Initialising rotation matrices...")
+ self._init_rotation_matrices()
+
+ # Step 3: Train if requested (needs full model forward/backward)
+ if self.config.trainable_rotation or self.config.trainable_smooth:
+ if dataloader is None:
+ raise ValueError("dataloader required when trainable=True")
+ logger.warning(
+ "[SpinQuant] Layer-wise mode with trainable_rotation: training "
+ "requires a full model forward/backward pass (higher memory). "
+ "Consider pre-training R matrices separately for large models."
+ )
+ # Trainable smooth needs norm replacement first
+ if self.config.trainable_smooth:
+ self._replace_norms_with_trainable()
+ self._train_rotations(dataloader)
+
+ # Pre-compute shared state consumed by rotate_layer()
+ if self.config.r1 and self.config.online_r1_rotation:
+ self._r1_shared = self._compute_online_r1_shared()
+ else:
+ self._r1_shared = None
+
+ if self.config.r2 and self.head_dim > 0:
+ R2_head = self._get_rotation_tensor("spinquant_R2_head")
+ if R2_head is not None:
+ R2 = R2_head.data.to(torch.float64)
+ self._r2_shared = (R2, R2.t())
+ else:
+ self._r2_shared = None
+ else:
+ self._r2_shared = None
+
+ if self.config.r4 and self.r4_rotation_size > 0:
+ use_random_r4 = self.config.random_r4
+ R4 = None
+ if use_random_r4:
+ R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
+ if R4_matrix is None:
+ raise RuntimeError("[SpinQuant] random_r4=True but spinquant_R4_matrix buffer not found.")
+ R4 = R4_matrix.to(torch.float64)
+ self._r4_shared = (self.r4_rotation_size, use_random_r4, R4)
+ else:
+ self._r4_shared = None
+
+ # Store config on model for downstream serialization
+ self.model._rotation_config = self.config
+ self.model._spinquant_config = self.config # legacy alias
+
+ logger.info("[SpinQuant] Layer-wise preparation complete (no weights modified yet).")
+
+ def rotate_layer(self, layer: nn.Module, layer_idx: int) -> None:
+ """Apply all configured rotations to a single decoder layer.
+
+ This is the per-block entry point called by the compressor inside the
+ block-wise quantization loop. The layer must already be materialised
+ and on the target device.
+
+ Operations performed (in order):
+ 1. **R1 (Online)**: rotate target-module weights + register hooks
+ 2. **R2 (Offline fuse)**: fuse into v_proj output + o_proj input
+ 3. **R4 (Offline fuse)**: fuse into down_proj input
+ 4. **R3 (Hook)**: monkeypatch apply_rotary_pos_emb
+ 5. **R4 (Hook)**: register forward_pre_hook on down_proj
+
+ Args:
+ layer: A single decoder layer, already on the target device.
+ layer_idx: Zero-based index for logging purposes.
+ """
+ if not hasattr(self, "_r1_shared"):
+ raise RuntimeError(
+ "[SpinQuant] prepare() must be called before rotate_layer(). "
+ "The layer-wise state has not been initialised."
+ )
+ if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
+ logger.warning(f"[SpinQuant] Layer {layer_idx}: no self_attn/mlp, skipping rotation.")
+ return
+
+ layer_device = next(layer.parameters()).device
+
+ # ── R1: Online weight rotation + hooks ──
+ if self._r1_shared is not None:
+ self._apply_online_r1_to_layer(layer, *self._r1_shared)
+
+ # ── R2: Offline fuse into v_proj/o_proj ──
+ if self._r2_shared is not None:
+ self._fuse_r2_to_layer(layer, *self._r2_shared)
+
+ # ── R4: Offline fuse into down_proj ──
+ if self._r4_shared is not None:
+ self._fuse_r4_to_layer(layer, *self._r4_shared)
+
+ # ── R3: Monkeypatch attention (after RoPE) ──
+ if self.config.r3 and self.head_dim > 0:
+ self._rotate_layer_r3_hook(layer, layer_idx)
+
+ # ── R4: Hook on down_proj ──
+ if self.config.r4 and self.r4_rotation_size > 0:
+ self._rotate_layer_r4_hook(layer, layer_idx, layer_device)
+
+ logger.debug(f"[SpinQuant] Layer {layer_idx}: rotation applied.")
+
+ def _rotate_layer_r3_hook(self, layer: nn.Module, layer_idx: int) -> None:
+ """Apply the R3 monkeypatch to a single layer's attention module.
+
+ Consistent with the model-level ``register_spinquant_hooks`` R3 path.
+ """
+ if not is_pow2(self.head_dim):
+ return
+
+ from auto_round.algorithms.transforms.spinquant.monkeypatch import (
+ add_qk_rotation_after_rope,
+ )
+
+ attn = layer.self_attn
+ if not (hasattr(attn, "q_proj") and hasattr(attn, "k_proj")):
+ return
+ if getattr(attn, "_spinquant_r3_patched", False):
+ return # Already patched (idempotent)
+
+ random_r3 = self.config.random_r3
+ r3_matrix = getattr(self.model, "spinquant_R3_head", None)
+
+ try:
+ wrapper = add_qk_rotation_after_rope(attn, rope_function_name="apply_rotary_pos_emb")
+ if random_r3 and r3_matrix is not None:
+ wrapper.set_matrix(r3_matrix)
+ else:
+ wrapper.set_hadamard(None, self.head_dim)
+ attn._spinquant_r3_patched = True
+ self._hook_handles.append(("r3_monkeypatch", f"layer_{layer_idx}", attn, wrapper))
+ except ValueError as e:
+ logger.warning(f"[SpinQuant] R3 monkeypatch failed for layer {layer_idx}: {e}")
+
+ def _rotate_layer_r4_hook(self, layer: nn.Module, layer_idx: int, layer_device: torch.device) -> None:
+ """Register the R4 ``forward_pre_hook`` on a single layer's down_proj.
+
+ Consistent with the model-level ``register_spinquant_hooks`` R4 path.
+ """
+ mlp = layer.mlp
+ if not hasattr(mlp, "down_proj"):
+ return
+
+ r4_size = self.r4_rotation_size
+ use_random = self.config.random_r4
+ need_block = r4_size < self.intermediate_size
+ module = mlp.down_proj
+
+ if use_random:
+ R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
+ if R4_matrix is None:
+ return
+ R4 = R4_matrix.to(device=layer_device, dtype=torch.float32)
+
+ def _make_hook(R, rot_size, block_mode):
+ def hook(mod, args):
+ x = args[0]
+ R_local = R.to(x.device, dtype=x.dtype)
+ if block_mode:
+ shape = x.shape
+ x = x.reshape(*shape[:-1], -1, rot_size)
+ x = (x @ R_local).reshape(shape)
+ else:
+ x = x @ R_local
+ return (x,) + args[1:]
+
+ return hook
+
+ hook = _make_hook(R4, r4_size, need_block)
+ else:
+ try:
+ had_K_mat, had_K_val = get_hadamard_K(r4_size)
+ except ValueError:
+ return
+ had_K_mat = had_K_mat.to(device=layer_device, dtype=torch.float32)
+
+ def _make_hook_butterfly(had_mat, k_val, rot_size, block_mode):
+ def hook(mod, args):
+ x = args[0]
+ if block_mode:
+ shape = x.shape
+ x = x.reshape(*shape[:-1], -1, rot_size)
+ x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)
+ x = x.reshape(shape)
+ else:
+ x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)
+ return (x,) + args[1:]
+
+ return hook
+
+ hook = _make_hook_butterfly(had_K_mat, had_K_val, r4_size, need_block)
+
+ hook._spinquant_hook = True
+ handle = module.register_forward_pre_hook(hook)
+ self._hook_handles.append(handle)
+
+ def finalize(self) -> None:
+ """Post-loop cleanup after all layers have been rotated layer-wise.
+
+ Mirrors :meth:`_cleanup` but for the layer-wise path.
+ """
+ self.model.eval()
+ for p in self.model.parameters():
+ p.requires_grad = False
+ self.rotation_params.clear()
+ self.smooth_params.clear()
+ self._rotated_modules.clear()
+
+ n_r1 = len(self._r1_hook_handles)
+ n_other = len(self._hook_handles)
+ logger.info(f"[SpinQuant] Layer-wise rotation finalized. " f"Active hooks: {n_r1} R1 + {n_other} R3/R4.")
+
def _validate_dimensions(self) -> None:
"""Validate dimension requirements and disable rotations that can't work."""
# R1: check r1_rotation_size divides hidden_size and is power of 2
@@ -694,6 +959,34 @@ class SpinQuantPreprocessor:
"need to save/reload the model."
)
+ r1_size, use_random, R1_full, hadamard_K, K = self._compute_online_r1_shared()
+
+ n_rotated = 0
+ n_hooked = 0
+
+ for layer in self._get_layers():
+ nr, nh = self._apply_online_r1_to_layer(layer, r1_size, use_random, R1_full, hadamard_K, K)
+ n_rotated += nr
+ n_hooked += nh
+
+ mode_str = "random matrix (x @ R)" if use_random else "deterministic butterfly"
+ logger.info(
+ f"[SpinQuant] Online R1: rotated {n_rotated} target modules, "
+ f"registered {n_hooked} activation hooks "
+ f"(rotation_size={r1_size}, mode={mode_str}, "
+ f"lm_head/embed_tokens/o_proj/down_proj unchanged)"
+ )
+
+ def _compute_online_r1_shared(self):
+ """Compute R1 rotation state shared across all layers (online R1).
+
+ Returns a tuple ``(r1_size, use_random, R1_full, hadamard_K, K)`` that
+ can be passed to :meth:`_apply_online_r1_to_layer`. Extracted so that
+ both full-model (:meth:`_apply_online_r1`) and layer-wise
+ (:meth:`rotate_layer`) paths share identical R1 conventions.
+ """
+ r1_size = self.r1_rotation_size
+ use_random = self.config.random_r1
model_device = next(self.model.parameters()).device
# For random R1: use the stored full matrix
@@ -711,88 +1004,86 @@ class SpinQuantPreprocessor:
R1_full = None
hadamard_K, K = get_hadamard_K(r1_size)
hadamard_K = hadamard_K.to(model_device)
+ return r1_size, use_random, R1_full, hadamard_K, K
- n_rotated = 0
- n_hooked = 0
+ def _apply_online_r1_to_layer(self, layer, r1_size, use_random, R1_full, hadamard_K, K):
+ """Apply online R1 to a single decoder layer: rotate target-module
+ weights and register the matching activation ``forward_pre_hook`` s.
- for layer in self._get_layers():
- if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
- continue
+ Returns ``(n_rotated, n_hooked)``.
+ """
+ if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
+ return 0, 0
- layer_device = next(layer.parameters()).device
+ layer_device = next(layer.parameters()).device
- attn = layer.self_attn
- mlp = layer.mlp
+ attn = layer.self_attn
+ mlp = layer.mlp
- # Target modules: (parent_module, attr_name)
- target_specs = []
- for proj_name in ("q_proj", "k_proj", "v_proj"):
- if hasattr(attn, proj_name):
- target_specs.append((attn, proj_name))
- for proj_name in ("gate_proj", "up_proj"):
- if hasattr(mlp, proj_name):
- target_specs.append((mlp, proj_name))
-
- for parent, attr_name in target_specs:
- module = getattr(parent, attr_name)
- dtype = module.weight.data.dtype
- in_features = module.weight.shape[-1]
-
- if use_random:
- # Random R1: explicit matrix multiply
- R = R1_full.to(layer_device)
- if r1_size == in_features:
- W = module.weight.data.to(torch.float64)
- module.weight.data = (W @ R).to(dtype)
- elif in_features % r1_size == 0:
- # Block rotation must use W @ R (not W @ R.T) to stay
- # consistent with the online hook (x @ R). The random
- # Hadamard matrix is orthonormal but NOT symmetric, so
- # rotate_in_channels_ (which applies R.T) would break
- # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.
- rotate_in_channels_(module, R_in=R.T)
- else:
- raise ValueError(
- f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
- )
+ # Target modules: (parent_module, attr_name)
+ target_specs = []
+ for proj_name in ("q_proj", "k_proj", "v_proj"):
+ if hasattr(attn, proj_name):
+ target_specs.append((attn, proj_name))
+ for proj_name in ("gate_proj", "up_proj"):
+ if hasattr(mlp, proj_name):
+ target_specs.append((mlp, proj_name))
+
+ n_rotated = 0
+ n_hooked = 0
+ for parent, attr_name in target_specs:
+ module = getattr(parent, attr_name)
+ dtype = module.weight.data.dtype
+ in_features = module.weight.shape[-1]
+
+ if use_random:
+ # Random R1: explicit matrix multiply
+ R = R1_full.to(layer_device)
+ if r1_size == in_features:
+ W = module.weight.data.to(torch.float64)
+ module.weight.data = (W @ R).to(dtype)
+ elif in_features % r1_size == 0:
+ # Block rotation must use W @ R (not W @ R.T) to stay
+ # consistent with the online hook (x @ R). The random
+ # Hadamard matrix is orthonormal but NOT symmetric, so
+ # rotate_in_channels_ (which applies R.T) would break
+ # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.
+ rotate_in_channels_(module, R_in=R.T)
else:
- # Deterministic Hadamard: butterfly algorithm
- had_K_local = hadamard_K.to(layer_device)
- if r1_size == in_features:
- module.weight.data = matmul_hadU(module.weight.data, hadamard_K=had_K_local, K=K).to(dtype)
- elif in_features % r1_size == 0:
- R_block = had_K_local.to(torch.float64)
- if R_block.shape[0] != r1_size:
- had_1, _ = get_hadamard_K(r1_size // K)
- R_block = torch.kron(
- had_K_local.to(device="cpu", dtype=torch.float64),
- had_1.to(device="cpu", dtype=torch.float64),
- )
- R_block = R_block / math.sqrt(r1_size)
- rotate_in_channels_(module, R_in=R_block)
- else:
- raise ValueError(
- f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
+ raise ValueError(
+ f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
+ )
+ else:
+ # Deterministic Hadamard: butterfly algorithm
+ had_K_local = hadamard_K.to(layer_device)
+ if r1_size == in_features:
+ module.weight.data = matmul_hadU(module.weight.data, hadamard_K=had_K_local, K=K).to(dtype)
+ elif in_features % r1_size == 0:
+ R_block = had_K_local.to(torch.float64)
+ if R_block.shape[0] != r1_size:
+ had_1, _ = get_hadamard_K(r1_size // K)
+ R_block = torch.kron(
+ had_K_local.to(device="cpu", dtype=torch.float64),
+ had_1.to(device="cpu", dtype=torch.float64),
)
- n_rotated += 1
-
- # Register forward_pre_hook for online activation rotation
- if use_random:
- hook = self._make_online_r1_hook_matrix(R1_full.to(layer_device), r1_size, in_features)
+ R_block = R_block / math.sqrt(r1_size)
+ rotate_in_channels_(module, R_in=R_block)
else:
- hook = self._make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K.to(layer_device), K)
- hook._spinquant_hook = True # tag for selective removal
- handle = module.register_forward_pre_hook(hook)
- self._r1_hook_handles.append(handle)
- n_hooked += 1
+ raise ValueError(
+ f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
+ )
+ n_rotated += 1
- mode_str = "random matrix (x @ R)" if use_random else "deterministic butterfly"
- logger.info(
- f"[SpinQuant] Online R1: rotated {n_rotated} target modules, "
- f"registered {n_hooked} activation hooks "
- f"(rotation_size={r1_size}, mode={mode_str}, "
- f"lm_head/embed_tokens/o_proj/down_proj unchanged)"
- )
+ # Register forward_pre_hook for online activation rotation
+ if use_random:
+ hook = self._make_online_r1_hook_matrix(R1_full.to(layer_device), r1_size, in_features)
+ else:
+ hook = self._make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K.to(layer_device), K)
+ hook._spinquant_hook = True # tag for selective removal
+ handle = module.register_forward_pre_hook(hook)
+ self._r1_hook_handles.append(handle)
+ n_hooked += 1
+ return n_rotated, n_hooked
@staticmethod
def _make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K, K):
@@ -957,34 +1248,41 @@ class SpinQuantPreprocessor:
n_fused = 0
for layer in self._get_layers():
- if not hasattr(layer, "self_attn"):
- continue
- attn = layer.self_attn
+ n_fused += self._fuse_r2_to_layer(layer, R2, R2_T)
- # v_proj: W_new = R2^T @ W per head on output dimension
- if hasattr(attn, "v_proj"):
- W = attn.v_proj.weight.data
- dtype = W.dtype
- W = W.to(torch.float64)
- n_heads = W.shape[0] // self.head_dim
- W_reshaped = W.reshape(n_heads, self.head_dim, W.shape[1])
- W_reshaped = torch.einsum("ij,kjl->kil", R2_T, W_reshaped)
- attn.v_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
-
- # o_proj: W_new = W @ R2 per head on input dimension
- # (R2^{-1} = R2^T on the activation side ↔ W @ R2 on the weight side)
- if hasattr(attn, "o_proj"):
- W = attn.o_proj.weight.data
- dtype = W.dtype
- W = W.to(torch.float64)
- n_heads = W.shape[1] // self.head_dim
- W_reshaped = W.reshape(W.shape[0], n_heads, self.head_dim)
- W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R2)
- attn.o_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
+ logger.info(f"[SpinQuant] R2 fused into {n_fused} layers (v_proj out + o_proj in, head_dim={self.head_dim})")
- n_fused += 1
+ def _fuse_r2_to_layer(self, layer, R2, R2_T) -> int:
+ """Fuse R2 per-head rotation into a single layer's v_proj/o_proj.
- logger.info(f"[SpinQuant] R2 fused into {n_fused} layers (v_proj out + o_proj in, head_dim={self.head_dim})")
+ Returns 1 if the layer was fused, else 0.
+ """
+ if not hasattr(layer, "self_attn"):
+ return 0
+ attn = layer.self_attn
+
+ # v_proj: W_new = R2^T @ W per head on output dimension
+ if hasattr(attn, "v_proj"):
+ W = attn.v_proj.weight.data
+ dtype = W.dtype
+ W = W.to(torch.float64)
+ n_heads = W.shape[0] // self.head_dim
+ W_reshaped = W.reshape(n_heads, self.head_dim, W.shape[1])
+ W_reshaped = torch.einsum("ij,kjl->kil", R2_T.to(W.device), W_reshaped)
+ attn.v_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
+
+ # o_proj: W_new = W @ R2 per head on input dimension
+ # (R2^{-1} = R2^T on the activation side ↔ W @ R2 on the weight side)
+ if hasattr(attn, "o_proj"):
+ W = attn.o_proj.weight.data
+ dtype = W.dtype
+ W = W.to(torch.float64)
+ n_heads = W.shape[1] // self.head_dim
+ W_reshaped = W.reshape(W.shape[0], n_heads, self.head_dim)
+ W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R2.to(W.device))
+ attn.o_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
+
+ return 1
def _fuse_r4_rotation(self) -> None:
"""Fuse R4 rotation into down_proj's input side.
@@ -1004,6 +1302,7 @@ class SpinQuantPreprocessor:
use_random = self.config.random_r4
# Get the rotation matrix
+ R4 = None
if use_random:
R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
if R4_matrix is None:
@@ -1012,38 +1311,7 @@ class SpinQuantPreprocessor:
n_fused = 0
for layer in self._get_layers():
- if not hasattr(layer, "mlp"):
- continue
- mlp = layer.mlp
- if hasattr(mlp, "down_proj"):
- W = mlp.down_proj.weight.data
- dtype = W.dtype
-
- if use_random:
- # Random: explicit W @ R per block
- W = W.to(torch.float64)
- if r4_size == W.shape[1]:
- mlp.down_proj.weight.data = (W @ R4).to(dtype)
- else:
- out_feat, in_feat = W.shape
- n_blocks = in_feat // r4_size
- W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
- W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R4)
- mlp.down_proj.weight.data = W_reshaped.reshape(out_feat, in_feat).to(dtype)
- else:
- # Deterministic: matmul_hadU (butterfly algorithm)
- # matmul_hadU operates on the last dimension — for weight
- # shape [out, in], last dim = in_features which is what
- # we want to rotate (input channels of down_proj).
- if r4_size == W.shape[1]:
- mlp.down_proj.weight.data = matmul_hadU(W).to(dtype)
- else:
- out_feat, in_feat = W.shape
- n_blocks = in_feat // r4_size
- W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
- W_rotated = matmul_hadU(W_reshaped)
- mlp.down_proj.weight.data = W_rotated.reshape(out_feat, in_feat).to(dtype)
- n_fused += 1
+ n_fused += self._fuse_r4_to_layer(layer, r4_size, use_random, R4)
mode_str = "random x @ R" if use_random else "deterministic butterfly"
logger.info(
@@ -1051,6 +1319,47 @@ class SpinQuantPreprocessor:
f"(r4_rotation_size={r4_size}, mode={mode_str})"
)
+ def _fuse_r4_to_layer(self, layer, r4_size, use_random, R4) -> int:
+ """Fuse R4 rotation into a single layer's down_proj input side.
+
+ Returns 1 if the layer was fused, else 0.
+ """
+ if not hasattr(layer, "mlp"):
+ return 0
+ mlp = layer.mlp
+ if not hasattr(mlp, "down_proj"):
+ return 0
+
+ W = mlp.down_proj.weight.data
+ dtype = W.dtype
+
+ if use_random:
+ # Random: explicit W @ R per block
+ W = W.to(torch.float64)
+ R4 = R4.to(W.device)
+ if r4_size == W.shape[1]:
+ mlp.down_proj.weight.data = (W @ R4).to(dtype)
+ else:
+ out_feat, in_feat = W.shape
+ n_blocks = in_feat // r4_size
+ W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
+ W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R4)
+ mlp.down_proj.weight.data = W_reshaped.reshape(out_feat, in_feat).to(dtype)
+ else:
+ # Deterministic: matmul_hadU (butterfly algorithm)
+ # matmul_hadU operates on the last dimension — for weight
+ # shape [out, in], last dim = in_features which is what
+ # we want to rotate (input channels of down_proj).
+ if r4_size == W.shape[1]:
+ mlp.down_proj.weight.data = matmul_hadU(W).to(dtype)
+ else:
+ out_feat, in_feat = W.shape
+ n_blocks = in_feat // r4_size
+ W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
+ W_rotated = matmul_hadU(W_reshaped)
+ mlp.down_proj.weight.data = W_rotated.reshape(out_feat, in_feat).to(dtype)
+ return 1
+
# ------------------------------------------------------------------
# Step 8: Cleanup
# ------------------------------------------------------------------
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index d5617e7aa..fc722ef53 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -24,7 +24,6 @@ from transformers import AutoConfig, set_seed
from auto_round.algorithms.quantization import BaseQuantizer, QuantizationConfig
from auto_round.algorithms.transforms import (
BaseRotationConfig,
- apply_rotation,
)
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.compressors.config_resolution import (
@@ -1408,7 +1407,6 @@ class BaseOrchestrator(object):
self._resolve_formats()
self._patch_model()
self._build_layer_config()
- self._apply_rotations()
# Reclaim temporaries from Phases 1-4 (scheme resolution, format
# parsing, model patching, layer-config walk) before Phase 5
@@ -1423,6 +1421,14 @@ class BaseOrchestrator(object):
# so _build_composer must run first.
self._build_composer()
+ # Phase 4.5 – Model-level pre-quantisation transforms (rotation).
+ # Applies full-model rotation up-front (or prepares layer-wise rotation
+ # matrices). Runs here so every entry point — the full quantize() loop,
+ # the zero-shot loop, and the external single-block quantize_block() API —
+ # sees a consistently transformed model before any calibration data is
+ # collected. Rotation is now owned by the composer's rotation members.
+ self.model_context.model = self.alg_composer.apply_model_transforms(self.model_context.model)
+
# Set block_forward torch compile for block forward
# Final trim after all init phases.
gc.collect()
@@ -1632,39 +1638,6 @@ class BaseOrchestrator(object):
# Defer ShardWriter construction to _ensure_shard_writer() to avoid
# heap fragmentation during post_init (parameter iteration).
- def _apply_rotations(self) -> None:
- """Phase 4.5 – Apply Hadamard / rotation transforms to the model.
-
- Preconditions:
- - Phase 3 complete: model topology is final (``apply_patches`` has
- replaced / merged layers, e.g. MoE experts), so rotation operates
- on the same modules that quantization will later see.
- - Phase 4 complete: ``self.layer_config`` is built; rotation only
- transforms weights and does not change layer names, so this
- ordering matches the old arch where rotation ran after
- ``configure_layer_config``.
- - ``self.quantize_config.data_type`` is final (rotation backend
- dispatch depends on it).
-
- Work performed:
- - Iterates ``self.rotation_configs`` and calls
- :func:`~auto_round.algorithms.transforms.apply_rotation` on the
- model for each config.
-
- Postconditions:
- - ``self.model_context.model`` carries the rotated weights and any
- inserted online-Hadamard hooks.
- """
- if not self.rotation_configs:
- return
- logger.info("Applying Hadamard transform to the model.")
- for rotation_cfg in self.rotation_configs:
- self.model_context.model = apply_rotation(
- self.model_context.model,
- rotation_cfg,
- data_type=self.quantize_config.data_type,
- )
-
def _patch_model(self) -> None:
"""Phase 3 – Model structure patching.
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index cc2a9f7b5..350c7b486 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -440,6 +440,7 @@ class CompressionOrchestrator(BaseOrchestrator):
all_blocks = self.quant_block_list or get_block_names(self.model)
pbar = tqdm(range(sum(len(block) for block in all_blocks)))
+ _zs_block_idx = 0
for block_names in all_blocks:
for block_name in block_names:
pbar.set_description(f"Quantizing {block_name}")
@@ -461,12 +462,15 @@ class CompressionOrchestrator(BaseOrchestrator):
materialize_model_(block)
# ── Pure algorithm ────────────────────────────────────────
+ # ``block_index`` carries the global block index so compress_block
+ # can drive layer-wise rotation with the correct layer_idx.
ctx = BlockContext(
model=self.model,
block_names=[block_name],
block_name=block_name,
- block_index=0,
+ block_index=_zs_block_idx,
)
+ _zs_block_idx += 1
# ── MoE scale alignment for FP8 dispatch efficiency ────────────────
if is_nv_fp(self.act_data_type) or not self.act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
@@ -512,6 +516,9 @@ class CompressionOrchestrator(BaseOrchestrator):
memory_monitor.log_summary()
pbar.update(1)
+ # ── Pipeline lifecycle: model-level teardown (also finalizes rotation) ─
+ self.alg_composer.finalize_run()
+
remain_layer_names = []
block_name_set = set(name for block in all_blocks for name in block)
for n, m in self.model.named_modules():
@@ -771,6 +778,10 @@ class CompressionOrchestrator(BaseOrchestrator):
for rs in resume_states:
rs.clear()
+ # ── Pipeline lifecycle: model-level teardown (also finalizes any
+ # layer-wise rotation). Symmetric with ``prepare_run`` above. ──────
+ self.alg_composer.finalize_run()
+
pbar.set_description("Quantizing done")
pbar.close()
if self.compress_context.low_cpu_mem_usage:
@@ -1024,6 +1035,20 @@ class CompressionOrchestrator(BaseOrchestrator):
if not self._post_init_done:
self.post_init()
+ # Layer-wise rotation is driven by the internal block loop inside
+ # ``AlgorithmComposer.compress_block`` (rotate as step 0, cleanup in
+ # ``finalize_run``). This externally-driven single-block API cannot
+ # guarantee that lifecycle, and rotating here would desync the caller's
+ # own reference/teacher outputs (collected on the un-rotated block).
+ # Fail loudly instead of producing silently wrong results.
+ if self.alg_composer.has_layerwise_rotation:
+ raise NotImplementedError(
+ "Layer-wise rotation (rotation config `layerwise=True`) is not supported "
+ "through the single-block quantize_block() API (e.g. LLM-Compressor). Use "
+ "the full AutoRound quantize() entry point, or set `layerwise=False` on the "
+ "rotation config to apply full-model rotation up-front."
+ )
+
# ── Zero-shot (RTN) path: no calibration data needed ──────────────────
if not self.need_calib:
from auto_round.algorithms.composer import BlockContext
diff --git a/docs/rotation_details.md b/docs/rotation_details.md
index 2b9f21819..2409ee02c 100644
--- a/docs/rotation_details.md
+++ b/docs/rotation_details.md
@@ -87,6 +87,7 @@ Defined in `auto_round/algorithms/transforms/spinquant/preprocessor.py`.
| `kl_top_k` | `1000` | Top-k logits used by the `kl_top` loss. |
| `fuse_rmsnorm` | `True` | Fuse RMSNorm scales into following linear layers. |
| `untie_embeddings` | `True` | Untie shared embedding / LM-head weights before rotating. |
+| `layerwise` | `False` | Apply rotation **per decoder block**, in lock-step with AutoRound's block-wise quantization, instead of rotating the whole model up-front (see §1.8). Supported by QuaRot / SpinQuant and the per-Linear Hadamard *transform* backend; the Hadamard *inplace* backend ignores it and falls back to full-model rotation. |
| `dtype` | `torch.float32` | Numerical dtype used for rotation math. |
| `device` | `None` (auto) | Defaults to `"cuda"` if available, else `"cpu"`. |
@@ -208,6 +209,69 @@ with a `*_type` code: `0` = deterministic, `1` = random, `2` = trained):
re-patches the QuantLinear forward (R1/R4) and re-applies the R3 RoPE monkeypatch.
R3 is rebuilt purely from `config.json`, not from stored buffers.
+### 1.8 Block-wise (layer-wise) rotation
+
+By default QuaRot / SpinQuant rotate the **entire model up-front**: R1 is fused into
+the embedding and all linears, R2 into attention, and R3/R4 are installed as online
+hooks — after which quantization runs. This requires the whole model to be resident at
+once, which is heavy for large models and decoupled from AutoRound's block-wise
+quantization loop.
+
+Setting `layerwise=True` on the rotation config aligns rotation with block-wise
+quantization: only the rotation matrices (R1–R4) are initialised up-front, and each
+decoder block is rotated **just before it is quantized**, so only one block needs to be
+on-device at a time.
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig
+
+# Block-wise QuaRot: rotate each decoder block in lock-step with quantization.
+ar = AutoRound(
+ "Qwen/Qwen3-0.6B",
+ scheme="MXFP4",
+ alg_configs=["rtn", SpinQuantConfig(r1=True, r2=True, layerwise=True)],
+)
+ar.quantize_and_save(output_dir="./Qwen3-0.6B-mxfp4-quarot-blockwise", format="auto_round")
+```
+
+The per-Linear Hadamard rotation also supports block-wise execution. Because it fuses an
+independent Hadamard into each Linear (no cross-layer residual-stream coupling), rotating
+one decoder block at a time is mathematically identical to the full-model pass:
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.transforms.hadamard.config import RotationConfig
+
+# Block-wise Hadamard (per-Linear transform backend, MXFP4 / NVFP4).
+ar = AutoRound(
+ "Qwen/Qwen3-0.6B",
+ scheme="MXFP4",
+ iters=0,
+ rotation_config=RotationConfig(hadamard_type="hadamard", layerwise=True),
+)
+ar.quantize_and_save(output_dir="./Qwen3-0.6B-mxfp4-hadamard-blockwise", format="auto_round")
+```
+
+Notes and limitations:
+
+- **Supported algorithms.** `layerwise` is honoured by QuaRot / SpinQuant and by the
+ per-Linear Hadamard **transform** backend (`BaseRotation.supports_layerwise == True`).
+ The Hadamard **inplace** / QuaRot residual-stream backend (`backend="inplace"`) couples
+ consecutive layers, so it reports `supports_layerwise == False` and transparently falls
+ back to full-model rotation. Requesting `layerwise=True` for the transform backend on a
+ non-MXFP4/NVFP4 dtype raises `NotImplementedError` (block-wise Hadamard needs the
+ per-Linear kernel path).
+- **`nblocks` aware.** When several decoder layers are fused into one scheduling group
+ (`nblocks > 1`), each fused layer receives its correct global `layer_idx`, so R2/R3/R4
+ head-dim math stays aligned with the model topology.
+- **Single-block API.** Layer-wise rotation is driven by AutoRound's internal block loop
+ and is **not** supported through the external single-block `quantize_block()` API
+ (e.g. LLM-Compressor); attempting it raises `NotImplementedError`. Use the full
+ `quantize()` entry point, or set `layerwise=False`.
+- **Accuracy parity.** Block-wise rotation is mathematically equivalent to full-model
+ rotation for the R1/R2/R3/R4 positions it supports; see §4 for measured parity.
+
---
## 2. Per-Linear Block Rotation
@@ -248,6 +312,7 @@ Application modes (`hadamard/apply.py`):
| `hadamard_type` | `"hadamard"` | One of `hadamard`, `random_hadamard`, `inplace_quarot_hadamard`, `inplace_hadamard`, `inplace_random`. Deterministic Hadamard uses Sylvester construction (`block_size` must be a power of 2); `random_hadamard` supports non-power-of-2 sizes from the known-matrix library. |
| `fuse_online_to_weight` | `None` | Fuse online Hadamard rotation into weights when supported (`inplace` backend only). |
| `allow_online_rotation` | `True` | Allow online activation rotation. |
+| `layerwise` | `False` | Apply the Hadamard rotation **per decoder block**, in lock-step with AutoRound's block-wise quantization (see §1.8). Supported by the per-Linear `transform` backend (MXFP4 / NVFP4); the `inplace` backend couples layers and falls back to full-model rotation. |
### 2.2 Usage
diff --git a/test/unit/common/algorithms/test_rotation.py b/test/unit/common/algorithms/test_rotation.py
index 472f88648..4f8b02016 100644
--- a/test/unit/common/algorithms/test_rotation.py
+++ b/test/unit/common/algorithms/test_rotation.py
@@ -923,7 +923,11 @@ class TestRandomHadamardCache:
m1 = get_or_create_random_hadamard(8)
clear_random_hadamard_cache()
m2 = get_or_create_random_hadamard(8)
- assert not torch.equal(m1, m2)
+ # Clearing the cache must force a freshly-built matrix object. We assert
+ # object identity rather than value inequality: a "random" Hadamard only
+ # randomizes ``dim`` sign bits (H @ diag(±1)), so for dim=8 two draws
+ # collide with probability 1/256, which would flake a value-based check.
+ assert m1 is not m2
def test_device_transfer(self):
clear_random_hadamard_cache()
diff --git a/test/unit/common/core/test_pipeline_fast.py b/test/unit/common/core/test_pipeline_fast.py
index ae77014e1..d31ef8ef1 100644
--- a/test/unit/common/core/test_pipeline_fast.py
+++ b/test/unit/common/core/test_pipeline_fast.py
@@ -81,7 +81,9 @@ def test_block_quantizer_is_terminal_regardless_of_config_order(configs):
members = pipeline.members()
- assert members[:-1] == pipeline.preprocessors
+ # Preprocessors always lead the pipeline; rotation members (added by
+ # RotationConfig) may sit between them and the terminal block quantizer.
+ assert members[: len(pipeline.preprocessors)] == pipeline.preprocessors
assert members[-1] is pipeline.block_quantizer
assert isinstance(members[-1], RTNQuantizer)