PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add blockwise rotation. (#2087)
lkk9b746ff95859
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 26bdffd6d..4ea01e8d8 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -122,10 +122,20 @@ class AlgorithmComposer:"""from auto_round.algorithms.quantization.base import BaseQuantizerfrom auto_round.algorithms.quantization.config import QuantizationConfig- from auto_round.algorithms.transforms.base import BasePreprocessor+ from auto_round.algorithms.transforms.base import BasePreprocessor, BaseRotationConfigconfigs = list(configs)+ # Rotation configs travel in the same config list but are not block+ # quantizers / preprocessors (they are ``BaseRotationConfig``, not+ # ``QuantizationConfig``). Capture them here and wrap each in a+ # ``RotationPreprocessor`` member below so the orchestrator stays+ # rotation-agnostic and every rotation owns its own lifecycle.+ #+ # Whether rotation runs per-block (layer-wise) is a field on each+ # rotation config (``BaseRotationConfig.layerwise``).+ self._rotation_configs = [c for c in configs if isinstance(c, BaseRotationConfig)]+_, block_quantizer_configs = split_quantization_configs(configs)if not block_quantizer_configs:from auto_round.algorithms.quantization.rtn.config import RTNConfig@@ -212,6 +222,19 @@ class AlgorithmComposer:self.block_quantizer.bind_block_forward_runner(self.block_forward)self.scheme = getattr(orchestrator, "scheme_context", None)+ # Rotation is modelled as ordinary pipeline members (see+ # ``RotationPreprocessor``) that own their entire lifecycle. They are+ # built from the rotation configs captured above and bound like every+ # other member so ``finalize_run`` can reach the live model.+ from auto_round.algorithms.transforms.member import RotationPreprocessor++ self._rotation_members = [RotationPreprocessor(cfg) for cfg in self._rotation_configs]+ if orchestrator is not None:+ for rotation_member in self._rotation_members:+ rotation_member.bind(orchestrator)+ # Guards the one-shot model-level rotation stage (apply_model_transforms).+ self._rotation_prepared: bool = False+# ── Internal hook helpers (act_max calibration) ───────────────────────────def _register_act_max_hooks(self, block: "torch.nn.Module") -> list:@@ -353,6 +376,13 @@ class AlgorithmComposer:"""block_forward_fn = self.block_forward+ # ── Step 0: Layer-wise rotation (before any reference/calibration) ────+ # Each rotation member rotates this block's weights and installs online+ # hooks so all downstream calibration and reference collection operate on+ # the rotated block. No-op unless a member prepared layer-wise rotation.+ for rotation_member in self._rotation_members:+ rotation_member.on_block_ready(block, block_ctx)+# ── Step 1: Preprocessor calibration (e.g. AWQ activation stats) ──────with torch.no_grad():pre_hooks = []@@ -504,13 +534,15 @@ class AlgorithmComposer:# ── Convenience act-calib helpers ────────────────────────────────────────def members(self) -> list:- """Return the canonical execution order: preprocessors, then block quantizer.+ """Return the canonical execution order: preprocessors, rotations, then block quantizer.Preprocessor order follows the input config list. The block quantizeris always returned last, so its position in that input list has no- effect on pipeline execution.+ effect on pipeline execution. Rotation members sit between the+ preprocessors and the block quantizer so ``prepare_run`` / ``finalize_run``+ cover them like any other member."""- return list(self.preprocessors) + [self.block_quantizer]+ return list(self.preprocessors) + list(self._rotation_members) + [self.block_quantizer]def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):"""Dispatch block to device(s) via the pipeline's algorithms.@@ -545,5 +577,58 @@ class AlgorithmComposer:alg.prepare_run(composer=self)def finalize_run(self):+ # Rotation members are part of ``members()`` and finalize themselves+ # (layer-wise teardown lives in ``RotationPreprocessor.finalize_run``).for alg in self.members():alg.finalize_run()++ # ------------------------------------------------------------------+ # Rotation lifecycle (delegated to rotation members)+ # ------------------------------------------------------------------+ #+ # Rotation is a model-level pre-quantisation transform. Full-model rotation+ # must run *before* calibration data is cached, which is earlier than the+ # per-member ``prepare_run`` stage; layer-wise rotation instead prepares its+ # matrices here and rotates each block from within ``compress_block``. All+ # algorithm-specific decisions live in the rotation members; the composer+ # only drives the single generic entry point :meth:`apply_model_transforms`.++ def _resolve_rotation_data_type(self) -> str:+ """Best-effort resolution of the quantization data_type for rotation dispatch."""+ if self.scheme is not None and getattr(self.scheme, "data_type", None):+ return self.scheme.data_type+ if self.block_quantizer is not None:+ return getattr(self.block_quantizer.config, "data_type", "mx_fp")+ return "mx_fp"++ def apply_model_transforms(self, model: "torch.nn.Module") -> "torch.nn.Module":+ """Apply model-level pre-quantisation transforms (rotation) to *model*.++ Generic entry point invoked once by the orchestrator before calibration+ caching / the block loop. Each rotation member either rotates the whole+ model immediately (full-model mode) or prepares its rotation matrices and+ defers per-block work to :meth:`compress_block` (layer-wise mode).+ Idempotent — repeated calls are a no-op.++ Returns:+ The (possibly mutated) model.+ """+ if self._rotation_prepared:+ return model+ if not self._rotation_members:+ self._rotation_prepared = True+ return model++ data_type = self._resolve_rotation_data_type()+ logger.info("Applying Hadamard transform to the model.")+ for rotation_member in self._rotation_members:+ # The member honours its own config (``layerwise`` field).+ model = rotation_member.rotate_model(model, data_type=data_type)++ self._rotation_prepared = True+ return model++ @property+ def has_layerwise_rotation(self) -> bool:+ """Whether any rotation member is driving per-block (layer-wise) rotation."""+ return any(rotation_member.is_layerwise_active for rotation_member in self._rotation_members)diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.pyindex 46fcbb370..d83935c55 100644--- a/auto_round/algorithms/transforms/base.py+++ b/auto_round/algorithms/transforms/base.py@@ -67,6 +67,13 @@ class BaseRotationConfig:#: Human-readable algorithm name, must be unique across all subclasses.algorithm: str = "base"+ #: Apply rotation per decoder block (layer-wise), in lock-step with+ #: AutoRound's block-wise quantization, instead of rotating the whole model+ #: up-front. Only honoured by rotation algorithms whose+ #: :attr:`BaseRotation.supports_layerwise` is ``True`` (SpinQuant/QuaRot);+ #: unsupported algorithms transparently fall back to full-model rotation.+ layerwise: bool = False+def can_compile_block_forward(self) -> bool:"""Whether block replay can be compiled after applying this rotation."""return True@@ -117,6 +124,89 @@ class BaseRotation(ABC):The transformed model."""+ # ------------------------------------------------------------------+ # Layer-wise rotation interface (optional)+ # ------------------------------------------------------------------++ @property+ def supports_layerwise(self) -> bool:+ """Whether this rotation algorithm supports layer-wise execution.++ When ``True``, the composer can call :meth:`prepare_layerwise`+ during ``apply_model_transforms`` and then :meth:`rotate_layer`+ per-block inside the block-quantization loop — avoiding the need to+ load the entire model onto GPU at once.+ """+ return False++ def prepare_layerwise(+ self,+ model: torch.nn.Module,+ data_type: str = "mx_fp",+ **kwargs: Any,+ ) -> "BaseRotation":+ """Prepare for layer-wise rotation without modifying model weights.++ Called once when the rotation config has ``layerwise=True``.+ Implementations should initialise rotation matrices (as model+ buffers) and any other lightweight state, but must **not** modify+ model weights or register hooks yet.++ Args:+ model: The model to prepare (not modified).+ data_type: Quantization data type.+ **kwargs: Algorithm-specific arguments (e.g. ``dataloader``).++ Returns:+ ``self``, so the composer can later call :meth:`rotate_layer`.++ Raises:+ NotImplementedError: If the algorithm does not support+ layer-wise rotation.+ """+ raise NotImplementedError(+ f"{self.__class__.__name__} does not support layer-wise rotation. " f"Use full-model rotation instead."+ )++ def rotate_layer(+ self,+ layer: torch.nn.Module,+ layer_idx: int,+ **kwargs: Any,+ ) -> None:+ """Apply rotation to a single decoder layer.++ Called per-block in the block-quantization loop, after the block+ is materialised and placed on the target device, **before**+ reference-output collection.++ Implementations should:+ - Rotate weights of target modules inside *layer*.+ - Register any online hooks (R1/R3/R4) needed at inference time.+ - Be idempotent — calling twice on the same layer should be safe.++ Args:+ layer: A single decoder layer, already on the target device.+ layer_idx: Zero-based index of this layer in the model.+ **kwargs: Algorithm-specific arguments.++ Raises:+ NotImplementedError: If the algorithm does not support+ layer-wise rotation.+ """+ raise NotImplementedError(f"{self.__class__.__name__} does not support layer-wise rotation.")++ def finalize_layerwise(self, model: torch.nn.Module) -> None:+ """Post-loop cleanup after all layers have been rotated.++ Called once after the block-quantization loop completes.+ Default implementation is a no-op.++ Args:+ model: The fully-rotated model.+ """+ pass+# ------------------------------------------------------------------# Factory# ------------------------------------------------------------------diff --git a/auto_round/algorithms/transforms/hadamard/apply.py b/auto_round/algorithms/transforms/hadamard/apply.pyindex 73706083d..9e7d03570 100644--- a/auto_round/algorithms/transforms/hadamard/apply.py+++ b/auto_round/algorithms/transforms/hadamard/apply.py@@ -162,6 +162,84 @@ class HadamardRotation(BaseRotation, SerializerMixin):setattr(model, "_rotation_config", cfg)return model+ # ------------------------------------------------------------------+ # Layer-wise (block-wise) rotation interface+ # ------------------------------------------------------------------+ #+ # The per-Linear ``transform`` backend fuses an independent Hadamard into+ # each Linear's weight/input; there is no cross-layer (residual-stream)+ # coupling, so applying it one decoder layer at a time is mathematically+ # identical to the full-model pass. That makes block-wise rotation a trivial+ # scoping of :meth:`apply_to_model` down to a single layer's sub-modules.+ #+ # The ``inplace`` / QuaRot backend rotates the shared residual stream and+ # therefore *couples* consecutive layers; it cannot be decomposed per-block,+ # so :attr:`supports_layerwise` reports ``False`` for it (clean fall back to+ # full-model rotation) and :meth:`prepare_layerwise` refuses it explicitly.++ @property+ def supports_layerwise(self) -> bool:+ """Hadamard supports layer-wise rotation for the per-Linear backend.++ The QuaRot ``inplace`` backend couples layers through the residual+ stream and cannot run per-block, so it reports ``False`` (the composer+ then falls back to full-model rotation).+ """+ cfg = self.config+ backend = getattr(cfg, "backend", "auto")+ hadamard_type = getattr(cfg, "hadamard_type", "") or ""+ if backend == "inplace" or "inplace" in hadamard_type:+ return False+ return True++ def prepare_layerwise(+ self,+ model: torch.nn.Module,+ data_type: str = "mx_fp",+ location: str = "weight",+ **kwargs: Any,+ ) -> "HadamardRotation":+ """Prepare for per-block Hadamard rotation without touching weights.++ The per-Linear Hadamard needs no global pre-computation — matrices are+ built lazily inside :meth:`rotate_layer`. This only validates that the+ resolved backend is the per-Linear ``transform`` path and records the+ config on *model* (so serialization still works, since the model-level+ :meth:`apply_to_model` — which normally sets ``_rotation_config`` — is+ skipped in layer-wise mode).+ """+ from auto_round.algorithms.transforms.hadamard.dispatcher import resolve_hadamard_backend++ backend = resolve_hadamard_backend(self.config, data_type)+ if backend != "transform":+ raise NotImplementedError(+ "Layer-wise Hadamard rotation only supports the per-Linear 'transform' backend "+ f"(MXFP4 / NVFP4). Resolved backend={backend!r} for data_type={data_type!r}. "+ "Use full-model rotation (layerwise=False), or the inplace/QuaRot rotation path."+ )+ self._layerwise_location = location+ self._layerwise_data_type = data_type+ setattr(model, "_rotation_config", self.config)+ return self++ def rotate_layer(+ self,+ layer: torch.nn.Module,+ layer_idx: int,+ **kwargs: Any,+ ) -> None:+ """Apply the per-Linear Hadamard to every target module inside *layer*."""+ cfg = self.config+ location = getattr(self, "_layerwise_location", "weight")+ data_type = getattr(self, "_layerwise_data_type", "mx_fp")+ target_types = (torch.nn.Linear, QModuleBase)+ for name, module in layer.named_modules():+ if not isinstance(module, target_types):+ continue+ if "lm_head" in name:+ continue+ _apply_to_module(layer, module, cfg, location, data_type)+# ------------------------------------------------------------------# SerializerMixin — Save side# ------------------------------------------------------------------diff --git a/auto_round/algorithms/transforms/hadamard/config.py b/auto_round/algorithms/transforms/hadamard/config.pyindex ed72ad1c4..f2f2bf85a 100644--- a/auto_round/algorithms/transforms/hadamard/config.py+++ b/auto_round/algorithms/transforms/hadamard/config.py@@ -77,6 +77,11 @@ class RotationConfig(AlgorithmConfig, BaseModel, BaseRotationConfig):backend: str = Field(default="auto")block_size: Optional[int] = Field(default=None)hadamard_type: str = Field(default="hadamard")+ # Apply the Hadamard rotation per decoder block, in lock-step with block-wise+ # quantization, instead of rotating the whole model up-front. Honoured by the+ # per-Linear transform backend; unsupported cases fall back to full-model+ # rotation (see ``RotationPreprocessor.rotate_model``).+ layerwise: bool = Field(default=False)# ---- inplace-only ----fuse_online_to_weight: Optional[bool] = Field(default=None)diff --git a/auto_round/algorithms/transforms/member.py b/auto_round/algorithms/transforms/member.pynew file mode 100644index 000000000..448e98ff3--- /dev/null+++ b/auto_round/algorithms/transforms/member.py@@ -0,0 +1,178 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Rotation as a first-class pipeline member.++``RotationPreprocessor`` wraps a :class:`~auto_round.algorithms.transforms.base.BaseRotation`+and plugs it into the ordinary :class:`~auto_round.algorithms.composer.AlgorithmComposer`+member lifecycle. It owns the *entire* rotation lifecycle so the composer stays+rotation-agnostic:++* :meth:`rotate_model` – model-level (Phase 4.5) full-model rotation, or+ layer-wise preparation (R matrices only).+* :meth:`on_block_ready` – per-block rotation for layer-wise mode, run as the+ first step of ``compress_block``. Handles ``nblocks``+ block fusion (``WrapperMultiblock``) so each fused+ decoder layer receives the correct ``layer_idx``.+* :meth:`finalize_run` – layer-wise teardown, invoked automatically via the+ composer's ``members()`` finalize loop.++The composer only ever calls these generic entry points in a thin loop; all+algorithm-specific decisions (does this rotation support layer-wise? which+sub-modules to rotate?) live here / in the concrete ``BaseRotation``.+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++from auto_round.algorithms.transforms.base import BasePreprocessor+from auto_round.utils import logger++if TYPE_CHECKING: # pragma: no cover+ import torch++ from auto_round.algorithms.composer import BlockContext+ from auto_round.algorithms.transforms.base import BaseRotation+++class RotationPreprocessor(BasePreprocessor):+ """Pipeline member that owns a single rotation transform's lifecycle.++ Instances are built by the composer from the rotation configs found in the+ algorithm config list and are added both to ``members()`` (so ``prepare_run``+ / ``finalize_run`` cover them) and to the composer's dedicated rotation list+ (so the model-level and per-block entry points can reach them).+ """++ def __init__(self, config: Any) -> None:+ super().__init__(config)+ self._rotation: "BaseRotation | None" = None+ # Set once :meth:`rotate_model` decides layer-wise preparation succeeded.+ self._layerwise_active: bool = False++ # ------------------------------------------------------------------+ # Lazy rotation construction+ # ------------------------------------------------------------------+ @property+ def rotation(self) -> "BaseRotation":+ """The concrete :class:`BaseRotation` for this member (built lazily)."""+ if self._rotation is None:+ from auto_round.algorithms.transforms import normalize_rotation_config+ from auto_round.algorithms.transforms.base import BaseRotation++ normalised = normalize_rotation_config(self.config)+ if normalised is None:+ raise ValueError(f"Rotation member received an empty config: {self.config!r}.")+ self._rotation = BaseRotation.from_config(normalised)+ return self._rotation++ @property+ def is_layerwise(self) -> bool:+ """Whether the config *requests* per-block rotation (intent only; see+ :attr:`is_layerwise_active` for whether it actually took effect)."""+ return bool(getattr(self.config, "layerwise", False))++ @property+ def is_layerwise_active(self) -> bool:+ """Whether this member prepared (and will drive) per-block rotation."""+ return self._layerwise_active++ # ------------------------------------------------------------------+ # Model-level entry (Phase 4.5)+ # ------------------------------------------------------------------+ def rotate_model(+ self,+ model: "torch.nn.Module",+ data_type: str = "mx_fp",+ ) -> "torch.nn.Module":+ """Rotate *model* up-front, or prepare layer-wise rotation matrices.++ Whether to rotate per-block is taken solely from the rotation config's+ ``layerwise`` field (via :attr:`is_layerwise`).++ For full-model rotation the model is rotated immediately and returned.+ For layer-wise rotation (when the underlying algorithm supports it) only+ the rotation matrices are initialised; per-block work is deferred to+ :meth:`on_block_ready`. When layer-wise is requested but unsupported, the+ member transparently falls back to full-model rotation.++ Returns:+ The (possibly mutated) model.+ """+ rotation = self.rotation+ if self.is_layerwise:+ if rotation.supports_layerwise:+ logger.info(+ "[Rotation] Layer-wise mode: preparing R matrices only " "(rotation deferred to per-block hook)."+ )+ rotation.prepare_layerwise(model, data_type=data_type)+ self._layerwise_active = True+ return model+ logger.warning(+ f"[Rotation] {rotation.__class__.__name__} does not support "+ "layer-wise mode. Falling back to full-model rotation."+ )++ return rotation.apply_to_model(model, data_type=data_type)++ # ------------------------------------------------------------------+ # Per-block entry (compress_block step 0)+ # ------------------------------------------------------------------+ def on_block_ready(self, block: "torch.nn.Module", ctx: "BlockContext") -> None:+ """Rotate the block about to be quantized (layer-wise mode only).++ No-op unless :meth:`rotate_model` prepared layer-wise rotation. Iterates+ the (possibly fused) decoder layers inside *block* and rotates each with+ its global ``layer_idx``.+ """+ if not self._layerwise_active:+ return++ for layer, layer_idx in self._iter_layers(block, ctx.block_index):+ self.rotation.rotate_layer(layer, layer_idx=layer_idx)++ # ------------------------------------------------------------------+ # Teardown (members() finalize loop)+ # ------------------------------------------------------------------+ def finalize_run(self) -> None:+ """Finalize layer-wise rotation once all blocks have been processed."""+ if not self._layerwise_active:+ return+ model = self.model_context.model if self.model_context is not None else None+ if model is None:+ return+ self.rotation.finalize_layerwise(model)++ # ------------------------------------------------------------------+ # Helpers+ # ------------------------------------------------------------------+ @staticmethod+ def _iter_layers(block: "torch.nn.Module", block_index: int):+ """Yield ``(decoder_layer, layer_idx)`` for every layer inside *block*.++ ``block_index`` is the global index of the *first* layer in this block.+ When ``nblocks > 1`` several consecutive decoder layers are fused into a+ single :class:`~auto_round.wrapper.WrapperMultiblock`; each fused layer+ ``j`` maps to global index ``block_index + j``. A plain (non-fused) block+ maps directly to ``block_index``.+ """+ from auto_round.wrapper import WrapperMultiblock++ if isinstance(block, WrapperMultiblock):+ sub_layers = list(block.layers)+ else:+ sub_layers = [block]++ for j, layer in enumerate(sub_layers):+ yield layer, block_index + jdiff --git a/auto_round/algorithms/transforms/spinquant/apply.py b/auto_round/algorithms/transforms/spinquant/apply.pyindex 108820bb5..f3c0e0c22 100644--- a/auto_round/algorithms/transforms/spinquant/apply.py+++ b/auto_round/algorithms/transforms/spinquant/apply.py@@ -106,6 +106,67 @@ class SpinQuantRotation(BaseRotation, SerializerMixin):preprocessor = SpinQuantPreprocessor(model, self.config)return preprocessor.preprocess(dataloader)+ # ------------------------------------------------------------------+ # Layer-wise rotation interface (block-wise quantization)+ # ------------------------------------------------------------------++ @property+ def supports_layerwise(self) -> bool:+ """SpinQuant supports layer-wise rotation (Online R1 + R2/R3/R4)."""+ return True++ def prepare_layerwise(+ self,+ model: torch.nn.Module,+ data_type: str = "mx_fp",+ **kwargs: Any,+ ) -> "SpinQuantRotation":+ """Prepare for layer-wise rotation: init R matrices only.++ Creates a :class:`SpinQuantPreprocessor`, calls its+ :meth:`~SpinQuantPreprocessor.prepare` method (which initialises+ rotation matrices without modifying weights), and stores the+ preprocessor for later :meth:`rotate_layer` calls.++ Args:+ model: The model to prepare (weights not modified).+ data_type: Quantization data type (informational).+ **kwargs: ``dataloader`` forwarded when ``trainable_rotation=True``.++ Returns:+ ``self``, for chaining.+ """+ from auto_round.algorithms.transforms.spinquant.preprocessor import (+ SpinQuantPreprocessor,+ )++ dataloader = kwargs.get("dataloader")+ self._preprocessor = SpinQuantPreprocessor(model, self.config)+ self._preprocessor.prepare(dataloader)+ return self++ def rotate_layer(+ self,+ layer: torch.nn.Module,+ layer_idx: int,+ **kwargs: Any,+ ) -> None:+ """Apply rotation to a single decoder layer.++ Delegates to :meth:`SpinQuantPreprocessor.rotate_layer`.+ """+ if not hasattr(self, "_preprocessor"):+ raise RuntimeError(+ "prepare_layerwise() must be called before rotate_layer(). "+ "The preprocessor has not been initialised."+ )+ self._preprocessor.rotate_layer(layer, layer_idx)++ def finalize_layerwise(self, model: torch.nn.Module) -> None:+ """Cleanup after all layers have been rotated."""+ if hasattr(self, "_preprocessor"):+ self._preprocessor.finalize()+# ------------------------------------------------------------------# SerializerMixin — Save side# ------------------------------------------------------------------diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex f13df42be..0c41271b6 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -333,6 +333,271 @@ class SpinQuantPreprocessor:logger.info("[SpinQuant] Preprocessing complete!")return self.model+ # ------------------------------------------------------------------+ # Layer-wise API (for block-lifecycle / block-wise quantization)+ # ------------------------------------------------------------------++ def prepare(self, dataloader: Optional[Any] = None) -> None:+ """Global preparation for layer-wise (block-wise) rotation.++ Performs all lightweight, non-destructive steps:+ - Validates dimensions+ - Initialises rotation matrices (stored as model buffers)+ - Optionally trains rotation matrices (requires full model + dataloader)+ - Pre-computes rotation state shared across layers++ Does **NOT** modify model weights or register hooks — that is done+ per-layer by :meth:`rotate_layer`.++ This method is the layer-wise counterpart of :meth:`preprocess`.++ Args:+ dataloader: Required only when ``trainable_rotation=True``.++ Raises:+ ValueError: If ``online_r1_rotation=False`` while R1 is enabled+ (offline R1 is incompatible with layer-wise mode because it+ rotates the inter-layer hidden-state space via embed_tokens /+ lm_head, which breaks pre-cached block inputs).+ """+ # Layer-wise rotation only works with Online R1 (offline R1 rotates the+ # shared residual stream, which is incompatible with pre-cached block+ # inputs).+ if self.config.r1 and not self.config.online_r1_rotation:+ raise ValueError(+ "[SpinQuant] Layer-wise rotation requires online_r1_rotation=True. "+ "Offline R1 changes inter-layer hidden state space, which is "+ "incompatible with pre-cached block inputs. Use full-model "+ "rotation (config `layerwise=False`) for offline R1."+ )++ logger.info("[SpinQuant] Preparing for layer-wise rotation...")+ logger.info(+ f"[SpinQuant] Model architecture: hidden_size={self.hidden_size}, "+ f"head_dim={self.head_dim}, intermediate_size={self.intermediate_size}"+ )+ logger.info(+ f"[SpinQuant] Rotation config: R1={self.config.r1}, R2={self.config.r2}, "+ f"R3={self.config.r3}, R4={self.config.r4}, "+ f"online_r1={self.config.online_r1_rotation}"+ )++ # Step 1: Validate dimensions+ self._validate_dimensions()++ # Step 2: Initialise rotation matrices (lightweight — only R matrix buffers)+ logger.info("[SpinQuant] Initialising rotation matrices...")+ self._init_rotation_matrices()++ # Step 3: Train if requested (needs full model forward/backward)+ if self.config.trainable_rotation or self.config.trainable_smooth:+ if dataloader is None:+ raise ValueError("dataloader required when trainable=True")+ logger.warning(+ "[SpinQuant] Layer-wise mode with trainable_rotation: training "+ "requires a full model forward/backward pass (higher memory). "+ "Consider pre-training R matrices separately for large models."+ )+ # Trainable smooth needs norm replacement first+ if self.config.trainable_smooth:+ self._replace_norms_with_trainable()+ self._train_rotations(dataloader)++ # Pre-compute shared state consumed by rotate_layer()+ if self.config.r1 and self.config.online_r1_rotation:+ self._r1_shared = self._compute_online_r1_shared()+ else:+ self._r1_shared = None++ if self.config.r2 and self.head_dim > 0:+ R2_head = self._get_rotation_tensor("spinquant_R2_head")+ if R2_head is not None:+ R2 = R2_head.data.to(torch.float64)+ self._r2_shared = (R2, R2.t())+ else:+ self._r2_shared = None+ else:+ self._r2_shared = None++ if self.config.r4 and self.r4_rotation_size > 0:+ use_random_r4 = self.config.random_r4+ R4 = None+ if use_random_r4:+ R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)+ if R4_matrix is None:+ raise RuntimeError("[SpinQuant] random_r4=True but spinquant_R4_matrix buffer not found.")+ R4 = R4_matrix.to(torch.float64)+ self._r4_shared = (self.r4_rotation_size, use_random_r4, R4)+ else:+ self._r4_shared = None++ # Store config on model for downstream serialization+ self.model._rotation_config = self.config+ self.model._spinquant_config = self.config # legacy alias++ logger.info("[SpinQuant] Layer-wise preparation complete (no weights modified yet).")++ def rotate_layer(self, layer: nn.Module, layer_idx: int) -> None:+ """Apply all configured rotations to a single decoder layer.++ This is the per-block entry point called by the compressor inside the+ block-wise quantization loop. The layer must already be materialised+ and on the target device.++ Operations performed (in order):+ 1. **R1 (Online)**: rotate target-module weights + register hooks+ 2. **R2 (Offline fuse)**: fuse into v_proj output + o_proj input+ 3. **R4 (Offline fuse)**: fuse into down_proj input+ 4. **R3 (Hook)**: monkeypatch apply_rotary_pos_emb+ 5. **R4 (Hook)**: register forward_pre_hook on down_proj++ Args:+ layer: A single decoder layer, already on the target device.+ layer_idx: Zero-based index for logging purposes.+ """+ if not hasattr(self, "_r1_shared"):+ raise RuntimeError(+ "[SpinQuant] prepare() must be called before rotate_layer(). "+ "The layer-wise state has not been initialised."+ )+ if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):+ logger.warning(f"[SpinQuant] Layer {layer_idx}: no self_attn/mlp, skipping rotation.")+ return++ layer_device = next(layer.parameters()).device++ # ── R1: Online weight rotation + hooks ──+ if self._r1_shared is not None:+ self._apply_online_r1_to_layer(layer, *self._r1_shared)++ # ── R2: Offline fuse into v_proj/o_proj ──+ if self._r2_shared is not None:+ self._fuse_r2_to_layer(layer, *self._r2_shared)++ # ── R4: Offline fuse into down_proj ──+ if self._r4_shared is not None:+ self._fuse_r4_to_layer(layer, *self._r4_shared)++ # ── R3: Monkeypatch attention (after RoPE) ──+ if self.config.r3 and self.head_dim > 0:+ self._rotate_layer_r3_hook(layer, layer_idx)++ # ── R4: Hook on down_proj ──+ if self.config.r4 and self.r4_rotation_size > 0:+ self._rotate_layer_r4_hook(layer, layer_idx, layer_device)++ logger.debug(f"[SpinQuant] Layer {layer_idx}: rotation applied.")++ def _rotate_layer_r3_hook(self, layer: nn.Module, layer_idx: int) -> None:+ """Apply the R3 monkeypatch to a single layer's attention module.++ Consistent with the model-level ``register_spinquant_hooks`` R3 path.+ """+ if not is_pow2(self.head_dim):+ return++ from auto_round.algorithms.transforms.spinquant.monkeypatch import (+ add_qk_rotation_after_rope,+ )++ attn = layer.self_attn+ if not (hasattr(attn, "q_proj") and hasattr(attn, "k_proj")):+ return+ if getattr(attn, "_spinquant_r3_patched", False):+ return # Already patched (idempotent)++ random_r3 = self.config.random_r3+ r3_matrix = getattr(self.model, "spinquant_R3_head", None)++ try:+ wrapper = add_qk_rotation_after_rope(attn, rope_function_name="apply_rotary_pos_emb")+ if random_r3 and r3_matrix is not None:+ wrapper.set_matrix(r3_matrix)+ else:+ wrapper.set_hadamard(None, self.head_dim)+ attn._spinquant_r3_patched = True+ self._hook_handles.append(("r3_monkeypatch", f"layer_{layer_idx}", attn, wrapper))+ except ValueError as e:+ logger.warning(f"[SpinQuant] R3 monkeypatch failed for layer {layer_idx}: {e}")++ def _rotate_layer_r4_hook(self, layer: nn.Module, layer_idx: int, layer_device: torch.device) -> None:+ """Register the R4 ``forward_pre_hook`` on a single layer's down_proj.++ Consistent with the model-level ``register_spinquant_hooks`` R4 path.+ """+ mlp = layer.mlp+ if not hasattr(mlp, "down_proj"):+ return++ r4_size = self.r4_rotation_size+ use_random = self.config.random_r4+ need_block = r4_size < self.intermediate_size+ module = mlp.down_proj++ if use_random:+ R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)+ if R4_matrix is None:+ return+ R4 = R4_matrix.to(device=layer_device, dtype=torch.float32)++ def _make_hook(R, rot_size, block_mode):+ def hook(mod, args):+ x = args[0]+ R_local = R.to(x.device, dtype=x.dtype)+ if block_mode:+ shape = x.shape+ x = x.reshape(*shape[:-1], -1, rot_size)+ x = (x @ R_local).reshape(shape)+ else:+ x = x @ R_local+ return (x,) + args[1:]++ return hook++ hook = _make_hook(R4, r4_size, need_block)+ else:+ try:+ had_K_mat, had_K_val = get_hadamard_K(r4_size)+ except ValueError:+ return+ had_K_mat = had_K_mat.to(device=layer_device, dtype=torch.float32)++ def _make_hook_butterfly(had_mat, k_val, rot_size, block_mode):+ def hook(mod, args):+ x = args[0]+ if block_mode:+ shape = x.shape+ x = x.reshape(*shape[:-1], -1, rot_size)+ x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)+ x = x.reshape(shape)+ else:+ x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)+ return (x,) + args[1:]++ return hook++ hook = _make_hook_butterfly(had_K_mat, had_K_val, r4_size, need_block)++ hook._spinquant_hook = True+ handle = module.register_forward_pre_hook(hook)+ self._hook_handles.append(handle)++ def finalize(self) -> None:+ """Post-loop cleanup after all layers have been rotated layer-wise.++ Mirrors :meth:`_cleanup` but for the layer-wise path.+ """+ self.model.eval()+ for p in self.model.parameters():+ p.requires_grad = False+ self.rotation_params.clear()+ self.smooth_params.clear()+ self._rotated_modules.clear()++ n_r1 = len(self._r1_hook_handles)+ n_other = len(self._hook_handles)+ logger.info(f"[SpinQuant] Layer-wise rotation finalized. " f"Active hooks: {n_r1} R1 + {n_other} R3/R4.")+def _validate_dimensions(self) -> None:"""Validate dimension requirements and disable rotations that can't work."""# R1: check r1_rotation_size divides hidden_size and is power of 2@@ -694,6 +959,34 @@ class SpinQuantPreprocessor:"need to save/reload the model.")+ r1_size, use_random, R1_full, hadamard_K, K = self._compute_online_r1_shared()++ n_rotated = 0+ n_hooked = 0++ for layer in self._get_layers():+ nr, nh = self._apply_online_r1_to_layer(layer, r1_size, use_random, R1_full, hadamard_K, K)+ n_rotated += nr+ n_hooked += nh++ mode_str = "random matrix (x @ R)" if use_random else "deterministic butterfly"+ logger.info(+ f"[SpinQuant] Online R1: rotated {n_rotated} target modules, "+ f"registered {n_hooked} activation hooks "+ f"(rotation_size={r1_size}, mode={mode_str}, "+ f"lm_head/embed_tokens/o_proj/down_proj unchanged)"+ )++ def _compute_online_r1_shared(self):+ """Compute R1 rotation state shared across all layers (online R1).++ Returns a tuple ``(r1_size, use_random, R1_full, hadamard_K, K)`` that+ can be passed to :meth:`_apply_online_r1_to_layer`. Extracted so that+ both full-model (:meth:`_apply_online_r1`) and layer-wise+ (:meth:`rotate_layer`) paths share identical R1 conventions.+ """+ r1_size = self.r1_rotation_size+ use_random = self.config.random_r1model_device = next(self.model.parameters()).device# For random R1: use the stored full matrix@@ -711,88 +1004,86 @@ class SpinQuantPreprocessor:R1_full = Nonehadamard_K, K = get_hadamard_K(r1_size)hadamard_K = hadamard_K.to(model_device)+ return r1_size, use_random, R1_full, hadamard_K, K- n_rotated = 0- n_hooked = 0+ def _apply_online_r1_to_layer(self, layer, r1_size, use_random, R1_full, hadamard_K, K):+ """Apply online R1 to a single decoder layer: rotate target-module+ weights and register the matching activation ``forward_pre_hook`` s.- for layer in self._get_layers():- if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):- continue+ Returns ``(n_rotated, n_hooked)``.+ """+ if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):+ return 0, 0- layer_device = next(layer.parameters()).device+ layer_device = next(layer.parameters()).device- attn = layer.self_attn- mlp = layer.mlp+ attn = layer.self_attn+ mlp = layer.mlp- # Target modules: (parent_module, attr_name)- target_specs = []- for proj_name in ("q_proj", "k_proj", "v_proj"):- if hasattr(attn, proj_name):- target_specs.append((attn, proj_name))- for proj_name in ("gate_proj", "up_proj"):- if hasattr(mlp, proj_name):- target_specs.append((mlp, proj_name))-- for parent, attr_name in target_specs:- module = getattr(parent, attr_name)- dtype = module.weight.data.dtype- in_features = module.weight.shape[-1]-- if use_random:- # Random R1: explicit matrix multiply- R = R1_full.to(layer_device)- if r1_size == in_features:- W = module.weight.data.to(torch.float64)- module.weight.data = (W @ R).to(dtype)- elif in_features % r1_size == 0:- # Block rotation must use W @ R (not W @ R.T) to stay- # consistent with the online hook (x @ R). The random- # Hadamard matrix is orthonormal but NOT symmetric, so- # rotate_in_channels_ (which applies R.T) would break- # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.- rotate_in_channels_(module, R_in=R.T)- else:- raise ValueError(- f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"- )+ # Target modules: (parent_module, attr_name)+ target_specs = []+ for proj_name in ("q_proj", "k_proj", "v_proj"):+ if hasattr(attn, proj_name):+ target_specs.append((attn, proj_name))+ for proj_name in ("gate_proj", "up_proj"):+ if hasattr(mlp, proj_name):+ target_specs.append((mlp, proj_name))++ n_rotated = 0+ n_hooked = 0+ for parent, attr_name in target_specs:+ module = getattr(parent, attr_name)+ dtype = module.weight.data.dtype+ in_features = module.weight.shape[-1]++ if use_random:+ # Random R1: explicit matrix multiply+ R = R1_full.to(layer_device)+ if r1_size == in_features:+ W = module.weight.data.to(torch.float64)+ module.weight.data = (W @ R).to(dtype)+ elif in_features % r1_size == 0:+ # Block rotation must use W @ R (not W @ R.T) to stay+ # consistent with the online hook (x @ R). The random+ # Hadamard matrix is orthonormal but NOT symmetric, so+ # rotate_in_channels_ (which applies R.T) would break+ # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.+ rotate_in_channels_(module, R_in=R.T)else:- # Deterministic Hadamard: butterfly algorithm- had_K_local = hadamard_K.to(layer_device)- if r1_size == in_features:- module.weight.data = matmul_hadU(module.weight.data, hadamard_K=had_K_local, K=K).to(dtype)- elif in_features % r1_size == 0:- R_block = had_K_local.to(torch.float64)- if R_block.shape[0] != r1_size:- had_1, _ = get_hadamard_K(r1_size // K)- R_block = torch.kron(- had_K_local.to(device="cpu", dtype=torch.float64),- had_1.to(device="cpu", dtype=torch.float64),- )- R_block = R_block / math.sqrt(r1_size)- rotate_in_channels_(module, R_in=R_block)- else:- raise ValueError(- f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"+ raise ValueError(+ f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"+ )+ else:+ # Deterministic Hadamard: butterfly algorithm+ had_K_local = hadamard_K.to(layer_device)+ if r1_size == in_features:+ module.weight.data = matmul_hadU(module.weight.data, hadamard_K=had_K_local, K=K).to(dtype)+ elif in_features % r1_size == 0:+ R_block = had_K_local.to(torch.float64)+ if R_block.shape[0] != r1_size:+ had_1, _ = get_hadamard_K(r1_size // K)+ R_block = torch.kron(+ had_K_local.to(device="cpu", dtype=torch.float64),+ had_1.to(device="cpu", dtype=torch.float64),)- n_rotated += 1-- # Register forward_pre_hook for online activation rotation- if use_random:- hook = self._make_online_r1_hook_matrix(R1_full.to(layer_device), r1_size, in_features)+ R_block = R_block / math.sqrt(r1_size)+ rotate_in_channels_(module, R_in=R_block)else:- hook = self._make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K.to(layer_device), K)- hook._spinquant_hook = True # tag for selective removal- handle = module.register_forward_pre_hook(hook)- self._r1_hook_handles.append(handle)- n_hooked += 1+ raise ValueError(+ f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"+ )+ n_rotated += 1- mode_str = "random matrix (x @ R)" if use_random else "deterministic butterfly"- logger.info(- f"[SpinQuant] Online R1: rotated {n_rotated} target modules, "- f"registered {n_hooked} activation hooks "- f"(rotation_size={r1_size}, mode={mode_str}, "- f"lm_head/embed_tokens/o_proj/down_proj unchanged)"- )+ # Register forward_pre_hook for online activation rotation+ if use_random:+ hook = self._make_online_r1_hook_matrix(R1_full.to(layer_device), r1_size, in_features)+ else:+ hook = self._make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K.to(layer_device), K)+ hook._spinquant_hook = True # tag for selective removal+ handle = module.register_forward_pre_hook(hook)+ self._r1_hook_handles.append(handle)+ n_hooked += 1+ return n_rotated, n_hooked@staticmethoddef _make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K, K):@@ -957,34 +1248,41 @@ class SpinQuantPreprocessor:n_fused = 0for layer in self._get_layers():- if not hasattr(layer, "self_attn"):- continue- attn = layer.self_attn+ n_fused += self._fuse_r2_to_layer(layer, R2, R2_T)- # v_proj: W_new = R2^T @ W per head on output dimension- if hasattr(attn, "v_proj"):- W = attn.v_proj.weight.data- dtype = W.dtype- W = W.to(torch.float64)- n_heads = W.shape[0] // self.head_dim- W_reshaped = W.reshape(n_heads, self.head_dim, W.shape[1])- W_reshaped = torch.einsum("ij,kjl->kil", R2_T, W_reshaped)- attn.v_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)-- # o_proj: W_new = W @ R2 per head on input dimension- # (R2^{-1} = R2^T on the activation side ↔ W @ R2 on the weight side)- if hasattr(attn, "o_proj"):- W = attn.o_proj.weight.data- dtype = W.dtype- W = W.to(torch.float64)- n_heads = W.shape[1] // self.head_dim- W_reshaped = W.reshape(W.shape[0], n_heads, self.head_dim)- W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R2)- attn.o_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)+ logger.info(f"[SpinQuant] R2 fused into {n_fused} layers (v_proj out + o_proj in, head_dim={self.head_dim})")- n_fused += 1+ def _fuse_r2_to_layer(self, layer, R2, R2_T) -> int:+ """Fuse R2 per-head rotation into a single layer's v_proj/o_proj.- logger.info(f"[SpinQuant] R2 fused into {n_fused} layers (v_proj out + o_proj in, head_dim={self.head_dim})")+ Returns 1 if the layer was fused, else 0.+ """+ if not hasattr(layer, "self_attn"):+ return 0+ attn = layer.self_attn++ # v_proj: W_new = R2^T @ W per head on output dimension+ if hasattr(attn, "v_proj"):+ W = attn.v_proj.weight.data+ dtype = W.dtype+ W = W.to(torch.float64)+ n_heads = W.shape[0] // self.head_dim+ W_reshaped = W.reshape(n_heads, self.head_dim, W.shape[1])+ W_reshaped = torch.einsum("ij,kjl->kil", R2_T.to(W.device), W_reshaped)+ attn.v_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)++ # o_proj: W_new = W @ R2 per head on input dimension+ # (R2^{-1} = R2^T on the activation side ↔ W @ R2 on the weight side)+ if hasattr(attn, "o_proj"):+ W = attn.o_proj.weight.data+ dtype = W.dtype+ W = W.to(torch.float64)+ n_heads = W.shape[1] // self.head_dim+ W_reshaped = W.reshape(W.shape[0], n_heads, self.head_dim)+ W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R2.to(W.device))+ attn.o_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)++ return 1def _fuse_r4_rotation(self) -> None:"""Fuse R4 rotation into down_proj's input side.@@ -1004,6 +1302,7 @@ class SpinQuantPreprocessor:use_random = self.config.random_r4# Get the rotation matrix+ R4 = Noneif use_random:R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)if R4_matrix is None:@@ -1012,38 +1311,7 @@ class SpinQuantPreprocessor:n_fused = 0for layer in self._get_layers():- if not hasattr(layer, "mlp"):- continue- mlp = layer.mlp- if hasattr(mlp, "down_proj"):- W = mlp.down_proj.weight.data- dtype = W.dtype-- if use_random:- # Random: explicit W @ R per block- W = W.to(torch.float64)- if r4_size == W.shape[1]:- mlp.down_proj.weight.data = (W @ R4).to(dtype)- else:- out_feat, in_feat = W.shape- n_blocks = in_feat // r4_size- W_reshaped = W.reshape(out_feat, n_blocks, r4_size)- W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R4)- mlp.down_proj.weight.data = W_reshaped.reshape(out_feat, in_feat).to(dtype)- else:- # Deterministic: matmul_hadU (butterfly algorithm)- # matmul_hadU operates on the last dimension — for weight- # shape [out, in], last dim = in_features which is what- # we want to rotate (input channels of down_proj).- if r4_size == W.shape[1]:- mlp.down_proj.weight.data = matmul_hadU(W).to(dtype)- else:- out_feat, in_feat = W.shape- n_blocks = in_feat // r4_size- W_reshaped = W.reshape(out_feat, n_blocks, r4_size)- W_rotated = matmul_hadU(W_reshaped)- mlp.down_proj.weight.data = W_rotated.reshape(out_feat, in_feat).to(dtype)- n_fused += 1+ n_fused += self._fuse_r4_to_layer(layer, r4_size, use_random, R4)mode_str = "random x @ R" if use_random else "deterministic butterfly"logger.info(@@ -1051,6 +1319,47 @@ class SpinQuantPreprocessor:f"(r4_rotation_size={r4_size}, mode={mode_str})")+ def _fuse_r4_to_layer(self, layer, r4_size, use_random, R4) -> int:+ """Fuse R4 rotation into a single layer's down_proj input side.++ Returns 1 if the layer was fused, else 0.+ """+ if not hasattr(layer, "mlp"):+ return 0+ mlp = layer.mlp+ if not hasattr(mlp, "down_proj"):+ return 0++ W = mlp.down_proj.weight.data+ dtype = W.dtype++ if use_random:+ # Random: explicit W @ R per block+ W = W.to(torch.float64)+ R4 = R4.to(W.device)+ if r4_size == W.shape[1]:+ mlp.down_proj.weight.data = (W @ R4).to(dtype)+ else:+ out_feat, in_feat = W.shape+ n_blocks = in_feat // r4_size+ W_reshaped = W.reshape(out_feat, n_blocks, r4_size)+ W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R4)+ mlp.down_proj.weight.data = W_reshaped.reshape(out_feat, in_feat).to(dtype)+ else:+ # Deterministic: matmul_hadU (butterfly algorithm)+ # matmul_hadU operates on the last dimension — for weight+ # shape [out, in], last dim = in_features which is what+ # we want to rotate (input channels of down_proj).+ if r4_size == W.shape[1]:+ mlp.down_proj.weight.data = matmul_hadU(W).to(dtype)+ else:+ out_feat, in_feat = W.shape+ n_blocks = in_feat // r4_size+ W_reshaped = W.reshape(out_feat, n_blocks, r4_size)+ W_rotated = matmul_hadU(W_reshaped)+ mlp.down_proj.weight.data = W_rotated.reshape(out_feat, in_feat).to(dtype)+ return 1+# ------------------------------------------------------------------# Step 8: Cleanup# ------------------------------------------------------------------diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex d5617e7aa..fc722ef53 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -24,7 +24,6 @@ from transformers import AutoConfig, set_seedfrom auto_round.algorithms.quantization import BaseQuantizer, QuantizationConfigfrom auto_round.algorithms.transforms import (BaseRotationConfig,- apply_rotation,)from auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.config_resolution import (@@ -1408,7 +1407,6 @@ class BaseOrchestrator(object):self._resolve_formats()self._patch_model()self._build_layer_config()- self._apply_rotations()# Reclaim temporaries from Phases 1-4 (scheme resolution, format# parsing, model patching, layer-config walk) before Phase 5@@ -1423,6 +1421,14 @@ class BaseOrchestrator(object):# so _build_composer must run first.self._build_composer()+ # Phase 4.5 – Model-level pre-quantisation transforms (rotation).+ # Applies full-model rotation up-front (or prepares layer-wise rotation+ # matrices). Runs here so every entry point — the full quantize() loop,+ # the zero-shot loop, and the external single-block quantize_block() API —+ # sees a consistently transformed model before any calibration data is+ # collected. Rotation is now owned by the composer's rotation members.+ self.model_context.model = self.alg_composer.apply_model_transforms(self.model_context.model)+# Set block_forward torch compile for block forward# Final trim after all init phases.gc.collect()@@ -1632,39 +1638,6 @@ class BaseOrchestrator(object):# Defer ShardWriter construction to _ensure_shard_writer() to avoid# heap fragmentation during post_init (parameter iteration).- def _apply_rotations(self) -> None:- """Phase 4.5 – Apply Hadamard / rotation transforms to the model.-- Preconditions:- - Phase 3 complete: model topology is final (``apply_patches`` has- replaced / merged layers, e.g. MoE experts), so rotation operates- on the same modules that quantization will later see.- - Phase 4 complete: ``self.layer_config`` is built; rotation only- transforms weights and does not change layer names, so this- ordering matches the old arch where rotation ran after- ``configure_layer_config``.- - ``self.quantize_config.data_type`` is final (rotation backend- dispatch depends on it).-- Work performed:- - Iterates ``self.rotation_configs`` and calls- :func:`~auto_round.algorithms.transforms.apply_rotation` on the- model for each config.-- Postconditions:- - ``self.model_context.model`` carries the rotated weights and any- inserted online-Hadamard hooks.- """- if not self.rotation_configs:- return- logger.info("Applying Hadamard transform to the model.")- for rotation_cfg in self.rotation_configs:- self.model_context.model = apply_rotation(- self.model_context.model,- rotation_cfg,- data_type=self.quantize_config.data_type,- )-def _patch_model(self) -> None:"""Phase 3 – Model structure patching.diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex cc2a9f7b5..350c7b486 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -440,6 +440,7 @@ class CompressionOrchestrator(BaseOrchestrator):all_blocks = self.quant_block_list or get_block_names(self.model)pbar = tqdm(range(sum(len(block) for block in all_blocks)))+ _zs_block_idx = 0for block_names in all_blocks:for block_name in block_names:pbar.set_description(f"Quantizing {block_name}")@@ -461,12 +462,15 @@ class CompressionOrchestrator(BaseOrchestrator):materialize_model_(block)# ── Pure algorithm ────────────────────────────────────────+ # ``block_index`` carries the global block index so compress_block+ # can drive layer-wise rotation with the correct layer_idx.ctx = BlockContext(model=self.model,block_names=[block_name],block_name=block_name,- block_index=0,+ block_index=_zs_block_idx,)+ _zs_block_idx += 1# ── MoE scale alignment for FP8 dispatch efficiency ────────────────if is_nv_fp(self.act_data_type) or not self.act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")@@ -512,6 +516,9 @@ class CompressionOrchestrator(BaseOrchestrator):memory_monitor.log_summary()pbar.update(1)+ # ── Pipeline lifecycle: model-level teardown (also finalizes rotation) ─+ self.alg_composer.finalize_run()+remain_layer_names = []block_name_set = set(name for block in all_blocks for name in block)for n, m in self.model.named_modules():@@ -771,6 +778,10 @@ class CompressionOrchestrator(BaseOrchestrator):for rs in resume_states:rs.clear()+ # ── Pipeline lifecycle: model-level teardown (also finalizes any+ # layer-wise rotation). Symmetric with ``prepare_run`` above. ──────+ self.alg_composer.finalize_run()+pbar.set_description("Quantizing done")pbar.close()if self.compress_context.low_cpu_mem_usage:@@ -1024,6 +1035,20 @@ class CompressionOrchestrator(BaseOrchestrator):if not self._post_init_done:self.post_init()+ # Layer-wise rotation is driven by the internal block loop inside+ # ``AlgorithmComposer.compress_block`` (rotate as step 0, cleanup in+ # ``finalize_run``). This externally-driven single-block API cannot+ # guarantee that lifecycle, and rotating here would desync the caller's+ # own reference/teacher outputs (collected on the un-rotated block).+ # Fail loudly instead of producing silently wrong results.+ if self.alg_composer.has_layerwise_rotation:+ raise NotImplementedError(+ "Layer-wise rotation (rotation config `layerwise=True`) is not supported "+ "through the single-block quantize_block() API (e.g. LLM-Compressor). Use "+ "the full AutoRound quantize() entry point, or set `layerwise=False` on the "+ "rotation config to apply full-model rotation up-front."+ )+# ── Zero-shot (RTN) path: no calibration data needed ──────────────────if not self.need_calib:from auto_round.algorithms.composer import BlockContextdiff --git a/docs/rotation_details.md b/docs/rotation_details.mdindex 2b9f21819..2409ee02c 100644--- a/docs/rotation_details.md+++ b/docs/rotation_details.md@@ -87,6 +87,7 @@ Defined in `auto_round/algorithms/transforms/spinquant/preprocessor.py`.| `kl_top_k` | `1000` | Top-k logits used by the `kl_top` loss. || `fuse_rmsnorm` | `True` | Fuse RMSNorm scales into following linear layers. || `untie_embeddings` | `True` | Untie shared embedding / LM-head weights before rotating. |+| `layerwise` | `False` | Apply rotation **per decoder block**, in lock-step with AutoRound's block-wise quantization, instead of rotating the whole model up-front (see §1.8). Supported by QuaRot / SpinQuant and the per-Linear Hadamard *transform* backend; the Hadamard *inplace* backend ignores it and falls back to full-model rotation. || `dtype` | `torch.float32` | Numerical dtype used for rotation math. || `device` | `None` (auto) | Defaults to `"cuda"` if available, else `"cpu"`. |@@ -208,6 +209,69 @@ with a `*_type` code: `0` = deterministic, `1` = random, `2` = trained):re-patches the QuantLinear forward (R1/R4) and re-applies the R3 RoPE monkeypatch.R3 is rebuilt purely from `config.json`, not from stored buffers.+### 1.8 Block-wise (layer-wise) rotation++By default QuaRot / SpinQuant rotate the **entire model up-front**: R1 is fused into+the embedding and all linears, R2 into attention, and R3/R4 are installed as online+hooks — after which quantization runs. This requires the whole model to be resident at+once, which is heavy for large models and decoupled from AutoRound's block-wise+quantization loop.++Setting `layerwise=True` on the rotation config aligns rotation with block-wise+quantization: only the rotation matrices (R1–R4) are initialised up-front, and each+decoder block is rotated **just before it is quantized**, so only one block needs to be+on-device at a time.++```python+from auto_round import AutoRound+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig++# Block-wise QuaRot: rotate each decoder block in lock-step with quantization.+ar = AutoRound(+ "Qwen/Qwen3-0.6B",+ scheme="MXFP4",+ alg_configs=["rtn", SpinQuantConfig(r1=True, r2=True, layerwise=True)],+)+ar.quantize_and_save(output_dir="./Qwen3-0.6B-mxfp4-quarot-blockwise", format="auto_round")+```++The per-Linear Hadamard rotation also supports block-wise execution. Because it fuses an+independent Hadamard into each Linear (no cross-layer residual-stream coupling), rotating+one decoder block at a time is mathematically identical to the full-model pass:++```python+from auto_round import AutoRound+from auto_round.algorithms.transforms.hadamard.config import RotationConfig++# Block-wise Hadamard (per-Linear transform backend, MXFP4 / NVFP4).+ar = AutoRound(+ "Qwen/Qwen3-0.6B",+ scheme="MXFP4",+ iters=0,+ rotation_config=RotationConfig(hadamard_type="hadamard", layerwise=True),+)+ar.quantize_and_save(output_dir="./Qwen3-0.6B-mxfp4-hadamard-blockwise", format="auto_round")+```++Notes and limitations:++- **Supported algorithms.** `layerwise` is honoured by QuaRot / SpinQuant and by the+ per-Linear Hadamard **transform** backend (`BaseRotation.supports_layerwise == True`).+ The Hadamard **inplace** / QuaRot residual-stream backend (`backend="inplace"`) couples+ consecutive layers, so it reports `supports_layerwise == False` and transparently falls+ back to full-model rotation. Requesting `layerwise=True` for the transform backend on a+ non-MXFP4/NVFP4 dtype raises `NotImplementedError` (block-wise Hadamard needs the+ per-Linear kernel path).+- **`nblocks` aware.** When several decoder layers are fused into one scheduling group+ (`nblocks > 1`), each fused layer receives its correct global `layer_idx`, so R2/R3/R4+ head-dim math stays aligned with the model topology.+- **Single-block API.** Layer-wise rotation is driven by AutoRound's internal block loop+ and is **not** supported through the external single-block `quantize_block()` API+ (e.g. LLM-Compressor); attempting it raises `NotImplementedError`. Use the full+ `quantize()` entry point, or set `layerwise=False`.+- **Accuracy parity.** Block-wise rotation is mathematically equivalent to full-model+ rotation for the R1/R2/R3/R4 positions it supports; see §4 for measured parity.+---## 2. Per-Linear Block Rotation@@ -248,6 +312,7 @@ Application modes (`hadamard/apply.py`):| `hadamard_type` | `"hadamard"` | One of `hadamard`, `random_hadamard`, `inplace_quarot_hadamard`, `inplace_hadamard`, `inplace_random`. Deterministic Hadamard uses Sylvester construction (`block_size` must be a power of 2); `random_hadamard` supports non-power-of-2 sizes from the known-matrix library. || `fuse_online_to_weight` | `None` | Fuse online Hadamard rotation into weights when supported (`inplace` backend only). || `allow_online_rotation` | `True` | Allow online activation rotation. |+| `layerwise` | `False` | Apply the Hadamard rotation **per decoder block**, in lock-step with AutoRound's block-wise quantization (see §1.8). Supported by the per-Linear `transform` backend (MXFP4 / NVFP4); the `inplace` backend couples layers and falls back to full-model rotation. |### 2.2 Usagediff --git a/test/unit/common/algorithms/test_rotation.py b/test/unit/common/algorithms/test_rotation.pyindex 472f88648..4f8b02016 100644--- a/test/unit/common/algorithms/test_rotation.py+++ b/test/unit/common/algorithms/test_rotation.py@@ -923,7 +923,11 @@ class TestRandomHadamardCache:m1 = get_or_create_random_hadamard(8)clear_random_hadamard_cache()m2 = get_or_create_random_hadamard(8)- assert not torch.equal(m1, m2)+ # Clearing the cache must force a freshly-built matrix object. We assert+ # object identity rather than value inequality: a "random" Hadamard only+ # randomizes ``dim`` sign bits (H @ diag(±1)), so for dim=8 two draws+ # collide with probability 1/256, which would flake a value-based check.+ assert m1 is not m2def test_device_transfer(self):clear_random_hadamard_cache()diff --git a/test/unit/common/core/test_pipeline_fast.py b/test/unit/common/core/test_pipeline_fast.pyindex ae77014e1..d31ef8ef1 100644--- a/test/unit/common/core/test_pipeline_fast.py+++ b/test/unit/common/core/test_pipeline_fast.py@@ -81,7 +81,9 @@ def test_block_quantizer_is_terminal_regardless_of_config_order(configs):members = pipeline.members()- assert members[:-1] == pipeline.preprocessors+ # Preprocessors always lead the pipeline; rotation members (added by+ # RotationConfig) may sit between them and the terminal block quantizer.+ assert members[: len(pipeline.preprocessors)] == pipeline.preprocessorsassert members[-1] is pipeline.block_quantizerassert isinstance(members[-1], RTNQuantizer)