PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# # Copyright (C) 2026 Intel Corporation
# # SPDX-License-Identifier: Apache-2.0
"""
SpinQuant preprocessor for Intel AutoRound.
Main class that orchestrates the SpinQuant / QuaRot rotation pipeline:
1. Fuse RMSNorm parameters into linear layers
2. Replace RMSNorm with TrainableRMSNorm (if ``trainable_smooth``)
3. Initialise rotation matrices (R1, R2, R3, R4)
4. (If trainable) train rotations & smooth values via KL divergence
5. Fuse offline rotations (R1, R2) into model weights
6. Register online hooks (R3, R4) and clean up
The module delegates in-place hook registration to
``spinquant.inplace.apply`` so that the online-rotation logic follows the
same pattern as AutoRound's ``rotation.inplace`` package.
"""
from __future__ import annotations
import logging
import math
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn as nn
from auto_round.algorithms.registry import register_algorithm
from auto_round.algorithms.transforms.base import BaseRotationConfig
# ---------------------------------------------------------------------------
# Delegate online-hook registration to the inplace sub-package.
# ---------------------------------------------------------------------------
from auto_round.algorithms.transforms.spinquant.inplace.apply import (
register_spinquant_hooks,
remove_spinquant_hooks,
)
from auto_round.algorithms.transforms.spinquant.rotation_utils import (
apply_hadamard_to_linear,
create_block_diag_from_head_matrix,
deterministic_hadamard_matrix,
fuse_rmsnorm_in_model,
get_hadamard_K,
get_model_arch_info,
is_pow2,
matmul_hadU,
random_hadamard_matrix,
rotate_in_channels_,
rotate_out_channels_,
untie_word_embeddings_if_needed,
)
logger = logging.getLogger("autoround.spinquant")
# NOTE: Online R1 uses forward_pre_hook on target modules. Hooks are
# compatible with auto-round's WrapperLinear (which runs orig_layer hooks)
# and WrapperWALayer (which steals and runs them at inference).
@dataclass
class SpinQuantConfig(BaseRotationConfig):
"""Configuration for SpinQuant / QuaRot preprocessing.
Inherits from :class:`BaseRotationConfig` so that instances can be
collected by :class:`BaseCompressor` alongside Hadamard configs and
dispatched through the unified ``apply_rotation()`` / ``BaseRotation``
registry.
Feature Status:
✅ QuaRot mode (``trainable_rotation=False``): Fully supported.
Fixed Hadamard rotation (R1–R4), no training needed, no calibration data.
⚠️ SpinQuant mode (``trainable_rotation=True``): Experimental.
Training loop exists but not fully validated on real models.
✅ Model save/load: fully implemented with rebuilt hooks.
"""
#: Registry key — used by ``BaseRotation.from_config()`` to dispatch.
algorithm: str = "spinquant"
# Rotation dimensions
r1: bool = True # R1: hidden_size rotation (offline fused)
r2: bool = True # R2: head_dim rotation (offline fused)
r3: bool = False # R3: Q/K online rotation
r4: bool = False # R4: MLP activation online rotation
# Rotation size override (None = use full dimension from model config)
# When set, R1 uses rotation_size instead of hidden_size,
# and R4 uses rotation_size instead of intermediate_size.
# R2 always uses head_dim, R3 does not support custom size.
# This follows the same convention as Quark's rotation_size.
rotation_size: int | None = None
# Rotation matrix type for R1–R4
# - False (default): deterministic Hadamard (same matrix every time, no need to persist)
# - True: random Hadamard = H × D where D = diag(±1) random (must persist matrix)
# Only relevant when trainable_rotation=False (QuaRot mode).
# When trainable_rotation=True (SpinQuant mode), R1/R2 init from identity regardless.
# For online rotations (R1/R3/R4), random uses explicit x @ R instead of
# butterfly algorithm — slightly slower but functionally equivalent.
random_r1: bool = False
random_r2: bool = False
random_r3: bool = False
random_r4: bool = False
# Training control
# ⚠️ trainable_rotation=True (SpinQuant mode) is experimental — training
# loop exists but not validated end-to-end. Use trainable_rotation=False
# (QuaRot mode) for production use.
trainable_rotation: bool = False # Learn R via Cayley SGD (True = SpinQuant experimental)
trainable_smooth: bool = False # Learn smooth_values via Adam (joint SmoothQuant)
online_r1_rotation: bool = True # Online R1: rotate target weights + hook (Quark default)
# Training hyperparameters
iters: int = 200 # Training iterations
lr: float = 1e-4 # SGDG learning rate (rotation matrices)
smooth_lr: float = 1e-3 # Adam learning rate (smooth values)
batch_size: int = 1
# Loss
loss_type: str = "kl_top" # "kl_top" | "kl_full" | "mse"
kl_top_k: int = 1000
# Pipeline steps
fuse_rmsnorm: bool = True
untie_embeddings: bool = True
# Numerics
dtype: torch.dtype = torch.float32
device: str | None = None
#: CLI descriptions for the flags derived from this dataclass's signature.
cli_field_help = {
"layerwise": "Rotate each decoder block in lock-step with block-wise quantization "
"instead of rotating the whole model up front.",
"r1": "Rotate the residual stream (R1).",
"r2": "Rotate the attention head dimension (R2).",
"r3": "Rotate the attention value path online (R3).",
"r4": "Rotate the MLP down-projection online (R4).",
"rotation_size": "Override the R1/R4 rotation dimension; must be a power of two. " "R2/R3 always use head_dim.",
"random_r1": "Use a random Hadamard matrix for R1 instead of the deterministic one.",
"random_r2": "Use a random Hadamard matrix for R2 instead of the deterministic one.",
"random_r3": "Use a random Hadamard matrix for R3 instead of the deterministic one.",
"random_r4": "Use a random Hadamard matrix for R4 instead of the deterministic one.",
"trainable_rotation": "Learn the rotation matrices (SpinQuant mode) instead of using fixed "
"Hadamard matrices (QuaRot mode).",
"trainable_smooth": "Learn the SmoothQuant smoothing scales jointly with the rotation.",
"online_r1_rotation": "Apply R1 through an online forward hook instead of fusing it into the weights.",
"smooth_lr": "Learning rate of the smoothing-scale optimizer.",
"loss_type": "Training loss: 'kl_top', 'kl_full' or 'mse'.",
"kl_top_k": "Number of top logits used by the 'kl_top' loss.",
"fuse_rmsnorm": "Fuse the RMSNorm scaling into the adjacent linear layers.",
"untie_embeddings": "Untie the input/output embeddings before rotating.",
}
def __post_init__(self):
if self.device is None:
self.device = "cuda" if torch.cuda.is_available() else "cpu"
if self.rotation_size is not None:
if self.rotation_size <= 0:
raise ValueError(f"rotation_size must be positive, got {self.rotation_size}")
if not is_pow2(self.rotation_size):
raise ValueError(
f"rotation_size must be a power of 2, got {self.rotation_size}. "
f"Valid values: 16, 32, 64, 128, 256, 512, 1024, ..."
)
register_algorithm(
"quarot",
aliases=("quarot",),
config_factory=lambda: SpinQuantConfig(trainable_rotation=False, trainable_smooth=False),
summary="QuaRot fixed-Hadamard rotation (no training, no calibration data).",
)
register_algorithm(
"spinquant",
aliases=("spinquant",),
config_factory=lambda: SpinQuantConfig(trainable_rotation=True, trainable_smooth=True),
summary="SpinQuant trainable rotation (experimental).",
)
class TrainableRMSNorm(nn.Module):
"""
RMSNorm wrapper with trainable ``smooth_values`` for joint
SpinQuant + SmoothQuant.
Original RMSNorm::
output = x / RMS(x) * gamma
Trainable version::
output = x / RMS(x) * gamma * smooth_values
The ``smooth_values`` (diagonal scaling ``D``) are learned jointly
with rotation matrices to minimise quantisation error.
"""
def __init__(self, original_norm: nn.Module, trainable: bool = True) -> None:
super().__init__()
self.original_norm = original_norm
self.trainable = trainable
if hasattr(original_norm, "weight"):
shape = original_norm.weight.shape
self.smooth_values = nn.Parameter(
torch.ones(shape, device=original_norm.weight.device, dtype=original_norm.weight.dtype),
requires_grad=trainable,
)
else:
self.smooth_values = None
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
out = self.original_norm(hidden_states)
if self.smooth_values is not None:
out = out * self.smooth_values
return out
class SpinQuantPreprocessor:
"""
SpinQuant preprocessor following AutoRound's transform conventions.
After preprocessing the model is mathematically equivalent to the
original but with weight distributions better suited for quantisation.
"""
def __init__(self, model: nn.Module, config: SpinQuantConfig | None = None) -> None:
self.model = model
self.config = config or SpinQuantConfig()
# Architecture metadata
info = get_model_arch_info(model)
self.hidden_size = info.get("hidden_size", 0)
self.head_dim = info.get("head_dim", 0)
self.num_q_heads = info.get("num_q_heads", 0)
self.num_kv_heads = info.get("num_kv_heads", 0)
self.intermediate_size = info.get("intermediate_size", 0)
# Resolve effective rotation sizes based on config.rotation_size
# R1: rotation_size or hidden_size
self.r1_rotation_size = self.config.rotation_size or self.hidden_size
# R2: always head_dim (not affected by rotation_size)
self.r2_rotation_size = self.head_dim
# R3: always head_dim (not affected by rotation_size, same as Quark)
self.r3_rotation_size = self.head_dim
# R4: rotation_size or intermediate_size
self.r4_rotation_size = self.config.rotation_size or self.intermediate_size
# Training state
self.rotation_params: list[nn.Parameter] = []
self.smooth_params: list[nn.Parameter] = []
# Fusion deduplication
self._rotated_modules: set[nn.Module] = set()
# Hook handles (managed by inplace sub-package)
self._hook_handles: list[Any] = []
# R1 online hook handles (managed by _apply_online_r1)
self._r1_hook_handles: list[Any] = []
# ------------------------------------------------------------------
# Main entry point
# ------------------------------------------------------------------
def preprocess(self, dataloader: Any | None = None) -> nn.Module:
logger.info("[SpinQuant] Starting preprocessing...")
logger.info(
f"[SpinQuant] Model architecture info: hidden_size={self.hidden_size}, "
f"head_dim={self.head_dim}, num_q_heads={self.num_q_heads}, "
f"num_kv_heads={self.num_kv_heads}, intermediate_size={self.intermediate_size}"
)
if self.config.rotation_size is not None:
logger.info(
f"[SpinQuant] Custom rotation_size={self.config.rotation_size} → "
f"R1 size={self.r1_rotation_size}, R4 size={self.r4_rotation_size} "
f"(R2/R3 always use head_dim={self.head_dim})"
)
logger.info(
f"[SpinQuant] Rotation config: R1={self.config.r1}, R2={self.config.r2}, "
f"R3={self.config.r3}, R4={self.config.r4}, "
f"online_r1={self.config.online_r1_rotation}, "
f"trainable_rotation={self.config.trainable_rotation}, "
f"trainable_smooth={self.config.trainable_smooth}"
)
# Validate dimensions for enabled rotations
self._validate_dimensions()
# Step 1: untie embeddings (only needed for offline R1 — it rotates embed_tokens)
if self.config.untie_embeddings and not self.config.online_r1_rotation:
if untie_word_embeddings_if_needed(self.model):
logger.info("[SpinQuant] Untied input/output embeddings")
elif self.config.online_r1_rotation:
logger.info("[SpinQuant] Online R1: skipping untie embeddings (embed_tokens unchanged)")
# Step 2: fuse RMSNorm gamma into linear weights
# Online R1 does NOT fuse RMSNorm (matching Quark's behavior):
# the rotation is local per-module, so gamma doesn't need to commute.
if self.config.fuse_rmsnorm and not self.config.online_r1_rotation:
logger.info("[SpinQuant] Fusing RMSNorm parameters into linear weights...")
fuse_rmsnorm_in_model(self.model)
elif self.config.online_r1_rotation:
logger.info("[SpinQuant] Online R1: skipping RMSNorm fusion (not needed)")
# Step 3: replace RMSNorm with TrainableRMSNorm (SmoothQuant)
if self.config.trainable_smooth:
logger.info("[SpinQuant] Adding trainable smooth values...")
self._replace_norms_with_trainable()
# Step 4: initialise rotation matrices
logger.info("[SpinQuant] Initialising rotation matrices...")
self._init_rotation_matrices()
# Step 5: train if requested
if self.config.trainable_rotation or self.config.trainable_smooth:
if dataloader is None:
raise ValueError("dataloader required when trainable=True")
logger.info(f"[SpinQuant] Training for {self.config.iters} iterations...")
self._train_rotations(dataloader)
# Step 6: apply R1 rotation
if self.config.r1 and self.config.online_r1_rotation:
logger.info("[SpinQuant] Applying online R1 rotation (weight + wrapper)...")
self._apply_online_r1()
# Still fuse R2 and R4 offline
self._fuse_r2_rotation()
self._fuse_r4_rotation()
else:
logger.info("[SpinQuant] Fusing offline rotations into weights...")
self._fuse_offline_rotations()
# Step 7: register online hooks (R3 / R4)
if self.config.r3 or self.config.r4:
logger.info("[SpinQuant] Registering online rotation hooks...")
self._hook_handles = register_spinquant_hooks(
self.model,
self.config,
head_dim=self.head_dim,
intermediate_size=self.intermediate_size,
r4_rotation_size=self.r4_rotation_size,
)
# Step 8: cleanup training artefacts
self._cleanup()
# Store config on model for downstream serialization (export pipeline)
self.model._rotation_config = self.config
self.model._spinquant_config = self.config # legacy alias
# Print per-layer transformation summary table
self._print_transformation_summary()
logger.info("[SpinQuant] Preprocessing complete!")
return self.model
# ------------------------------------------------------------------
# Layer-wise API (for block-lifecycle / block-wise quantization)
# ------------------------------------------------------------------
def prepare(self, dataloader: Any | None = None) -> None:
"""Global preparation for layer-wise (block-wise) rotation.
Performs all lightweight, non-destructive steps:
- Validates dimensions
- Initialises rotation matrices (stored as model buffers)
- Optionally trains rotation matrices (requires full model + dataloader)
- Pre-computes rotation state shared across layers
Does **NOT** modify model weights or register hooks — that is done
per-layer by :meth:`rotate_layer`.
This method is the layer-wise counterpart of :meth:`preprocess`.
Args:
dataloader: Required only when ``trainable_rotation=True``.
Raises:
ValueError: If ``online_r1_rotation=False`` while R1 is enabled
(offline R1 is incompatible with layer-wise mode because it
rotates the inter-layer hidden-state space via embed_tokens /
lm_head, which breaks pre-cached block inputs).
"""
# Layer-wise rotation only works with Online R1 (offline R1 rotates the
# shared residual stream, which is incompatible with pre-cached block
# inputs).
if self.config.r1 and not self.config.online_r1_rotation:
raise ValueError(
"[SpinQuant] Layer-wise rotation requires online_r1_rotation=True. "
"Offline R1 changes inter-layer hidden state space, which is "
"incompatible with pre-cached block inputs. Use full-model "
"rotation (config `layerwise=False`) for offline R1."
)
logger.info("[SpinQuant] Preparing for layer-wise rotation...")
logger.info(
f"[SpinQuant] Model architecture: hidden_size={self.hidden_size}, "
f"head_dim={self.head_dim}, intermediate_size={self.intermediate_size}"
)
logger.info(
f"[SpinQuant] Rotation config: R1={self.config.r1}, R2={self.config.r2}, "
f"R3={self.config.r3}, R4={self.config.r4}, "
f"online_r1={self.config.online_r1_rotation}"
)
# Step 1: Validate dimensions
self._validate_dimensions()
# Step 2: Initialise rotation matrices (lightweight — only R matrix buffers)
logger.info("[SpinQuant] Initialising rotation matrices...")
self._init_rotation_matrices()
# Step 3: Train if requested (needs full model forward/backward)
if self.config.trainable_rotation or self.config.trainable_smooth:
if dataloader is None:
raise ValueError("dataloader required when trainable=True")
logger.warning(
"[SpinQuant] Layer-wise mode with trainable_rotation: training "
"requires a full model forward/backward pass (higher memory). "
"Consider pre-training R matrices separately for large models."
)
# Trainable smooth needs norm replacement first
if self.config.trainable_smooth:
self._replace_norms_with_trainable()
self._train_rotations(dataloader)
# Pre-compute shared state consumed by rotate_layer()
if self.config.r1 and self.config.online_r1_rotation:
self._r1_shared = self._compute_online_r1_shared()
else:
self._r1_shared = None
if self.config.r2 and self.head_dim > 0:
R2_head = self._get_rotation_tensor("spinquant_R2_head")
if R2_head is not None:
R2 = R2_head.data.to(torch.float64)
self._r2_shared = (R2, R2.t())
else:
self._r2_shared = None
else:
self._r2_shared = None
if self.config.r4 and self.r4_rotation_size > 0:
use_random_r4 = self.config.random_r4
R4 = None
if use_random_r4:
R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
if R4_matrix is None:
raise RuntimeError("[SpinQuant] random_r4=True but spinquant_R4_matrix buffer not found.")
R4 = R4_matrix.to(torch.float64)
self._r4_shared = (self.r4_rotation_size, use_random_r4, R4)
else:
self._r4_shared = None
# Store config on model for downstream serialization
self.model._rotation_config = self.config
self.model._spinquant_config = self.config # legacy alias
logger.info("[SpinQuant] Layer-wise preparation complete (no weights modified yet).")
def rotate_layer(self, layer: nn.Module, layer_idx: int) -> None:
"""Apply all configured rotations to a single decoder layer.
This is the per-block entry point called by the compressor inside the
block-wise quantization loop. The layer must already be materialised
and on the target device.
Operations performed (in order):
1. **R1 (Online)**: rotate target-module weights + register hooks
2. **R2 (Offline fuse)**: fuse into v_proj output + o_proj input
3. **R4 (Offline fuse)**: fuse into down_proj input
4. **R3 (Hook)**: monkeypatch apply_rotary_pos_emb
5. **R4 (Hook)**: register forward_pre_hook on down_proj
Args:
layer: A single decoder layer, already on the target device.
layer_idx: Zero-based index for logging purposes.
"""
if not hasattr(self, "_r1_shared"):
raise RuntimeError(
"[SpinQuant] prepare() must be called before rotate_layer(). "
"The layer-wise state has not been initialised."
)
if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
logger.warning(f"[SpinQuant] Layer {layer_idx}: no self_attn/mlp, skipping rotation.")
return
layer_device = next(layer.parameters()).device
# ── R1: Online weight rotation + hooks ──
if self._r1_shared is not None:
self._apply_online_r1_to_layer(layer, *self._r1_shared)
# ── R2: Offline fuse into v_proj/o_proj ──
if self._r2_shared is not None:
self._fuse_r2_to_layer(layer, *self._r2_shared)
# ── R4: Offline fuse into down_proj ──
if self._r4_shared is not None:
self._fuse_r4_to_layer(layer, *self._r4_shared)
# ── R3: Monkeypatch attention (after RoPE) ──
if self.config.r3 and self.head_dim > 0:
self._rotate_layer_r3_hook(layer, layer_idx)
# ── R4: Hook on down_proj ──
if self.config.r4 and self.r4_rotation_size > 0:
self._rotate_layer_r4_hook(layer, layer_idx, layer_device)
logger.debug(f"[SpinQuant] Layer {layer_idx}: rotation applied.")
def _rotate_layer_r3_hook(self, layer: nn.Module, layer_idx: int) -> None:
"""Apply the R3 monkeypatch to a single layer's attention module.
Consistent with the model-level ``register_spinquant_hooks`` R3 path.
"""
if not is_pow2(self.head_dim):
return
from auto_round.algorithms.transforms.spinquant.monkeypatch import (
add_qk_rotation_after_rope,
)
attn = layer.self_attn
if not (hasattr(attn, "q_proj") and hasattr(attn, "k_proj")):
return
if getattr(attn, "_spinquant_r3_patched", False):
return # Already patched (idempotent)
random_r3 = self.config.random_r3
r3_matrix = getattr(self.model, "spinquant_R3_head", None)
try:
wrapper = add_qk_rotation_after_rope(attn, rope_function_name="apply_rotary_pos_emb")
if random_r3 and r3_matrix is not None:
wrapper.set_matrix(r3_matrix)
else:
wrapper.set_hadamard(None, self.head_dim)
attn._spinquant_r3_patched = True
self._hook_handles.append(("r3_monkeypatch", f"layer_{layer_idx}", attn, wrapper))
except ValueError as e:
logger.warning(f"[SpinQuant] R3 monkeypatch failed for layer {layer_idx}: {e}")
def _rotate_layer_r4_hook(self, layer: nn.Module, layer_idx: int, layer_device: torch.device) -> None:
"""Register the R4 ``forward_pre_hook`` on a single layer's down_proj.
Consistent with the model-level ``register_spinquant_hooks`` R4 path.
"""
mlp = layer.mlp
if not hasattr(mlp, "down_proj"):
return
r4_size = self.r4_rotation_size
use_random = self.config.random_r4
need_block = r4_size < self.intermediate_size
module = mlp.down_proj
if use_random:
R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
if R4_matrix is None:
return
R4 = R4_matrix.to(device=layer_device, dtype=torch.float32)
def _make_hook(R, rot_size, block_mode):
def hook(mod, args):
x = args[0]
R_local = R.to(x.device, dtype=x.dtype)
if block_mode:
shape = x.shape
x = x.reshape(*shape[:-1], -1, rot_size)
x = (x @ R_local).reshape(shape)
else:
x = x @ R_local
return (x,) + args[1:]
return hook
hook = _make_hook(R4, r4_size, need_block)
else:
try:
had_K_mat, had_K_val = get_hadamard_K(r4_size)
except ValueError:
return
had_K_mat = had_K_mat.to(device=layer_device, dtype=torch.float32)
def _make_hook_butterfly(had_mat, k_val, rot_size, block_mode):
def hook(mod, args):
x = args[0]
if block_mode:
shape = x.shape
x = x.reshape(*shape[:-1], -1, rot_size)
x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)
x = x.reshape(shape)
else:
x = matmul_hadU(x, hadamard_K=had_mat.to(x.device), K=k_val)
return (x,) + args[1:]
return hook
hook = _make_hook_butterfly(had_K_mat, had_K_val, r4_size, need_block)
hook._spinquant_hook = True
handle = module.register_forward_pre_hook(hook)
self._hook_handles.append(handle)
def finalize(self) -> None:
"""Post-loop cleanup after all layers have been rotated layer-wise.
Mirrors :meth:`_cleanup` but for the layer-wise path.
"""
self.model.eval()
for p in self.model.parameters():
p.requires_grad = False
self.rotation_params.clear()
self.smooth_params.clear()
self._rotated_modules.clear()
n_r1 = len(self._r1_hook_handles)
n_other = len(self._hook_handles)
logger.info(f"[SpinQuant] Layer-wise rotation finalized. " f"Active hooks: {n_r1} R1 + {n_other} R3/R4.")
def _validate_dimensions(self) -> None:
"""Validate dimension requirements and disable rotations that can't work."""
# R1: check r1_rotation_size divides hidden_size and is power of 2
if self.config.r1 and self.r1_rotation_size > 0:
if not is_pow2(self.r1_rotation_size):
logger.warning(
f"[SpinQuant] R1 rotation_size must be a power of 2, "
f"but got {self.r1_rotation_size}. Disabling R1."
)
self.config.r1 = False
elif self.hidden_size % self.r1_rotation_size != 0:
logger.warning(
f"[SpinQuant] R1 rotation_size={self.r1_rotation_size} must divide "
f"hidden_size={self.hidden_size}. Disabling R1."
)
self.config.r1 = False
if self.config.r2 and self.head_dim > 0 and not is_pow2(self.head_dim):
logger.warning(
f"[SpinQuant] R2 requires head_dim to be a power of 2, "
f"but got head_dim={self.head_dim}. Disabling R2."
)
self.config.r2 = False
if self.config.r3 and self.head_dim > 0 and not is_pow2(self.head_dim):
logger.warning(
f"[SpinQuant] R3 requires head_dim to be a power of 2, "
f"but got head_dim={self.head_dim}. Disabling R3."
)
self.config.r3 = False
if self.config.r3 and self.config.rotation_size is not None:
logger.warning(
f"[SpinQuant] R3 does not support custom rotation_size "
f"(always uses head_dim={self.head_dim}). Ignoring rotation_size for R3."
)
if self.config.r4 and self.r4_rotation_size > 0:
# Validate using get_hadamard_K — same decomposition used by matmul_hadU
inter = self.r4_rotation_size
try:
_, K = get_hadamard_K(inter)
except ValueError:
logger.warning(
f"[SpinQuant] R4 cannot find Hadamard decomposition for " f"r4_rotation_size={inter}. Disabling R4."
)
self.config.r4 = False
K = 0
if self.config.r4 and self.intermediate_size % self.r4_rotation_size != 0:
logger.warning(
f"[SpinQuant] R4 rotation_size={self.r4_rotation_size} must divide "
f"intermediate_size={self.intermediate_size}. Disabling R4."
)
self.config.r4 = False
elif self.config.r4:
logger.info(
f"[SpinQuant] R4 Hadamard: K={K}, "
f"r4_rotation_size={inter}, intermediate_size={self.intermediate_size}"
)
# ------------------------------------------------------------------
# Step 3: Trainable RMSNorm
# ------------------------------------------------------------------
def _replace_norms_with_trainable(self) -> None:
"""Replace RMSNorm modules with TrainableRMSNorm wrappers."""
layers = list(self._get_layers())
if not layers:
return
for layer in layers:
if hasattr(layer, "input_layernorm"):
layer.input_layernorm = TrainableRMSNorm(layer.input_layernorm, trainable=self.config.trainable_smooth)
if self.config.trainable_smooth:
self.smooth_params.append(layer.input_layernorm.smooth_values)
if hasattr(layer, "post_attention_layernorm"):
layer.post_attention_layernorm = TrainableRMSNorm(
layer.post_attention_layernorm, trainable=self.config.trainable_smooth
)
if self.config.trainable_smooth:
self.smooth_params.append(layer.post_attention_layernorm.smooth_values)
# Final norm
final_norm = None
for path in ("model.norm", "norm"):
parts = path.split(".")
obj = self.model
for p in parts:
if not hasattr(obj, p):
break
obj = getattr(obj, p)
else:
final_norm = obj
break
if final_norm is not None:
wrapped = TrainableRMSNorm(final_norm, trainable=self.config.trainable_smooth)
# Replace on model
for path in ("model.norm", "norm"):
parts = path.split(".")
if len(parts) == 1:
if hasattr(self.model, parts[0]):
setattr(self.model, parts[0], wrapped)
break
elif len(parts) == 2:
parent = getattr(self.model, parts[0], None)
if parent is not None and hasattr(parent, parts[1]):
setattr(parent, parts[1], wrapped)
break
if self.config.trainable_smooth:
self.smooth_params.append(wrapped.smooth_values)
# ------------------------------------------------------------------
# Step 4: Initialise rotation matrices
# ------------------------------------------------------------------
def _init_rotation_matrices(self) -> None:
# Use the model's actual device, not config.device, to avoid mismatch
model_device = next(self.model.parameters()).device
dtype = self.config.dtype
# R1: r1_rotation_size x r1_rotation_size
if self.config.r1:
r1_size = self.r1_rotation_size
if self.config.online_r1_rotation and not self.config.trainable_rotation and not self.config.random_r1:
# Online deterministic R1: Hadamard computed on-the-fly via
# butterfly algorithm — no need to store a matrix buffer.
logger.info(
f"[SpinQuant] R1: Online mode — deterministic Hadamard will be computed "
f"on-the-fly via butterfly algorithm (rotation_size={r1_size})"
)
elif self.config.trainable_rotation and not self.config.online_r1_rotation:
R1 = nn.Parameter(torch.eye(r1_size, device=model_device, dtype=dtype))
logger.info(f"[SpinQuant] R1: Trainable rotation matrix [{r1_size}×{r1_size}] (identity init)")
self._register_rotation("spinquant_R1", R1)
else:
if self.config.random_r1:
R1 = nn.Parameter(
random_hadamard_matrix(r1_size, dtype=dtype, device=model_device),
requires_grad=False,
)
matrix_type = "Random Hadamard"
else:
R1 = nn.Parameter(
deterministic_hadamard_matrix(r1_size, dtype=dtype, device=model_device),
requires_grad=False,
)
matrix_type = "Deterministic Hadamard"
mode = "online" if self.config.online_r1_rotation else "offline fuse"
if r1_size < self.hidden_size:
logger.info(
f"[SpinQuant] R1: {matrix_type} [{r1_size}×{r1_size}] block rotation "
f"({self.hidden_size // r1_size} blocks, fixed, {mode})"
)
else:
logger.info(f"[SpinQuant] R1: {matrix_type} [{r1_size}×{r1_size}] (fixed, {mode})")
self._register_rotation("spinquant_R1", R1)
# R2_head: head_dim x head_dim
if self.config.r2 and self.head_dim > 0:
if self.config.trainable_rotation:
R2_head = nn.Parameter(torch.eye(self.head_dim, device=model_device, dtype=dtype))
logger.info(
f"[SpinQuant] R2: Trainable per-head rotation [{self.head_dim}×{self.head_dim}] (identity init)"
)
else:
if self.config.random_r2:
R2_head = nn.Parameter(
random_hadamard_matrix(self.head_dim, dtype=dtype, device=model_device),
requires_grad=False,
)
matrix_type = "Random Hadamard"
else:
R2_head = nn.Parameter(
deterministic_hadamard_matrix(self.head_dim, dtype=dtype, device=model_device),
requires_grad=False,
)
matrix_type = "Deterministic Hadamard"
logger.info(
f"[SpinQuant] R2: {matrix_type} [{self.head_dim}×{self.head_dim}] per head (fixed, offline fuse)"
)
self._register_rotation("spinquant_R2_head", R2_head)
# R3_head: head_dim × head_dim (online, after RoPE on Q/K)
if self.config.r3 and self.head_dim > 0:
if self.config.random_r3:
R3 = random_hadamard_matrix(self.head_dim, dtype=dtype, device=model_device)
matrix_type = "Random Hadamard"
mode_str = "online, x @ R"
else:
R3 = deterministic_hadamard_matrix(self.head_dim, dtype=dtype, device=model_device)
matrix_type = "Deterministic Hadamard"
mode_str = "online, butterfly"
self.model.register_buffer("spinquant_R3_head", R3)
logger.info(f"[SpinQuant] R3: {matrix_type} " f"[{self.head_dim}×{self.head_dim}] after RoPE ({mode_str})")
# R4: r4_rotation_size (online hook on down_proj + offline fuse)
if self.config.r4 and self.r4_rotation_size > 0:
r4_size = self.r4_rotation_size
if self.config.random_r4:
R4 = random_hadamard_matrix(r4_size, dtype=dtype, device=model_device)
self.model.register_buffer("spinquant_R4_matrix", R4)
matrix_type = "Random Hadamard"
mode_str = "online x @ R + offline fuse"
else:
matrix_type = "Deterministic Hadamard"
mode_str = "online butterfly + offline fuse"
# Always store Hadamard decomposition (needed for deterministic path
# and for backward compat with serialize.py buffer detection)
had_K, K = get_hadamard_K(r4_size)
had_K = had_K.to(dtype=dtype, device=model_device)
self.model.register_buffer("spinquant_R4_had_K", had_K)
self.model.register_buffer("spinquant_R4_K", torch.tensor(K, device=model_device))
if r4_size < self.intermediate_size:
logger.info(
f"[SpinQuant] R4: {matrix_type} K={K}, "
f"rotation_size={r4_size}, "
f"{self.intermediate_size // r4_size} rotation blocks on down_proj ({mode_str})"
)
else:
logger.info(f"[SpinQuant] R4: {matrix_type} K={K}, " f"rotation_size={r4_size} ({mode_str})")
def _register_rotation(self, name: str, param: nn.Parameter) -> None:
self.model.register_parameter(name, param)
if param.requires_grad:
self.rotation_params.append(param)
# ------------------------------------------------------------------
# Step 5: Training
# ------------------------------------------------------------------
def _train_rotations(self, dataloader: Any) -> None:
"""Train rotation matrices using the shared training loop.
Delegates to :func:`training.run_training_loop` which is the
single implementation shared with :class:`RotationTrainer`.
For advanced training features (callbacks, evaluation, checkpointing,
custom loss), use ``RotationTrainer`` from ``training.py`` directly.
"""
from auto_round.algorithms.transforms.spinquant.training import (
clone_model_for_reference,
create_dual_optimizer,
run_training_loop,
)
optimizer = create_dual_optimizer(
self.model,
lr=self.config.lr,
smooth_lr=self.config.smooth_lr,
)
if optimizer is None:
return
logger.info("[SpinQuant] Cloning original model for KL reference...")
original_model = clone_model_for_reference(self.model)
result = run_training_loop(
model=self.model,
original_model=original_model,
optimizer=optimizer,
dataloader=dataloader,
max_iters=self.config.iters,
loss_type=self.config.loss_type,
kl_top_k=self.config.kl_top_k,
log_interval=50,
)
logger.info(
f"[SpinQuant] Training complete: {result.steps} steps, "
f"best_loss={result.best_loss:.3e}, ortho_dev={result.final_ortho_deviation:.2e}"
)
del original_model
if torch.cuda.is_available():
torch.cuda.empty_cache()
def _get_embed_tokens(self) -> nn.Module | None:
"""Get embedding module, supporting both model.embed_tokens and model.model.embed_tokens."""
for attr_path in ("embed_tokens", "model.embed_tokens"):
parts = attr_path.split(".")
obj = self.model
for p in parts:
if not hasattr(obj, p):
break
obj = getattr(obj, p)
else:
return obj
return None
def _get_layers(self):
"""Yield transformer layers, supporting multiple nesting patterns."""
for attr_path in ("layers", "model.layers", "transformer.h", "model.decoder.layers"):
parts = attr_path.split(".")
obj = self.model
for p in parts:
if not hasattr(obj, p):
break
obj = getattr(obj, p)
else:
if hasattr(obj, "__iter__"):
for layer in obj:
yield layer
return
# Fallback: search recursively
for name, module in self.model.named_modules():
if name.endswith(".layers") or name == "layers":
if hasattr(module, "__iter__"):
for layer in module:
yield layer
return
def _get_lm_head(self) -> nn.Module | None:
"""Get LM head module."""
return getattr(self.model, "lm_head", None)
# ------------------------------------------------------------------
# Step 6a: Online R1 rotation (matching Quark's default behavior)
# ------------------------------------------------------------------
def _apply_online_r1(self) -> None:
"""Apply online R1 rotation: rotate target module weights and register
``forward_pre_hook`` s so the matching activation rotation is applied at
runtime.
Supports two modes:
- **Deterministic Hadamard** (default): uses butterfly algorithm
(``matmul_hadU``) for both weight rotation and activation hooks —
no matrix needs to be stored.
- **Random Hadamard** (``random_r1=True``): uses the stored
``spinquant_R1`` matrix for explicit ``x @ R`` in both weight
rotation and activation hooks.
This matches Quark's ``apply_online_r1()`` behavior:
- Target modules (q/k/v_proj, gate/up_proj) get their weights rotated
- A ``forward_pre_hook`` on each target module applies the same rotation
to activations at runtime
- The two transforms cancel out: ``R(x) @ (W @ R).T = x @ W.T``
- prev_modules (embed_tokens, o_proj, down_proj) are NOT modified
- RMSNorm gamma is NOT fused
- lm_head is NOT modified (last_layer skipped, matching Quark)
Hooks are compatible with auto-round's quantization pipeline:
WrapperLinear.forward runs ``orig_layer._forward_pre_hooks`` before
the linear computation, and WrapperWALayer steals & runs them at
inference time.
.. warning::
Hook-based online R1 is **not serializable** — ``save_pretrained()``
will NOT save the activation hooks. If you need to save and reload
the rotated model, use offline R1 instead
(``SpinQuantConfig(online_r1_rotation=False)``).
"""
r1_size = self.r1_rotation_size
use_random = self.config.random_r1
logger.warning(
"[SpinQuant] Online R1 uses forward_pre_hooks which are NOT saved by "
"save_pretrained(). The saved model will lose activation rotation hooks. "
"Use SpinQuantConfig(online_r1_rotation=False) for offline R1 if you "
"need to save/reload the model."
)
r1_size, use_random, R1_full, hadamard_K, K = self._compute_online_r1_shared()
n_rotated = 0
n_hooked = 0
for layer in self._get_layers():
nr, nh = self._apply_online_r1_to_layer(layer, r1_size, use_random, R1_full, hadamard_K, K)
n_rotated += nr
n_hooked += nh
mode_str = "random matrix (x @ R)" if use_random else "deterministic butterfly"
logger.info(
f"[SpinQuant] Online R1: rotated {n_rotated} target modules, "
f"registered {n_hooked} activation hooks "
f"(rotation_size={r1_size}, mode={mode_str}, "
f"lm_head/embed_tokens/o_proj/down_proj unchanged)"
)
def _compute_online_r1_shared(self):
"""Compute R1 rotation state shared across all layers (online R1).
Returns a tuple ``(r1_size, use_random, R1_full, hadamard_K, K)`` that
can be passed to :meth:`_apply_online_r1_to_layer`. Extracted so that
both full-model (:meth:`_apply_online_r1`) and layer-wise
(:meth:`rotate_layer`) paths share identical R1 conventions.
"""
r1_size = self.r1_rotation_size
use_random = self.config.random_r1
model_device = next(self.model.parameters()).device
# For random R1: use the stored full matrix
# For deterministic R1: use butterfly algorithm
if use_random:
R1_full = self._get_rotation_tensor("spinquant_R1")
if R1_full is None:
raise RuntimeError(
"[SpinQuant] random_r1=True but spinquant_R1 buffer not found. "
"Ensure _init_rotation_matrices() was called first."
)
R1_full = R1_full.data.to(torch.float64)
hadamard_K, K = None, None
else:
R1_full = None
hadamard_K, K = get_hadamard_K(r1_size)
hadamard_K = hadamard_K.to(model_device)
return r1_size, use_random, R1_full, hadamard_K, K
def _apply_online_r1_to_layer(self, layer, r1_size, use_random, R1_full, hadamard_K, K):
"""Apply online R1 to a single decoder layer: rotate target-module
weights and register the matching activation ``forward_pre_hook`` s.
Returns ``(n_rotated, n_hooked)``.
"""
if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
return 0, 0
layer_device = next(layer.parameters()).device
attn = layer.self_attn
mlp = layer.mlp
# Target modules: (parent_module, attr_name)
target_specs = []
for proj_name in ("q_proj", "k_proj", "v_proj"):
if hasattr(attn, proj_name):
target_specs.append((attn, proj_name))
for proj_name in ("gate_proj", "up_proj"):
if hasattr(mlp, proj_name):
target_specs.append((mlp, proj_name))
n_rotated = 0
n_hooked = 0
for parent, attr_name in target_specs:
module = getattr(parent, attr_name)
dtype = module.weight.data.dtype
in_features = module.weight.shape[-1]
if use_random:
# Random R1: explicit matrix multiply
R = R1_full.to(layer_device)
if r1_size == in_features:
W = module.weight.data.to(torch.float64)
module.weight.data = (W @ R).to(dtype)
elif in_features % r1_size == 0:
# Block rotation must use W @ R (not W @ R.T) to stay
# consistent with the online hook (x @ R). The random
# Hadamard matrix is orthonormal but NOT symmetric, so
# rotate_in_channels_ (which applies R.T) would break
# equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.
rotate_in_channels_(module, R_in=R.T)
else:
raise ValueError(
f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
)
else:
# Deterministic Hadamard: butterfly algorithm
had_K_local = hadamard_K.to(layer_device)
if r1_size == in_features:
module.weight.data = matmul_hadU(module.weight.data, hadamard_K=had_K_local, K=K).to(dtype)
elif in_features % r1_size == 0:
R_block = had_K_local.to(torch.float64)
if R_block.shape[0] != r1_size:
had_1, _ = get_hadamard_K(r1_size // K)
R_block = torch.kron(
had_K_local.to(device="cpu", dtype=torch.float64),
had_1.to(device="cpu", dtype=torch.float64),
)
R_block = R_block / math.sqrt(r1_size)
rotate_in_channels_(module, R_in=R_block)
else:
raise ValueError(
f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
)
n_rotated += 1
# Register forward_pre_hook for online activation rotation
if use_random:
hook = self._make_online_r1_hook_matrix(R1_full.to(layer_device), r1_size, in_features)
else:
hook = self._make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K.to(layer_device), K)
hook._spinquant_hook = True # tag for selective removal
handle = module.register_forward_pre_hook(hook)
self._r1_hook_handles.append(handle)
n_hooked += 1
return n_rotated, n_hooked
@staticmethod
def _make_online_r1_hook_butterfly(r1_size, in_features, hadamard_K, K):
"""Create a forward_pre_hook using butterfly algorithm (deterministic Hadamard)."""
if r1_size == in_features:
def hook(module, args):
x = args[0]
x = matmul_hadU(x, hadamard_K=hadamard_K.to(x.device), K=K)
return (x,) + args[1:]
else:
R_block = hadamard_K.to(torch.float64)
if R_block.shape[0] != r1_size:
had_1, _ = get_hadamard_K(r1_size // K)
R_block = torch.kron(
hadamard_K.to(device="cpu", dtype=torch.float64),
had_1.to(device="cpu", dtype=torch.float64),
)
R_block = R_block / math.sqrt(r1_size)
R_block_f32 = R_block.float()
def hook(module, args):
x = args[0]
dtype = x.dtype
shape = x.shape
R = R_block_f32.to(x.device, dtype=x.dtype)
x = x.reshape(*shape[:-1], -1, r1_size)
x = (x @ R).reshape(shape).to(dtype)
return (x,) + args[1:]
return hook
@staticmethod
def _make_online_r1_hook_matrix(R1_matrix, r1_size, in_features):
"""Create a forward_pre_hook using stored full matrix (random/trained R1)."""
R1_f32 = R1_matrix.float()
if r1_size == in_features:
def hook(module, args):
x = args[0]
R = R1_f32.to(x.device, dtype=x.dtype)
x = x @ R
return (x,) + args[1:]
else:
def hook(module, args):
x = args[0]
dtype = x.dtype
shape = x.shape
R = R1_f32.to(x.device, dtype=x.dtype)
x = x.reshape(*shape[:-1], -1, r1_size)
x = (x @ R).reshape(shape).to(dtype)
return (x,) + args[1:]
return hook
# ------------------------------------------------------------------
# Step 6b: Fuse offline rotations
# ------------------------------------------------------------------
def _fuse_offline_rotations(self) -> None:
self._rotated_modules.clear()
if not self.config.r1:
# Even without R1, we may still need R2 and R4
self._fuse_r2_rotation()
self._fuse_r4_rotation()
return
R1 = self._get_rotation_tensor("spinquant_R1")
if R1 is None or R1.numel() == 0:
return
R1_inv = R1.t()
r1_size = R1.shape[0]
# Embed tokens (output rotation: W_embed @ R1)
embed = self._get_embed_tokens()
if embed is not None:
with torch.no_grad():
W_f64 = embed.weight.data.to(torch.float64)
R_f64 = R1.to(embed.weight.device).to(torch.float64)
if W_f64.shape[-1] == r1_size:
new_w = torch.matmul(W_f64, R_f64)
elif W_f64.shape[-1] % r1_size == 0:
# Block rotation for embedding
w_reshaped = W_f64.reshape(*W_f64.shape[:-1], -1, r1_size)
new_w = (w_reshaped @ R_f64).reshape(W_f64.shape)
else:
raise ValueError(f"embed_tokens dim={W_f64.shape[-1]} not divisible by R1 size={r1_size}")
embed.weight.data = new_w.to(embed.weight.dtype)
# Transformer layers
n_layers = 0
for layer in self._get_layers():
if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
continue
attn = layer.self_attn
mlp = layer.mlp
# Ensure R1_inv is on the same device as layer weights
layer_device = next(layer.parameters()).device
R1_inv_local = R1_inv.to(layer_device)
R1_local = R1.to(layer_device)
# Attention: in-channel uses R1_inv (→ W @ R1), out-channel uses R1 (→ R1_inv @ W)
if hasattr(attn, "q_proj"):
rotate_in_channels_(attn.q_proj, R_in=R1_inv_local, rotated_modules=self._rotated_modules)
if hasattr(attn, "k_proj"):
rotate_in_channels_(attn.k_proj, R_in=R1_inv_local, rotated_modules=self._rotated_modules)
if hasattr(attn, "v_proj"):
rotate_in_channels_(attn.v_proj, R_in=R1_inv_local, rotated_modules=self._rotated_modules)
if hasattr(attn, "o_proj"):
rotate_out_channels_(attn.o_proj, R_out=R1_local, rotated_modules=self._rotated_modules)
# MLP: same convention
if hasattr(mlp, "gate_proj"):
rotate_in_channels_(mlp.gate_proj, R_in=R1_inv_local, rotated_modules=self._rotated_modules)
if hasattr(mlp, "up_proj"):
rotate_in_channels_(mlp.up_proj, R_in=R1_inv_local, rotated_modules=self._rotated_modules)
if hasattr(mlp, "down_proj"):
rotate_out_channels_(mlp.down_proj, R_out=R1_local, rotated_modules=self._rotated_modules)
n_layers += 1
logger.info(
f"[SpinQuant] R1 fused into {n_layers} layers (embed_tokens, q/k/v/o_proj, gate/up/down_proj, lm_head)"
)
# LM head: in-channel uses R1_inv (→ W @ R1)
lm_head = self._get_lm_head()
if lm_head is not None:
lm_device = lm_head.weight.device
rotate_in_channels_(lm_head, R_in=R1_inv.to(lm_device), rotated_modules=self._rotated_modules)
# R2 head-dim rotation (per-head Hadamard on v_proj output + o_proj input)
self._fuse_r2_rotation()
# R4 offline fusion (Hadamard on down_proj input side)
self._fuse_r4_rotation()
def _fuse_r2_rotation(self) -> None:
"""Fuse R2 per-head rotation into v_proj and o_proj.
Uses the stored ``spinquant_R2_head`` matrix (which may be deterministic
Hadamard, random Hadamard, or a trained orthogonal matrix) to rotate
v_proj output channels and o_proj input channels per attention head.
Math:
v_rotated = v @ R2 per head → fuse into v_proj: W_new = R2^T @ W
o_proj input is R2-rotated → fuse into o_proj: W_new = W @ R2
"""
if not self.config.r2 or self.head_dim <= 0:
return
R2_head = self._get_rotation_tensor("spinquant_R2_head")
if R2_head is None:
return
R2 = R2_head.data.to(torch.float64)
R2_T = R2.t()
n_fused = 0
for layer in self._get_layers():
n_fused += self._fuse_r2_to_layer(layer, R2, R2_T)
logger.info(f"[SpinQuant] R2 fused into {n_fused} layers (v_proj out + o_proj in, head_dim={self.head_dim})")
def _fuse_r2_to_layer(self, layer, R2, R2_T) -> int:
"""Fuse R2 per-head rotation into a single layer's v_proj/o_proj.
Returns 1 if the layer was fused, else 0.
"""
if not hasattr(layer, "self_attn"):
return 0
attn = layer.self_attn
# v_proj: W_new = R2^T @ W per head on output dimension
if hasattr(attn, "v_proj"):
W = attn.v_proj.weight.data
dtype = W.dtype
W = W.to(torch.float64)
n_heads = W.shape[0] // self.head_dim
W_reshaped = W.reshape(n_heads, self.head_dim, W.shape[1])
W_reshaped = torch.einsum("ij,kjl->kil", R2_T.to(W.device), W_reshaped)
attn.v_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
# o_proj: W_new = W @ R2 per head on input dimension
# (R2^{-1} = R2^T on the activation side ↔ W @ R2 on the weight side)
if hasattr(attn, "o_proj"):
W = attn.o_proj.weight.data
dtype = W.dtype
W = W.to(torch.float64)
n_heads = W.shape[1] // self.head_dim
W_reshaped = W.reshape(W.shape[0], n_heads, self.head_dim)
W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R2.to(W.device))
attn.o_proj.weight.data = W_reshaped.reshape(W.shape).to(dtype)
return 1
def _fuse_r4_rotation(self) -> None:
"""Fuse R4 rotation into down_proj's input side.
For deterministic Hadamard: uses :func:`matmul_hadU` (butterfly algorithm).
For random Hadamard: uses the stored ``spinquant_R4_matrix`` buffer with
explicit ``W @ R`` matrix multiply.
The R4 hook applies ``x → x @ R`` on activations before down_proj.
To cancel this in the weight: ``(x @ R) @ W^T = x @ (W @ R)^T``,
so we fuse ``W_new = W @ R`` on the input dimension.
"""
if not self.config.r4 or self.r4_rotation_size <= 0:
return
r4_size = self.r4_rotation_size
use_random = self.config.random_r4
# Get the rotation matrix
R4 = None
if use_random:
R4_matrix = getattr(self.model, "spinquant_R4_matrix", None)
if R4_matrix is None:
raise RuntimeError("[SpinQuant] random_r4=True but spinquant_R4_matrix buffer not found.")
R4 = R4_matrix.to(torch.float64)
n_fused = 0
for layer in self._get_layers():
n_fused += self._fuse_r4_to_layer(layer, r4_size, use_random, R4)
mode_str = "random x @ R" if use_random else "deterministic butterfly"
logger.info(
f"[SpinQuant] R4 offline fused into {n_fused} down_proj layers "
f"(r4_rotation_size={r4_size}, mode={mode_str})"
)
def _fuse_r4_to_layer(self, layer, r4_size, use_random, R4) -> int:
"""Fuse R4 rotation into a single layer's down_proj input side.
Returns 1 if the layer was fused, else 0.
"""
if not hasattr(layer, "mlp"):
return 0
mlp = layer.mlp
if not hasattr(mlp, "down_proj"):
return 0
W = mlp.down_proj.weight.data
dtype = W.dtype
if use_random:
# Random: explicit W @ R per block
W = W.to(torch.float64)
R4 = R4.to(W.device)
if r4_size == W.shape[1]:
mlp.down_proj.weight.data = (W @ R4).to(dtype)
else:
out_feat, in_feat = W.shape
n_blocks = in_feat // r4_size
W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
W_reshaped = torch.einsum("ijk,kl->ijl", W_reshaped, R4)
mlp.down_proj.weight.data = W_reshaped.reshape(out_feat, in_feat).to(dtype)
else:
# Deterministic: matmul_hadU (butterfly algorithm)
# matmul_hadU operates on the last dimension — for weight
# shape [out, in], last dim = in_features which is what
# we want to rotate (input channels of down_proj).
if r4_size == W.shape[1]:
mlp.down_proj.weight.data = matmul_hadU(W).to(dtype)
else:
out_feat, in_feat = W.shape
n_blocks = in_feat // r4_size
W_reshaped = W.reshape(out_feat, n_blocks, r4_size)
W_rotated = matmul_hadU(W_reshaped)
mlp.down_proj.weight.data = W_rotated.reshape(out_feat, in_feat).to(dtype)
return 1
# ------------------------------------------------------------------
# Step 8: Cleanup
# ------------------------------------------------------------------
def _cleanup(self) -> None:
self.model.eval()
# Keep rotation matrices on the model — they are needed by
# inject_buffers_on_layer during serialization (random / trained
# matrices must be read back via _get_stored_rotation).
# Only the internal tracking lists are cleared below.
# NOTE: Do NOT remove online hooks/monkeypatches here. R3/R4 hooks
# must persist for correct inference. Only remove training-related state.
# Freeze all parameters
for p in self.model.parameters():
p.requires_grad = False
# Clear internal tracking
self.rotation_params.clear()
self.smooth_params.clear()
self._rotated_modules.clear()
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _get_rotation_tensor(self, name: str) -> torch.Tensor | None:
if hasattr(self.model, name):
tensor = getattr(self.model, name)
if isinstance(tensor, (nn.Parameter, torch.Tensor)):
return tensor.data if isinstance(tensor, nn.Parameter) else tensor
return None
def _print_transformation_summary(self) -> None:
"""Print a per-layer table summarizing all applied transformations and hooks."""
lines = []
lines.append("")
lines.append("=" * 100)
lines.append("SpinQuant Transformation Summary")
lines.append("=" * 100)
# --- Global transforms ---
lines.append("")
lines.append("Global Transforms:")
lines.append(f" {'Component':<30} {'Transform':<50} {'Status'}")
lines.append(f" {'-'*30} {'-'*50} {'-'*10}")
# Untie embeddings
tied = getattr(self.model.config, "tie_word_embeddings", False) if hasattr(self.model, "config") else "unknown"
if self.config.online_r1_rotation:
lines.append(
f" {'embed_tokens / lm_head':<30} {'Untie word embeddings (skipped: online R1)':<50} {'- n/a'}"
)
else:
lines.append(
f" {'embed_tokens / lm_head':<30} {'Untie word embeddings':<50} "
f"{'✓ untied' if not tied else '✗ still tied'}"
)
# RMSNorm fusion
if self.config.online_r1_rotation:
lines.append(f" {'All RMSNorm layers':<30} {'Fuse gamma (skipped: online R1)':<50} {'- n/a'}")
else:
lines.append(
f" {'All RMSNorm layers':<30} {'Fuse gamma into linear weights':<50} "
f"{'✓ fused' if self.config.fuse_rmsnorm else '✗ skipped'}"
)
# R1
if self.config.r1:
r1_size = self.r1_rotation_size
mode = "online" if self.config.online_r1_rotation else "offline"
if r1_size < self.hidden_size:
r1_desc = f"Block Hadamard {r1_size}×{r1_size}, {self.hidden_size // r1_size} blocks ({mode})"
else:
r1_desc = f"Hadamard {r1_size}×{r1_size} ({mode})"
else:
r1_desc = "Disabled"
lines.append(f" {'Residual stream (R1)':<30} {r1_desc:<50} {'✓' if self.config.r1 else '✗'}")
# --- Per-layer table ---
lines.append("")
lines.append("Per-Layer Transforms:")
# Determine R4 K
r4_K = 0
if self.config.r4 and self.r4_rotation_size > 0:
K = 1
r4_size = self.r4_rotation_size
while K * 2 <= r4_size and r4_size % (K * 2) == 0:
K *= 2
if K > 1:
r4_K = K
# Table header
col_layer = "Layer"
col_r1 = "R1 (online)" if self.config.online_r1_rotation else "R1 (weight fuse)"
col_r2 = "R2 (weight fuse)"
col_r3 = "R3 (online hook)"
col_r4 = "R4 (online hook)"
header = f" {col_layer:<35} {col_r1:<20} {col_r2:<20} {col_r3:<22} {col_r4:<22}"
lines.append(header)
lines.append(f" {'-'*35} {'-'*20} {'-'*20} {'-'*22} {'-'*22}")
online_r1 = self.config.online_r1_rotation
# embed_tokens
if online_r1:
r1_embed = "-" # online R1 doesn't touch embed
else:
r1_embed = f"W@R ({self.r1_rotation_size})" if self.config.r1 else "-"
lines.append(f" {'model.embed_tokens':<35} {r1_embed:<20} {'-':<20} {'-':<22} {'-':<22}")
# Transformer layers
layers = list(self._get_layers())
n_layers = sum(1 for l in layers if hasattr(l, "self_attn") and hasattr(l, "mlp"))
for i, layer in enumerate(layers):
if not (hasattr(layer, "self_attn") and hasattr(layer, "mlp")):
continue
# Only show first 2 and last layer to keep output concise
if n_layers > 5 and 2 <= i < n_layers - 1:
if i == 2:
lines.append(f" {' ... (same pattern for all layers)':<35}")
continue
layer_name = f"layers.{i}"
if online_r1 and self.config.r1:
r1_attn = "q/k/v:W@H+hook"
r1_mlp = "g/u:W@H+hook"
elif self.config.r1:
r1_attn = "q/k/v:R⁻¹@W o:W@R"
r1_mlp = "g/u:R⁻¹@W d:W@R"
else:
r1_attn = "-"
r1_mlp = "-"
# R2 for this layer: affects v_proj out, o_proj in
r2_status = f"H({self.head_dim}) v↔o" if self.config.r2 else "-"
# R3: check if monkeypatch applied (look for wrapper in self_attn)
r3_status = "-"
if self.config.r3 and self.head_dim > 0 and is_pow2(self.head_dim):
r3_status = f"H({self.head_dim}) Q,K post-RoPE"
# R4: check if hook on down_proj
r4_status = "-"
if self.config.r4 and r4_K > 0:
n_blocks = self.r4_rotation_size // r4_K
r4_status = f"blockH(K={r4_K},b={n_blocks})"
# Print attention row
lines.append(f" {layer_name + '.self_attn':<35} {r1_attn:<20} {r2_status:<20} {r3_status:<22} {'-':<22}")
# Print MLP row
lines.append(f" {layer_name + '.mlp':<35} {r1_mlp:<20} {'-':<20} {'-':<22} {r4_status:<22}")
# lm_head
if online_r1:
r1_lm = "-" # online R1 doesn't touch lm_head
else:
r1_lm = f"R⁻¹@W ({self.r1_rotation_size})" if self.config.r1 else "-"
lines.append(f" {'model.lm_head':<35} {r1_lm:<20} {'-':<20} {'-':<22} {'-':<22}")
# --- Hook summary ---
lines.append("")
lines.append("Registered Hooks:")
# R1 online hooks are stored in self._r1_hook_handles
r1_hooks = len(self._r1_hook_handles)
# R3/R4 hooks are stored in self._hook_handles (from register_spinquant_hooks)
r3_hooks = sum(1 for h in self._hook_handles if isinstance(h, tuple) and h[0] == "r3_monkeypatch")
r4_hooks = sum(1 for h in self._hook_handles if not (isinstance(h, tuple) and h[0] == "r3_monkeypatch"))
lines.append(f" R1 online hooks (Hadamard on target module input): {r1_hooks} modules")
lines.append(f" R3 monkeypatch (apply_rotary_pos_emb → QKRotationWrapper): {r3_hooks} attention layers")
lines.append(f" R4 forward_pre_hook (block Hadamard on down_proj input): {r4_hooks} MLP layers")
# --- Summary totals ---
lines.append("")
lines.append("Totals:")
lines.append(f" Transformer layers: {n_layers}")
if online_r1 and self.config.r1:
lines.append(f" Online R1 targets: {n_layers}×(q/k/v_proj, gate/up_proj) = {r1_hooks} modules")
fused_parts = []
else:
fused_parts = ["embed_tokens"]
if self.config.r1:
fused_parts.append(f"{n_layers}×(q/k/v/o_proj, gate/up/down_proj), lm_head")
if self.config.r2:
fused_parts.append(f"{n_layers}×(v_proj↔o_proj)")
if self.config.r4 and r4_K > 0:
fused_parts.append(f"{n_layers}×(down_proj)")
lines.append(f" Offline-fused params: {', '.join(fused_parts) if fused_parts else 'none'}")
total_hooks = r1_hooks + r3_hooks + r4_hooks
lines.append(f" Online hooks: {total_hooks} total ({r1_hooks} R1 + {r3_hooks} R3 + {r4_hooks} R4)")
lines.append(f" Inference overhead: R1={'O(seq×hidden×log₂H) per module' if r1_hooks > 0 else 'none'}")
lines.append(
f" R3={'O(seq×heads×d_head×log₂d_head) per layer' if r3_hooks > 0 else 'none'}"
)
lines.append(f" R4={'O(seq×inter×log₂K) per layer' if r4_hooks > 0 else 'none'}")
lines.append("=" * 100)
# Log as a single multi-line message
logger.info("\n".join(lines))
def remove_spinquant_hooks_from_model(model: nn.Module) -> None:
"""Remove all SpinQuant hooks and R3 monkeypatches from a model.
Removes:
- Forward hooks / pre-hooks tagged with ``_spinquant_hook = True`` (R1/R4)
- R3 monkeypatches on attention modules (tagged with ``_spinquant_r3_patched``)
Hooks from other frameworks are left untouched.
"""
for module in model.modules():
# Remove tagged forward hooks (R4) and pre-hooks (R1 online)
if hasattr(module, "_forward_hooks"):
for hook_id in list(module._forward_hooks.keys()):
hook = module._forward_hooks[hook_id]
if getattr(hook, "_spinquant_hook", False):
del module._forward_hooks[hook_id]
if hasattr(module, "_forward_pre_hooks"):
for hook_id in list(module._forward_pre_hooks.keys()):
hook = module._forward_pre_hooks[hook_id]
if getattr(hook, "_spinquant_hook", False):
del module._forward_pre_hooks[hook_id]
# Remove R3 monkeypatches (instance-level forward override)
if getattr(module, "_spinquant_r3_patched", False):
if "forward" in module.__dict__:
del module.__dict__["forward"]
delattr(module, "_spinquant_r3_patched")