Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix random rotation and update rotation doc. (#1884)

lkk4e2ad7728b34
diff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.py
index d2b63d9e7..5e48cd8c9 100644
--- a/auto_round/algorithms/transforms/__init__.py
+++ b/auto_round/algorithms/transforms/__init__.py
@@ -127,10 +127,11 @@ def normalize_rotation_config(
if key in ("spinquant", "quarot"):
from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig
- # "quarot" → QuaRot defaults (no training)
+ # "quarot" → QuaRot defaults (fixed Hadamard, no training)
if key == "quarot":
return SpinQuantConfig(trainable_rotation=False, trainable_smooth=False)
- return SpinQuantConfig()
+ # "spinquant" → experimental trainable mode
+ return SpinQuantConfig(trainable_rotation=True, trainable_smooth=True)
# Otherwise treat as Hadamard config.
return RotationConfig.model_validate(_normalize_hadamard_config(config))
diff --git a/auto_round/algorithms/transforms/spinquant/__init__.py b/auto_round/algorithms/transforms/spinquant/__init__.py
index 9c90d3077..b94ef5dc1 100644
--- a/auto_round/algorithms/transforms/spinquant/__init__.py
+++ b/auto_round/algorithms/transforms/spinquant/__init__.py
@@ -43,8 +43,7 @@ Example (QuaRot — fixed Hadamard, recommended)::
config = SpinQuantConfig(
r1=True, r2=True, r3=False, r4=False,
- trainable_rotation=False, # QuaRot: fixed Hadamard, no training needed
- trainable_smooth=False,
+ # Default: trainable_rotation=False, trainable_smooth=False (QuaRot mode)
online_r1_rotation=True,
)
SpinQuantPreprocessor(model, config).preprocess() # no dataloader needed
diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.py
index a1f2cf178..7c2944ea1 100644
--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py
+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py
@@ -109,8 +109,8 @@ class SpinQuantConfig(BaseRotationConfig):
# ⚠️ trainable_rotation=True (SpinQuant mode) is experimental — training
# loop exists but not validated end-to-end. Use trainable_rotation=False
# (QuaRot mode) for production use.
- trainable_rotation: bool = True # Learn R via Cayley SGD (False = QuaRot fixed Hadamard)
- trainable_smooth: bool = True # Learn smooth_values via Adam (joint SmoothQuant)
+ trainable_rotation: bool = False # Learn R via Cayley SGD (True = SpinQuant experimental)
+ trainable_smooth: bool = False # Learn smooth_values via Adam (joint SmoothQuant)
online_r1_rotation: bool = True # Online R1: rotate target weights + hook (Quark default)
# Training hyperparameters
@@ -730,7 +730,12 @@ class SpinQuantPreprocessor:
W = module.weight.data.to(torch.float64)
module.weight.data = (W @ R).to(dtype)
elif in_features % r1_size == 0:
- rotate_in_channels_(module, R_in=R)
+ # Block rotation must use W @ R (not W @ R.T) to stay
+ # consistent with the online hook (x @ R). The random
+ # Hadamard matrix is orthonormal but NOT symmetric, so
+ # rotate_in_channels_ (which applies R.T) would break
+ # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.
+ rotate_in_channels_(module, R_in=R.T)
else:
raise ValueError(
f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 4a2a8052d..ad3b33922 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -34,7 +34,9 @@ This document presents step-by-step instructions for auto-round llm quantization
+ [Device/Multi-GPU setting in Quantization](#devicemulti-gpu-setting-in-quantization)
- [Enable multiple gpus calibration in lm_head quantization](#enable-multiple-gpus-calibration-in-lm_head-quantization)
+ [Adjust Hyperparameters](#adjust-hyperparameters)
- + [Hadamard Transform-Research Feature](#hadamard-transform)
+ + [Rotation](#rotation)
+ - [QuaRot / SpinQuant](#quarot--spinquant)
+ - [Per-Linear Block Rotation (Experimental)](#per-linear-block-rotation-experimental)
* [4 Inference](#4-inference)
+ [CPU](#cpu)
+ [Intel GPU](#intel-gpu)
@@ -800,31 +802,131 @@ autoround.save_quantized(format="auto_awq", output_dir="tmp_autoround")
Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.
-### Hadamard Transform
+### Rotation
-**Research feature with no effective kernels currently available and typically low community adoption.**
+AutoRound supports rotation-based transforms to improve quantization accuracy. Rotation redistributes outliers in weights and activations before quantization, making the distribution more uniform and quantization-friendly.
-AutoRound supports Hadamard transform as an optional weight/activation transformation technique, which can improve quantization accuracy by rotating the weight/activation matrix. This is particularly useful for certain quantization scenarios.
+Two rotation approaches are available:
-#### Overview
+- **QuaRot / SpinQuant** — full-model rotation across multiple positions (R1–R4), providing comprehensive outlier suppression for aggressive quantization (e.g., MXFP4, NVFP4, W4A4). **Recommended for most use cases.**
+- **Per-Linear Block Rotation** — block-diagonal rotation applied uniformly to every linear layer. An earlier experimental implementation; the QuaRot/SpinQuant approach above is generally preferred.
-The Hadamard transform is particularly useful in scenarios where activation outliers hurt quantization accuracy. In practice, it helps suppress such outliers, making it especially effective when `act_bits < 8`. Users can enable this feature when they need more stable activation distributions and better accuracy in low‑bit quantization settings.
+#### QuaRot / SpinQuant
-#### Implementation
+QuaRot applies deterministic Hadamard rotations at up to 4 positions in the transformer architecture. Unlike the per-linear block rotation, it operates at the model architecture level — rotating the residual stream, attention heads, and MLP activations at specific positions for targeted outlier suppression.
-AutoRound provides two types of Hadamard transforms:
+##### Rotation Positions
-1. **Deterministic Hadamard Transform** (`hadamard`): Uses Sylvester's construction to create a deterministic Hadamard matrix. The size must be a power of 2.
+| Position | Target | Mode | Effect |
+|----------|--------|------|--------|
+| **R1** | Residual stream (hidden_size) | Online or Offline | Smooths weight outliers across all linear layers |
+| **R2** | V/O projections (head_dim) | Offline (fused) | Balances per-head value distributions |
+| **R3** | Q/K after RoPE (head_dim) | Online (hook) | Improves KV-cache quantization friendliness |
+| **R4** | MLP up/down (intermediate_size) | Online (hook) | Suppresses activation outliers in FFN |
-2. **Random Hadamard Transform** (`random_hadamard`): Uses known Hadamard matrices from N. J. A. Sloane's Library of Hadamard Matrices. Supports non-power-of-2 sizes and deterministic seeding.
+- **Online**: Applied at runtime via forward hooks (no weight modification, zero-cost at save time)
+- **Offline (fused)**: Absorbed into adjacent weight matrices (no runtime overhead)
+##### Quick Start
-#### Quantization with Hadamard Transform
+```python
+from auto_round import AutoRound
+
+model_name = "Qwen/Qwen3-0.6B"
+output_dir = "./Qwen3-0.6B-mxfp4-quarot"
+
+# QuaRot preset: R1+R2+R3+R4 with deterministic Hadamard
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config="quarot")
+ar.quantize_and_save(output_dir=output_dir, format="auto_round")
+```
+
+##### Rotation Levels
+
+Choose how many positions to rotate based on your accuracy needs:
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig
+
+# R1 only (fast, good baseline improvement)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True))
+
+# R1 + R2 (better, no runtime overhead after fuse)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True))
+
+# R1 + R2 + R3 + R4 (best accuracy, slight runtime overhead from hooks)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))
+```
+
+##### String Shortcuts
+
+| Value | Equivalent |
+|-------|-----------|
+| `"quarot"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=False, trainable_smooth=False)` — deterministic Hadamard, no training |
+| `"spinquant"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=True, trainable_smooth=True)` — **experimental** (requires a dataloader) |
+
+> ⚠️ **SpinQuant trainable rotation** (`trainable_rotation=True`) enables learnable rotation matrices optimized via Cayley SGD. This feature is **experimental** and not fully validated. Use `"quarot"` (fixed Hadamard) for production workloads.
+
+##### Deterministic vs Random Hadamard
+
+```python
+# Deterministic (default): fixed Hadamard matrix, no extra storage needed
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))
+
+# Random: H × diag(±1), slightly better outlier suppression, requires saving the rotation matrix
+ar = AutoRound(
+ model_name,
+ scheme="MXFP4",
+ rotation_config=SpinQuantConfig(
+ r1=True, r2=True, r3=True, r4=True, random_r1=True, random_r2=True, random_r3=True, random_r4=True
+ ),
+)
+```
+
+##### Key Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `r1` / `r2` / `r3` / `r4` | `True / True / False / False` | Enable rotation at each position |
+| `online_r1_rotation` | `True` | R1 via hook (`True`) or fused into weights (`False`) |
+| `random_r1` / `random_r2` / `random_r3` / `random_r4` | `False` | Use random Hadamard (H×diag(±1)) instead of deterministic |
+| `rotation_size` | `None` (auto) | Block rotation dimension; auto-detected from model dimensions |
+| `trainable_rotation` | `False` | Enable SpinQuant learnable rotation (**experimental**, requires dataloader) |
+| `trainable_smooth` | `False` | Enable learnable smooth values (**experimental**, requires dataloader) |
+
+##### Save & Load
+
+Quantized models with rotation are saved and loaded transparently:
+
+```python
+# Save (rotation matrices stored automatically if needed)
+ar.quantize_and_save(output_dir="./my_model", format="auto_round")
+
+# Load (rotation hooks restored automatically)
+from transformers import AutoModelForCausalLM
+
+model = AutoModelForCausalLM.from_pretrained("./my_model", device_map="auto")
+```
+
+- **Deterministic rotations**: Only metadata (type + rotation_size) is stored — matrices are reconstructed on load
+- **Random rotations**: An `int8` (±1) rotation matrix is stored (size ~rotation_size² bytes)
+- **Online rotations**: Rebuilt during model loading (R1/R4 via QuantLinear forward patching; R3 via monkeypatch from config)
+
+#### Per-Linear Block Rotation (Experimental)
+
+> ⚠️ This is an earlier experimental implementation that applies block-diagonal Hadamard rotation **per linear layer** by patching every `nn.Linear` module in the model. For most use cases, the [QuaRot / SpinQuant](#quarot--spinquant) approach above is preferred — it provides architecture-aware rotation at specific positions (R1–R4) with better accuracy and lower overhead.
+
+The per-linear block rotation works by iterating over all linear layers and either:
+- **Weight mode**: Fusing the Hadamard matrix directly into the weight tensor (offline)
+- **Input mode**: Registering a forward pre-hook that rotates the input activation before each linear layer (online)
+
+This approach uses a configurable `block_size` (default 32) applied uniformly to every linear layer. It is simpler than QuaRot but less targeted — it does not distinguish between residual stream, attention, and MLP layers, nor does it handle RoPE or activation-side rotation. Use `rotation_config="default"` to enable it.
+
+##### Usage
```python
from auto_round import AutoRound
-# Load a model (supports FP8/BF16/FP16/FP32)
model_name_or_path = "meta-llama/Llama-3.1-8B-Instruct"
output_dir = "./Llama-3.1-8B-Instruct-mxfp4-ht"
@@ -834,19 +936,20 @@ ar = AutoRound(model_name_or_path, scheme="MXFP4", rotation_config="default")
ar.quantize_and_save(output_dir=output_dir, format="auto_round")
```
-#### Transform Classes
+##### Types
-| Class | Description |
-|-------|-------------|
-| `HadamardTransform` | Applies deterministic Hadamard transform |
-| `RandomHadamardTransform` | Applies random Hadamard transform with optional seeding |
+| Type | Description |
+|------|-------------|
+| `hadamard` (default) | Deterministic Hadamard matrix (Sylvester construction, block_size must be power of 2) |
+| `random_hadamard` | Randomly signed Hadamard from known matrix library; supports non-power-of-2 sizes |
-#### Parameters
+##### Parameters
-| Parameter | Description |
-|-----------|-------------|
-| `block_size` | Size of the transformation block (default: 32) |
-| `seed` | Random seed (for RandomHadamardTransform) |
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `block_size` | `32` | Size of the Hadamard block applied to each linear layer |
+| `hadamard_type` | `"hadamard"` | `"hadamard"` or `"random_hadamard"` |
+| `seed` | `None` | Random seed (for `random_hadamard` only) |
## 4 Inference
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index cf4347df0..8677fe185 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -35,7 +35,9 @@
- [lm_head 量化中开启多 GPU 标定](#lm_head-量化中开启多-gpu-标定)
- [手动配置设备映射](#手动配置设备映射)
+ [超参数调整](#超参数调整)
- + [Hadamard变换-研究功能](#hadamard变换)
+ + [旋转(Rotation)](#旋转rotation)
+ - [QuaRot / SpinQuant](#quarot--spinquant)
+ - [逐线性层块旋转(实验性)](#逐线性层块旋转实验性)
* [4 推理部署](#4-推理部署)
+ [CPU](#cpu)
+ [英特尔 GPU](#英特尔-gpu)
@@ -767,29 +769,131 @@ auto-round --model_name Qwen/Qwen3-0.6B --scheme "W4A16" --quant_lm_head --form
#### 使用 AdamW 优化器
添加 `--adam` 参数即可启用;**注意**:在我们的多项测试场景中,AdamW 优化器的效果均不如符号梯度下降(sign gradient descent)。
+### 旋转(Rotation)
+AutoRound 支持基于旋转的变换技术来提升量化精度。旋转在量化前对权重和激活中的离群点进行重分布,使分布更加均匀,从而对量化更友好。
-### Hadamard变换
+目前提供两种旋转方式:
-AutoRound 支持将 Hadamard 变换作为可选的权重/激活变换技术,通过旋转权重/激活矩阵来提升量化精度。这在某些量化场景中尤其有用。
+- **QuaRot / SpinQuant** — 在多个位置(R1–R4)进行全模型旋转,为激进量化(如 MXFP4、NVFP4、W4A4)提供全面的离群点抑制。**推荐在大多数场景中使用。**
+- **逐线性层块旋转(Per-Linear Block Rotation)** — 对每个线性层均匀应用块对角旋转。为较早期的实验性实现,通常建议使用上述 QuaRot/SpinQuant 方式。
-#### 概述
+#### QuaRot / SpinQuant
-Hadamard 变换在激活值存在离群点且影响量化精度的场景中特别有效。在实践中,它能够抑制这些离群点,因此在 `act_bits < 8` 的低比特激活量化设置中效果尤为显著。用户可以在需要更稳定的激活分布和更高低比特精度时启用该功能。
+QuaRot 在 Transformer 架构的最多 4 个位置应用确定性 Hadamard 旋转。与逐线性层块旋转不同,它在模型架构级别操作——在特定位置对残差流、注意力头和 MLP 激活进行旋转,以实现针对性的离群点抑制。
-#### 实现方式
+##### 旋转位置
-AutoRound 提供两种类型的 Hadamard 变换:
+| 位置 | 目标 | 模式 | 效果 |
+|------|------|------|------|
+| **R1** | 残差流(hidden_size) | 在线或离线 | 平滑所有线性层的权重离群点 |
+| **R2** | V/O 投影(head_dim) | 离线(融合) | 平衡各注意力头的值分布 |
+| **R3** | RoPE 后的 Q/K(head_dim) | 在线(hook) | 提升 KV-cache 的量化友好性 |
+| **R4** | MLP up/down(intermediate_size) | 在线(hook) | 抑制 FFN 中的激活离群点 |
-1. **确定性 Hadamard 变换**(`hadamard`):使用 Sylvester 构造法生成确定性的 Hadamard 矩阵,尺寸必须为 2 的幂次。
-2. **随机 Hadamard 变换**(`random_hadamard`):使用 N. J. A. Sloane 的 Hadamard 矩阵库中已知的矩阵。支持非 2 的幂次的尺寸,并支持确定性随机种子。
+- **在线(Online)**:通过 forward hook 在运行时应用(不修改权重,保存时零开销)
+- **离线/融合(Offline/Fused)**:吸收到相邻权重矩阵中(无运行时开销)
+
+##### 快速开始
+
+```python
+from auto_round import AutoRound
+
+model_name = "Qwen/Qwen3-0.6B"
+output_dir = "./Qwen3-0.6B-mxfp4-quarot"
+
+# QuaRot 预设:R1+R2+R3+R4,确定性 Hadamard
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config="quarot")
+ar.quantize_and_save(output_dir=output_dir, format="auto_round")
+```
+
+##### 旋转级别
+
+根据精度需求选择旋转的位置数量:
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig
+
+# 仅 R1(速度快,良好的基准提升)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True))
+
+# R1 + R2(更好,融合后无运行时开销)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True))
+
+# R1 + R2 + R3 + R4(最佳精度,hook 带来少许运行时开销)
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))
+```
+
+##### 字符串快捷方式
+
+| 值 | 等价配置 |
+|----|---------|
+| `"quarot"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=False, trainable_smooth=False)` — 确定性 Hadamard,无需训练 |
+| `"spinquant"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=True, trainable_smooth=True)` — **实验性**(需要提供 dataloader) |
+
+> ⚠️ **SpinQuant 可训练旋转**(`trainable_rotation=True`)启用通过 Cayley SGD 优化的可学习旋转矩阵。此功能为**实验性**,尚未完全验证。生产环境建议使用 `"quarot"`(固定 Hadamard)。
+
+##### 确定性与随机 Hadamard
+
+```python
+# 确定性(默认):固定 Hadamard 矩阵,无需额外存储
+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))
+
+# 随机:H × diag(±1),离群点抑制效果略好,需保存旋转矩阵
+ar = AutoRound(
+ model_name,
+ scheme="MXFP4",
+ rotation_config=SpinQuantConfig(
+ r1=True, r2=True, r3=True, r4=True, random_r1=True, random_r2=True, random_r3=True, random_r4=True
+ ),
+)
+```
+
+##### 关键参数
+
+| 参数 | 默认值 | 描述 |
+|------|--------|------|
+| `r1` / `r2` / `r3` / `r4` | `True / True / False / False` | 启用各位置的旋转 |
+| `online_r1_rotation` | `True` | R1 通过 hook 应用(`True`)或融合到权重中(`False`) |
+| `random_r1` / `random_r2` / `random_r3` / `random_r4` | `False` | 使用随机 Hadamard(H×diag(±1))而非确定性 |
+| `rotation_size` | `None`(自动) | 块旋转维度;从模型维度自动检测 |
+| `trainable_rotation` | `False` | 启用 SpinQuant 可学习旋转(**实验性**,需要 dataloader) |
+| `trainable_smooth` | `False` | 启用可学习 smooth 值(**实验性**,需要 dataloader) |
+
+##### 保存与加载
+
+带旋转的量化模型可透明地保存和加载:
+
+```python
+# 保存(旋转矩阵在需要时自动存储)
+ar.quantize_and_save(output_dir="./my_model", format="auto_round")
+
+# 加载(旋转 hook 自动恢复)
+from transformers import AutoModelForCausalLM
+
+model = AutoModelForCausalLM.from_pretrained("./my_model", device_map="auto")
+```
+
+- **确定性旋转**:仅存储元数据(类型 + rotation_size)——矩阵在加载时重建
+- **随机旋转**:以 `int8`(±1)旋转矩阵形式存储(大小约为 rotation_size² 字节)
+- **在线旋转**:在模型加载时重建(R1/R4 通过 QuantLinear forward patch;R3 通过配置触发的 monkeypatch)
+
+#### 逐线性层块旋转(实验性)
+
+> ⚠️ 这是较早期的实验性实现,通过对模型中每个 `nn.Linear` 模块应用块对角 Hadamard 旋转来工作。对于大多数场景,推荐使用上述 [QuaRot / SpinQuant](#quarot--spinquant) 方式——它在特定位置(R1–R4)提供架构感知的旋转,具有更好的精度和更低的开销。
+
+逐线性层块旋转通过遍历所有线性层实现,可选择以下模式:
+- **权重模式**:将 Hadamard 矩阵直接融合到权重张量中(离线)
+- **输入模式**:注册 forward pre-hook 在每个线性层前旋转输入激活(在线)
+
+该方式使用可配置的 `block_size`(默认 32),对每个线性层统一应用。它比 QuaRot 更简单,但不够精确——不区分残差流、注意力和 MLP 层,也不处理 RoPE 或激活侧旋转。使用 `rotation_config="default"` 启用。
+
+##### 使用方式
-#### 使用 Hadamard 旋转进行量化
-研究性功能:当前暂无高效可用的 kernel 支持,且社区兼容性通常较低。
```python
from auto_round import AutoRound
-# 加载模型(支持 FP8/BF16/FP16/FP32)
model_name_or_path = "meta-llama/Llama-3.1-8B-Instruct"
output_dir = "./Llama-3.1-8B-Instruct-mxfp4-ht"
@@ -799,19 +903,20 @@ ar = AutoRound(model_name_or_path, scheme="MXFP4", rotation_config="default")
ar.quantize_and_save(output_dir=output_dir, format="auto_round")
```
-#### 变换类
+##### 类型
-| 类名 | 描述 |
-| ------------------------- | --------------------------- |
-| `HadamardTransform` | 应用确定性 Hadamard 变换 |
-| `RandomHadamardTransform` | 应用随机 Hadamard 变换,并可选随机种子 |
+| 类型 | 描述 |
+|------|------|
+| `hadamard`(默认) | 确定性 Hadamard 矩阵(Sylvester 构造,block_size 必须为 2 的幂次) |
+| `random_hadamard` | 随机签名 Hadamard,来自已知矩阵库;支持非 2 的幂次尺寸 |
-#### 参数说明
+##### 参数说明
-| 参数 | 描述 |
-| ------------ | -------------------------------- |
-| `block_size` | 变换块大小(默认:32) |
-| `seed` | 随机种子(用于 RandomHadamardTransform) |
+| 参数 | 默认值 | 描述 |
+|------|--------|------|
+| `block_size` | `32` | 应用于每个线性层的 Hadamard 块大小 |
+| `hadamard_type` | `"hadamard"` | `"hadamard"` 或 `"random_hadamard"` |
+| `seed` | `None` | 随机种子(仅用于 `random_hadamard`) |
## 4 推理部署
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index c60f157e9..cb77b6858 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -8,3 +8,5 @@ pillow
torchvision
numba
sentencepiece
+# Remove after new GPTQModel release, https://github.com/intel/auto-round/issues/1891
+transformers <= 5.9.0
diff --git a/test/test_cuda/transform/test_spinquant.py b/test/test_cuda/transform/test_spinquant.py
index 29545b929..8c5136b20 100644
--- a/test/test_cuda/transform/test_spinquant.py
+++ b/test/test_cuda/transform/test_spinquant.py
@@ -13,6 +13,7 @@ import shutil
import pytest
import torch
+import torch.nn.functional as F
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
@@ -41,15 +42,15 @@ class TestSpinQuantConfig:
assert cfg.r3 is False
assert cfg.r4 is False
assert cfg.online_r1_rotation is True
- assert cfg.trainable_rotation is True
- assert cfg.trainable_smooth is True
-
- def test_quarot_config(self):
- """QuaRot = fixed Hadamard, no training."""
- cfg = SpinQuantConfig(trainable_rotation=False, trainable_smooth=False)
assert cfg.trainable_rotation is False
assert cfg.trainable_smooth is False
+ def test_trainable_config(self):
+ """SpinQuant mode = explicitly enable trainable."""
+ cfg = SpinQuantConfig(trainable_rotation=True, trainable_smooth=True)
+ assert cfg.trainable_rotation is True
+ assert cfg.trainable_smooth is True
+
def test_selective_rotation_levels(self):
cfg = SpinQuantConfig(r1=True, r2=True, r3=False, r4=False)
assert cfg.r1 is True
@@ -85,6 +86,7 @@ class TestNormalizeRotationConfig:
cfg = normalize_rotation_config("spinquant")
assert isinstance(cfg, SpinQuantConfig)
assert cfg.trainable_rotation is True
+ assert cfg.trainable_smooth is True
def test_string_hadamard(self):
"""'hadamard' should return the original RotationConfig, not SpinQuantConfig."""
@@ -364,3 +366,158 @@ class TestPipelineIntegration:
output_dir=self.save_dir + "_dict", format="auto_round"
)
shutil.rmtree(self.save_dir + "_dict", ignore_errors=True)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Trainable Mode Validation Tests
+# ═══════════════════════════════════════════════════════════════════════════════
+
+
+class TestTrainableValidation:
+ """Test that trainable mode correctly requires a dataloader."""
+
+ def test_trainable_without_dataloader_raises(self):
+ """trainable_rotation=True without dataloader should raise ValueError."""
+ model_name = get_model_path("Qwen/Qwen3-0.6B")
+ model = (
+ AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)
+ .cuda()
+ .eval()
+ )
+ cfg = SpinQuantConfig(
+ r1=True,
+ r2=True,
+ r3=False,
+ r4=False,
+ trainable_rotation=True,
+ trainable_smooth=True,
+ )
+ preprocessor = SpinQuantPreprocessor(model, cfg)
+ with pytest.raises(ValueError, match="dataloader required"):
+ preprocessor.preprocess(dataloader=None)
+ del model
+ torch.cuda.empty_cache()
+
+ def test_default_config_no_dataloader_ok(self):
+ """Default config (trainable=False) should work without dataloader."""
+ model_name = get_model_path("Qwen/Qwen3-0.6B")
+ model = (
+ AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)
+ .cuda()
+ .eval()
+ )
+ cfg = SpinQuantConfig(r1=True, r2=True, r3=False, r4=False)
+ preprocessor = SpinQuantPreprocessor(model, cfg)
+ # Should not raise — no dataloader needed for fixed Hadamard
+ preprocessor.preprocess(dataloader=None)
+ del model
+ torch.cuda.empty_cache()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Rotation Equivalence Tests
+# ═══════════════════════════════════════════════════════════════════════════════
+
+
+class TestRotationEquivalence:
+ """Test that rotation is mathematically equivalent (preserves model output).
+
+ All rotation combinations (R1, R2, R3, R4 and their combinations) should
+ preserve functional equivalence:
+ - R1 (online): activation hook + weight compensation → x·R·(R^T·W)^T = x·W^T
+ - R2 (offline): head rotation fused into o_proj/next-layer weights
+ - R3 (online): same rotation on Q and K after RoPE → (Q@R)(K@R)^T = Q@K^T
+ - R4 (online + offline fuse): activation rotation + down_proj compensation
+ """
+
+ MODEL_NAME = None
+ TOKENIZER = None
+ BASELINE_LOGITS = None
+
+ @classmethod
+ def _ensure_baseline(cls):
+ """Compute baseline logits once, reuse across all parametrized tests."""
+ if cls.BASELINE_LOGITS is not None:
+ return
+ cls.MODEL_NAME = get_model_path("Qwen/Qwen3-0.6B")
+ cls.TOKENIZER = AutoTokenizer.from_pretrained(cls.MODEL_NAME, trust_remote_code=True)
+ model = (
+ AutoModelForCausalLM.from_pretrained(cls.MODEL_NAME, torch_dtype=torch.float32, trust_remote_code=True)
+ .cuda()
+ .eval()
+ )
+ inputs = cls.TOKENIZER("The capital of France is", return_tensors="pt").to(model.device)
+ with torch.no_grad():
+ cls.BASELINE_LOGITS = model(**inputs).logits.cpu()
+ del model
+ torch.cuda.empty_cache()
+
+ @staticmethod
+ def _get_logits(model, tokenizer, text="The capital of France is"):
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ with torch.no_grad():
+ return model(**inputs).logits.cpu()
+
+ @pytest.mark.parametrize(
+ "r1,r2,r3,r4,random_r1,random_r2,random_r3,random_r4,label",
+ [
+ # --- Single rotations (deterministic) ---
+ (True, False, False, False, False, False, False, False, "R1"),
+ (False, True, False, False, False, False, False, False, "R2"),
+ (False, False, True, False, False, False, False, False, "R3"),
+ (False, False, False, True, False, False, False, False, "R4"),
+ # --- Pairwise combinations (deterministic) ---
+ (True, True, False, False, False, False, False, False, "R1+R2"),
+ (True, False, True, False, False, False, False, False, "R1+R3"),
+ (True, False, False, True, False, False, False, False, "R1+R4"),
+ (False, False, True, True, False, False, False, False, "R3+R4"),
+ # --- Full combination (deterministic) ---
+ (True, True, True, True, False, False, False, False, "R1+R2+R3+R4"),
+ # --- Random Hadamard variants ---
+ (True, False, False, False, True, False, False, False, "R1-random"),
+ (False, True, False, False, False, True, False, False, "R2-random"),
+ (False, False, True, False, False, False, True, False, "R3-random"),
+ (False, False, False, True, False, False, False, True, "R4-random"),
+ (True, True, True, True, True, True, True, True, "R1+R2+R3+R4-all-random"),
+ ],
+ )
+ def test_rotation_equivalence(self, r1, r2, r3, r4, random_r1, random_r2, random_r3, random_r4, label):
+ """Rotation should preserve model output (functional equivalence)."""
+ self._ensure_baseline()
+
+ model = (
+ AutoModelForCausalLM.from_pretrained(self.MODEL_NAME, torch_dtype=torch.float32, trust_remote_code=True)
+ .cuda()
+ .eval()
+ )
+ cfg = SpinQuantConfig(
+ r1=r1,
+ r2=r2,
+ r3=r3,
+ r4=r4,
+ random_r1=random_r1,
+ random_r2=random_r2,
+ random_r3=random_r3,
+ random_r4=random_r4,
+ online_r1_rotation=True,
+ trainable_rotation=False,
+ trainable_smooth=False,
+ )
+ model = apply_rotation(model, cfg)
+ logits = self._get_logits(model, self.TOKENIZER)
+ del model
+ torch.cuda.empty_cache()
+
+ # Use cosine similarity — robust metric for rotation equivalence on real
+ # models. Online rotation hooks introduce a different float32 computation
+ # path (butterfly ops vs direct matmul), causing expected numerical drift
+ # that accumulates across 28 layers. Max absolute diffs of 0.01–0.15 are
+ # normal for float32; cosine similarity captures structural preservation.
+ cos_sim = F.cosine_similarity(
+ self.BASELINE_LOGITS.flatten().unsqueeze(0).float(),
+ logits.flatten().unsqueeze(0).float(),
+ ).item()
+ max_diff = (self.BASELINE_LOGITS - logits).abs().max().item()
+ assert cos_sim > 0.9999, (
+ f"{label} rotation broke model equivalence: " f"cos_sim = {cos_sim:.6f}, max_diff = {max_diff:.4f}"
+ )