PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix random rotation and update rotation doc. (#1884)
lkk4e2ad7728b34
diff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.pyindex d2b63d9e7..5e48cd8c9 100644--- a/auto_round/algorithms/transforms/__init__.py+++ b/auto_round/algorithms/transforms/__init__.py@@ -127,10 +127,11 @@ def normalize_rotation_config(if key in ("spinquant", "quarot"):from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig- # "quarot" → QuaRot defaults (no training)+ # "quarot" → QuaRot defaults (fixed Hadamard, no training)if key == "quarot":return SpinQuantConfig(trainable_rotation=False, trainable_smooth=False)- return SpinQuantConfig()+ # "spinquant" → experimental trainable mode+ return SpinQuantConfig(trainable_rotation=True, trainable_smooth=True)# Otherwise treat as Hadamard config.return RotationConfig.model_validate(_normalize_hadamard_config(config))diff --git a/auto_round/algorithms/transforms/spinquant/__init__.py b/auto_round/algorithms/transforms/spinquant/__init__.pyindex 9c90d3077..b94ef5dc1 100644--- a/auto_round/algorithms/transforms/spinquant/__init__.py+++ b/auto_round/algorithms/transforms/spinquant/__init__.py@@ -43,8 +43,7 @@ Example (QuaRot — fixed Hadamard, recommended)::config = SpinQuantConfig(r1=True, r2=True, r3=False, r4=False,- trainable_rotation=False, # QuaRot: fixed Hadamard, no training needed- trainable_smooth=False,+ # Default: trainable_rotation=False, trainable_smooth=False (QuaRot mode)online_r1_rotation=True,)SpinQuantPreprocessor(model, config).preprocess() # no dataloader neededdiff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex a1f2cf178..7c2944ea1 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -109,8 +109,8 @@ class SpinQuantConfig(BaseRotationConfig):# ⚠️ trainable_rotation=True (SpinQuant mode) is experimental — training# loop exists but not validated end-to-end. Use trainable_rotation=False# (QuaRot mode) for production use.- trainable_rotation: bool = True # Learn R via Cayley SGD (False = QuaRot fixed Hadamard)- trainable_smooth: bool = True # Learn smooth_values via Adam (joint SmoothQuant)+ trainable_rotation: bool = False # Learn R via Cayley SGD (True = SpinQuant experimental)+ trainable_smooth: bool = False # Learn smooth_values via Adam (joint SmoothQuant)online_r1_rotation: bool = True # Online R1: rotate target weights + hook (Quark default)# Training hyperparameters@@ -730,7 +730,12 @@ class SpinQuantPreprocessor:W = module.weight.data.to(torch.float64)module.weight.data = (W @ R).to(dtype)elif in_features % r1_size == 0:- rotate_in_channels_(module, R_in=R)+ # Block rotation must use W @ R (not W @ R.T) to stay+ # consistent with the online hook (x @ R). The random+ # Hadamard matrix is orthonormal but NOT symmetric, so+ # rotate_in_channels_ (which applies R.T) would break+ # equivalence. Pass R.T so it computes W @ (R.T).T = W @ R.+ rotate_in_channels_(module, R_in=R.T)else:raise ValueError(f"Online R1: in_features={in_features} not compatible " f"with r1_rotation_size={r1_size}"diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 4a2a8052d..ad3b33922 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -34,7 +34,9 @@ This document presents step-by-step instructions for auto-round llm quantization+ [Device/Multi-GPU setting in Quantization](#devicemulti-gpu-setting-in-quantization)- [Enable multiple gpus calibration in lm_head quantization](#enable-multiple-gpus-calibration-in-lm_head-quantization)+ [Adjust Hyperparameters](#adjust-hyperparameters)- + [Hadamard Transform-Research Feature](#hadamard-transform)+ + [Rotation](#rotation)+ - [QuaRot / SpinQuant](#quarot--spinquant)+ - [Per-Linear Block Rotation (Experimental)](#per-linear-block-rotation-experimental)* [4 Inference](#4-inference)+ [CPU](#cpu)+ [Intel GPU](#intel-gpu)@@ -800,31 +802,131 @@ autoround.save_quantized(format="auto_awq", output_dir="tmp_autoround")Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.-### Hadamard Transform+### Rotation-**Research feature with no effective kernels currently available and typically low community adoption.**+AutoRound supports rotation-based transforms to improve quantization accuracy. Rotation redistributes outliers in weights and activations before quantization, making the distribution more uniform and quantization-friendly.-AutoRound supports Hadamard transform as an optional weight/activation transformation technique, which can improve quantization accuracy by rotating the weight/activation matrix. This is particularly useful for certain quantization scenarios.+Two rotation approaches are available:-#### Overview+- **QuaRot / SpinQuant** — full-model rotation across multiple positions (R1–R4), providing comprehensive outlier suppression for aggressive quantization (e.g., MXFP4, NVFP4, W4A4). **Recommended for most use cases.**+- **Per-Linear Block Rotation** — block-diagonal rotation applied uniformly to every linear layer. An earlier experimental implementation; the QuaRot/SpinQuant approach above is generally preferred.-The Hadamard transform is particularly useful in scenarios where activation outliers hurt quantization accuracy. In practice, it helps suppress such outliers, making it especially effective when `act_bits < 8`. Users can enable this feature when they need more stable activation distributions and better accuracy in low‑bit quantization settings.+#### QuaRot / SpinQuant-#### Implementation+QuaRot applies deterministic Hadamard rotations at up to 4 positions in the transformer architecture. Unlike the per-linear block rotation, it operates at the model architecture level — rotating the residual stream, attention heads, and MLP activations at specific positions for targeted outlier suppression.-AutoRound provides two types of Hadamard transforms:+##### Rotation Positions-1. **Deterministic Hadamard Transform** (`hadamard`): Uses Sylvester's construction to create a deterministic Hadamard matrix. The size must be a power of 2.+| Position | Target | Mode | Effect |+|----------|--------|------|--------|+| **R1** | Residual stream (hidden_size) | Online or Offline | Smooths weight outliers across all linear layers |+| **R2** | V/O projections (head_dim) | Offline (fused) | Balances per-head value distributions |+| **R3** | Q/K after RoPE (head_dim) | Online (hook) | Improves KV-cache quantization friendliness |+| **R4** | MLP up/down (intermediate_size) | Online (hook) | Suppresses activation outliers in FFN |-2. **Random Hadamard Transform** (`random_hadamard`): Uses known Hadamard matrices from N. J. A. Sloane's Library of Hadamard Matrices. Supports non-power-of-2 sizes and deterministic seeding.+- **Online**: Applied at runtime via forward hooks (no weight modification, zero-cost at save time)+- **Offline (fused)**: Absorbed into adjacent weight matrices (no runtime overhead)+##### Quick Start-#### Quantization with Hadamard Transform+```python+from auto_round import AutoRound++model_name = "Qwen/Qwen3-0.6B"+output_dir = "./Qwen3-0.6B-mxfp4-quarot"++# QuaRot preset: R1+R2+R3+R4 with deterministic Hadamard+ar = AutoRound(model_name, scheme="MXFP4", rotation_config="quarot")+ar.quantize_and_save(output_dir=output_dir, format="auto_round")+```++##### Rotation Levels++Choose how many positions to rotate based on your accuracy needs:++```python+from auto_round import AutoRound+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig++# R1 only (fast, good baseline improvement)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True))++# R1 + R2 (better, no runtime overhead after fuse)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True))++# R1 + R2 + R3 + R4 (best accuracy, slight runtime overhead from hooks)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))+```++##### String Shortcuts++| Value | Equivalent |+|-------|-----------|+| `"quarot"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=False, trainable_smooth=False)` — deterministic Hadamard, no training |+| `"spinquant"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=True, trainable_smooth=True)` — **experimental** (requires a dataloader) |++> ⚠️ **SpinQuant trainable rotation** (`trainable_rotation=True`) enables learnable rotation matrices optimized via Cayley SGD. This feature is **experimental** and not fully validated. Use `"quarot"` (fixed Hadamard) for production workloads.++##### Deterministic vs Random Hadamard++```python+# Deterministic (default): fixed Hadamard matrix, no extra storage needed+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))++# Random: H × diag(±1), slightly better outlier suppression, requires saving the rotation matrix+ar = AutoRound(+ model_name,+ scheme="MXFP4",+ rotation_config=SpinQuantConfig(+ r1=True, r2=True, r3=True, r4=True, random_r1=True, random_r2=True, random_r3=True, random_r4=True+ ),+)+```++##### Key Parameters++| Parameter | Default | Description |+|-----------|---------|-------------|+| `r1` / `r2` / `r3` / `r4` | `True / True / False / False` | Enable rotation at each position |+| `online_r1_rotation` | `True` | R1 via hook (`True`) or fused into weights (`False`) |+| `random_r1` / `random_r2` / `random_r3` / `random_r4` | `False` | Use random Hadamard (H×diag(±1)) instead of deterministic |+| `rotation_size` | `None` (auto) | Block rotation dimension; auto-detected from model dimensions |+| `trainable_rotation` | `False` | Enable SpinQuant learnable rotation (**experimental**, requires dataloader) |+| `trainable_smooth` | `False` | Enable learnable smooth values (**experimental**, requires dataloader) |++##### Save & Load++Quantized models with rotation are saved and loaded transparently:++```python+# Save (rotation matrices stored automatically if needed)+ar.quantize_and_save(output_dir="./my_model", format="auto_round")++# Load (rotation hooks restored automatically)+from transformers import AutoModelForCausalLM++model = AutoModelForCausalLM.from_pretrained("./my_model", device_map="auto")+```++- **Deterministic rotations**: Only metadata (type + rotation_size) is stored — matrices are reconstructed on load+- **Random rotations**: An `int8` (±1) rotation matrix is stored (size ~rotation_size² bytes)+- **Online rotations**: Rebuilt during model loading (R1/R4 via QuantLinear forward patching; R3 via monkeypatch from config)++#### Per-Linear Block Rotation (Experimental)++> ⚠️ This is an earlier experimental implementation that applies block-diagonal Hadamard rotation **per linear layer** by patching every `nn.Linear` module in the model. For most use cases, the [QuaRot / SpinQuant](#quarot--spinquant) approach above is preferred — it provides architecture-aware rotation at specific positions (R1–R4) with better accuracy and lower overhead.++The per-linear block rotation works by iterating over all linear layers and either:+- **Weight mode**: Fusing the Hadamard matrix directly into the weight tensor (offline)+- **Input mode**: Registering a forward pre-hook that rotates the input activation before each linear layer (online)++This approach uses a configurable `block_size` (default 32) applied uniformly to every linear layer. It is simpler than QuaRot but less targeted — it does not distinguish between residual stream, attention, and MLP layers, nor does it handle RoPE or activation-side rotation. Use `rotation_config="default"` to enable it.++##### Usage```pythonfrom auto_round import AutoRound-# Load a model (supports FP8/BF16/FP16/FP32)model_name_or_path = "meta-llama/Llama-3.1-8B-Instruct"output_dir = "./Llama-3.1-8B-Instruct-mxfp4-ht"@@ -834,19 +936,20 @@ ar = AutoRound(model_name_or_path, scheme="MXFP4", rotation_config="default")ar.quantize_and_save(output_dir=output_dir, format="auto_round")```-#### Transform Classes+##### Types-| Class | Description |-|-------|-------------|-| `HadamardTransform` | Applies deterministic Hadamard transform |-| `RandomHadamardTransform` | Applies random Hadamard transform with optional seeding |+| Type | Description |+|------|-------------|+| `hadamard` (default) | Deterministic Hadamard matrix (Sylvester construction, block_size must be power of 2) |+| `random_hadamard` | Randomly signed Hadamard from known matrix library; supports non-power-of-2 sizes |-#### Parameters+##### Parameters-| Parameter | Description |-|-----------|-------------|-| `block_size` | Size of the transformation block (default: 32) |-| `seed` | Random seed (for RandomHadamardTransform) |+| Parameter | Default | Description |+|-----------|---------|-------------|+| `block_size` | `32` | Size of the Hadamard block applied to each linear layer |+| `hadamard_type` | `"hadamard"` | `"hadamard"` or `"random_hadamard"` |+| `seed` | `None` | Random seed (for `random_hadamard` only) |## 4 Inferencediff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex cf4347df0..8677fe185 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -35,7 +35,9 @@- [lm_head 量化中开启多 GPU 标定](#lm_head-量化中开启多-gpu-标定)- [手动配置设备映射](#手动配置设备映射)+ [超参数调整](#超参数调整)- + [Hadamard变换-研究功能](#hadamard变换)+ + [旋转(Rotation)](#旋转rotation)+ - [QuaRot / SpinQuant](#quarot--spinquant)+ - [逐线性层块旋转(实验性)](#逐线性层块旋转实验性)* [4 推理部署](#4-推理部署)+ [CPU](#cpu)+ [英特尔 GPU](#英特尔-gpu)@@ -767,29 +769,131 @@ auto-round --model_name Qwen/Qwen3-0.6B --scheme "W4A16" --quant_lm_head --form#### 使用 AdamW 优化器添加 `--adam` 参数即可启用;**注意**:在我们的多项测试场景中,AdamW 优化器的效果均不如符号梯度下降(sign gradient descent)。+### 旋转(Rotation)+AutoRound 支持基于旋转的变换技术来提升量化精度。旋转在量化前对权重和激活中的离群点进行重分布,使分布更加均匀,从而对量化更友好。-### Hadamard变换+目前提供两种旋转方式:-AutoRound 支持将 Hadamard 变换作为可选的权重/激活变换技术,通过旋转权重/激活矩阵来提升量化精度。这在某些量化场景中尤其有用。+- **QuaRot / SpinQuant** — 在多个位置(R1–R4)进行全模型旋转,为激进量化(如 MXFP4、NVFP4、W4A4)提供全面的离群点抑制。**推荐在大多数场景中使用。**+- **逐线性层块旋转(Per-Linear Block Rotation)** — 对每个线性层均匀应用块对角旋转。为较早期的实验性实现,通常建议使用上述 QuaRot/SpinQuant 方式。-#### 概述+#### QuaRot / SpinQuant-Hadamard 变换在激活值存在离群点且影响量化精度的场景中特别有效。在实践中,它能够抑制这些离群点,因此在 `act_bits < 8` 的低比特激活量化设置中效果尤为显著。用户可以在需要更稳定的激活分布和更高低比特精度时启用该功能。+QuaRot 在 Transformer 架构的最多 4 个位置应用确定性 Hadamard 旋转。与逐线性层块旋转不同,它在模型架构级别操作——在特定位置对残差流、注意力头和 MLP 激活进行旋转,以实现针对性的离群点抑制。-#### 实现方式+##### 旋转位置-AutoRound 提供两种类型的 Hadamard 变换:+| 位置 | 目标 | 模式 | 效果 |+|------|------|------|------|+| **R1** | 残差流(hidden_size) | 在线或离线 | 平滑所有线性层的权重离群点 |+| **R2** | V/O 投影(head_dim) | 离线(融合) | 平衡各注意力头的值分布 |+| **R3** | RoPE 后的 Q/K(head_dim) | 在线(hook) | 提升 KV-cache 的量化友好性 |+| **R4** | MLP up/down(intermediate_size) | 在线(hook) | 抑制 FFN 中的激活离群点 |-1. **确定性 Hadamard 变换**(`hadamard`):使用 Sylvester 构造法生成确定性的 Hadamard 矩阵,尺寸必须为 2 的幂次。-2. **随机 Hadamard 变换**(`random_hadamard`):使用 N. J. A. Sloane 的 Hadamard 矩阵库中已知的矩阵。支持非 2 的幂次的尺寸,并支持确定性随机种子。+- **在线(Online)**:通过 forward hook 在运行时应用(不修改权重,保存时零开销)+- **离线/融合(Offline/Fused)**:吸收到相邻权重矩阵中(无运行时开销)++##### 快速开始++```python+from auto_round import AutoRound++model_name = "Qwen/Qwen3-0.6B"+output_dir = "./Qwen3-0.6B-mxfp4-quarot"++# QuaRot 预设:R1+R2+R3+R4,确定性 Hadamard+ar = AutoRound(model_name, scheme="MXFP4", rotation_config="quarot")+ar.quantize_and_save(output_dir=output_dir, format="auto_round")+```++##### 旋转级别++根据精度需求选择旋转的位置数量:++```python+from auto_round import AutoRound+from auto_round.algorithms.transforms.spinquant import SpinQuantConfig++# 仅 R1(速度快,良好的基准提升)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True))++# R1 + R2(更好,融合后无运行时开销)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True))++# R1 + R2 + R3 + R4(最佳精度,hook 带来少许运行时开销)+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))+```++##### 字符串快捷方式++| 值 | 等价配置 |+|----|---------|+| `"quarot"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=False, trainable_smooth=False)` — 确定性 Hadamard,无需训练 |+| `"spinquant"` | `SpinQuantConfig(r1=True, r2=True, trainable_rotation=True, trainable_smooth=True)` — **实验性**(需要提供 dataloader) |++> ⚠️ **SpinQuant 可训练旋转**(`trainable_rotation=True`)启用通过 Cayley SGD 优化的可学习旋转矩阵。此功能为**实验性**,尚未完全验证。生产环境建议使用 `"quarot"`(固定 Hadamard)。++##### 确定性与随机 Hadamard++```python+# 确定性(默认):固定 Hadamard 矩阵,无需额外存储+ar = AutoRound(model_name, scheme="MXFP4", rotation_config=SpinQuantConfig(r1=True, r2=True, r3=True, r4=True))++# 随机:H × diag(±1),离群点抑制效果略好,需保存旋转矩阵+ar = AutoRound(+ model_name,+ scheme="MXFP4",+ rotation_config=SpinQuantConfig(+ r1=True, r2=True, r3=True, r4=True, random_r1=True, random_r2=True, random_r3=True, random_r4=True+ ),+)+```++##### 关键参数++| 参数 | 默认值 | 描述 |+|------|--------|------|+| `r1` / `r2` / `r3` / `r4` | `True / True / False / False` | 启用各位置的旋转 |+| `online_r1_rotation` | `True` | R1 通过 hook 应用(`True`)或融合到权重中(`False`) |+| `random_r1` / `random_r2` / `random_r3` / `random_r4` | `False` | 使用随机 Hadamard(H×diag(±1))而非确定性 |+| `rotation_size` | `None`(自动) | 块旋转维度;从模型维度自动检测 |+| `trainable_rotation` | `False` | 启用 SpinQuant 可学习旋转(**实验性**,需要 dataloader) |+| `trainable_smooth` | `False` | 启用可学习 smooth 值(**实验性**,需要 dataloader) |++##### 保存与加载++带旋转的量化模型可透明地保存和加载:++```python+# 保存(旋转矩阵在需要时自动存储)+ar.quantize_and_save(output_dir="./my_model", format="auto_round")++# 加载(旋转 hook 自动恢复)+from transformers import AutoModelForCausalLM++model = AutoModelForCausalLM.from_pretrained("./my_model", device_map="auto")+```++- **确定性旋转**:仅存储元数据(类型 + rotation_size)——矩阵在加载时重建+- **随机旋转**:以 `int8`(±1)旋转矩阵形式存储(大小约为 rotation_size² 字节)+- **在线旋转**:在模型加载时重建(R1/R4 通过 QuantLinear forward patch;R3 通过配置触发的 monkeypatch)++#### 逐线性层块旋转(实验性)++> ⚠️ 这是较早期的实验性实现,通过对模型中每个 `nn.Linear` 模块应用块对角 Hadamard 旋转来工作。对于大多数场景,推荐使用上述 [QuaRot / SpinQuant](#quarot--spinquant) 方式——它在特定位置(R1–R4)提供架构感知的旋转,具有更好的精度和更低的开销。++逐线性层块旋转通过遍历所有线性层实现,可选择以下模式:+- **权重模式**:将 Hadamard 矩阵直接融合到权重张量中(离线)+- **输入模式**:注册 forward pre-hook 在每个线性层前旋转输入激活(在线)++该方式使用可配置的 `block_size`(默认 32),对每个线性层统一应用。它比 QuaRot 更简单,但不够精确——不区分残差流、注意力和 MLP 层,也不处理 RoPE 或激活侧旋转。使用 `rotation_config="default"` 启用。++##### 使用方式-#### 使用 Hadamard 旋转进行量化-研究性功能:当前暂无高效可用的 kernel 支持,且社区兼容性通常较低。```pythonfrom auto_round import AutoRound-# 加载模型(支持 FP8/BF16/FP16/FP32)model_name_or_path = "meta-llama/Llama-3.1-8B-Instruct"output_dir = "./Llama-3.1-8B-Instruct-mxfp4-ht"@@ -799,19 +903,20 @@ ar = AutoRound(model_name_or_path, scheme="MXFP4", rotation_config="default")ar.quantize_and_save(output_dir=output_dir, format="auto_round")```-#### 变换类+##### 类型-| 类名 | 描述 |-| ------------------------- | --------------------------- |-| `HadamardTransform` | 应用确定性 Hadamard 变换 |-| `RandomHadamardTransform` | 应用随机 Hadamard 变换,并可选随机种子 |+| 类型 | 描述 |+|------|------|+| `hadamard`(默认) | 确定性 Hadamard 矩阵(Sylvester 构造,block_size 必须为 2 的幂次) |+| `random_hadamard` | 随机签名 Hadamard,来自已知矩阵库;支持非 2 的幂次尺寸 |-#### 参数说明+##### 参数说明-| 参数 | 描述 |-| ------------ | -------------------------------- |-| `block_size` | 变换块大小(默认:32) |-| `seed` | 随机种子(用于 RandomHadamardTransform) |+| 参数 | 默认值 | 描述 |+|------|--------|------|+| `block_size` | `32` | 应用于每个线性层的 Hadamard 块大小 |+| `hadamard_type` | `"hadamard"` | `"hadamard"` 或 `"random_hadamard"` |+| `seed` | `None` | 随机种子(仅用于 `random_hadamard`) |## 4 推理部署diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txtindex c60f157e9..cb77b6858 100644--- a/test/test_cuda/requirements.txt+++ b/test/test_cuda/requirements.txt@@ -8,3 +8,5 @@ pillowtorchvisionnumbasentencepiece+# Remove after new GPTQModel release, https://github.com/intel/auto-round/issues/1891+transformers <= 5.9.0diff --git a/test/test_cuda/transform/test_spinquant.py b/test/test_cuda/transform/test_spinquant.pyindex 29545b929..8c5136b20 100644--- a/test/test_cuda/transform/test_spinquant.py+++ b/test/test_cuda/transform/test_spinquant.py@@ -13,6 +13,7 @@ import shutilimport pytestimport torch+import torch.nn.functional as Ffrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound@@ -41,15 +42,15 @@ class TestSpinQuantConfig:assert cfg.r3 is Falseassert cfg.r4 is Falseassert cfg.online_r1_rotation is True- assert cfg.trainable_rotation is True- assert cfg.trainable_smooth is True-- def test_quarot_config(self):- """QuaRot = fixed Hadamard, no training."""- cfg = SpinQuantConfig(trainable_rotation=False, trainable_smooth=False)assert cfg.trainable_rotation is Falseassert cfg.trainable_smooth is False+ def test_trainable_config(self):+ """SpinQuant mode = explicitly enable trainable."""+ cfg = SpinQuantConfig(trainable_rotation=True, trainable_smooth=True)+ assert cfg.trainable_rotation is True+ assert cfg.trainable_smooth is True+def test_selective_rotation_levels(self):cfg = SpinQuantConfig(r1=True, r2=True, r3=False, r4=False)assert cfg.r1 is True@@ -85,6 +86,7 @@ class TestNormalizeRotationConfig:cfg = normalize_rotation_config("spinquant")assert isinstance(cfg, SpinQuantConfig)assert cfg.trainable_rotation is True+ assert cfg.trainable_smooth is Truedef test_string_hadamard(self):"""'hadamard' should return the original RotationConfig, not SpinQuantConfig."""@@ -364,3 +366,158 @@ class TestPipelineIntegration:output_dir=self.save_dir + "_dict", format="auto_round")shutil.rmtree(self.save_dir + "_dict", ignore_errors=True)+++# ═══════════════════════════════════════════════════════════════════════════════+# Trainable Mode Validation Tests+# ═══════════════════════════════════════════════════════════════════════════════+++class TestTrainableValidation:+ """Test that trainable mode correctly requires a dataloader."""++ def test_trainable_without_dataloader_raises(self):+ """trainable_rotation=True without dataloader should raise ValueError."""+ model_name = get_model_path("Qwen/Qwen3-0.6B")+ model = (+ AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)+ .cuda()+ .eval()+ )+ cfg = SpinQuantConfig(+ r1=True,+ r2=True,+ r3=False,+ r4=False,+ trainable_rotation=True,+ trainable_smooth=True,+ )+ preprocessor = SpinQuantPreprocessor(model, cfg)+ with pytest.raises(ValueError, match="dataloader required"):+ preprocessor.preprocess(dataloader=None)+ del model+ torch.cuda.empty_cache()++ def test_default_config_no_dataloader_ok(self):+ """Default config (trainable=False) should work without dataloader."""+ model_name = get_model_path("Qwen/Qwen3-0.6B")+ model = (+ AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)+ .cuda()+ .eval()+ )+ cfg = SpinQuantConfig(r1=True, r2=True, r3=False, r4=False)+ preprocessor = SpinQuantPreprocessor(model, cfg)+ # Should not raise — no dataloader needed for fixed Hadamard+ preprocessor.preprocess(dataloader=None)+ del model+ torch.cuda.empty_cache()+++# ═══════════════════════════════════════════════════════════════════════════════+# Rotation Equivalence Tests+# ═══════════════════════════════════════════════════════════════════════════════+++class TestRotationEquivalence:+ """Test that rotation is mathematically equivalent (preserves model output).++ All rotation combinations (R1, R2, R3, R4 and their combinations) should+ preserve functional equivalence:+ - R1 (online): activation hook + weight compensation → x·R·(R^T·W)^T = x·W^T+ - R2 (offline): head rotation fused into o_proj/next-layer weights+ - R3 (online): same rotation on Q and K after RoPE → (Q@R)(K@R)^T = Q@K^T+ - R4 (online + offline fuse): activation rotation + down_proj compensation+ """++ MODEL_NAME = None+ TOKENIZER = None+ BASELINE_LOGITS = None++ @classmethod+ def _ensure_baseline(cls):+ """Compute baseline logits once, reuse across all parametrized tests."""+ if cls.BASELINE_LOGITS is not None:+ return+ cls.MODEL_NAME = get_model_path("Qwen/Qwen3-0.6B")+ cls.TOKENIZER = AutoTokenizer.from_pretrained(cls.MODEL_NAME, trust_remote_code=True)+ model = (+ AutoModelForCausalLM.from_pretrained(cls.MODEL_NAME, torch_dtype=torch.float32, trust_remote_code=True)+ .cuda()+ .eval()+ )+ inputs = cls.TOKENIZER("The capital of France is", return_tensors="pt").to(model.device)+ with torch.no_grad():+ cls.BASELINE_LOGITS = model(**inputs).logits.cpu()+ del model+ torch.cuda.empty_cache()++ @staticmethod+ def _get_logits(model, tokenizer, text="The capital of France is"):+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ with torch.no_grad():+ return model(**inputs).logits.cpu()++ @pytest.mark.parametrize(+ "r1,r2,r3,r4,random_r1,random_r2,random_r3,random_r4,label",+ [+ # --- Single rotations (deterministic) ---+ (True, False, False, False, False, False, False, False, "R1"),+ (False, True, False, False, False, False, False, False, "R2"),+ (False, False, True, False, False, False, False, False, "R3"),+ (False, False, False, True, False, False, False, False, "R4"),+ # --- Pairwise combinations (deterministic) ---+ (True, True, False, False, False, False, False, False, "R1+R2"),+ (True, False, True, False, False, False, False, False, "R1+R3"),+ (True, False, False, True, False, False, False, False, "R1+R4"),+ (False, False, True, True, False, False, False, False, "R3+R4"),+ # --- Full combination (deterministic) ---+ (True, True, True, True, False, False, False, False, "R1+R2+R3+R4"),+ # --- Random Hadamard variants ---+ (True, False, False, False, True, False, False, False, "R1-random"),+ (False, True, False, False, False, True, False, False, "R2-random"),+ (False, False, True, False, False, False, True, False, "R3-random"),+ (False, False, False, True, False, False, False, True, "R4-random"),+ (True, True, True, True, True, True, True, True, "R1+R2+R3+R4-all-random"),+ ],+ )+ def test_rotation_equivalence(self, r1, r2, r3, r4, random_r1, random_r2, random_r3, random_r4, label):+ """Rotation should preserve model output (functional equivalence)."""+ self._ensure_baseline()++ model = (+ AutoModelForCausalLM.from_pretrained(self.MODEL_NAME, torch_dtype=torch.float32, trust_remote_code=True)+ .cuda()+ .eval()+ )+ cfg = SpinQuantConfig(+ r1=r1,+ r2=r2,+ r3=r3,+ r4=r4,+ random_r1=random_r1,+ random_r2=random_r2,+ random_r3=random_r3,+ random_r4=random_r4,+ online_r1_rotation=True,+ trainable_rotation=False,+ trainable_smooth=False,+ )+ model = apply_rotation(model, cfg)+ logits = self._get_logits(model, self.TOKENIZER)+ del model+ torch.cuda.empty_cache()++ # Use cosine similarity — robust metric for rotation equivalence on real+ # models. Online rotation hooks introduce a different float32 computation+ # path (butterfly ops vs direct matmul), causing expected numerical drift+ # that accumulates across 28 layers. Max absolute diffs of 0.01–0.15 are+ # normal for float32; cosine similarity captures structural preservation.+ cos_sim = F.cosine_similarity(+ self.BASELINE_LOGITS.flatten().unsqueeze(0).float(),+ logits.flatten().unsqueeze(0).float(),+ ).item()+ max_diff = (self.BASELINE_LOGITS - logits).abs().max().item()+ assert cos_sim > 0.9999, (+ f"{label} rotation broke model equivalence: " f"cos_sim = {cos_sim:.6f}, max_diff = {max_diff:.4f}"+ )