Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refactor quarot/spinquant rotation with simplying code. (#1849)

lkkc73f36b59139
diff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.py
index 8a707cab1..d2b63d9e7 100644
--- a/auto_round/algorithms/transforms/__init__.py
+++ b/auto_round/algorithms/transforms/__init__.py
@@ -47,7 +47,7 @@ import torch
from auto_round.algorithms.transforms.base import (
BaseRotation,
BaseRotationConfig,
- RotationSerializer,
+ SerializerMixin,
ROTATION_SUPPORTED_SCHEMES,
check_supported_schemes,
_ensure_registry_populated,
@@ -63,7 +63,7 @@ __all__ = [
# Base interfaces
"BaseRotation",
"BaseRotationConfig",
- "RotationSerializer",
+ "SerializerMixin",
"ROTATION_SUPPORTED_SCHEMES",
"check_supported_schemes",
# Config
@@ -185,8 +185,8 @@ import logging as _logging
_dispatch_logger = _logging.getLogger("autoround.transforms.dispatch")
-def _get_serializer(model: torch.nn.Module) -> RotationSerializer | None:
- """Get the :class:`RotationSerializer` for the model's rotation config.
+def _get_serializer(model: torch.nn.Module) -> SerializerMixin | None:
+ """Get the :class:`SerializerMixin` for the model's rotation config.
Returns ``None`` if no rotation was applied or the rotation algorithm
does not support serialization.
@@ -200,15 +200,15 @@ def _get_serializer(model: torch.nn.Module) -> RotationSerializer | None:
rotation = BaseRotation.from_config(config)
except (ValueError, KeyError):
return None
- if not isinstance(rotation, RotationSerializer):
+ if not isinstance(rotation, SerializerMixin):
return None
return rotation
def _get_serializer_for_config(
config_dict: dict,
-) -> RotationSerializer | None:
- """Get :class:`RotationSerializer` from a JSON config dict (load side).
+) -> SerializerMixin | None:
+ """Get :class:`SerializerMixin` from a JSON config dict (load side).
The dict should have an ``"algorithm"`` field; defaults to
``"spinquant"`` for backward compatibility.
@@ -224,7 +224,7 @@ def _get_serializer_for_config(
rotation = rotation_cls(config)
except Exception:
return None
- if not isinstance(rotation, RotationSerializer):
+ if not isinstance(rotation, SerializerMixin):
return None
return rotation
@@ -239,7 +239,7 @@ def inject_rotation_buffers_on_layer(
) -> None:
"""Per-layer rotation buffer injection (ShardWriter path).
- Dispatches to the correct :class:`RotationSerializer` based on
+ Dispatches to the correct :class:`SerializerMixin` based on
``model._rotation_config``. No-op if no rotation config is present.
Called from ``pack_layer()`` in all export files.
@@ -255,7 +255,7 @@ def inject_rotation_buffers_bulk(
) -> None:
"""Bulk rotation buffer injection (non-ShardWriter path).
- Dispatches to the correct :class:`RotationSerializer` based on
+ Dispatches to the correct :class:`SerializerMixin` based on
``model._rotation_config``. No-op if no rotation config is present.
Called from ``save_quantized_as_*()`` in all export files.
@@ -271,7 +271,7 @@ def save_rotation_config(
) -> None:
"""Persist rotation config to ``config.json``.
- Dispatches to the correct :class:`RotationSerializer` based on
+ Dispatches to the correct :class:`SerializerMixin` based on
``model._rotation_config``. No-op if no rotation config is present.
Called from ``save_quantized_as_*()`` in all export files, after
@@ -292,7 +292,7 @@ def preregister_rotation_buffers(
"""Pre-register empty rotation buffers before state_dict loading.
Extracts the rotation config dict from *quantization_config* by
- checking each registered :class:`RotationSerializer`'s
+ checking each registered :class:`SerializerMixin`'s
:meth:`config_key` (e.g. ``"spinquant_config"``).
Note: does NOT use ``"rotation_config"`` — that key is reserved for
@@ -307,7 +307,7 @@ def preregister_rotation_buffers(
if isinstance(quantization_config, dict):
for _name, _cls in BaseRotation._REGISTRY.items():
- if not (isinstance(_cls, type) and issubclass(_cls, RotationSerializer)):
+ if not (isinstance(_cls, type) and issubclass(_cls, SerializerMixin)):
continue
try:
legacy_key = _cls.config_key()
@@ -322,7 +322,7 @@ def preregister_rotation_buffers(
# Object-style quantization_config (e.g. QuantizationConfig)
# Check each registered serializer's config_key as attribute
for _name, _cls in BaseRotation._REGISTRY.items():
- if not (isinstance(_cls, type) and issubclass(_cls, RotationSerializer)):
+ if not (isinstance(_cls, type) and issubclass(_cls, SerializerMixin)):
continue
try:
legacy_key = _cls.config_key()
@@ -347,7 +347,7 @@ def preregister_rotation_buffers(
def rebuild_rotation_if_needed(model: torch.nn.Module) -> None:
"""Rebuild online rotation hooks after weights are loaded.
- Scans all registered :class:`RotationSerializer` implementations to
+ Scans all registered :class:`SerializerMixin` implementations to
find one whose buffers are present on the model, then calls its
:meth:`rebuild_online`.
"""
@@ -358,7 +358,7 @@ def rebuild_rotation_if_needed(model: torch.nn.Module) -> None:
temp = rotation_cls(BaseRotationConfig(algorithm=name))
except Exception:
continue
- if not isinstance(temp, RotationSerializer):
+ if not isinstance(temp, SerializerMixin):
continue
# Quick scan: does any module have this method's buffers?
diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.py
index b18debc3e..db48c6059 100644
--- a/auto_round/algorithms/transforms/base.py
+++ b/auto_round/algorithms/transforms/base.py
@@ -172,7 +172,7 @@ def _ensure_registry_populated() -> None:
# ---------------------------------------------------------------------------
-class RotationSerializer(ABC):
+class SerializerMixin(ABC):
"""Mixin interface for rotation method serialization.
Any :class:`BaseRotation` subclass that needs save/load support should
@@ -185,7 +185,7 @@ class RotationSerializer(ABC):
Usage::
@BaseRotation.register("my_rotation")
- class MyRotation(BaseRotation, RotationSerializer):
+ class MyRotation(BaseRotation, SerializerMixin):
def apply_to_model(self, ...): ...
def inject_buffers_on_layer(self, ...): ...
...
diff --git a/auto_round/algorithms/transforms/spinquant/__init__.py b/auto_round/algorithms/transforms/spinquant/__init__.py
index ab8426994..9c90d3077 100644
--- a/auto_round/algorithms/transforms/spinquant/__init__.py
+++ b/auto_round/algorithms/transforms/spinquant/__init__.py
@@ -70,7 +70,7 @@ Example (SpinQuant — trainable, ⚠️ experimental)::
autoround.quantize()
"""
-from auto_round.algorithms.transforms.spinquant.algorithm import (
+from auto_round.algorithms.transforms.spinquant.apply import (
SpinQuantRotation,
)
from auto_round.algorithms.transforms.spinquant.cayley_optimizer import (
@@ -87,12 +87,16 @@ from auto_round.algorithms.transforms.spinquant.preprocessor import (
SpinQuantPreprocessor,
TrainableRMSNorm,
)
-from auto_round.algorithms.transforms.spinquant.trainer import (
+from auto_round.algorithms.transforms.spinquant.training import (
LossLogger,
OrthogonalityMonitor,
RotationTrainer,
RotationTrainerCallback,
RotationTrainerConfig,
+ SpinQuantState,
+ SpinQuantTrainingHook,
+ create_spinquant_optimizer,
+ spinquant_loss_fn,
)
from auto_round.algorithms.transforms.spinquant.rotation_utils import (
InputRotationWrapperHadamard,
@@ -102,12 +106,6 @@ from auto_round.algorithms.transforms.spinquant.serialize import (
rebuild_spinquant_online,
save_spinquant_config,
)
-from auto_round.algorithms.transforms.spinquant.training import (
- SpinQuantState,
- SpinQuantTrainingHook,
- create_spinquant_optimizer,
- spinquant_loss_fn,
-)
__all__ = [
# -- Registry algorithm (unified apply_rotation() entry) --
diff --git a/auto_round/algorithms/transforms/spinquant/algorithm.py b/auto_round/algorithms/transforms/spinquant/apply.py
similarity index 95%
rename from auto_round/algorithms/transforms/spinquant/algorithm.py
rename to auto_round/algorithms/transforms/spinquant/apply.py
index 13b60e96d..e8e4c8f68 100644
--- a/auto_round/algorithms/transforms/spinquant/algorithm.py
+++ b/auto_round/algorithms/transforms/spinquant/apply.py
@@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""SpinQuant / QuaRot rotation — ``BaseRotation`` + ``RotationSerializer``.
+"""SpinQuant / QuaRot rotation — ``BaseRotation`` + ``SerializerMixin``.
Registers ``"spinquant"`` in the :class:`BaseRotation` registry so that
the unified ``apply_rotation(model, config)`` entry point can dispatch to
the SpinQuant preprocessing pipeline automatically.
-Also implements :class:`RotationSerializer` so that the generic dispatch
+Also implements :class:`SerializerMixin` so that the generic dispatch
functions (``inject_rotation_buffers_on_layer``, etc.) route to SpinQuant
serialization logic without export files ever naming "spinquant".
@@ -50,13 +50,13 @@ from typing import Any
import torch
import torch.nn as nn
-from auto_round.algorithms.transforms.base import BaseRotation, RotationSerializer
+from auto_round.algorithms.transforms.base import BaseRotation, SerializerMixin
logger = logging.getLogger("autoround")
@BaseRotation.register("spinquant")
-class SpinQuantRotation(BaseRotation, RotationSerializer):
+class SpinQuantRotation(BaseRotation, SerializerMixin):
"""QuaRot / SpinQuant rotation registered as ``"spinquant"`` in
:class:`BaseRotation`.
@@ -64,7 +64,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):
pipeline (RMSNorm fusion → rotation matrix init → optional training →
weight fusion → online hook registration).
- Implements :class:`RotationSerializer` for save/load support — all
+ Implements :class:`SerializerMixin` for save/load support — all
serialization calls are delegated to the existing ``serialize.py``
functions.
@@ -107,7 +107,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):
return preprocessor.preprocess(dataloader)
# ------------------------------------------------------------------
- # RotationSerializer — Save side
+ # SerializerMixin — Save side
# ------------------------------------------------------------------
def inject_buffers_on_layer(
@@ -203,7 +203,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):
logger.warning(f"Failed to save SpinQuant config: {e}")
# ------------------------------------------------------------------
- # RotationSerializer — Load side
+ # SerializerMixin — Load side
# ------------------------------------------------------------------
def preregister_buffers(
diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.py
index cf498b0c1..a1f2cf178 100644
--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py
+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py
@@ -551,13 +551,13 @@ class SpinQuantPreprocessor:
def _train_rotations(self, dataloader: Any) -> None:
"""Train rotation matrices using the shared training loop.
- Delegates to :func:`training_core.run_training_loop` which is the
+ Delegates to :func:`training.run_training_loop` which is the
single implementation shared with :class:`RotationTrainer`.
For advanced training features (callbacks, evaluation, checkpointing,
- custom loss), use ``RotationTrainer`` from ``trainer.py`` directly.
+ custom loss), use ``RotationTrainer`` from ``training.py`` directly.
"""
- from auto_round.algorithms.transforms.spinquant.training_core import (
+ from auto_round.algorithms.transforms.spinquant.training import (
clone_model_for_reference,
create_dual_optimizer,
run_training_loop,
diff --git a/auto_round/algorithms/transforms/spinquant/trainer.py b/auto_round/algorithms/transforms/spinquant/trainer.py
deleted file mode 100644
index c691ff60a..000000000
--- a/auto_round/algorithms/transforms/spinquant/trainer.py
+++ /dev/null
@@ -1,532 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-
-"""
-SpinQuant / QuaRot Trainer for AutoRound.
-
-Provides a HuggingFace-Trainer-style interface for SpinQuant training,
-while remaining dependency-light (does not require ``transformers.Trainer``).
-
-The trainer manages the full training lifecycle:
-
- 1. Setup – init rotations, smooth values, optimiser, original-model clone
- 2. Train – customisable training loop with logging / evaluation
- 3. Fuse – offline rotation fusion into weights
- 4. Export – return the transformed model for AutoRound quantisation
-
-Key differences from Quark's ``SpinQuantTrainer``:
- * No hard dependency on ``transformers.Trainer`` (works standalone).
- * Plug-and-play with AutoRound's dataloader format.
- * Supports mid-training evaluation (e.g. perplexity) via callbacks.
- * Can be used both as an independent trainer and as a component inside
- ``AutoRound.quantize()``.
-
-Usage::
-
- from auto_round.algorithms.transforms.spinquant import (
- RotationTrainer, RotationTrainerConfig
- )
-
- trainer = RotationTrainer(
- model,
- config=RotationTrainerConfig(iters=200, lr=1e-4),
- )
- trainer.train(dataloader)
- model_rotated = trainer.fuse()
-
- # Proceed to AutoRound quantisation
- autoround = AutoRound(model_rotated, tokenizer, bits=4, ...)
- autoround.quantize()
-"""
-
-from __future__ import annotations
-
-import time
-from dataclasses import dataclass, field
-from typing import Any, Callable, Optional
-
-import torch
-import torch.nn as nn
-
-from auto_round.algorithms.transforms.spinquant.inplace.apply import (
- register_spinquant_hooks,
-)
-from auto_round.algorithms.transforms.spinquant.preprocessor import (
- SpinQuantConfig,
- SpinQuantPreprocessor,
- get_model_arch_info,
-)
-from auto_round.algorithms.transforms.spinquant.rotation_utils import (
- fuse_rmsnorm_in_model,
- untie_word_embeddings_if_needed,
-)
-
-
-@dataclass
-class RotationTrainerConfig:
- """Training hyperparameters for ``RotationTrainer``.
-
- ⚠️ **Experimental**: The SpinQuant training loop has basic infrastructure
- (Cayley SGD, KL loss, callbacks, checkpointing) but has NOT been validated
- end-to-end on real models. For production use, prefer QuaRot mode via
- ``SpinQuantPreprocessor`` with ``trainable_rotation=False``.
- """
-
- # ---------- Rotation knobs ----------
- r1: bool = True
- r2: bool = True
- r3: bool = True
- r4: bool = True
-
- trainable_rotation: bool = True # False = fixed Hadamard (QuaRot)
- trainable_smooth: bool = True # joint SmoothQuant training
- online_r1_rotation: bool = False
-
- # ---------- Optimiser ----------
- lr: float = 1e-4 # SGDG lr (rotation matrices)
- smooth_lr: float = 1e-3 # Adam lr (smooth values)
- iters: int = 200
- batch_size: int = 1
-
- # ---------- Loss ----------
- loss_type: str = "kl_top" # kl_top | kl_full | mse
- kl_top_k: int = 1000
-
- # ---------- Pipeline ----------
- fuse_rmsnorm: bool = True
- untie_embeddings: bool = True
-
- # ---------- Misc ----------
- dtype: torch.dtype = torch.float32
- device: Optional[str] = None
- log_interval: int = 50 # print every N steps
- eval_interval: int = 0 # 0 = never
- save_interval: int = 0 # 0 = never
- checkpoint_dir: Optional[str] = None
-
- def __post_init__(self):
- if self.device is None:
- self.device = "cuda" if torch.cuda.is_available() else "cpu"
-
-
-class RotationTrainerCallback:
- """
- Minimal callback interface (inspired by HF ``TrainerCallback``).
-
- Subclass and override any of the following hooks:
-
- * ``on_train_begin``
- * ``on_train_end``
- * ``on_step_begin``
- * ``on_step_end``
- * ``on_evaluate``
- """
-
- def on_train_begin(self, args: RotationTrainerConfig, state: dict, control: dict):
- pass
-
- def on_train_end(self, args: RotationTrainerConfig, state: dict, control: dict):
- pass
-
- def on_step_begin(self, args: RotationTrainerConfig, state: dict, control: dict):
- pass
-
- def on_step_end(self, args: RotationTrainerConfig, state: dict, control: dict):
- pass
-
- def on_evaluate(self, args: RotationTrainerConfig, state: dict, metrics: dict):
- pass
-
-
-class OrthogonalityMonitor(RotationTrainerCallback):
- """
- Callback that monitors ``R @ R.T ≈ I`` during training.
-
- Prints a warning when the deviation exceeds a threshold.
- """
-
- def __init__(self, threshold: float = 1e-3, log_interval: int = 50):
- self.threshold = threshold
- self.log_interval = log_interval
-
- def on_step_end(self, args, state, control):
- step = state.get("step", 0)
- if step % self.log_interval != 0:
- return
-
- model = state.get("model")
- if model is None:
- return
-
- max_dev = 0.0
- for name, param in model.named_parameters():
- if not param.requires_grad or "spinquant_R" not in name:
- continue
- R = param.data
- if R.dim() != 2 or R.shape[0] != R.shape[1]:
- continue
- I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)
- dev = (torch.matmul(R, R.t()) - I).abs().max().item()
- max_dev = max(max_dev, dev)
-
- state["ortho_deviation"] = max_dev
- if max_dev > self.threshold:
- print(f" [OrthogonalityMonitor] step={step} dev={max_dev:.2e} ⚠️ > {self.threshold}")
-
-
-class LossLogger(RotationTrainerCallback):
- """Callback that prints loss every ``log_interval`` steps."""
-
- def __init__(self, log_interval: int = 50):
- self.log_interval = log_interval
-
- def on_step_end(self, args, state, control):
- step = state.get("step", 0)
- if step % self.log_interval == 0:
- loss = state.get("loss", 0.0)
- avg_loss = state.get("avg_loss", 0.0)
- print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")
-
-
-class RotationTrainer:
- """
- Trainer for SpinQuant / QuaRot orthogonal rotation matrices.
-
- ⚠️ **Experimental**: This trainer has basic infrastructure but the
- SpinQuant training path has NOT been validated end-to-end on real models.
- For production QuaRot usage (fixed Hadamard, no training), use
- ``SpinQuantPreprocessor`` directly instead.
-
- Similar to Quark's ``SpinQuantTrainer`` but decoupled from
- ``transformers.Trainer``, making it usable inside AutoRound without
- extra heavy dependencies.
-
- Lifecycle::
-
- trainer = RotationTrainer(model, config, callbacks=[...])
- trainer.train(dataloader) # training loop
- model = trainer.fuse() # fuse into weights
- # ... now feed model to AutoRound ...
-
- Arguments:
- model: The transformer model (modified **in place**).
- config: ``RotationTrainerConfig`` instance.
- callbacks: List of ``RotationTrainerCallback`` subclasses.
- compute_loss_fn: Optional custom loss callable. Signature::
-
- compute_loss_fn(logits, ori_logits, config) -> Tensor
- """
-
- def __init__(
- self,
- model: nn.Module,
- config: Optional[RotationTrainerConfig] = None,
- callbacks: Optional[list[RotationTrainerCallback]] = None,
- compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,
- ):
- self.model = model
- self.config = config or RotationTrainerConfig()
- self.callbacks = callbacks or [LossLogger(), OrthogonalityMonitor()]
- self.compute_loss_fn = compute_loss_fn or self._default_compute_loss
-
- # Internal state (shared with callbacks)
- self.state: dict[str, Any] = {
- "step": 0,
- "global_step": 0,
- "loss": 0.0,
- "avg_loss": 0.0,
- "best_loss": float("inf"),
- "ortho_deviation": 0.0,
- "model": model,
- }
-
- # Architecture info
- info = get_model_arch_info(model)
- self.hidden_size = info.get("hidden_size", 0)
- self.head_dim = info.get("head_dim", 0)
- self.num_q_heads = info.get("num_q_heads", 0)
- self.num_kv_heads = info.get("num_kv_heads", 0)
- self.intermediate_size = info.get("intermediate_size", 0)
-
- # Training components (created lazily)
- self.optimizer = None
- self._original_model: Optional[nn.Module] = None
- self._hook_handles: list[Any] = []
- self._rotated_modules: set[nn.Module] = set()
- self._loss_buffer: list[float] = []
-
- # SpinQuant preprocessor (reused for init / fusion)
- self._preprocessor = SpinQuantPreprocessor(model, self._to_sq_config(self.config))
-
- # ------------------------------------------------------------------
- # Public API
- # ------------------------------------------------------------------
- def train(self, dataloader: Any) -> dict[str, Any]:
- """
- Run the full SpinQuant training loop.
-
- Returns a dict of training metrics::
-
- {
- "loss_history": list[float],
- "best_loss": float,
- "final_ortho_deviation": float,
- "steps": int,
- "elapsed_sec": float,
- }
- """
- self._setup_training(dataloader)
- self._trigger_event("on_train_begin")
-
- t0 = time.time()
- for batch in dataloader:
- if self.state["step"] >= self.config.iters:
- break
-
- self._trigger_event("on_step_begin")
- loss = self._training_step(batch)
- self._loss_buffer.append(loss)
- self.state["loss"] = loss
- self.state["avg_loss"] = sum(self._loss_buffer[-50:]) / len(self._loss_buffer[-50:])
- self.state["best_loss"] = min(self.state["best_loss"], loss)
- self.state["step"] += 1
- self.state["global_step"] += 1
-
- # Mid-training evaluation
- if self.config.eval_interval > 0 and self.state["step"] % self.config.eval_interval == 0:
- metrics = self.evaluate(dataloader)
- self._trigger_event("on_evaluate", metrics=metrics)
-
- # Checkpointing
- if self.config.save_interval > 0 and self.state["step"] % self.config.save_interval == 0:
- self.save_checkpoint()
-
- self._trigger_event("on_step_end")
-
- elapsed = time.time() - t0
- self._trigger_event("on_train_end")
- self._teardown_training()
-
- return {
- "loss_history": self._loss_buffer,
- "best_loss": self.state["best_loss"],
- "final_ortho_deviation": self.state.get("ortho_deviation", 0.0),
- "steps": self.state["step"],
- "elapsed_sec": elapsed,
- }
-
- def evaluate(self, dataloader: Any) -> dict[str, float]:
- """
- Evaluate current rotated model against the original on a subset.
-
- Returns a dict with ``loss``, ``max_diff``, etc.
- """
- from auto_round.algorithms.transforms.spinquant.training_core import (
- compute_rotation_loss,
- move_batch_to_device,
- )
-
- if self._original_model is None:
- return {}
-
- self.model.eval()
- device = next(self.model.parameters()).device
- losses = []
- max_diffs = []
-
- for i, batch in enumerate(dataloader):
- if i >= 10: # evaluate on first 10 batches only
- break
- batch = move_batch_to_device(batch, device)
- with torch.no_grad():
- out_rot = self.model(**batch)
- logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot
- out_ori = self._original_model(**batch)
- logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori
-
- loss = compute_rotation_loss(
- logits_rot,
- logits_ori,
- loss_type=self.config.loss_type,
- kl_top_k=self.config.kl_top_k,
- )
- losses.append(loss.item())
- max_diffs.append((logits_rot - logits_ori).abs().max().item())
-
- self.model.train()
- return {
- "eval_loss": sum(losses) / len(losses),
- "eval_max_diff": sum(max_diffs) / len(max_diffs),
- }
-
- def fuse(self) -> nn.Module:
- """
- Fuse offline rotations into model weights and clean up.
-
- Returns the modified ``model`` (same object, in-place).
- """
- self._preprocessor._fuse_offline_rotations()
- self._preprocessor._cleanup()
- return self.model
-
- def save_checkpoint(self, path: Optional[str] = None) -> str:
- """Save rotation + smooth params to disk."""
- if path is None:
- path = f"{self.config.checkpoint_dir or '.'}/spinquant_ckpt_step{self.state['step']}.pt"
- ckpt = {
- "step": self.state["step"],
- "config": self.config,
- "rotation_params": {
- n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "spinquant_R" in n
- },
- "smooth_params": {
- n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "smooth_values" in n
- },
- }
- torch.save(ckpt, path)
- print(f"[RotationTrainer] Checkpoint saved: {path}")
- return path
-
- def load_checkpoint(self, path: str) -> None:
- """Restore rotation + smooth params from disk."""
- ckpt = torch.load(path, map_location="cpu", weights_only=True)
- for n, p in self.model.named_parameters():
- if n in ckpt["rotation_params"]:
- p.data.copy_(ckpt["rotation_params"][n].to(p.device))
- if n in ckpt["smooth_params"]:
- p.data.copy_(ckpt["smooth_params"][n].to(p.device))
- self.state["step"] = ckpt["step"]
- print(f"[RotationTrainer] Checkpoint loaded: {path}")
-
- # ------------------------------------------------------------------
- # Internal helpers
- # ------------------------------------------------------------------
- def _setup_training(self, dataloader: Any) -> None:
- """Initialise rotations, optimiser, and original-model clone."""
- from auto_round.algorithms.transforms.spinquant.training_core import (
- clone_model_for_reference,
- create_dual_optimizer,
- )
-
- cfg = self.config
-
- # 1. Untie + RMSNorm fusion
- if cfg.untie_embeddings:
- untie_word_embeddings_if_needed(self.model)
- if cfg.fuse_rmsnorm:
- fuse_rmsnorm_in_model(self.model)
-
- # 2. Replace norms with trainable wrappers (SmoothQuant)
- if cfg.trainable_smooth:
- self._preprocessor._replace_norms_with_trainable()
-
- # 3. Initialise rotation matrices
- self._preprocessor._init_rotation_matrices()
-
- # 4. Register online hooks (R3 / R4)
- if cfg.r3 or cfg.r4:
- self._hook_handles = register_spinquant_hooks(
- self.model,
- self._to_sq_config(cfg),
- head_dim=self.head_dim,
- intermediate_size=self.intermediate_size,
- )
-
- # 5. Create dual optimiser (shared helper)
- self.optimizer = create_dual_optimizer(
- self.model,
- lr=cfg.lr,
- smooth_lr=cfg.smooth_lr,
- )
-
- # 6. Clone original model for KL reference (shared helper)
- self._original_model = clone_model_for_reference(self.model)
-
- # 7. Use model's existing device (don't move)
- self.model.train()
-
- def _training_step(self, batch: Any) -> float:
- """Single training step. Returns loss scalar."""
- from auto_round.algorithms.transforms.spinquant.training_core import (
- compute_rotation_loss,
- move_batch_to_device,
- )
-
- device = next(self.model.parameters()).device
- batch = move_batch_to_device(batch, device)
-
- # Forward: rotated model
- out_rot = self.model(**batch)
- logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot
-
- # Forward: original model (no grad)
- with torch.no_grad():
- assert self._original_model is not None
- out_ori = self._original_model(**batch)
- logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori
-
- # Loss & backward — use custom loss_fn if set, else shared core
- if self.compute_loss_fn is not self._default_compute_loss:
- loss = self.compute_loss_fn(logits_rot, logits_ori, self.config)
- else:
- loss = compute_rotation_loss(
- logits_rot,
- logits_ori,
- loss_type=self.config.loss_type,
- kl_top_k=self.config.kl_top_k,
- )
- loss.backward()
- self.optimizer.step() # type: ignore[union-attr]
- self.optimizer.zero_grad() # type: ignore[union-attr]
-
- return loss.item()
-
- def _teardown_training(self) -> None:
- """Free original model and set eval mode."""
- if self._original_model is not None:
- del self._original_model
- self._original_model = None
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
- self.model.eval()
-
- def _trigger_event(self, event_name: str, **kwargs) -> None:
- """Fire callback hooks."""
- for cb in self.callbacks:
- method = getattr(cb, event_name, None)
- if callable(method):
- method(self.config, self.state, {"should_training_stop": False, **kwargs})
-
- @staticmethod
- def _default_compute_loss(
- logits: torch.Tensor,
- ori_logits: torch.Tensor,
- config: RotationTrainerConfig,
- ) -> torch.Tensor:
- """Default loss — delegates to :func:`training_core.compute_rotation_loss`."""
- from auto_round.algorithms.transforms.spinquant.training_core import compute_rotation_loss
-
- return compute_rotation_loss(logits, ori_logits, config.loss_type, config.kl_top_k)
-
- @staticmethod
- def _to_sq_config(cfg: RotationTrainerConfig) -> SpinQuantConfig:
- """Convert trainer config to SpinQuantConfig for internal reuse."""
- return SpinQuantConfig(
- r1=cfg.r1,
- r2=cfg.r2,
- r3=cfg.r3,
- r4=cfg.r4,
- trainable_rotation=cfg.trainable_rotation,
- trainable_smooth=cfg.trainable_smooth,
- online_r1_rotation=cfg.online_r1_rotation,
- iters=cfg.iters,
- lr=cfg.lr,
- smooth_lr=cfg.smooth_lr,
- batch_size=cfg.batch_size,
- loss_type=cfg.loss_type,
- kl_top_k=cfg.kl_top_k,
- fuse_rmsnorm=cfg.fuse_rmsnorm,
- untie_embeddings=cfg.untie_embeddings,
- dtype=cfg.dtype,
- device=cfg.device,
- )
diff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.py
index 99d55ae8d..e3e1e2b64 100644
--- a/auto_round/algorithms/transforms/spinquant/training.py
+++ b/auto_round/algorithms/transforms/spinquant/training.py
@@ -1,37 +1,311 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""SpinQuant / QuaRot training infrastructure.
+
+⚠️ **Experimental**: The SpinQuant training path has basic infrastructure
+(Cayley SGD optimizer, KL-divergence loss, callbacks, checkpointing) but
+has NOT been validated end-to-end on real models. For production use, prefer
+QuaRot mode (fixed Hadamard, no training) via ``SpinQuantPreprocessor``.
-"""
-Training utilities for SpinQuant in AutoRound.
+This module consolidates all training-related functionality:
-⚠️ **Experimental**: These training utilities provide hooks and helpers for
-integrating SpinQuant training into AutoRound's calibration pipeline, but the
-training path has NOT been validated end-to-end. For production use, prefer
-QuaRot mode (fixed Hadamard, no training) via ``SpinQuantPreprocessor``.
+* **Core primitives** — loss computation, training loop, optimizer creation
+* **Pipeline hooks** — ``SpinQuantTrainingHook`` for AutoRound integration
+* **Standalone trainer** — ``RotationTrainer`` (HF Trainer style)
+* **Callbacks** — orthogonality monitoring, loss logging
+* **State tracking** — ``SpinQuantState`` for metrics
+
+Usage (standalone trainer)::
-Provides hooks, callbacks, and helper functions for integrating
-SpinQuant training into AutoRound's calibration pipeline.
+ from auto_round.algorithms.transforms.spinquant.training import (
+ RotationTrainer, RotationTrainerConfig
+ )
+
+ trainer = RotationTrainer(model, config=RotationTrainerConfig(iters=200))
+ trainer.train(dataloader)
+ model = trainer.fuse()
-Key design: SpinQuant training happens BEFORE AutoRound's block-wise
-quantization calibration. After rotation matrices are learned and fused,
-the model is mathematically equivalent to the original, so AutoRound
-can apply its standard quantization without any modifications.
+Usage (pipeline hook)::
+
+ hook = SpinQuantTrainingHook(model, config)
+ hook.preprocess(dataloader)
"""
from __future__ import annotations
-from typing import Any, Optional
+import copy
+import logging
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any, Callable, Optional
import torch
import torch.nn as nn
+import torch.nn.functional as F
+
+from auto_round.algorithms.transforms.spinquant.cayley_optimizer import AdamAndSGDG
+
+logger = logging.getLogger("autoround.spinquant")
+
+
+# ===========================================================================
+# Core primitives: loss, utilities, training loop
+# ===========================================================================
+
+
+def compute_rotation_loss(
+ logits: torch.Tensor,
+ ori_logits: torch.Tensor,
+ loss_type: str = "kl_top",
+ kl_top_k: int = 1000,
+) -> torch.Tensor:
+ """Compute loss between rotated-model logits and original-model logits.
+
+ This is the single implementation shared by both the preprocessor's
+ embedded loop and the standalone :class:`RotationTrainer`.
+
+ Args:
+ logits: Rotated model output logits.
+ ori_logits: Original (un-rotated) model output logits (detached).
+ loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.
+ kl_top_k: Number of top logits to use for ``kl_top``.
+
+ Returns:
+ Scalar loss tensor.
+ """
+ if loss_type == "kl_top":
+ k = min(kl_top_k, logits.size(-1))
+ top_ori, indices = ori_logits.topk(k, dim=-1, sorted=False)
+ top_logits = logits.gather(-1, indices)
+ return F.kl_div(
+ F.log_softmax(top_logits.flatten(0, -2), dim=-1),
+ F.softmax(top_ori.flatten(0, -2), dim=-1),
+ reduction="batchmean",
+ )
+ if loss_type == "kl_full":
+ return F.kl_div(
+ F.log_softmax(logits.flatten(0, -2), dim=-1),
+ F.softmax(ori_logits.flatten(0, -2), dim=-1),
+ reduction="batchmean",
+ )
+ if loss_type == "mse":
+ return F.mse_loss(logits, ori_logits)
+ raise ValueError(f"Unknown loss_type={loss_type!r}")
+
+
+# Alias for backward compatibility (training.py used to export this)
+spinquant_loss_fn = compute_rotation_loss
+
+
+def move_batch_to_device(batch: Any, device: torch.device) -> Any:
+ """Move a batch (tensor or dict of tensors) to *device*."""
+ if hasattr(batch, "to"):
+ return batch.to(device)
+ if isinstance(batch, dict):
+ return {k: v.to(device) if hasattr(v, "to") else v for k, v in batch.items()}
+ return batch
+
-from auto_round.algorithms.transforms.spinquant.cayley_optimizer import (
- AdamAndSGDG,
-)
-from auto_round.algorithms.transforms.spinquant.preprocessor import (
- SpinQuantConfig,
- SpinQuantPreprocessor,
-)
+def check_orthogonality(model: nn.Module, threshold: float = 1e-3) -> float:
+ """Check orthogonality of all ``spinquant_R*`` parameters.
+
+ Returns:
+ Maximum deviation from identity (``max ||R @ R^T - I||_inf``).
+ """
+ max_dev = 0.0
+ for name, param in model.named_parameters():
+ if not param.requires_grad or "spinquant_R" not in name:
+ continue
+ R = param.data
+ if R.dim() != 2 or R.shape[0] != R.shape[1] or R.numel() == 0:
+ continue
+ I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)
+ dev = (torch.matmul(R, R.t()) - I).abs().max().item()
+ max_dev = max(max_dev, dev)
+ if dev > 1e-4:
+ logger.warning(f" {name} orthogonality deviation={dev:.2e}")
+ if max_dev > 0:
+ logger.info(f"[SpinQuant] Max orthogonality deviation={max_dev:.2e}")
+ if max_dev > threshold:
+ logger.warning(" Orthogonality constraint significantly violated!")
+ return max_dev
+
+
+def clone_model_for_reference(model: nn.Module) -> nn.Module:
+ """Create a frozen deep-copy of *model* for KL-divergence reference.
+
+ Hooks registered by SpinQuant are removed from the clone so it runs
+ as the un-rotated baseline.
+ """
+ from auto_round.algorithms.transforms.spinquant.preprocessor import (
+ remove_spinquant_hooks_from_model,
+ )
+
+ original = copy.deepcopy(model)
+ original.eval()
+ for p in original.parameters():
+ p.requires_grad = False
+ remove_spinquant_hooks_from_model(original)
+ return original
+
+
+def create_dual_optimizer(
+ model: nn.Module,
+ lr: float = 1e-4,
+ smooth_lr: float = 1e-3,
+) -> Optional[AdamAndSGDG]:
+ """Create the Adam (smooth) + SGDG (rotation) dual optimiser.
+
+ Returns ``None`` if no trainable parameters are found.
+ """
+ rot_params = [p for n, p in model.named_parameters() if p.requires_grad and "spinquant_R" in n]
+ smooth_params = [p for n, p in model.named_parameters() if p.requires_grad and "smooth_values" in n]
+
+ if not rot_params and not smooth_params:
+ logger.info("[SpinQuant] No trainable parameters — nothing to train.")
+ return None
+
+ n_rot = len(rot_params)
+ n_smooth = len(smooth_params)
+ total_params = sum(p.numel() for p in rot_params) + sum(p.numel() for p in smooth_params)
+ logger.info(
+ f"[SpinQuant] Training: {n_rot} rotation params + {n_smooth} smooth params "
+ f"= {total_params:,} trainable parameters"
+ )
+
+ return AdamAndSGDG(
+ adam_params=smooth_params,
+ sgdg_params=rot_params,
+ learning_rate=lr,
+ smooth_learning_rate=smooth_lr,
+ )
+
+
+# Alias for backward compatibility
+create_spinquant_optimizer = create_dual_optimizer
+
+
+@dataclass
+class TrainingResult:
+ """Result of :func:`run_training_loop`."""
+
+ loss_history: list[float]
+ best_loss: float
+ final_ortho_deviation: float
+ steps: int
+
+
+def run_training_loop(
+ model: nn.Module,
+ original_model: nn.Module,
+ optimizer: AdamAndSGDG,
+ dataloader: Any,
+ *,
+ max_iters: int = 200,
+ loss_type: str = "kl_top",
+ kl_top_k: int = 1000,
+ compute_loss_fn: Optional[Callable] = None,
+ on_step_end: Optional[Callable[[int, float, float], None]] = None,
+ log_interval: int = 50,
+) -> TrainingResult:
+ """Run the SpinQuant rotation training loop.
+
+ This is the **single** training loop implementation shared by
+ :class:`SpinQuantPreprocessor` (embedded) and :class:`RotationTrainer`
+ (standalone).
+
+ Args:
+ model: The model being trained (rotation params + optional smooth params).
+ original_model: Frozen reference model for KL divergence.
+ optimizer: :class:`AdamAndSGDG` dual optimiser.
+ dataloader: Iterable of batches.
+ max_iters: Stop after this many steps.
+ loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.
+ kl_top_k: Top-k for ``kl_top`` loss.
+ compute_loss_fn: Optional override for loss computation. Signature:
+ ``(logits, ori_logits) -> Tensor``. If ``None``, uses
+ :func:`compute_rotation_loss`.
+ on_step_end: Optional callback ``(step, loss, avg_loss) -> None``
+ called after each gradient step.
+ log_interval: Default logging interval (used when no ``on_step_end``
+ is provided).
+
+ Returns:
+ :class:`TrainingResult` with loss history and orthogonality stats.
+ """
+ device = next(model.parameters()).device
+ model.train()
+
+ loss_fn = compute_loss_fn or (
+ lambda logits, ori_logits: compute_rotation_loss(logits, ori_logits, loss_type, kl_top_k)
+ )
+
+ loss_history: list[float] = []
+ best_loss = float("inf")
+ step = 0
+
+ for batch in dataloader:
+ if step >= max_iters:
+ break
+
+ batch = move_batch_to_device(batch, device)
+
+ # Forward: rotated model
+ out_rot = model(**batch)
+ logits = out_rot.logits if hasattr(out_rot, "logits") else out_rot
+
+ # Forward: original model (no grad)
+ with torch.no_grad():
+ out_ori = original_model(**batch)
+ ori_logits = out_ori.logits if hasattr(out_ori, "logits") else out_ori
+
+ # Loss + backward
+ loss = loss_fn(logits, ori_logits)
+ loss.backward()
+ optimizer.step()
+ optimizer.zero_grad()
+
+ loss_val = loss.item()
+ loss_history.append(loss_val)
+ best_loss = min(best_loss, loss_val)
+
+ # Logging / callbacks
+ if on_step_end is not None:
+ avg = sum(loss_history[-50:]) / len(loss_history[-50:])
+ on_step_end(step, loss_val, avg)
+ elif step % log_interval == 0:
+ avg = sum(loss_history[-50:]) / len(loss_history[-50:])
+ logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")
+
+ step += 1
+
+ # Final orthogonality check
+ ortho_dev = check_orthogonality(model)
+
+ model.eval()
+
+ return TrainingResult(
+ loss_history=loss_history,
+ best_loss=best_loss,
+ final_ortho_deviation=ortho_dev,
+ steps=step,
+ )
+
+
+# ===========================================================================
+# Pipeline hooks: SpinQuantTrainingHook for AutoRound integration
+# ===========================================================================
class SpinQuantTrainingHook:
@@ -53,9 +327,14 @@ class SpinQuantTrainingHook:
def __init__(
self,
model: nn.Module,
- config: Optional[SpinQuantConfig] = None,
+ config=None,
enabled: bool = True,
):
+ from auto_round.algorithms.transforms.spinquant.preprocessor import (
+ SpinQuantConfig,
+ SpinQuantPreprocessor,
+ )
+
self.model = model
self.config = config or SpinQuantConfig()
self.enabled = enabled
@@ -66,6 +345,8 @@ class SpinQuantTrainingHook:
if not self.enabled:
return self.model
+ from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantPreprocessor
+
self.preprocessor = SpinQuantPreprocessor(self.model, self.config)
return self.preprocessor.preprocess(dataloader)
@@ -112,97 +393,10 @@ class OrthogonalTrainingCallback:
self.max_deviation_history.append(max_dev)
if max_dev > 1e-4:
- print(f"[SpinQuant] Step {self.step}: max orthogonality deviation = {max_dev:.2e}")
+ logger.info(f"[SpinQuant] Step {self.step}: max orthogonality deviation = {max_dev:.2e}")
if max_dev > 1e-2:
- print(" 🔴 WARNING: Orthogonality significantly violated! Consider reducing lr.")
-
-
-def create_spinquant_optimizer(
- model: nn.Module,
- lr: float = 1e-4,
- smooth_lr: float = 1e-3,
-) -> Optional[torch.optim.Optimizer]:
- """
- Create an AdamAndSGDG optimizer for SpinQuant parameters in a model.
-
- This is useful when AutoRound's own training loop wants to manage
- the optimizer externally rather than letting SpinQuant create it internally.
-
- Args:
- model: The model that has been set up with SpinQuant parameters
- lr: Learning rate for rotation matrices (SGDG)
- smooth_lr: Learning rate for smooth values (Adam)
-
- Returns:
- AdamAndSGDG optimizer or None if no SpinQuant parameters found
- """
- rotation_params = []
- smooth_params = []
-
- for name, param in model.named_parameters():
- if not param.requires_grad:
- continue
- if "spinquant_rotation" in name or "spinquant_R" in name:
- rotation_params.append(param)
- elif "spinquant_smooth" in name or "smooth_values" in name:
- smooth_params.append(param)
-
- if len(rotation_params) == 0 and len(smooth_params) == 0:
- return None
-
- return AdamAndSGDG(
- adam_params=smooth_params,
- sgdg_params=rotation_params,
- learning_rate=lr,
- smooth_learning_rate=smooth_lr,
- )
-
-
-def spinquant_loss_fn(
- logits: torch.Tensor,
- original_logits: torch.Tensor,
- loss_type: str = "kl_top",
- kl_top_k: int = 1000,
-) -> torch.Tensor:
- """
- Compute the SpinQuant training loss.
-
- Supports KL divergence (top-k or full) and MSE.
-
- Args:
- logits: Logits from the rotated model
- original_logits: Logits from the original (unrotated) model
- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``
- kl_top_k: Number of top logits for KL divergence
-
- Returns:
- Loss tensor
- """
- import torch.nn.functional as F
-
- if loss_type == "kl_top":
- k = min(kl_top_k, logits.size(-1))
- top_ori_logits, indices = original_logits.topk(k, dim=-1, sorted=False)
- top_logits = logits.gather(-1, indices)
-
- loss = F.kl_div(
- F.log_softmax(top_logits.flatten(0, -2), dim=-1),
- F.softmax(top_ori_logits.flatten(0, -2), dim=-1),
- reduction="batchmean",
- )
- elif loss_type == "kl_full":
- loss = F.kl_div(
- F.log_softmax(logits.flatten(0, -2), dim=-1),
- F.softmax(original_logits.flatten(0, -2), dim=-1),
- reduction="batchmean",
- )
- elif loss_type == "mse":
- loss = F.mse_loss(logits, original_logits)
- else:
- raise ValueError(f"Unknown loss_type: {loss_type}. Supported: kl_top, kl_full, mse")
-
- return loss
+ logger.warning("[SpinQuant] Orthogonality significantly violated! Consider reducing lr.")
class SpinQuantState:
@@ -245,3 +439,488 @@ class SpinQuantState:
"rotation_params": self.rotation_names,
"orthogonality_deviation": self.orthogonality_deviation[-1] if self.orthogonality_deviation else None,
}
+
+
+# ===========================================================================
+# Standalone trainer: RotationTrainer (HF Trainer style)
+# ===========================================================================
+
+
+@dataclass
+class RotationTrainerConfig:
+ """Training hyperparameters for ``RotationTrainer``.
+
+ ⚠️ **Experimental**: The SpinQuant training loop has basic infrastructure
+ (Cayley SGD, KL loss, callbacks, checkpointing) but has NOT been validated
+ end-to-end on real models. For production use, prefer QuaRot mode via
+ ``SpinQuantPreprocessor`` with ``trainable_rotation=False``.
+ """
+
+ # ---------- Rotation knobs ----------
+ r1: bool = True
+ r2: bool = True
+ r3: bool = True
+ r4: bool = True
+
+ trainable_rotation: bool = True # False = fixed Hadamard (QuaRot)
+ trainable_smooth: bool = True # joint SmoothQuant training
+ online_r1_rotation: bool = False
+
+ # ---------- Optimiser ----------
+ lr: float = 1e-4 # SGDG lr (rotation matrices)
+ smooth_lr: float = 1e-3 # Adam lr (smooth values)
+ iters: int = 200
+ batch_size: int = 1
+
+ # ---------- Loss ----------
+ loss_type: str = "kl_top" # kl_top | kl_full | mse
+ kl_top_k: int = 1000
+
+ # ---------- Pipeline ----------
+ fuse_rmsnorm: bool = True
+ untie_embeddings: bool = True
+
+ # ---------- Misc ----------
+ dtype: torch.dtype = torch.float32
+ device: Optional[str] = None
+ log_interval: int = 50 # print every N steps
+ eval_interval: int = 0 # 0 = never
+ save_interval: int = 0 # 0 = never
+ checkpoint_dir: Optional[str] = None
+
+ def __post_init__(self):
+ if self.device is None:
+ self.device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+class RotationTrainerCallback:
+ """
+ Minimal callback interface (inspired by HF ``TrainerCallback``).
+
+ Subclass and override any of the following hooks:
+
+ * ``on_train_begin``
+ * ``on_train_end``
+ * ``on_step_begin``
+ * ``on_step_end``
+ * ``on_evaluate``
+ """
+
+ def on_train_begin(self, args: RotationTrainerConfig, state: dict, control: dict):
+ pass
+
+ def on_train_end(self, args: RotationTrainerConfig, state: dict, control: dict):
+ pass
+
+ def on_step_begin(self, args: RotationTrainerConfig, state: dict, control: dict):
+ pass
+
+ def on_step_end(self, args: RotationTrainerConfig, state: dict, control: dict):
+ pass
+
+ def on_evaluate(self, args: RotationTrainerConfig, state: dict, metrics: dict):
+ pass
+
+
+class OrthogonalityMonitor(RotationTrainerCallback):
+ """
+ Callback that monitors ``R @ R.T ≈ I`` during training.
+
+ Prints a warning when the deviation exceeds a threshold.
+ """
+
+ def __init__(self, threshold: float = 1e-3, log_interval: int = 50):
+ self.threshold = threshold
+ self.log_interval = log_interval
+
+ def on_step_end(self, args, state, control):
+ step = state.get("step", 0)
+ if step % self.log_interval != 0:
+ return
+
+ model = state.get("model")
+ if model is None:
+ return
+
+ max_dev = 0.0
+ for name, param in model.named_parameters():
+ if not param.requires_grad or "spinquant_R" not in name:
+ continue
+ R = param.data
+ if R.dim() != 2 or R.shape[0] != R.shape[1]:
+ continue
+ I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)
+ dev = (torch.matmul(R, R.t()) - I).abs().max().item()
+ max_dev = max(max_dev, dev)
+
+ state["ortho_deviation"] = max_dev
+ if max_dev > self.threshold:
+ print(f" [OrthogonalityMonitor] step={step} dev={max_dev:.2e} > {self.threshold}")
+
+
+class LossLogger(RotationTrainerCallback):
+ """Callback that prints loss every ``log_interval`` steps."""
+
+ def __init__(self, log_interval: int = 50):
+ self.log_interval = log_interval
+
+ def on_step_end(self, args, state, control):
+ step = state.get("step", 0)
+ if step % self.log_interval == 0:
+ loss = state.get("loss", 0.0)
+ avg_loss = state.get("avg_loss", 0.0)
+ print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")
+
+
+class RotationTrainer:
+ """
+ Trainer for SpinQuant / QuaRot orthogonal rotation matrices.
+
+ ⚠️ **Experimental**: This trainer has basic infrastructure but the
+ SpinQuant training path has NOT been validated end-to-end on real models.
+ For production QuaRot usage (fixed Hadamard, no training), use
+ ``SpinQuantPreprocessor`` directly instead.
+
+ Similar to Quark's ``SpinQuantTrainer`` but decoupled from
+ ``transformers.Trainer``, making it usable inside AutoRound without
+ extra heavy dependencies.
+
+ Lifecycle::
+
+ trainer = RotationTrainer(model, config, callbacks=[...])
+ trainer.train(dataloader) # training loop
+ model = trainer.fuse() # fuse into weights
+ # ... now feed model to AutoRound ...
+
+ Arguments:
+ model: The transformer model (modified **in place**).
+ config: ``RotationTrainerConfig`` instance.
+ callbacks: List of ``RotationTrainerCallback`` subclasses.
+ compute_loss_fn: Optional custom loss callable. Signature::
+
+ compute_loss_fn(logits, ori_logits, config) -> Tensor
+ """
+
+ def __init__(
+ self,
+ model: nn.Module,
+ config: Optional[RotationTrainerConfig] = None,
+ callbacks: Optional[list[RotationTrainerCallback]] = None,
+ compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,
+ ):
+ from auto_round.algorithms.transforms.spinquant.preprocessor import (
+ SpinQuantConfig,
+ SpinQuantPreprocessor,
+ get_model_arch_info,
+ )
+
+ self.model = model
+ self.config = config or RotationTrainerConfig()
+ self.callbacks = callbacks or [LossLogger(), OrthogonalityMonitor()]
+ self.compute_loss_fn = compute_loss_fn or self._default_compute_loss
+
+ # Internal state (shared with callbacks)
+ self.state: dict[str, Any] = {
+ "step": 0,
+ "global_step": 0,
+ "loss": 0.0,
+ "avg_loss": 0.0,
+ "best_loss": float("inf"),
+ "ortho_deviation": 0.0,
+ "model": model,
+ }
+
+ # Architecture info
+ info = get_model_arch_info(model)
+ self.hidden_size = info.get("hidden_size", 0)
+ self.head_dim = info.get("head_dim", 0)
+ self.num_q_heads = info.get("num_q_heads", 0)
+ self.num_kv_heads = info.get("num_kv_heads", 0)
+ self.intermediate_size = info.get("intermediate_size", 0)
+
+ # Training components (created lazily)
+ self.optimizer = None
+ self._original_model: Optional[nn.Module] = None
+ self._hook_handles: list[Any] = []
+ self._rotated_modules: set[nn.Module] = set()
+ self._loss_buffer: list[float] = []
+
+ # SpinQuant preprocessor (reused for init / fusion)
+ self._preprocessor = SpinQuantPreprocessor(model, self._to_sq_config(self.config))
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+ def train(self, dataloader: Any) -> dict[str, Any]:
+ """
+ Run the full SpinQuant training loop.
+
+ Returns a dict of training metrics::
+
+ {
+ "loss_history": list[float],
+ "best_loss": float,
+ "final_ortho_deviation": float,
+ "steps": int,
+ "elapsed_sec": float,
+ }
+ """
+ self._setup_training(dataloader)
+ self._trigger_event("on_train_begin")
+
+ t0 = time.time()
+ for batch in dataloader:
+ if self.state["step"] >= self.config.iters:
+ break
+
+ self._trigger_event("on_step_begin")
+ loss = self._training_step(batch)
+ self._loss_buffer.append(loss)
+ self.state["loss"] = loss
+ self.state["avg_loss"] = sum(self._loss_buffer[-50:]) / len(self._loss_buffer[-50:])
+ self.state["best_loss"] = min(self.state["best_loss"], loss)
+ self.state["step"] += 1
+ self.state["global_step"] += 1
+
+ # Mid-training evaluation
+ if self.config.eval_interval > 0 and self.state["step"] % self.config.eval_interval == 0:
+ metrics = self.evaluate(dataloader)
+ self._trigger_event("on_evaluate", metrics=metrics)
+
+ # Checkpointing
+ if self.config.save_interval > 0 and self.state["step"] % self.config.save_interval == 0:
+ self.save_checkpoint()
+
+ self._trigger_event("on_step_end")
+
+ elapsed = time.time() - t0
+ self._trigger_event("on_train_end")
+ self._teardown_training()
+
+ return {
+ "loss_history": self._loss_buffer,
+ "best_loss": self.state["best_loss"],
+ "final_ortho_deviation": self.state.get("ortho_deviation", 0.0),
+ "steps": self.state["step"],
+ "elapsed_sec": elapsed,
+ }
+
+ def evaluate(self, dataloader: Any) -> dict[str, float]:
+ """
+ Evaluate current rotated model against the original on a subset.
+
+ Returns a dict with ``loss``, ``max_diff``, etc.
+ """
+ if self._original_model is None:
+ return {}
+
+ self.model.eval()
+ device = next(self.model.parameters()).device
+ losses = []
+ max_diffs = []
+
+ for i, batch in enumerate(dataloader):
+ if i >= 10: # evaluate on first 10 batches only
+ break
+ batch = move_batch_to_device(batch, device)
+ with torch.no_grad():
+ out_rot = self.model(**batch)
+ logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot
+ out_ori = self._original_model(**batch)
+ logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori
+
+ loss = compute_rotation_loss(
+ logits_rot,
+ logits_ori,
+ loss_type=self.config.loss_type,
+ kl_top_k=self.config.kl_top_k,
+ )
+ losses.append(loss.item())
+ max_diffs.append((logits_rot - logits_ori).abs().max().item())
+
+ self.model.train()
+ if not losses:
+ return {"eval_loss": 0.0, "eval_max_diff": 0.0}
+ return {
+ "eval_loss": sum(losses) / len(losses),
+ "eval_max_diff": sum(max_diffs) / len(max_diffs),
+ }
+
+ def fuse(self) -> nn.Module:
+ """
+ Fuse offline rotations into model weights and clean up.
+
+ Returns the modified ``model`` (same object, in-place).
+ """
+ self._preprocessor._fuse_offline_rotations()
+ self._preprocessor._cleanup()
+ return self.model
+
+ def save_checkpoint(self, path: Optional[str] = None) -> str:
+ """Save rotation + smooth params to disk."""
+ if path is None:
+ path = f"{self.config.checkpoint_dir or '.'}/spinquant_ckpt_step{self.state['step']}.pt"
+ ckpt = {
+ "step": self.state["step"],
+ "config": asdict(self.config),
+ "rotation_params": {
+ n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "spinquant_R" in n
+ },
+ "smooth_params": {
+ n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "smooth_values" in n
+ },
+ }
+ torch.save(ckpt, path)
+ logger.info(f"[RotationTrainer] Checkpoint saved: {path}")
+ return path
+
+ def load_checkpoint(self, path: str) -> None:
+ """Restore rotation + smooth params from disk."""
+ try:
+ ckpt = torch.load(path, map_location="cpu", weights_only=True)
+ except TypeError:
+ ckpt = torch.load(path, map_location="cpu")
+ for n, p in self.model.named_parameters():
+ if n in ckpt["rotation_params"]:
+ p.data.copy_(ckpt["rotation_params"][n].to(p.device))
+ if n in ckpt["smooth_params"]:
+ p.data.copy_(ckpt["smooth_params"][n].to(p.device))
+ self.state["step"] = ckpt["step"]
+ logger.info(f"[RotationTrainer] Checkpoint loaded: {path}")
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+ def _setup_training(self, dataloader: Any) -> None:
+ """Initialise rotations, optimiser, and original-model clone."""
+ from auto_round.algorithms.transforms.spinquant.inplace.apply import (
+ register_spinquant_hooks,
+ )
+ from auto_round.algorithms.transforms.spinquant.rotation_utils import (
+ fuse_rmsnorm_in_model,
+ untie_word_embeddings_if_needed,
+ )
+
+ cfg = self.config
+
+ # 1. Untie + RMSNorm fusion
+ if cfg.untie_embeddings:
+ untie_word_embeddings_if_needed(self.model)
+ if cfg.fuse_rmsnorm:
+ fuse_rmsnorm_in_model(self.model)
+
+ # 2. Replace norms with trainable wrappers (SmoothQuant)
+ if cfg.trainable_smooth:
+ self._preprocessor._replace_norms_with_trainable()
+
+ # 3. Initialise rotation matrices
+ self._preprocessor._init_rotation_matrices()
+
+ # 4. Register online hooks (R3 / R4)
+ if cfg.r3 or cfg.r4:
+ self._hook_handles = register_spinquant_hooks(
+ self.model,
+ self._to_sq_config(cfg),
+ head_dim=self.head_dim,
+ intermediate_size=self.intermediate_size,
+ )
+
+ # 5. Create dual optimiser
+ self.optimizer = create_dual_optimizer(
+ self.model,
+ lr=cfg.lr,
+ smooth_lr=cfg.smooth_lr,
+ )
+ if self.optimizer is None:
+ raise ValueError(
+ "SpinQuant training requires at least one trainable parameter group, "
+ "but create_dual_optimizer() returned None. Ensure that training is "
+ "configured with trainable rotation and/or smooth parameters enabled."
+ )
+
+ # 6. Clone original model for KL reference
+ self._original_model = clone_model_for_reference(self.model)
+
+ # 7. Use model's existing device (don't move)
+ self.model.train()
+
+ def _training_step(self, batch: Any) -> float:
+ """Single training step. Returns loss scalar."""
+ device = next(self.model.parameters()).device
+ batch = move_batch_to_device(batch, device)
+
+ # Forward: rotated model
+ out_rot = self.model(**batch)
+ logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot
+
+ # Forward: original model (no grad)
+ with torch.no_grad():
+ assert self._original_model is not None
+ out_ori = self._original_model(**batch)
+ logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori
+
+ # Loss & backward
+ if self.compute_loss_fn is not self._default_compute_loss:
+ loss = self.compute_loss_fn(logits_rot, logits_ori, self.config)
+ else:
+ loss = compute_rotation_loss(
+ logits_rot,
+ logits_ori,
+ loss_type=self.config.loss_type,
+ kl_top_k=self.config.kl_top_k,
+ )
+ loss.backward()
+ self.optimizer.step() # type: ignore[union-attr]
+ self.optimizer.zero_grad() # type: ignore[union-attr]
+
+ return loss.item()
+
+ def _teardown_training(self) -> None:
+ """Free original model and set eval mode."""
+ if self._original_model is not None:
+ del self._original_model
+ self._original_model = None
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ self.model.eval()
+
+ def _trigger_event(self, event_name: str, **kwargs) -> None:
+ """Fire callback hooks."""
+ for cb in self.callbacks:
+ method = getattr(cb, event_name, None)
+ if callable(method):
+ method(self.config, self.state, {"should_training_stop": False, **kwargs})
+
+ @staticmethod
+ def _default_compute_loss(
+ logits: torch.Tensor,
+ ori_logits: torch.Tensor,
+ config: RotationTrainerConfig,
+ ) -> torch.Tensor:
+ """Default loss — delegates to :func:`compute_rotation_loss`."""
+ return compute_rotation_loss(logits, ori_logits, config.loss_type, config.kl_top_k)
+
+ @staticmethod
+ def _to_sq_config(cfg: RotationTrainerConfig):
+ """Convert trainer config to SpinQuantConfig for internal reuse."""
+ from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig
+
+ return SpinQuantConfig(
+ r1=cfg.r1,
+ r2=cfg.r2,
+ r3=cfg.r3,
+ r4=cfg.r4,
+ trainable_rotation=cfg.trainable_rotation,
+ trainable_smooth=cfg.trainable_smooth,
+ online_r1_rotation=cfg.online_r1_rotation,
+ iters=cfg.iters,
+ lr=cfg.lr,
+ smooth_lr=cfg.smooth_lr,
+ batch_size=cfg.batch_size,
+ loss_type=cfg.loss_type,
+ kl_top_k=cfg.kl_top_k,
+ fuse_rmsnorm=cfg.fuse_rmsnorm,
+ untie_embeddings=cfg.untie_embeddings,
+ dtype=cfg.dtype,
+ device=cfg.device,
+ )
diff --git a/auto_round/algorithms/transforms/spinquant/training_core.py b/auto_round/algorithms/transforms/spinquant/training_core.py
deleted file mode 100644
index 18a68cbc8..000000000
--- a/auto_round/algorithms/transforms/spinquant/training_core.py
+++ /dev/null
@@ -1,285 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Shared training primitives for SpinQuant rotation learning.
-
-This module provides the **single implementation** of the rotation training
-loop used by both:
-
-* :meth:`SpinQuantPreprocessor._train_rotations` — minimal embedded loop
-* :class:`RotationTrainer` — full-featured trainer with callbacks
-
-By centralising the training step and loss computation here, both entry
-points stay in sync and avoid code duplication.
-"""
-
-from __future__ import annotations
-
-import copy
-import logging
-from dataclasses import dataclass
-from typing import Any, Callable, Optional
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from auto_round.algorithms.transforms.spinquant.cayley_optimizer import AdamAndSGDG
-
-logger = logging.getLogger("autoround.spinquant")
-
-
-# ---------------------------------------------------------------------------
-# Shared loss computation
-# ---------------------------------------------------------------------------
-
-
-def compute_rotation_loss(
- logits: torch.Tensor,
- ori_logits: torch.Tensor,
- loss_type: str = "kl_top",
- kl_top_k: int = 1000,
-) -> torch.Tensor:
- """Compute loss between rotated-model logits and original-model logits.
-
- This is the single implementation shared by both the preprocessor's
- embedded loop and the standalone :class:`RotationTrainer`.
-
- Args:
- logits: Rotated model output logits.
- ori_logits: Original (un-rotated) model output logits (detached).
- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.
- kl_top_k: Number of top logits to use for ``kl_top``.
-
- Returns:
- Scalar loss tensor.
- """
- if loss_type == "kl_top":
- k = min(kl_top_k, logits.size(-1))
- top_ori, indices = ori_logits.topk(k, dim=-1, sorted=False)
- top_logits = logits.gather(-1, indices)
- return F.kl_div(
- F.log_softmax(top_logits.flatten(0, -2), dim=-1),
- F.softmax(top_ori.flatten(0, -2), dim=-1),
- reduction="batchmean",
- )
- if loss_type == "kl_full":
- return F.kl_div(
- F.log_softmax(logits.flatten(0, -2), dim=-1),
- F.softmax(ori_logits.flatten(0, -2), dim=-1),
- reduction="batchmean",
- )
- if loss_type == "mse":
- return F.mse_loss(logits, ori_logits)
- raise ValueError(f"Unknown loss_type={loss_type!r}")
-
-
-# ---------------------------------------------------------------------------
-# Shared utilities
-# ---------------------------------------------------------------------------
-
-
-def move_batch_to_device(batch: Any, device: torch.device) -> Any:
- """Move a batch (tensor or dict of tensors) to *device*."""
- if hasattr(batch, "to"):
- return batch.to(device)
- if isinstance(batch, dict):
- return {k: v.to(device) if hasattr(v, "to") else v for k, v in batch.items()}
- return batch
-
-
-def check_orthogonality(model: nn.Module, threshold: float = 1e-3) -> float:
- """Check orthogonality of all ``spinquant_R*`` parameters.
-
- Returns:
- Maximum deviation from identity (``max ||R @ R^T - I||_inf``).
- """
- max_dev = 0.0
- for name, param in model.named_parameters():
- if not param.requires_grad or "spinquant_R" not in name:
- continue
- R = param.data
- if R.dim() != 2 or R.shape[0] != R.shape[1] or R.numel() == 0:
- continue
- I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)
- dev = (torch.matmul(R, R.t()) - I).abs().max().item()
- max_dev = max(max_dev, dev)
- if dev > 1e-4:
- logger.warning(f" {name} orthogonality deviation={dev:.2e}")
- if max_dev > 0:
- logger.info(f"[SpinQuant] Max orthogonality deviation={max_dev:.2e}")
- if max_dev > threshold:
- logger.warning(" Orthogonality constraint significantly violated!")
- return max_dev
-
-
-def clone_model_for_reference(model: nn.Module) -> nn.Module:
- """Create a frozen deep-copy of *model* for KL-divergence reference.
-
- Hooks registered by SpinQuant are removed from the clone so it runs
- as the un-rotated baseline.
- """
- from auto_round.algorithms.transforms.spinquant.preprocessor import (
- remove_spinquant_hooks_from_model,
- )
-
- original = copy.deepcopy(model)
- original.eval()
- for p in original.parameters():
- p.requires_grad = False
- remove_spinquant_hooks_from_model(original)
- return original
-
-
-def create_dual_optimizer(
- model: nn.Module,
- lr: float = 1e-4,
- smooth_lr: float = 1e-3,
-) -> Optional[AdamAndSGDG]:
- """Create the Adam (smooth) + SGDG (rotation) dual optimiser.
-
- Returns ``None`` if no trainable parameters are found.
- """
- rot_params = [p for n, p in model.named_parameters() if p.requires_grad and "spinquant_R" in n]
- smooth_params = [p for n, p in model.named_parameters() if p.requires_grad and "smooth_values" in n]
-
- if not rot_params and not smooth_params:
- logger.info("[SpinQuant] No trainable parameters — nothing to train.")
- return None
-
- n_rot = len(rot_params)
- n_smooth = len(smooth_params)
- total_params = sum(p.numel() for p in rot_params) + sum(p.numel() for p in smooth_params)
- logger.info(
- f"[SpinQuant] Training: {n_rot} rotation params + {n_smooth} smooth params "
- f"= {total_params:,} trainable parameters"
- )
-
- return AdamAndSGDG(
- adam_params=smooth_params,
- sgdg_params=rot_params,
- learning_rate=lr,
- smooth_learning_rate=smooth_lr,
- )
-
-
-# ---------------------------------------------------------------------------
-# Shared training loop
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class TrainingResult:
- """Result of :func:`run_training_loop`."""
-
- loss_history: list[float]
- best_loss: float
- final_ortho_deviation: float
- steps: int
-
-
-def run_training_loop(
- model: nn.Module,
- original_model: nn.Module,
- optimizer: AdamAndSGDG,
- dataloader: Any,
- *,
- max_iters: int = 200,
- loss_type: str = "kl_top",
- kl_top_k: int = 1000,
- compute_loss_fn: Optional[Callable] = None,
- on_step_end: Optional[Callable[[int, float, float], None]] = None,
- log_interval: int = 50,
-) -> TrainingResult:
- """Run the SpinQuant rotation training loop.
-
- This is the **single** training loop implementation shared by
- :class:`SpinQuantPreprocessor` (embedded) and :class:`RotationTrainer`
- (standalone).
-
- Args:
- model: The model being trained (rotation params + optional smooth params).
- original_model: Frozen reference model for KL divergence.
- optimizer: :class:`AdamAndSGDG` dual optimiser.
- dataloader: Iterable of batches.
- max_iters: Stop after this many steps.
- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.
- kl_top_k: Top-k for ``kl_top`` loss.
- compute_loss_fn: Optional override for loss computation. Signature:
- ``(logits, ori_logits) -> Tensor``. If ``None``, uses
- :func:`compute_rotation_loss`.
- on_step_end: Optional callback ``(step, loss, avg_loss) -> None``
- called after each gradient step.
- log_interval: Default logging interval (used when no ``on_step_end``
- is provided).
-
- Returns:
- :class:`TrainingResult` with loss history and orthogonality stats.
- """
- device = next(model.parameters()).device
- model.train()
-
- loss_fn = compute_loss_fn or (
- lambda logits, ori_logits: compute_rotation_loss(logits, ori_logits, loss_type, kl_top_k)
- )
-
- loss_history: list[float] = []
- best_loss = float("inf")
- step = 0
-
- for batch in dataloader:
- if step >= max_iters:
- break
-
- batch = move_batch_to_device(batch, device)
-
- # Forward: rotated model
- out_rot = model(**batch)
- logits = out_rot.logits if hasattr(out_rot, "logits") else out_rot
-
- # Forward: original model (no grad)
- with torch.no_grad():
- out_ori = original_model(**batch)
- ori_logits = out_ori.logits if hasattr(out_ori, "logits") else out_ori
-
- # Loss + backward
- loss = loss_fn(logits, ori_logits)
- loss.backward()
- optimizer.step()
- optimizer.zero_grad()
-
- loss_val = loss.item()
- loss_history.append(loss_val)
- best_loss = min(best_loss, loss_val)
-
- # Logging / callbacks
- if on_step_end is not None:
- avg = sum(loss_history[-50:]) / len(loss_history[-50:])
- on_step_end(step, loss_val, avg)
- elif step % log_interval == 0:
- avg = sum(loss_history[-50:]) / len(loss_history[-50:])
- logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")
-
- step += 1
-
- # Final orthogonality check
- ortho_dev = check_orthogonality(model)
-
- model.eval()
-
- return TrainingResult(
- loss_history=loss_history,
- best_loss=best_loss,
- final_ortho_deviation=ortho_dev,
- steps=step,
- )
diff --git a/test/test_cuda/transform/test_spinquant.py b/test/test_cuda/transform/test_spinquant.py
index 68099963c..29545b929 100644
--- a/test/test_cuda/transform/test_spinquant.py
+++ b/test/test_cuda/transform/test_spinquant.py
@@ -9,7 +9,6 @@ Tests cover:
- Rotation correctness: R1, R1+R2, R1+R2+R3+R4 produce valid logits
"""
-import copy
import shutil
import pytest
@@ -245,89 +244,22 @@ class TestHookLifecycle:
class TestRotationCorrectness:
"""Test that rotation produces valid (non-NaN, non-zero) logits."""
- @pytest.fixture(scope="class")
- def model_and_tokenizer(self):
- model_name = get_model_path("Qwen/Qwen3-0.6B")
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- model = AutoModelForCausalLM.from_pretrained(
- model_name, torch_dtype=torch.float16, trust_remote_code=True
- ).cuda()
- model.eval()
- yield model, tokenizer
- del model
- torch.cuda.empty_cache()
-
- @pytest.fixture
- def baseline_logits(self, model_and_tokenizer):
- """Get FP16 baseline logits for comparison."""
- model, tokenizer = model_and_tokenizer
- inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device)
- with torch.no_grad():
- logits = model(**inputs).logits
- return logits.clone()
-
- def _get_logits(self, model, tokenizer, text="The capital of France is"):
+ @staticmethod
+ def _get_logits(model, tokenizer, text="The capital of France is"):
inputs = tokenizer(text, return_tensors="pt").to(model.device)
with torch.no_grad():
return model(**inputs).logits
- def test_r1_only(self, model_and_tokenizer):
- """R1-only rotation should produce valid logits."""
- model_name = get_model_path("Qwen/Qwen3-0.6B")
- model = (
- AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)
- .cuda()
- .eval()
- )
-
- cfg = SpinQuantConfig(
- r1=True,
- r2=False,
- r3=False,
- r4=False,
- online_r1_rotation=True,
- trainable_rotation=False,
- trainable_smooth=False,
- )
- model = apply_rotation(model, cfg)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- logits = self._get_logits(model, tokenizer)
-
- assert not torch.isnan(logits).any(), "R1 rotation produced NaN logits"
- assert not torch.isinf(logits).any(), "R1 rotation produced Inf logits"
- assert logits.abs().sum() > 0, "R1 rotation produced all-zero logits"
- del model
- torch.cuda.empty_cache()
-
- def test_r1_r2(self, model_and_tokenizer):
- """R1+R2 rotation should produce valid logits."""
- model_name = get_model_path("Qwen/Qwen3-0.6B")
- model = (
- AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)
- .cuda()
- .eval()
- )
-
- cfg = SpinQuantConfig(
- r1=True,
- r2=True,
- r3=False,
- r4=False,
- online_r1_rotation=True,
- trainable_rotation=False,
- trainable_smooth=False,
- )
- model = apply_rotation(model, cfg)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- logits = self._get_logits(model, tokenizer)
-
- assert not torch.isnan(logits).any(), "R1+R2 rotation produced NaN logits"
- assert not torch.isinf(logits).any(), "R1+R2 rotation produced Inf logits"
- del model
- torch.cuda.empty_cache()
-
- def test_r1_r2_r3_r4(self, model_and_tokenizer):
- """Full R1+R2+R3+R4 rotation should produce valid logits."""
+ @pytest.mark.parametrize(
+ "r1,r2,r3,r4,label",
+ [
+ (True, False, False, False, "R1"),
+ (True, True, False, False, "R1+R2"),
+ (True, True, True, True, "R1+R2+R3+R4"),
+ ],
+ )
+ def test_rotation_produces_valid_logits(self, r1, r2, r3, r4, label):
+ """Rotation configurations should produce valid (non-NaN, non-Inf) logits."""
model_name = get_model_path("Qwen/Qwen3-0.6B")
model = (
AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)
@@ -336,10 +268,10 @@ class TestRotationCorrectness:
)
cfg = SpinQuantConfig(
- r1=True,
- r2=True,
- r3=True,
- r4=True,
+ r1=r1,
+ r2=r2,
+ r3=r3,
+ r4=r4,
online_r1_rotation=True,
trainable_rotation=False,
trainable_smooth=False,
@@ -348,8 +280,9 @@ class TestRotationCorrectness:
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
logits = self._get_logits(model, tokenizer)
- assert not torch.isnan(logits).any(), "Full rotation produced NaN logits"
- assert not torch.isinf(logits).any(), "Full rotation produced Inf logits"
+ assert not torch.isnan(logits).any(), f"{label} rotation produced NaN logits"
+ assert not torch.isinf(logits).any(), f"{label} rotation produced Inf logits"
+ assert logits.abs().sum() > 0, f"{label} rotation produced all-zero logits"
del model
torch.cuda.empty_cache()