PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor quarot/spinquant rotation with simplying code. (#1849)
lkkc73f36b59139
diff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.pyindex 8a707cab1..d2b63d9e7 100644--- a/auto_round/algorithms/transforms/__init__.py+++ b/auto_round/algorithms/transforms/__init__.py@@ -47,7 +47,7 @@ import torchfrom auto_round.algorithms.transforms.base import (BaseRotation,BaseRotationConfig,- RotationSerializer,+ SerializerMixin,ROTATION_SUPPORTED_SCHEMES,check_supported_schemes,_ensure_registry_populated,@@ -63,7 +63,7 @@ __all__ = [# Base interfaces"BaseRotation","BaseRotationConfig",- "RotationSerializer",+ "SerializerMixin","ROTATION_SUPPORTED_SCHEMES","check_supported_schemes",# Config@@ -185,8 +185,8 @@ import logging as _logging_dispatch_logger = _logging.getLogger("autoround.transforms.dispatch")-def _get_serializer(model: torch.nn.Module) -> RotationSerializer | None:- """Get the :class:`RotationSerializer` for the model's rotation config.+def _get_serializer(model: torch.nn.Module) -> SerializerMixin | None:+ """Get the :class:`SerializerMixin` for the model's rotation config.Returns ``None`` if no rotation was applied or the rotation algorithmdoes not support serialization.@@ -200,15 +200,15 @@ def _get_serializer(model: torch.nn.Module) -> RotationSerializer | None:rotation = BaseRotation.from_config(config)except (ValueError, KeyError):return None- if not isinstance(rotation, RotationSerializer):+ if not isinstance(rotation, SerializerMixin):return Nonereturn rotationdef _get_serializer_for_config(config_dict: dict,-) -> RotationSerializer | None:- """Get :class:`RotationSerializer` from a JSON config dict (load side).+) -> SerializerMixin | None:+ """Get :class:`SerializerMixin` from a JSON config dict (load side).The dict should have an ``"algorithm"`` field; defaults to``"spinquant"`` for backward compatibility.@@ -224,7 +224,7 @@ def _get_serializer_for_config(rotation = rotation_cls(config)except Exception:return None- if not isinstance(rotation, RotationSerializer):+ if not isinstance(rotation, SerializerMixin):return Nonereturn rotation@@ -239,7 +239,7 @@ def inject_rotation_buffers_on_layer() -> None:"""Per-layer rotation buffer injection (ShardWriter path).- Dispatches to the correct :class:`RotationSerializer` based on+ Dispatches to the correct :class:`SerializerMixin` based on``model._rotation_config``. No-op if no rotation config is present.Called from ``pack_layer()`` in all export files.@@ -255,7 +255,7 @@ def inject_rotation_buffers_bulk() -> None:"""Bulk rotation buffer injection (non-ShardWriter path).- Dispatches to the correct :class:`RotationSerializer` based on+ Dispatches to the correct :class:`SerializerMixin` based on``model._rotation_config``. No-op if no rotation config is present.Called from ``save_quantized_as_*()`` in all export files.@@ -271,7 +271,7 @@ def save_rotation_config() -> None:"""Persist rotation config to ``config.json``.- Dispatches to the correct :class:`RotationSerializer` based on+ Dispatches to the correct :class:`SerializerMixin` based on``model._rotation_config``. No-op if no rotation config is present.Called from ``save_quantized_as_*()`` in all export files, after@@ -292,7 +292,7 @@ def preregister_rotation_buffers("""Pre-register empty rotation buffers before state_dict loading.Extracts the rotation config dict from *quantization_config* by- checking each registered :class:`RotationSerializer`'s+ checking each registered :class:`SerializerMixin`'s:meth:`config_key` (e.g. ``"spinquant_config"``).Note: does NOT use ``"rotation_config"`` — that key is reserved for@@ -307,7 +307,7 @@ def preregister_rotation_buffers(if isinstance(quantization_config, dict):for _name, _cls in BaseRotation._REGISTRY.items():- if not (isinstance(_cls, type) and issubclass(_cls, RotationSerializer)):+ if not (isinstance(_cls, type) and issubclass(_cls, SerializerMixin)):continuetry:legacy_key = _cls.config_key()@@ -322,7 +322,7 @@ def preregister_rotation_buffers(# Object-style quantization_config (e.g. QuantizationConfig)# Check each registered serializer's config_key as attributefor _name, _cls in BaseRotation._REGISTRY.items():- if not (isinstance(_cls, type) and issubclass(_cls, RotationSerializer)):+ if not (isinstance(_cls, type) and issubclass(_cls, SerializerMixin)):continuetry:legacy_key = _cls.config_key()@@ -347,7 +347,7 @@ def preregister_rotation_buffers(def rebuild_rotation_if_needed(model: torch.nn.Module) -> None:"""Rebuild online rotation hooks after weights are loaded.- Scans all registered :class:`RotationSerializer` implementations to+ Scans all registered :class:`SerializerMixin` implementations tofind one whose buffers are present on the model, then calls its:meth:`rebuild_online`."""@@ -358,7 +358,7 @@ def rebuild_rotation_if_needed(model: torch.nn.Module) -> None:temp = rotation_cls(BaseRotationConfig(algorithm=name))except Exception:continue- if not isinstance(temp, RotationSerializer):+ if not isinstance(temp, SerializerMixin):continue# Quick scan: does any module have this method's buffers?diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.pyindex b18debc3e..db48c6059 100644--- a/auto_round/algorithms/transforms/base.py+++ b/auto_round/algorithms/transforms/base.py@@ -172,7 +172,7 @@ def _ensure_registry_populated() -> None:# ----------------------------------------------------------------------------class RotationSerializer(ABC):+class SerializerMixin(ABC):"""Mixin interface for rotation method serialization.Any :class:`BaseRotation` subclass that needs save/load support should@@ -185,7 +185,7 @@ class RotationSerializer(ABC):Usage::@BaseRotation.register("my_rotation")- class MyRotation(BaseRotation, RotationSerializer):+ class MyRotation(BaseRotation, SerializerMixin):def apply_to_model(self, ...): ...def inject_buffers_on_layer(self, ...): ......diff --git a/auto_round/algorithms/transforms/spinquant/__init__.py b/auto_round/algorithms/transforms/spinquant/__init__.pyindex ab8426994..9c90d3077 100644--- a/auto_round/algorithms/transforms/spinquant/__init__.py+++ b/auto_round/algorithms/transforms/spinquant/__init__.py@@ -70,7 +70,7 @@ Example (SpinQuant — trainable, ⚠️ experimental)::autoround.quantize()"""-from auto_round.algorithms.transforms.spinquant.algorithm import (+from auto_round.algorithms.transforms.spinquant.apply import (SpinQuantRotation,)from auto_round.algorithms.transforms.spinquant.cayley_optimizer import (@@ -87,12 +87,16 @@ from auto_round.algorithms.transforms.spinquant.preprocessor import (SpinQuantPreprocessor,TrainableRMSNorm,)-from auto_round.algorithms.transforms.spinquant.trainer import (+from auto_round.algorithms.transforms.spinquant.training import (LossLogger,OrthogonalityMonitor,RotationTrainer,RotationTrainerCallback,RotationTrainerConfig,+ SpinQuantState,+ SpinQuantTrainingHook,+ create_spinquant_optimizer,+ spinquant_loss_fn,)from auto_round.algorithms.transforms.spinquant.rotation_utils import (InputRotationWrapperHadamard,@@ -102,12 +106,6 @@ from auto_round.algorithms.transforms.spinquant.serialize import (rebuild_spinquant_online,save_spinquant_config,)-from auto_round.algorithms.transforms.spinquant.training import (- SpinQuantState,- SpinQuantTrainingHook,- create_spinquant_optimizer,- spinquant_loss_fn,-)__all__ = [# -- Registry algorithm (unified apply_rotation() entry) --diff --git a/auto_round/algorithms/transforms/spinquant/algorithm.py b/auto_round/algorithms/transforms/spinquant/apply.pysimilarity index 95%rename from auto_round/algorithms/transforms/spinquant/algorithm.pyrename to auto_round/algorithms/transforms/spinquant/apply.pyindex 13b60e96d..e8e4c8f68 100644--- a/auto_round/algorithms/transforms/spinquant/algorithm.py+++ b/auto_round/algorithms/transforms/spinquant/apply.py@@ -11,13 +11,13 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-"""SpinQuant / QuaRot rotation — ``BaseRotation`` + ``RotationSerializer``.+"""SpinQuant / QuaRot rotation — ``BaseRotation`` + ``SerializerMixin``.Registers ``"spinquant"`` in the :class:`BaseRotation` registry so thatthe unified ``apply_rotation(model, config)`` entry point can dispatch tothe SpinQuant preprocessing pipeline automatically.-Also implements :class:`RotationSerializer` so that the generic dispatch+Also implements :class:`SerializerMixin` so that the generic dispatchfunctions (``inject_rotation_buffers_on_layer``, etc.) route to SpinQuantserialization logic without export files ever naming "spinquant".@@ -50,13 +50,13 @@ from typing import Anyimport torchimport torch.nn as nn-from auto_round.algorithms.transforms.base import BaseRotation, RotationSerializer+from auto_round.algorithms.transforms.base import BaseRotation, SerializerMixinlogger = logging.getLogger("autoround")@BaseRotation.register("spinquant")-class SpinQuantRotation(BaseRotation, RotationSerializer):+class SpinQuantRotation(BaseRotation, SerializerMixin):"""QuaRot / SpinQuant rotation registered as ``"spinquant"`` in:class:`BaseRotation`.@@ -64,7 +64,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):pipeline (RMSNorm fusion → rotation matrix init → optional training →weight fusion → online hook registration).- Implements :class:`RotationSerializer` for save/load support — all+ Implements :class:`SerializerMixin` for save/load support — allserialization calls are delegated to the existing ``serialize.py``functions.@@ -107,7 +107,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):return preprocessor.preprocess(dataloader)# ------------------------------------------------------------------- # RotationSerializer — Save side+ # SerializerMixin — Save side# ------------------------------------------------------------------def inject_buffers_on_layer(@@ -203,7 +203,7 @@ class SpinQuantRotation(BaseRotation, RotationSerializer):logger.warning(f"Failed to save SpinQuant config: {e}")# ------------------------------------------------------------------- # RotationSerializer — Load side+ # SerializerMixin — Load side# ------------------------------------------------------------------def preregister_buffers(diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex cf498b0c1..a1f2cf178 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -551,13 +551,13 @@ class SpinQuantPreprocessor:def _train_rotations(self, dataloader: Any) -> None:"""Train rotation matrices using the shared training loop.- Delegates to :func:`training_core.run_training_loop` which is the+ Delegates to :func:`training.run_training_loop` which is thesingle implementation shared with :class:`RotationTrainer`.For advanced training features (callbacks, evaluation, checkpointing,- custom loss), use ``RotationTrainer`` from ``trainer.py`` directly.+ custom loss), use ``RotationTrainer`` from ``training.py`` directly."""- from auto_round.algorithms.transforms.spinquant.training_core import (+ from auto_round.algorithms.transforms.spinquant.training import (clone_model_for_reference,create_dual_optimizer,run_training_loop,diff --git a/auto_round/algorithms/transforms/spinquant/trainer.py b/auto_round/algorithms/transforms/spinquant/trainer.pydeleted file mode 100644index c691ff60a..000000000--- a/auto_round/algorithms/transforms/spinquant/trainer.py+++ /dev/null@@ -1,532 +0,0 @@-# # Copyright (C) 2026 Intel Corporation-# # SPDX-License-Identifier: Apache-2.0--"""-SpinQuant / QuaRot Trainer for AutoRound.--Provides a HuggingFace-Trainer-style interface for SpinQuant training,-while remaining dependency-light (does not require ``transformers.Trainer``).--The trainer manages the full training lifecycle:-- 1. Setup – init rotations, smooth values, optimiser, original-model clone- 2. Train – customisable training loop with logging / evaluation- 3. Fuse – offline rotation fusion into weights- 4. Export – return the transformed model for AutoRound quantisation--Key differences from Quark's ``SpinQuantTrainer``:- * No hard dependency on ``transformers.Trainer`` (works standalone).- * Plug-and-play with AutoRound's dataloader format.- * Supports mid-training evaluation (e.g. perplexity) via callbacks.- * Can be used both as an independent trainer and as a component inside- ``AutoRound.quantize()``.--Usage::-- from auto_round.algorithms.transforms.spinquant import (- RotationTrainer, RotationTrainerConfig- )-- trainer = RotationTrainer(- model,- config=RotationTrainerConfig(iters=200, lr=1e-4),- )- trainer.train(dataloader)- model_rotated = trainer.fuse()-- # Proceed to AutoRound quantisation- autoround = AutoRound(model_rotated, tokenizer, bits=4, ...)- autoround.quantize()-"""--from __future__ import annotations--import time-from dataclasses import dataclass, field-from typing import Any, Callable, Optional--import torch-import torch.nn as nn--from auto_round.algorithms.transforms.spinquant.inplace.apply import (- register_spinquant_hooks,-)-from auto_round.algorithms.transforms.spinquant.preprocessor import (- SpinQuantConfig,- SpinQuantPreprocessor,- get_model_arch_info,-)-from auto_round.algorithms.transforms.spinquant.rotation_utils import (- fuse_rmsnorm_in_model,- untie_word_embeddings_if_needed,-)---@dataclass-class RotationTrainerConfig:- """Training hyperparameters for ``RotationTrainer``.-- ⚠️ **Experimental**: The SpinQuant training loop has basic infrastructure- (Cayley SGD, KL loss, callbacks, checkpointing) but has NOT been validated- end-to-end on real models. For production use, prefer QuaRot mode via- ``SpinQuantPreprocessor`` with ``trainable_rotation=False``.- """-- # ---------- Rotation knobs ----------- r1: bool = True- r2: bool = True- r3: bool = True- r4: bool = True-- trainable_rotation: bool = True # False = fixed Hadamard (QuaRot)- trainable_smooth: bool = True # joint SmoothQuant training- online_r1_rotation: bool = False-- # ---------- Optimiser ----------- lr: float = 1e-4 # SGDG lr (rotation matrices)- smooth_lr: float = 1e-3 # Adam lr (smooth values)- iters: int = 200- batch_size: int = 1-- # ---------- Loss ----------- loss_type: str = "kl_top" # kl_top | kl_full | mse- kl_top_k: int = 1000-- # ---------- Pipeline ----------- fuse_rmsnorm: bool = True- untie_embeddings: bool = True-- # ---------- Misc ----------- dtype: torch.dtype = torch.float32- device: Optional[str] = None- log_interval: int = 50 # print every N steps- eval_interval: int = 0 # 0 = never- save_interval: int = 0 # 0 = never- checkpoint_dir: Optional[str] = None-- def __post_init__(self):- if self.device is None:- self.device = "cuda" if torch.cuda.is_available() else "cpu"---class RotationTrainerCallback:- """- Minimal callback interface (inspired by HF ``TrainerCallback``).-- Subclass and override any of the following hooks:-- * ``on_train_begin``- * ``on_train_end``- * ``on_step_begin``- * ``on_step_end``- * ``on_evaluate``- """-- def on_train_begin(self, args: RotationTrainerConfig, state: dict, control: dict):- pass-- def on_train_end(self, args: RotationTrainerConfig, state: dict, control: dict):- pass-- def on_step_begin(self, args: RotationTrainerConfig, state: dict, control: dict):- pass-- def on_step_end(self, args: RotationTrainerConfig, state: dict, control: dict):- pass-- def on_evaluate(self, args: RotationTrainerConfig, state: dict, metrics: dict):- pass---class OrthogonalityMonitor(RotationTrainerCallback):- """- Callback that monitors ``R @ R.T ≈ I`` during training.-- Prints a warning when the deviation exceeds a threshold.- """-- def __init__(self, threshold: float = 1e-3, log_interval: int = 50):- self.threshold = threshold- self.log_interval = log_interval-- def on_step_end(self, args, state, control):- step = state.get("step", 0)- if step % self.log_interval != 0:- return-- model = state.get("model")- if model is None:- return-- max_dev = 0.0- for name, param in model.named_parameters():- if not param.requires_grad or "spinquant_R" not in name:- continue- R = param.data- if R.dim() != 2 or R.shape[0] != R.shape[1]:- continue- I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)- dev = (torch.matmul(R, R.t()) - I).abs().max().item()- max_dev = max(max_dev, dev)-- state["ortho_deviation"] = max_dev- if max_dev > self.threshold:- print(f" [OrthogonalityMonitor] step={step} dev={max_dev:.2e} ⚠️ > {self.threshold}")---class LossLogger(RotationTrainerCallback):- """Callback that prints loss every ``log_interval`` steps."""-- def __init__(self, log_interval: int = 50):- self.log_interval = log_interval-- def on_step_end(self, args, state, control):- step = state.get("step", 0)- if step % self.log_interval == 0:- loss = state.get("loss", 0.0)- avg_loss = state.get("avg_loss", 0.0)- print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")---class RotationTrainer:- """- Trainer for SpinQuant / QuaRot orthogonal rotation matrices.-- ⚠️ **Experimental**: This trainer has basic infrastructure but the- SpinQuant training path has NOT been validated end-to-end on real models.- For production QuaRot usage (fixed Hadamard, no training), use- ``SpinQuantPreprocessor`` directly instead.-- Similar to Quark's ``SpinQuantTrainer`` but decoupled from- ``transformers.Trainer``, making it usable inside AutoRound without- extra heavy dependencies.-- Lifecycle::-- trainer = RotationTrainer(model, config, callbacks=[...])- trainer.train(dataloader) # training loop- model = trainer.fuse() # fuse into weights- # ... now feed model to AutoRound ...-- Arguments:- model: The transformer model (modified **in place**).- config: ``RotationTrainerConfig`` instance.- callbacks: List of ``RotationTrainerCallback`` subclasses.- compute_loss_fn: Optional custom loss callable. Signature::-- compute_loss_fn(logits, ori_logits, config) -> Tensor- """-- def __init__(- self,- model: nn.Module,- config: Optional[RotationTrainerConfig] = None,- callbacks: Optional[list[RotationTrainerCallback]] = None,- compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,- ):- self.model = model- self.config = config or RotationTrainerConfig()- self.callbacks = callbacks or [LossLogger(), OrthogonalityMonitor()]- self.compute_loss_fn = compute_loss_fn or self._default_compute_loss-- # Internal state (shared with callbacks)- self.state: dict[str, Any] = {- "step": 0,- "global_step": 0,- "loss": 0.0,- "avg_loss": 0.0,- "best_loss": float("inf"),- "ortho_deviation": 0.0,- "model": model,- }-- # Architecture info- info = get_model_arch_info(model)- self.hidden_size = info.get("hidden_size", 0)- self.head_dim = info.get("head_dim", 0)- self.num_q_heads = info.get("num_q_heads", 0)- self.num_kv_heads = info.get("num_kv_heads", 0)- self.intermediate_size = info.get("intermediate_size", 0)-- # Training components (created lazily)- self.optimizer = None- self._original_model: Optional[nn.Module] = None- self._hook_handles: list[Any] = []- self._rotated_modules: set[nn.Module] = set()- self._loss_buffer: list[float] = []-- # SpinQuant preprocessor (reused for init / fusion)- self._preprocessor = SpinQuantPreprocessor(model, self._to_sq_config(self.config))-- # ------------------------------------------------------------------- # Public API- # ------------------------------------------------------------------- def train(self, dataloader: Any) -> dict[str, Any]:- """- Run the full SpinQuant training loop.-- Returns a dict of training metrics::-- {- "loss_history": list[float],- "best_loss": float,- "final_ortho_deviation": float,- "steps": int,- "elapsed_sec": float,- }- """- self._setup_training(dataloader)- self._trigger_event("on_train_begin")-- t0 = time.time()- for batch in dataloader:- if self.state["step"] >= self.config.iters:- break-- self._trigger_event("on_step_begin")- loss = self._training_step(batch)- self._loss_buffer.append(loss)- self.state["loss"] = loss- self.state["avg_loss"] = sum(self._loss_buffer[-50:]) / len(self._loss_buffer[-50:])- self.state["best_loss"] = min(self.state["best_loss"], loss)- self.state["step"] += 1- self.state["global_step"] += 1-- # Mid-training evaluation- if self.config.eval_interval > 0 and self.state["step"] % self.config.eval_interval == 0:- metrics = self.evaluate(dataloader)- self._trigger_event("on_evaluate", metrics=metrics)-- # Checkpointing- if self.config.save_interval > 0 and self.state["step"] % self.config.save_interval == 0:- self.save_checkpoint()-- self._trigger_event("on_step_end")-- elapsed = time.time() - t0- self._trigger_event("on_train_end")- self._teardown_training()-- return {- "loss_history": self._loss_buffer,- "best_loss": self.state["best_loss"],- "final_ortho_deviation": self.state.get("ortho_deviation", 0.0),- "steps": self.state["step"],- "elapsed_sec": elapsed,- }-- def evaluate(self, dataloader: Any) -> dict[str, float]:- """- Evaluate current rotated model against the original on a subset.-- Returns a dict with ``loss``, ``max_diff``, etc.- """- from auto_round.algorithms.transforms.spinquant.training_core import (- compute_rotation_loss,- move_batch_to_device,- )-- if self._original_model is None:- return {}-- self.model.eval()- device = next(self.model.parameters()).device- losses = []- max_diffs = []-- for i, batch in enumerate(dataloader):- if i >= 10: # evaluate on first 10 batches only- break- batch = move_batch_to_device(batch, device)- with torch.no_grad():- out_rot = self.model(**batch)- logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot- out_ori = self._original_model(**batch)- logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori-- loss = compute_rotation_loss(- logits_rot,- logits_ori,- loss_type=self.config.loss_type,- kl_top_k=self.config.kl_top_k,- )- losses.append(loss.item())- max_diffs.append((logits_rot - logits_ori).abs().max().item())-- self.model.train()- return {- "eval_loss": sum(losses) / len(losses),- "eval_max_diff": sum(max_diffs) / len(max_diffs),- }-- def fuse(self) -> nn.Module:- """- Fuse offline rotations into model weights and clean up.-- Returns the modified ``model`` (same object, in-place).- """- self._preprocessor._fuse_offline_rotations()- self._preprocessor._cleanup()- return self.model-- def save_checkpoint(self, path: Optional[str] = None) -> str:- """Save rotation + smooth params to disk."""- if path is None:- path = f"{self.config.checkpoint_dir or '.'}/spinquant_ckpt_step{self.state['step']}.pt"- ckpt = {- "step": self.state["step"],- "config": self.config,- "rotation_params": {- n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "spinquant_R" in n- },- "smooth_params": {- n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "smooth_values" in n- },- }- torch.save(ckpt, path)- print(f"[RotationTrainer] Checkpoint saved: {path}")- return path-- def load_checkpoint(self, path: str) -> None:- """Restore rotation + smooth params from disk."""- ckpt = torch.load(path, map_location="cpu", weights_only=True)- for n, p in self.model.named_parameters():- if n in ckpt["rotation_params"]:- p.data.copy_(ckpt["rotation_params"][n].to(p.device))- if n in ckpt["smooth_params"]:- p.data.copy_(ckpt["smooth_params"][n].to(p.device))- self.state["step"] = ckpt["step"]- print(f"[RotationTrainer] Checkpoint loaded: {path}")-- # ------------------------------------------------------------------- # Internal helpers- # ------------------------------------------------------------------- def _setup_training(self, dataloader: Any) -> None:- """Initialise rotations, optimiser, and original-model clone."""- from auto_round.algorithms.transforms.spinquant.training_core import (- clone_model_for_reference,- create_dual_optimizer,- )-- cfg = self.config-- # 1. Untie + RMSNorm fusion- if cfg.untie_embeddings:- untie_word_embeddings_if_needed(self.model)- if cfg.fuse_rmsnorm:- fuse_rmsnorm_in_model(self.model)-- # 2. Replace norms with trainable wrappers (SmoothQuant)- if cfg.trainable_smooth:- self._preprocessor._replace_norms_with_trainable()-- # 3. Initialise rotation matrices- self._preprocessor._init_rotation_matrices()-- # 4. Register online hooks (R3 / R4)- if cfg.r3 or cfg.r4:- self._hook_handles = register_spinquant_hooks(- self.model,- self._to_sq_config(cfg),- head_dim=self.head_dim,- intermediate_size=self.intermediate_size,- )-- # 5. Create dual optimiser (shared helper)- self.optimizer = create_dual_optimizer(- self.model,- lr=cfg.lr,- smooth_lr=cfg.smooth_lr,- )-- # 6. Clone original model for KL reference (shared helper)- self._original_model = clone_model_for_reference(self.model)-- # 7. Use model's existing device (don't move)- self.model.train()-- def _training_step(self, batch: Any) -> float:- """Single training step. Returns loss scalar."""- from auto_round.algorithms.transforms.spinquant.training_core import (- compute_rotation_loss,- move_batch_to_device,- )-- device = next(self.model.parameters()).device- batch = move_batch_to_device(batch, device)-- # Forward: rotated model- out_rot = self.model(**batch)- logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot-- # Forward: original model (no grad)- with torch.no_grad():- assert self._original_model is not None- out_ori = self._original_model(**batch)- logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori-- # Loss & backward — use custom loss_fn if set, else shared core- if self.compute_loss_fn is not self._default_compute_loss:- loss = self.compute_loss_fn(logits_rot, logits_ori, self.config)- else:- loss = compute_rotation_loss(- logits_rot,- logits_ori,- loss_type=self.config.loss_type,- kl_top_k=self.config.kl_top_k,- )- loss.backward()- self.optimizer.step() # type: ignore[union-attr]- self.optimizer.zero_grad() # type: ignore[union-attr]-- return loss.item()-- def _teardown_training(self) -> None:- """Free original model and set eval mode."""- if self._original_model is not None:- del self._original_model- self._original_model = None- if torch.cuda.is_available():- torch.cuda.empty_cache()- self.model.eval()-- def _trigger_event(self, event_name: str, **kwargs) -> None:- """Fire callback hooks."""- for cb in self.callbacks:- method = getattr(cb, event_name, None)- if callable(method):- method(self.config, self.state, {"should_training_stop": False, **kwargs})-- @staticmethod- def _default_compute_loss(- logits: torch.Tensor,- ori_logits: torch.Tensor,- config: RotationTrainerConfig,- ) -> torch.Tensor:- """Default loss — delegates to :func:`training_core.compute_rotation_loss`."""- from auto_round.algorithms.transforms.spinquant.training_core import compute_rotation_loss-- return compute_rotation_loss(logits, ori_logits, config.loss_type, config.kl_top_k)-- @staticmethod- def _to_sq_config(cfg: RotationTrainerConfig) -> SpinQuantConfig:- """Convert trainer config to SpinQuantConfig for internal reuse."""- return SpinQuantConfig(- r1=cfg.r1,- r2=cfg.r2,- r3=cfg.r3,- r4=cfg.r4,- trainable_rotation=cfg.trainable_rotation,- trainable_smooth=cfg.trainable_smooth,- online_r1_rotation=cfg.online_r1_rotation,- iters=cfg.iters,- lr=cfg.lr,- smooth_lr=cfg.smooth_lr,- batch_size=cfg.batch_size,- loss_type=cfg.loss_type,- kl_top_k=cfg.kl_top_k,- fuse_rmsnorm=cfg.fuse_rmsnorm,- untie_embeddings=cfg.untie_embeddings,- dtype=cfg.dtype,- device=cfg.device,- )diff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.pyindex 99d55ae8d..e3e1e2b64 100644--- a/auto_round/algorithms/transforms/spinquant/training.py+++ b/auto_round/algorithms/transforms/spinquant/training.py@@ -1,37 +1,311 @@-# # Copyright (C) 2026 Intel Corporation-# # SPDX-License-Identifier: Apache-2.0+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""SpinQuant / QuaRot training infrastructure.++⚠️ **Experimental**: The SpinQuant training path has basic infrastructure+(Cayley SGD optimizer, KL-divergence loss, callbacks, checkpointing) but+has NOT been validated end-to-end on real models. For production use, prefer+QuaRot mode (fixed Hadamard, no training) via ``SpinQuantPreprocessor``.-"""-Training utilities for SpinQuant in AutoRound.+This module consolidates all training-related functionality:-⚠️ **Experimental**: These training utilities provide hooks and helpers for-integrating SpinQuant training into AutoRound's calibration pipeline, but the-training path has NOT been validated end-to-end. For production use, prefer-QuaRot mode (fixed Hadamard, no training) via ``SpinQuantPreprocessor``.+* **Core primitives** — loss computation, training loop, optimizer creation+* **Pipeline hooks** — ``SpinQuantTrainingHook`` for AutoRound integration+* **Standalone trainer** — ``RotationTrainer`` (HF Trainer style)+* **Callbacks** — orthogonality monitoring, loss logging+* **State tracking** — ``SpinQuantState`` for metrics++Usage (standalone trainer)::-Provides hooks, callbacks, and helper functions for integrating-SpinQuant training into AutoRound's calibration pipeline.+ from auto_round.algorithms.transforms.spinquant.training import (+ RotationTrainer, RotationTrainerConfig+ )++ trainer = RotationTrainer(model, config=RotationTrainerConfig(iters=200))+ trainer.train(dataloader)+ model = trainer.fuse()-Key design: SpinQuant training happens BEFORE AutoRound's block-wise-quantization calibration. After rotation matrices are learned and fused,-the model is mathematically equivalent to the original, so AutoRound-can apply its standard quantization without any modifications.+Usage (pipeline hook)::++ hook = SpinQuantTrainingHook(model, config)+ hook.preprocess(dataloader)"""from __future__ import annotations-from typing import Any, Optional+import copy+import logging+import time+from dataclasses import asdict, dataclass, field+from typing import Any, Callable, Optionalimport torchimport torch.nn as nn+import torch.nn.functional as F++from auto_round.algorithms.transforms.spinquant.cayley_optimizer import AdamAndSGDG++logger = logging.getLogger("autoround.spinquant")+++# ===========================================================================+# Core primitives: loss, utilities, training loop+# ===========================================================================+++def compute_rotation_loss(+ logits: torch.Tensor,+ ori_logits: torch.Tensor,+ loss_type: str = "kl_top",+ kl_top_k: int = 1000,+) -> torch.Tensor:+ """Compute loss between rotated-model logits and original-model logits.++ This is the single implementation shared by both the preprocessor's+ embedded loop and the standalone :class:`RotationTrainer`.++ Args:+ logits: Rotated model output logits.+ ori_logits: Original (un-rotated) model output logits (detached).+ loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.+ kl_top_k: Number of top logits to use for ``kl_top``.++ Returns:+ Scalar loss tensor.+ """+ if loss_type == "kl_top":+ k = min(kl_top_k, logits.size(-1))+ top_ori, indices = ori_logits.topk(k, dim=-1, sorted=False)+ top_logits = logits.gather(-1, indices)+ return F.kl_div(+ F.log_softmax(top_logits.flatten(0, -2), dim=-1),+ F.softmax(top_ori.flatten(0, -2), dim=-1),+ reduction="batchmean",+ )+ if loss_type == "kl_full":+ return F.kl_div(+ F.log_softmax(logits.flatten(0, -2), dim=-1),+ F.softmax(ori_logits.flatten(0, -2), dim=-1),+ reduction="batchmean",+ )+ if loss_type == "mse":+ return F.mse_loss(logits, ori_logits)+ raise ValueError(f"Unknown loss_type={loss_type!r}")+++# Alias for backward compatibility (training.py used to export this)+spinquant_loss_fn = compute_rotation_loss+++def move_batch_to_device(batch: Any, device: torch.device) -> Any:+ """Move a batch (tensor or dict of tensors) to *device*."""+ if hasattr(batch, "to"):+ return batch.to(device)+ if isinstance(batch, dict):+ return {k: v.to(device) if hasattr(v, "to") else v for k, v in batch.items()}+ return batch+-from auto_round.algorithms.transforms.spinquant.cayley_optimizer import (- AdamAndSGDG,-)-from auto_round.algorithms.transforms.spinquant.preprocessor import (- SpinQuantConfig,- SpinQuantPreprocessor,-)+def check_orthogonality(model: nn.Module, threshold: float = 1e-3) -> float:+ """Check orthogonality of all ``spinquant_R*`` parameters.++ Returns:+ Maximum deviation from identity (``max ||R @ R^T - I||_inf``).+ """+ max_dev = 0.0+ for name, param in model.named_parameters():+ if not param.requires_grad or "spinquant_R" not in name:+ continue+ R = param.data+ if R.dim() != 2 or R.shape[0] != R.shape[1] or R.numel() == 0:+ continue+ I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)+ dev = (torch.matmul(R, R.t()) - I).abs().max().item()+ max_dev = max(max_dev, dev)+ if dev > 1e-4:+ logger.warning(f" {name} orthogonality deviation={dev:.2e}")+ if max_dev > 0:+ logger.info(f"[SpinQuant] Max orthogonality deviation={max_dev:.2e}")+ if max_dev > threshold:+ logger.warning(" Orthogonality constraint significantly violated!")+ return max_dev+++def clone_model_for_reference(model: nn.Module) -> nn.Module:+ """Create a frozen deep-copy of *model* for KL-divergence reference.++ Hooks registered by SpinQuant are removed from the clone so it runs+ as the un-rotated baseline.+ """+ from auto_round.algorithms.transforms.spinquant.preprocessor import (+ remove_spinquant_hooks_from_model,+ )++ original = copy.deepcopy(model)+ original.eval()+ for p in original.parameters():+ p.requires_grad = False+ remove_spinquant_hooks_from_model(original)+ return original+++def create_dual_optimizer(+ model: nn.Module,+ lr: float = 1e-4,+ smooth_lr: float = 1e-3,+) -> Optional[AdamAndSGDG]:+ """Create the Adam (smooth) + SGDG (rotation) dual optimiser.++ Returns ``None`` if no trainable parameters are found.+ """+ rot_params = [p for n, p in model.named_parameters() if p.requires_grad and "spinquant_R" in n]+ smooth_params = [p for n, p in model.named_parameters() if p.requires_grad and "smooth_values" in n]++ if not rot_params and not smooth_params:+ logger.info("[SpinQuant] No trainable parameters — nothing to train.")+ return None++ n_rot = len(rot_params)+ n_smooth = len(smooth_params)+ total_params = sum(p.numel() for p in rot_params) + sum(p.numel() for p in smooth_params)+ logger.info(+ f"[SpinQuant] Training: {n_rot} rotation params + {n_smooth} smooth params "+ f"= {total_params:,} trainable parameters"+ )++ return AdamAndSGDG(+ adam_params=smooth_params,+ sgdg_params=rot_params,+ learning_rate=lr,+ smooth_learning_rate=smooth_lr,+ )+++# Alias for backward compatibility+create_spinquant_optimizer = create_dual_optimizer+++@dataclass+class TrainingResult:+ """Result of :func:`run_training_loop`."""++ loss_history: list[float]+ best_loss: float+ final_ortho_deviation: float+ steps: int+++def run_training_loop(+ model: nn.Module,+ original_model: nn.Module,+ optimizer: AdamAndSGDG,+ dataloader: Any,+ *,+ max_iters: int = 200,+ loss_type: str = "kl_top",+ kl_top_k: int = 1000,+ compute_loss_fn: Optional[Callable] = None,+ on_step_end: Optional[Callable[[int, float, float], None]] = None,+ log_interval: int = 50,+) -> TrainingResult:+ """Run the SpinQuant rotation training loop.++ This is the **single** training loop implementation shared by+ :class:`SpinQuantPreprocessor` (embedded) and :class:`RotationTrainer`+ (standalone).++ Args:+ model: The model being trained (rotation params + optional smooth params).+ original_model: Frozen reference model for KL divergence.+ optimizer: :class:`AdamAndSGDG` dual optimiser.+ dataloader: Iterable of batches.+ max_iters: Stop after this many steps.+ loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.+ kl_top_k: Top-k for ``kl_top`` loss.+ compute_loss_fn: Optional override for loss computation. Signature:+ ``(logits, ori_logits) -> Tensor``. If ``None``, uses+ :func:`compute_rotation_loss`.+ on_step_end: Optional callback ``(step, loss, avg_loss) -> None``+ called after each gradient step.+ log_interval: Default logging interval (used when no ``on_step_end``+ is provided).++ Returns:+ :class:`TrainingResult` with loss history and orthogonality stats.+ """+ device = next(model.parameters()).device+ model.train()++ loss_fn = compute_loss_fn or (+ lambda logits, ori_logits: compute_rotation_loss(logits, ori_logits, loss_type, kl_top_k)+ )++ loss_history: list[float] = []+ best_loss = float("inf")+ step = 0++ for batch in dataloader:+ if step >= max_iters:+ break++ batch = move_batch_to_device(batch, device)++ # Forward: rotated model+ out_rot = model(**batch)+ logits = out_rot.logits if hasattr(out_rot, "logits") else out_rot++ # Forward: original model (no grad)+ with torch.no_grad():+ out_ori = original_model(**batch)+ ori_logits = out_ori.logits if hasattr(out_ori, "logits") else out_ori++ # Loss + backward+ loss = loss_fn(logits, ori_logits)+ loss.backward()+ optimizer.step()+ optimizer.zero_grad()++ loss_val = loss.item()+ loss_history.append(loss_val)+ best_loss = min(best_loss, loss_val)++ # Logging / callbacks+ if on_step_end is not None:+ avg = sum(loss_history[-50:]) / len(loss_history[-50:])+ on_step_end(step, loss_val, avg)+ elif step % log_interval == 0:+ avg = sum(loss_history[-50:]) / len(loss_history[-50:])+ logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")++ step += 1++ # Final orthogonality check+ ortho_dev = check_orthogonality(model)++ model.eval()++ return TrainingResult(+ loss_history=loss_history,+ best_loss=best_loss,+ final_ortho_deviation=ortho_dev,+ steps=step,+ )+++# ===========================================================================+# Pipeline hooks: SpinQuantTrainingHook for AutoRound integration+# ===========================================================================class SpinQuantTrainingHook:@@ -53,9 +327,14 @@ class SpinQuantTrainingHook:def __init__(self,model: nn.Module,- config: Optional[SpinQuantConfig] = None,+ config=None,enabled: bool = True,):+ from auto_round.algorithms.transforms.spinquant.preprocessor import (+ SpinQuantConfig,+ SpinQuantPreprocessor,+ )+self.model = modelself.config = config or SpinQuantConfig()self.enabled = enabled@@ -66,6 +345,8 @@ class SpinQuantTrainingHook:if not self.enabled:return self.model+ from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantPreprocessor+self.preprocessor = SpinQuantPreprocessor(self.model, self.config)return self.preprocessor.preprocess(dataloader)@@ -112,97 +393,10 @@ class OrthogonalTrainingCallback:self.max_deviation_history.append(max_dev)if max_dev > 1e-4:- print(f"[SpinQuant] Step {self.step}: max orthogonality deviation = {max_dev:.2e}")+ logger.info(f"[SpinQuant] Step {self.step}: max orthogonality deviation = {max_dev:.2e}")if max_dev > 1e-2:- print(" 🔴 WARNING: Orthogonality significantly violated! Consider reducing lr.")---def create_spinquant_optimizer(- model: nn.Module,- lr: float = 1e-4,- smooth_lr: float = 1e-3,-) -> Optional[torch.optim.Optimizer]:- """- Create an AdamAndSGDG optimizer for SpinQuant parameters in a model.-- This is useful when AutoRound's own training loop wants to manage- the optimizer externally rather than letting SpinQuant create it internally.-- Args:- model: The model that has been set up with SpinQuant parameters- lr: Learning rate for rotation matrices (SGDG)- smooth_lr: Learning rate for smooth values (Adam)-- Returns:- AdamAndSGDG optimizer or None if no SpinQuant parameters found- """- rotation_params = []- smooth_params = []-- for name, param in model.named_parameters():- if not param.requires_grad:- continue- if "spinquant_rotation" in name or "spinquant_R" in name:- rotation_params.append(param)- elif "spinquant_smooth" in name or "smooth_values" in name:- smooth_params.append(param)-- if len(rotation_params) == 0 and len(smooth_params) == 0:- return None-- return AdamAndSGDG(- adam_params=smooth_params,- sgdg_params=rotation_params,- learning_rate=lr,- smooth_learning_rate=smooth_lr,- )---def spinquant_loss_fn(- logits: torch.Tensor,- original_logits: torch.Tensor,- loss_type: str = "kl_top",- kl_top_k: int = 1000,-) -> torch.Tensor:- """- Compute the SpinQuant training loss.-- Supports KL divergence (top-k or full) and MSE.-- Args:- logits: Logits from the rotated model- original_logits: Logits from the original (unrotated) model- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``- kl_top_k: Number of top logits for KL divergence-- Returns:- Loss tensor- """- import torch.nn.functional as F-- if loss_type == "kl_top":- k = min(kl_top_k, logits.size(-1))- top_ori_logits, indices = original_logits.topk(k, dim=-1, sorted=False)- top_logits = logits.gather(-1, indices)-- loss = F.kl_div(- F.log_softmax(top_logits.flatten(0, -2), dim=-1),- F.softmax(top_ori_logits.flatten(0, -2), dim=-1),- reduction="batchmean",- )- elif loss_type == "kl_full":- loss = F.kl_div(- F.log_softmax(logits.flatten(0, -2), dim=-1),- F.softmax(original_logits.flatten(0, -2), dim=-1),- reduction="batchmean",- )- elif loss_type == "mse":- loss = F.mse_loss(logits, original_logits)- else:- raise ValueError(f"Unknown loss_type: {loss_type}. Supported: kl_top, kl_full, mse")-- return loss+ logger.warning("[SpinQuant] Orthogonality significantly violated! Consider reducing lr.")class SpinQuantState:@@ -245,3 +439,488 @@ class SpinQuantState:"rotation_params": self.rotation_names,"orthogonality_deviation": self.orthogonality_deviation[-1] if self.orthogonality_deviation else None,}+++# ===========================================================================+# Standalone trainer: RotationTrainer (HF Trainer style)+# ===========================================================================+++@dataclass+class RotationTrainerConfig:+ """Training hyperparameters for ``RotationTrainer``.++ ⚠️ **Experimental**: The SpinQuant training loop has basic infrastructure+ (Cayley SGD, KL loss, callbacks, checkpointing) but has NOT been validated+ end-to-end on real models. For production use, prefer QuaRot mode via+ ``SpinQuantPreprocessor`` with ``trainable_rotation=False``.+ """++ # ---------- Rotation knobs ----------+ r1: bool = True+ r2: bool = True+ r3: bool = True+ r4: bool = True++ trainable_rotation: bool = True # False = fixed Hadamard (QuaRot)+ trainable_smooth: bool = True # joint SmoothQuant training+ online_r1_rotation: bool = False++ # ---------- Optimiser ----------+ lr: float = 1e-4 # SGDG lr (rotation matrices)+ smooth_lr: float = 1e-3 # Adam lr (smooth values)+ iters: int = 200+ batch_size: int = 1++ # ---------- Loss ----------+ loss_type: str = "kl_top" # kl_top | kl_full | mse+ kl_top_k: int = 1000++ # ---------- Pipeline ----------+ fuse_rmsnorm: bool = True+ untie_embeddings: bool = True++ # ---------- Misc ----------+ dtype: torch.dtype = torch.float32+ device: Optional[str] = None+ log_interval: int = 50 # print every N steps+ eval_interval: int = 0 # 0 = never+ save_interval: int = 0 # 0 = never+ checkpoint_dir: Optional[str] = None++ def __post_init__(self):+ if self.device is None:+ self.device = "cuda" if torch.cuda.is_available() else "cpu"+++class RotationTrainerCallback:+ """+ Minimal callback interface (inspired by HF ``TrainerCallback``).++ Subclass and override any of the following hooks:++ * ``on_train_begin``+ * ``on_train_end``+ * ``on_step_begin``+ * ``on_step_end``+ * ``on_evaluate``+ """++ def on_train_begin(self, args: RotationTrainerConfig, state: dict, control: dict):+ pass++ def on_train_end(self, args: RotationTrainerConfig, state: dict, control: dict):+ pass++ def on_step_begin(self, args: RotationTrainerConfig, state: dict, control: dict):+ pass++ def on_step_end(self, args: RotationTrainerConfig, state: dict, control: dict):+ pass++ def on_evaluate(self, args: RotationTrainerConfig, state: dict, metrics: dict):+ pass+++class OrthogonalityMonitor(RotationTrainerCallback):+ """+ Callback that monitors ``R @ R.T ≈ I`` during training.++ Prints a warning when the deviation exceeds a threshold.+ """++ def __init__(self, threshold: float = 1e-3, log_interval: int = 50):+ self.threshold = threshold+ self.log_interval = log_interval++ def on_step_end(self, args, state, control):+ step = state.get("step", 0)+ if step % self.log_interval != 0:+ return++ model = state.get("model")+ if model is None:+ return++ max_dev = 0.0+ for name, param in model.named_parameters():+ if not param.requires_grad or "spinquant_R" not in name:+ continue+ R = param.data+ if R.dim() != 2 or R.shape[0] != R.shape[1]:+ continue+ I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)+ dev = (torch.matmul(R, R.t()) - I).abs().max().item()+ max_dev = max(max_dev, dev)++ state["ortho_deviation"] = max_dev+ if max_dev > self.threshold:+ print(f" [OrthogonalityMonitor] step={step} dev={max_dev:.2e} > {self.threshold}")+++class LossLogger(RotationTrainerCallback):+ """Callback that prints loss every ``log_interval`` steps."""++ def __init__(self, log_interval: int = 50):+ self.log_interval = log_interval++ def on_step_end(self, args, state, control):+ step = state.get("step", 0)+ if step % self.log_interval == 0:+ loss = state.get("loss", 0.0)+ avg_loss = state.get("avg_loss", 0.0)+ print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")+++class RotationTrainer:+ """+ Trainer for SpinQuant / QuaRot orthogonal rotation matrices.++ ⚠️ **Experimental**: This trainer has basic infrastructure but the+ SpinQuant training path has NOT been validated end-to-end on real models.+ For production QuaRot usage (fixed Hadamard, no training), use+ ``SpinQuantPreprocessor`` directly instead.++ Similar to Quark's ``SpinQuantTrainer`` but decoupled from+ ``transformers.Trainer``, making it usable inside AutoRound without+ extra heavy dependencies.++ Lifecycle::++ trainer = RotationTrainer(model, config, callbacks=[...])+ trainer.train(dataloader) # training loop+ model = trainer.fuse() # fuse into weights+ # ... now feed model to AutoRound ...++ Arguments:+ model: The transformer model (modified **in place**).+ config: ``RotationTrainerConfig`` instance.+ callbacks: List of ``RotationTrainerCallback`` subclasses.+ compute_loss_fn: Optional custom loss callable. Signature::++ compute_loss_fn(logits, ori_logits, config) -> Tensor+ """++ def __init__(+ self,+ model: nn.Module,+ config: Optional[RotationTrainerConfig] = None,+ callbacks: Optional[list[RotationTrainerCallback]] = None,+ compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,+ ):+ from auto_round.algorithms.transforms.spinquant.preprocessor import (+ SpinQuantConfig,+ SpinQuantPreprocessor,+ get_model_arch_info,+ )++ self.model = model+ self.config = config or RotationTrainerConfig()+ self.callbacks = callbacks or [LossLogger(), OrthogonalityMonitor()]+ self.compute_loss_fn = compute_loss_fn or self._default_compute_loss++ # Internal state (shared with callbacks)+ self.state: dict[str, Any] = {+ "step": 0,+ "global_step": 0,+ "loss": 0.0,+ "avg_loss": 0.0,+ "best_loss": float("inf"),+ "ortho_deviation": 0.0,+ "model": model,+ }++ # Architecture info+ info = get_model_arch_info(model)+ self.hidden_size = info.get("hidden_size", 0)+ self.head_dim = info.get("head_dim", 0)+ self.num_q_heads = info.get("num_q_heads", 0)+ self.num_kv_heads = info.get("num_kv_heads", 0)+ self.intermediate_size = info.get("intermediate_size", 0)++ # Training components (created lazily)+ self.optimizer = None+ self._original_model: Optional[nn.Module] = None+ self._hook_handles: list[Any] = []+ self._rotated_modules: set[nn.Module] = set()+ self._loss_buffer: list[float] = []++ # SpinQuant preprocessor (reused for init / fusion)+ self._preprocessor = SpinQuantPreprocessor(model, self._to_sq_config(self.config))++ # ------------------------------------------------------------------+ # Public API+ # ------------------------------------------------------------------+ def train(self, dataloader: Any) -> dict[str, Any]:+ """+ Run the full SpinQuant training loop.++ Returns a dict of training metrics::++ {+ "loss_history": list[float],+ "best_loss": float,+ "final_ortho_deviation": float,+ "steps": int,+ "elapsed_sec": float,+ }+ """+ self._setup_training(dataloader)+ self._trigger_event("on_train_begin")++ t0 = time.time()+ for batch in dataloader:+ if self.state["step"] >= self.config.iters:+ break++ self._trigger_event("on_step_begin")+ loss = self._training_step(batch)+ self._loss_buffer.append(loss)+ self.state["loss"] = loss+ self.state["avg_loss"] = sum(self._loss_buffer[-50:]) / len(self._loss_buffer[-50:])+ self.state["best_loss"] = min(self.state["best_loss"], loss)+ self.state["step"] += 1+ self.state["global_step"] += 1++ # Mid-training evaluation+ if self.config.eval_interval > 0 and self.state["step"] % self.config.eval_interval == 0:+ metrics = self.evaluate(dataloader)+ self._trigger_event("on_evaluate", metrics=metrics)++ # Checkpointing+ if self.config.save_interval > 0 and self.state["step"] % self.config.save_interval == 0:+ self.save_checkpoint()++ self._trigger_event("on_step_end")++ elapsed = time.time() - t0+ self._trigger_event("on_train_end")+ self._teardown_training()++ return {+ "loss_history": self._loss_buffer,+ "best_loss": self.state["best_loss"],+ "final_ortho_deviation": self.state.get("ortho_deviation", 0.0),+ "steps": self.state["step"],+ "elapsed_sec": elapsed,+ }++ def evaluate(self, dataloader: Any) -> dict[str, float]:+ """+ Evaluate current rotated model against the original on a subset.++ Returns a dict with ``loss``, ``max_diff``, etc.+ """+ if self._original_model is None:+ return {}++ self.model.eval()+ device = next(self.model.parameters()).device+ losses = []+ max_diffs = []++ for i, batch in enumerate(dataloader):+ if i >= 10: # evaluate on first 10 batches only+ break+ batch = move_batch_to_device(batch, device)+ with torch.no_grad():+ out_rot = self.model(**batch)+ logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot+ out_ori = self._original_model(**batch)+ logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori++ loss = compute_rotation_loss(+ logits_rot,+ logits_ori,+ loss_type=self.config.loss_type,+ kl_top_k=self.config.kl_top_k,+ )+ losses.append(loss.item())+ max_diffs.append((logits_rot - logits_ori).abs().max().item())++ self.model.train()+ if not losses:+ return {"eval_loss": 0.0, "eval_max_diff": 0.0}+ return {+ "eval_loss": sum(losses) / len(losses),+ "eval_max_diff": sum(max_diffs) / len(max_diffs),+ }++ def fuse(self) -> nn.Module:+ """+ Fuse offline rotations into model weights and clean up.++ Returns the modified ``model`` (same object, in-place).+ """+ self._preprocessor._fuse_offline_rotations()+ self._preprocessor._cleanup()+ return self.model++ def save_checkpoint(self, path: Optional[str] = None) -> str:+ """Save rotation + smooth params to disk."""+ if path is None:+ path = f"{self.config.checkpoint_dir or '.'}/spinquant_ckpt_step{self.state['step']}.pt"+ ckpt = {+ "step": self.state["step"],+ "config": asdict(self.config),+ "rotation_params": {+ n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "spinquant_R" in n+ },+ "smooth_params": {+ n: p.data.cpu() for n, p in self.model.named_parameters() if p.requires_grad and "smooth_values" in n+ },+ }+ torch.save(ckpt, path)+ logger.info(f"[RotationTrainer] Checkpoint saved: {path}")+ return path++ def load_checkpoint(self, path: str) -> None:+ """Restore rotation + smooth params from disk."""+ try:+ ckpt = torch.load(path, map_location="cpu", weights_only=True)+ except TypeError:+ ckpt = torch.load(path, map_location="cpu")+ for n, p in self.model.named_parameters():+ if n in ckpt["rotation_params"]:+ p.data.copy_(ckpt["rotation_params"][n].to(p.device))+ if n in ckpt["smooth_params"]:+ p.data.copy_(ckpt["smooth_params"][n].to(p.device))+ self.state["step"] = ckpt["step"]+ logger.info(f"[RotationTrainer] Checkpoint loaded: {path}")++ # ------------------------------------------------------------------+ # Internal helpers+ # ------------------------------------------------------------------+ def _setup_training(self, dataloader: Any) -> None:+ """Initialise rotations, optimiser, and original-model clone."""+ from auto_round.algorithms.transforms.spinquant.inplace.apply import (+ register_spinquant_hooks,+ )+ from auto_round.algorithms.transforms.spinquant.rotation_utils import (+ fuse_rmsnorm_in_model,+ untie_word_embeddings_if_needed,+ )++ cfg = self.config++ # 1. Untie + RMSNorm fusion+ if cfg.untie_embeddings:+ untie_word_embeddings_if_needed(self.model)+ if cfg.fuse_rmsnorm:+ fuse_rmsnorm_in_model(self.model)++ # 2. Replace norms with trainable wrappers (SmoothQuant)+ if cfg.trainable_smooth:+ self._preprocessor._replace_norms_with_trainable()++ # 3. Initialise rotation matrices+ self._preprocessor._init_rotation_matrices()++ # 4. Register online hooks (R3 / R4)+ if cfg.r3 or cfg.r4:+ self._hook_handles = register_spinquant_hooks(+ self.model,+ self._to_sq_config(cfg),+ head_dim=self.head_dim,+ intermediate_size=self.intermediate_size,+ )++ # 5. Create dual optimiser+ self.optimizer = create_dual_optimizer(+ self.model,+ lr=cfg.lr,+ smooth_lr=cfg.smooth_lr,+ )+ if self.optimizer is None:+ raise ValueError(+ "SpinQuant training requires at least one trainable parameter group, "+ "but create_dual_optimizer() returned None. Ensure that training is "+ "configured with trainable rotation and/or smooth parameters enabled."+ )++ # 6. Clone original model for KL reference+ self._original_model = clone_model_for_reference(self.model)++ # 7. Use model's existing device (don't move)+ self.model.train()++ def _training_step(self, batch: Any) -> float:+ """Single training step. Returns loss scalar."""+ device = next(self.model.parameters()).device+ batch = move_batch_to_device(batch, device)++ # Forward: rotated model+ out_rot = self.model(**batch)+ logits_rot = out_rot.logits if hasattr(out_rot, "logits") else out_rot++ # Forward: original model (no grad)+ with torch.no_grad():+ assert self._original_model is not None+ out_ori = self._original_model(**batch)+ logits_ori = out_ori.logits if hasattr(out_ori, "logits") else out_ori++ # Loss & backward+ if self.compute_loss_fn is not self._default_compute_loss:+ loss = self.compute_loss_fn(logits_rot, logits_ori, self.config)+ else:+ loss = compute_rotation_loss(+ logits_rot,+ logits_ori,+ loss_type=self.config.loss_type,+ kl_top_k=self.config.kl_top_k,+ )+ loss.backward()+ self.optimizer.step() # type: ignore[union-attr]+ self.optimizer.zero_grad() # type: ignore[union-attr]++ return loss.item()++ def _teardown_training(self) -> None:+ """Free original model and set eval mode."""+ if self._original_model is not None:+ del self._original_model+ self._original_model = None+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ self.model.eval()++ def _trigger_event(self, event_name: str, **kwargs) -> None:+ """Fire callback hooks."""+ for cb in self.callbacks:+ method = getattr(cb, event_name, None)+ if callable(method):+ method(self.config, self.state, {"should_training_stop": False, **kwargs})++ @staticmethod+ def _default_compute_loss(+ logits: torch.Tensor,+ ori_logits: torch.Tensor,+ config: RotationTrainerConfig,+ ) -> torch.Tensor:+ """Default loss — delegates to :func:`compute_rotation_loss`."""+ return compute_rotation_loss(logits, ori_logits, config.loss_type, config.kl_top_k)++ @staticmethod+ def _to_sq_config(cfg: RotationTrainerConfig):+ """Convert trainer config to SpinQuantConfig for internal reuse."""+ from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig++ return SpinQuantConfig(+ r1=cfg.r1,+ r2=cfg.r2,+ r3=cfg.r3,+ r4=cfg.r4,+ trainable_rotation=cfg.trainable_rotation,+ trainable_smooth=cfg.trainable_smooth,+ online_r1_rotation=cfg.online_r1_rotation,+ iters=cfg.iters,+ lr=cfg.lr,+ smooth_lr=cfg.smooth_lr,+ batch_size=cfg.batch_size,+ loss_type=cfg.loss_type,+ kl_top_k=cfg.kl_top_k,+ fuse_rmsnorm=cfg.fuse_rmsnorm,+ untie_embeddings=cfg.untie_embeddings,+ dtype=cfg.dtype,+ device=cfg.device,+ )diff --git a/auto_round/algorithms/transforms/spinquant/training_core.py b/auto_round/algorithms/transforms/spinquant/training_core.pydeleted file mode 100644index 18a68cbc8..000000000--- a/auto_round/algorithms/transforms/spinquant/training_core.py+++ /dev/null@@ -1,285 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-"""Shared training primitives for SpinQuant rotation learning.--This module provides the **single implementation** of the rotation training-loop used by both:--* :meth:`SpinQuantPreprocessor._train_rotations` — minimal embedded loop-* :class:`RotationTrainer` — full-featured trainer with callbacks--By centralising the training step and loss computation here, both entry-points stay in sync and avoid code duplication.-"""--from __future__ import annotations--import copy-import logging-from dataclasses import dataclass-from typing import Any, Callable, Optional--import torch-import torch.nn as nn-import torch.nn.functional as F--from auto_round.algorithms.transforms.spinquant.cayley_optimizer import AdamAndSGDG--logger = logging.getLogger("autoround.spinquant")---# ----------------------------------------------------------------------------# Shared loss computation-# ------------------------------------------------------------------------------def compute_rotation_loss(- logits: torch.Tensor,- ori_logits: torch.Tensor,- loss_type: str = "kl_top",- kl_top_k: int = 1000,-) -> torch.Tensor:- """Compute loss between rotated-model logits and original-model logits.-- This is the single implementation shared by both the preprocessor's- embedded loop and the standalone :class:`RotationTrainer`.-- Args:- logits: Rotated model output logits.- ori_logits: Original (un-rotated) model output logits (detached).- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.- kl_top_k: Number of top logits to use for ``kl_top``.-- Returns:- Scalar loss tensor.- """- if loss_type == "kl_top":- k = min(kl_top_k, logits.size(-1))- top_ori, indices = ori_logits.topk(k, dim=-1, sorted=False)- top_logits = logits.gather(-1, indices)- return F.kl_div(- F.log_softmax(top_logits.flatten(0, -2), dim=-1),- F.softmax(top_ori.flatten(0, -2), dim=-1),- reduction="batchmean",- )- if loss_type == "kl_full":- return F.kl_div(- F.log_softmax(logits.flatten(0, -2), dim=-1),- F.softmax(ori_logits.flatten(0, -2), dim=-1),- reduction="batchmean",- )- if loss_type == "mse":- return F.mse_loss(logits, ori_logits)- raise ValueError(f"Unknown loss_type={loss_type!r}")---# ----------------------------------------------------------------------------# Shared utilities-# ------------------------------------------------------------------------------def move_batch_to_device(batch: Any, device: torch.device) -> Any:- """Move a batch (tensor or dict of tensors) to *device*."""- if hasattr(batch, "to"):- return batch.to(device)- if isinstance(batch, dict):- return {k: v.to(device) if hasattr(v, "to") else v for k, v in batch.items()}- return batch---def check_orthogonality(model: nn.Module, threshold: float = 1e-3) -> float:- """Check orthogonality of all ``spinquant_R*`` parameters.-- Returns:- Maximum deviation from identity (``max ||R @ R^T - I||_inf``).- """- max_dev = 0.0- for name, param in model.named_parameters():- if not param.requires_grad or "spinquant_R" not in name:- continue- R = param.data- if R.dim() != 2 or R.shape[0] != R.shape[1] or R.numel() == 0:- continue- I = torch.eye(R.shape[0], device=R.device, dtype=R.dtype)- dev = (torch.matmul(R, R.t()) - I).abs().max().item()- max_dev = max(max_dev, dev)- if dev > 1e-4:- logger.warning(f" {name} orthogonality deviation={dev:.2e}")- if max_dev > 0:- logger.info(f"[SpinQuant] Max orthogonality deviation={max_dev:.2e}")- if max_dev > threshold:- logger.warning(" Orthogonality constraint significantly violated!")- return max_dev---def clone_model_for_reference(model: nn.Module) -> nn.Module:- """Create a frozen deep-copy of *model* for KL-divergence reference.-- Hooks registered by SpinQuant are removed from the clone so it runs- as the un-rotated baseline.- """- from auto_round.algorithms.transforms.spinquant.preprocessor import (- remove_spinquant_hooks_from_model,- )-- original = copy.deepcopy(model)- original.eval()- for p in original.parameters():- p.requires_grad = False- remove_spinquant_hooks_from_model(original)- return original---def create_dual_optimizer(- model: nn.Module,- lr: float = 1e-4,- smooth_lr: float = 1e-3,-) -> Optional[AdamAndSGDG]:- """Create the Adam (smooth) + SGDG (rotation) dual optimiser.-- Returns ``None`` if no trainable parameters are found.- """- rot_params = [p for n, p in model.named_parameters() if p.requires_grad and "spinquant_R" in n]- smooth_params = [p for n, p in model.named_parameters() if p.requires_grad and "smooth_values" in n]-- if not rot_params and not smooth_params:- logger.info("[SpinQuant] No trainable parameters — nothing to train.")- return None-- n_rot = len(rot_params)- n_smooth = len(smooth_params)- total_params = sum(p.numel() for p in rot_params) + sum(p.numel() for p in smooth_params)- logger.info(- f"[SpinQuant] Training: {n_rot} rotation params + {n_smooth} smooth params "- f"= {total_params:,} trainable parameters"- )-- return AdamAndSGDG(- adam_params=smooth_params,- sgdg_params=rot_params,- learning_rate=lr,- smooth_learning_rate=smooth_lr,- )---# ----------------------------------------------------------------------------# Shared training loop-# ------------------------------------------------------------------------------@dataclass-class TrainingResult:- """Result of :func:`run_training_loop`."""-- loss_history: list[float]- best_loss: float- final_ortho_deviation: float- steps: int---def run_training_loop(- model: nn.Module,- original_model: nn.Module,- optimizer: AdamAndSGDG,- dataloader: Any,- *,- max_iters: int = 200,- loss_type: str = "kl_top",- kl_top_k: int = 1000,- compute_loss_fn: Optional[Callable] = None,- on_step_end: Optional[Callable[[int, float, float], None]] = None,- log_interval: int = 50,-) -> TrainingResult:- """Run the SpinQuant rotation training loop.-- This is the **single** training loop implementation shared by- :class:`SpinQuantPreprocessor` (embedded) and :class:`RotationTrainer`- (standalone).-- Args:- model: The model being trained (rotation params + optional smooth params).- original_model: Frozen reference model for KL divergence.- optimizer: :class:`AdamAndSGDG` dual optimiser.- dataloader: Iterable of batches.- max_iters: Stop after this many steps.- loss_type: ``"kl_top"`` | ``"kl_full"`` | ``"mse"``.- kl_top_k: Top-k for ``kl_top`` loss.- compute_loss_fn: Optional override for loss computation. Signature:- ``(logits, ori_logits) -> Tensor``. If ``None``, uses- :func:`compute_rotation_loss`.- on_step_end: Optional callback ``(step, loss, avg_loss) -> None``- called after each gradient step.- log_interval: Default logging interval (used when no ``on_step_end``- is provided).-- Returns:- :class:`TrainingResult` with loss history and orthogonality stats.- """- device = next(model.parameters()).device- model.train()-- loss_fn = compute_loss_fn or (- lambda logits, ori_logits: compute_rotation_loss(logits, ori_logits, loss_type, kl_top_k)- )-- loss_history: list[float] = []- best_loss = float("inf")- step = 0-- for batch in dataloader:- if step >= max_iters:- break-- batch = move_batch_to_device(batch, device)-- # Forward: rotated model- out_rot = model(**batch)- logits = out_rot.logits if hasattr(out_rot, "logits") else out_rot-- # Forward: original model (no grad)- with torch.no_grad():- out_ori = original_model(**batch)- ori_logits = out_ori.logits if hasattr(out_ori, "logits") else out_ori-- # Loss + backward- loss = loss_fn(logits, ori_logits)- loss.backward()- optimizer.step()- optimizer.zero_grad()-- loss_val = loss.item()- loss_history.append(loss_val)- best_loss = min(best_loss, loss_val)-- # Logging / callbacks- if on_step_end is not None:- avg = sum(loss_history[-50:]) / len(loss_history[-50:])- on_step_end(step, loss_val, avg)- elif step % log_interval == 0:- avg = sum(loss_history[-50:]) / len(loss_history[-50:])- logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")-- step += 1-- # Final orthogonality check- ortho_dev = check_orthogonality(model)-- model.eval()-- return TrainingResult(- loss_history=loss_history,- best_loss=best_loss,- final_ortho_deviation=ortho_dev,- steps=step,- )diff --git a/test/test_cuda/transform/test_spinquant.py b/test/test_cuda/transform/test_spinquant.pyindex 68099963c..29545b929 100644--- a/test/test_cuda/transform/test_spinquant.py+++ b/test/test_cuda/transform/test_spinquant.py@@ -9,7 +9,6 @@ Tests cover:- Rotation correctness: R1, R1+R2, R1+R2+R3+R4 produce valid logits"""-import copyimport shutilimport pytest@@ -245,89 +244,22 @@ class TestHookLifecycle:class TestRotationCorrectness:"""Test that rotation produces valid (non-NaN, non-zero) logits."""- @pytest.fixture(scope="class")- def model_and_tokenizer(self):- model_name = get_model_path("Qwen/Qwen3-0.6B")- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- model = AutoModelForCausalLM.from_pretrained(- model_name, torch_dtype=torch.float16, trust_remote_code=True- ).cuda()- model.eval()- yield model, tokenizer- del model- torch.cuda.empty_cache()-- @pytest.fixture- def baseline_logits(self, model_and_tokenizer):- """Get FP16 baseline logits for comparison."""- model, tokenizer = model_and_tokenizer- inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device)- with torch.no_grad():- logits = model(**inputs).logits- return logits.clone()-- def _get_logits(self, model, tokenizer, text="The capital of France is"):+ @staticmethod+ def _get_logits(model, tokenizer, text="The capital of France is"):inputs = tokenizer(text, return_tensors="pt").to(model.device)with torch.no_grad():return model(**inputs).logits- def test_r1_only(self, model_and_tokenizer):- """R1-only rotation should produce valid logits."""- model_name = get_model_path("Qwen/Qwen3-0.6B")- model = (- AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)- .cuda()- .eval()- )-- cfg = SpinQuantConfig(- r1=True,- r2=False,- r3=False,- r4=False,- online_r1_rotation=True,- trainable_rotation=False,- trainable_smooth=False,- )- model = apply_rotation(model, cfg)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- logits = self._get_logits(model, tokenizer)-- assert not torch.isnan(logits).any(), "R1 rotation produced NaN logits"- assert not torch.isinf(logits).any(), "R1 rotation produced Inf logits"- assert logits.abs().sum() > 0, "R1 rotation produced all-zero logits"- del model- torch.cuda.empty_cache()-- def test_r1_r2(self, model_and_tokenizer):- """R1+R2 rotation should produce valid logits."""- model_name = get_model_path("Qwen/Qwen3-0.6B")- model = (- AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)- .cuda()- .eval()- )-- cfg = SpinQuantConfig(- r1=True,- r2=True,- r3=False,- r4=False,- online_r1_rotation=True,- trainable_rotation=False,- trainable_smooth=False,- )- model = apply_rotation(model, cfg)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- logits = self._get_logits(model, tokenizer)-- assert not torch.isnan(logits).any(), "R1+R2 rotation produced NaN logits"- assert not torch.isinf(logits).any(), "R1+R2 rotation produced Inf logits"- del model- torch.cuda.empty_cache()-- def test_r1_r2_r3_r4(self, model_and_tokenizer):- """Full R1+R2+R3+R4 rotation should produce valid logits."""+ @pytest.mark.parametrize(+ "r1,r2,r3,r4,label",+ [+ (True, False, False, False, "R1"),+ (True, True, False, False, "R1+R2"),+ (True, True, True, True, "R1+R2+R3+R4"),+ ],+ )+ def test_rotation_produces_valid_logits(self, r1, r2, r3, r4, label):+ """Rotation configurations should produce valid (non-NaN, non-Inf) logits."""model_name = get_model_path("Qwen/Qwen3-0.6B")model = (AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, trust_remote_code=True)@@ -336,10 +268,10 @@ class TestRotationCorrectness:)cfg = SpinQuantConfig(- r1=True,- r2=True,- r3=True,- r4=True,+ r1=r1,+ r2=r2,+ r3=r3,+ r4=r4,online_r1_rotation=True,trainable_rotation=False,trainable_smooth=False,@@ -348,8 +280,9 @@ class TestRotationCorrectness:tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)logits = self._get_logits(model, tokenizer)- assert not torch.isnan(logits).any(), "Full rotation produced NaN logits"- assert not torch.isinf(logits).any(), "Full rotation produced Inf logits"+ assert not torch.isnan(logits).any(), f"{label} rotation produced NaN logits"+ assert not torch.isinf(logits).any(), f"{label} rotation produced Inf logits"+ assert logits.abs().sum() > 0, f"{label} rotation produced all-zero logits"del modeltorch.cuda.empty_cache()