Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

26.0 KB
Raw Blame
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""SpinQuant / QuaRot rotation serialization for quantized models.

This module handles saving and loading online rotation state so that
rotated + quantized models can be saved with ``save_pretrained()`` and
loaded back for inference without manual hook re-registration.

Strategy (Plan A + C):
- **R1/R4 online rotations**: Stored as buffers on QuantLinear modules.
  At inference, QuantLinear.forward() detects these buffers and applies
  the rotation before the quantized matmul.
- **R3 online rotation**: Stored in ``config.json`` only (deterministic
  Hadamard, reconstructible from head_dim). Rebuilt via monkeypatch after
  model loading.
- **R1/R2 offline (fused)**: No serialization needed — already in weights.

Buffer storage format:
- Deterministic Hadamard: ``int8`` (±1 values) + rotation_size metadata
- Random Hadamard: ``int8`` (±1 values, full matrix after random sign flip)
- Trained orthogonal: ``float32`` (full matrix, cannot be compressed)
"""

from __future__ import annotations

import json
import logging
import math
import os
from dataclasses import asdict
from typing import TYPE_CHECKING

import torch
import torch.nn as nn

from auto_round.algorithms.transforms.spinquant.rotation_utils import (
    get_hadamard_K,
    matmul_hadU,
)

if TYPE_CHECKING:
    from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig

logger = logging.getLogger("autoround.spinquant.serialize")

# --------------------------------------------------------------------------
# Constants
# --------------------------------------------------------------------------

# Buffer name prefixes (avoid collisions with other rotation implementations)
_R1_PREFIX = "spinquant_r1"
_R4_PREFIX = "spinquant_r4"

# Rotation type flags stored as scalar tensors
ROTATION_TYPE_HADAMARD = 0  # Deterministic, reconstruct from size
ROTATION_TYPE_RANDOM = 1  # Random Hadamard, stored as int8 (±1)
ROTATION_TYPE_TRAINED = 2  # Trained orthogonal, stored as float32


# --------------------------------------------------------------------------
# Save-side: inject rotation buffers into QuantLinear before save
# --------------------------------------------------------------------------


def inject_spinquant_buffers(
    model: nn.Module,
    config: SpinQuantConfig,
) -> int:
    """Inject SpinQuant rotation buffers into QuantLinear modules for serialization.

    Must be called AFTER quantization/packing (QuantLinear modules exist) and
    BEFORE ``model.save_pretrained()``.

    Args:
        model: Quantized model with QuantLinear modules.
        config: SpinQuantConfig used during preprocessing.

    Returns:
        Number of QuantLinear modules that received rotation buffers.
    """
    from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig

    n_injected = 0

    # Determine rotation parameters
    hidden_size = _get_hidden_size(model)
    head_dim = _get_head_dim(model)
    intermediate_size = _get_intermediate_size(model)

    r1_size = config.rotation_size or hidden_size
    r4_size = config.rotation_size or intermediate_size

    # Online R1 targets: q_proj, k_proj, v_proj, gate_proj, up_proj
    if config.r1 and config.online_r1_rotation:
        r1_targets = _get_online_r1_target_names(model)
        for name, module in model.named_modules():
            if not _is_quantlinear(module):
                continue
            if name in r1_targets:
                _inject_rotation_buffers(
                    module,
                    _R1_PREFIX,
                    r1_size,
                    config.random_r1,
                    is_trained=False,
                    rotation_matrix=_get_stored_rotation(model, "spinquant_R1"),
                )
                n_injected += 1

    # Online R4 targets: down_proj
    if config.r4:
        r4_targets = _get_r4_target_names(model)
        for name, module in model.named_modules():
            if not _is_quantlinear(module):
                continue
            if name in r4_targets:
                _inject_rotation_buffers(
                    module,
                    _R4_PREFIX,
                    r4_size,
                    random=config.random_r4,
                    is_trained=False,
                    rotation_matrix=_get_stored_rotation(model, "spinquant_R4_matrix") if config.random_r4 else None,
                )
                n_injected += 1

    logger.info(f"[SpinQuant Serialize] Injected rotation buffers into " f"{n_injected} QuantLinear modules")
    return n_injected


def save_spinquant_config(
    model: nn.Module,
    save_dir: str,
    config: SpinQuantConfig,
) -> None:
    """Save SpinQuant config into the model's config.json for load-time reconstruction.

    This enables R3 monkeypatch reconstruction and general rotation metadata
    persistence. Called after model.save_pretrained().

    Args:
        model: The model (used to extract architecture info).
        save_dir: Directory where config.json was saved.
        config: SpinQuantConfig to persist.
    """
    config_path = os.path.join(save_dir, "config.json")
    if not os.path.exists(config_path):
        logger.warning(f"[SpinQuant Serialize] config.json not found at {save_dir}, " f"cannot save spinquant_config")
        return

    with open(config_path, "r") as f:
        model_config = json.load(f)

    # Build serializable spinquant config
    spinquant_dict = _config_to_serializable(config, model)
    spinquant_dict["algorithm"] = "spinquant"

    # Store under quantization_config if it exists, else at top level.
    # Key is "spinquant_config" (not "rotation_config" — that key is
    # already used by the Hadamard rotation system with a different schema).
    if "quantization_config" in model_config:
        qcfg = model_config["quantization_config"]
        qcfg["spinquant_config"] = spinquant_dict
    else:
        model_config["spinquant_config"] = spinquant_dict

    with open(config_path, "w") as f:
        json.dump(model_config, f, indent=2)

    logger.info(f"[SpinQuant Serialize] Saved spinquant_config to {config_path}")


# --------------------------------------------------------------------------
# Load-side: pre-register empty buffers before weight loading
# --------------------------------------------------------------------------


def preregister_spinquant_buffers(
    model: nn.Module,
    spinquant_config: dict,
) -> int:
    """Pre-register empty SpinQuant buffers on QuantLinear modules.

    Must be called AFTER ``convert_hf_model()`` replaces Linear → QuantLinear
    but BEFORE HuggingFace loads the state_dict from safetensors.
    This ensures the spinquant buffers in the checkpoint are not silently
    dropped by ``load_state_dict(strict=False)``.

    Args:
        model: Model with QuantLinear modules (post convert_hf_model).
        spinquant_config: Dict from config.json's spinquant_config.

    Returns:
        Number of modules that received pre-registered buffers.
    """
    if not isinstance(spinquant_config, dict):
        return 0

    r1_online = spinquant_config.get("online_r1_rotation", True)
    r1_enabled = spinquant_config.get("r1", True)
    r4_enabled = spinquant_config.get("r4", False)
    random_r1 = spinquant_config.get("random_r1", False)
    trained = spinquant_config.get("trainable_rotation", False)
    rotation_size_cfg = spinquant_config.get("rotation_size")

    hidden_size = spinquant_config.get("hidden_size", _get_hidden_size(model))
    intermediate_size = spinquant_config.get("intermediate_size", _get_intermediate_size(model))

    r1_size = rotation_size_cfg or hidden_size
    r4_size = rotation_size_cfg or intermediate_size

    # Determine which type of matrix storage
    if trained:
        needs_matrix = True
        matrix_dtype = torch.float32
    elif random_r1:
        needs_matrix = True
        matrix_dtype = torch.int8
    else:
        needs_matrix = False
        matrix_dtype = torch.int8

    n_registered = 0

    r1_targets = _get_online_r1_target_names(model) if (r1_enabled and r1_online) else set()
    r4_targets = _get_r4_target_names(model) if r4_enabled else set()

    for name, module in model.named_modules():
        if not _is_quantlinear(module):
            continue

        if name in r1_targets:
            _preregister_buffers_on_module(module, _R1_PREFIX, r1_size, needs_matrix, matrix_dtype)
            n_registered += 1

        if name in r4_targets:
            random_r4 = spinquant_config.get("random_r4", False)
            r4_needs_matrix = random_r4 or trained
            _preregister_buffers_on_module(
                module,
                _R4_PREFIX,
                r4_size,
                needs_matrix=r4_needs_matrix,
                matrix_dtype=torch.float32 if trained else torch.int8,
            )
            n_registered += 1

    if n_registered > 0:
        buf_types = []
        if r1_targets:
            buf_types.append(f"R1({len(r1_targets)} modules)")
        if r4_targets:
            buf_types.append(f"R4({len(r4_targets)} modules)")
        logger.info(
            f"[SpinQuant] Pre-registered empty buffers on "
            f"{n_registered} QuantLinear modules for state_dict loading "
            f"[{', '.join(buf_types)}]"
        )
    return n_registered


def _preregister_buffers_on_module(
    module: nn.Module,
    prefix: str,
    rotation_size: int,
    needs_matrix: bool,
    matrix_dtype: torch.dtype,
) -> None:
    """Register empty spinquant buffers on a module for state_dict loading."""
    # Use CPU: at pre-registration time QuantLinear params may be on meta
    # device; HF's state_dict loader will overwrite these with real values.
    device = torch.device("cpu")
    module.register_buffer(
        f"{prefix}_type",
        torch.tensor(0, dtype=torch.int32, device=device),
    )
    module.register_buffer(
        f"{prefix}_size",
        torch.tensor(0, dtype=torch.int32, device=device),
    )
    if needs_matrix:
        module.register_buffer(
            f"{prefix}_matrix",
            torch.zeros(
                rotation_size,
                rotation_size,
                dtype=matrix_dtype,
                device=device,
            ),
        )


# --------------------------------------------------------------------------
# Load-side: reconstruct online rotations from buffers and config
# --------------------------------------------------------------------------


def rebuild_spinquant_online(
    model: nn.Module,
    config: SpinQuantConfig | None = None,
) -> nn.Module:
    """Rebuild online SpinQuant rotations after loading a quantized model.

    Handles:
    - R1/R4: QuantLinear forward is patched to apply rotation from buffers.
    - R3: If config indicates R3 was enabled, re-applies monkeypatch.

    Args:
        model: Loaded quantized model (QuantLinear modules have spinquant buffers).
        config: SpinQuantConfig. If None, attempts to read from model.config.

    Returns:
        Model with online rotations restored.
    """
    if config is None:
        config = _load_config_from_model(model)
        if config is None:
            logger.warning("[SpinQuant] No spinquant_config found on model. " "Cannot rebuild online rotations.")
            return model

    # Patch QuantLinear forward for R1/R4 buffer-based rotation
    n_patched = _patch_quantlinear_forward_spinquant(model)

    # Rebuild R3 monkeypatch if enabled
    if config.r3:
        head_dim = _get_head_dim(model)
        if head_dim > 0:
            # Build a minimal config with R3=True, R4=False for hook rebuild.
            # R1/R4 are handled by QuantLinear buffer-based forward patching.
            from types import SimpleNamespace

            from auto_round.algorithms.transforms.spinquant.inplace.apply import (
                register_spinquant_hooks,
            )

            r3_config = SimpleNamespace(
                r3=True,
                r4=False,
                random_r3=getattr(config, "random_r3", False),
                random_r4=False,
                head_dim=head_dim,
                intermediate_size=0,
            )
            register_spinquant_hooks(
                model,
                config=r3_config,
                head_dim=head_dim,
                r4_rotation_size=0,
            )
            logger.info(f"[SpinQuant] Rebuilt R3 monkeypatch (head_dim={head_dim})")

    # Build descriptive summary of which rotations are active
    active = []
    if config.r1:
        r1_mode = "online" if getattr(config, "online_r1_rotation", True) else "offline"
        active.append(f"R1({r1_mode})")
    if getattr(config, "r2", False):
        active.append("R2(offline, fused into weights)")
    if config.r3:
        active.append("R3(hook)")
    if config.r4:
        active.append("R4(buffer)")
    active_str = ", ".join(active) if active else "none"

    logger.info(
        f"[SpinQuant] Rebuilt online rotations: " f"{n_patched} QuantLinear patched, " f"active rotations: {active_str}"
    )
    return model


# --------------------------------------------------------------------------
# QuantLinear forward patching
# --------------------------------------------------------------------------

_QUANTLINEAR_PATCHED = False


def _patch_quantlinear_forward_spinquant(model: nn.Module) -> int:
    """Patch QuantLinear.forward() to apply spinquant rotation from buffers.

    This is a class-level patch applied once. Each QuantLinear instance checks
    at forward time whether it has spinquant buffers.

    Returns:
        Number of QuantLinear modules with spinquant buffers found.
    """

    n_with_buffers = 0
    quantlinear_classes = set()

    for _, module in model.named_modules():
        if _is_quantlinear(module) and _has_spinquant_buffers(module):
            n_with_buffers += 1
            quantlinear_classes.add(type(module))

    if n_with_buffers == 0:
        return 0

    # Patch each QuantLinear class variant found
    for cls in quantlinear_classes:
        if getattr(cls, "_spinquant_forward_patched", False):
            continue
        _monkey_patch_forward(cls)
        cls._spinquant_forward_patched = True

    return n_with_buffers


def _monkey_patch_forward(cls: type) -> None:
    """Replace cls.forward with a version that applies spinquant rotation."""
    original_forward = cls.forward

    def forward_with_spinquant(self, x):
        # Apply R1 rotation if buffer present
        if hasattr(self, f"{_R1_PREFIX}_type"):
            x = _apply_rotation_from_buffer(self, x, _R1_PREFIX)

        # Apply R4 rotation if buffer present
        if hasattr(self, f"{_R4_PREFIX}_type"):
            x = _apply_rotation_from_buffer(self, x, _R4_PREFIX)

        return original_forward(self, x)

    cls.forward = forward_with_spinquant


def _apply_rotation_from_buffer(
    module: nn.Module,
    x: torch.Tensor,
    prefix: str,
) -> torch.Tensor:
    """Apply rotation using stored buffers on a QuantLinear module.

    Args:
        module: QuantLinear with spinquant buffers.
        x: Input tensor [..., in_features].
        prefix: Buffer name prefix ("spinquant_r1" or "spinquant_r4").

    Returns:
        Rotated tensor.
    """
    rot_type = int(getattr(module, f"{prefix}_type"))
    rot_size = int(getattr(module, f"{prefix}_size"))
    in_features = x.shape[-1]

    if rot_type == ROTATION_TYPE_HADAMARD:
        # Deterministic Hadamard — reconstruct from size
        # Cache the decomposition on the module for efficiency
        cache_key = f"_cached_{prefix}_had"
        if not hasattr(module, cache_key):
            had_K, K = get_hadamard_K(rot_size)
            setattr(module, cache_key, (had_K, K))
        had_K, K = getattr(module, cache_key)
        had_K = had_K.to(x.device)

        if rot_size == in_features:
            x = matmul_hadU(x, hadamard_K=had_K, K=K)
        else:
            # Block rotation
            x = _apply_block_rotation_butterfly(x, had_K, K, rot_size)

    elif rot_type in (ROTATION_TYPE_RANDOM, ROTATION_TYPE_TRAINED):
        # Full matrix stored in buffer
        matrix = getattr(module, f"{prefix}_matrix")
        R = matrix.to(x.device, dtype=x.dtype)

        if rot_type == ROTATION_TYPE_RANDOM:
            # int8 ±1 matrix, normalize
            R = R.float() / math.sqrt(rot_size)
            R = R.to(x.dtype)

        if rot_size == in_features:
            x = x @ R
        else:
            # Block rotation
            shape = x.shape
            x = x.reshape(*shape[:-1], -1, rot_size)
            x = (x @ R).reshape(shape)

    return x


def _apply_block_rotation_butterfly(
    x: torch.Tensor,
    had_K: torch.Tensor,
    K: int,
    rot_size: int,
) -> torch.Tensor:
    """Apply block-wise Hadamard rotation using butterfly or direct matmul."""
    in_features = x.shape[-1]
    if in_features == rot_size:
        return matmul_hadU(x, hadamard_K=had_K, K=K)

    # Block-wise: reshape and apply per block
    shape = x.shape
    dtype = x.dtype

    # Build full rotation matrix for the block
    R = had_K.to(torch.float64)
    if R.shape[0] != rot_size:
        had_1, _ = get_hadamard_K(rot_size // K)
        R = torch.kron(
            had_K.to(device="cpu", dtype=torch.float64),
            had_1.to(device="cpu", dtype=torch.float64),
        )
    R = (R / math.sqrt(rot_size)).to(device=x.device, dtype=dtype)

    x = x.reshape(*shape[:-1], -1, rot_size)
    x = (x @ R).reshape(shape)
    return x


# --------------------------------------------------------------------------
# Internal helpers
# --------------------------------------------------------------------------


def _inject_rotation_buffers(
    module: nn.Module,
    prefix: str,
    rotation_size: int,
    random: bool,
    is_trained: bool,
    rotation_matrix: torch.Tensor | None = None,
) -> None:
    """Register rotation buffers on a QuantLinear module.

    Args:
        module: Target QuantLinear.
        prefix: Buffer name prefix.
        rotation_size: Size of the rotation block.
        random: Whether this is a random Hadamard (needs full matrix storage).
        is_trained: Whether this is a trained orthogonal matrix.
        rotation_matrix: Pre-computed rotation matrix (for random/trained).
    """
    # Always use CPU for rotation metadata buffers.  After packing,
    # QuantLinear parameters may live on the ``meta`` device and
    # ``save_pretrained`` silently skips meta tensors.  CPU ensures the
    # buffers are persisted to safetensors.
    device = torch.device("cpu")

    # Determine storage type
    if is_trained:
        rot_type = ROTATION_TYPE_TRAINED
    elif random:
        rot_type = ROTATION_TYPE_RANDOM
    else:
        rot_type = ROTATION_TYPE_HADAMARD

    # Always store: type and size
    module.register_buffer(
        f"{prefix}_type",
        torch.tensor(rot_type, dtype=torch.int32, device=device),
    )
    module.register_buffer(
        f"{prefix}_size",
        torch.tensor(rotation_size, dtype=torch.int32, device=device),
    )

    # Store matrix for random/trained types
    if rot_type == ROTATION_TYPE_RANDOM:
        if rotation_matrix is None:
            logger.warning(
                f"Random rotation ({prefix}) requires rotation_matrix but got None. "
                "This usually means the matrix was deleted during cleanup. "
                "Falling back to deterministic Hadamard — the saved model may be INCORRECT."
            )
            had_K, K = get_hadamard_K(rotation_size)
            if had_K.shape[0] != rotation_size:
                had_1, _ = get_hadamard_K(rotation_size // K)
                rotation_matrix = torch.kron(had_K, had_1)
            else:
                rotation_matrix = had_K
        # Store as int8 (±1 values only)
        module.register_buffer(
            f"{prefix}_matrix",
            rotation_matrix.sign().to(torch.int8).to(device),
        )

    elif rot_type == ROTATION_TYPE_TRAINED:
        if rotation_matrix is None:
            raise ValueError("Trained rotation requires rotation_matrix to be provided")
        # Store as float32 (arbitrary values)
        module.register_buffer(
            f"{prefix}_matrix",
            rotation_matrix.to(torch.float32).to(device),
        )
    # For HADAMARD type: no matrix needed (reconstructed from size at runtime)


def _is_quantlinear(module: nn.Module) -> bool:
    """Check if a module is any variant of quantized linear layer.

    Covers:
    - QuantLinear (INT W4A16/W3A16/W8A16 from export.py / qlinear_int.py / qlinear_fp.py)
    - NVFP4QuantLinear, MXFP4QuantLinear, MXFP8QuantLinear, MXINT4QuantLinear (from qmodules)
    - WeightFP8ActFP8StaticQuantLinear (FP8 static)
    - Any class with 'QuantLinear' in its name or inheriting QModuleBase
    """
    cls_name = type(module).__name__
    # Direct name match for standard QuantLinear
    if cls_name == "QuantLinear":
        return True
    # Match FP-family quantized linears (NVFP4QuantLinear, MXFP4QuantLinear, etc.)
    if "QuantLinear" in cls_name:
        return True
    # Match QModuleBase subclasses (FP8/MXFP/NVFP inference modules)
    for base in type(module).__mro__:
        if base.__name__ == "QModuleBase":
            return True
    return False


def _has_spinquant_buffers(module: nn.Module) -> bool:
    """Check if a module has any spinquant rotation buffers."""
    return hasattr(module, f"{_R1_PREFIX}_type") or hasattr(module, f"{_R4_PREFIX}_type")


def _get_online_r1_target_names(model: nn.Module) -> set:
    """Get full qualified names of modules that should have online R1 rotation.

    These are: q_proj, k_proj, v_proj, gate_proj, up_proj
    (the modules whose input channels were rotated during preprocessing).
    """
    targets = set()
    r1_proj_names = ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj")

    for name, module in model.named_modules():
        # Match by suffix (handles both attn.q_proj and self_attn.q_proj patterns)
        parts = name.split(".")
        if parts and parts[-1] in r1_proj_names:
            targets.add(name)

    return targets


def _get_r4_target_names(model: nn.Module) -> set:
    """Get full qualified names of modules that should have R4 rotation.

    R4 is applied to the input of down_proj.
    """
    targets = set()
    for name, module in model.named_modules():
        parts = name.split(".")
        if parts and parts[-1] == "down_proj":
            targets.add(name)
    return targets


def _get_stored_rotation(model: nn.Module, param_name: str) -> torch.Tensor | None:
    """Get a stored rotation matrix/parameter from the model.

    During preprocessing, rotation matrices are stored as model-level
    parameters (e.g., model.spinquant_R1).
    """
    if hasattr(model, param_name):
        param = getattr(model, param_name)
        if isinstance(param, (torch.Tensor, nn.Parameter)):
            return param.data
    return None


def _get_hidden_size(model: nn.Module) -> int:
    """Extract hidden_size from model config."""
    if hasattr(model, "config"):
        return getattr(model.config, "hidden_size", 0)
    return 0


def _get_head_dim(model: nn.Module) -> int:
    """Extract head_dim from model config."""
    if hasattr(model, "config"):
        cfg = model.config
        if hasattr(cfg, "head_dim"):
            return cfg.head_dim
        if hasattr(cfg, "hidden_size") and hasattr(cfg, "num_attention_heads"):
            return cfg.hidden_size // cfg.num_attention_heads
    return 0


def _get_intermediate_size(model: nn.Module) -> int:
    """Extract intermediate_size from model config."""
    if hasattr(model, "config"):
        return getattr(model.config, "intermediate_size", 0)
    return 0


def _config_to_serializable(config: SpinQuantConfig, model: nn.Module) -> dict:
    """Convert SpinQuantConfig to a JSON-serializable dict with model info."""
    from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig

    # Get relevant fields from config
    result = {
        "r1": config.r1,
        "r2": config.r2,
        "r3": config.r3,
        "r4": config.r4,
        "online_r1_rotation": config.online_r1_rotation,
        "rotation_size": config.rotation_size,
        "random_r1": config.random_r1,
        "random_r2": config.random_r2,
        "random_r3": config.random_r3,
        "random_r4": config.random_r4,
        "trainable_rotation": config.trainable_rotation,
        # Model architecture info for reconstruction
        "head_dim": _get_head_dim(model),
        "hidden_size": _get_hidden_size(model),
        "intermediate_size": _get_intermediate_size(model),
    }
    return result


def _load_config_from_model(
    model: nn.Module,
) -> SpinQuantConfig | None:
    """Try to load SpinQuantConfig from model.config."""
    from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig

    if not hasattr(model, "config"):
        return None

    cfg = model.config
    spinquant_dict = None

    # Check in quantization_config (may be a dict or QuantizationConfigMixin)
    if hasattr(cfg, "quantization_config"):
        qcfg = cfg.quantization_config
        if isinstance(qcfg, dict):
            spinquant_dict = qcfg.get("spinquant_config")
        else:
            # QuantizationConfigMixin or similar object
            spinquant_dict = getattr(qcfg, "spinquant_config", None)

    # Check at top level
    if spinquant_dict is None and hasattr(cfg, "spinquant_config"):
        spinquant_dict = cfg.spinquant_config

    if spinquant_dict is None:
        return None

    # Reconstruct SpinQuantConfig from dict
    try:
        return SpinQuantConfig(
            r1=spinquant_dict.get("r1", True),
            r2=spinquant_dict.get("r2", True),
            r3=spinquant_dict.get("r3", False),
            r4=spinquant_dict.get("r4", False),
            online_r1_rotation=spinquant_dict.get("online_r1_rotation", True),
            rotation_size=spinquant_dict.get("rotation_size"),
            random_r1=spinquant_dict.get("random_r1", False),
            random_r2=spinquant_dict.get("random_r2", False),
            random_r3=spinquant_dict.get("random_r3", False),
            random_r4=spinquant_dict.get("random_r4", False),
            trainable_rotation=spinquant_dict.get("trainable_rotation", False),
        )
    except Exception as e:
        logger.warning(f"[SpinQuant] Failed to parse spinquant_config: {e}")
        return None