PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# # Copyright (C) 2026 Intel Corporation
# # SPDX-License-Identifier: Apache-2.0
"""
SpinQuant / QuaRot rotation utilities.
This module provides rotation fusion functions for SpinQuant. Where possible,
it delegates to AutoRound's ``rotation.utils.matrix`` and
``rotation.utils.math`` modules for Hadamard matrix generation and
linear-algebra helpers.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
# ---------------------------------------------------------------------------
# Hadamard matrix generation (always use our own, to avoid AutoRound
# version incompatibilities that may produce unnormalized / low-precision
# matrices).
# ---------------------------------------------------------------------------
def deterministic_hadamard_matrix(
size: int, dtype: torch.dtype = torch.float32, device: torch.device = torch.device("cpu")
) -> torch.Tensor:
"""Generate a normalized Sylvester Hadamard matrix (H / sqrt(N))."""
if size <= 0 or size & (size - 1) != 0:
raise ValueError(f"deterministic_hadamard_matrix requires power-of-2 size, got {size}")
H = torch.tensor([[1.0]], dtype=dtype, device=device)
while H.size(0) < size:
H = torch.cat([torch.cat([H, H], dim=1), torch.cat([H, -H], dim=1)], dim=0)
return H / math.sqrt(size)
def random_hadamard_matrix(
size: int, dtype: torch.dtype = torch.float32, device: torch.device = torch.device("cpu")
) -> torch.Tensor:
"""Generate a random Hadamard matrix: H @ diag(±1) / sqrt(N).
Works for any size supported by :func:`get_hadamard_K` (power-of-2 or
known non-power-of-2 sizes like 12, 20, 28, etc.).
"""
D = torch.randint(0, 2, (size,), dtype=torch.float64, device=device) * 2 - 1
Q = torch.diag(D)
hadamard_K, K = get_hadamard_K(size)
hadamard_K = hadamard_K.to(dtype=torch.float64, device=device)
result = matmul_hadU(Q, hadamard_K=hadamard_K, K=K)
return result.to(dtype=dtype)
def is_pow2(n: int) -> bool:
"""Check if n is a power of 2."""
return n > 0 and (n & (n - 1)) == 0
def get_hadamard_K(n: int) -> tuple[torch.Tensor, int]:
"""Get the Hadamard matrix and block dimension K for a given input size.
For power-of-2 sizes, K=1 (full Walsh-Hadamard via butterfly).
For non-power-of-2 sizes, K is the largest known Hadamard size that
divides n such that n/K is a power of 2. Known sizes are sourced from
:mod:`known_hadamard` (ported from Quark/AMD), covering sizes like 12,
20, 28, 36, 40, 52, 60, 108, 140, 156, 172.
Examples:
- n=3072: 3072 = 12 × 256 → K=12, returns 12×12 Hadamard
- n=1024: power-of-2 → K=1, returns 1024×1024 Hadamard
Returns:
(hadamard_K, K): The K×K Hadamard matrix and the value K.
"""
from auto_round.algorithms.transforms.spinquant.known_hadamard import (
KNOWN_HADAMARD_MATRICES,
)
if is_pow2(n):
# Sylvester construction (unnormalized) — replaces scipy.linalg.hadamard
H = torch.ones(1, 1)
while H.shape[0] < n:
H = torch.cat([torch.cat([H, H], dim=1), torch.cat([H, -H], dim=1)], dim=0)
return H, 1
else:
# Search known Hadamard matrices: find K such that n % K == 0 and n/K is pow2
for size in KNOWN_HADAMARD_MATRICES:
if n % size == 0 and is_pow2(n // size):
had_K = KNOWN_HADAMARD_MATRICES[size]()
return had_K, size
raise ValueError(
f"Cannot find suitable Hadamard decomposition for n={n}. "
f"Known non-pow2 sizes: {sorted(KNOWN_HADAMARD_MATRICES.keys())}. "
f"n must be a power of 2, or n = K × 2^m for a known K."
)
def matmul_hadU(X: torch.Tensor, hadamard_K: torch.Tensor | None = None, K: int | None = None) -> torch.Tensor:
"""Apply normalized Hadamard transform to the last dimension of X.
Uses the efficient butterfly algorithm for power-of-2 dimensions,
combined with an explicit K×K Hadamard matrix for any residual block.
This is equivalent to X @ H where H is the normalized Hadamard matrix,
but computed more efficiently via recursive butterfly operations.
Based on QuaRot/Quark's implementation.
Args:
X: Input tensor with shape [..., n].
hadamard_K: Optional pre-computed K×K Hadamard matrix (unnormalized).
If None, computed automatically.
K: Block size for the explicit Hadamard step. If None, computed automatically.
Returns:
Transformed tensor of same shape as X.
"""
n = X.shape[-1]
if hadamard_K is None or K is None:
hadamard_K, K = get_hadamard_K(n)
hadamard_K = hadamard_K.to(dtype=X.dtype, device=X.device)
inp = X.clone().reshape(-1, n, 1)
output = inp.clone()
# Butterfly decomposition (Walsh-Hadamard for the n/K part)
while inp.shape[1] > K:
inp = inp.view(inp.shape[0], inp.shape[1] // 2, 2, inp.shape[2])
output = output.view(inp.shape)
output[:, :, 0, :] = inp[:, :, 0, :] + inp[:, :, 1, :]
output[:, :, 1, :] = inp[:, :, 0, :] - inp[:, :, 1, :]
output = output.view(inp.shape[0], inp.shape[1], -1)
inp, output = (output, inp)
del output
# Apply the K×K Hadamard block (if K > 1)
if K > 1:
had = hadamard_K.to(inp.device).to(inp.dtype)
inp = had.view(1, K, K) @ inp
# Normalize: butterfly + K-block gives unnormalized result, divide by √n
result = inp.view(X.shape) / math.sqrt(n)
return result
# ---------------------------------------------------------------------------
# Delegate to AutoRound's rotation utilities where available.
# If the AutoRound modules are not present (e.g. during standalone tests),
# the fallback implementations below are used.
# ---------------------------------------------------------------------------
try:
from auto_round.algorithms.transforms.hadamard.utils.matrix import apply_transform_weight
except ImportError:
# Fallback for standalone usage.
def apply_transform_weight(
transform_weight: torch.Tensor,
value: torch.Tensor,
location: str,
module_type: type[nn.Module],
) -> torch.Tensor:
if location == "input":
return value @ transform_weight.T
if location == "output":
return transform_weight.T @ value
raise NotImplementedError(f"apply_transform_weight: unsupported location={location!r}")
__all__ = [
"InputRotationWrapperHadamard",
"apply_hadamard_to_linear",
"create_block_diag_from_head_matrix",
"deterministic_hadamard_matrix",
"fuse_rmsnorm_in_model",
"get_hadamard_K",
"get_model_arch_info",
"is_pow2",
"matmul_hadU",
"random_hadamard_matrix",
"rotate_in_channels_",
"rotate_out_channels_",
"untie_word_embeddings_if_needed",
]
# ---------------------------------------------------------------------------
# InputRotationWrapperHadamard — nn.Module wrapper for online rotation.
#
# Replaces a Linear layer with a wrapper that applies Hadamard rotation to
# the input activation before the linear forward pass. Since the wrapper is
# a proper nn.Module, it is serialised by ``save_pretrained`` / ``state_dict``
# and automatically reconstructed on ``load_state_dict``.
#
# The wrapper takes ownership of the original Linear's weight and bias
# parameters (they are NOT stored as a submodule). This prevents auto-round's
# quantization pipeline from discovering and double-wrapping the inner Linear.
#
# Matches Quark's ``InputRotationWrapperHadamard`` semantics.
# ---------------------------------------------------------------------------
class InputRotationWrapperHadamard(nn.Module):
"""nn.Module wrapper that applies Hadamard rotation to input before Linear.
When online R1 rotation is used, target modules (q/k/v/gate/up_proj) have
their weights pre-rotated as ``W' = W @ R``. This wrapper applies the
matching rotation ``x' = x @ R`` to the activation at runtime so that the
full-precision result is preserved::
y = (x @ R) @ (W @ R).T = x @ R @ R.T @ W.T = x @ W.T
The wrapper stores the Hadamard matrix as a buffer (``hadamard_K``), so it
is saved alongside the model weights and automatically restored on load.
Weight and bias are owned directly by this module (not via a submodule),
so ``state_dict()`` produces compatible key names (``q_proj.weight``, not
``q_proj.original_module.weight``). This also ensures that auto-round's
quantization pipeline treats this module as a leaf and doesn't try to wrap
the inner Linear separately.
Args:
original_module: The ``nn.Linear`` layer (with already-rotated weights).
Its weight and bias are *moved* into this wrapper.
rotation_size: Size of the block rotation (= in_features for full rotation).
hadamard_K: Pre-computed Hadamard matrix from ``get_hadamard_K()``.
If None, computed automatically from ``rotation_size``.
K: Hadamard block dimension (1 for power-of-2 sizes).
"""
def __init__(
self,
original_module: nn.Linear,
rotation_size: int,
hadamard_K: torch.Tensor | None = None,
K: int | None = None,
) -> None:
super().__init__()
if not isinstance(original_module, nn.Linear):
raise ValueError(
f"InputRotationWrapperHadamard only supports nn.Linear, " f"got {type(original_module).__name__}"
)
self._rotation_size = rotation_size
self._in_features = original_module.in_features
self._out_features = original_module.out_features
# Take ownership of weight and bias from the original module.
# The original module is NOT stored as a submodule — this prevents
# auto-round from discovering it via named_modules() and wrapping it.
self.weight = original_module.weight # nn.Parameter, auto-registered
if original_module.bias is not None:
self.bias = original_module.bias # nn.Parameter
else:
self.register_parameter("bias", None)
if self._in_features == rotation_size:
self._use_butterfly = True
elif self._in_features % rotation_size == 0:
self._use_butterfly = False
else:
raise ValueError(f"rotation_size={rotation_size} not compatible with " f"in_features={self._in_features}")
# Compute / store Hadamard matrix
if hadamard_K is None or K is None:
hadamard_K, K = get_hadamard_K(rotation_size)
self._K = K
device = self.weight.device
# Store hadamard_K as buffer (serialised with model)
self.register_buffer(
"hadamard_K",
hadamard_K.to(device=device, dtype=torch.float32),
)
# For block rotation, pre-build the full rotation matrix
if not self._use_butterfly:
rot_mat = hadamard_K.to(torch.float64)
if rot_mat.shape[0] != rotation_size:
had_1, _ = get_hadamard_K(rotation_size // K)
rot_mat = torch.kron(
hadamard_K.to(device="cpu", dtype=torch.float64),
had_1.to(device="cpu", dtype=torch.float64),
)
self.register_buffer(
"rotation_matrix",
rot_mat.to(device=device, dtype=torch.float32),
)
# --- Properties for compatibility with nn.Linear consumers ---
@property
def in_features(self) -> int:
return self._in_features
@property
def out_features(self) -> int:
return self._out_features
# --- Forward: rotate input, then F.linear ---
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self._use_butterfly:
x = matmul_hadU(
x,
hadamard_K=self.hadamard_K.to(x.device),
K=self._K,
)
else:
dtype = x.dtype
shape = x.shape
R = self.rotation_matrix.to(x.device, dtype=x.dtype)
x = x.reshape(*shape[:-1], -1, self._rotation_size)
x = (x @ R).reshape(shape).to(dtype)
return nn.functional.linear(x, self.weight, self.bias) # pylint: disable=not-callable
def __repr__(self) -> str:
return (
f"InputRotationWrapperHadamard(\n"
f" in_features={self._in_features}, out_features={self._out_features},\n"
f" rotation_size={self._rotation_size}, "
f"butterfly={self._use_butterfly}, K={self._K}\n"
f")"
)
def rotate_in_channels_(
layer: nn.Linear,
rotation_matrix: torch.Tensor | None = None,
R_in: torch.Tensor | None = None,
rotated_modules: set | None = None,
) -> None:
"""Fuse an input-side rotation into a linear layer's weight.
Mathematically::
W_new = W @ R.T
When ``R`` is smaller than the last dimension of ``W`` (block rotation),
the rotation is applied as a block-diagonal transform: the last dimension
is reshaped into ``(-1, rotation_size)``, ``@ R.T`` is applied, and the
result is flattened back. This follows the same convention as Quark's
``rotate_with_size``.
Args:
layer: The ``nn.Linear`` layer whose weight will be rotated.
rotation_matrix: Full rotation matrix ``R`` (uses ``R.T`` internally).
R_in: Alias for ``rotation_matrix`` provided for API symmetry with
``rotate_out_channels_``.
rotated_modules: Optional ``set`` used to deduplicate rotations
when a module is shared across layers (e.g. MoE).
"""
if rotated_modules is not None:
if layer in rotated_modules:
return
rotated_modules.add(layer)
W = layer.weight.data
R = R_in if R_in is not None else rotation_matrix
if R is None:
return
rot_size = R.shape[0]
W_f64 = W.to(torch.float64)
R_f64 = R.to(torch.float64)
if W.shape[-1] == rot_size:
# Full rotation: W_new = W @ R.T
layer.weight.data = (W_f64 @ R_f64.T).to(W.dtype)
elif W.shape[-1] % rot_size == 0:
# Block rotation: reshape → rotate → flatten
w_reshaped = W_f64.reshape(*W_f64.shape[:-1], -1, rot_size)
w_rotated = w_reshaped @ R_f64.T
layer.weight.data = w_rotated.reshape(W.shape).to(W.dtype)
else:
raise ValueError(f"rotate_in_channels_: rotation_size={rot_size} does not divide " f"weight dim={W.shape[-1]}")
# input-side rotation does NOT affect bias:
# y = (W @ R.T) @ (R @ x) + b = W @ x + b
# bias unchanged.
def rotate_out_channels_(
layer: nn.Linear,
rotation_matrix: torch.Tensor | None = None,
R_out: torch.Tensor | None = None,
rotated_modules: set | None = None,
) -> None:
"""Fuse an output-side rotation into a linear layer's weight.
Mathematically::
W_new = R.T @ W
When ``R`` is smaller than the output dimension (block rotation), the
first dimension (output channels / rows) is reshaped into
``(-1, rotation_size)``, transposed to apply ``@ R.T``, and reshaped back.
This follows the same convention as Quark's ``rotate_with_size``.
Args:
layer: The ``nn.Linear`` layer whose weight will be rotated.
rotation_matrix: Full rotation matrix ``R`` (uses ``R.T`` internally).
R_out: Alias for ``rotation_matrix``.
rotated_modules: Optional ``set`` for deduplication.
"""
if rotated_modules is not None:
if layer in rotated_modules:
return
rotated_modules.add(layer)
W = layer.weight.data
R = R_out if R_out is not None else rotation_matrix
if R is None:
return
rot_size = R.shape[0]
W_f64 = W.to(torch.float64)
R_f64 = R.to(torch.float64)
# output rotation: we need to rotate rows (output dim) of W
# For block rotation, use W.T so rotation acts on the last dim,
# then transpose back (same as Quark's approach)
out_dim = W.shape[0]
if out_dim == rot_size:
# Full rotation: W_new = R.T @ W
layer.weight.data = (R_f64.T @ W_f64).to(W.dtype)
elif out_dim % rot_size == 0:
# Block rotation on output dim: transpose, block rotate, transpose back
WT = W_f64.T # [in_features, out_features]
wt_reshaped = WT.reshape(*WT.shape[:-1], -1, rot_size)
wt_rotated = wt_reshaped @ R_f64
layer.weight.data = wt_rotated.reshape(WT.shape).T.contiguous().to(W.dtype)
else:
raise ValueError(f"rotate_out_channels_: rotation_size={rot_size} does not divide " f"output dim={out_dim}")
# Rotate bias if present
if layer.bias is not None:
bias_f64 = layer.bias.data.to(torch.float64)
if bias_f64.shape[0] == rot_size:
layer.bias.data = (R_f64.T @ bias_f64).to(layer.bias.dtype)
elif bias_f64.shape[0] % rot_size == 0:
b_reshaped = bias_f64.reshape(-1, rot_size)
b_rotated = (b_reshaped @ R_f64).reshape(-1)
layer.bias.data = b_rotated.to(layer.bias.dtype)
else:
raise ValueError(
f"rotate_out_channels_: rotation_size={rot_size} does not divide " f"bias dim={bias_f64.shape[0]}"
)
def fuse_rmsnorm_in_model(model: nn.Module) -> None:
"""
Fuse RMSNorm / LayerNorm ``gamma`` into the subsequent linear layers.
After fusion the norm layers become pure normalisation (``gamma == 1``).
Uses ``float64`` intermediate computation to avoid precision loss.
"""
# Attempt to use AutoRound's model_config layer discovery if available.
try:
from auto_round.algorithms.transforms.hadamard.inplace.model_config import get_scaling_layers
layer_paths = get_scaling_layers(model.config.model_type if hasattr(model, "config") else "")
if layer_paths:
# Model-config-driven fusion (supports GPT-2, OPT, etc.)
_fuse_rmsnorm_with_layer_paths(model, layer_paths)
return
except ImportError:
pass
# Fallback: hard-coded Llama-like traversal.
_fuse_rmsnorm_llama_like(model)
def _fuse_rmsnorm_llama_like(model: nn.Module) -> None:
"""Traverse layers, supporting both model.layers and model.model.layers patterns."""
# Try different layer container paths
layers = None
for path in ("model.layers", "layers", "transformer.h"):
parts = path.split(".")
obj = model
for p in parts:
if not hasattr(obj, p):
break
obj = getattr(obj, p)
else:
layers = obj
break
if layers is None:
# Try recursive search
for name, module in model.named_modules():
if name.endswith(".layers") or name == "layers":
if hasattr(module, "__iter__"):
layers = module
break
if layers is None:
return
for layer in layers:
# 1. input_layernorm -> q / k / v
if hasattr(layer, "input_layernorm") and hasattr(layer.input_layernorm, "weight"):
gamma = layer.input_layernorm.weight.data.to(torch.float64)
if hasattr(layer, "self_attn"):
for proj_name in ("q_proj", "k_proj", "v_proj"):
if hasattr(layer.self_attn, proj_name):
proj = getattr(layer.self_attn, proj_name)
w = proj.weight.data.to(torch.float64)
proj.weight.data = (w * gamma.view(1, -1)).to(proj.weight.dtype)
layer.input_layernorm.weight.data.fill_(1.0)
# 2. post_attention_layernorm -> gate / up
if hasattr(layer, "post_attention_layernorm") and hasattr(layer.post_attention_layernorm, "weight"):
gamma = layer.post_attention_layernorm.weight.data.to(torch.float64)
if hasattr(layer, "mlp"):
for proj_name in ("gate_proj", "up_proj"):
if hasattr(layer.mlp, proj_name):
proj = getattr(layer.mlp, proj_name)
w = proj.weight.data.to(torch.float64)
proj.weight.data = (w * gamma.view(1, -1)).to(proj.weight.dtype)
layer.post_attention_layernorm.weight.data.fill_(1.0)
# 3. final norm -> lm_head
final_norm = None
for path in ("model.norm", "norm"):
parts = path.split(".")
obj = model
for p in parts:
if not hasattr(obj, p):
break
obj = getattr(obj, p)
else:
final_norm = obj
break
lm_head = getattr(model, "lm_head", None)
if final_norm is not None and lm_head is not None and hasattr(final_norm, "weight"):
gamma = final_norm.weight.data.to(torch.float64)
w = lm_head.weight.data.to(torch.float64)
lm_head.weight.data = (w * gamma.view(1, -1)).to(lm_head.weight.dtype)
final_norm.weight.data.fill_(1.0)
def _fuse_rmsnorm_with_layer_paths(model: nn.Module, layer_paths: list[str]) -> None:
"""Model-config-driven RMSNorm fusion (supports non-Llama architectures)."""
raise NotImplementedError(
"Model-config-driven RMSNorm fusion is not yet implemented. "
"Use fuse_rmsnorm_into_linear() for Llama-family models, or set "
"r1=False to skip R1 rotation for unsupported architectures."
)
def untie_word_embeddings_if_needed(model: nn.Module) -> bool:
"""Untie input and output embeddings if they share storage.
Also sets ``config.tie_word_embeddings = False`` so that downstream
frameworks (e.g. lm_eval's HFLM, HuggingFace Trainer) do not re-tie
the weights after we've separated them.
"""
if hasattr(model, "model") and hasattr(model.model, "embed_tokens") and hasattr(model, "lm_head"):
embed = model.model.embed_tokens.weight
head = model.lm_head.weight
if embed.data_ptr() == head.data_ptr():
model.lm_head.weight = nn.Parameter(head.clone())
# Prevent frameworks from re-tying the now-separate weights
if hasattr(model, "config"):
model.config.tie_word_embeddings = False
return True
return False
def create_block_diag_from_head_matrix(R_head: torch.Tensor, num_heads: int) -> torch.Tensor:
"""Build a block-diagonal matrix from a per-head rotation matrix."""
return torch.block_diag(*[R_head for _ in range(num_heads)])
def apply_hadamard_to_linear(
module: nn.Linear,
had_dim: int = -1,
output: bool = False,
) -> None:
"""Apply deterministic Hadamard transform to a linear layer's weight.
This is a CPU-compatible equivalent of Quark's ``apply_exact_had_to_linear``.
It applies a normalized Hadamard transform per chunk of size ``had_dim``.
Args:
module: The linear layer to modify in-place.
had_dim: Hadamard block dimension. If -1, uses the full dimension
(in_features for input side, out_features for output side).
output: If True, apply to output channels (rows). If False, apply
to input channels (columns).
"""
# Support both nn.Linear and InputRotationWrapperHadamard
assert isinstance(
module, (nn.Linear, InputRotationWrapperHadamard)
), f"module must be nn.Linear or InputRotationWrapperHadamard, got {type(module).__name__}"
in_features = module.in_features
out_features = module.out_features
W = module.weight.data
dtype = W.dtype
device = W.device
W = W.to(torch.float64)
if had_dim == -1:
# Full Hadamard on the relevant dimension
dim = out_features if output else in_features
H = deterministic_hadamard_matrix(dim, dtype=torch.float64, device=device)
if output:
# W_new = H @ W (rotate output channels)
W = H @ W
else:
# W_new = W @ H (rotate input channels)
W = W @ H
else:
# Per-chunk Hadamard (block-diagonal application)
H = deterministic_hadamard_matrix(had_dim, dtype=torch.float64, device=device)
if output:
# Apply H per had_dim chunk on the output (row) dimension
# W shape: [out_features, in_features]
# Reshape to [out_features // had_dim, had_dim, in_features]
assert out_features % had_dim == 0, f"out_features={out_features} not divisible by had_dim={had_dim}"
n_chunks = out_features // had_dim
W_reshaped = W.reshape(n_chunks, had_dim, in_features)
# Apply H to each chunk: H @ chunk (along dim 1)
W_reshaped = torch.einsum("ij,kjl->kil", H, W_reshaped)
W = W_reshaped.reshape(out_features, in_features)
else:
# Apply H per had_dim chunk on the input (column) dimension
# W shape: [out_features, in_features]
# Reshape to [out_features, in_features // had_dim, had_dim]
assert in_features % had_dim == 0, f"in_features={in_features} not divisible by had_dim={had_dim}"
n_chunks = in_features // had_dim
W_reshaped = W.reshape(out_features, n_chunks, had_dim)
# Apply H to each chunk: chunk @ H.T (along last dim)
W_reshaped = torch.einsum("ijk,lk->ijl", W_reshaped, H)
W = W_reshaped.reshape(out_features, in_features)
module.weight.data = W.to(dtype)
# Handle bias for output-side rotation
if output and module.bias is not None:
b = module.bias.data.to(torch.float64)
if had_dim == -1:
b = H @ b
else:
b_reshaped = b.view(n_chunks, had_dim)
b_reshaped = torch.einsum("ij,kj->ki", H, b_reshaped)
b = b_reshaped.view(-1)
module.bias.data = b.to(module.bias.dtype)
def get_model_arch_info(model: nn.Module) -> dict:
"""
Extract architecture metadata from a model.
Returns a dict with keys: ``hidden_size``, ``head_dim``, ``num_q_heads``,
``num_kv_heads``, ``intermediate_size``, ``model_type``.
"""
info: dict = {"model_type": "unknown"}
if hasattr(model, "config"):
cfg = model.config
info["model_type"] = getattr(cfg, "model_type", "unknown")
info["hidden_size"] = getattr(cfg, "hidden_size", 0)
info["num_q_heads"] = getattr(cfg, "num_attention_heads", 0)
info["num_kv_heads"] = getattr(cfg, "num_key_value_heads", info["num_q_heads"])
info["intermediate_size"] = getattr(cfg, "intermediate_size", 0)
info["head_dim"] = getattr(cfg, "head_dim", info["hidden_size"] // max(info["num_q_heads"], 1))
if all(v for v in [info["hidden_size"], info["head_dim"], info["intermediate_size"]]):
return info
# Fallback: inspect model layers directly when .config is absent or incomplete
# Try common attribute paths for embeddings / decoder layers
embed = None
for path in ("embed_tokens", "model.embed_tokens", "transformer.wte", "decoder.embed_tokens"):
parts = path.split(".")
obj = model
for p in parts:
obj = getattr(obj, p, None)
if obj is None:
break
else:
embed = obj
break
if embed is not None:
info["hidden_size"] = getattr(embed, "embedding_dim", getattr(embed, "weight", torch.empty(0)).shape[-1])
# Inspect first decoder layer for attention / MLP dimensions
first_layer = None
for path in ("layers", "model.layers", "transformer.h", "model.decoder.layers"):
parts = path.split(".")
obj = model
for p in parts:
obj = getattr(obj, p, None)
if obj is None:
break
else:
if hasattr(obj, "__iter__"):
for layer in obj:
first_layer = layer
break
break
if first_layer is not None:
# Attention
attn = getattr(first_layer, "self_attn", None)
if attn is None:
attn = getattr(first_layer, "attn", None)
if attn is not None:
q_proj = getattr(attn, "q_proj", None)
if q_proj is not None:
out_dim = q_proj.weight.shape[0] # hidden_size
in_dim = q_proj.weight.shape[1] # hidden_size
info.setdefault("hidden_size", out_dim)
# Infer head count from output projection if available
o_proj = getattr(attn, "o_proj", None)
if o_proj is not None:
info.setdefault("hidden_size", o_proj.weight.shape[1])
# Try to infer head_dim from rotary_emb or k_proj
k_proj = getattr(attn, "k_proj", None)
if k_proj is not None:
kv_dim = k_proj.weight.shape[0]
# num_kv_heads = hidden_size // head_dim (approximate)
# We'll set head_dim = hidden_size // num_q_heads if we can find it
# MLP intermediate size
mlp = getattr(first_layer, "mlp", None)
if mlp is None:
mlp = getattr(first_layer, "ffn", None)
if mlp is not None:
gate = getattr(mlp, "gate_proj", None)
if gate is not None:
info.setdefault("intermediate_size", gate.weight.shape[0])
up = getattr(mlp, "up_proj", None)
if up is not None:
info.setdefault("intermediate_size", up.weight.shape[0])
# Final defaults
info.setdefault("hidden_size", 0)
info.setdefault("intermediate_size", 0)
info.setdefault("num_q_heads", 0)
info.setdefault("num_kv_heads", info["num_q_heads"])
info.setdefault("head_dim", info["hidden_size"] // max(info["num_q_heads"], 1))
return info
def get_attention_layers(model: nn.Module):
"""Yield attention modules using model_config if available, else fall back."""
try:
from auto_round.algorithms.transforms.hadamard.inplace.model_config import get_attention_layers as _get
return _get(model)
except ImportError:
pass
if hasattr(model, "model") and hasattr(model.model, "layers"):
for layer in model.model.layers:
if hasattr(layer, "self_attn"):
yield layer.self_attn
elif hasattr(model, "transformer") and hasattr(model.transformer, "h"):
for block in model.transformer.h:
if hasattr(block, "attn"):
yield block.attn