Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine awq, bugfix (#2115)

Weiweia3af36712a38
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
index e99494fed..338599e3a 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -130,8 +130,8 @@ class SignRoundOptimizedWrapperLinear(WrapperLinear):
Reproduces the layout the quant func sees at tuning time: the (optionally
transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in
- ``clip_as_init`` mode and stored a per-group ``awq_clip_max`` on the layer,
- the weight is clamped to that range first so the searched ``init_scale``
+ ``clip_as_init`` mode and stored a per-group clip range on the layer, the
+ weight is clamped to that range first so the searched ``init_scale``
already reflects the AWQ clip (``max_scale`` then tunes a coefficient on
top of it).
"""
@@ -141,15 +141,22 @@ class SignRoundOptimizedWrapperLinear(WrapperLinear):
weight = weight.t()
weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)
+ clip_min = getattr(layer, "awq_clip_min", None)
clip_max = getattr(layer, "awq_clip_max", None)
if clip_max is not None:
clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
- if clip_max.shape[0] == weight_reshape.shape[0]:
- weight_reshape = torch.clamp(weight_reshape, -clip_max, clip_max)
+ clip_min = (
+ -clip_max
+ if clip_min is None
+ else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
+ )
+ if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:
+ weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)
else:
logger.warning_once(
- "SignRoundV2: ignoring awq_clip_max with shape %s incompatible with "
- "grouped weight shape %s." % (tuple(clip_max.shape), tuple(weight_reshape.shape))
+ "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "
+ "grouped weight shape %s."
+ % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))
)
return weight_reshape
diff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.py
index 7b1bf5437..3f62ca205 100644
--- a/auto_round/algorithms/transforms/awq/base.py
+++ b/auto_round/algorithms/transforms/awq/base.py
@@ -14,7 +14,7 @@
"""AWQ (Activation-Aware Weight Quantization) quantizer.
Algorithm:
-1. Collect per-channel activation magnitudes during calibration.
+1. Collect per-channel balance-layer input magnitudes during calibration.
2. For each smooth-balance mapping, perform a grid search over scaling ratios
to find the one that minimises quantization error (output-based loss).
3. Apply the best channel-wise scaling:
@@ -38,9 +38,10 @@ import torch
from auto_round.algorithms.registry import register_pipeline_member
from auto_round.algorithms.transforms.awq.config import AWQConfig
from auto_round.algorithms.transforms.awq.mappings import (
+ AWQ_DYNAMIC_MAPPING_REGISTRY,
+ AWQ_MAPPING_REGISTRY,
ResolvedMapping,
_extract_block_prefix,
- check_model_compatibility,
resolve_mappings,
)
from auto_round.algorithms.transforms.awq.qdq import QDQTool
@@ -50,6 +51,7 @@ from auto_round.data_type.utils import (
revert_tensor_by_pad,
)
from auto_round.logger import logger
+from auto_round.utils.model import move_to_device
if TYPE_CHECKING:
from auto_round.algorithms.composer import AlgorithmComposer, BlockContext
@@ -102,6 +104,113 @@ def _rmsnorm_has_unit_offset(module: torch.nn.Module) -> bool:
return result
+def _slice_seq_tensor(v: Any, actual_seq: int, seqlen: int) -> Any:
+ """Slice a value's sequence dimension to *seqlen*, recursing into containers."""
+ if isinstance(v, torch.Tensor):
+ if v.ndim == 1 and v.shape[0] == actual_seq:
+ return v[:seqlen]
+ if v.ndim == 3 and v.shape[1] == actual_seq:
+ return v[:, :seqlen]
+ if v.ndim == 2 and v.shape[1] == actual_seq:
+ return v[:, :seqlen]
+ if v.ndim == 4 and v.shape[2] == actual_seq and v.shape[3] == actual_seq:
+ return v[:, :, :seqlen, :seqlen]
+ if v.ndim == 4 and v.shape[2] == actual_seq:
+ return v[:, :, :seqlen]
+ return v
+ if isinstance(v, (tuple, list)):
+ return type(v)(_slice_seq_tensor(t, actual_seq, seqlen) for t in v)
+ if isinstance(v, dict):
+ return {k: _slice_seq_tensor(t, actual_seq, seqlen) for k, t in v.items()}
+ return v
+
+
+def _detect_actual_seq(values) -> int | None:
+ """Infer sequence length from the first 3-D tensor, including nested containers."""
+ for v in values:
+ if isinstance(v, torch.Tensor) and v.ndim == 3:
+ return v.shape[1]
+ if isinstance(v, (tuple, list)):
+ nested = _detect_actual_seq(v)
+ if nested is not None:
+ return nested
+ if isinstance(v, dict):
+ nested = _detect_actual_seq(v.values())
+ if nested is not None:
+ return nested
+ return None
+
+
+def _truncate_args_kwargs(args: tuple, kwargs: dict, seqlen: int) -> tuple[tuple, dict]:
+ """Truncate parent-forward positional args and kwargs to at most *seqlen* tokens."""
+ actual_seq = _detect_actual_seq(list(args) + list(kwargs.values()))
+ if actual_seq is None or actual_seq <= seqlen:
+ return args, kwargs
+ new_args = tuple(_slice_seq_tensor(v, actual_seq, seqlen) for v in args)
+ new_kwargs = {k: _slice_seq_tensor(v, actual_seq, seqlen) for k, v in kwargs.items()}
+ return new_args, new_kwargs
+
+
+def _truncate_awq_tensor(x: torch.Tensor, seqlen: int) -> torch.Tensor:
+ """Truncate an AWQ activation tensor to at most *seqlen* tokens."""
+ actual_seq = _detect_actual_seq((x,))
+ if actual_seq is None or actual_seq <= seqlen:
+ return x
+ return _slice_seq_tensor(x, actual_seq, seqlen)
+
+
+def _detect_actual_batch(values) -> int | None:
+ """Infer batch size from the first tensor with an explicit batch axis."""
+ for v in values:
+ if isinstance(v, torch.Tensor) and v.ndim >= 2:
+ return v.shape[0]
+ if isinstance(v, (tuple, list)):
+ nested = _detect_actual_batch(v)
+ if nested is not None:
+ return nested
+ if isinstance(v, dict):
+ nested = _detect_actual_batch(v.values())
+ if nested is not None:
+ return nested
+ return None
+
+
+def _detect_parent_batch(args: tuple, kwargs: dict) -> int | None:
+ """Infer parent replay batch size, preferring positional model inputs."""
+ actual_batch = _detect_actual_batch(args)
+ if actual_batch is not None:
+ return actual_batch
+ return _detect_actual_batch(kwargs.values())
+
+
+def _slice_batch_value(v: Any, actual_batch: int, start: int, end: int) -> Any:
+ """Slice a value's leading batch dimension, recursing into containers."""
+ if isinstance(v, torch.Tensor):
+ if v.ndim >= 2 and v.shape[0] == actual_batch:
+ return v[start:end]
+ return v
+ if isinstance(v, (tuple, list)):
+ return type(v)(_slice_batch_value(t, actual_batch, start, end) for t in v)
+ if isinstance(v, dict):
+ return {k: _slice_batch_value(t, actual_batch, start, end) for k, t in v.items()}
+ return v
+
+
+def _slice_batch_args_kwargs(args: tuple, kwargs: dict, actual_batch: int, start: int, end: int) -> tuple[tuple, dict]:
+ """Slice parent-forward positional args and kwargs to a batch microchunk."""
+ new_args = tuple(_slice_batch_value(v, actual_batch, start, end) for v in args)
+ new_kwargs = {k: _slice_batch_value(v, actual_batch, start, end) for k, v in kwargs.items()}
+ return new_args, new_kwargs
+
+
+def _iter_block_names_for_mapping(model: torch.nn.Module) -> list[str]:
+ """Return block names sorted from most-specific to least-specific for mapping lookup."""
+ from auto_round.utils.common import flatten_list
+ from auto_round.utils.model import get_block_names
+
+ return sorted((name for name in flatten_list(get_block_names(model)) if name), key=len, reverse=True)
+
+
@register_pipeline_member(AWQConfig)
class AWQTransform(BasePreprocessor):
"""AWQ transform: activation-aware weight smoothing pre-processor.
@@ -124,6 +233,13 @@ class AWQTransform(BasePreprocessor):
self.clip_max_shrink: float = getattr(config, "clip_max_shrink", 0.5)
self.clip_n_sample_token: int = getattr(config, "clip_n_sample_token", 512)
+ # Cap sequence length consistently across AWQ calibration.
+ # A value <= 0 disables truncation and uses the full calibration sequence.
+ awq_seqlen = getattr(config, "awq_seqlen", 512)
+ self._awq_seqlen: int | None = awq_seqlen if awq_seqlen > 0 else None
+ smooth_batch_size = getattr(config, "smooth_batch_size", None)
+ self._smooth_batch_size: int | None = smooth_batch_size if smooth_batch_size and smooth_batch_size > 0 else None
+
# Single source of truth for "QDQ a candidate weight under the target
# block-quantizer scheme", used as AWQ's grid-search / clip loss. AWQ
# composes this instead of re-implementing block-quantizer dispatch;
@@ -135,6 +251,7 @@ class AWQTransform(BasePreprocessor):
)
self._user_mappings: list[dict] | None = config.mappings
+ self._skip_moe: bool = getattr(config, "skip_moe", False)
# Set at runtime by the compressor's post_init() via ``pre.layer_config = self.layer_config``.
self.layer_config: dict | None = None
@@ -143,8 +260,7 @@ class AWQTransform(BasePreprocessor):
self._block_mappings: dict[str, list[ResolvedMapping]] = {}
self._activation_stats: dict[str, list] = {}
- self._parent_args_cache: dict[torch.nn.Module, list[dict]] = {}
- self._parent_signatures: dict[int, inspect.Signature] = {}
+ self._parent_args_cache: dict[torch.nn.Module, list[tuple[tuple, dict]]] = {}
# Per-mapping balance-layer input features captured for the clip search
# (keyed by smooth_name). Only populated when ``apply_clip`` is set.
self._clip_input_feat: dict[str, torch.Tensor] = {}
@@ -165,14 +281,11 @@ class AWQTransform(BasePreprocessor):
exit(-1)
def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:
- """Validate compatibility, resolve model-wide mappings, and group by block prefix."""
+ """Resolve model-wide mappings and group them by transformer block."""
model = self.model
- report = check_model_compatibility(model, self._user_mappings)
- for warning in report["warnings"]:
- logger.warning(warning)
# ── Resolve all model-level mappings (name-only, no module caching) ──
- self._resolved_mappings = resolve_mappings(model, self._user_mappings)
+ self._resolved_mappings = resolve_mappings(model, self._user_mappings, skip_moe=self._skip_moe)
if not self._resolved_mappings:
raise ValueError(
"AWQ: no layer mappings were resolved for this model. "
@@ -181,11 +294,31 @@ class AWQTransform(BasePreprocessor):
"add an entry to auto_round/algorithms/transforms/awq/mappings.py."
)
- # Group mappings by block prefix for O(1) lookup during block iteration.
+ cls_name = type(model).__name__
+ if (
+ self._user_mappings is None
+ and cls_name not in AWQ_MAPPING_REGISTRY
+ and cls_name not in AWQ_DYNAMIC_MAPPING_REGISTRY
+ ):
+ logger.warning(
+ "AWQ: model class '%s' is not in any AWQ mapping registry; using "
+ "default Llama-like mappings. If quantization quality is poor, "
+ "provide explicit mappings via AWQConfig(mappings=[...]).",
+ cls_name,
+ )
+
+ iter_block_names = _iter_block_names_for_mapping(model)
+
self._block_mappings = {}
for m in self._resolved_mappings:
- prefix = _extract_block_prefix(m.smooth_name)
- self._block_mappings.setdefault(prefix, []).append(m)
+ key = None
+ for block_name in iter_block_names:
+ if m.smooth_name == block_name or m.smooth_name.startswith(block_name + "."):
+ key = block_name
+ break
+ if key is None:
+ key = _extract_block_prefix(m.smooth_name)
+ self._block_mappings.setdefault(key, []).append(m)
if composer is not None:
self._qdq_tool.configure(composer)
@@ -226,11 +359,22 @@ class AWQTransform(BasePreprocessor):
# The compressor sets ``layer_config`` after ``prepare_run``; keep the
# QDQ service in sync before it is used for the grid-search / clip loss.
self._qdq_tool.layer_config = self.layer_config
- self._smooth_block(block_name, block_mappings)
+ active_mappings = [m for m in block_mappings if self._mapping_is_smoothable(m)]
+ skipped = len(block_mappings) - len(active_mappings)
+ if skipped:
+ logger.warning_once(
+ "AWQ: skipped %d smoothing mapping(s) in block '%s' that include "
+ "ignore_layers / full-precision layers or incompatible per-layer quantization parameters.",
+ skipped,
+ block_name,
+ )
+ if not active_mappings:
+ return
+ self._smooth_block(block_name, active_mappings)
if self.apply_clip:
- self._clip_block(block_name, block_mappings)
+ self._clip_block(block_name, active_mappings)
modified = []
- for mapping in block_mappings:
+ for mapping in active_mappings:
modified.extend(mapping.balance_names)
modified.append(mapping.smooth_name)
@@ -245,6 +389,8 @@ class AWQTransform(BasePreprocessor):
# Drop the transient clip attribute once the block quantizer has
# consumed it (the persistent copy lives on the model context).
for bl in m.balance_layers:
+ if hasattr(bl, "awq_clip_min"):
+ delattr(bl, "awq_clip_min")
if hasattr(bl, "awq_clip_max"):
delattr(bl, "awq_clip_max")
seen_parents: set[int] = set()
@@ -260,7 +406,6 @@ class AWQTransform(BasePreprocessor):
return
self._activation_stats.clear()
self._parent_args_cache.clear()
- self._parent_signatures.clear()
self._clip_input_feat.clear()
self._finalized = True
logger.debug("AWQ: finalize_quantization complete.")
@@ -276,85 +421,90 @@ class AWQTransform(BasePreprocessor):
"""Register activation-stats and parent-kwargs hooks for one block."""
handles = []
mappings = self._block_mappings.get(block_name, [])
- smooth_names = {m.smooth_name for m in mappings}
-
- # ── Smooth-layer activation-stats hooks ───────────────────────────────
- # Priority: smooth source forward_hook (output stats).
- # Each smooth source is hooked exactly once (set de-duplication via name).
- for name, module in block.named_modules():
- full_name = f"{block_name}.{name}" if name else block_name
- if full_name not in smooth_names:
+ module_lookup = dict(model.named_modules())
+
+ def _resolve_activation_hook_layer(mapping: ResolvedMapping) -> torch.nn.Module | None:
+ if not mapping.balance_layers:
+ return None
+
+ hook_target = mapping.activation_hook_target
+ if not hook_target:
+ return mapping.balance_layers[0]
+
+ target_layer = module_lookup.get(hook_target)
+ if target_layer is None and mapping.parent_name:
+ target_layer = module_lookup.get(f"{mapping.parent_name}.{hook_target}")
+ if target_layer is None:
+ try:
+ target_layer = mapping.parent.get_submodule(hook_target)
+ except AttributeError:
+ target_layer = None
+ if target_layer is None:
+ logger.warning(
+ "AWQ: activation_hook_target '%s' for '%s' was not found; using first balance layer '%s'.",
+ hook_target,
+ mapping.smooth_name,
+ mapping.balance_names[0] if mapping.balance_names else "<unknown>",
+ )
+ return mapping.balance_layers[0]
+ return target_layer
+
+ # ── Balance-layer input activation hooks ─────────────────────────────
+ # AWQ scales are derived from the tensor entering the balance layer. For
+ # gated MLPs, smooth-layer output (for example up_proj(x)) is not the same
+ # tensor consumed by down_proj (act(gate_proj(x)) * up_proj(x)).
+ for mapping in mappings:
+ target_layer = _resolve_activation_hook_layer(mapping)
+ if target_layer is None:
continue
- def _make_stats_hook(layer_name: str):
+ def _make_activation_hook(smooth_name: str):
- def hook_fn(mod, args, output):
- x = output[0] if isinstance(output, tuple) else output
- if x is None or x.numel() == 0:
+ def hook_fn(mod, args):
+ x = args[0] if isinstance(args, tuple) else args
+ if x is None or not isinstance(x, torch.Tensor) or x.numel() == 0:
return
- channel_sum = x.detach().float().flatten(0, -2).abs().sum(dim=0).cpu()
- count = x[..., 0].numel()
- if layer_name not in self._activation_stats:
- self._activation_stats[layer_name] = [
+
+ feat = x.detach()
+ if self._awq_seqlen is not None:
+ feat = _truncate_awq_tensor(feat, self._awq_seqlen)
+ if feat.ndim == 1:
+ feat = feat.view(1, -1)
+ else:
+ feat = feat.flatten(0, -2)
+
+ channel_sum = feat.float().abs().sum(dim=0).cpu()
+ count = feat.shape[0]
+ if smooth_name not in self._activation_stats:
+ self._activation_stats[smooth_name] = [
torch.zeros_like(channel_sum),
0,
]
- self._activation_stats[layer_name][0] += channel_sum
- self._activation_stats[layer_name][1] += count
-
- return hook_fn
-
- h = module.register_forward_hook(_make_stats_hook(full_name))
- handles.append(h)
-
- # ── Clip input-feature hooks (only when apply_clip is enabled) ────────
- # The balance layers of a mapping share the same input; capture it once
- # per mapping (keyed by smooth_name) for the post-smooth clip search.
- if self.apply_clip:
- for mapping in mappings:
- if not mapping.balance_layers:
- continue
- target_layer = mapping.balance_layers[0]
-
- def _make_clip_hook(smooth_name: str):
+ self._activation_stats[smooth_name][0] += channel_sum
+ self._activation_stats[smooth_name][1] += count
- def hook_fn(mod, args):
- x = args[0] if isinstance(args, tuple) else args
- if x is None or not isinstance(x, torch.Tensor) or x.numel() == 0:
- return
- feat = x.detach().reshape(-1, x.shape[-1])
+ if self.apply_clip:
+ clip_feat = feat
# Subsample tokens to bound memory.
- if feat.shape[0] > self.clip_n_sample_token:
- step = max(1, feat.shape[0] // self.clip_n_sample_token)
- feat = feat[::step]
- feat = feat.float().cpu()
+ if clip_feat.shape[0] > self.clip_n_sample_token:
+ step = max(1, clip_feat.shape[0] // self.clip_n_sample_token)
+ clip_feat = clip_feat[::step]
+ clip_feat = clip_feat.float().cpu()
prev = self._clip_input_feat.get(smooth_name)
if prev is None:
- self._clip_input_feat[smooth_name] = feat
+ self._clip_input_feat[smooth_name] = clip_feat
else:
- self._clip_input_feat[smooth_name] = torch.cat([prev, feat], dim=0)
+ self._clip_input_feat[smooth_name] = torch.cat([prev, clip_feat], dim=0)
- return hook_fn
+ return hook_fn
- h = target_layer.register_forward_pre_hook(_make_clip_hook(mapping.smooth_name))
- handles.append(h)
+ h = target_layer.register_forward_pre_hook(_make_activation_hook(mapping.smooth_name))
+ handles.append(h)
# One forward_pre_hook per unique parent module in the current block.
parent_modules_hooked: set[int] = set()
for mapping in mappings:
parent = mapping.parent
- hook_target = mapping.activation_hook_target
- if hook_target:
- target_parent = dict(model.named_modules()).get(hook_target)
- if target_parent is None:
- logger.warning(
- "AWQ: activation_hook_target '%s' for '%s' was not found; using resolved parent '%s'.",
- hook_target,
- mapping.smooth_name,
- mapping.parent_name,
- )
- else:
- parent = target_parent
if id(parent) in parent_modules_hooked:
continue
parent_modules_hooked.add(id(parent))
@@ -365,41 +515,31 @@ class AWQTransform(BasePreprocessor):
def _make_parent_hook(parent_module: torch.nn.Module):
def hook_fn(mod, args, kwargs):
- cls_id = id(type(mod))
- if cls_id not in self._parent_signatures:
- self._parent_signatures[cls_id] = inspect.signature(mod.forward)
- sig = self._parent_signatures[cls_id]
- try:
- bound = sig.bind(*args, **kwargs)
- bound.apply_defaults()
- except TypeError:
- return # signature mismatch; skip this sample
-
param = next(mod.parameters(), None)
w_dtype = param.dtype if param is not None else None
- stored: dict[str, Any] = {}
- for k, v in bound.arguments.items():
+ def _proc(v):
+ if hasattr(v, "key_cache"):
+ return None
if isinstance(v, torch.Tensor):
v = v.detach()
if w_dtype and v.is_floating_point() and v.dtype != w_dtype:
v = v.to(w_dtype)
- stored[k] = v
- elif isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v):
- stored[k] = tuple(
- (
- t.detach().to(w_dtype)
- if (w_dtype and isinstance(t, torch.Tensor) and t.is_floating_point())
- else (t.detach() if isinstance(t, torch.Tensor) else t)
- )
- for t in v
- )
- elif hasattr(v, "key_cache"):
- stored[k] = None # Null out KV cache objects
- else:
- stored[k] = v
-
- self._parent_args_cache[parent_module].append(stored)
+ return v.to("cpu", non_blocking=False)
+ if isinstance(v, tuple):
+ return tuple(_proc(t) for t in v)
+ if isinstance(v, list):
+ return [_proc(t) for t in v]
+ if isinstance(v, dict):
+ return {k: _proc(t) for k, t in v.items()}
+ return v
+
+ proc_args = tuple(_proc(a) for a in args)
+ proc_kwargs = {k: _proc(v) for k, v in kwargs.items()}
+
+ if self._awq_seqlen is not None:
+ proc_args, proc_kwargs = _truncate_args_kwargs(proc_args, proc_kwargs, self._awq_seqlen)
+ self._parent_args_cache[parent_module].append((proc_args, proc_kwargs))
return hook_fn
@@ -410,6 +550,64 @@ class AWQTransform(BasePreprocessor):
# ── Smoothing (grid search + scale apply) ─────────────────────────────────
+ def _mapping_has_ignored_layer(self, mapping: ResolvedMapping) -> bool:
+ """Return True if the smooth layer or any balance layer is kept full precision."""
+ layer_config = self._qdq_tool.layer_config or {}
+ if not layer_config:
+ return False
+
+ def _is_fp(layer: torch.nn.Module) -> bool:
+ name = getattr(layer, "global_name", None)
+ if not name or name not in layer_config:
+ return False
+ bits = layer_config[name].get("bits", None)
+ return bits is not None and bits >= 16
+
+ if _is_fp(mapping.smooth_layer):
+ return True
+ return any(_is_fp(bl) for bl in mapping.balance_layers)
+
+ @staticmethod
+ def _freeze_quant_param(value):
+ if isinstance(value, list):
+ return tuple(AWQTransform._freeze_quant_param(item) for item in value)
+ if isinstance(value, tuple):
+ return tuple(AWQTransform._freeze_quant_param(item) for item in value)
+ return value
+
+ def _balance_quant_signature(self, layer: torch.nn.Module) -> tuple:
+ """Return the resolved quantization signature that must match within one AWQ mapping."""
+ params = self._qdq_tool.resolve_params(layer)
+ keys = ("bits", "group_size", "sym", "data_type", "super_bits", "super_group_size")
+ return tuple((key, self._freeze_quant_param(params.get(key))) for key in keys)
+
+ def _mapping_has_mixed_quant_params(self, mapping: ResolvedMapping) -> bool:
+ """Return True when balance layers in one AWQ smoothing group do not share quant params."""
+ if len(mapping.balance_layers) <= 1:
+ return False
+
+ signatures = [self._balance_quant_signature(layer) for layer in mapping.balance_layers]
+ first = signatures[0]
+ if all(signature == first for signature in signatures[1:]):
+ return False
+
+ details = {name: dict(signature) for name, signature in zip(mapping.balance_names, signatures)}
+ logger.warning(
+ "AWQ: skipping smoothing for '%s' because balance layers in the same mapping "
+ "have different quantization parameters: %s.",
+ mapping.smooth_name,
+ details,
+ )
+ return True
+
+ def _mapping_is_smoothable(self, mapping: ResolvedMapping) -> bool:
+ """AWQ smoothing is all-or-nothing for layers sharing one smooth scale."""
+ if self._mapping_has_ignored_layer(mapping):
+ return False
+ if self._mapping_has_mixed_quant_params(mapping):
+ return False
+ return True
+
def _smooth_block(self, block_prefix: str, block_mappings: list) -> None:
"""Run grid search and apply AWQ scales for one block.
@@ -503,7 +701,8 @@ class AWQTransform(BasePreprocessor):
device = mapping.balance_layers[0].weight.device
x_mean = x_mean.to(device)
- group_size = self._normalize_group_size(self._qdq_tool.group_size, -1)
+ bl_params = {bl: self._qdq_tool.resolve_params(bl) for bl in mapping.balance_layers}
+ group_size = self._normalize_group_size(bl_params[mapping.balance_layers[0]]["group_size"], -1)
if self.duo_scaling is not False:
w_mean = self._compute_layer_means(mapping.balance_layers, group_size).to(device)
@@ -511,7 +710,11 @@ class AWQTransform(BasePreprocessor):
use_parent_forward = len(parent_kwargs_list) > 0
if use_parent_forward:
- fp16_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)
+ fp16_outputs = self._run_parent_samples(
+ mapping.parent,
+ parent_kwargs_list,
+ offload_to_cpu=self._smooth_batch_size is not None,
+ )
if not fp16_outputs or all(f.numel() == 0 for f in fp16_outputs):
use_parent_forward = False
@@ -519,11 +722,11 @@ class AWQTransform(BasePreprocessor):
if not use_parent_forward:
orig_weights = orig_state # same reference is fine
- # Resolve each balance layer's scheme once, then pre-resolve the quant
- # functions for the grid-search loop. ``opt_quant_func`` is non-None only
- # when the SignRoundV2 optimized init-scale path applies for this mapping.
- bl_params = {bl: self._qdq_tool.resolve_params(bl) for bl in mapping.balance_layers}
- cached_quant_func, opt_quant_func = self._qdq_tool.resolve_quant_funcs(bl_params[mapping.balance_layers[0]])
+ # Resolve each balance layer's quant functions once, then reuse them in
+ # the grid-search loop. Normal AWQ flow requires one mapping to have
+ # compatible quant params, but keeping this per-layer avoids hidden
+ # coupling to the first layer and makes direct calls robust.
+ bl_quant_funcs = {bl: self._qdq_tool.resolve_quant_funcs(bl_params[bl]) for bl in mapping.balance_layers}
best_error = float("inf")
best_scales = None
@@ -544,28 +747,28 @@ class AWQTransform(BasePreprocessor):
# de-smoothed result back, so the parent forward below sees the
# weights the layer would actually compute with.
for bl in mapping.balance_layers:
+ quant_func, opt_quant_func = bl_quant_funcs[bl]
w_qdq = self._qdq_tool.qdq(
orig_state[bl] * scales_view,
bl_params[bl],
- quant_func=cached_quant_func,
+ quant_func=quant_func,
opt_quant_func=opt_quant_func,
imatrix=getattr(bl, "imatrix", None),
)
bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)
- int_w_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)
- total_loss = self._compute_loss(fp16_outputs, int_w_outputs)
- del int_w_outputs
+ total_loss = self._compute_parent_loss(mapping.parent, parent_kwargs_list, fp16_outputs)
for bl in mapping.balance_layers:
bl.weight.data.copy_(orig_state[bl])
else:
total_loss = 0.0
for bl in mapping.balance_layers:
+ quant_func, opt_quant_func = bl_quant_funcs[bl]
w_orig = orig_weights[bl].to(device)
w_qdq = self._qdq_tool.qdq(
w_orig * scales_view,
bl_params[bl],
- quant_func=cached_quant_func,
+ quant_func=quant_func,
opt_quant_func=opt_quant_func,
imatrix=getattr(bl, "imatrix", None),
)
@@ -583,20 +786,80 @@ class AWQTransform(BasePreprocessor):
logger.debug("AWQ '%s': best_ratio=%.2f, best_error=%.3e", mapping.smooth_name, best_ratio, best_error)
return best_scales
+ def _iter_parent_calls(self, stored_args: tuple, stored_kwargs: dict):
+ """Yield full or microbatched parent-call args from one cached calibration batch."""
+ actual_batch = _detect_parent_batch(stored_args, stored_kwargs)
+ if self._smooth_batch_size is None or actual_batch is None or actual_batch <= self._smooth_batch_size:
+ yield stored_args, stored_kwargs
+ return
+
+ for start in range(0, actual_batch, self._smooth_batch_size):
+ end = min(actual_batch, start + self._smooth_batch_size)
+ yield _slice_batch_args_kwargs(stored_args, stored_kwargs, actual_batch, start, end)
+
+ @staticmethod
+ def _normalize_parent_output(out: Any) -> torch.Tensor:
+ """Extract the tensor output used by AWQ parent-output loss."""
+ if isinstance(out, tuple):
+ return out[0]
+ return out
+
@torch.no_grad()
def _run_parent_samples(
self,
parent: torch.nn.Module,
- kwargs_list: list[dict],
+ kwargs_list: list[tuple[tuple, dict]],
+ offload_to_cpu: bool = False,
) -> list[torch.Tensor]:
+ param = next(parent.parameters(), None)
+ device = param.device if param is not None else torch.device("cpu")
+
outputs = []
- for stored_kwargs in kwargs_list:
- out = parent(**stored_kwargs)
- if isinstance(out, tuple):
- out = out[0]
- outputs.append(out)
+ for stored_args, stored_kwargs in kwargs_list:
+ for micro_args, micro_kwargs in self._iter_parent_calls(stored_args, stored_kwargs):
+ call_args = tuple(move_to_device(a, device) for a in micro_args)
+ call_kwargs = {k: move_to_device(v, device) for k, v in micro_kwargs.items()}
+ out = self._normalize_parent_output(parent(*call_args, **call_kwargs)).detach()
+ if offload_to_cpu:
+ out = out.to("cpu", non_blocking=False)
+ outputs.append(out)
return outputs
+ @torch.no_grad()
+ def _compute_parent_loss(
+ self,
+ parent: torch.nn.Module,
+ kwargs_list: list[tuple[tuple, dict]],
+ fp16_outputs: list[torch.Tensor],
+ ) -> float:
+ """Replay parent samples and stream MSE loss without storing candidate outputs."""
+ param = next(parent.parameters(), None)
+ device = param.device if param is not None else torch.device("cpu")
+
+ loss = torch.tensor(0.0, device=device)
+ num_elements = torch.tensor(0, device=device, dtype=torch.long)
+ output_idx = 0
+ for stored_args, stored_kwargs in kwargs_list:
+ for micro_args, micro_kwargs in self._iter_parent_calls(stored_args, stored_kwargs):
+ if output_idx >= len(fp16_outputs):
+ return float("inf")
+ call_args = tuple(move_to_device(a, device) for a in micro_args)
+ call_kwargs = {k: move_to_device(v, device) for k, v in micro_kwargs.items()}
+ out = self._normalize_parent_output(parent(*call_args, **call_kwargs))
+ fp16_out = fp16_outputs[output_idx].to(device, non_blocking=False)
+ loss += torch.nn.functional.mse_loss(
+ fp16_out.float(),
+ out.float(),
+ reduction="sum",
+ )
+ num_elements += fp16_out.numel()
+ output_idx += 1
+ del out, fp16_out
+
+ if output_idx != len(fp16_outputs) or num_elements == 0:
+ return float("inf")
+ return (loss / num_elements).item()
+
@staticmethod
@torch.no_grad()
def _compute_loss(
@@ -684,15 +947,16 @@ class AWQTransform(BasePreprocessor):
recorded on the model context (and, in ``clip_as_init`` mode, on the
balance layer) so it is kept for downstream use. Two modes:
- * ``clip_as_init=False`` (default): the clip is hard-clamped in place on
- the (already smoothed) balance-layer weights, so any downstream block
- quantizer (RTN / SignRound / SignRoundV2) re-derives its min/max
- range from the clipped weights.
- * ``clip_as_init=True``: the weights are left untouched and the clip is
- stored on the layer (``awq_clip_max``); the downstream SignRound /
- SignRoundV2 quantizer uses it to *initialize* its tunable weight range
- (capping ``weight_min``/``weight_max`` or clamping before the scale
- search) and then tunes ``min_scale``/``max_scale`` on top.
+ * ``clip_as_init=False`` (default): the clip range is hard-clamped in
+ place on the (already smoothed) balance-layer weights, so any
+ downstream block quantizer (RTN / SignRound / SignRoundV2) re-derives
+ its min/max range from the clipped weights.
+ * ``clip_as_init=True``: the weights are left untouched and the clip
+ range is stored on the layer (``awq_clip_min`` / ``awq_clip_max``);
+ the downstream SignRound / SignRoundV2 quantizer uses it to
+ *initialize* its tunable weight range (capping
+ ``weight_min``/``weight_max`` or clamping before the scale search) and
+ then tunes ``min_scale``/``max_scale`` on top.
"""
clip_store = getattr(self.model_context, "awq_clip_values", None)
for mapping in block_mappings:
@@ -708,29 +972,37 @@ class AWQTransform(BasePreprocessor):
if self._should_skip_clip(name):
logger.debug("AWQ: skip clip for '%s' (avoid-clipping layer).", name)
continue
- max_val = self._compute_best_clip(bl, feat)
- if max_val is None:
+ clip_range = self._compute_best_clip(bl, feat)
+ if clip_range is None:
continue
+ min_val, max_val = clip_range
key = getattr(bl, "global_name", None) or name
if clip_store is not None:
- clip_store[key] = max_val.detach().to("cpu")
+ if torch.allclose(min_val, -max_val):
+ clip_store[key] = max_val.detach().to("cpu")
+ else:
+ clip_store[key] = {
+ "min": min_val.detach().to("cpu"),
+ "max": max_val.detach().to("cpu"),
+ }
if self.clip_as_init:
# Keep the weights intact; hand the clip to the block
# quantizer as the initialization of its weight range.
+ bl.awq_clip_min = min_val.detach()
bl.awq_clip_max = max_val.detach()
else:
- self._apply_clip(bl, max_val)
+ self._apply_clip(bl, min_val, max_val)
@torch.no_grad()
def _compute_best_clip(
self,
layer: torch.nn.Module,
input_feat: torch.Tensor,
- ) -> torch.Tensor | None:
+ ) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Search the per-group clip threshold that minimizes output MSE.
- Returns a ``[out_channels, n_group]`` tensor of clip magnitudes, or
- ``None`` if clipping is not applicable to this layer.
+ Returns ``(min_val, max_val)`` per-group tensors, or ``None`` if
+ clipping is not applicable to this layer.
"""
params = self._qdq_tool.resolve_params(layer)
bits = params["bits"]
@@ -772,39 +1044,53 @@ class AWQTransform(BasePreprocessor):
# Batch output channels to bound peak memory.
oc_batch = 256 if out_features % 256 == 0 else (64 if out_features % 64 == 0 else out_features)
+ use_asym_clip = params["sym"] is False
+ best_min_val_all = []
best_max_val_all = []
n_steps = max(1, int(self.clip_max_shrink * self.clip_n_grid))
for i_b in range(0, out_features, oc_batch):
w_b = w[i_b : i_b + oc_batch]
- org_max_val = w_b.abs().amax(dim=-1, keepdim=True) # [oc_b, 1, n_group, 1]
+ if use_asym_clip:
+ org_min_val = w_b.amin(dim=-1, keepdim=True).clamp(max=0)
+ org_max_val = w_b.amax(dim=-1, keepdim=True).clamp(min=0)
+ else:
+ org_max_val = w_b.abs().amax(dim=-1, keepdim=True) # [oc_b, 1, n_group, 1]
+ org_min_val = -org_max_val
+ best_min_val = org_min_val.clone()
best_max_val = org_max_val.clone()
min_errs = torch.full_like(org_max_val, 1e9)
org_out = (feat * w_b).sum(dim=-1) # [oc_b, n_token, n_group]
for i_s in range(n_steps):
- max_val = org_max_val * (1 - i_s / self.clip_n_grid)
- cur_w = torch.clamp(w_b, -max_val, max_val)
+ shrink = 1 - i_s / self.clip_n_grid
+ min_val = org_min_val * shrink
+ max_val = org_max_val * shrink
+ cur_w = torch.clamp(w_b, min_val, max_val)
cur_w_flat = cur_w.reshape(cur_w.shape[0], n_group * gs)
q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quant_func=quant_func).reshape(cur_w.shape)
cur_out = (feat * q_w).sum(dim=-1)
err = (cur_out - org_out).pow(2).mean(dim=1).view(min_errs.shape)
improved = err < min_errs
min_errs[improved] = err[improved]
+ best_min_val[improved] = min_val[improved]
best_max_val[improved] = max_val[improved]
del cur_w, q_w, cur_out
+ best_min_val_all.append(best_min_val)
best_max_val_all.append(best_max_val)
+ best_min_val = torch.cat(best_min_val_all, dim=0)
best_max_val = torch.cat(best_max_val_all, dim=0)
- return best_max_val.squeeze(1) # [out_features, n_group, 1]
+ return best_min_val.squeeze(1), best_max_val.squeeze(1) # [out_features, n_group, 1]
@torch.no_grad()
- def _apply_clip(self, layer: torch.nn.Module, max_val: torch.Tensor) -> None:
- """Hard-clamp the layer weight to ``[-max_val, max_val]`` per group."""
+ def _apply_clip(self, layer: torch.nn.Module, min_val: torch.Tensor, max_val: torch.Tensor) -> None:
+ """Hard-clamp the layer weight to ``[min_val, max_val]`` per group."""
org_dtype = layer.weight.dtype
+ min_val = min_val.to(device=layer.weight.device, dtype=org_dtype)
max_val = max_val.to(device=layer.weight.device, dtype=org_dtype)
org_shape = layer.weight.shape
w = layer.weight.data.reshape(*max_val.shape[:2], -1)
- w = torch.clamp(w, -max_val, max_val)
+ w = torch.clamp(w, min_val, max_val)
layer.weight.data = w.reshape(org_shape).to(org_dtype)
diff --git a/auto_round/algorithms/transforms/awq/config.py b/auto_round/algorithms/transforms/awq/config.py
index b5b620735..520846011 100644
--- a/auto_round/algorithms/transforms/awq/config.py
+++ b/auto_round/algorithms/transforms/awq/config.py
@@ -47,6 +47,9 @@ class AWQConfig(QuantizationConfig):
clip_n_grid: int = 20,
clip_max_shrink: float = 0.5,
clip_n_sample_token: int = 512,
+ awq_seqlen: int | None = None,
+ smooth_batch_size: int | None = None,
+ skip_moe: bool = True,
mappings: list[dict] | None = None,
**kwargs,
):
@@ -75,8 +78,10 @@ class AWQConfig(QuantizationConfig):
resulting scales. ``1`` reproduces the original single-pass AWQ.
apply_clip: Whether to search and apply AWQ weight clipping after
smoothing. When True, AWQ searches a per-group clipping
- threshold for each balance layer (minimizing output MSE) and
- hard-clamps the weights to ``[-max_val, max_val]`` in place.
+ range for each balance layer (minimizing output MSE) and
+ hard-clamps the weights in place. Symmetric quantization uses a
+ symmetric ``[-max_val, max_val]`` range; asymmetric quantization
+ searches separate ``min_val`` and ``max_val`` bounds.
This is a pure weight transformation, so it composes with any
downstream block quantizer: a SignRound/SignRoundV2 quantizer
re-derives its ``weight_min``/``weight_max`` from the clamped
@@ -87,7 +92,7 @@ class AWQConfig(QuantizationConfig):
clip_as_init: Selects how the searched clip threshold is consumed
(only relevant when ``apply_clip`` is True). When False
(default), the weights are hard-clamped in place. When True,
- the weights are left untouched and the per-group clip magnitude
+ the weights are left untouched and the per-group clip range
is instead stored (on the model context and on each balance
layer) so the downstream block quantizer uses it to *initialize*
its tunable weight range: SignRound caps its
@@ -104,6 +109,24 @@ class AWQConfig(QuantizationConfig):
clip_n_sample_token: Maximum number of calibration tokens used per
balance layer when searching the clip threshold (subsampled to
bound memory).
+ awq_seqlen: Maximum sequence length (number of tokens) used per
+ calibration sample during AWQ calibration, including activation
+ statistics, smoothing scale search, and clip-search input
+ features. Defaults to ``512``. Set a larger positive integer to
+ use longer sequences, or a value ``<= 0`` to disable truncation
+ entirely.
+ smooth_batch_size: Optional microbatch size used when replaying AWQ
+ parent modules during scale grid search. Smaller values reduce
+ peak VRAM while preserving the AWQ parent-output loss, at the
+ cost of more parent forward calls. ``None`` or ``<= 0`` replays
+ the cached calibration batch as-is.
+ skip_moe: Whether to exclude routed MoE experts from AWQ smoothing.
+ When True, balance layers belonging to routed experts (module
+ names matching ``.experts.<N>.``) are dropped from the resolved
+ mappings, so AWQ only smooths attention and dense/shared paths
+ and leaves routed experts to the downstream block quantizer. This
+ has no effect on dense models and is ignored when explicit
+ ``mappings`` are provided.
mappings: Optional explicit AWQ smooth/balance mappings. Each
item should contain ``smooth_layer`` and
``balance_layers`` entries. If None, mappings are inferred
@@ -114,6 +137,9 @@ class AWQConfig(QuantizationConfig):
"""
super().__init__(**kwargs)
+ if awq_seqlen is None:
+ awq_seqlen = 512
+
if isinstance(duo_scaling, str) and duo_scaling != "both":
raise ValueError(f"duo_scaling must be True, False, or 'both', got '{duo_scaling!r}'")
self.duo_scaling = duo_scaling
@@ -135,9 +161,14 @@ class AWQConfig(QuantizationConfig):
raise ValueError(f"`clip_max_shrink` must be in (0, 1), got {clip_max_shrink!r}")
if clip_n_sample_token is None or clip_n_sample_token < 1:
raise ValueError(f"`clip_n_sample_token` must be a positive integer, got {clip_n_sample_token!r}")
+ if smooth_batch_size is not None and smooth_batch_size < 0:
+ raise ValueError(f"`smooth_batch_size` must be a non-negative integer or None, got {smooth_batch_size!r}")
self.clip_n_grid = clip_n_grid
self.clip_max_shrink = clip_max_shrink
self.clip_n_sample_token = clip_n_sample_token
+ self.awq_seqlen = awq_seqlen
+ self.smooth_batch_size = smooth_batch_size
+ self.skip_moe = skip_moe
self.mappings = mappings
self.infer_bs_coeff = 1
self.batch_dim = None
@@ -159,6 +190,8 @@ class AWQConfig(QuantizationConfig):
f"AWQConfig(duo_scaling={self.duo_scaling!r}, n_grid={self.n_grid}, "
f"smooth_iters={self.smooth_iters}, "
f"apply_clip={self.apply_clip}, clip_as_init={self.clip_as_init}, "
+ f"awq_seqlen={self.awq_seqlen}, smooth_batch_size={self.smooth_batch_size}, "
+ f"skip_moe={self.skip_moe}, "
f"bits={self.bits}, group_size={self.group_size}, sym={self.sym}, "
f"mappings={'<explicit>' if self.mappings else 'auto'})"
)
diff --git a/auto_round/algorithms/transforms/awq/mappings.py b/auto_round/algorithms/transforms/awq/mappings.py
index d7f6fd147..e122c2bea 100644
--- a/auto_round/algorithms/transforms/awq/mappings.py
+++ b/auto_round/algorithms/transforms/awq/mappings.py
@@ -70,6 +70,10 @@ class ResolvedMapping:
activation_hook_target: str | None = None
+# Matches routed MoE expert modules, e.g. "...mlp.experts.3.gate_proj".
+_ROUTED_EXPERT_RE = re.compile(r"\.experts\.\d+\.")
+
+
# ── Mapping definitions ─────────────────────────
# Reference: vllm-project/llm-compressor src/llmcompressor/modifiers/awq/mappings.py
@@ -173,6 +177,15 @@ _afmoe_mappings = [
AWQMapping(r"up_proj$", [r"down_proj$"]),
]
+_bagel_mappings = [
+ AWQMapping(
+ r"input_layernorm$",
+ [r"\.self_attn\.q_proj$", r"\.self_attn\.k_proj$", r"\.self_attn\.v_proj$"],
+ ),
+ AWQMapping(r"post_attention_layernorm$", [r"\.mlp\.gate_proj$", r"\.mlp\.up_proj$"]),
+ AWQMapping(r"\.mlp\.up_proj$", [r"\.mlp\.down_proj$"]),
+]
+
# ── Model class name → mappings registry ──────────────────────────────────────
# Aligned with llm-compressor AWQ_MAPPING_REGISTRY (llmcompressor v0.10.0).
# Models not in this registry fall back to default_mappings.
@@ -220,6 +233,10 @@ AWQ_MAPPING_REGISTRY: dict[str, list[AWQMapping]] = {
# Other models using default mappings
"SeedOssForCausalLM": default_mappings,
"Ernie4_5_MoeForCausalLM": default_mappings,
+ # BAGEL wraps a Qwen2 language model and carries parallel *_moe_gen modules
+ # for image generation. Keep AWQ smoothing on the normal text path only.
+ "BagelForQuantization": _bagel_mappings,
+ "BagelForConditionalGeneration": _bagel_mappings,
}
@@ -331,6 +348,14 @@ def _build_hybrid_attention_mappings(model: torch.nn.Module) -> list[AWQMapping]
layer_types, num_layers = result
+ if len(layer_types) < num_layers:
+ logger.warning(
+ "Hybrid attention model config has num_hidden_layers=%d but only %d layer_types entries. Falling back.",
+ num_layers,
+ len(layer_types),
+ )
+ return None
+
full_indices = [i for i in range(num_layers) if layer_types[i] == "full_attention"]
linear_indices = [i for i in range(num_layers) if layer_types[i] == "linear_attention"]
@@ -435,6 +460,7 @@ def _get_mappings_for_model(model: torch.nn.Module) -> list[AWQMapping]:
def resolve_mappings(
model: torch.nn.Module,
user_mappings: list[dict] | None = None,
+ skip_moe: bool = False,
) -> list[ResolvedMapping]:
"""Resolve AWQ mappings for the given model.
@@ -444,15 +470,54 @@ def resolve_mappings(
2. ``AWQ_MAPPING_REGISTRY`` — model-class-name lookup
3. ``default_mappings`` — Llama-like fallback.
+ Args:
+ model: The model to resolve mappings against.
+ user_mappings: Optional explicit mappings; when provided they are used
+ verbatim and ``skip_moe`` is ignored.
+ skip_moe: When True, drop routed MoE experts (module names matching
+ ``.experts.<N>.``) from the resolved mappings so AWQ leaves each
+ routed expert to the downstream block quantizer.
+
Returns:
List of ``ResolvedMapping`` objects ready for AWQ grid search.
"""
if user_mappings is not None:
- mapping_defs = [AWQMapping(m["smooth_layer"], m["balance_layers"]) for m in user_mappings]
+ mapping_defs = [
+ AWQMapping(m["smooth_layer"], m["balance_layers"], m.get("activation_hook_target")) for m in user_mappings
+ ]
else:
mapping_defs = _get_mappings_for_model(model)
- return _resolve_mapping_defs(model, mapping_defs)
+ resolved = _resolve_mapping_defs(model, mapping_defs)
+
+ if skip_moe and user_mappings is None:
+ resolved = _drop_routed_experts(model, resolved)
+
+ return resolved
+
+
+def _drop_routed_experts(model: torch.nn.Module, resolved: list[ResolvedMapping]) -> list[ResolvedMapping]:
+ """Remove routed MoE experts from resolved mappings for ``skip_moe``."""
+ kept: list[ResolvedMapping] = []
+ dropped_experts = 0
+ for mapping in resolved:
+ if _ROUTED_EXPERT_RE.search(mapping.smooth_name):
+ dropped_experts += len(mapping.balance_names)
+ continue
+
+ keep_idx = [i for i, name in enumerate(mapping.balance_names) if not _ROUTED_EXPERT_RE.search(name)]
+ dropped_experts += len(mapping.balance_names) - len(keep_idx)
+ if not keep_idx:
+ continue
+ if len(keep_idx) < len(mapping.balance_names):
+ mapping.balance_names = [mapping.balance_names[i] for i in keep_idx]
+ mapping.balance_layers = [mapping.balance_layers[i] for i in keep_idx]
+ mapping.parent_name, mapping.parent = _find_parent(model, mapping.balance_names)
+ kept.append(mapping)
+
+ if dropped_experts:
+ logger.info(f"AWQ skip_moe: excluded {dropped_experts} routed-expert balance layer(s) from smoothing.")
+ return kept
def _resolve_mapping_defs(
@@ -554,13 +619,7 @@ def _resolve_mapping_defs(
"AWQConfig(mappings=[...])."
)
else:
- first_prefix = next(iter(block_modules))
- n_blocks = len(block_modules)
- mappings_per_block = sum(1 for r in resolved if r.smooth_name.startswith(first_prefix))
- logger.info(
- f"AWQ resolved {matched_count} smooth-balance mappings "
- f"({mappings_per_block} per block × {n_blocks} blocks)."
- )
+ logger.info(f"AWQ resolved {matched_count} smooth-balance mappings.")
return resolved
@@ -585,11 +644,11 @@ def check_model_compatibility(
"""
warnings_list = []
cls_name = _get_model_class_name(model)
- in_registry = cls_name in AWQ_MAPPING_REGISTRY
+ in_registry = cls_name in AWQ_MAPPING_REGISTRY or cls_name in AWQ_DYNAMIC_MAPPING_REGISTRY
if not in_registry and user_mappings is None:
warnings_list.append(
- f"Model class '{cls_name}' is not in AWQ_MAPPING_REGISTRY. "
+ f"Model class '{cls_name}' is not in any AWQ mapping registry. "
f"Using default Llama-like mappings. If quantization quality is "
f"poor, provide explicit mappings via AWQConfig(mappings=[...])."
)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 480ba113f..4f2bee857 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -381,6 +381,9 @@ _AWQ_FIELDS = {
"clip_n_grid",
"clip_max_shrink",
"clip_n_sample_token",
+ "awq_seqlen",
+ "smooth_batch_size",
+ "skip_moe",
"mappings",
}
_ROTATION_FIELDS = {
diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.py
index 193d70583..4a38c6e6d 100644
--- a/auto_round/cli/algorithms.py
+++ b/auto_round/cli/algorithms.py
@@ -191,7 +191,7 @@ class AWQ(AlgorithmHandler):
def register(self, group) -> None:
group.add_argument(
- "--awq-duo-scaling",
+ "--awq_duo_scaling",
dest="duo_scaling",
default=True,
type=_parse_bool_or_mode,
@@ -199,36 +199,56 @@ class AWQ(AlgorithmHandler):
help="Use activation+weight duo scaling (true/false/both).",
)
group.add_argument(
- "--awq-n-grid",
+ "--awq_n_grid",
dest="n_grid",
default=20,
type=int,
help="Number of grid-search points for AWQ scaling ratio.",
)
group.add_argument(
- "--awq-apply-clip",
+ "--awq_seqlen",
+ dest="awq_seqlen",
+ default=None,
+ type=int,
+ help=(
+ "Maximum sequence length used by AWQ calibration. "
+ "This is distinct from the global calibration --seqlen."
+ ),
+ )
+ group.add_argument(
+ "--awq_smooth_batch_size",
+ dest="awq_smooth_batch_size",
+ default=None,
+ type=int,
+ help="Microbatch size for AWQ parent replay during scale search; <=0 disables microbatching.",
+ )
+ group.add_argument(
+ "--awq_apply_clip",
dest="awq_apply_clip",
action="store_true",
help="Search and hard-clamp per-group AWQ weight clipping after smoothing.",
)
group.add_argument(
- "--awq-clip-as-init",
+ "--awq_clip_as_init",
dest="awq_clip_as_init",
action="store_true",
help=(
"Use the searched AWQ clip to initialize the block quantizer's "
- "weight range instead of hard-clamping (requires --awq-apply-clip)."
+ "weight range instead of hard-clamping (requires --awq_apply_clip)."
),
)
def build(self, args, common_kwargs: dict[str, Any]):
from auto_round.algorithms.transforms.awq.config import AWQConfig
+ awq_seqlen = getattr(args, "awq_seqlen", None)
return AWQConfig(
duo_scaling=getattr(args, "duo_scaling", True),
n_grid=getattr(args, "n_grid", 20),
apply_clip=getattr(args, "awq_apply_clip", False),
clip_as_init=getattr(args, "awq_clip_as_init", False),
+ awq_seqlen=512 if awq_seqlen is None else awq_seqlen,
+ smooth_batch_size=getattr(args, "awq_smooth_batch_size", None),
**common_kwargs,
)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 009b8adba..f8cc7a276 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -18,7 +18,7 @@ import os
import re
from collections import UserDict
from pathlib import Path
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
import psutil
import torch
@@ -1792,6 +1792,23 @@ def to_device(input, device=torch.device("cpu")):
return input
+def move_to_device(input: Any, device=torch.device("cpu"), non_blocking: bool = False) -> Any:
+ """Return a nested tensor container moved to *device* without mutating it."""
+ if input is None:
+ return None
+ if isinstance(input, torch.Tensor):
+ return input.to(device, non_blocking=non_blocking)
+ if isinstance(input, UserDict):
+ return type(input)({k: move_to_device(v, device, non_blocking) for k, v in input.items()})
+ if isinstance(input, dict):
+ return {k: move_to_device(v, device, non_blocking) for k, v in input.items()}
+ if isinstance(input, tuple):
+ return tuple(move_to_device(v, device, non_blocking) for v in input)
+ if isinstance(input, list):
+ return [move_to_device(v, device, non_blocking) for v in input]
+ return input
+
+
def mv_module_from_gpu(module):
"""Moves module from gpu to cpu.
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 6e8928855..f36b67e00 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -166,16 +166,21 @@ class WrapperLinear(torch.nn.Module):
self.weight_min = None
self.weight_max = None
# AWQ clip-as-init: cap the tunable weight range to the per-group clip
- # magnitude searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
+ # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
# This initializes the range used by quant_tensor_sym/asym, leaving
# min_scale/max_scale to tune a coefficient on top. Only the standard
# (non-tuple) group layout maps onto weight_min/weight_max here.
+ awq_clip_min = getattr(orig_layer, "awq_clip_min", None)
awq_clip_max = getattr(orig_layer, "awq_clip_max", None)
if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):
- clip_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
- if clip_flat.numel() == self.weight_max.numel():
- self.weight_max = torch.minimum(self.weight_max, clip_flat)
- self.weight_min = torch.maximum(self.weight_min, -clip_flat)
+ clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
+ if awq_clip_min is None:
+ clip_min_flat = -clip_max_flat
+ else:
+ clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)
+ if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():
+ self.weight_max = torch.minimum(self.weight_max, clip_max_flat)
+ self.weight_min = torch.maximum(self.weight_min, clip_min_flat)
self._init_params(
"value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16
)
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 09faf2043..b63882653 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -344,11 +344,11 @@ W2G64 Average Accuracy of 13 tasks and Time Cost Results(Testing was conducted o
### AWQ Algorithm
-**Experimental feature: our current implementation does not apply weight clipping yet, so accuracy may drop compared to the original AWQ algorithm.**
+**Experimental feature:** AWQ weight clipping is optional. Enable it with `--awq_apply_clip` when you want to match the original AWQ flow more closely.
AWQ (Activation-Aware Weight Quantization) is available as an alternative quantization algorithm. AWQ protects salient weight channels by analyzing activation patterns and applying channel-wise scaling before standard RTN quantization.
-The canonical AWQ deployment path is **W4A16** served by vLLM's AWQ/Marlin CUDA kernels. **W8A8** with AWQ smoothing can also be served via vLLM's compressed_tensors backend (cutlass INT8 GEMM).
+The canonical AWQ deployment path is **W4A16** served by vLLM's AWQ/Marlin CUDA kernels. **INT8** is AutoRound's W8A8 scheme and can use AWQ smoothing before RTN quantization for vLLM's compressed_tensors backend (cutlass INT8 GEMM).
#### CLI Usage
@@ -356,25 +356,69 @@ The canonical AWQ deployment path is **W4A16** served by vLLM's AWQ/Marlin CUDA
auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"
```
+AWQ can also be paired with AutoRound optimization:
+
+```bash
+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq,auto_round
+```
+
+Recommended INT8/W8A8 recipe using AWQ smoothing:
+
+```bash
+auto-round \
+ --model Qwen/Qwen3-0.6B \
+ --scheme INT8 \
+ --algorithm awq \
+ --nsamples 256 \
+ --awq_seqlen 512 \
+ --awq_apply_clip \
+ --format auto_round:llm_compressor
+```
+
+The explicit `--nsamples 256` and `--awq_seqlen 512` settings are recommended for W8A8 AWQ calibration. The default
+AutoRound values are tuned for AutoRound optimization, not plain AWQ smoothing.
+
AWQ-specific options:
-- `--duo_scaling`: Use both activations and weights for scaling. Options: `true`, `false`, or `both` (searches both modes and picks the best). (default: True).
-- `--n_grid`: Number of grid points for scaling ratio search (default: 20).
+- `--awq_duo_scaling`: Use both activations and weights for scaling. Options: `true`, `false`, or `both` (searches both modes and picks the best). (default: True).
+- `--awq_n_grid`: Number of grid points for scaling ratio search (default: 20).
+- `--awq_apply_clip`: Search and apply AWQ weight clipping after smoothing.
+- `--awq_seqlen`: Maximum sequence length used by AWQ calibration, including activation statistics, smoothing
+ scale search, and clip-search input features. This is separate from the global `--seqlen`, which controls
+ calibration sample construction. Set a value `<= 0` to use the full calibration sequence.
+
+API-only AWQ options:
+- `AWQConfig(skip_moe=True)`: Skips routed MoE experts during AWQ smoothing while keeping attention and dense/shared paths. Explicit `mappings` are used as provided.
#### API Usage
+For default AWQ settings, the string alias is sufficient:
+
+```python
+from auto_round import AutoRound
+
+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
+```
+
+Use `AWQConfig` when you need AWQ-specific options such as `apply_clip=True`:
+
```python
from auto_round import AWQConfig, AutoRound
ar = AutoRound(
"Qwen/Qwen3-0.6B",
- scheme="INT8",
- alg_configs=AWQConfig(),
+ scheme="W4A16",
+ alg_configs=AWQConfig(apply_clip=True),
)
output_dir = "./tmp_awq"
ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
```
+`alg_configs="awq"` or `alg_configs=AWQConfig()` selects the AWQ algorithm. This is independent from export
+format selection such as `format="auto_awq"`. For example:
+- `alg_configs="awq"` + `format="auto_round"`: AWQ smoothing with AutoRound packing.
+- `alg_configs="signround"` + `format="auto_awq"`: AutoAWQ packing without AWQ smoothing.
+
### AutoScheme
AutoScheme automatically generates adaptive mixed-bit and mixed-data-type quantization recipes. For accuracy results, see [AutoScheme Accuracy Report](./auto_scheme_acc.md).
@@ -487,42 +531,6 @@ Embedding layer is not supported in AutoScheme, it will use the best scheme in o
When using AutoScheme with `model_free=True`, only INT (`W2A16`/`W4A16`/`W8A16`) and MXFP (`MXFP4`/`MXFP8`) option families are supported. Options like `W3A16`, `GGUF:*`, and `NVFP4` will raise a `ValueError`. INT and MXFP families cannot be mixed in the same `AutoScheme`.
-### AWQ Quantization Algorithm
-
-AWQ (`alg_configs="awq"` or `alg_configs=AWQConfig()`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).
-
-#### CLI Usage
-```bash
-# AWQ + default RTN (iters=0 auto-selected)
-auto-round --model Qwen/Qwen3-0.6B --algorithm awq --scheme W4A16
-
-# AWQ + AutoRound optimization
-auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16
-
-# AWQ flags
---duo-scaling true|false|both (default: true)
---n-grid 20 (default: 20)
-```
-
-#### API Usage
-```python
-from auto_round import AWQConfig, AutoRound, SignRoundConfig
-
-# String alias (AWQ defaults, with RTN appended automatically)
-ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
-
-# AWQ + default RTN (simplest)
-ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")
-
-# AWQ + AutoRound via alg_configs (explicit pipeline)
-ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")
-ar.quantize_and_save(output_dir="./qmodel")
-```
-
-**Important Note**: `alg_configs="awq"` or `alg_configs=AWQConfig()` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:
-- `alg_configs="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing
-- `alg_configs="signround"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing
-
### OPT RTN Mode
AutoRound also supports Optimized RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. Setting `iters=0` tp enable it and we recommend using `group_size=32` for better results. Check [accuracy comparison](./opt_rtn.md) between RTN and OPT RTN mode
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index df6f2e890..b6341bfd0 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -339,11 +339,11 @@ W2G64 在 13 个任务上的平均精度与耗时
### AWQ 算法
-实验性功能:原始实现中未使用 weight clipping(权重裁剪)逻辑,因此相比原版 AWQ 算法,可能会存在一定精度下降
+实验性功能:AWQ weight clipping(权重裁剪)为可选功能。如需更接近原始 AWQ 流程,可使用 `--awq_apply_clip` 开启。
AWQ(Activation-Aware Weight Quantization,激活感知权重量化)是一种可选的量化算法。AWQ 通过分析激活模式来保护关键权重通道,在标准量化前对权重施加通道级缩放,从而降低量化误差。
-AWQ 的标准部署路径是 **W4A16**,通过 vLLM 的 AWQ/Marlin CUDA 内核提供服务。**W8A8** 搭配 AWQ 平滑化也可通过 vLLM 的 compressed_tensors 后端(cutlass INT8 GEMM)提供服务。
+AWQ 的标准部署路径是 **W4A16**,通过 vLLM 的 AWQ/Marlin CUDA 内核提供服务。**INT8** 是 AutoRound 的 W8A8 scheme,可在 RTN 量化前使用 AWQ 平滑化,并通过 vLLM 的 compressed_tensors 后端(cutlass INT8 GEMM)提供服务。
#### 命令行用法
@@ -351,27 +351,67 @@ AWQ 的标准部署路径是 **W4A16**,通过 vLLM 的 AWQ/Marlin CUDA 内核
auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"
```
+AWQ 也可以与 AutoRound 优化组合使用:
+
+```bash
+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq,auto_round
+```
+
+推荐的 INT8/W8A8 配方,使用 AWQ 平滑化:
+
+```bash
+auto-round \
+ --model Qwen/Qwen3-0.6B \
+ --scheme INT8 \
+ --algorithm awq \
+ --nsamples 256 \
+ --awq_seqlen 512 \
+ --awq_apply_clip \
+ --format auto_round:llm_compressor
+```
+
+这里显式设置 `--nsamples 256` 和 `--awq_seqlen 512` 是 W8A8 AWQ 标定的推荐配方。AutoRound 默认值主要面向
+AutoRound 优化调优,并不适合作为 plain AWQ smoothing 的默认选择。
+
AWQ 专用选项:
-- `--duo_scaling`:同时使用激活和权重计算缩放因子。选项:`true`、`false` 或 `both`(搜索两种模式并选择最佳)。(默认:True)。
-- `--n_grid`:缩放比率搜索的网格点数(默认:20)。
+- `--awq_duo_scaling`:同时使用激活和权重计算缩放因子。选项:`true`、`false` 或 `both`(搜索两种模式并选择最佳)。(默认:True)。
+- `--awq_n_grid`:缩放比率搜索的网格点数(默认:20)。
+- `--awq_apply_clip`:在 AWQ 平滑后搜索并应用权重裁剪。
+- `--awq_seqlen`:AWQ 标定使用的最大序列长度,包括激活统计、smoothing scale search 以及 clip-search
+ 输入特征。它不同于全局 `--seqlen`,后者用于控制标定样本构造。设为 `<= 0` 时使用完整标定序列。
+
+仅 API 支持的 AWQ 选项:
+- `AWQConfig(skip_moe=True)`:AWQ 平滑时跳过 routed MoE experts,仅保留 attention 和 dense/shared 路径。显式传入的 `mappings` 会按原样使用。
#### API 用法
-W8A8 搭配 AWQ 平滑化:
+使用默认 AWQ 参数时,字符串别名就足够:
+
+```python
+from auto_round import AutoRound
+
+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
+```
+
+需要 `apply_clip=True` 等 AWQ 专用选项时,再使用 `AWQConfig`:
```python
from auto_round import AWQConfig, AutoRound
ar = AutoRound(
"Qwen/Qwen3-0.6B",
- scheme="INT8",
- alg_configs=AWQConfig(),
+ scheme="W4A16",
+ alg_configs=AWQConfig(apply_clip=True),
)
output_dir = "./tmp_awq"
ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
```
+`alg_configs="awq"` 或 `alg_configs=AWQConfig()` 选择的是 AWQ 算法;这与 `format="auto_awq"` 等导出格式相互独立。例如:
+- `alg_configs="awq"` + `format="auto_round"`:使用 AWQ 平滑,并采用 AutoRound 打包。
+- `alg_configs="signround"` + `format="auto_awq"`:不使用 AWQ 平滑,但采用 AutoAWQ 打包。
+
### AutoScheme 自动混合精度量化方案
@@ -485,42 +525,6 @@ AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量
当 AutoScheme 与 `model_free=True` 联合使用时,仅支持 INT(`W2A16`/`W4A16`/`W8A16`)和 MXFP(`MXFP4`/`MXFP8`)两种选项族。`W3A16`、`GGUF:*`、`NVFP4` 等不支持的选项会直接抛出 `ValueError`;同一 `AutoScheme` 中也不允许混用 INT 和 MXFP 选项族。
-### AWQ 量化算法
-
-AWQ(`alg_configs="awq"` 或 `alg_configs=AWQConfig()`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。
-
-#### 命令行用法
-```bash
-# AWQ + 默认 RTN (自动选择 iters=0)
-auto-round --model Qwen/Qwen3-0.6B --algorithm awq --scheme W4A16
-
-# AWQ + AutoRound 优化
-auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16
-
-# AWQ 相关参数
---duo-scaling true|false|both (默认: true)
---n-grid 20 (默认: 20)
-```
-
-#### API 用法
-```python
-from auto_round import AWQConfig, AutoRound, SignRoundConfig
-
-# 字符串别名(使用 AWQ 默认参数,并自动追加 RTN)
-ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
-
-# AWQ + 默认 RTN (最简用法)
-ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")
-
-# 通过 alg_configs 指定 AWQ + AutoRound (显式流水线)
-ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")
-ar.quantize_and_save(output_dir="./qmodel")
-```
-
-**重要提示**:`alg_configs="awq"` 或 `alg_configs=AWQConfig()`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:
-- `alg_configs="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包
-- `alg_configs="signround"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包
-
### OPT-RTN 模式
AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。
diff --git a/test/unit/test_cpu/algorithms/test_awq.py b/test/unit/test_cpu/algorithms/test_awq.py
index 854451fbd..a9f933c07 100644
--- a/test/unit/test_cpu/algorithms/test_awq.py
+++ b/test/unit/test_cpu/algorithms/test_awq.py
@@ -44,6 +44,12 @@ class TestAWQNormalLLM:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
+ def test_awq_config_uses_awq_seqlen(self):
+ """awq_seqlen should configure AWQ calibration truncation."""
+ cfg = AWQConfig(awq_seqlen=128)
+
+ assert cfg.awq_seqlen == 128
+
@pytest.mark.timeout(60)
def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):
"""W4A16 AWQ quantization produces valid layer_config and model can generate."""
@@ -244,6 +250,452 @@ class TestAWQMoE:
del model
+ def test_awq_moe_skip_moe(self):
+ """skip_moe should drop routed-expert balance layers/mappings but keep dense paths."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping, _drop_routed_experts
+
+ model = nn.Module()
+ ln = nn.LayerNorm(8)
+ q = nn.Linear(8, 8, bias=False)
+ shared_gate = nn.Linear(8, 16, bias=False)
+ e0_gate = nn.Linear(8, 16, bias=False)
+ e1_gate = nn.Linear(8, 16, bias=False)
+ e0_up = nn.Linear(8, 16, bias=False)
+ e0_down = nn.Linear(16, 8, bias=False)
+
+ resolved = [
+ ResolvedMapping(
+ smooth_name="model.layers.0.input_layernorm",
+ smooth_layer=ln,
+ balance_names=["model.layers.0.self_attn.q_proj"],
+ balance_layers=[q],
+ parent_name="model.layers.0.self_attn",
+ parent=nn.Module(),
+ ),
+ ResolvedMapping(
+ smooth_name="model.layers.0.post_attention_layernorm",
+ smooth_layer=ln,
+ balance_names=[
+ "model.layers.0.mlp.shared_expert.gate_proj",
+ "model.layers.0.mlp.experts.0.gate_proj",
+ "model.layers.0.mlp.experts.1.gate_proj",
+ ],
+ balance_layers=[shared_gate, e0_gate, e1_gate],
+ parent_name="model.layers.0.mlp",
+ parent=nn.Module(),
+ ),
+ ResolvedMapping(
+ smooth_name="model.layers.0.mlp.experts.0.up_proj",
+ smooth_layer=e0_up,
+ balance_names=["model.layers.0.mlp.experts.0.down_proj"],
+ balance_layers=[e0_down],
+ parent_name="model.layers.0.mlp.experts.0",
+ parent=nn.Module(),
+ ),
+ ]
+
+ kept = _drop_routed_experts(model, resolved)
+ smooth_names = [m.smooth_name for m in kept]
+
+ assert "model.layers.0.mlp.experts.0.up_proj" not in smooth_names
+ assert "model.layers.0.input_layernorm" in smooth_names
+ mixed = next(m for m in kept if m.smooth_name.endswith("post_attention_layernorm"))
+ assert mixed.balance_names == ["model.layers.0.mlp.shared_expert.gate_proj"]
+
+ del model
+
+ def test_explicit_awq_mapping_preserves_activation_hook_target(self):
+ """Custom AWQ mappings should keep activation_hook_target for non-standard balance inputs."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.mappings import resolve_mappings
+
+ class TinyBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.smooth = nn.Linear(4, 4, bias=False)
+ self.hook = nn.Linear(4, 4, bias=False)
+ self.balance = nn.Linear(4, 4, bias=False)
+
+ class TinyModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = nn.ModuleList([TinyBlock()])
+
+ resolved = resolve_mappings(
+ TinyModel(),
+ user_mappings=[
+ {
+ "smooth_layer": "smooth$",
+ "balance_layers": ["balance$"],
+ "activation_hook_target": "hook",
+ }
+ ],
+ )
+
+ assert len(resolved) == 1
+ assert resolved[0].activation_hook_target == "hook"
+
+ def test_hybrid_attention_mapping_short_layer_types_falls_back(self):
+ """Malformed hybrid configs should fall back instead of raising IndexError."""
+ from types import SimpleNamespace
+
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.mappings import _build_hybrid_attention_mappings
+
+ class BadHybridModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(
+ layer_types=["full_attention", "linear_attention"],
+ num_hidden_layers=3,
+ )
+
+ assert _build_hybrid_attention_mappings(BadHybridModel()) is None
+
+ def test_awq_ignored_layer_skips_mapping(self):
+ """A mapping containing an ignore_layers / bits>=16 layer is skipped as one smooth-scale group."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping
+
+ transform = AWQTransform(AWQConfig(bits=4, group_size=128, sym=True, data_type="int"))
+
+ ln = nn.LayerNorm(8)
+ q = nn.Linear(8, 8, bias=False)
+ q.global_name = "model.layers.0.self_attn.q_proj"
+ k = nn.Linear(8, 8, bias=False)
+ k.global_name = "model.layers.0.self_attn.k_proj"
+
+ mapping = ResolvedMapping(
+ smooth_name="model.layers.0.input_layernorm",
+ smooth_layer=ln,
+ balance_names=[q.global_name, k.global_name],
+ balance_layers=[q, k],
+ parent_name="model.layers.0.self_attn",
+ parent=nn.Module(),
+ )
+
+ transform._qdq_tool.layer_config = {}
+ assert transform._mapping_has_ignored_layer(mapping) is False
+
+ transform._qdq_tool.layer_config = {q.global_name: {"bits": 4}, k.global_name: {"bits": 4}}
+ assert transform._mapping_has_ignored_layer(mapping) is False
+
+ transform._qdq_tool.layer_config = {q.global_name: {"bits": 4}, k.global_name: {"bits": 16}}
+ assert transform._mapping_has_ignored_layer(mapping) is True
+ assert transform._mapping_is_smoothable(mapping) is False
+
+ def test_awq_mixed_balance_quant_params_skip_mapping(self, monkeypatch):
+ """Balance layers sharing one AWQ scale must share the same resolved quantization params."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq import base as awq_base
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping
+
+ transform = AWQTransform(AWQConfig(bits=4, group_size=128, sym=True, data_type="int"))
+
+ ln = nn.LayerNorm(8)
+ q = nn.Linear(8, 8, bias=False)
+ q.global_name = "model.layers.0.self_attn.q_proj"
+ k = nn.Linear(8, 8, bias=False)
+ k.global_name = "model.layers.0.self_attn.k_proj"
+
+ mapping = ResolvedMapping(
+ smooth_name="model.layers.0.input_layernorm",
+ smooth_layer=ln,
+ balance_names=[q.global_name, k.global_name],
+ balance_layers=[q, k],
+ parent_name="model.layers.0.self_attn",
+ parent=nn.Module(),
+ )
+
+ warnings = []
+
+ def fake_warning(message, *args, **kwargs):
+ warnings.append(message % args if args else message)
+
+ monkeypatch.setattr(awq_base.logger, "warning", fake_warning)
+
+ transform._qdq_tool.layer_config = {
+ q.global_name: {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},
+ k.global_name: {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},
+ }
+ assert transform._mapping_has_mixed_quant_params(mapping) is False
+ assert transform._mapping_is_smoothable(mapping) is True
+
+ transform._qdq_tool.layer_config = {
+ q.global_name: {
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "disable_opt_rtn": False,
+ },
+ k.global_name: {
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "disable_opt_rtn": True,
+ },
+ }
+ assert transform._mapping_has_mixed_quant_params(mapping) is False
+ assert transform._mapping_is_smoothable(mapping) is True
+
+ transform._qdq_tool.layer_config = {
+ q.global_name: {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},
+ k.global_name: {"bits": 4, "group_size": 128, "sym": True, "data_type": "mx_fp"},
+ }
+ assert transform._mapping_has_mixed_quant_params(mapping) is True
+ assert transform._mapping_is_smoothable(mapping) is False
+ assert any("different quantization parameters" in warning for warning in warnings)
+
+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
+ """Direct grid search should pass each balance layer's own resolved quant function."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping
+
+ transform = AWQTransform(AWQConfig(bits=4, group_size=8, sym=True, data_type="int", duo_scaling=False))
+
+ ln = nn.LayerNorm(8)
+ q = nn.Linear(8, 8, bias=False)
+ q.global_name = "model.layers.0.self_attn.q_proj"
+ k = nn.Linear(8, 8, bias=False)
+ k.global_name = "model.layers.0.self_attn.k_proj"
+ mapping = ResolvedMapping(
+ smooth_name="model.layers.0.input_layernorm",
+ smooth_layer=ln,
+ balance_names=[q.global_name, k.global_name],
+ balance_layers=[q, k],
+ parent_name="model.layers.0.self_attn",
+ parent=nn.Module(),
+ )
+
+ transform._qdq_tool.layer_config = {
+ q.global_name: {"bits": 4, "group_size": 8, "sym": True, "data_type": "int"},
+ k.global_name: {"bits": 8, "group_size": 8, "sym": True, "data_type": "mx_fp"},
+ }
+ records = []
+
+ def fake_resolve_quant_funcs(params):
+ return f"{params['data_type']}_{params['bits']}", None
+
+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quant_func))
+ return weight
+
+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
+ transform._qdq_tool.qdq = fake_qdq
+
+ transform._grid_search_scales(mapping, torch.ones(8))
+
+ assert ("int", 4, "int_4") in records
+ assert ("mx_fp", 8, "mx_fp_8") in records
+
+ def test_awq_activation_stats_use_balance_layer_input_for_gated_mlp(self):
+ """up_proj -> down_proj stats must use down_proj input, not raw up_proj output."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping
+
+ class TinyGatedBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.up_proj = nn.Linear(4, 6, bias=False)
+ self.gate_proj = nn.Linear(4, 6, bias=False)
+ self.down_proj = nn.Linear(6, 4, bias=False)
+
+ def forward(self, x):
+ return self.down_proj(torch.nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))
+
+ class TinyModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.block = TinyGatedBlock()
+
+ def forward(self, x):
+ return self.block(x)
+
+ torch.manual_seed(0)
+ model = TinyModel()
+ transform = AWQTransform(
+ AWQConfig(bits=4, group_size=2, sym=True, data_type="int", apply_clip=True, clip_n_sample_token=32)
+ )
+ mapping = ResolvedMapping(
+ smooth_name="block.up_proj",
+ smooth_layer=model.block.up_proj,
+ balance_names=["block.down_proj"],
+ balance_layers=[model.block.down_proj],
+ parent_name="block",
+ parent=model.block,
+ )
+ transform._block_mappings = {"block": [mapping]}
+
+ x = torch.randn(2, 3, 4)
+ handles = transform._register_awq_hooks(model, model.block, "block")
+ try:
+ model(x)
+ finally:
+ for handle in handles:
+ handle.remove()
+
+ expected = torch.nn.functional.silu(model.block.gate_proj(x)) * model.block.up_proj(x)
+ expected_feat = expected.detach().flatten(0, -2)
+ raw_up_feat = model.block.up_proj(x).detach().flatten(0, -2)
+
+ act_sum, act_count = transform._activation_stats["block.up_proj"]
+ assert torch.allclose(act_sum, expected_feat.abs().sum(dim=0))
+ assert act_count == expected_feat.shape[0]
+ assert torch.allclose(transform._clip_input_feat["block.up_proj"], expected_feat.float().cpu())
+ assert not torch.allclose(act_sum, raw_up_feat.abs().sum(dim=0))
+
+ def test_awq_parent_cache_recursively_detaches_tensor_containers(self):
+ """Parent replay cache should not retain tensors inside list/dict containers."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+ from auto_round.algorithms.transforms.awq.mappings import ResolvedMapping
+
+ class ContainerParent(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.smooth = nn.Linear(4, 4, bias=False)
+ self.balance = nn.Linear(4, 4, bias=False)
+
+ def forward(self, items=None, payload=None):
+ return self.balance(items[0]) + self.balance(payload["x"])
+
+ class TinyModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.block = ContainerParent()
+
+ def forward(self, items=None, payload=None):
+ return self.block(items=items, payload=payload)
+
+ model = TinyModel()
+ transform = AWQTransform(AWQConfig(bits=4, group_size=4, sym=True, data_type="int"))
+ mapping = ResolvedMapping(
+ smooth_name="block.smooth",
+ smooth_layer=model.block.smooth,
+ balance_names=["block.balance"],
+ balance_layers=[model.block.balance],
+ parent_name="block",
+ parent=model.block,
+ )
+ transform._block_mappings = {"block": [mapping]}
+
+ items = [torch.randn(1, 4, requires_grad=True)]
+ payload = {"x": torch.randn(1, 4, requires_grad=True)}
+ handles = transform._register_awq_hooks(model, model.block, "block")
+ try:
+ model(items=items, payload=payload)
+ finally:
+ for handle in handles:
+ handle.remove()
+
+ _, cached_kwargs = transform._parent_args_cache[model.block][0]
+ assert cached_kwargs["items"] is not items
+ assert cached_kwargs["payload"] is not payload
+ assert cached_kwargs["items"][0].device.type == "cpu"
+ assert cached_kwargs["payload"]["x"].device.type == "cpu"
+ assert cached_kwargs["items"][0].requires_grad is False
+ assert cached_kwargs["payload"]["x"].requires_grad is False
+
+ def test_awq_parent_replay_microbatch_matches_full_batch(self):
+ """AWQ parent replay microbatching should preserve exact parent-output loss."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+
+ class RecordingParent(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = nn.Linear(4, 4, bias=False)
+ self.batch_sizes = []
+
+ def forward(self, hidden_states, *, position_ids=None, cache_position=None, payload=None):
+ self.batch_sizes.append(hidden_states.shape[0])
+ assert position_ids is not None and position_ids.shape[0] == hidden_states.shape[0]
+ assert cache_position is not None and cache_position.shape == (3,)
+ add = payload["add"] if payload is not None else 0
+ return self.proj(hidden_states + add)
+
+ torch.manual_seed(0)
+ parent = RecordingParent()
+ hidden_states = torch.randn(5, 3, 4)
+ position_ids = torch.arange(15).reshape(5, 3)
+ cache_position = torch.arange(3)
+ payload = {"add": torch.randn(5, 3, 4)}
+ kwargs_list = [
+ ((hidden_states,), {"position_ids": position_ids, "cache_position": cache_position, "payload": payload})
+ ]
+
+ full = AWQTransform(AWQConfig(bits=4, group_size=4, sym=True, data_type="int"))
+ micro = AWQTransform(AWQConfig(bits=4, group_size=4, sym=True, data_type="int", smooth_batch_size=2))
+
+ full_outputs = full._run_parent_samples(parent, kwargs_list)
+ parent.batch_sizes.clear()
+ micro_outputs = micro._run_parent_samples(parent, kwargs_list)
+
+ assert parent.batch_sizes == [2, 2, 1]
+ assert len(full_outputs) == 1
+ assert len(micro_outputs) == 3
+ assert torch.allclose(torch.cat(micro_outputs, dim=0), full_outputs[0])
+
+ ref_outputs = [out + 0.125 for out in micro_outputs]
+ parent.batch_sizes.clear()
+ streamed_loss = micro._compute_parent_loss(parent, kwargs_list, ref_outputs)
+ expected_loss = micro._compute_loss(ref_outputs, micro_outputs)
+
+ assert parent.batch_sizes == [2, 2, 1]
+ assert streamed_loss == expected_loss
+
+ def test_awq_seqlen_truncates_awq_inputs(self):
+ """awq_seqlen should truncate matching sequence dimensions consistently."""
+ from auto_round.algorithms.transforms.awq.base import _truncate_args_kwargs, _truncate_awq_tensor
+
+ hidden_states = torch.zeros(1, 16, 8)
+ position_ids = torch.arange(16).reshape(1, 16)
+ cache_position = torch.arange(16)
+ attention_mask = torch.zeros(1, 1, 16, 16)
+ rotary = (torch.zeros(1, 16, 8), torch.ones(1, 16, 8))
+ feat = _truncate_awq_tensor(hidden_states, seqlen=4)
+
+ args, kwargs = _truncate_args_kwargs(
+ (hidden_states,),
+ {
+ "position_ids": position_ids,
+ "cache_position": cache_position,
+ "attention_mask": attention_mask,
+ "position_embeddings": rotary,
+ },
+ seqlen=4,
+ )
+
+ assert feat.shape == (1, 4, 8)
+ assert args[0].shape == (1, 4, 8)
+ assert kwargs["position_ids"].shape == (1, 4)
+ assert kwargs["cache_position"].shape == (4,)
+ assert kwargs["attention_mask"].shape == (1, 1, 4, 4)
+ assert kwargs["position_embeddings"][0].shape == (1, 4, 8)
+ assert kwargs["position_embeddings"][1].shape == (1, 4, 8)
+
@pytest.mark.timeout(420)
def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):
"""AWQ on MoE: expert layers should be quantized, gates/routers stay FP."""
@@ -323,6 +775,45 @@ class TestAWQWeightClip:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
+ def test_awq_asym_clip_uses_separate_min_max_bounds(self):
+ """Asymmetric AWQ clip should not use a symmetric absmax range."""
+ import torch.nn as nn
+
+ from auto_round.algorithms.transforms.awq.base import AWQTransform
+ from auto_round.algorithms.transforms.awq.config import AWQConfig
+
+ layer = nn.Linear(4, 2, bias=False)
+ layer.weight.data = torch.tensor([[-8.0, -2.0, 1.0, 2.0], [-1.0, 3.0, 4.0, 5.0]])
+ transform = AWQTransform(
+ AWQConfig(
+ bits=4,
+ group_size=4,
+ sym=False,
+ data_type="int",
+ apply_clip=True,
+ clip_n_grid=2,
+ clip_n_sample_token=32,
+ )
+ )
+
+ input_feat = torch.randn(32, 4)
+ clip_range = transform._compute_best_clip(layer, input_feat)
+ assert clip_range is not None
+ min_val, max_val = clip_range
+
+ grouped = layer.weight.data.reshape(*max_val.shape[:2], -1)
+ org_min = grouped.amin(dim=-1, keepdim=True).clamp(max=0)
+ org_max = grouped.amax(dim=-1, keepdim=True).clamp(min=0)
+
+ assert torch.all(min_val >= org_min)
+ assert torch.all(max_val <= org_max)
+ assert max_val[0, 0, 0] <= 2.0
+
+ transform._apply_clip(layer, min_val, max_val)
+ clipped = layer.weight.data.reshape(*max_val.shape[:2], -1)
+ assert torch.all(clipped >= min_val)
+ assert torch.all(clipped <= max_val)
+
def test_awq_clip_then_rtn(self, tiny_opt_model_path):
"""AWQ smooth+clip → RTN: produces a valid W4 model that can generate."""
from auto_round.algorithms.quantization.rtn.config import RTNConfig