Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Awq algorithm (#1749)

Weiweie8ccff294d74
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 7372916ad..656d7e2cf 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -74,9 +74,19 @@ class BasicArgumentParser(argparse.ArgumentParser):
# choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],
help="Quantization scheme to use. "
"W4A16: 4-bit weights with 16-bit activations (default). "
- "Other options include W2A16, W3A16, W8A16 for different bit widths, "
+ "Other options include W2A16, W3A16, W8A16, W8A8 for different bit widths, "
"and MXFP4/MXFP8/NVFP4 for different data type.",
)
+ basic.add_argument(
+ "--algorithm",
+ default=None,
+ type=str.lower,
+ choices=["auto_round", "rtn", "awq"],
+ help="Quantization algorithm to use. "
+ "auto_round: SignSGD-based optimization (default when iters > 0). "
+ "rtn: Round-to-nearest (default when iters == 0). "
+ "awq: Activation-Aware Weight Quantization (AWQ smoothing + RTN).",
+ )
basic.add_argument(
"--batch_size",
"--train_bs",
@@ -313,6 +323,22 @@ class BasicArgumentParser(argparse.ArgumentParser):
"Use this to force the regular AutoRound flow.",
)
+ awq_group = self.add_argument_group("AWQ Arguments")
+ awq_group.add_argument(
+ "--duo_scaling",
+ default=True,
+ type=lambda s: "both" if s.lower() == "both" else s.lower() in ("true", "1", "yes"),
+ help="Whether to use both activations and weights for AWQ scaling. "
+ "Options: true/false/'both'. 'both' searches both modes and picks the best. "
+ "(default: True).",
+ )
+ awq_group.add_argument(
+ "--n_grid",
+ default=20,
+ type=int,
+ help="Number of grid points for AWQ scaling ratio search (default: 20).",
+ )
+
scheme = self.add_argument_group("Scheme Arguments")
scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")
scheme.add_argument(
@@ -767,6 +793,12 @@ def tune(args):
rotation_config=rot_config,
model_free=args.model_free,
disable_model_free=args.disable_model_free,
+ algorithm=getattr(args, "algorithm", None),
+ **(
+ {"duo_scaling": args.duo_scaling, "n_grid": args.n_grid}
+ if getattr(args, "algorithm", None) == "awq"
+ else {}
+ ),
)
# ======================= Quantize and save model =======================
diff --git a/auto_round/algorithms/quantization/__init__.py b/auto_round/algorithms/quantization/__init__.py
index 3d8ca932b..5b297df1d 100644
--- a/auto_round/algorithms/quantization/__init__.py
+++ b/auto_round/algorithms/quantization/__init__.py
@@ -18,5 +18,7 @@ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
from auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizer
from auto_round.algorithms.quantization.adam_round.adam import AdamRoundQuantizer
+from auto_round.algorithms.quantization.awq.config import AWQConfig
+from auto_round.algorithms.quantization.awq.quantizer import AWQQuantizer
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer, OptimizedRTNQuantizer
diff --git a/auto_round/algorithms/quantization/awq/__init__.py b/auto_round/algorithms/quantization/awq/__init__.py
new file mode 100644
index 000000000..14a492441
--- /dev/null
+++ b/auto_round/algorithms/quantization/awq/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/auto_round/algorithms/quantization/awq/config.py b/auto_round/algorithms/quantization/awq/config.py
new file mode 100644
index 000000000..50340eeda
--- /dev/null
+++ b/auto_round/algorithms/quantization/awq/config.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.algorithms.quantization.config import QuantizationConfig
+
+
+class AWQConfig(QuantizationConfig):
+ """Configuration for AWQ (Activation-Aware Weight Quantization).
+
+ AWQ protects salient weight channels by analyzing activation patterns and
+ applying channel-wise scaling to reduce quantization error. The scaling
+ factors are computed offline via a grid search over calibration data.
+ After smoothing, standard RTN quantization is applied to the adjusted weights.
+
+ Args:
+ duo_scaling: Use both activations and weights for the scaling factor.
+ True: always use duo scaling. False: only activation scaling.
+ "both": search both modes and pick the best.
+ n_grid: Number of grid points for the scaling-ratio search.
+ seqlen: Calibration sequence length.
+ nsamples: Number of calibration samples. Grid search time scales
+ linearly with nsamples (each batch triggers one parent forward
+ per grid point).
+ batch_size: Batch size for calibration forward passes.
+ apply_smooth: Whether to apply AWQ smoothing (channel scaling).
+ True: run both smoothing and quantization (default AWQ behavior).
+ False: skip smoothing, only run RTN quantization.
+ mappings: Explicit AWQ mappings. Each mapping is a dict with keys
+ ``smooth_layer`` (str) and ``balance_layers`` (list[str]).
+ If None, mappings are inferred automatically from the model structure.
+ **kwargs: Forwarded to ``QuantizationConfig`` (bits, group_size, sym, …).
+ """
+
+ _alg_cls = "AWQQuantizer"
+
+ def __init__(
+ self,
+ *,
+ duo_scaling: bool | str = True,
+ n_grid: int = 20,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ apply_smooth: bool = True,
+ mappings: list[dict] = None,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ if isinstance(duo_scaling, str) and duo_scaling != "both":
+ raise ValueError(f"duo_scaling must be True, False, or 'both', got '{duo_scaling}'")
+ self.duo_scaling = duo_scaling
+ self.n_grid = n_grid
+ self.seqlen = seqlen
+ self.nsamples = nsamples
+ self.batch_size = batch_size
+ self.apply_smooth = apply_smooth
+ self.mappings = mappings
+ self.infer_bs_coeff = 1
+ self.batch_dim = None
+
+ # TODO adjust those args defaults after architecture refactoring:
+ self.enable_quanted_input = False # AWQ doesn't cascade quantized block outputs
+ # AWQ uses plain RTN (no iterative optimization) for the quantization step.
+ self.disable_opt_rtn = True
+ self.orig_disable_opt_rtn = True
diff --git a/auto_round/algorithms/quantization/awq/mappings.py b/auto_round/algorithms/quantization/awq/mappings.py
new file mode 100644
index 000000000..32617632f
--- /dev/null
+++ b/auto_round/algorithms/quantization/awq/mappings.py
@@ -0,0 +1,624 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file contains AWQ mapping definitions and registry structure adapted from
+# llm-compressor (https://github.com/vllm-project/llm-compressor), originally
+# developed by Neural Magic, Inc. and licensed under the Apache License 2.0.
+# The mapping patterns, model-class registry, and AWQMapping/ResolvedMapping
+# data structures are aligned with llm-compressor's AWQ modifier so that
+# auto-round AWQ produces models compatible with vllm's AWQ inference kernels.
+# Reference: llmcompressor/modifiers/awq/mappings.py
+
+"""AWQ layer mapping resolution
+
+Defines the relationship between *smooth layers* (whose output channels are
+inversely scaled) and *balance layers* (whose input channels are scaled up) for
+the AWQ smoothing algorithm.
+
+Mappings are resolved in order of priority:
+ 1. User-provided explicit mappings (``AWQConfig.mappings``).
+ 2. Model-class-name registry (``AWQ_MAPPING_REGISTRY``), matching
+ llm-compressor's(vLLM) AWQ architecture support.
+ 3. ``default_mappings`` fallback for unknown Llama-like architectures.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import torch
+
+from auto_round.logger import logger
+from auto_round.utils.model import is_moe_model
+
+# ── Data structures ───────────────────────────────────────────────────────────
+
+
+@dataclass
+class AWQMapping:
+ """Declarative mapping: smooth_layer regex → list of balance_layer regexes.
+ Aligned with ``llmcompressor.modifiers.awq.mappings.AWQMapping``.
+ """
+
+ smooth_layer: str
+ balance_layers: list[str]
+ activation_hook_target: str | None = None
+
+
+@dataclass
+class ResolvedMapping:
+ """A fully resolved AWQ mapping with concrete module references."""
+
+ smooth_name: str
+ smooth_layer: torch.nn.Module
+ balance_names: list[str]
+ balance_layers: list[torch.nn.Module]
+ parent_name: str
+ parent: torch.nn.Module
+ activation_hook_target: str | None = None
+
+
+# ── Mapping definitions ─────────────────────────
+# Reference: vllm-project/llm-compressor src/llmcompressor/modifiers/awq/mappings.py
+
+default_mappings = [
+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+_gemma_mappings = [
+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(r"pre_feedforward_layernorm$", [r"gate_proj$", r"up_proj$"]),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# Cohere: MLP runs in parallel with attention; both fed by input_layernorm.
+_cohere_mappings = [
+ AWQMapping(
+ r"input_layernorm$",
+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$", r"mlp\.gate_proj$", r"mlp\.up_proj$"],
+ ),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# Phi: fused qkv_proj and gate_up_proj layers.
+_phi_mappings = [
+ AWQMapping(r"input_layernorm$", [r"qkv_proj$"]),
+ AWQMapping(r"qkv_proj$", [r"o_proj$"]),
+ AWQMapping(r"post_attention_layernorm$", [r"gate_up_proj$"]),
+ AWQMapping(r"gate_up_proj$", [r"down_proj$"]),
+]
+
+# OPT: uses self_attn_layer_norm / final_layer_norm / out_proj / fc1 / fc2.
+_opt_mappings = [
+ AWQMapping(r"self_attn_layer_norm$", [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$"]),
+ AWQMapping(r"self_attn\.v_proj$", [r"self_attn\.out_proj$"]),
+ AWQMapping(r"final_layer_norm$", [r"fc1$"]),
+ AWQMapping(r"fc1$", [r"fc2$"]),
+]
+
+# Bloom: different naming convention.
+# Note: query_key_value → dense mapping, see
+# https://github.com/mit-han-lab/llm-awq/issues/2#issuecomment-1606297469
+_bloom_mappings = [
+ AWQMapping(r"input_layernorm$", [r"query_key_value$"]),
+ AWQMapping(r"post_attention_layernorm$", [r"dense_h_to_4h$"]),
+ AWQMapping(r"gelu_impl$", [r"dense_4h_to_h$"]),
+]
+
+# DeepSeek V2/V3: multi-head latent attention with compressed KV.
+_deepseek_mappings = [
+ AWQMapping(r"input_layernorm$", [r"(q|q_a)_proj$", r"kv_a_proj_with_mqa$"]),
+ AWQMapping(r"q_a_layernorm$", [r"q_b_proj$"]),
+ AWQMapping(r"kv_a_layernorm$", [r"kv_b_proj$"]),
+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# MoE default: expert-parallel gate/up projections.
+_moe_default_mappings = [
+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(
+ r"post_attention_layernorm$",
+ [r"mlp\.experts\.\d+\.gate_proj$", r"mlp\.experts\.\d+\.up_proj$"],
+ ),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# Llama4: uses feed_forward instead of mlp, and includes router.
+_llama4_default_mappings = [
+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(
+ r"post_attention_layernorm$",
+ [r"feed_forward\.router$", r"feed_forward\..*gate_proj$", r"feed_forward\..*up_proj$"],
+ ),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# Exaone4 / Olmo3: only v_proj→o_proj and up_proj→down_proj smoothing.
+_exaone4_mappings = [
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# AFMOE: dual normalization — pre_mlp_layernorm feeds MLP, attention has gate_proj.
+_afmoe_mappings = [
+ AWQMapping(
+ r"input_layernorm$",
+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$", r"self_attn\.gate_proj$"],
+ ),
+ AWQMapping(r"v_proj$", [r"o_proj$"]),
+ AWQMapping(
+ r"pre_mlp_layernorm$",
+ [r"mlp\..*gate_proj$", r"mlp\..*up_proj$"],
+ ),
+ AWQMapping(r"up_proj$", [r"down_proj$"]),
+]
+
+# ── Model class name → mappings registry ──────────────────────────────────────
+# Aligned with llm-compressor AWQ_MAPPING_REGISTRY (llmcompressor v0.10.0).
+# Models not in this registry fall back to default_mappings.
+
+AWQ_MAPPING_REGISTRY: dict[str, list[AWQMapping]] = {
+ # AFMOE
+ "AfmoeForCausalLM": _afmoe_mappings,
+ # Bloom
+ "BloomForCausalLM": _bloom_mappings,
+ # OPT
+ "OPTForCausalLM": _opt_mappings,
+ # Cohere / Command-R
+ "CohereForCausalLM": _cohere_mappings,
+ "Cohere2ForCausalLM": _cohere_mappings,
+ "Cohere2VisionForConditionalGeneration": _cohere_mappings,
+ # DeepSeek V2/V3
+ "DeepseekV3ForCausalLM": _deepseek_mappings,
+ # Exaone4 / Olmo3
+ "Exaone4ForCausalLM": _exaone4_mappings,
+ # Gemma 2/3
+ "Gemma2ForCausalLM": _gemma_mappings,
+ "Gemma3ForCausalLM": _gemma_mappings,
+ "Gemma3ForConditionalGeneration": _gemma_mappings,
+ # Llama
+ "LlamaForCausalLM": default_mappings,
+ "Llama4ForConditionalGeneration": _llama4_default_mappings,
+ # Mistral
+ "MistralForCausalLM": default_mappings,
+ "Mistral3ForConditionalGeneration": default_mappings,
+ # Olmo3 (same as Exaone4)
+ "Olmo3ForCausalLM": _exaone4_mappings,
+ # Phi
+ "Phi3ForCausalLM": _phi_mappings,
+ "Phi3VForCausalLM": _phi_mappings,
+ # Qwen
+ "Qwen2ForCausalLM": default_mappings,
+ "Qwen2_5OmniThinkerForConditionalGeneration": default_mappings,
+ "Qwen3ForCausalLM": default_mappings,
+ # Qwen MoE
+ "Qwen2MoeForCausalLM": _moe_default_mappings,
+ "Qwen3MoeForCausalLM": _moe_default_mappings,
+ # GLM
+ "Glm4MoeForCausalLM": _moe_default_mappings,
+ "GlmMoeDsaForCausalLM": _deepseek_mappings,
+ # Other models using default mappings
+ "SeedOssForCausalLM": default_mappings,
+ "Ernie4_5_MoeForCausalLM": default_mappings,
+}
+
+
+# ── Helper functions ──────────────────────────────────────────────────────────
+
+
+def _get_module(model: torch.nn.Module, name: str) -> torch.nn.Module | None:
+ """Safely retrieve a sub-module by dotted name."""
+ try:
+ parts = name.split(".")
+ m = model
+ for p in parts:
+ m = getattr(m, p)
+ return m
+ except AttributeError:
+ return None
+
+
+def _find_parent(model: torch.nn.Module, names: list[str]) -> tuple[str, torch.nn.Module]:
+ """Find the lowest common ancestor module for a list of module names."""
+ if not names:
+ return "", model
+
+ parts_list = [n.split(".") for n in names]
+ common = []
+ for level_parts in zip(*parts_list):
+ if len(set(level_parts)) == 1:
+ common.append(level_parts[0])
+ else:
+ break
+
+ ancestor_name = ".".join(common)
+ if ancestor_name:
+ ancestor = _get_module(model, ancestor_name)
+ else:
+ ancestor = model
+ return ancestor_name, ancestor
+
+
+def _extract_block_prefix(name: str) -> str | None:
+ """Extract the transformer block prefix from a module name.
+
+ E.g., "model.layers.5.input_layernorm" → "model.layers.5"
+ """
+ match = re.match(r"(.*\.\d+)\.", name)
+ return match.group(1) if match else None
+
+
+def _get_model_class_name(model: torch.nn.Module) -> str:
+ """Get the model class name, handling nested model wrappers."""
+ return type(model).__name__
+
+
+# ── Dynamic mapping builders ─────────────────────────────────────────────────
+# For hybrid attention models (mix of full self-attention and linear/Gated
+# DeltaNet attention) that need layer-index-specific AWQ mappings.
+
+
+def _get_hybrid_attention_config(model: torch.nn.Module) -> tuple[list[str], int] | None:
+ """Extract layer_types and num_hidden_layers from a hybrid attention model.
+
+ Checks text_config for VL models, then top-level config.
+ Returns (layer_types, num_hidden_layers) or None if not hybrid.
+ """
+ config = getattr(model, "config", None)
+ if config is None:
+ return None
+
+ text_config = getattr(config, "text_config", config)
+ layer_types = getattr(text_config, "layer_types", None)
+ num_layers = getattr(text_config, "num_hidden_layers", None)
+
+ if layer_types is None or num_layers is None:
+ return None
+
+ if "full_attention" not in layer_types or "linear_attention" not in layer_types:
+ return None
+
+ return layer_types, num_layers
+
+
+def _detect_linear_attn_projections(model: torch.nn.Module) -> list[str]:
+ """Detect linear attention projection names from the model.
+
+ Different architectures use different projection layouts:
+ - Qwen3Next: in_proj_qkvz, in_proj_ba
+ - Qwen3.5: in_proj_qkv, in_proj_z, in_proj_b, in_proj_a
+ """
+ proj_names = []
+ for name, _ in model.named_modules():
+ if ".linear_attn." not in name:
+ continue
+ sub = name.rsplit("linear_attn.", 1)[-1]
+ if sub.startswith("in_proj_"):
+ proj_names.append(sub)
+ return list(dict.fromkeys(proj_names))
+
+
+def _build_hybrid_attention_mappings(model: torch.nn.Module) -> list[AWQMapping] | None:
+ """Dynamically build AWQ mappings for hybrid attention models.
+
+ Reads ``layer_types`` from model config to determine which layers use
+ full vs linear attention, then inspects module names to detect the correct
+ linear attention projection names and MLP structure (MoE vs dense).
+ """
+ result = _get_hybrid_attention_config(model)
+ if result is None:
+ return None
+
+ layer_types, num_layers = result
+
+ full_indices = [i for i in range(num_layers) if layer_types[i] == "full_attention"]
+ linear_indices = [i for i in range(num_layers) if layer_types[i] == "linear_attention"]
+
+ if not full_indices or not linear_indices:
+ logger.warning(
+ "Hybrid attention model detected but missing indices for "
+ "both full and linear attention layers. Falling back."
+ )
+ return None
+
+ full_re = "|".join(str(i) for i in full_indices)
+ linear_re = "|".join(str(i) for i in linear_indices)
+
+ linear_proj_names = _detect_linear_attn_projections(model)
+ is_moe = is_moe_model(model)
+
+ mappings = []
+
+ # Full attention layers: input_layernorm → q/k/v_proj
+ mappings.append(
+ AWQMapping(
+ rf"layers\.({full_re})\.input_layernorm$",
+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$"],
+ )
+ )
+
+ # Linear attention layers: input_layernorm → linear_attn projections
+ if linear_proj_names:
+ mappings.append(
+ AWQMapping(
+ rf"layers\.({linear_re})\.input_layernorm$",
+ [rf"linear_attn\.{p}$" for p in linear_proj_names],
+ )
+ )
+
+ # MLP: MoE vs dense
+ if is_moe:
+ mappings.append(
+ AWQMapping(
+ r"post_attention_layernorm$",
+ [
+ r"mlp\.experts\.\d+\.gate_proj$",
+ r"mlp\.experts\.\d+\.up_proj$",
+ r"mlp\.shared_expert\.gate_proj$",
+ r"mlp\.shared_expert\.up_proj$",
+ ],
+ )
+ )
+ else:
+ mappings.append(
+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),
+ )
+
+ mappings.append(AWQMapping(r"up_proj$", [r"down_proj$"]))
+
+ logger.warning(
+ f"Built dynamic hybrid-attention AWQ mappings: "
+ f"{len(full_indices)} full-attention, {len(linear_indices)} linear-attention, "
+ f"projections={linear_proj_names}, MoE={is_moe}"
+ )
+ return mappings
+
+
+AWQ_DYNAMIC_MAPPING_REGISTRY: dict[str, callable] = {
+ "Qwen3NextForCausalLM": _build_hybrid_attention_mappings,
+ "Qwen3_5ForCausalLM": _build_hybrid_attention_mappings,
+ "Qwen3_5ForConditionalGeneration": _build_hybrid_attention_mappings,
+ "Qwen3_5MoeForCausalLM": _build_hybrid_attention_mappings,
+ "Qwen3_5MoeForConditionalGeneration": _build_hybrid_attention_mappings,
+}
+
+
+def _get_mappings_for_model(model: torch.nn.Module) -> list[AWQMapping]:
+ """Look up mappings for a model from the registry, with default fallback.
+
+ Resolution order:
+ 1. Dynamic registry — runtime-generated per-layer mappings
+ (e.g. hybrid attention models like Qwen3.5 MoE).
+ 2. Static registry — ``AWQ_MAPPING_REGISTRY``.
+ 3. ``default_mappings`` — Llama-like fallback.
+ """
+ cls_name = _get_model_class_name(model)
+
+ if cls_name in AWQ_DYNAMIC_MAPPING_REGISTRY:
+ mappings = AWQ_DYNAMIC_MAPPING_REGISTRY[cls_name](model)
+ if mappings is not None:
+ return mappings
+
+ if cls_name in AWQ_MAPPING_REGISTRY:
+ logger.info(f"Using registered AWQ mappings for {cls_name}.")
+ return AWQ_MAPPING_REGISTRY[cls_name]
+
+ logger.info(
+ f"Architecture '{cls_name}' not found in AWQ mapping registry. " f"Using default mappings (Llama-like)."
+ )
+ return default_mappings
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+
+def resolve_mappings(
+ model: torch.nn.Module,
+ user_mappings: list[dict] | None = None,
+) -> list[ResolvedMapping]:
+ """Resolve AWQ mappings for the given model.
+
+ Resolution order:
+ 1. ``user_mappings`` — explicit dicts with ``smooth_layer`` /
+ ``balance_layers`` regex keys.
+ 2. ``AWQ_MAPPING_REGISTRY`` — model-class-name lookup
+ 3. ``default_mappings`` — Llama-like fallback.
+
+ Returns:
+ List of ``ResolvedMapping`` objects ready for AWQ grid search.
+ """
+ if user_mappings is not None:
+ mapping_defs = [AWQMapping(m["smooth_layer"], m["balance_layers"]) for m in user_mappings]
+ else:
+ mapping_defs = _get_mappings_for_model(model)
+
+ return _resolve_mapping_defs(model, mapping_defs)
+
+
+def _resolve_mapping_defs(
+ model: torch.nn.Module,
+ mapping_defs: list[AWQMapping],
+) -> list[ResolvedMapping]:
+ """Resolve a list of AWQMapping definitions against the model."""
+ resolved = []
+ all_names = [n for n, _ in model.named_modules()]
+
+ # Group modules by block prefix
+ block_modules: dict[str, list[str]] = {}
+ for name in all_names:
+ prefix = _extract_block_prefix(name)
+ if prefix is not None:
+ block_modules.setdefault(prefix, []).append(name)
+
+ if not block_modules:
+ logger.warning(
+ "AWQ found no repeating block structure in the model. "
+ "Provide explicit mappings via AWQConfig(mappings=[...])."
+ )
+ return resolved
+
+ matched_count = 0
+
+ for prefix, names_in_block in block_modules.items():
+ for mapping_def in mapping_defs:
+ # Find smooth layer(s) in this block
+ smooth_matches = [n for n in names_in_block if re.search(mapping_def.smooth_layer, n)]
+ if not smooth_matches:
+ continue
+
+ for smooth_name in smooth_matches:
+ smooth_layer = _get_module(model, smooth_name)
+ if smooth_layer is None:
+ continue
+ if not hasattr(smooth_layer, "weight"):
+ continue
+
+ # Find balance layers in the same block
+ balance_names = []
+ balance_layers = []
+ for bp in mapping_def.balance_layers:
+ for n in names_in_block:
+ if re.search(bp, n):
+ m = _get_module(model, n)
+ if m is not None and isinstance(m, torch.nn.Linear):
+ balance_names.append(n)
+ balance_layers.append(m)
+
+ # Fallback: search child blocks for MoE cross-level mappings
+ # (e.g., post_attention_layernorm → experts.N.gate_proj/up_proj)
+ if not balance_layers:
+ for bp in mapping_def.balance_layers:
+ for child_prefix, child_names in block_modules.items():
+ if child_prefix != prefix and child_prefix.startswith(prefix + "."):
+ for n in child_names:
+ if re.search(bp, n):
+ m = _get_module(model, n)
+ if m is not None and isinstance(m, torch.nn.Linear):
+ balance_names.append(n)
+ balance_layers.append(m)
+
+ if not balance_layers:
+ continue
+
+ # Verify dimensional compatibility (filters out GQA
+ # mismatches for v_proj → o_proj automatically)
+ smooth_dim = smooth_layer.weight.shape[0]
+ compatible = all(bl.in_features == smooth_dim for bl in balance_layers)
+ if not compatible:
+ logger.warning_once(
+ f"Skipping AWQ for '{smooth_name}': incompatible "
+ f"balance layers (smooth_dim={smooth_dim}, "
+ f"balance in_features="
+ f"{[bl.in_features for bl in balance_layers]})"
+ )
+ continue
+
+ parent_name, parent = _find_parent(model, balance_names)
+ resolved.append(
+ ResolvedMapping(
+ smooth_name=smooth_name,
+ smooth_layer=smooth_layer,
+ balance_names=balance_names,
+ balance_layers=balance_layers,
+ parent_name=parent_name,
+ parent=parent,
+ activation_hook_target=mapping_def.activation_hook_target,
+ )
+ )
+ matched_count += 1
+
+ if matched_count == 0:
+ logger.warning(
+ "AWQ resolved 0 mappings. The model architecture may not match "
+ "any known pattern. Provide explicit mappings via "
+ "AWQConfig(mappings=[...])."
+ )
+ else:
+ first_prefix = next(iter(block_modules))
+ n_blocks = len(block_modules)
+ mappings_per_block = sum(1 for r in resolved if r.smooth_name.startswith(first_prefix))
+ logger.info(
+ f"AWQ resolved {matched_count} smooth-balance mappings "
+ f"({mappings_per_block} per block × {n_blocks} blocks)."
+ )
+
+ return resolved
+
+
+# ── Model compatibility diagnostics ───────────────────────────────────────────
+
+
+def check_model_compatibility(
+ model: torch.nn.Module,
+ user_mappings: list[dict] | None = None,
+) -> dict:
+ """Check AWQ compatibility and return a diagnostic report.
+
+ Returns a dict with:
+ - ``compatible`` (bool): True if at least one mapping was resolved.
+ - ``n_mappings`` (int): Number of resolved mappings.
+ - ``mappings_per_block`` (int): Mappings in the first block.
+ - ``n_blocks`` (int): Number of transformer blocks.
+ - ``model_class`` (str): Model class name.
+ - ``in_registry`` (bool): Whether model class is in AWQ_MAPPING_REGISTRY.
+ - ``warnings`` (list[str]): Any compatibility warnings.
+ """
+ warnings_list = []
+ cls_name = _get_model_class_name(model)
+ in_registry = cls_name in AWQ_MAPPING_REGISTRY
+
+ if not in_registry and user_mappings is None:
+ warnings_list.append(
+ f"Model class '{cls_name}' is not in AWQ_MAPPING_REGISTRY. "
+ f"Using default Llama-like mappings. If quantization quality is "
+ f"poor, provide explicit mappings via AWQConfig(mappings=[...])."
+ )
+
+ resolved = resolve_mappings(model, user_mappings)
+
+ all_prefixes = set()
+ for r in resolved:
+ prefix = _extract_block_prefix(r.smooth_name)
+ if prefix:
+ all_prefixes.add(prefix)
+
+ n_blocks = len(all_prefixes)
+ mappings_per_block = 0
+ if n_blocks > 0 and resolved:
+ first_prefix = min(all_prefixes)
+ mappings_per_block = sum(1 for r in resolved if r.smooth_name.startswith(first_prefix))
+
+ if not resolved:
+ warnings_list.append(
+ "No AWQ mappings could be resolved. The model architecture may " "not be supported for auto-detection."
+ )
+
+ return {
+ "compatible": len(resolved) > 0,
+ "n_mappings": len(resolved),
+ "mappings_per_block": mappings_per_block,
+ "n_blocks": n_blocks,
+ "model_class": cls_name,
+ "in_registry": in_registry,
+ "warnings": warnings_list,
+ }
diff --git a/auto_round/algorithms/quantization/awq/quantizer.py b/auto_round/algorithms/quantization/awq/quantizer.py
new file mode 100644
index 000000000..f6aca2f75
--- /dev/null
+++ b/auto_round/algorithms/quantization/awq/quantizer.py
@@ -0,0 +1,935 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""AWQ (Activation-Aware Weight Quantization) quantizer.
+
+The algorithm:
+1. Collects per-channel activation magnitudes during calibration.
+2. For each smooth-balance mapping, performs a grid search over scaling ratios
+ to find the one that minimizes quantization error (output-based loss).
+3. Applies the best channel-wise scaling: balance_layer.weight *= scales,
+ smooth_layer.weight /= scales (or smooth_layer.bias /= scales if 1-D).
+
+
+Reference implementations:
+ - AutoAWQ: https://github.com/casper-hansen/AutoAWQ
+ - llm-compressor: https://github.com/vllm-project/llm-compressor
+"""
+
+from __future__ import annotations
+
+import contextlib
+import inspect
+import traceback
+
+import torch
+from tqdm import tqdm
+
+from auto_round.algorithms.quantization.awq.config import AWQConfig
+from auto_round.algorithms.quantization.awq.mappings import (
+ ResolvedMapping,
+ _extract_block_prefix,
+ resolve_mappings,
+)
+from auto_round.algorithms.quantization.base import BaseQuantizers
+from auto_round.compressors.shard_writer import ShardWriter
+from auto_round.compressors.utils import immediate_pack
+from auto_round.data_type.utils import (
+ get_quant_func,
+ reshape_pad_tensor_by_group_size,
+ revert_tensor_by_pad,
+ update_block_global_scale_if_needed,
+)
+from auto_round.logger import logger
+from auto_round.utils import (
+ check_to_quantized,
+ clear_memory,
+ convert_module_to_hp_if_necessary,
+ get_module,
+ set_amax_for_all_moe_layers,
+ set_module,
+)
+from auto_round.wrapper import WrapperLinear
+
+
+class AWQQuantizer(BaseQuantizers):
+ """AWQ quantizer: activation-aware channel scaling + delegated quantization.
+
+ AWQ is a pre-quantization transform that applies channel-wise scaling to reduce
+ quantization error. The actual quantization is delegated to an inner
+ quantizer (currently RTN by default).
+ """
+
+ def __init__(self, config: AWQConfig):
+ super().__init__(config)
+ self.duo_scaling = config.duo_scaling
+ self.n_grid = config.n_grid
+ self.apply_smooth = config.apply_smooth
+
+ # Populated during calibration
+ self._user_mappings = config.mappings
+ self._resolved_mappings: list[ResolvedMapping] = []
+ self._block_groups: dict[str, list[ResolvedMapping]] = {}
+ self._activation_stats: dict[str, list[torch.Tensor]] = {}
+ # Parent module kwargs cache: parent_module → list of kwargs dicts
+ self._parent_args_cache: dict[torch.nn.Module, list[dict]] = {}
+ self._parent_signatures: dict[int, inspect.BoundArguments] = {}
+ self._smoothing_applied: bool = False
+
+ # Fail fast: validate scheme at construction time
+ self._check_scheme_compatibility()
+
+ def __setattr__(self, name, value):
+ """Trigger model-level AWQ setup when compress_context is assigned."""
+ super().__setattr__(name, value)
+ if name == "compress_context" and value is not None:
+ self._prepare_model()
+
+ # ── Public API ────────────────────────────────────────────────────────────
+
+ def resolve_all_mappings(self, model: torch.nn.Module) -> dict[str, list[ResolvedMapping]]:
+ """Resolve all AWQ mappings and group by block prefix.
+
+ Call this once before the block-by-block loop. The resolved mappings
+ are stored internally and used by ``register_activation_hooks`` and
+ ``smooth_block`` for per-block filtering.
+
+ Returns:
+ Dictionary mapping block prefix (e.g. "model.layers.0") to the
+ list of resolved mappings for that block.
+ """
+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)
+ block_groups: dict[str, list[ResolvedMapping]] = {}
+ for m in self._resolved_mappings:
+ prefix = _extract_block_prefix(m.smooth_name)
+ block_groups.setdefault(prefix, []).append(m)
+ self._block_groups = block_groups
+ return block_groups
+
+ def apply_smoothing(
+ self,
+ model: torch.nn.Module,
+ device: torch.device | str | None = None,
+ ) -> None:
+ """Apply AWQ smoothing to the model weights.
+
+ This is the core AWQ step: for each mapping, find the best scaling
+ factor and apply it to smooth/balance layers.
+
+ When *device* is provided the model is assumed to reside on CPU and
+ only the modules needed for each mapping are temporarily moved to
+ *device* during the grid search. This dramatically reduces peak VRAM
+ (~14 GB savings for an 8B model).
+
+ Should be called AFTER calibration data has been collected and BEFORE
+ RTN quantization.
+ """
+ if self._smoothing_applied:
+ return
+
+ # Resolve mappings
+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)
+ if not self._resolved_mappings:
+ logger.warning("No AWQ mappings resolved; skipping smoothing.")
+ self._smoothing_applied = True
+ return
+
+ if not self._activation_stats:
+ logger.error("No activation statistics collected for AWQ smoothing.")
+
+ logger.info(f"Applying AWQ smoothing to {len(self._resolved_mappings)} mappings...")
+
+ for mapping in tqdm(self._resolved_mappings, desc="AWQ Smoothing"):
+ if mapping.smooth_name not in self._activation_stats:
+ logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping this layer.")
+ continue
+
+ act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)
+ if act_count == 0:
+ logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping this layer.")
+ continue
+
+ # Mean activation per channel (act_count is a plain int now,
+ # avoiding a CPU tensor allocation per channel).
+ x_mean = (act_sum / act_count).to(torch.float32)
+ del act_sum
+
+ with self._align_modules(mapping, device):
+ best_scales = self._grid_search_scales(mapping, x_mean)
+
+ if best_scales is not None:
+ self._apply_scales(mapping, best_scales)
+
+ # Eagerly free per-mapping parent cache to reduce RAM
+ parent = mapping.parent
+ self._parent_args_cache.pop(parent, None)
+
+ self._activation_stats.clear()
+ self._parent_args_cache.clear()
+ self._parent_signatures.clear()
+ self._smoothing_applied = True
+ logger.info("AWQ smoothing complete.")
+
+ def smooth_block(self, block_prefix: str) -> None:
+ """Apply AWQ smoothing for one block's mappings.
+
+ Assumes activation stats and parent kwargs have already been collected
+ for this block via ``register_activation_hooks`` + block forward.
+ The block's modules are expected to be on the compute device, so
+ no device alignment is needed (unlike ``apply_smoothing`` which uses
+ ``_align_modules`` for CPU-offloaded models).
+
+ Args:
+ block_prefix: Block prefix (e.g. "model.layers.0") identifying
+ which mappings to smooth.
+ """
+ block_mappings = [m for m in self._resolved_mappings if _extract_block_prefix(m.smooth_name) == block_prefix]
+ for mapping in block_mappings:
+ if mapping.smooth_name not in self._activation_stats:
+ logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping.")
+ continue
+
+ act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)
+ if act_count == 0:
+ logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping.")
+ continue
+
+ x_mean = (act_sum / act_count).to(torch.float32)
+ del act_sum
+
+ # Block is on the compute device — grid search runs in-place
+ best_scales = self._grid_search_scales(mapping, x_mean)
+
+ if best_scales is not None:
+ self._apply_scales(mapping, best_scales)
+
+ # Free parent kwargs cache after ALL mappings for this block are done.
+ # (All mappings in a block typically share the same parent module;
+ # popping inside the loop would break output-based loss for
+ # subsequent mappings.)
+ seen_parents = set()
+ for mapping in block_mappings:
+ pid = id(mapping.parent)
+ if pid not in seen_parents:
+ seen_parents.add(pid)
+ self._parent_args_cache.pop(mapping.parent, None)
+
+ # ── Model-level initialization ─────────────────────────────────────────────
+
+ def _prepare_model(self) -> None:
+ """One-time model-level AWQ setup: check compatibility and resolve mappings.
+
+ Triggered automatically when model_context is assigned (via __setattr__).
+ """
+ from auto_round.algorithms.quantization.awq.mappings import check_model_compatibility
+
+ model = self.model_context.model
+ report = check_model_compatibility(model, self._user_mappings)
+ for w in report["warnings"]:
+ logger.warning(w)
+ if not report["compatible"]:
+ model_class = report.get("model_class", "unknown")
+ raise ValueError(
+ f"AWQ: no smooth-balance mappings could be resolved for "
+ f"'{model_class}'. Either the model architecture is not "
+ f"supported for automatic AWQ mapping detection, or the model "
+ f"has no repeating transformer block structure. "
+ f"You can provide explicit mappings via "
+ f"AWQConfig(mappings=[{{'smooth_layer': '<regex>', "
+ f"'balance_layers': ['<regex>', ...]}}])."
+ )
+
+ self.resolve_all_mappings(model)
+
+ # AWQ caches block I/O on CPU — only one block lives on GPU at a time.
+ self.compress_context.cache_device = torch.device("cpu")
+
+ def _check_scheme_compatibility(self) -> None:
+ """Validate the quantization scheme against AWQ inference backends."""
+ bits = self.bits
+ act_bits = self.act_bits or 16
+ data_type = self.data_type or "int"
+
+ if "int" not in data_type:
+ raise ValueError(
+ f"AWQ requires integer data_type, got '{data_type}'. "
+ f"AWQ channel scaling is designed for integer quantization "
+ f"grids. Use algorithm='autoround' for FP8/MXFP quantization."
+ )
+
+ if act_bits is not None and act_bits < 16 and act_bits != 8:
+ raise ValueError(
+ f"AWQ with act_bits={act_bits} is not supported. "
+ f"No inference kernel exists for W{bits}A{act_bits} in vllm "
+ f"or sglang. Supported schemes: W4A16 (canonical AWQ) or "
+ f"W8A8 (compressed_tensors INT8 backend)."
+ )
+
+ if bits == 4 and act_bits >= 16:
+ pass
+ elif bits == 8 and act_bits == 8:
+ logger.info(
+ "AWQ with W8A8: AWQ smoothing will be applied, followed by "
+ "INT8 quantization. This is served by vllm's "
+ "compressed_tensors backend (cutlass INT8 GEMM), not AWQ "
+ "kernels."
+ )
+ elif bits not in (4, 8):
+ logger.warning(
+ f"AWQ with bits={bits}: vllm AWQ kernels only support "
+ f"bits=4 (AWQ/Marlin) natively. bits=8 is supported via "
+ f"compressed_tensors. Other bit widths may not have "
+ f"optimized inference kernels."
+ )
+
+ # ── Weight quantization (delegated RTN) ─────────────────────────────────────
+
+ def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:
+ """Quantize a single layer using RTN after AWQ smoothing has been applied.
+
+ AWQ's quantize_layer is simpler than RTN's because:
+ - AWQ always uses plain RTN (disable_opt_rtn=True, no imatrix)
+ - No MoE-specific RTN disabling (AWQ handles MoE via mappings)
+ - No GGUF special path (AWQ targets AWQ/Marlin inference kernels)
+
+ Args:
+ name: Fully-qualified module name (e.g. "model.layers.0.self_attn.q_proj").
+ dtype: Optional dtype to cast the layer to before quantization.
+ """
+ m = get_module(self.model, name)
+ if dtype is not None:
+ m = m.to(dtype)
+
+ m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)
+ set_module(self.model, name, m)
+ tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.compress_context.device
+
+ try:
+ m = m.to(tuning_device)
+ m = WrapperLinear(
+ m,
+ device=tuning_device,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ disable_opt_rtn=self.config.disable_opt_rtn,
+ iters=0,
+ )
+ m = m.unwrapper({})
+ except torch.OutOfMemoryError:
+ cuda_error_msg = traceback.format_exc()
+ m = m.orig_layer if hasattr(m, "orig_layer") else m
+ try:
+ logger.error(cuda_error_msg)
+ logger.warning("AWQ quantize_layer: falling back to CPU.")
+ m.to("cpu")
+ m = WrapperLinear(
+ m,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ disable_opt_rtn=self.config.disable_opt_rtn,
+ iters=0,
+ )
+ m = m.unwrapper({})
+ except Exception:
+ raise
+
+ set_module(self.model, name, m)
+ self._immediate_pack_and_save_module(name)
+
+ def quantize_layer_outside_block(self, *args, **kwargs):
+ """Quantize layers outside blocks (e.g. lm_head). Delegates to quantize_layer."""
+ return self.quantize_layer(*args, **kwargs)
+
+ def _immediate_pack_and_save_module(self, module_name: str) -> None:
+ """Pack and/or save a quantized module if immediate mode is enabled."""
+ shard_writer = ShardWriter.get_shard_writer()
+ to_cpu = self.compress_context.low_gpu_mem_usage
+ module = get_module(self.model, module_name)
+ if self.compress_context.is_immediate_packing:
+ immediate_pack(module_name, self.layer_config)
+ if to_cpu:
+ module = module.to("cpu")
+ packed_module = get_module(self.model, module_name)
+ set_module(self.model, module_name, packed_module.to("cpu"))
+ else:
+ if to_cpu:
+ module = module.to("cpu")
+ set_module(self.model, module_name, module)
+ if self.compress_context.is_immediate_saving:
+ module = get_module(self.model, module_name)
+ module.to("cpu")
+ shard_writer.write(module, module_name, False)
+ module.to("meta")
+
+ # ── Block-level quantization ──────────────────────────────────────────────
+
+ def quantize_block(
+ self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs
+ ) -> dict:
+ """AWQ block quantization: collect stats → smooth → quantize.
+
+ The full per-block AWQ pipeline:
+ 1. Register activation hooks for this block's mappings
+ 2. Run block forward to collect activation stats + parent kwargs
+ 3. Apply AWQ smoothing (grid search + scale application)
+ 4. Collect act_max AFTER smoothing (for activation quantization)
+ 5. Quantize the smoothed block via RTN
+
+ Args:
+ block: Module already on the compute device.
+ input_ids: Calibration inputs for this block.
+ input_others: Additional kwargs for block forward.
+ reference_output: FP16 block output (kept for interface consistency).
+ **kwargs: Must include 'block_name' for mapping lookup.
+ """
+ block_name = kwargs.get("block_name")
+ if block_name is None:
+ # Infer block_name from resolved mappings by matching the block's modules
+ for prefix, mappings in self._block_groups.items():
+ if any(m.smooth_layer is mod or m.parent is mod for m in mappings for mod in block.modules()):
+ block_name = prefix
+ break
+ if block_name is None:
+ raise ValueError(
+ "AWQQuantizer.quantize_block() could not determine block_name. "
+ "Pass block_name explicitly or ensure resolved mappings cover this block."
+ )
+
+ model = self.model
+ bs = self.batch_size * self.infer_bs_coeff
+ if self.compress_context.low_gpu_mem_usage:
+ bs = 1
+ logger.info("AWQ: low_gpu_mem_usage enabled, setting inference batch size to 1.")
+
+ # Step 1 & 2: AWQ smoothing (optional)
+ if self.apply_smooth:
+ awq_hooks = self.register_activation_hooks(model, block_prefix=block_name)
+ self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)
+ for h in awq_hooks:
+ h.remove()
+
+ self.smooth_block(block_name)
+
+ # Step 3: Collect act_max AFTER smoothing
+ # AWQ smoothing changes internal activations (LayerNorm output /= scales),
+ # so act_max must be collected post-smoothing. Reset any pre-smoothing
+ # act_max values first to avoid stale data persisting via max().
+ act_max_hooks = self.register_calibration_hooks(block, imatrix=False)
+ if act_max_hooks:
+ for _name, m in block.named_modules():
+ if hasattr(m, "act_max"):
+ del m.act_max
+ self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)
+ for h in act_max_hooks:
+ h.remove()
+
+ # Step 4: Quantize the smoothed block (delegated RTN)
+ # This is equivalent to RTNQuantizer.quantize_block but inlined here
+ # to avoid inheritance coupling.
+ update_block_global_scale_if_needed(block, self.data_type, self.group_size)
+ if (
+ self.config.is_act_nv_fp
+ or self.config.is_static_afp8
+ or (self.config.is_wfp8afp8 and not self.config.act_dynamic)
+ ):
+ set_amax_for_all_moe_layers(block, attr_name="act_max")
+
+ for _name, m in block.named_modules():
+ if hasattr(m, "global_name") and check_to_quantized(m):
+ self.quantize_layer(m.global_name)
+
+ return {}
+
+ # ── Module alignment (device onload/offload) ─────────────────────────────
+
+ @contextlib.contextmanager
+ def _align_modules(
+ self,
+ mapping,
+ device: torch.device | str | None,
+ ):
+ """Temporarily move mapping-related modules to *device* for grid search.
+
+ If *device* is None (model already on the compute device), this is a
+ no-op passthrough.
+ """
+ if device is None:
+ yield
+ return
+
+ modules_to_move = [mapping.parent, mapping.smooth_layer] + list(mapping.balance_layers)
+ # Deduplicate (parent may overlap with smooth/balance)
+ seen = set()
+ unique = []
+ for m in modules_to_move:
+ if id(m) not in seen:
+ seen.add(id(m))
+ unique.append(m)
+
+ # Onload
+ for m in unique:
+ m.to(device)
+
+ try:
+ yield
+ finally:
+ # Offload back to CPU
+ for m in unique:
+ m.to("cpu")
+ clear_memory()
+
+ # ── Activation statistics collection ──────────────────────────────────────
+
+ def register_activation_hooks(
+ self,
+ model: torch.nn.Module,
+ block_prefix: str | None = None,
+ ) -> list:
+ """Register activation hooks for AWQ and return hook handles.
+
+ Hooks are registered for:
+ 1. Smooth layers: accumulate abs activation sums/counts for x_mean.
+ 2. Parent modules: cache forward kwargs for output-based loss
+
+ Args:
+ model: The model (hooks are registered on the actual module
+ objects, which are shared with blocks obtained via
+ ``get_module``).
+ block_prefix: When provided, only register hooks for mappings
+ whose smooth layer belongs to this block (e.g.
+ "model.layers.0"). Used in the block-by-block pipeline.
+
+ Should be called by the compressor before calibration, and handles
+ should be removed after calibration.
+ """
+ if not self._resolved_mappings:
+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)
+
+ mappings = self._resolved_mappings
+ if block_prefix is not None:
+ mappings = [m for m in mappings if _extract_block_prefix(m.smooth_name) == block_prefix]
+
+ hooks = []
+ smooth_names = {m.smooth_name for m in mappings}
+
+ # Hook smooth layers for activation statistics
+ for name, module in model.named_modules():
+ if name not in smooth_names:
+ continue
+
+ def _make_hook(layer_name):
+ def hook_fn(mod, args, output):
+ if isinstance(output, tuple):
+ x = output[0]
+ else:
+ x = output
+ if x is None or x.numel() == 0:
+ return
+
+ # Compute abs-sum directly on GPU without materialising a
+ # full float32 intermediate. flatten→abs→sum produces a
+ # single [channels] tensor; .cpu() immediately frees the
+ # GPU allocation (aligned with AR's "immediate CPU move"
+ # pattern from CalibCompressor hooks).
+ channel_sum = x.detach().float().flatten(0, -2).abs().sum(dim=0).cpu()
+ count = x.shape[:-1].numel()
+
+ if layer_name not in self._activation_stats:
+ self._activation_stats[layer_name] = [
+ torch.zeros_like(channel_sum),
+ 0,
+ ]
+ self._activation_stats[layer_name][0] += channel_sum
+ self._activation_stats[layer_name][1] += count
+
+ return hook_fn
+
+ h = module.register_forward_hook(_make_hook(name))
+ hooks.append(h)
+
+ # Hook parent modules to cache kwargs for output-based loss
+ parent_modules_hooked = set()
+ for mapping in mappings:
+ parent = mapping.parent
+ if id(parent) in parent_modules_hooked:
+ continue
+ parent_modules_hooked.add(id(parent))
+
+ self._parent_args_cache[parent] = []
+
+ def _make_parent_hook(parent_module):
+ def hook_fn(mod, args, kwargs):
+ # Cache ALL calibration batches for output-based grid
+ # search loss, use all calibration data without subsampling.
+ mod_cls_id = id(type(mod))
+ if mod_cls_id not in self._parent_signatures:
+ self._parent_signatures[mod_cls_id] = inspect.signature(mod.forward)
+ sig = self._parent_signatures[mod_cls_id]
+ bound = sig.bind(*args, **kwargs)
+ bound.apply_defaults()
+
+ # Infer the parent's compute dtype so that tensors
+ # upcasted by nn.LayerNorm (float32) are stored in the
+ # weight dtype (e.g. bfloat16). This avoids repeated
+ # per-sample casting in the grid search stage.
+ param = next(mod.parameters(), None)
+ w_dtype = param.dtype if param is not None else None
+
+ stored = {}
+ for k, v in bound.arguments.items():
+ if isinstance(v, torch.Tensor):
+ v = v.detach()
+ if w_dtype is not None and v.is_floating_point() and v.dtype != w_dtype:
+ v = v.to(w_dtype)
+ stored[k] = v
+ elif isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v):
+ # Detach tensors in tuples (e.g. position_embeddings
+ # = (cos, sin)) to release computation graph refs.
+ stored[k] = tuple(
+ (
+ (
+ t.detach().to(w_dtype)
+ if w_dtype and t.is_floating_point() and t.dtype != w_dtype
+ else t.detach()
+ )
+ if isinstance(t, torch.Tensor)
+ else t
+ )
+ for t in v
+ )
+ elif hasattr(v, "key_cache"):
+ # Null out KV cache objects (DynamicCache etc.)
+ stored[k] = None
+ else:
+ stored[k] = v
+
+ self._parent_args_cache[parent_module].append(stored)
+
+ return hook_fn
+
+ h = parent.register_forward_pre_hook(_make_parent_hook(parent), with_kwargs=True)
+ hooks.append(h)
+
+ return hooks
+
+ # ── Grid search ───────────────────────────────────────────────────────────
+
+ def _get_grid_search_params(self) -> list[tuple[float, bool]]:
+ """Get grid search parameters (ratio, duo_scaling).
+
+ Returns:
+ List of (ratio, use_duo_scaling) tuples for the grid search.
+ """
+ match self.duo_scaling:
+ # "both": half grid with duo off, half with duo on
+ case "both":
+ n_grid = max(int(self.n_grid / 2), 2)
+ return [
+ (grid_idx / (n_grid - 1), duo_scaling)
+ for grid_idx in range(n_grid)
+ for duo_scaling in [False, True]
+ ]
+ case False:
+ n_grid = max(self.n_grid, 2)
+ return [(grid_idx / (n_grid - 1), False) for grid_idx in range(n_grid)]
+ # True: include identity (0.0, False) as first, then duo points
+ case True:
+ n_grid = max(self.n_grid, 3)
+ return [(0.0, False)] + [(grid_idx / (n_grid - 2), True) for grid_idx in range(n_grid - 1)]
+ case _:
+ raise ValueError(f"Found unexpected duo_scaling configuration {self.duo_scaling}")
+
+ @staticmethod
+ def _compute_layer_means(layers: list[torch.nn.Module], group_size: int) -> torch.Tensor:
+ """Compute per-channel mean of normalised weights.
+
+ Within each quantization group, weights are normalized by their group max
+ (so values are on a 0-1 scale), then averaged across all groups to get
+ per-channel importance.
+
+ Args:
+ layers: Balance layers whose weights are concatenated.
+ group_size: Quantization group size. If -1, uses full channel width.
+
+ Returns:
+ Per-channel mean of normalised weights [in_features].
+ """
+ # Concatenate all balance layer weights [total_out, in_features]
+ weight = torch.cat([m.weight.detach().float() for m in layers], dim=0)
+ org_shape = weight.shape
+
+ # Determine effective group size
+ gs = group_size if group_size > 0 else org_shape[1]
+
+ # Pad when needed so AWQ works with layers whose input width is not a
+ # multiple of group_size, matching grouped RTN quantization behavior.
+ weight, _, pad_len = reshape_pad_tensor_by_group_size(weight, gs)
+ # Normalize within each group: abs / max (0-1 scale per group)
+ w_scale = weight.abs() / (weight.abs().amax(dim=1, keepdim=True) + 1e-6)
+ # Remove padding, then take mean across output channels.
+ w_scale = revert_tensor_by_pad(w_scale, orig_shape=org_shape, pad_len=pad_len)
+ w_mean = w_scale.mean(0)
+ return w_mean
+
+ @torch.no_grad()
+ def _grid_search_scales(
+ self,
+ mapping: ResolvedMapping,
+ x_mean: torch.Tensor,
+ ) -> torch.Tensor | None:
+ """Find the best scaling ratio via grid search.
+
+ Uses output-based error:
+ ``L(s) = || fp16_output - Q(W*s) @ (X/s) ||^2``
+
+ For each candidate scaling ratio, applies scales to balance layer
+ weights, quantize-dequantizes them, runs all cached calibration
+ batches through the parent module, and computes the output MSE
+ against the fp16 baseline.
+
+ Returns:
+ Best scales tensor, or None if no valid scale was found.
+ """
+ device = mapping.balance_layers[0].weight.device
+ x_mean = x_mean.to(device)
+
+ # Compute normalised weight means
+ group_size = self.group_size if self.group_size > 0 else -1
+ if self.duo_scaling is not False:
+ w_mean = self._compute_layer_means(mapping.balance_layers, group_size).to(device)
+
+ # Try to run parent module forward for output-based loss
+ parent_kwargs_list = self._parent_args_cache.get(mapping.parent, [])
+ use_parent_forward = len(parent_kwargs_list) > 0
+
+ if use_parent_forward:
+ # Compute fp16 baseline outputs for loss computation
+ fp16_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)
+ if not fp16_outputs or all(f.numel() == 0 for f in fp16_outputs):
+ use_parent_forward = False
+
+ # Save original weights for restoration during grid search
+ orig_state = {bl: bl.weight.data.clone() for bl in mapping.balance_layers}
+
+ best_error = float("inf")
+ best_scales = None
+ best_ratio = -1
+
+ # Pre-resolve quant function once.
+ ref_layer = mapping.balance_layers[0]
+ ref_name = getattr(ref_layer, "global_name", None) or ""
+ ref_cfg = self.layer_config.get(ref_name, {})
+ try:
+ cached_quant_func, _ = get_quant_func(
+ ref_cfg.get("data_type", self.data_type),
+ ref_cfg.get("bits", self.bits),
+ ref_cfg.get("sym", self.sym),
+ disable_opt_rtn=True,
+ group_size=ref_cfg.get("group_size", self.group_size),
+ iters=0,
+ )
+ except Exception:
+ cached_quant_func = None
+
+ grid_params = self._get_grid_search_params()
+
+ for ratio, use_duo in grid_params:
+ # Compute scales
+ if use_duo:
+ scales = (x_mean.pow(ratio) / (w_mean.pow(1 - ratio) + 1e-4)).clamp(min=1e-4)
+ else:
+ scales = x_mean.pow(ratio).clamp(min=1e-4).view(-1)
+ scales = scales / (scales.max() * scales.min()).sqrt()
+ scales[torch.isinf(scales)] = 1
+ scales[torch.isnan(scales)] = 1
+ scales_view = scales.view(1, -1).to(device)
+
+ if use_parent_forward:
+ # Q(W * s) / s: mutate balance layer weights
+ for bl in mapping.balance_layers:
+ bl.weight.data.copy_(orig_state[bl] * scales_view)
+ w_qdq = self._quantize_dequantize_weight(
+ bl,
+ bl.weight.data.float(),
+ quant_func=cached_quant_func,
+ )
+ if w_qdq is not None:
+ bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)
+ else:
+ bl.weight.data.copy_(orig_state[bl])
+
+ # Collect quantized outputs then compute loss
+ int_w_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)
+ total_loss = self._compute_loss(fp16_outputs, int_w_outputs)
+ del int_w_outputs
+
+ # Restore original weights
+ for bl in mapping.balance_layers:
+ bl.weight.data.copy_(orig_state[bl])
+ else:
+ # Weight-only fallback: || W - Q(W*s)/s ||^2
+ total_loss = 0.0
+ for bl in mapping.balance_layers:
+ w_orig = orig_state[bl].to(device)
+ w_scaled = w_orig * scales_view
+ w_qdq = self._quantize_dequantize_weight(
+ bl,
+ w_scaled,
+ quant_func=cached_quant_func,
+ )
+ if w_qdq is None:
+ total_loss = float("inf")
+ break
+ w_qdq_unscaled = w_qdq / scales_view
+ total_loss += (w_orig - w_qdq_unscaled).pow(2).sum().item()
+
+ if total_loss < best_error:
+ best_error = total_loss
+ best_scales = scales.clone()
+ best_ratio = ratio
+
+ if best_ratio < 0:
+ logger.warning(f"AWQ grid search failed for '{mapping.smooth_name}': " "no finite error found.")
+ return None
+
+ logger.debug(f"AWQ '{mapping.smooth_name}': best_ratio={best_ratio:.2f}, " f"best_error={best_error:.3e}")
+ return best_scales
+
+ # ── Parent module forward ─────────────────────────────
+
+ @torch.no_grad()
+ def _run_parent_samples(
+ self,
+ parent: torch.nn.Module,
+ kwargs_list: list[dict],
+ ) -> list[torch.Tensor]:
+ """Run cached samples through the parent module.
+
+ Feeds cached kwargs through the parent forward without any CUDA
+ synchronisation between batches, so the GPU can pipeline all forwards.
+ Outputs are kept on-device for loss computation.
+ """
+ outputs = []
+ for stored_kwargs in kwargs_list:
+ out = parent(**stored_kwargs)
+ if isinstance(out, tuple):
+ out = out[0]
+ outputs.append(out)
+ return outputs
+
+ @staticmethod
+ @torch.no_grad()
+ def _compute_loss(
+ fp16_outputs: list[torch.Tensor],
+ int_w_outputs: list[torch.Tensor],
+ ) -> float:
+ """Compute normalised MSE between fp16 and quantized outputs."""
+ device = fp16_outputs[0].device
+ loss = torch.tensor(0.0, device=device)
+ num_elements = torch.tensor(0, device=device, dtype=torch.long)
+
+ for fp16_out, int_w_out in zip(fp16_outputs, int_w_outputs):
+ loss += torch.nn.functional.mse_loss(
+ fp16_out.float(),
+ int_w_out.to(fp16_out.device).float(),
+ reduction="sum",
+ )
+ num_elements += fp16_out.numel()
+
+ if num_elements == 0:
+ return float("inf")
+ return (loss / num_elements).item()
+
+ def _quantize_dequantize_weight(
+ self,
+ layer: torch.nn.Module,
+ weight: torch.Tensor,
+ quant_func=None,
+ ) -> torch.Tensor | None:
+ """Quantize-dequantize a weight tensor using the layer's config.
+
+ Args:
+ quant_func: Pre-resolved quantization function. When provided,
+ skips the ``get_quant_func`` dispatch (avoids redundant
+ lookups in the inner grid search loop).
+
+ Returns the quantized-then-dequantized weight, or None on failure.
+ """
+ layer_name = getattr(layer, "global_name", None) or ""
+ config = self.layer_config.get(layer_name, {})
+ bits = config.get("bits", self.bits)
+ group_size = config.get("group_size", self.group_size)
+ sym = config.get("sym", self.sym)
+ data_type = config.get("data_type", self.data_type)
+
+ if quant_func is None:
+ try:
+ quant_func, _ = get_quant_func(
+ data_type,
+ bits,
+ sym,
+ disable_opt_rtn=True, # AWQ always uses plain RTN
+ group_size=group_size,
+ iters=0, # Route to rtn_int_sym, not int_sym
+ )
+ except Exception:
+ return None
+
+ if quant_func is None:
+ return None
+
+ try:
+ qdq_weight, scale, zp = quant_func(
+ weight,
+ bits=bits,
+ group_size=group_size,
+ )
+ return qdq_weight
+ except Exception:
+ return None
+
+ # ── Apply scales ──────────────────────────────────────────────────────────
+
+ @torch.no_grad()
+ def _apply_scales(self, mapping: ResolvedMapping, scales: torch.Tensor) -> None:
+ """Apply the computed AWQ scales to smooth and balance layers.
+
+ - Balance layers (Linear): weight *= scales (along input channels)
+ - Smooth layer (LayerNorm/RMSNorm): weight /= scales, bias /= scales
+ """
+ for bl in mapping.balance_layers:
+ device = bl.weight.device
+ s = scales.to(device).view(1, -1)
+ bl.weight.data.mul_(s)
+
+ smooth = mapping.smooth_layer
+ device = smooth.weight.device
+ s = scales.to(device)
+
+ if smooth.weight.ndim == 1:
+ # LayerNorm / RMSNorm: 1-D weight (per-channel)
+ smooth.weight.data.div_(s)
+ else:
+ # Edge case: when smooth layer's out_features != balance layer's
+ # in_features (e.g. fused qkv_proj smoothing o_proj). Scale the
+ # last output features (aligned with AutoAWQ).
+ # https://github.com/casper-hansen/AutoAWQ/blob/main/awq/quantize/scale.py#L123
+ smooth.weight.data[-s.size(0) :].div_(s.view(-1, 1))
+
+ if hasattr(smooth, "bias") and smooth.bias is not None:
+ smooth.bias.data.div_(s)
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index a5303371a..10ac8332f 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -1014,6 +1014,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
# ── Infrastructure: register act_max hook and run forward pass ──
hook_handles = self.quantizer.register_calibration_hooks(block, imatrix=False)
+ block_input_ids = input_ids # keep reference for quantize_block
input_ids = self.quantizer._get_block_outputs(
block,
input_ids,
@@ -1031,7 +1032,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
self.compress_context.clear_memory()
# ── Pure algorithm ────────────────────────────────────────────
- self.quantizer.quantize_block(block)
+ self.quantizer.quantize_block(block, block_input_ids, input_others, block_name=block_name)
# ── Infrastructure: cleanup ───────────────────────────────────
mv_module_from_gpu(block)
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
index e7f3fd67b..a7018ea71 100644
--- a/auto_round/compressors/entry.py
+++ b/auto_round/compressors/entry.py
@@ -7,6 +7,7 @@ from typing import Any, Callable, Optional, Union
import torch
from auto_round.algorithms.alg_config import AlgConfig
+from auto_round.algorithms.quantization.awq.config import AWQConfig
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.transforms.rotation.config import RotationConfig as _NewArchRotationConfig
@@ -266,6 +267,11 @@ class AutoRound(object):
if isinstance(quant_config, SignRoundConfig):
return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **kwargs)
+ elif isinstance(quant_config, AWQConfig):
+ # AWQ requires calibration for activation collection + smoothing
+ quant_config._alg_cls = "AWQQuantizer"
+ return _get_compressor_class(model_type, CalibratedRTNCompressor)(alg_configs, **local_args, **kwargs)
+
elif isinstance(quant_config, RTNConfig):
enable_imatrix = False
_resolved = {}
@@ -425,11 +431,18 @@ class AutoRoundCompatible:
enable_torch_compile: bool = False,
seed: int = 42,
low_cpu_mem_usage: bool = True,
+ algorithm: str = None,
**kwargs,
):
"""Create AutoRoundCompatible instance using new AutoRound architecture.
This method translates old AutoRoundCompatible API to new AutoRound API.
+
+ Args:
+ algorithm: Quantization algorithm to use. Options:
+ - None or "auto_round": SignSGD-based optimization (default when iters > 0)
+ - "rtn": Round-to-nearest (default when iters == 0)
+ - "awq": Activation-Aware Weight Quantization (AWQ smoothing + RTN)
"""
from auto_round.utils import is_diffusion_model, is_mllm_model
from auto_round.utils.model import is_model_free_route
@@ -481,8 +494,33 @@ class AutoRoundCompatible:
enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)
enable_quanted_input = kwargs.pop("enable_quanted_input", True)
+ # Pop AWQ-only kwargs early so they don't leak into non-AWQ constructors
+ duo_scaling = kwargs.pop("duo_scaling", True)
+ n_grid = kwargs.pop("n_grid", 20)
+ awq_mappings = kwargs.pop("mappings", None)
+
# Decide which algorithm to use
- if iters == 0:
+ if algorithm and algorithm.lower() == "awq":
+ # AWQ mode: activation-aware weight quantization
+ config = AWQConfig(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ data_type=data_type,
+ act_bits=act_bits,
+ act_group_size=act_group_size,
+ act_sym=act_sym,
+ act_data_type=act_data_type,
+ act_dynamic=act_dynamic,
+ duo_scaling=duo_scaling,
+ n_grid=n_grid,
+ seqlen=seqlen,
+ nsamples=nsamples,
+ batch_size=batch_size,
+ mappings=awq_mappings,
+ **common_config_kwargs,
+ )
+ elif (algorithm and algorithm.lower() == "rtn") or iters == 0:
# RTN mode
disable_opt_rtn = kwargs.pop("disable_opt_rtn", None)
config = RTNConfig(
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index b9ffc7749..06c905ec8 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -27,6 +27,7 @@ This document presents step-by-step instructions for auto-round llm quantization
- [API Usage](#api-usage-1)
- [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)
+ [OPT RTN mode](#opt-rtn-mode)
+ + [AWQ Algorithm](#awq-algorithm)
+ [Model-Free Mode](#model-free-mode)
+ [GGUF format](#gguf-format)
+ [Quantization Costs](#quantization-costs)
@@ -322,6 +323,37 @@ W2G64 Average Accuracy of 13 tasks and Time Cost Results(Testing was conducted o
</details>
+### AWQ Algorithm
+
+AWQ (Activation-Aware Weight Quantization) is available as an alternative quantization algorithm. AWQ protects salient weight channels by analyzing activation patterns and applying channel-wise scaling before standard RTN quantization.
+
+The canonical AWQ deployment path is **W4A16** served by vLLM's AWQ/Marlin CUDA kernels. **W8A8** with AWQ smoothing can also be served via vLLM's compressed_tensors backend (cutlass INT8 GEMM).
+
+#### CLI Usage
+
+```bash
+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"
+```
+
+AWQ-specific options:
+- `--duo_scaling`: Use both activations and weights for scaling. Options: `true`, `false`, or `both` (searches both modes and picks the best). (default: True).
+- `--n_grid`: Number of grid points for scaling ratio search (default: 20).
+
+#### API Usage
+
+```python
+from auto_round import AutoRound
+
+ar = AutoRound(
+ "Qwen/Qwen3-0.6B",
+ scheme="INT8",
+ algorithm="awq",
+)
+
+output_dir = "./tmp_awq"
+ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
+```
+
### AutoScheme
AutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md)
@@ -955,5 +987,3 @@ Some VLMs require manual support.
Mamba is not supported.
-
-
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index b990166d8..d74dcd486 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -27,6 +27,7 @@
- [API 用法](#api-用法)
- [AutoScheme 中的超参数](#autoscheme-超参数说明)
+ [OPT RTN 模式](#opt-rtn-模式)
+ + [AWQ 算法](#awq-算法)
+ [免模型量化模式](#免模型量化模式)
+ [GGUF 格式](#gguf-格式量化)
+ [量化成本](#量化成本)
@@ -317,6 +318,40 @@ W2G64 在 13 个任务上的平均精度与耗时
</details>
+### AWQ 算法
+
+AWQ(Activation-Aware Weight Quantization,激活感知权重量化)是一种可选的量化算法。AWQ 通过分析激活模式来保护关键权重通道,在标准量化前对权重施加通道级缩放,从而降低量化误差。
+
+AWQ 的标准部署路径是 **W4A16**,通过 vLLM 的 AWQ/Marlin CUDA 内核提供服务。**W8A8** 搭配 AWQ 平滑化也可通过 vLLM 的 compressed_tensors 后端(cutlass INT8 GEMM)提供服务。
+
+#### 命令行用法
+
+```bash
+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"
+```
+
+AWQ 专用选项:
+- `--duo_scaling`:同时使用激活和权重计算缩放因子。选项:`true`、`false` 或 `both`(搜索两种模式并选择最佳)。(默认:True)。
+- `--n_grid`:缩放比率搜索的网格点数(默认:20)。
+
+#### API 用法
+
+W8A8 搭配 AWQ 平滑化:
+
+```python
+from auto_round import AutoRound
+
+ar = AutoRound(
+ "Qwen/Qwen3-0.6B",
+ scheme="INT8",
+ algorithm="awq",
+)
+
+output_dir = "./tmp_awq"
+ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
+```
+
+
### AutoScheme 自动混合精度量化方案
AutoScheme 采用自动化算法,可生成 **自适应的混合精度与数据类型** 的量化方案(mixed bits/data type quantization recipes)。相关测试结果请参考[《AutoScheme精度报告》](./auto_scheme_acc.md)。
diff --git a/pyproject.toml b/pyproject.toml
index 9fd3b8337..dccbd31ce 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,6 +23,7 @@ extend-exclude = [
"auto_round_extension/ark"
]
[tool.typos.default.extend-words]
+ba = "ba"
ue = "ue"
endianess = "endianess"
thw = "thw"
diff --git a/test/test_cpu/algorithms/__init__.py b/test/test_cpu/algorithms/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/test_cpu/algorithms/test_awq.py b/test/test_cpu/algorithms/test_awq.py
new file mode 100644
index 000000000..85a3e815f
--- /dev/null
+++ b/test/test_cpu/algorithms/test_awq.py
@@ -0,0 +1,294 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CPU CI tests for AWQ (Activation-Aware Weight Quantization) algorithm.
+
+Covers:
+- Normal LLM (OPT-125m): W4A16 quantization, inference, export args verification
+- INT W8A8 export via llm_compressor format with config validation
+- Tiny Qwen MoE: dynamic smoothing, quantized layer checks, quant config saving
+"""
+
+import json
+import os
+import shutil
+
+import pytest
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+from ...helpers import generate_prompt, get_model_path, opt_name_or_path, save_tiny_model
+
+
+class TestAWQNormalLLM:
+ """AWQ quantization on a normal LLM (OPT-125m style tiny model).
+
+ Tests W4A16 quantize → inference → export args validation.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):
+ """W4A16 AWQ quantization produces valid layer_config and model can generate."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ model, layer_config = ar.quantize()
+
+ # Verify quantization happened
+ assert model is not None
+ assert len(layer_config) > 0
+ for name, cfg in layer_config.items():
+ assert cfg["bits"] == 4, f"Layer {name} expected bits=4, got {cfg['bits']}"
+
+ # Inference smoke test
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path)
+ output = generate_prompt(model, tokenizer, device="cpu")
+ assert len(output) > 0, "Model should produce non-empty output"
+
+ def test_awq_w4a16_export_auto_round_format(self, tiny_opt_model_path):
+ """AWQ W4A16 export to auto_round format: verify quantization_config in saved config."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ # Verify config.json has quantization_config
+ config = AutoConfig.from_pretrained(save_path)
+ qconfig = config.quantization_config
+ assert qconfig is not None, "quantization_config should be present"
+ assert qconfig["bits"] == 4
+ assert qconfig["group_size"] == 128
+ assert "auto-round" in qconfig["quant_method"]
+
+ def test_awq_w4a16_export_args_check(self, tiny_opt_model_path):
+ """Verify key export args in saved quantization_config match input parameters."""
+ bits, group_size, sym = 8, 64, True
+ ar = AutoRound(
+ tiny_opt_model_path,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ algorithm="awq",
+ nsamples=2,
+ seqlen=8,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ config = AutoConfig.from_pretrained(save_path)
+ qconfig = config.quantization_config
+ assert qconfig["bits"] == bits
+ assert qconfig["group_size"] == group_size
+ assert qconfig["sym"] == sym
+
+
+class TestAWQW8A8LLMCompressor:
+ """AWQ INT W8A8 quantization with llm_compressor export format.
+
+ Validates compressed-tensors config structure, num_bits, and key fields.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_awq_w8a8_llmc_export(self, tiny_opt_model_path):
+ """W8A8 AWQ → llm_compressor: verify compressed-tensors metadata fields."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="INT8",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")
+
+ config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)
+ qconfig = config.quantization_config
+
+ assert qconfig["quant_method"] == "compressed-tensors"
+
+ group0 = qconfig["config_groups"]["group_0"]
+ # Weight args
+ assert group0["weights"]["num_bits"] == 8
+ assert group0["weights"]["type"] == "int"
+ assert group0["weights"]["symmetric"] is True
+ # Activation args
+ assert group0["input_activations"]["num_bits"] == 8
+ # Targets
+ targets = group0.get("targets")
+ assert targets is not None and len(targets) > 0
+
+ # QuantLinear check: verify saved weights are int8 with per-channel scales
+ from safetensors import safe_open
+
+ st_files = [f for f in os.listdir(save_path) if f.endswith(".safetensors")]
+ assert len(st_files) > 0, f"No safetensors files in {save_path}"
+ with safe_open(os.path.join(save_path, st_files[0]), framework="pt") as f:
+ weight = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight")
+ assert weight.dtype == torch.int8, f"Expected int8 weight, got {weight.dtype}"
+ scale = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight_scale")
+ assert scale.shape[1] == 1, f"Expected per-channel scale shape (out, 1), got {scale.shape}"
+
+
+class TestAWQMoE:
+ """AWQ on a tiny MoE model: dynamic smoothing, layer checks, config saving.
+
+ Uses the session-scoped tiny_qwen_moe_model_path fixture.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):
+ """AWQ dynamic smoothing should resolve mappings on a MoE model without error."""
+ from auto_round.algorithms.quantization.awq.mappings import resolve_mappings
+
+ model = AutoModelForCausalLM.from_pretrained(
+ tiny_qwen_moe_model_path,
+ torch_dtype=torch.bfloat16,
+ device_map="cpu",
+ trust_remote_code=True,
+ )
+ resolved = resolve_mappings(model, user_mappings=None)
+
+ # Must resolve a non-trivial number of mappings
+ assert len(resolved) > 0, "Expected non-empty resolved mappings"
+
+ # Verify smooth layers are unique (no duplicate smoothing targets)
+ smooth_names = [r.smooth_name for r in resolved]
+ assert len(smooth_names) == len(
+ set(smooth_names)
+ ), f"Duplicate smooth names: {[n for n in smooth_names if smooth_names.count(n) > 1]}"
+
+ # Must have attention-related mappings (input_layernorm→qkv, v_proj→o_proj)
+ n_layers = model.config.num_hidden_layers
+ attn_smooths = [n for n in smooth_names if "input_layernorm" in n or "self_attn.v_proj" in n]
+ assert len(attn_smooths) == 2 * n_layers, f"Expected {2 * n_layers} attn smooth layers, got {len(attn_smooths)}"
+
+ # Shared expert up→down should resolve at block level
+ if hasattr(model.model.layers[0].mlp, "shared_expert"):
+ shared_smooths = [n for n in smooth_names if "shared_expert" in n]
+ assert (
+ len(shared_smooths) == n_layers
+ ), f"Expected {n_layers} shared_expert smooth layers, got {len(shared_smooths)}"
+
+ del model
+
+ def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):
+ """AWQ on MoE: expert layers should be quantized, gates/routers stay FP."""
+ ar = AutoRound(
+ tiny_qwen_moe_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ model, layer_config = ar.quantize()
+ assert model is not None
+ assert len(layer_config) > 0
+
+ q4_layers = {k for k, v in layer_config.items() if v["bits"] == 4}
+ fp_layers = {k for k, v in layer_config.items() if v["bits"] >= 16}
+ other_layers = {k: v["bits"] for k, v in layer_config.items() if v["bits"] != 4 and v["bits"] < 16}
+
+ # Tiny Qwen MoE: mlp.gate is a TopKRouter (not Linear) so it's not in layer_config.
+ # Only mlp.shared_expert_gate (a Linear) stays FP → 1 FP gate layer per block.
+ assert len(other_layers) == 0, f"Unexpected bit widths: {other_layers}"
+ n_layers = model.config.num_hidden_layers
+ assert len(fp_layers) == n_layers, (
+ f"Expected {n_layers} FP gate layers (mlp.shared_expert_gate per block), "
+ f"got {len(fp_layers)}: {sorted(fp_layers)}"
+ )
+ assert len(q4_layers) == len(layer_config) - len(
+ fp_layers
+ ), f"Expected {len(layer_config) - len(fp_layers)} W4 layers, got {len(q4_layers)}"
+
+ for name in fp_layers:
+ assert name.endswith("gate"), f"Unexpected FP layer: {name}"
+
+ def test_awq_moe_save_quant_config(self, tiny_qwen_moe_model_path):
+ """AWQ MoE: saved quantization_config should be consistent and loadable."""
+ ar = AutoRound(
+ tiny_qwen_moe_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ # Load and verify config
+ config_path = os.path.join(save_path, "config.json")
+ assert os.path.exists(config_path), f"config.json not found at {save_path}"
+
+ with open(config_path, "r") as f:
+ config_data = json.load(f)
+
+ qconfig = config_data.get("quantization_config")
+ assert qconfig is not None, "quantization_config missing from saved config.json"
+ assert qconfig["bits"] == 4
+ assert qconfig["group_size"] == 128
+ assert "auto-round" in qconfig["quant_method"]
+
+ def test_awq_moe_save_compressed_size(self, tiny_qwen_moe_model_path):
+ """AWQ MoE W4: quantized safetensors should be smaller than original."""
+ ar = AutoRound(
+ tiny_qwen_moe_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ def _safetensors_size(path):
+ return sum(os.path.getsize(os.path.join(path, f)) for f in os.listdir(path) if f.endswith(".safetensors"))
+
+ original_size = _safetensors_size(tiny_qwen_moe_model_path)
+ quantized_size = _safetensors_size(save_path)
+ assert quantized_size > 0, f"No safetensors files in {save_path}"
+
+ ratio = quantized_size / original_size
+ # W4G128 packing: (4-bit weights + scale/zp overhead)
+ assert 0.35 < ratio < 0.65, (
+ f"Compression ratio {ratio:.4f} outside expected range (0.35, 0.65): "
+ f"original={original_size / (1024**2):.1f}MB, quantized={quantized_size / (1024**2):.1f}MB"
+ )
diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.py
index 48a66ba75..9a3082be6 100644
--- a/test/test_cpu/integrations/test_llmc_integration.py
+++ b/test/test_cpu/integrations/test_llmc_integration.py
@@ -1,10 +1,13 @@
+import os
+
import pytest
import torch
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
from llmcompressor import oneshot
from llmcompressor.modifiers.autoround import AutoRoundModifier
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+from auto_round import AutoRound
from auto_round.calib_dataset import get_dataset
from ...envs import multi_card
@@ -239,3 +242,47 @@ def test_rtn_oneshot(recipe, tmp_path, tiny_tiny_llama_model_path):
# Check lm-head is not quantized
not_targeted = model_loaded.lm_head
assert not hasattr(not_targeted, "quantization_scheme")
+
+
+# ---------------------------------------------------------------------------
+# AWQ W8A8 llm_compressor export config args
+# ---------------------------------------------------------------------------
+def test_llmc_awq_w8a8_export_config_args(tiny_opt_model_path, tmp_path):
+ """W8A8 AWQ → llm_compressor: verify compressed-tensors metadata fields."""
+ save_dir = str(tmp_path / "saved")
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="INT8",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")
+
+ config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)
+ qconfig = config.quantization_config
+
+ assert qconfig["quant_method"] == "compressed-tensors"
+
+ group0 = qconfig["config_groups"]["group_0"]
+ # Weight args
+ assert group0["weights"]["num_bits"] == 8
+ assert group0["weights"]["type"] == "int"
+ assert group0["weights"]["symmetric"] is True
+ # Activation args
+ assert group0["input_activations"]["num_bits"] == 8
+ # Targets
+ targets = group0.get("targets")
+ assert targets is not None and len(targets) > 0
+
+ # QuantLinear check: verify saved weights are int8 with per-channel scales
+ from safetensors import safe_open
+
+ st_files = [f for f in os.listdir(save_path) if f.endswith(".safetensors")]
+ assert len(st_files) > 0, f"No safetensors files in {save_path}"
+ with safe_open(os.path.join(save_path, st_files[0]), framework="pt") as f:
+ weight = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight")
+ assert weight.dtype == torch.int8, f"Expected int8 weight, got {weight.dtype}"
+ scale = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight_scale")
+ assert scale.shape[1] == 1, f"Expected per-channel scale shape (out, 1), got {scale.shape}"
diff --git a/test/test_cuda/algorithms/test_awq.py b/test/test_cuda/algorithms/test_awq.py
new file mode 100644
index 000000000..5222a98da
--- /dev/null
+++ b/test/test_cuda/algorithms/test_awq.py
@@ -0,0 +1,257 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CUDA CI tests for AWQ (Activation-Aware Weight Quantization) algorithm.
+
+Covers:
+- Normal LLM (Like OPT-125m): W4A16 quantization, inference, export check
+- INT W8A8: llm_compressor export args, vLLM inference
+- Tiny Qwen MoE: dynamic smoothing, quantized layer checks, quant config saving
+- Accuracy: lm_eval on OPT-125m AWQ W4A16 (lambada_openai, limit=50)
+"""
+
+import json
+import os
+import shutil
+
+import pytest
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+from ...helpers import evaluate_accuracy, generate_prompt, get_model_path, opt_name_or_path, save_tiny_model
+
+# ---------------------------------------------------------------------------
+# Section 1: Normal LLM (OPT-125m) – W4A16 quantize, inference, export args
+# ---------------------------------------------------------------------------
+
+
+class TestAWQLLM:
+ """AWQ W4A16 on a tiny OPT model with CUDA acceleration."""
+
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):
+ """W4A16 AWQ quantization and CUDA inference smoke test."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ model, layer_config = ar.quantize()
+
+ assert model is not None
+ assert len(layer_config) > 0
+ for name, cfg in layer_config.items():
+ assert cfg["bits"] == 4, f"Layer {name} expected bits=4, got {cfg['bits']}"
+
+ # CUDA inference
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path)
+ output = generate_prompt(model, tokenizer, device="cuda")
+ assert len(output) > 0 and "!!!" not in output, f"Unexpected generation output: {output}"
+
+ def test_awq_w4a16_export_auto_round_args(self, tiny_opt_model_path):
+ """Export to auto_round format: verify quantization_config fields."""
+ bits, group_size, sym = 8, 64, True
+ ar = AutoRound(
+ tiny_opt_model_path,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ config = AutoConfig.from_pretrained(save_path)
+ qconfig = config.quantization_config
+ assert qconfig is not None
+ assert qconfig["bits"] == bits
+ assert qconfig["group_size"] == group_size
+ assert qconfig["sym"] == sym
+ assert "auto-round" in qconfig["quant_method"]
+
+ def test_awq_w4a16_load_and_generate(self, tiny_opt_model_path):
+ """Quantize, save, reload, and generate on CUDA to verify round-trip."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ loaded_model = AutoModelForCausalLM.from_pretrained(save_path, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(save_path)
+ output = generate_prompt(loaded_model, tokenizer, device="cuda")
+ assert len(output) > 0 and "!!!" not in output, f"Unexpected generation output: {output}"
+
+
+# ---------------------------------------------------------------------------
+# Section 2: Tiny Qwen MoE – dynamic smoothing, quantized layers, config
+# ---------------------------------------------------------------------------
+
+
+class TestAWQMoE:
+ """AWQ on a tiny MoE model with CUDA: smoothing, layers, config saving."""
+
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):
+ """AWQ mapping resolution works on MoE model."""
+ from auto_round.algorithms.quantization.awq.mappings import resolve_mappings
+
+ model = AutoModelForCausalLM.from_pretrained(
+ tiny_qwen_moe_model_path,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ trust_remote_code=True,
+ )
+ resolved = resolve_mappings(model, user_mappings=None)
+
+ assert len(resolved) > 0, "Expected non-empty resolved mappings"
+
+ # Smooth layers should be unique
+ smooth_names = [r.smooth_name for r in resolved]
+ assert len(smooth_names) == len(
+ set(smooth_names)
+ ), f"Duplicate smooth names: {[n for n in smooth_names if smooth_names.count(n) > 1]}"
+
+ n_layers = model.config.num_hidden_layers
+ attn_smooths = [n for n in smooth_names if "input_layernorm" in n or "self_attn.v_proj" in n]
+ assert len(attn_smooths) == 2 * n_layers, f"Expected {2 * n_layers} attn smooth layers, got {len(attn_smooths)}"
+
+ if hasattr(model.model.layers[0].mlp, "shared_expert"):
+ shared_smooths = [n for n in smooth_names if "shared_expert" in n]
+ assert (
+ len(shared_smooths) == n_layers
+ ), f"Expected {n_layers} shared_expert smooth layers, got {len(shared_smooths)}"
+
+ del model
+
+ def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):
+ """Expert layers quantized to W4, gates/routers stay fp."""
+ ar = AutoRound(
+ tiny_qwen_moe_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ model, layer_config = ar.quantize()
+ assert model is not None
+ assert len(layer_config) > 0
+
+ # Categorize layers
+ q4_layers = {k for k, v in layer_config.items() if v["bits"] == 4}
+ fp_layers = {k for k, v in layer_config.items() if v["bits"] >= 16}
+ other_layers = {k: v["bits"] for k, v in layer_config.items() if v["bits"] != 4 and v["bits"] < 16}
+
+ # Tiny Qwen MoE: mlp.gate is a TopKRouter (not Linear, excluded from
+ # layer_config), mlp.shared_expert_gate is Linear but excluded from
+ # quantization → 1 FP gate layer per block.
+ assert len(other_layers) == 0, f"Unexpected bit widths: {other_layers}"
+ n_layers = model.config.num_hidden_layers
+ assert len(fp_layers) == n_layers, (
+ f"Expected {n_layers} FP gate layers (mlp.shared_expert_gate per block), "
+ f"got {len(fp_layers)}: {sorted(fp_layers)}"
+ )
+ assert len(q4_layers) == len(layer_config) - len(
+ fp_layers
+ ), f"Expected {len(layer_config) - len(fp_layers)} W4 layers, got {len(q4_layers)}"
+
+ for name in fp_layers:
+ assert name.endswith("gate"), f"Unexpected FP layer: {name}"
+
+ def test_awq_moe_save_compressed_size(self, tiny_qwen_moe_model_path):
+ """AWQ MoE W4: quantized safetensors should be smaller than original."""
+ ar = AutoRound(
+ tiny_qwen_moe_model_path,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ def _safetensors_size(path):
+ return sum(os.path.getsize(os.path.join(path, f)) for f in os.listdir(path) if f.endswith(".safetensors"))
+
+ original_size = _safetensors_size(tiny_qwen_moe_model_path)
+ quantized_size = _safetensors_size(save_path)
+ assert quantized_size > 0, f"No safetensors files in {save_path}"
+
+ ratio = quantized_size / original_size
+ assert 0.35 < ratio < 0.65, (
+ f"Compression ratio {ratio:.4f} outside expected range (0.35, 0.65): "
+ f"original={original_size / (1024**2):.1f}MB, quantized={quantized_size / (1024**2):.1f}MB"
+ )
+
+
+class TestAWQEval:
+ """AWQ accuracy evaluation on OPT-125m using lm_eval.
+
+ Uses the full OPT-125m model (not tiny) for meaningful accuracy check.
+ lambada_openai task, add limit for CI speed.
+ """
+
+ @classmethod
+ def setup_class(cls):
+ model_name = opt_name_or_path
+ cls.model_name = model_name
+ cls.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+ @classmethod
+ def teardown_class(cls):
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_awq_w4a16_lmeval(self):
+ """AWQ W4A16 on OPT-125m: lambada_openai accuracy check."""
+ ar = AutoRound(
+ self.model_name,
+ scheme="W4A16",
+ algorithm="awq",
+ nsamples=32,
+ seqlen=32,
+ batch_size=8,
+ )
+ model, _ = ar.quantize()
+
+ evaluate_accuracy(
+ model,
+ self.tokenizer,
+ task="lambada_openai",
+ threshold=0.3,
+ batch_size="auto:8",
+ limit=50,
+ device="cuda",
+ )
diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.py
index e9faa3e14..63636e395 100644
--- a/test/test_cuda/integrations/test_vllm.py
+++ b/test/test_cuda/integrations/test_vllm.py
@@ -175,6 +175,36 @@ class TestVllmEvaluation:
assert ret == 0, f"vllm evaluation with iters=0 failed (rc={ret})"
+ not current_platform.is_cuda(),
+ reason="only supports CUDA backend.",
+)
[email protected]_ci(reason="due to vllm and CT Dependency conflict.")
+def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):
+ """W8A8 AWQ → llm_compressor → vLLM: end-to-end inference test."""
+ save_dir = str(tmp_path / "saved")
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="INT8",
+ algorithm="awq",
+ nsamples=2,
+ seqlen=32,
+ batch_size=2,
+ )
+ _, save_path = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")
+
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=50)
+ llm = LLM(
+ model=save_path,
+ trust_remote_code=True,
+ tensor_parallel_size=1,
+ gpu_memory_utilization=0.8,
+ )
+ outputs = llm.generate(["Hello, my name is"], sampling_params)
+ generated_text = outputs[0].outputs[0].text
+ assert len(generated_text.strip()) > 0 and "!!!" not in generated_text, "vLLM produced empty/meaningless output"
+
+
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="only supports CUDA backend.",