PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Awq algorithm (#1749)
Weiweie8ccff294d74
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 7372916ad..656d7e2cf 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -74,9 +74,19 @@ class BasicArgumentParser(argparse.ArgumentParser):# choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],help="Quantization scheme to use. ""W4A16: 4-bit weights with 16-bit activations (default). "- "Other options include W2A16, W3A16, W8A16 for different bit widths, "+ "Other options include W2A16, W3A16, W8A16, W8A8 for different bit widths, ""and MXFP4/MXFP8/NVFP4 for different data type.",)+ basic.add_argument(+ "--algorithm",+ default=None,+ type=str.lower,+ choices=["auto_round", "rtn", "awq"],+ help="Quantization algorithm to use. "+ "auto_round: SignSGD-based optimization (default when iters > 0). "+ "rtn: Round-to-nearest (default when iters == 0). "+ "awq: Activation-Aware Weight Quantization (AWQ smoothing + RTN).",+ )basic.add_argument("--batch_size","--train_bs",@@ -313,6 +323,22 @@ class BasicArgumentParser(argparse.ArgumentParser):"Use this to force the regular AutoRound flow.",)+ awq_group = self.add_argument_group("AWQ Arguments")+ awq_group.add_argument(+ "--duo_scaling",+ default=True,+ type=lambda s: "both" if s.lower() == "both" else s.lower() in ("true", "1", "yes"),+ help="Whether to use both activations and weights for AWQ scaling. "+ "Options: true/false/'both'. 'both' searches both modes and picks the best. "+ "(default: True).",+ )+ awq_group.add_argument(+ "--n_grid",+ default=20,+ type=int,+ help="Number of grid points for AWQ scaling ratio search (default: 20).",+ )+scheme = self.add_argument_group("Scheme Arguments")scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")scheme.add_argument(@@ -767,6 +793,12 @@ def tune(args):rotation_config=rot_config,model_free=args.model_free,disable_model_free=args.disable_model_free,+ algorithm=getattr(args, "algorithm", None),+ **(+ {"duo_scaling": args.duo_scaling, "n_grid": args.n_grid}+ if getattr(args, "algorithm", None) == "awq"+ else {}+ ),)# ======================= Quantize and save model =======================diff --git a/auto_round/algorithms/quantization/__init__.py b/auto_round/algorithms/quantization/__init__.pyindex 3d8ca932b..5b297df1d 100644--- a/auto_round/algorithms/quantization/__init__.py+++ b/auto_round/algorithms/quantization/__init__.py@@ -18,5 +18,7 @@ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizerfrom auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizerfrom auto_round.algorithms.quantization.adam_round.adam import AdamRoundQuantizer+from auto_round.algorithms.quantization.awq.config import AWQConfig+from auto_round.algorithms.quantization.awq.quantizer import AWQQuantizerfrom auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer, OptimizedRTNQuantizerdiff --git a/auto_round/algorithms/quantization/awq/__init__.py b/auto_round/algorithms/quantization/awq/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/algorithms/quantization/awq/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/algorithms/quantization/awq/config.py b/auto_round/algorithms/quantization/awq/config.pynew file mode 100644index 000000000..50340eeda--- /dev/null+++ b/auto_round/algorithms/quantization/awq/config.py@@ -0,0 +1,77 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.algorithms.quantization.config import QuantizationConfig+++class AWQConfig(QuantizationConfig):+ """Configuration for AWQ (Activation-Aware Weight Quantization).++ AWQ protects salient weight channels by analyzing activation patterns and+ applying channel-wise scaling to reduce quantization error. The scaling+ factors are computed offline via a grid search over calibration data.+ After smoothing, standard RTN quantization is applied to the adjusted weights.++ Args:+ duo_scaling: Use both activations and weights for the scaling factor.+ True: always use duo scaling. False: only activation scaling.+ "both": search both modes and pick the best.+ n_grid: Number of grid points for the scaling-ratio search.+ seqlen: Calibration sequence length.+ nsamples: Number of calibration samples. Grid search time scales+ linearly with nsamples (each batch triggers one parent forward+ per grid point).+ batch_size: Batch size for calibration forward passes.+ apply_smooth: Whether to apply AWQ smoothing (channel scaling).+ True: run both smoothing and quantization (default AWQ behavior).+ False: skip smoothing, only run RTN quantization.+ mappings: Explicit AWQ mappings. Each mapping is a dict with keys+ ``smooth_layer`` (str) and ``balance_layers`` (list[str]).+ If None, mappings are inferred automatically from the model structure.+ **kwargs: Forwarded to ``QuantizationConfig`` (bits, group_size, sym, …).+ """++ _alg_cls = "AWQQuantizer"++ def __init__(+ self,+ *,+ duo_scaling: bool | str = True,+ n_grid: int = 20,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ apply_smooth: bool = True,+ mappings: list[dict] = None,+ **kwargs,+ ):+ super().__init__(**kwargs)++ if isinstance(duo_scaling, str) and duo_scaling != "both":+ raise ValueError(f"duo_scaling must be True, False, or 'both', got '{duo_scaling}'")+ self.duo_scaling = duo_scaling+ self.n_grid = n_grid+ self.seqlen = seqlen+ self.nsamples = nsamples+ self.batch_size = batch_size+ self.apply_smooth = apply_smooth+ self.mappings = mappings+ self.infer_bs_coeff = 1+ self.batch_dim = None++ # TODO adjust those args defaults after architecture refactoring:+ self.enable_quanted_input = False # AWQ doesn't cascade quantized block outputs+ # AWQ uses plain RTN (no iterative optimization) for the quantization step.+ self.disable_opt_rtn = True+ self.orig_disable_opt_rtn = Truediff --git a/auto_round/algorithms/quantization/awq/mappings.py b/auto_round/algorithms/quantization/awq/mappings.pynew file mode 100644index 000000000..32617632f--- /dev/null+++ b/auto_round/algorithms/quantization/awq/mappings.py@@ -0,0 +1,624 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# This file contains AWQ mapping definitions and registry structure adapted from+# llm-compressor (https://github.com/vllm-project/llm-compressor), originally+# developed by Neural Magic, Inc. and licensed under the Apache License 2.0.+# The mapping patterns, model-class registry, and AWQMapping/ResolvedMapping+# data structures are aligned with llm-compressor's AWQ modifier so that+# auto-round AWQ produces models compatible with vllm's AWQ inference kernels.+# Reference: llmcompressor/modifiers/awq/mappings.py++"""AWQ layer mapping resolution++Defines the relationship between *smooth layers* (whose output channels are+inversely scaled) and *balance layers* (whose input channels are scaled up) for+the AWQ smoothing algorithm.++Mappings are resolved in order of priority:+ 1. User-provided explicit mappings (``AWQConfig.mappings``).+ 2. Model-class-name registry (``AWQ_MAPPING_REGISTRY``), matching+ llm-compressor's(vLLM) AWQ architecture support.+ 3. ``default_mappings`` fallback for unknown Llama-like architectures.+"""++from __future__ import annotations++import re+from dataclasses import dataclass++import torch++from auto_round.logger import logger+from auto_round.utils.model import is_moe_model++# ── Data structures ───────────────────────────────────────────────────────────+++@dataclass+class AWQMapping:+ """Declarative mapping: smooth_layer regex → list of balance_layer regexes.+ Aligned with ``llmcompressor.modifiers.awq.mappings.AWQMapping``.+ """++ smooth_layer: str+ balance_layers: list[str]+ activation_hook_target: str | None = None+++@dataclass+class ResolvedMapping:+ """A fully resolved AWQ mapping with concrete module references."""++ smooth_name: str+ smooth_layer: torch.nn.Module+ balance_names: list[str]+ balance_layers: list[torch.nn.Module]+ parent_name: str+ parent: torch.nn.Module+ activation_hook_target: str | None = None+++# ── Mapping definitions ─────────────────────────+# Reference: vllm-project/llm-compressor src/llmcompressor/modifiers/awq/mappings.py++default_mappings = [+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++_gemma_mappings = [+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(r"pre_feedforward_layernorm$", [r"gate_proj$", r"up_proj$"]),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# Cohere: MLP runs in parallel with attention; both fed by input_layernorm.+_cohere_mappings = [+ AWQMapping(+ r"input_layernorm$",+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$", r"mlp\.gate_proj$", r"mlp\.up_proj$"],+ ),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# Phi: fused qkv_proj and gate_up_proj layers.+_phi_mappings = [+ AWQMapping(r"input_layernorm$", [r"qkv_proj$"]),+ AWQMapping(r"qkv_proj$", [r"o_proj$"]),+ AWQMapping(r"post_attention_layernorm$", [r"gate_up_proj$"]),+ AWQMapping(r"gate_up_proj$", [r"down_proj$"]),+]++# OPT: uses self_attn_layer_norm / final_layer_norm / out_proj / fc1 / fc2.+_opt_mappings = [+ AWQMapping(r"self_attn_layer_norm$", [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$"]),+ AWQMapping(r"self_attn\.v_proj$", [r"self_attn\.out_proj$"]),+ AWQMapping(r"final_layer_norm$", [r"fc1$"]),+ AWQMapping(r"fc1$", [r"fc2$"]),+]++# Bloom: different naming convention.+# Note: query_key_value → dense mapping, see+# https://github.com/mit-han-lab/llm-awq/issues/2#issuecomment-1606297469+_bloom_mappings = [+ AWQMapping(r"input_layernorm$", [r"query_key_value$"]),+ AWQMapping(r"post_attention_layernorm$", [r"dense_h_to_4h$"]),+ AWQMapping(r"gelu_impl$", [r"dense_4h_to_h$"]),+]++# DeepSeek V2/V3: multi-head latent attention with compressed KV.+_deepseek_mappings = [+ AWQMapping(r"input_layernorm$", [r"(q|q_a)_proj$", r"kv_a_proj_with_mqa$"]),+ AWQMapping(r"q_a_layernorm$", [r"q_b_proj$"]),+ AWQMapping(r"kv_a_layernorm$", [r"kv_b_proj$"]),+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# MoE default: expert-parallel gate/up projections.+_moe_default_mappings = [+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(+ r"post_attention_layernorm$",+ [r"mlp\.experts\.\d+\.gate_proj$", r"mlp\.experts\.\d+\.up_proj$"],+ ),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# Llama4: uses feed_forward instead of mlp, and includes router.+_llama4_default_mappings = [+ AWQMapping(r"input_layernorm$", [r"q_proj$", r"k_proj$", r"v_proj$"]),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(+ r"post_attention_layernorm$",+ [r"feed_forward\.router$", r"feed_forward\..*gate_proj$", r"feed_forward\..*up_proj$"],+ ),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# Exaone4 / Olmo3: only v_proj→o_proj and up_proj→down_proj smoothing.+_exaone4_mappings = [+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# AFMOE: dual normalization — pre_mlp_layernorm feeds MLP, attention has gate_proj.+_afmoe_mappings = [+ AWQMapping(+ r"input_layernorm$",+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$", r"self_attn\.gate_proj$"],+ ),+ AWQMapping(r"v_proj$", [r"o_proj$"]),+ AWQMapping(+ r"pre_mlp_layernorm$",+ [r"mlp\..*gate_proj$", r"mlp\..*up_proj$"],+ ),+ AWQMapping(r"up_proj$", [r"down_proj$"]),+]++# ── Model class name → mappings registry ──────────────────────────────────────+# Aligned with llm-compressor AWQ_MAPPING_REGISTRY (llmcompressor v0.10.0).+# Models not in this registry fall back to default_mappings.++AWQ_MAPPING_REGISTRY: dict[str, list[AWQMapping]] = {+ # AFMOE+ "AfmoeForCausalLM": _afmoe_mappings,+ # Bloom+ "BloomForCausalLM": _bloom_mappings,+ # OPT+ "OPTForCausalLM": _opt_mappings,+ # Cohere / Command-R+ "CohereForCausalLM": _cohere_mappings,+ "Cohere2ForCausalLM": _cohere_mappings,+ "Cohere2VisionForConditionalGeneration": _cohere_mappings,+ # DeepSeek V2/V3+ "DeepseekV3ForCausalLM": _deepseek_mappings,+ # Exaone4 / Olmo3+ "Exaone4ForCausalLM": _exaone4_mappings,+ # Gemma 2/3+ "Gemma2ForCausalLM": _gemma_mappings,+ "Gemma3ForCausalLM": _gemma_mappings,+ "Gemma3ForConditionalGeneration": _gemma_mappings,+ # Llama+ "LlamaForCausalLM": default_mappings,+ "Llama4ForConditionalGeneration": _llama4_default_mappings,+ # Mistral+ "MistralForCausalLM": default_mappings,+ "Mistral3ForConditionalGeneration": default_mappings,+ # Olmo3 (same as Exaone4)+ "Olmo3ForCausalLM": _exaone4_mappings,+ # Phi+ "Phi3ForCausalLM": _phi_mappings,+ "Phi3VForCausalLM": _phi_mappings,+ # Qwen+ "Qwen2ForCausalLM": default_mappings,+ "Qwen2_5OmniThinkerForConditionalGeneration": default_mappings,+ "Qwen3ForCausalLM": default_mappings,+ # Qwen MoE+ "Qwen2MoeForCausalLM": _moe_default_mappings,+ "Qwen3MoeForCausalLM": _moe_default_mappings,+ # GLM+ "Glm4MoeForCausalLM": _moe_default_mappings,+ "GlmMoeDsaForCausalLM": _deepseek_mappings,+ # Other models using default mappings+ "SeedOssForCausalLM": default_mappings,+ "Ernie4_5_MoeForCausalLM": default_mappings,+}+++# ── Helper functions ──────────────────────────────────────────────────────────+++def _get_module(model: torch.nn.Module, name: str) -> torch.nn.Module | None:+ """Safely retrieve a sub-module by dotted name."""+ try:+ parts = name.split(".")+ m = model+ for p in parts:+ m = getattr(m, p)+ return m+ except AttributeError:+ return None+++def _find_parent(model: torch.nn.Module, names: list[str]) -> tuple[str, torch.nn.Module]:+ """Find the lowest common ancestor module for a list of module names."""+ if not names:+ return "", model++ parts_list = [n.split(".") for n in names]+ common = []+ for level_parts in zip(*parts_list):+ if len(set(level_parts)) == 1:+ common.append(level_parts[0])+ else:+ break++ ancestor_name = ".".join(common)+ if ancestor_name:+ ancestor = _get_module(model, ancestor_name)+ else:+ ancestor = model+ return ancestor_name, ancestor+++def _extract_block_prefix(name: str) -> str | None:+ """Extract the transformer block prefix from a module name.++ E.g., "model.layers.5.input_layernorm" → "model.layers.5"+ """+ match = re.match(r"(.*\.\d+)\.", name)+ return match.group(1) if match else None+++def _get_model_class_name(model: torch.nn.Module) -> str:+ """Get the model class name, handling nested model wrappers."""+ return type(model).__name__+++# ── Dynamic mapping builders ─────────────────────────────────────────────────+# For hybrid attention models (mix of full self-attention and linear/Gated+# DeltaNet attention) that need layer-index-specific AWQ mappings.+++def _get_hybrid_attention_config(model: torch.nn.Module) -> tuple[list[str], int] | None:+ """Extract layer_types and num_hidden_layers from a hybrid attention model.++ Checks text_config for VL models, then top-level config.+ Returns (layer_types, num_hidden_layers) or None if not hybrid.+ """+ config = getattr(model, "config", None)+ if config is None:+ return None++ text_config = getattr(config, "text_config", config)+ layer_types = getattr(text_config, "layer_types", None)+ num_layers = getattr(text_config, "num_hidden_layers", None)++ if layer_types is None or num_layers is None:+ return None++ if "full_attention" not in layer_types or "linear_attention" not in layer_types:+ return None++ return layer_types, num_layers+++def _detect_linear_attn_projections(model: torch.nn.Module) -> list[str]:+ """Detect linear attention projection names from the model.++ Different architectures use different projection layouts:+ - Qwen3Next: in_proj_qkvz, in_proj_ba+ - Qwen3.5: in_proj_qkv, in_proj_z, in_proj_b, in_proj_a+ """+ proj_names = []+ for name, _ in model.named_modules():+ if ".linear_attn." not in name:+ continue+ sub = name.rsplit("linear_attn.", 1)[-1]+ if sub.startswith("in_proj_"):+ proj_names.append(sub)+ return list(dict.fromkeys(proj_names))+++def _build_hybrid_attention_mappings(model: torch.nn.Module) -> list[AWQMapping] | None:+ """Dynamically build AWQ mappings for hybrid attention models.++ Reads ``layer_types`` from model config to determine which layers use+ full vs linear attention, then inspects module names to detect the correct+ linear attention projection names and MLP structure (MoE vs dense).+ """+ result = _get_hybrid_attention_config(model)+ if result is None:+ return None++ layer_types, num_layers = result++ full_indices = [i for i in range(num_layers) if layer_types[i] == "full_attention"]+ linear_indices = [i for i in range(num_layers) if layer_types[i] == "linear_attention"]++ if not full_indices or not linear_indices:+ logger.warning(+ "Hybrid attention model detected but missing indices for "+ "both full and linear attention layers. Falling back."+ )+ return None++ full_re = "|".join(str(i) for i in full_indices)+ linear_re = "|".join(str(i) for i in linear_indices)++ linear_proj_names = _detect_linear_attn_projections(model)+ is_moe = is_moe_model(model)++ mappings = []++ # Full attention layers: input_layernorm → q/k/v_proj+ mappings.append(+ AWQMapping(+ rf"layers\.({full_re})\.input_layernorm$",+ [r"self_attn\.q_proj$", r"self_attn\.k_proj$", r"self_attn\.v_proj$"],+ )+ )++ # Linear attention layers: input_layernorm → linear_attn projections+ if linear_proj_names:+ mappings.append(+ AWQMapping(+ rf"layers\.({linear_re})\.input_layernorm$",+ [rf"linear_attn\.{p}$" for p in linear_proj_names],+ )+ )++ # MLP: MoE vs dense+ if is_moe:+ mappings.append(+ AWQMapping(+ r"post_attention_layernorm$",+ [+ r"mlp\.experts\.\d+\.gate_proj$",+ r"mlp\.experts\.\d+\.up_proj$",+ r"mlp\.shared_expert\.gate_proj$",+ r"mlp\.shared_expert\.up_proj$",+ ],+ )+ )+ else:+ mappings.append(+ AWQMapping(r"post_attention_layernorm$", [r"gate_proj$", r"up_proj$"]),+ )++ mappings.append(AWQMapping(r"up_proj$", [r"down_proj$"]))++ logger.warning(+ f"Built dynamic hybrid-attention AWQ mappings: "+ f"{len(full_indices)} full-attention, {len(linear_indices)} linear-attention, "+ f"projections={linear_proj_names}, MoE={is_moe}"+ )+ return mappings+++AWQ_DYNAMIC_MAPPING_REGISTRY: dict[str, callable] = {+ "Qwen3NextForCausalLM": _build_hybrid_attention_mappings,+ "Qwen3_5ForCausalLM": _build_hybrid_attention_mappings,+ "Qwen3_5ForConditionalGeneration": _build_hybrid_attention_mappings,+ "Qwen3_5MoeForCausalLM": _build_hybrid_attention_mappings,+ "Qwen3_5MoeForConditionalGeneration": _build_hybrid_attention_mappings,+}+++def _get_mappings_for_model(model: torch.nn.Module) -> list[AWQMapping]:+ """Look up mappings for a model from the registry, with default fallback.++ Resolution order:+ 1. Dynamic registry — runtime-generated per-layer mappings+ (e.g. hybrid attention models like Qwen3.5 MoE).+ 2. Static registry — ``AWQ_MAPPING_REGISTRY``.+ 3. ``default_mappings`` — Llama-like fallback.+ """+ cls_name = _get_model_class_name(model)++ if cls_name in AWQ_DYNAMIC_MAPPING_REGISTRY:+ mappings = AWQ_DYNAMIC_MAPPING_REGISTRY[cls_name](model)+ if mappings is not None:+ return mappings++ if cls_name in AWQ_MAPPING_REGISTRY:+ logger.info(f"Using registered AWQ mappings for {cls_name}.")+ return AWQ_MAPPING_REGISTRY[cls_name]++ logger.info(+ f"Architecture '{cls_name}' not found in AWQ mapping registry. " f"Using default mappings (Llama-like)."+ )+ return default_mappings+++# ── Public API ────────────────────────────────────────────────────────────────+++def resolve_mappings(+ model: torch.nn.Module,+ user_mappings: list[dict] | None = None,+) -> list[ResolvedMapping]:+ """Resolve AWQ mappings for the given model.++ Resolution order:+ 1. ``user_mappings`` — explicit dicts with ``smooth_layer`` /+ ``balance_layers`` regex keys.+ 2. ``AWQ_MAPPING_REGISTRY`` — model-class-name lookup+ 3. ``default_mappings`` — Llama-like fallback.++ Returns:+ List of ``ResolvedMapping`` objects ready for AWQ grid search.+ """+ if user_mappings is not None:+ mapping_defs = [AWQMapping(m["smooth_layer"], m["balance_layers"]) for m in user_mappings]+ else:+ mapping_defs = _get_mappings_for_model(model)++ return _resolve_mapping_defs(model, mapping_defs)+++def _resolve_mapping_defs(+ model: torch.nn.Module,+ mapping_defs: list[AWQMapping],+) -> list[ResolvedMapping]:+ """Resolve a list of AWQMapping definitions against the model."""+ resolved = []+ all_names = [n for n, _ in model.named_modules()]++ # Group modules by block prefix+ block_modules: dict[str, list[str]] = {}+ for name in all_names:+ prefix = _extract_block_prefix(name)+ if prefix is not None:+ block_modules.setdefault(prefix, []).append(name)++ if not block_modules:+ logger.warning(+ "AWQ found no repeating block structure in the model. "+ "Provide explicit mappings via AWQConfig(mappings=[...])."+ )+ return resolved++ matched_count = 0++ for prefix, names_in_block in block_modules.items():+ for mapping_def in mapping_defs:+ # Find smooth layer(s) in this block+ smooth_matches = [n for n in names_in_block if re.search(mapping_def.smooth_layer, n)]+ if not smooth_matches:+ continue++ for smooth_name in smooth_matches:+ smooth_layer = _get_module(model, smooth_name)+ if smooth_layer is None:+ continue+ if not hasattr(smooth_layer, "weight"):+ continue++ # Find balance layers in the same block+ balance_names = []+ balance_layers = []+ for bp in mapping_def.balance_layers:+ for n in names_in_block:+ if re.search(bp, n):+ m = _get_module(model, n)+ if m is not None and isinstance(m, torch.nn.Linear):+ balance_names.append(n)+ balance_layers.append(m)++ # Fallback: search child blocks for MoE cross-level mappings+ # (e.g., post_attention_layernorm → experts.N.gate_proj/up_proj)+ if not balance_layers:+ for bp in mapping_def.balance_layers:+ for child_prefix, child_names in block_modules.items():+ if child_prefix != prefix and child_prefix.startswith(prefix + "."):+ for n in child_names:+ if re.search(bp, n):+ m = _get_module(model, n)+ if m is not None and isinstance(m, torch.nn.Linear):+ balance_names.append(n)+ balance_layers.append(m)++ if not balance_layers:+ continue++ # Verify dimensional compatibility (filters out GQA+ # mismatches for v_proj → o_proj automatically)+ smooth_dim = smooth_layer.weight.shape[0]+ compatible = all(bl.in_features == smooth_dim for bl in balance_layers)+ if not compatible:+ logger.warning_once(+ f"Skipping AWQ for '{smooth_name}': incompatible "+ f"balance layers (smooth_dim={smooth_dim}, "+ f"balance in_features="+ f"{[bl.in_features for bl in balance_layers]})"+ )+ continue++ parent_name, parent = _find_parent(model, balance_names)+ resolved.append(+ ResolvedMapping(+ smooth_name=smooth_name,+ smooth_layer=smooth_layer,+ balance_names=balance_names,+ balance_layers=balance_layers,+ parent_name=parent_name,+ parent=parent,+ activation_hook_target=mapping_def.activation_hook_target,+ )+ )+ matched_count += 1++ if matched_count == 0:+ logger.warning(+ "AWQ resolved 0 mappings. The model architecture may not match "+ "any known pattern. Provide explicit mappings via "+ "AWQConfig(mappings=[...])."+ )+ else:+ first_prefix = next(iter(block_modules))+ n_blocks = len(block_modules)+ mappings_per_block = sum(1 for r in resolved if r.smooth_name.startswith(first_prefix))+ logger.info(+ f"AWQ resolved {matched_count} smooth-balance mappings "+ f"({mappings_per_block} per block × {n_blocks} blocks)."+ )++ return resolved+++# ── Model compatibility diagnostics ───────────────────────────────────────────+++def check_model_compatibility(+ model: torch.nn.Module,+ user_mappings: list[dict] | None = None,+) -> dict:+ """Check AWQ compatibility and return a diagnostic report.++ Returns a dict with:+ - ``compatible`` (bool): True if at least one mapping was resolved.+ - ``n_mappings`` (int): Number of resolved mappings.+ - ``mappings_per_block`` (int): Mappings in the first block.+ - ``n_blocks`` (int): Number of transformer blocks.+ - ``model_class`` (str): Model class name.+ - ``in_registry`` (bool): Whether model class is in AWQ_MAPPING_REGISTRY.+ - ``warnings`` (list[str]): Any compatibility warnings.+ """+ warnings_list = []+ cls_name = _get_model_class_name(model)+ in_registry = cls_name in AWQ_MAPPING_REGISTRY++ if not in_registry and user_mappings is None:+ warnings_list.append(+ f"Model class '{cls_name}' is not in AWQ_MAPPING_REGISTRY. "+ f"Using default Llama-like mappings. If quantization quality is "+ f"poor, provide explicit mappings via AWQConfig(mappings=[...])."+ )++ resolved = resolve_mappings(model, user_mappings)++ all_prefixes = set()+ for r in resolved:+ prefix = _extract_block_prefix(r.smooth_name)+ if prefix:+ all_prefixes.add(prefix)++ n_blocks = len(all_prefixes)+ mappings_per_block = 0+ if n_blocks > 0 and resolved:+ first_prefix = min(all_prefixes)+ mappings_per_block = sum(1 for r in resolved if r.smooth_name.startswith(first_prefix))++ if not resolved:+ warnings_list.append(+ "No AWQ mappings could be resolved. The model architecture may " "not be supported for auto-detection."+ )++ return {+ "compatible": len(resolved) > 0,+ "n_mappings": len(resolved),+ "mappings_per_block": mappings_per_block,+ "n_blocks": n_blocks,+ "model_class": cls_name,+ "in_registry": in_registry,+ "warnings": warnings_list,+ }diff --git a/auto_round/algorithms/quantization/awq/quantizer.py b/auto_round/algorithms/quantization/awq/quantizer.pynew file mode 100644index 000000000..f6aca2f75--- /dev/null+++ b/auto_round/algorithms/quantization/awq/quantizer.py@@ -0,0 +1,935 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""AWQ (Activation-Aware Weight Quantization) quantizer.++The algorithm:+1. Collects per-channel activation magnitudes during calibration.+2. For each smooth-balance mapping, performs a grid search over scaling ratios+ to find the one that minimizes quantization error (output-based loss).+3. Applies the best channel-wise scaling: balance_layer.weight *= scales,+ smooth_layer.weight /= scales (or smooth_layer.bias /= scales if 1-D).+++Reference implementations:+ - AutoAWQ: https://github.com/casper-hansen/AutoAWQ+ - llm-compressor: https://github.com/vllm-project/llm-compressor+"""++from __future__ import annotations++import contextlib+import inspect+import traceback++import torch+from tqdm import tqdm++from auto_round.algorithms.quantization.awq.config import AWQConfig+from auto_round.algorithms.quantization.awq.mappings import (+ ResolvedMapping,+ _extract_block_prefix,+ resolve_mappings,+)+from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.compressors.shard_writer import ShardWriter+from auto_round.compressors.utils import immediate_pack+from auto_round.data_type.utils import (+ get_quant_func,+ reshape_pad_tensor_by_group_size,+ revert_tensor_by_pad,+ update_block_global_scale_if_needed,+)+from auto_round.logger import logger+from auto_round.utils import (+ check_to_quantized,+ clear_memory,+ convert_module_to_hp_if_necessary,+ get_module,+ set_amax_for_all_moe_layers,+ set_module,+)+from auto_round.wrapper import WrapperLinear+++class AWQQuantizer(BaseQuantizers):+ """AWQ quantizer: activation-aware channel scaling + delegated quantization.++ AWQ is a pre-quantization transform that applies channel-wise scaling to reduce+ quantization error. The actual quantization is delegated to an inner+ quantizer (currently RTN by default).+ """++ def __init__(self, config: AWQConfig):+ super().__init__(config)+ self.duo_scaling = config.duo_scaling+ self.n_grid = config.n_grid+ self.apply_smooth = config.apply_smooth++ # Populated during calibration+ self._user_mappings = config.mappings+ self._resolved_mappings: list[ResolvedMapping] = []+ self._block_groups: dict[str, list[ResolvedMapping]] = {}+ self._activation_stats: dict[str, list[torch.Tensor]] = {}+ # Parent module kwargs cache: parent_module → list of kwargs dicts+ self._parent_args_cache: dict[torch.nn.Module, list[dict]] = {}+ self._parent_signatures: dict[int, inspect.BoundArguments] = {}+ self._smoothing_applied: bool = False++ # Fail fast: validate scheme at construction time+ self._check_scheme_compatibility()++ def __setattr__(self, name, value):+ """Trigger model-level AWQ setup when compress_context is assigned."""+ super().__setattr__(name, value)+ if name == "compress_context" and value is not None:+ self._prepare_model()++ # ── Public API ────────────────────────────────────────────────────────────++ def resolve_all_mappings(self, model: torch.nn.Module) -> dict[str, list[ResolvedMapping]]:+ """Resolve all AWQ mappings and group by block prefix.++ Call this once before the block-by-block loop. The resolved mappings+ are stored internally and used by ``register_activation_hooks`` and+ ``smooth_block`` for per-block filtering.++ Returns:+ Dictionary mapping block prefix (e.g. "model.layers.0") to the+ list of resolved mappings for that block.+ """+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)+ block_groups: dict[str, list[ResolvedMapping]] = {}+ for m in self._resolved_mappings:+ prefix = _extract_block_prefix(m.smooth_name)+ block_groups.setdefault(prefix, []).append(m)+ self._block_groups = block_groups+ return block_groups++ def apply_smoothing(+ self,+ model: torch.nn.Module,+ device: torch.device | str | None = None,+ ) -> None:+ """Apply AWQ smoothing to the model weights.++ This is the core AWQ step: for each mapping, find the best scaling+ factor and apply it to smooth/balance layers.++ When *device* is provided the model is assumed to reside on CPU and+ only the modules needed for each mapping are temporarily moved to+ *device* during the grid search. This dramatically reduces peak VRAM+ (~14 GB savings for an 8B model).++ Should be called AFTER calibration data has been collected and BEFORE+ RTN quantization.+ """+ if self._smoothing_applied:+ return++ # Resolve mappings+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)+ if not self._resolved_mappings:+ logger.warning("No AWQ mappings resolved; skipping smoothing.")+ self._smoothing_applied = True+ return++ if not self._activation_stats:+ logger.error("No activation statistics collected for AWQ smoothing.")++ logger.info(f"Applying AWQ smoothing to {len(self._resolved_mappings)} mappings...")++ for mapping in tqdm(self._resolved_mappings, desc="AWQ Smoothing"):+ if mapping.smooth_name not in self._activation_stats:+ logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping this layer.")+ continue++ act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)+ if act_count == 0:+ logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping this layer.")+ continue++ # Mean activation per channel (act_count is a plain int now,+ # avoiding a CPU tensor allocation per channel).+ x_mean = (act_sum / act_count).to(torch.float32)+ del act_sum++ with self._align_modules(mapping, device):+ best_scales = self._grid_search_scales(mapping, x_mean)++ if best_scales is not None:+ self._apply_scales(mapping, best_scales)++ # Eagerly free per-mapping parent cache to reduce RAM+ parent = mapping.parent+ self._parent_args_cache.pop(parent, None)++ self._activation_stats.clear()+ self._parent_args_cache.clear()+ self._parent_signatures.clear()+ self._smoothing_applied = True+ logger.info("AWQ smoothing complete.")++ def smooth_block(self, block_prefix: str) -> None:+ """Apply AWQ smoothing for one block's mappings.++ Assumes activation stats and parent kwargs have already been collected+ for this block via ``register_activation_hooks`` + block forward.+ The block's modules are expected to be on the compute device, so+ no device alignment is needed (unlike ``apply_smoothing`` which uses+ ``_align_modules`` for CPU-offloaded models).++ Args:+ block_prefix: Block prefix (e.g. "model.layers.0") identifying+ which mappings to smooth.+ """+ block_mappings = [m for m in self._resolved_mappings if _extract_block_prefix(m.smooth_name) == block_prefix]+ for mapping in block_mappings:+ if mapping.smooth_name not in self._activation_stats:+ logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping.")+ continue++ act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)+ if act_count == 0:+ logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping.")+ continue++ x_mean = (act_sum / act_count).to(torch.float32)+ del act_sum++ # Block is on the compute device — grid search runs in-place+ best_scales = self._grid_search_scales(mapping, x_mean)++ if best_scales is not None:+ self._apply_scales(mapping, best_scales)++ # Free parent kwargs cache after ALL mappings for this block are done.+ # (All mappings in a block typically share the same parent module;+ # popping inside the loop would break output-based loss for+ # subsequent mappings.)+ seen_parents = set()+ for mapping in block_mappings:+ pid = id(mapping.parent)+ if pid not in seen_parents:+ seen_parents.add(pid)+ self._parent_args_cache.pop(mapping.parent, None)++ # ── Model-level initialization ─────────────────────────────────────────────++ def _prepare_model(self) -> None:+ """One-time model-level AWQ setup: check compatibility and resolve mappings.++ Triggered automatically when model_context is assigned (via __setattr__).+ """+ from auto_round.algorithms.quantization.awq.mappings import check_model_compatibility++ model = self.model_context.model+ report = check_model_compatibility(model, self._user_mappings)+ for w in report["warnings"]:+ logger.warning(w)+ if not report["compatible"]:+ model_class = report.get("model_class", "unknown")+ raise ValueError(+ f"AWQ: no smooth-balance mappings could be resolved for "+ f"'{model_class}'. Either the model architecture is not "+ f"supported for automatic AWQ mapping detection, or the model "+ f"has no repeating transformer block structure. "+ f"You can provide explicit mappings via "+ f"AWQConfig(mappings=[{{'smooth_layer': '<regex>', "+ f"'balance_layers': ['<regex>', ...]}}])."+ )++ self.resolve_all_mappings(model)++ # AWQ caches block I/O on CPU — only one block lives on GPU at a time.+ self.compress_context.cache_device = torch.device("cpu")++ def _check_scheme_compatibility(self) -> None:+ """Validate the quantization scheme against AWQ inference backends."""+ bits = self.bits+ act_bits = self.act_bits or 16+ data_type = self.data_type or "int"++ if "int" not in data_type:+ raise ValueError(+ f"AWQ requires integer data_type, got '{data_type}'. "+ f"AWQ channel scaling is designed for integer quantization "+ f"grids. Use algorithm='autoround' for FP8/MXFP quantization."+ )++ if act_bits is not None and act_bits < 16 and act_bits != 8:+ raise ValueError(+ f"AWQ with act_bits={act_bits} is not supported. "+ f"No inference kernel exists for W{bits}A{act_bits} in vllm "+ f"or sglang. Supported schemes: W4A16 (canonical AWQ) or "+ f"W8A8 (compressed_tensors INT8 backend)."+ )++ if bits == 4 and act_bits >= 16:+ pass+ elif bits == 8 and act_bits == 8:+ logger.info(+ "AWQ with W8A8: AWQ smoothing will be applied, followed by "+ "INT8 quantization. This is served by vllm's "+ "compressed_tensors backend (cutlass INT8 GEMM), not AWQ "+ "kernels."+ )+ elif bits not in (4, 8):+ logger.warning(+ f"AWQ with bits={bits}: vllm AWQ kernels only support "+ f"bits=4 (AWQ/Marlin) natively. bits=8 is supported via "+ f"compressed_tensors. Other bit widths may not have "+ f"optimized inference kernels."+ )++ # ── Weight quantization (delegated RTN) ─────────────────────────────────────++ def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:+ """Quantize a single layer using RTN after AWQ smoothing has been applied.++ AWQ's quantize_layer is simpler than RTN's because:+ - AWQ always uses plain RTN (disable_opt_rtn=True, no imatrix)+ - No MoE-specific RTN disabling (AWQ handles MoE via mappings)+ - No GGUF special path (AWQ targets AWQ/Marlin inference kernels)++ Args:+ name: Fully-qualified module name (e.g. "model.layers.0.self_attn.q_proj").+ dtype: Optional dtype to cast the layer to before quantization.+ """+ m = get_module(self.model, name)+ if dtype is not None:+ m = m.to(dtype)++ m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)+ set_module(self.model, name, m)+ tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.compress_context.device++ try:+ m = m.to(tuning_device)+ m = WrapperLinear(+ m,+ device=tuning_device,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=self.config.disable_opt_rtn,+ iters=0,+ )+ m = m.unwrapper({})+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ m = m.orig_layer if hasattr(m, "orig_layer") else m+ try:+ logger.error(cuda_error_msg)+ logger.warning("AWQ quantize_layer: falling back to CPU.")+ m.to("cpu")+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=self.config.disable_opt_rtn,+ iters=0,+ )+ m = m.unwrapper({})+ except Exception:+ raise++ set_module(self.model, name, m)+ self._immediate_pack_and_save_module(name)++ def quantize_layer_outside_block(self, *args, **kwargs):+ """Quantize layers outside blocks (e.g. lm_head). Delegates to quantize_layer."""+ return self.quantize_layer(*args, **kwargs)++ def _immediate_pack_and_save_module(self, module_name: str) -> None:+ """Pack and/or save a quantized module if immediate mode is enabled."""+ shard_writer = ShardWriter.get_shard_writer()+ to_cpu = self.compress_context.low_gpu_mem_usage+ module = get_module(self.model, module_name)+ if self.compress_context.is_immediate_packing:+ immediate_pack(module_name, self.layer_config)+ if to_cpu:+ module = module.to("cpu")+ packed_module = get_module(self.model, module_name)+ set_module(self.model, module_name, packed_module.to("cpu"))+ else:+ if to_cpu:+ module = module.to("cpu")+ set_module(self.model, module_name, module)+ if self.compress_context.is_immediate_saving:+ module = get_module(self.model, module_name)+ module.to("cpu")+ shard_writer.write(module, module_name, False)+ module.to("meta")++ # ── Block-level quantization ──────────────────────────────────────────────++ def quantize_block(+ self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs+ ) -> dict:+ """AWQ block quantization: collect stats → smooth → quantize.++ The full per-block AWQ pipeline:+ 1. Register activation hooks for this block's mappings+ 2. Run block forward to collect activation stats + parent kwargs+ 3. Apply AWQ smoothing (grid search + scale application)+ 4. Collect act_max AFTER smoothing (for activation quantization)+ 5. Quantize the smoothed block via RTN++ Args:+ block: Module already on the compute device.+ input_ids: Calibration inputs for this block.+ input_others: Additional kwargs for block forward.+ reference_output: FP16 block output (kept for interface consistency).+ **kwargs: Must include 'block_name' for mapping lookup.+ """+ block_name = kwargs.get("block_name")+ if block_name is None:+ # Infer block_name from resolved mappings by matching the block's modules+ for prefix, mappings in self._block_groups.items():+ if any(m.smooth_layer is mod or m.parent is mod for m in mappings for mod in block.modules()):+ block_name = prefix+ break+ if block_name is None:+ raise ValueError(+ "AWQQuantizer.quantize_block() could not determine block_name. "+ "Pass block_name explicitly or ensure resolved mappings cover this block."+ )++ model = self.model+ bs = self.batch_size * self.infer_bs_coeff+ if self.compress_context.low_gpu_mem_usage:+ bs = 1+ logger.info("AWQ: low_gpu_mem_usage enabled, setting inference batch size to 1.")++ # Step 1 & 2: AWQ smoothing (optional)+ if self.apply_smooth:+ awq_hooks = self.register_activation_hooks(model, block_prefix=block_name)+ self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)+ for h in awq_hooks:+ h.remove()++ self.smooth_block(block_name)++ # Step 3: Collect act_max AFTER smoothing+ # AWQ smoothing changes internal activations (LayerNorm output /= scales),+ # so act_max must be collected post-smoothing. Reset any pre-smoothing+ # act_max values first to avoid stale data persisting via max().+ act_max_hooks = self.register_calibration_hooks(block, imatrix=False)+ if act_max_hooks:+ for _name, m in block.named_modules():+ if hasattr(m, "act_max"):+ del m.act_max+ self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)+ for h in act_max_hooks:+ h.remove()++ # Step 4: Quantize the smoothed block (delegated RTN)+ # This is equivalent to RTNQuantizer.quantize_block but inlined here+ # to avoid inheritance coupling.+ update_block_global_scale_if_needed(block, self.data_type, self.group_size)+ if (+ self.config.is_act_nv_fp+ or self.config.is_static_afp8+ or (self.config.is_wfp8afp8 and not self.config.act_dynamic)+ ):+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ for _name, m in block.named_modules():+ if hasattr(m, "global_name") and check_to_quantized(m):+ self.quantize_layer(m.global_name)++ return {}++ # ── Module alignment (device onload/offload) ─────────────────────────────++ @contextlib.contextmanager+ def _align_modules(+ self,+ mapping,+ device: torch.device | str | None,+ ):+ """Temporarily move mapping-related modules to *device* for grid search.++ If *device* is None (model already on the compute device), this is a+ no-op passthrough.+ """+ if device is None:+ yield+ return++ modules_to_move = [mapping.parent, mapping.smooth_layer] + list(mapping.balance_layers)+ # Deduplicate (parent may overlap with smooth/balance)+ seen = set()+ unique = []+ for m in modules_to_move:+ if id(m) not in seen:+ seen.add(id(m))+ unique.append(m)++ # Onload+ for m in unique:+ m.to(device)++ try:+ yield+ finally:+ # Offload back to CPU+ for m in unique:+ m.to("cpu")+ clear_memory()++ # ── Activation statistics collection ──────────────────────────────────────++ def register_activation_hooks(+ self,+ model: torch.nn.Module,+ block_prefix: str | None = None,+ ) -> list:+ """Register activation hooks for AWQ and return hook handles.++ Hooks are registered for:+ 1. Smooth layers: accumulate abs activation sums/counts for x_mean.+ 2. Parent modules: cache forward kwargs for output-based loss++ Args:+ model: The model (hooks are registered on the actual module+ objects, which are shared with blocks obtained via+ ``get_module``).+ block_prefix: When provided, only register hooks for mappings+ whose smooth layer belongs to this block (e.g.+ "model.layers.0"). Used in the block-by-block pipeline.++ Should be called by the compressor before calibration, and handles+ should be removed after calibration.+ """+ if not self._resolved_mappings:+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)++ mappings = self._resolved_mappings+ if block_prefix is not None:+ mappings = [m for m in mappings if _extract_block_prefix(m.smooth_name) == block_prefix]++ hooks = []+ smooth_names = {m.smooth_name for m in mappings}++ # Hook smooth layers for activation statistics+ for name, module in model.named_modules():+ if name not in smooth_names:+ continue++ def _make_hook(layer_name):+ def hook_fn(mod, args, output):+ if isinstance(output, tuple):+ x = output[0]+ else:+ x = output+ if x is None or x.numel() == 0:+ return++ # Compute abs-sum directly on GPU without materialising a+ # full float32 intermediate. flatten→abs→sum produces a+ # single [channels] tensor; .cpu() immediately frees the+ # GPU allocation (aligned with AR's "immediate CPU move"+ # pattern from CalibCompressor hooks).+ channel_sum = x.detach().float().flatten(0, -2).abs().sum(dim=0).cpu()+ count = x.shape[:-1].numel()++ if layer_name not in self._activation_stats:+ self._activation_stats[layer_name] = [+ torch.zeros_like(channel_sum),+ 0,+ ]+ self._activation_stats[layer_name][0] += channel_sum+ self._activation_stats[layer_name][1] += count++ return hook_fn++ h = module.register_forward_hook(_make_hook(name))+ hooks.append(h)++ # Hook parent modules to cache kwargs for output-based loss+ parent_modules_hooked = set()+ for mapping in mappings:+ parent = mapping.parent+ if id(parent) in parent_modules_hooked:+ continue+ parent_modules_hooked.add(id(parent))++ self._parent_args_cache[parent] = []++ def _make_parent_hook(parent_module):+ def hook_fn(mod, args, kwargs):+ # Cache ALL calibration batches for output-based grid+ # search loss, use all calibration data without subsampling.+ mod_cls_id = id(type(mod))+ if mod_cls_id not in self._parent_signatures:+ self._parent_signatures[mod_cls_id] = inspect.signature(mod.forward)+ sig = self._parent_signatures[mod_cls_id]+ bound = sig.bind(*args, **kwargs)+ bound.apply_defaults()++ # Infer the parent's compute dtype so that tensors+ # upcasted by nn.LayerNorm (float32) are stored in the+ # weight dtype (e.g. bfloat16). This avoids repeated+ # per-sample casting in the grid search stage.+ param = next(mod.parameters(), None)+ w_dtype = param.dtype if param is not None else None++ stored = {}+ for k, v in bound.arguments.items():+ if isinstance(v, torch.Tensor):+ v = v.detach()+ if w_dtype is not None and v.is_floating_point() and v.dtype != w_dtype:+ v = v.to(w_dtype)+ stored[k] = v+ elif isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v):+ # Detach tensors in tuples (e.g. position_embeddings+ # = (cos, sin)) to release computation graph refs.+ stored[k] = tuple(+ (+ (+ t.detach().to(w_dtype)+ if w_dtype and t.is_floating_point() and t.dtype != w_dtype+ else t.detach()+ )+ if isinstance(t, torch.Tensor)+ else t+ )+ for t in v+ )+ elif hasattr(v, "key_cache"):+ # Null out KV cache objects (DynamicCache etc.)+ stored[k] = None+ else:+ stored[k] = v++ self._parent_args_cache[parent_module].append(stored)++ return hook_fn++ h = parent.register_forward_pre_hook(_make_parent_hook(parent), with_kwargs=True)+ hooks.append(h)++ return hooks++ # ── Grid search ───────────────────────────────────────────────────────────++ def _get_grid_search_params(self) -> list[tuple[float, bool]]:+ """Get grid search parameters (ratio, duo_scaling).++ Returns:+ List of (ratio, use_duo_scaling) tuples for the grid search.+ """+ match self.duo_scaling:+ # "both": half grid with duo off, half with duo on+ case "both":+ n_grid = max(int(self.n_grid / 2), 2)+ return [+ (grid_idx / (n_grid - 1), duo_scaling)+ for grid_idx in range(n_grid)+ for duo_scaling in [False, True]+ ]+ case False:+ n_grid = max(self.n_grid, 2)+ return [(grid_idx / (n_grid - 1), False) for grid_idx in range(n_grid)]+ # True: include identity (0.0, False) as first, then duo points+ case True:+ n_grid = max(self.n_grid, 3)+ return [(0.0, False)] + [(grid_idx / (n_grid - 2), True) for grid_idx in range(n_grid - 1)]+ case _:+ raise ValueError(f"Found unexpected duo_scaling configuration {self.duo_scaling}")++ @staticmethod+ def _compute_layer_means(layers: list[torch.nn.Module], group_size: int) -> torch.Tensor:+ """Compute per-channel mean of normalised weights.++ Within each quantization group, weights are normalized by their group max+ (so values are on a 0-1 scale), then averaged across all groups to get+ per-channel importance.++ Args:+ layers: Balance layers whose weights are concatenated.+ group_size: Quantization group size. If -1, uses full channel width.++ Returns:+ Per-channel mean of normalised weights [in_features].+ """+ # Concatenate all balance layer weights [total_out, in_features]+ weight = torch.cat([m.weight.detach().float() for m in layers], dim=0)+ org_shape = weight.shape++ # Determine effective group size+ gs = group_size if group_size > 0 else org_shape[1]++ # Pad when needed so AWQ works with layers whose input width is not a+ # multiple of group_size, matching grouped RTN quantization behavior.+ weight, _, pad_len = reshape_pad_tensor_by_group_size(weight, gs)+ # Normalize within each group: abs / max (0-1 scale per group)+ w_scale = weight.abs() / (weight.abs().amax(dim=1, keepdim=True) + 1e-6)+ # Remove padding, then take mean across output channels.+ w_scale = revert_tensor_by_pad(w_scale, orig_shape=org_shape, pad_len=pad_len)+ w_mean = w_scale.mean(0)+ return w_mean++ @torch.no_grad()+ def _grid_search_scales(+ self,+ mapping: ResolvedMapping,+ x_mean: torch.Tensor,+ ) -> torch.Tensor | None:+ """Find the best scaling ratio via grid search.++ Uses output-based error:+ ``L(s) = || fp16_output - Q(W*s) @ (X/s) ||^2``++ For each candidate scaling ratio, applies scales to balance layer+ weights, quantize-dequantizes them, runs all cached calibration+ batches through the parent module, and computes the output MSE+ against the fp16 baseline.++ Returns:+ Best scales tensor, or None if no valid scale was found.+ """+ device = mapping.balance_layers[0].weight.device+ x_mean = x_mean.to(device)++ # Compute normalised weight means+ group_size = self.group_size if self.group_size > 0 else -1+ if self.duo_scaling is not False:+ w_mean = self._compute_layer_means(mapping.balance_layers, group_size).to(device)++ # Try to run parent module forward for output-based loss+ parent_kwargs_list = self._parent_args_cache.get(mapping.parent, [])+ use_parent_forward = len(parent_kwargs_list) > 0++ if use_parent_forward:+ # Compute fp16 baseline outputs for loss computation+ fp16_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)+ if not fp16_outputs or all(f.numel() == 0 for f in fp16_outputs):+ use_parent_forward = False++ # Save original weights for restoration during grid search+ orig_state = {bl: bl.weight.data.clone() for bl in mapping.balance_layers}++ best_error = float("inf")+ best_scales = None+ best_ratio = -1++ # Pre-resolve quant function once.+ ref_layer = mapping.balance_layers[0]+ ref_name = getattr(ref_layer, "global_name", None) or ""+ ref_cfg = self.layer_config.get(ref_name, {})+ try:+ cached_quant_func, _ = get_quant_func(+ ref_cfg.get("data_type", self.data_type),+ ref_cfg.get("bits", self.bits),+ ref_cfg.get("sym", self.sym),+ disable_opt_rtn=True,+ group_size=ref_cfg.get("group_size", self.group_size),+ iters=0,+ )+ except Exception:+ cached_quant_func = None++ grid_params = self._get_grid_search_params()++ for ratio, use_duo in grid_params:+ # Compute scales+ if use_duo:+ scales = (x_mean.pow(ratio) / (w_mean.pow(1 - ratio) + 1e-4)).clamp(min=1e-4)+ else:+ scales = x_mean.pow(ratio).clamp(min=1e-4).view(-1)+ scales = scales / (scales.max() * scales.min()).sqrt()+ scales[torch.isinf(scales)] = 1+ scales[torch.isnan(scales)] = 1+ scales_view = scales.view(1, -1).to(device)++ if use_parent_forward:+ # Q(W * s) / s: mutate balance layer weights+ for bl in mapping.balance_layers:+ bl.weight.data.copy_(orig_state[bl] * scales_view)+ w_qdq = self._quantize_dequantize_weight(+ bl,+ bl.weight.data.float(),+ quant_func=cached_quant_func,+ )+ if w_qdq is not None:+ bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)+ else:+ bl.weight.data.copy_(orig_state[bl])++ # Collect quantized outputs then compute loss+ int_w_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)+ total_loss = self._compute_loss(fp16_outputs, int_w_outputs)+ del int_w_outputs++ # Restore original weights+ for bl in mapping.balance_layers:+ bl.weight.data.copy_(orig_state[bl])+ else:+ # Weight-only fallback: || W - Q(W*s)/s ||^2+ total_loss = 0.0+ for bl in mapping.balance_layers:+ w_orig = orig_state[bl].to(device)+ w_scaled = w_orig * scales_view+ w_qdq = self._quantize_dequantize_weight(+ bl,+ w_scaled,+ quant_func=cached_quant_func,+ )+ if w_qdq is None:+ total_loss = float("inf")+ break+ w_qdq_unscaled = w_qdq / scales_view+ total_loss += (w_orig - w_qdq_unscaled).pow(2).sum().item()++ if total_loss < best_error:+ best_error = total_loss+ best_scales = scales.clone()+ best_ratio = ratio++ if best_ratio < 0:+ logger.warning(f"AWQ grid search failed for '{mapping.smooth_name}': " "no finite error found.")+ return None++ logger.debug(f"AWQ '{mapping.smooth_name}': best_ratio={best_ratio:.2f}, " f"best_error={best_error:.3e}")+ return best_scales++ # ── Parent module forward ─────────────────────────────++ @torch.no_grad()+ def _run_parent_samples(+ self,+ parent: torch.nn.Module,+ kwargs_list: list[dict],+ ) -> list[torch.Tensor]:+ """Run cached samples through the parent module.++ Feeds cached kwargs through the parent forward without any CUDA+ synchronisation between batches, so the GPU can pipeline all forwards.+ Outputs are kept on-device for loss computation.+ """+ outputs = []+ for stored_kwargs in kwargs_list:+ out = parent(**stored_kwargs)+ if isinstance(out, tuple):+ out = out[0]+ outputs.append(out)+ return outputs++ @staticmethod+ @torch.no_grad()+ def _compute_loss(+ fp16_outputs: list[torch.Tensor],+ int_w_outputs: list[torch.Tensor],+ ) -> float:+ """Compute normalised MSE between fp16 and quantized outputs."""+ device = fp16_outputs[0].device+ loss = torch.tensor(0.0, device=device)+ num_elements = torch.tensor(0, device=device, dtype=torch.long)++ for fp16_out, int_w_out in zip(fp16_outputs, int_w_outputs):+ loss += torch.nn.functional.mse_loss(+ fp16_out.float(),+ int_w_out.to(fp16_out.device).float(),+ reduction="sum",+ )+ num_elements += fp16_out.numel()++ if num_elements == 0:+ return float("inf")+ return (loss / num_elements).item()++ def _quantize_dequantize_weight(+ self,+ layer: torch.nn.Module,+ weight: torch.Tensor,+ quant_func=None,+ ) -> torch.Tensor | None:+ """Quantize-dequantize a weight tensor using the layer's config.++ Args:+ quant_func: Pre-resolved quantization function. When provided,+ skips the ``get_quant_func`` dispatch (avoids redundant+ lookups in the inner grid search loop).++ Returns the quantized-then-dequantized weight, or None on failure.+ """+ layer_name = getattr(layer, "global_name", None) or ""+ config = self.layer_config.get(layer_name, {})+ bits = config.get("bits", self.bits)+ group_size = config.get("group_size", self.group_size)+ sym = config.get("sym", self.sym)+ data_type = config.get("data_type", self.data_type)++ if quant_func is None:+ try:+ quant_func, _ = get_quant_func(+ data_type,+ bits,+ sym,+ disable_opt_rtn=True, # AWQ always uses plain RTN+ group_size=group_size,+ iters=0, # Route to rtn_int_sym, not int_sym+ )+ except Exception:+ return None++ if quant_func is None:+ return None++ try:+ qdq_weight, scale, zp = quant_func(+ weight,+ bits=bits,+ group_size=group_size,+ )+ return qdq_weight+ except Exception:+ return None++ # ── Apply scales ──────────────────────────────────────────────────────────++ @torch.no_grad()+ def _apply_scales(self, mapping: ResolvedMapping, scales: torch.Tensor) -> None:+ """Apply the computed AWQ scales to smooth and balance layers.++ - Balance layers (Linear): weight *= scales (along input channels)+ - Smooth layer (LayerNorm/RMSNorm): weight /= scales, bias /= scales+ """+ for bl in mapping.balance_layers:+ device = bl.weight.device+ s = scales.to(device).view(1, -1)+ bl.weight.data.mul_(s)++ smooth = mapping.smooth_layer+ device = smooth.weight.device+ s = scales.to(device)++ if smooth.weight.ndim == 1:+ # LayerNorm / RMSNorm: 1-D weight (per-channel)+ smooth.weight.data.div_(s)+ else:+ # Edge case: when smooth layer's out_features != balance layer's+ # in_features (e.g. fused qkv_proj smoothing o_proj). Scale the+ # last output features (aligned with AutoAWQ).+ # https://github.com/casper-hansen/AutoAWQ/blob/main/awq/quantize/scale.py#L123+ smooth.weight.data[-s.size(0) :].div_(s.view(-1, 1))++ if hasattr(smooth, "bias") and smooth.bias is not None:+ smooth.bias.data.div_(s)diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex a5303371a..10ac8332f 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -1014,6 +1014,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):# ── Infrastructure: register act_max hook and run forward pass ──hook_handles = self.quantizer.register_calibration_hooks(block, imatrix=False)+ block_input_ids = input_ids # keep reference for quantize_blockinput_ids = self.quantizer._get_block_outputs(block,input_ids,@@ -1031,7 +1032,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):self.compress_context.clear_memory()# ── Pure algorithm ────────────────────────────────────────────- self.quantizer.quantize_block(block)+ self.quantizer.quantize_block(block, block_input_ids, input_others, block_name=block_name)# ── Infrastructure: cleanup ───────────────────────────────────mv_module_from_gpu(block)diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex e7f3fd67b..a7018ea71 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -7,6 +7,7 @@ from typing import Any, Callable, Optional, Unionimport torchfrom auto_round.algorithms.alg_config import AlgConfig+from auto_round.algorithms.quantization.awq.config import AWQConfigfrom auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.transforms.rotation.config import RotationConfig as _NewArchRotationConfig@@ -266,6 +267,11 @@ class AutoRound(object):if isinstance(quant_config, SignRoundConfig):return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **kwargs)+ elif isinstance(quant_config, AWQConfig):+ # AWQ requires calibration for activation collection + smoothing+ quant_config._alg_cls = "AWQQuantizer"+ return _get_compressor_class(model_type, CalibratedRTNCompressor)(alg_configs, **local_args, **kwargs)+elif isinstance(quant_config, RTNConfig):enable_imatrix = False_resolved = {}@@ -425,11 +431,18 @@ class AutoRoundCompatible:enable_torch_compile: bool = False,seed: int = 42,low_cpu_mem_usage: bool = True,+ algorithm: str = None,**kwargs,):"""Create AutoRoundCompatible instance using new AutoRound architecture.This method translates old AutoRoundCompatible API to new AutoRound API.++ Args:+ algorithm: Quantization algorithm to use. Options:+ - None or "auto_round": SignSGD-based optimization (default when iters > 0)+ - "rtn": Round-to-nearest (default when iters == 0)+ - "awq": Activation-Aware Weight Quantization (AWQ smoothing + RTN)"""from auto_round.utils import is_diffusion_model, is_mllm_modelfrom auto_round.utils.model import is_model_free_route@@ -481,8 +494,33 @@ class AutoRoundCompatible:enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)enable_quanted_input = kwargs.pop("enable_quanted_input", True)+ # Pop AWQ-only kwargs early so they don't leak into non-AWQ constructors+ duo_scaling = kwargs.pop("duo_scaling", True)+ n_grid = kwargs.pop("n_grid", 20)+ awq_mappings = kwargs.pop("mappings", None)+# Decide which algorithm to use- if iters == 0:+ if algorithm and algorithm.lower() == "awq":+ # AWQ mode: activation-aware weight quantization+ config = AWQConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ duo_scaling=duo_scaling,+ n_grid=n_grid,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ mappings=awq_mappings,+ **common_config_kwargs,+ )+ elif (algorithm and algorithm.lower() == "rtn") or iters == 0:# RTN modedisable_opt_rtn = kwargs.pop("disable_opt_rtn", None)config = RTNConfig(diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex b9ffc7749..06c905ec8 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -27,6 +27,7 @@ This document presents step-by-step instructions for auto-round llm quantization- [API Usage](#api-usage-1)- [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)+ [OPT RTN mode](#opt-rtn-mode)+ + [AWQ Algorithm](#awq-algorithm)+ [Model-Free Mode](#model-free-mode)+ [GGUF format](#gguf-format)+ [Quantization Costs](#quantization-costs)@@ -322,6 +323,37 @@ W2G64 Average Accuracy of 13 tasks and Time Cost Results(Testing was conducted o</details>+### AWQ Algorithm++AWQ (Activation-Aware Weight Quantization) is available as an alternative quantization algorithm. AWQ protects salient weight channels by analyzing activation patterns and applying channel-wise scaling before standard RTN quantization.++The canonical AWQ deployment path is **W4A16** served by vLLM's AWQ/Marlin CUDA kernels. **W8A8** with AWQ smoothing can also be served via vLLM's compressed_tensors backend (cutlass INT8 GEMM).++#### CLI Usage++```bash+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"+```++AWQ-specific options:+- `--duo_scaling`: Use both activations and weights for scaling. Options: `true`, `false`, or `both` (searches both modes and picks the best). (default: True).+- `--n_grid`: Number of grid points for scaling ratio search (default: 20).++#### API Usage++```python+from auto_round import AutoRound++ar = AutoRound(+ "Qwen/Qwen3-0.6B",+ scheme="INT8",+ algorithm="awq",+)++output_dir = "./tmp_awq"+ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")+```+### AutoSchemeAutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md)@@ -955,5 +987,3 @@ Some VLMs require manual support.Mamba is not supported.--diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex b990166d8..d74dcd486 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -27,6 +27,7 @@- [API 用法](#api-用法)- [AutoScheme 中的超参数](#autoscheme-超参数说明)+ [OPT RTN 模式](#opt-rtn-模式)+ + [AWQ 算法](#awq-算法)+ [免模型量化模式](#免模型量化模式)+ [GGUF 格式](#gguf-格式量化)+ [量化成本](#量化成本)@@ -317,6 +318,40 @@ W2G64 在 13 个任务上的平均精度与耗时</details>+### AWQ 算法++AWQ(Activation-Aware Weight Quantization,激活感知权重量化)是一种可选的量化算法。AWQ 通过分析激活模式来保护关键权重通道,在标准量化前对权重施加通道级缩放,从而降低量化误差。++AWQ 的标准部署路径是 **W4A16**,通过 vLLM 的 AWQ/Marlin CUDA 内核提供服务。**W8A8** 搭配 AWQ 平滑化也可通过 vLLM 的 compressed_tensors 后端(cutlass INT8 GEMM)提供服务。++#### 命令行用法++```bash+auto-round --model Qwen/Qwen3-0.6B --scheme "W4A16" --algorithm awq --format "auto_round"+```++AWQ 专用选项:+- `--duo_scaling`:同时使用激活和权重计算缩放因子。选项:`true`、`false` 或 `both`(搜索两种模式并选择最佳)。(默认:True)。+- `--n_grid`:缩放比率搜索的网格点数(默认:20)。++#### API 用法++W8A8 搭配 AWQ 平滑化:++```python+from auto_round import AutoRound++ar = AutoRound(+ "Qwen/Qwen3-0.6B",+ scheme="INT8",+ algorithm="awq",+)++output_dir = "./tmp_awq"+ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")+```++### AutoScheme 自动混合精度量化方案AutoScheme 采用自动化算法,可生成 **自适应的混合精度与数据类型** 的量化方案(mixed bits/data type quantization recipes)。相关测试结果请参考[《AutoScheme精度报告》](./auto_scheme_acc.md)。diff --git a/pyproject.toml b/pyproject.tomlindex 9fd3b8337..dccbd31ce 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -23,6 +23,7 @@ extend-exclude = ["auto_round_extension/ark"][tool.typos.default.extend-words]+ba = "ba"ue = "ue"endianess = "endianess"thw = "thw"diff --git a/test/test_cpu/algorithms/__init__.py b/test/test_cpu/algorithms/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/test/test_cpu/algorithms/test_awq.py b/test/test_cpu/algorithms/test_awq.pynew file mode 100644index 000000000..85a3e815f--- /dev/null+++ b/test/test_cpu/algorithms/test_awq.py@@ -0,0 +1,294 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""CPU CI tests for AWQ (Activation-Aware Weight Quantization) algorithm.++Covers:+- Normal LLM (OPT-125m): W4A16 quantization, inference, export args verification+- INT W8A8 export via llm_compressor format with config validation+- Tiny Qwen MoE: dynamic smoothing, quantized layer checks, quant config saving+"""++import json+import os+import shutil++import pytest+import torch+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound++from ...helpers import generate_prompt, get_model_path, opt_name_or_path, save_tiny_model+++class TestAWQNormalLLM:+ """AWQ quantization on a normal LLM (OPT-125m style tiny model).++ Tests W4A16 quantize → inference → export args validation.+ """++ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):+ """W4A16 AWQ quantization produces valid layer_config and model can generate."""+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ model, layer_config = ar.quantize()++ # Verify quantization happened+ assert model is not None+ assert len(layer_config) > 0+ for name, cfg in layer_config.items():+ assert cfg["bits"] == 4, f"Layer {name} expected bits=4, got {cfg['bits']}"++ # Inference smoke test+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path)+ output = generate_prompt(model, tokenizer, device="cpu")+ assert len(output) > 0, "Model should produce non-empty output"++ def test_awq_w4a16_export_auto_round_format(self, tiny_opt_model_path):+ """AWQ W4A16 export to auto_round format: verify quantization_config in saved config."""+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ # Verify config.json has quantization_config+ config = AutoConfig.from_pretrained(save_path)+ qconfig = config.quantization_config+ assert qconfig is not None, "quantization_config should be present"+ assert qconfig["bits"] == 4+ assert qconfig["group_size"] == 128+ assert "auto-round" in qconfig["quant_method"]++ def test_awq_w4a16_export_args_check(self, tiny_opt_model_path):+ """Verify key export args in saved quantization_config match input parameters."""+ bits, group_size, sym = 8, 64, True+ ar = AutoRound(+ tiny_opt_model_path,+ bits=bits,+ group_size=group_size,+ sym=sym,+ algorithm="awq",+ nsamples=2,+ seqlen=8,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ config = AutoConfig.from_pretrained(save_path)+ qconfig = config.quantization_config+ assert qconfig["bits"] == bits+ assert qconfig["group_size"] == group_size+ assert qconfig["sym"] == sym+++class TestAWQW8A8LLMCompressor:+ """AWQ INT W8A8 quantization with llm_compressor export format.++ Validates compressed-tensors config structure, num_bits, and key fields.+ """++ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_awq_w8a8_llmc_export(self, tiny_opt_model_path):+ """W8A8 AWQ → llm_compressor: verify compressed-tensors metadata fields."""+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="INT8",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")++ config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)+ qconfig = config.quantization_config++ assert qconfig["quant_method"] == "compressed-tensors"++ group0 = qconfig["config_groups"]["group_0"]+ # Weight args+ assert group0["weights"]["num_bits"] == 8+ assert group0["weights"]["type"] == "int"+ assert group0["weights"]["symmetric"] is True+ # Activation args+ assert group0["input_activations"]["num_bits"] == 8+ # Targets+ targets = group0.get("targets")+ assert targets is not None and len(targets) > 0++ # QuantLinear check: verify saved weights are int8 with per-channel scales+ from safetensors import safe_open++ st_files = [f for f in os.listdir(save_path) if f.endswith(".safetensors")]+ assert len(st_files) > 0, f"No safetensors files in {save_path}"+ with safe_open(os.path.join(save_path, st_files[0]), framework="pt") as f:+ weight = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight")+ assert weight.dtype == torch.int8, f"Expected int8 weight, got {weight.dtype}"+ scale = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight_scale")+ assert scale.shape[1] == 1, f"Expected per-channel scale shape (out, 1), got {scale.shape}"+++class TestAWQMoE:+ """AWQ on a tiny MoE model: dynamic smoothing, layer checks, config saving.++ Uses the session-scoped tiny_qwen_moe_model_path fixture.+ """++ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):+ """AWQ dynamic smoothing should resolve mappings on a MoE model without error."""+ from auto_round.algorithms.quantization.awq.mappings import resolve_mappings++ model = AutoModelForCausalLM.from_pretrained(+ tiny_qwen_moe_model_path,+ torch_dtype=torch.bfloat16,+ device_map="cpu",+ trust_remote_code=True,+ )+ resolved = resolve_mappings(model, user_mappings=None)++ # Must resolve a non-trivial number of mappings+ assert len(resolved) > 0, "Expected non-empty resolved mappings"++ # Verify smooth layers are unique (no duplicate smoothing targets)+ smooth_names = [r.smooth_name for r in resolved]+ assert len(smooth_names) == len(+ set(smooth_names)+ ), f"Duplicate smooth names: {[n for n in smooth_names if smooth_names.count(n) > 1]}"++ # Must have attention-related mappings (input_layernorm→qkv, v_proj→o_proj)+ n_layers = model.config.num_hidden_layers+ attn_smooths = [n for n in smooth_names if "input_layernorm" in n or "self_attn.v_proj" in n]+ assert len(attn_smooths) == 2 * n_layers, f"Expected {2 * n_layers} attn smooth layers, got {len(attn_smooths)}"++ # Shared expert up→down should resolve at block level+ if hasattr(model.model.layers[0].mlp, "shared_expert"):+ shared_smooths = [n for n in smooth_names if "shared_expert" in n]+ assert (+ len(shared_smooths) == n_layers+ ), f"Expected {n_layers} shared_expert smooth layers, got {len(shared_smooths)}"++ del model++ def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):+ """AWQ on MoE: expert layers should be quantized, gates/routers stay FP."""+ ar = AutoRound(+ tiny_qwen_moe_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ model, layer_config = ar.quantize()+ assert model is not None+ assert len(layer_config) > 0++ q4_layers = {k for k, v in layer_config.items() if v["bits"] == 4}+ fp_layers = {k for k, v in layer_config.items() if v["bits"] >= 16}+ other_layers = {k: v["bits"] for k, v in layer_config.items() if v["bits"] != 4 and v["bits"] < 16}++ # Tiny Qwen MoE: mlp.gate is a TopKRouter (not Linear) so it's not in layer_config.+ # Only mlp.shared_expert_gate (a Linear) stays FP → 1 FP gate layer per block.+ assert len(other_layers) == 0, f"Unexpected bit widths: {other_layers}"+ n_layers = model.config.num_hidden_layers+ assert len(fp_layers) == n_layers, (+ f"Expected {n_layers} FP gate layers (mlp.shared_expert_gate per block), "+ f"got {len(fp_layers)}: {sorted(fp_layers)}"+ )+ assert len(q4_layers) == len(layer_config) - len(+ fp_layers+ ), f"Expected {len(layer_config) - len(fp_layers)} W4 layers, got {len(q4_layers)}"++ for name in fp_layers:+ assert name.endswith("gate"), f"Unexpected FP layer: {name}"++ def test_awq_moe_save_quant_config(self, tiny_qwen_moe_model_path):+ """AWQ MoE: saved quantization_config should be consistent and loadable."""+ ar = AutoRound(+ tiny_qwen_moe_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ # Load and verify config+ config_path = os.path.join(save_path, "config.json")+ assert os.path.exists(config_path), f"config.json not found at {save_path}"++ with open(config_path, "r") as f:+ config_data = json.load(f)++ qconfig = config_data.get("quantization_config")+ assert qconfig is not None, "quantization_config missing from saved config.json"+ assert qconfig["bits"] == 4+ assert qconfig["group_size"] == 128+ assert "auto-round" in qconfig["quant_method"]++ def test_awq_moe_save_compressed_size(self, tiny_qwen_moe_model_path):+ """AWQ MoE W4: quantized safetensors should be smaller than original."""+ ar = AutoRound(+ tiny_qwen_moe_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ def _safetensors_size(path):+ return sum(os.path.getsize(os.path.join(path, f)) for f in os.listdir(path) if f.endswith(".safetensors"))++ original_size = _safetensors_size(tiny_qwen_moe_model_path)+ quantized_size = _safetensors_size(save_path)+ assert quantized_size > 0, f"No safetensors files in {save_path}"++ ratio = quantized_size / original_size+ # W4G128 packing: (4-bit weights + scale/zp overhead)+ assert 0.35 < ratio < 0.65, (+ f"Compression ratio {ratio:.4f} outside expected range (0.35, 0.65): "+ f"original={original_size / (1024**2):.1f}MB, quantized={quantized_size / (1024**2):.1f}MB"+ )diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.pyindex 48a66ba75..9a3082be6 100644--- a/test/test_cpu/integrations/test_llmc_integration.py+++ b/test/test_cpu/integrations/test_llmc_integration.py@@ -1,10 +1,13 @@+import os+import pytestimport torchfrom compressed_tensors.quantization import QuantizationArgs, QuantizationSchemefrom llmcompressor import oneshotfrom llmcompressor.modifiers.autoround import AutoRoundModifier-from transformers import AutoModelForCausalLM, AutoTokenizer+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRoundfrom auto_round.calib_dataset import get_datasetfrom ...envs import multi_card@@ -239,3 +242,47 @@ def test_rtn_oneshot(recipe, tmp_path, tiny_tiny_llama_model_path):# Check lm-head is not quantizednot_targeted = model_loaded.lm_headassert not hasattr(not_targeted, "quantization_scheme")+++# ---------------------------------------------------------------------------+# AWQ W8A8 llm_compressor export config args+# ---------------------------------------------------------------------------+def test_llmc_awq_w8a8_export_config_args(tiny_opt_model_path, tmp_path):+ """W8A8 AWQ → llm_compressor: verify compressed-tensors metadata fields."""+ save_dir = str(tmp_path / "saved")+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="INT8",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")++ config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)+ qconfig = config.quantization_config++ assert qconfig["quant_method"] == "compressed-tensors"++ group0 = qconfig["config_groups"]["group_0"]+ # Weight args+ assert group0["weights"]["num_bits"] == 8+ assert group0["weights"]["type"] == "int"+ assert group0["weights"]["symmetric"] is True+ # Activation args+ assert group0["input_activations"]["num_bits"] == 8+ # Targets+ targets = group0.get("targets")+ assert targets is not None and len(targets) > 0++ # QuantLinear check: verify saved weights are int8 with per-channel scales+ from safetensors import safe_open++ st_files = [f for f in os.listdir(save_path) if f.endswith(".safetensors")]+ assert len(st_files) > 0, f"No safetensors files in {save_path}"+ with safe_open(os.path.join(save_path, st_files[0]), framework="pt") as f:+ weight = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight")+ assert weight.dtype == torch.int8, f"Expected int8 weight, got {weight.dtype}"+ scale = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight_scale")+ assert scale.shape[1] == 1, f"Expected per-channel scale shape (out, 1), got {scale.shape}"diff --git a/test/test_cuda/algorithms/test_awq.py b/test/test_cuda/algorithms/test_awq.pynew file mode 100644index 000000000..5222a98da--- /dev/null+++ b/test/test_cuda/algorithms/test_awq.py@@ -0,0 +1,257 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""CUDA CI tests for AWQ (Activation-Aware Weight Quantization) algorithm.++Covers:+- Normal LLM (Like OPT-125m): W4A16 quantization, inference, export check+- INT W8A8: llm_compressor export args, vLLM inference+- Tiny Qwen MoE: dynamic smoothing, quantized layer checks, quant config saving+- Accuracy: lm_eval on OPT-125m AWQ W4A16 (lambada_openai, limit=50)+"""++import json+import os+import shutil++import pytest+import torch+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound++from ...helpers import evaluate_accuracy, generate_prompt, get_model_path, opt_name_or_path, save_tiny_model++# ---------------------------------------------------------------------------+# Section 1: Normal LLM (OPT-125m) – W4A16 quantize, inference, export args+# ---------------------------------------------------------------------------+++class TestAWQLLM:+ """AWQ W4A16 on a tiny OPT model with CUDA acceleration."""++ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):+ """W4A16 AWQ quantization and CUDA inference smoke test."""+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ model, layer_config = ar.quantize()++ assert model is not None+ assert len(layer_config) > 0+ for name, cfg in layer_config.items():+ assert cfg["bits"] == 4, f"Layer {name} expected bits=4, got {cfg['bits']}"++ # CUDA inference+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path)+ output = generate_prompt(model, tokenizer, device="cuda")+ assert len(output) > 0 and "!!!" not in output, f"Unexpected generation output: {output}"++ def test_awq_w4a16_export_auto_round_args(self, tiny_opt_model_path):+ """Export to auto_round format: verify quantization_config fields."""+ bits, group_size, sym = 8, 64, True+ ar = AutoRound(+ tiny_opt_model_path,+ bits=bits,+ group_size=group_size,+ sym=sym,+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ config = AutoConfig.from_pretrained(save_path)+ qconfig = config.quantization_config+ assert qconfig is not None+ assert qconfig["bits"] == bits+ assert qconfig["group_size"] == group_size+ assert qconfig["sym"] == sym+ assert "auto-round" in qconfig["quant_method"]++ def test_awq_w4a16_load_and_generate(self, tiny_opt_model_path):+ """Quantize, save, reload, and generate on CUDA to verify round-trip."""+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ loaded_model = AutoModelForCausalLM.from_pretrained(save_path, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(save_path)+ output = generate_prompt(loaded_model, tokenizer, device="cuda")+ assert len(output) > 0 and "!!!" not in output, f"Unexpected generation output: {output}"+++# ---------------------------------------------------------------------------+# Section 2: Tiny Qwen MoE – dynamic smoothing, quantized layers, config+# ---------------------------------------------------------------------------+++class TestAWQMoE:+ """AWQ on a tiny MoE model with CUDA: smoothing, layers, config saving."""++ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):+ """AWQ mapping resolution works on MoE model."""+ from auto_round.algorithms.quantization.awq.mappings import resolve_mappings++ model = AutoModelForCausalLM.from_pretrained(+ tiny_qwen_moe_model_path,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ trust_remote_code=True,+ )+ resolved = resolve_mappings(model, user_mappings=None)++ assert len(resolved) > 0, "Expected non-empty resolved mappings"++ # Smooth layers should be unique+ smooth_names = [r.smooth_name for r in resolved]+ assert len(smooth_names) == len(+ set(smooth_names)+ ), f"Duplicate smooth names: {[n for n in smooth_names if smooth_names.count(n) > 1]}"++ n_layers = model.config.num_hidden_layers+ attn_smooths = [n for n in smooth_names if "input_layernorm" in n or "self_attn.v_proj" in n]+ assert len(attn_smooths) == 2 * n_layers, f"Expected {2 * n_layers} attn smooth layers, got {len(attn_smooths)}"++ if hasattr(model.model.layers[0].mlp, "shared_expert"):+ shared_smooths = [n for n in smooth_names if "shared_expert" in n]+ assert (+ len(shared_smooths) == n_layers+ ), f"Expected {n_layers} shared_expert smooth layers, got {len(shared_smooths)}"++ del model++ def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):+ """Expert layers quantized to W4, gates/routers stay fp."""+ ar = AutoRound(+ tiny_qwen_moe_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ model, layer_config = ar.quantize()+ assert model is not None+ assert len(layer_config) > 0++ # Categorize layers+ q4_layers = {k for k, v in layer_config.items() if v["bits"] == 4}+ fp_layers = {k for k, v in layer_config.items() if v["bits"] >= 16}+ other_layers = {k: v["bits"] for k, v in layer_config.items() if v["bits"] != 4 and v["bits"] < 16}++ # Tiny Qwen MoE: mlp.gate is a TopKRouter (not Linear, excluded from+ # layer_config), mlp.shared_expert_gate is Linear but excluded from+ # quantization → 1 FP gate layer per block.+ assert len(other_layers) == 0, f"Unexpected bit widths: {other_layers}"+ n_layers = model.config.num_hidden_layers+ assert len(fp_layers) == n_layers, (+ f"Expected {n_layers} FP gate layers (mlp.shared_expert_gate per block), "+ f"got {len(fp_layers)}: {sorted(fp_layers)}"+ )+ assert len(q4_layers) == len(layer_config) - len(+ fp_layers+ ), f"Expected {len(layer_config) - len(fp_layers)} W4 layers, got {len(q4_layers)}"++ for name in fp_layers:+ assert name.endswith("gate"), f"Unexpected FP layer: {name}"++ def test_awq_moe_save_compressed_size(self, tiny_qwen_moe_model_path):+ """AWQ MoE W4: quantized safetensors should be smaller than original."""+ ar = AutoRound(+ tiny_qwen_moe_model_path,+ scheme="W4A16",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ def _safetensors_size(path):+ return sum(os.path.getsize(os.path.join(path, f)) for f in os.listdir(path) if f.endswith(".safetensors"))++ original_size = _safetensors_size(tiny_qwen_moe_model_path)+ quantized_size = _safetensors_size(save_path)+ assert quantized_size > 0, f"No safetensors files in {save_path}"++ ratio = quantized_size / original_size+ assert 0.35 < ratio < 0.65, (+ f"Compression ratio {ratio:.4f} outside expected range (0.35, 0.65): "+ f"original={original_size / (1024**2):.1f}MB, quantized={quantized_size / (1024**2):.1f}MB"+ )+++class TestAWQEval:+ """AWQ accuracy evaluation on OPT-125m using lm_eval.++ Uses the full OPT-125m model (not tiny) for meaningful accuracy check.+ lambada_openai task, add limit for CI speed.+ """++ @classmethod+ def setup_class(cls):+ model_name = opt_name_or_path+ cls.model_name = model_name+ cls.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)++ @classmethod+ def teardown_class(cls):+ shutil.rmtree("runs", ignore_errors=True)++ def test_awq_w4a16_lmeval(self):+ """AWQ W4A16 on OPT-125m: lambada_openai accuracy check."""+ ar = AutoRound(+ self.model_name,+ scheme="W4A16",+ algorithm="awq",+ nsamples=32,+ seqlen=32,+ batch_size=8,+ )+ model, _ = ar.quantize()++ evaluate_accuracy(+ model,+ self.tokenizer,+ task="lambada_openai",+ threshold=0.3,+ batch_size="auto:8",+ limit=50,+ device="cuda",+ )diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.pyindex e9faa3e14..63636e395 100644--- a/test/test_cuda/integrations/test_vllm.py+++ b/test/test_cuda/integrations/test_vllm.py@@ -175,6 +175,36 @@ class TestVllmEvaluation:assert ret == 0, f"vllm evaluation with iters=0 failed (rc={ret})"+ not current_platform.is_cuda(),+ reason="only supports CUDA backend.",+)[email protected]_ci(reason="due to vllm and CT Dependency conflict.")+def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):+ """W8A8 AWQ → llm_compressor → vLLM: end-to-end inference test."""+ save_dir = str(tmp_path / "saved")+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="INT8",+ algorithm="awq",+ nsamples=2,+ seqlen=32,+ batch_size=2,+ )+ _, save_path = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")++ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=50)+ llm = LLM(+ model=save_path,+ trust_remote_code=True,+ tensor_parallel_size=1,+ gpu_memory_utilization=0.8,+ )+ outputs = llm.generate(["Hello, my name is"], sampling_params)+ generated_text = outputs[0].outputs[0].text+ assert len(generated_text.strip()) > 0 and "!!!" not in generated_text, "vLLM produced empty/meaningless output"++@pytest.mark.skipif(not current_platform.is_cuda(),reason="only supports CUDA backend.",