PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import json
import os
import re
import sys
from dataclasses import asdict, fields
from typing import Union
import torch
from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
from auto_round.compressors.config_resolution.contracts import LayerConfig, freeze_mapping
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, get_gguf_scheme, is_mx_fp, is_nv_fp
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_LAYER_TYPES,
check_to_quantized,
compress_layer_names,
expand_layer_config_for_weight_renames,
get_layer_names_in_block,
infer_bits_by_data_type,
to_standard_regex,
)
from auto_round.utils.model import get_module
def apply_layer_config_special_cases(
layer_config,
model,
default_dict,
supported_types,
inner_supported_types,
quant_block_list,
quant_lm_head,
gguf_name,
export_to_awq=False,
) -> tuple[dict, bool, str | None, bool]:
"""Apply lm-head, shape-divisibility, and block-membership layer rules."""
from auto_round.utils.model import get_lm_head_name
lm_head_name = get_lm_head_name(model)
tie_word_embeddings = bool(getattr(getattr(model, "config", None), "tie_word_embeddings", False))
if lm_head_name in layer_config:
quant_lm_head = True
if quant_lm_head and tie_word_embeddings and not gguf_name:
quant_lm_head = False
logger.warning(
"reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently"
)
if lm_head_name not in layer_config and quant_lm_head:
layer_config[lm_head_name] = copy.deepcopy(default_dict)
if not quant_lm_head and not gguf_name:
layer_config.pop(lm_head_name, None)
if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:
for n, m in model.named_modules():
if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
layer_config.setdefault(n, copy.deepcopy(default_dict))
layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
if export_to_awq and not gguf_name:
# Re-mark after regex/partial layer_config entries are expanded: the same
# marking already ran at format-resolution time, but raw user entries can
# overwrite it during expansion.
from auto_round.export.formats.backends.auto_awq import mark_awq_unservable_layers
layer_config = mark_awq_unservable_layers(model, layer_config, default_dict)
if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
skipped_layers = []
for n, m in model.named_modules():
if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
if m.weight.shape[1] % default_dict["group_size"]:
layer_config.setdefault(n, copy.deepcopy(default_dict))
layer_config[n].update(
{"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}
)
skipped_layers.append(n)
compressed_skipped_layers = compress_layer_names(skipped_layers)
if compressed_skipped_layers:
logger.warning_once(
f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "
f"{compressed_skipped_layers}"
)
for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):
if name not in layer_config:
layer_config[name] = copy.deepcopy(default_dict)
layer_config[name]["fixed_by_user"] = False
layer_config[name]["in_blocks"] = True
has_qlayer_outside_block = False
for cfg in layer_config.values():
if "in_blocks" not in cfg:
cfg["in_blocks"] = False
if not cfg["in_blocks"] and check_to_quantized(cfg):
has_qlayer_outside_block = True
return layer_config, has_qlayer_outside_block, lm_head_name, tie_word_embeddings
def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
"""Collect layer names from safetensor files that are not loaded into the model.
Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
instantiated by ``transformers``. This function discovers them so that regex
patterns in ``layer_config`` can still match them.
Returns:
List of layer names (the path without the ``.weight`` suffix) for weight
tensors present in the safetensor files but absent from *all_module_names*.
"""
name_or_path = None
if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
name_or_path = model.config.name_or_path
if not name_or_path:
return []
if not os.path.isdir(name_or_path):
# Resolution is intentionally local-only: model downloads belong to model loading,
# not to the pure layer-config planning phase.
return []
try:
from safetensors import safe_open
except ImportError:
return []
# Build tensor-name list from the safetensors index or single file
source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
source_single_file = os.path.join(name_or_path, "model.safetensors")
tensor_names: list = []
if os.path.exists(source_index_file):
with open(source_index_file) as f:
src_index = json.load(f)
tensor_names = list(src_index["weight_map"].keys())
elif os.path.exists(source_single_file):
with safe_open(source_single_file, framework="pt", device="cpu") as f:
tensor_names = list(f.keys())
else:
return []
module_name_set = set(all_module_names)
extra_layer_names = []
for tensor_name in tensor_names:
if not tensor_name.endswith(".weight"):
continue
layer_name = tensor_name[: -len(".weight")]
if layer_name not in module_name_set:
extra_layer_names.append(layer_name)
return extra_layer_names
def _resolve_layer_config_presets(
layer_config,
model,
ignore_layers,
default_scheme,
default_scale_dtype,
fill_default_value,
w8_asym_allowed: bool = False,
) -> tuple[dict, dict, tuple[str, ...], set[str]]:
"""Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,
bits-inference, and default-filling."""
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
"""Convert config entry into dict and validate keys."""
if isinstance(item, str):
config = asdict(preset_name_to_scheme(item.upper()))
elif isinstance(item, QuantizationScheme):
config = asdict(item)
elif isinstance(item, dict):
# "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
# silently drop it before validation.
item = {k: v for k, v in item.items() if k != "in_blocks"}
scheme_name = item.pop("scheme", None)
config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
if invalid:
raise ValueError(
f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
)
config.update(item)
else:
raise TypeError(
f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
f"Expected str, dict, or QuantizationScheme."
)
# Clean up
config = {k: v for k, v in config.items() if v is not None}
config["fixed_by_user"] = True
return config
extra_scheme_keys = ("scale_dtype",)
scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
layer_config = copy.deepcopy(layer_config) or {}
ignore_layer_patterns = set()
if ignore_layers:
ignore_layers = ignore_layers.replace(" ", "").split(",")
# "layers.1" -> "layers.1." so it does not also match "layers.10", but only when modules sit
# under that name; a layer whose own name ends with a digit (e.g. "fc1") is kept as is.
module_names = [n for n, _ in model.named_modules()]
ignore_layers = [
name + "." if name[-1].isdigit() and any(name + "." in n for n in module_names) else name
for name in ignore_layers
]
ignore_layer_patterns = set(ignore_layers)
# 1. ignore_layers -> force 16
for name in get_fp_layer_names(model, ignore_layers):
layer_config[name] = {
"bits": 16,
"act_bits": 16,
"data_type": "float",
"act_data_type": "float",
"fixed_by_user": True,
}
# 1b. Merge predefined model defaults; user entries win per-pattern and per-key.
from auto_round.special_model_handler import get_predefined_layer_config
predefined_only = set()
for _pattern, _overlay in get_predefined_layer_config(model).items():
_existing = layer_config.get(_pattern)
if isinstance(_existing, dict):
for _k, _v in _overlay.items():
_existing.setdefault(_k, _v)
elif _existing is None:
layer_config[_pattern] = dict(_overlay)
predefined_only.add(_pattern)
# 2. normalize
layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
# Entries added only from predefined model defaults are not user choices.
for _pattern in predefined_only:
layer_config[_pattern]["fixed_by_user"] = False
# entries whose sym was set by the user (dict key or a QuantizationScheme
# object) rather than inherited from the default scheme below
explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}
# 3. infer missing bits
for cfg in layer_config.values():
if "data_type" in cfg and "bits" not in cfg:
if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:
cfg["bits"] = b
if "act_data_type" in cfg and "act_bits" not in cfg:
if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:
cfg["act_bits"] = b
# 4. fill defaults
if isinstance(default_scheme, str):
default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))
elif default_scheme is None:
# Defensive fallback: callers that build/patch layer configs without a resolved
# scheme (e.g. external integrations invoking configure_layer_config() out of the
# normal init sequence) would otherwise crash `asdict(None)` below.
default_dict = asdict(QuantizationScheme())
else:
default_dict = asdict(default_scheme)
default_dict["scale_dtype"] = default_scale_dtype
# In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
# which should not be filled by default q4km again
for cfg in layer_config.values():
for key in scheme_keys:
if fill_default_value:
cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
else:
if key in extra_scheme_keys:
cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
else:
cfg.setdefault(key, None)
# 5. 8-bit asymmetric int entries are not servable in formats without a
# W8-asym path (vLLM GPTQ-format is symmetric-only, Marlin zp is 4-bit
# only). Refuse explicit requests up front; pin entries that would only
# inherit the global asym. llm_compressor exports (and AR_ALLOW_W8_ASYM)
# are exempt: compressed-tensors serves W8 asym.
pinned_w8 = []
for name, cfg in layer_config.items():
if cfg.get("data_type") == "int" and cfg.get("bits") == 8 and cfg.get("sym") is False:
if not w8_asym_allowed:
if name in explicit_sym:
raise ValueError(
f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "
"which this format cannot serve (vLLM W8 GPTQ-format weights are "
"symmetric-only; Marlin supports zero points at 4 bits only). Use a "
"symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "
"'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."
)
cfg["sym"] = True
pinned_w8.append(name)
if pinned_w8:
logger.info(
"layer_config: pinned %d 8-bit entry(s) to symmetric (inherited asym is not "
"servable at 8 bits in this format): %s",
len(pinned_w8),
", ".join(sorted(pinned_w8)),
)
return layer_config, default_dict, scheme_keys, ignore_layer_patterns
def _traverse_and_expand_layer_config(
layer_config, model, supported_types, inner_supported_types, ignore_layer_patterns, scheme_keys, gguf_name
) -> tuple[dict, dict, list[str], tuple]:
"""Steps 5-6 of layer-config resolution: supported-module collection (incl. the
GGUF embedding-type-detection branch) and regex-config expansion."""
from auto_round.utils.model import get_module
# 5. collect supported modules
embedding_types = (torch.nn.Embedding,)
if gguf_name:
if torch.nn.Embedding not in supported_types:
supported_types = (*supported_types, torch.nn.Embedding)
# for some Embedding which type() is not torch.nn.Embedding
# for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding
model_module_name = model.__class__.__module__
module_cls = sys.modules[model_module_name]
for name in module_cls.__dict__:
if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):
embedding_types = (*embedding_types, getattr(module_cls, name))
supported_types = (*supported_types, *embedding_types)
all_supported_layer_names, embedding_layer_names = [], []
all_module_names = []
for n, m in model.named_modules():
all_module_names.append(n)
if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:
continue
all_supported_layer_names.append(n)
if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
embedding_layer_names.append(n)
# Also include layer names from safetensor files not loaded into the model
# (e.g. MTP layers that transformers does not instantiate).
safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)
# 6. expand regex configs; exact layer names always take precedence over patterns.
regex_config = {}
exact_names = {name for name in layer_config if name in all_supported_layer_names}
for name in list(layer_config.keys()):
if name in all_supported_layer_names:
continue
if name in all_module_names:
m = get_module(model, name)
if len(list(m.children())) == 0 and type(m) not in supported_types:
val = layer_config.pop(name)
if name in ignore_layer_patterns:
# Keep unsupported ignore_layers entries so export can serialize
# them into regex-based extra_config for loaders like vLLM INC.
regex_config[name] = val
else:
logger.warning(
f"'{name}' exists in the model but is not a supported quantization target "
f"in the current scheme, ignoring its setting in `layer_config`"
)
continue
regex = re.compile(to_standard_regex(name))
matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
# skip it for mtp layers not loaded in transformers
if not matched and not safetensor_only_matched:
# type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear
logger.warning_once(
f"Layer name or regex '{name}' in layer_config does not match any supported layers. "
+ "Please check for typos or update the regex pattern, ignore it for now"
)
val = layer_config.pop(name)
regex_config[name] = val # keep regex config
for match in matched:
if match not in exact_names:
layer_config[match] = val
return layer_config, regex_config, embedding_layer_names, supported_types
def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
"""Identifies and returns layers in the model to exclude from quantization.
This function processes a comma-separated list of fully precision (FP) layers,
matches them to the names of layers in the model, and returns a list of such
layers to exclude from quantization.
Args:
model (torch.nn.Module): The model whose layers will be inspected.
ignore_layers (str): A comma-separated string of layer names to be excluded
from quantization. Whitespace is ignored in this string.
Returns:
list: A list of layer names that match the specified FP layers or are
subcomponents of those layers.
"""
from auto_round.utils import SUPPORTED_LAYER_TYPES
if not ignore_layers:
return []
all_layer_names = []
for n, m in model.named_modules():
if type(m) in SUPPORTED_LAYER_TYPES:
all_layer_names.append(n)
not_to_quantized_layers = []
for fp_layer in ignore_layers:
if fp_layer == "":
continue
if fp_layer in all_layer_names:
not_to_quantized_layers.append(fp_layer)
continue
for name in all_layer_names:
if fp_layer in name:
not_to_quantized_layers.append(name)
not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
if not_to_quantized_layers:
logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")
return not_to_quantized_layers
def resolve_layer_config(
*,
model,
scheme: ResolvedScheme,
layer_config,
scale_dtype=None,
supported_types=None,
inner_supported_types=None,
quant_block_list=None,
ignore_layers: str = "",
quant_lm_head: bool = False,
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
fill_default_value: bool = True,
format: str | None = None,
) -> LayerConfig:
"""Resolve final per-layer configuration without writing model attributes."""
layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)
supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
inner_supported_types = tuple(
INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
)
scheme_value = scheme.value
gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
from auto_round import envs
from auto_round.schemes import format_allows_w8_asym
w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
layer_config,
model,
ignore_layers,
scheme_value,
scale_dtype,
fill_default_value,
w8_asym_allowed=w8_asym_allowed,
)
resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(
resolved,
model,
supported_types,
inner_supported_types,
ignore_patterns,
scheme_keys,
gguf_name,
)
resolved, has_outside, lm_head_name, tied = apply_layer_config_special_cases(
resolved,
model,
default_dict,
supported_types,
inner_supported_types,
quant_block_list,
quant_lm_head,
gguf_name,
export_to_awq="awq" in (format or ""),
)
if gguf_name:
from auto_round.export.formats.backends.gguf import apply_gguf_layer_defaults
resolved, _ = apply_gguf_layer_defaults(
dict(resolved),
model,
gguf_name,
lm_head_name,
tied,
embedding_names,
scale_dtype,
enable_gguf_official_mixed,
is_mllm,
has_outside,
)
return freeze_mapping(resolved)
def extract_regex_config(
*,
model,
scheme: ResolvedScheme,
layer_config,
scale_dtype=None,
supported_types=None,
inner_supported_types=None,
ignore_layers: str = "",
fill_default_value: bool = True,
format: str | None = None,
) -> LayerConfig:
"""Resolve only the regex entries retained for export metadata."""
layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)
supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
inner_supported_types = tuple(
INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
)
scheme_value = scheme.value
gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
from auto_round import envs
from auto_round.schemes import format_allows_w8_asym
# Same allowance rule as resolve_layer_config: AutoScheme output can carry
# DP-selected W8-asym entries, which stay untouched only when the export
# format serves them (llm_compressor) or the opt-in env is set.
w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
layer_config,
model,
ignore_layers,
scheme_value,
scale_dtype,
fill_default_value,
w8_asym_allowed=w8_asym_allowed,
)
_, regex_config, _, _ = _traverse_and_expand_layer_config(
normalized,
model,
supported_types,
inner_supported_types,
ignore_patterns,
scheme_keys,
gguf_name,
)
return freeze_mapping(regex_config)
def has_quantized_layer_outside_blocks(layer_config: LayerConfig) -> bool:
"""Derive the block-outside flag from a final layer mapping."""
return any(
not config.get("in_blocks", False) and check_to_quantized(dict(config)) for config in layer_config.values()
)
def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:
"""Apply a resolved plan at the single explicit module-attribute write boundary."""
scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)
for module_name, module in model.named_modules():
# AutoRound only ever writes scheme attributes onto quantization-target
# layers (plus model-level keys such as ``rotation_config`` on the root
# module). Some scheme keys share a name with attributes a model class
# legitimately owns -- e.g. a grouped RMSNorm's own ``group_size`` -- so
# blindly clearing them from every module would corrupt the model
# (``AttributeError`` on the norm's next forward). Scope the reset to the
# same modules that receive the plan below.
is_quant_target = (
isinstance(module, (SUPPORTED_LAYER_TYPES, torch.nn.Embedding))
or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES
)
if module_name != "" and not is_quant_target:
continue
for key in scheme_keys:
if module_name == "" and key == "rotation_config":
continue
if hasattr(module, key):
delattr(module, key)
for layer_name, config in plan.layer_config.items():
module = get_module(model, layer_name)
if module is None:
continue
for name, value in config.items():
setattr(module, name, value)