Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

24.9 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import os
import re
import sys
from dataclasses import asdict, fields
from typing import Union

import torch

from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
from auto_round.compressors.config_resolution.contracts import LayerConfig, freeze_mapping
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, get_gguf_scheme, is_mx_fp, is_nv_fp
from auto_round.utils import (
    INNER_SUPPORTED_LAYER_TYPES,
    SUPPORTED_LAYER_TYPES,
    check_to_quantized,
    compress_layer_names,
    expand_layer_config_for_weight_renames,
    get_layer_names_in_block,
    infer_bits_by_data_type,
    to_standard_regex,
)
from auto_round.utils.model import get_module


def apply_layer_config_special_cases(
    layer_config,
    model,
    default_dict,
    supported_types,
    inner_supported_types,
    quant_block_list,
    quant_lm_head,
    gguf_name,
    export_to_awq=False,
) -> tuple[dict, bool, str | None, bool]:
    """Apply lm-head, shape-divisibility, and block-membership layer rules."""
    from auto_round.utils.model import get_lm_head_name

    lm_head_name = get_lm_head_name(model)
    tie_word_embeddings = bool(getattr(getattr(model, "config", None), "tie_word_embeddings", False))
    if lm_head_name in layer_config:
        quant_lm_head = True
    if quant_lm_head and tie_word_embeddings and not gguf_name:
        quant_lm_head = False
        logger.warning(
            "reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently"
        )
    if lm_head_name not in layer_config and quant_lm_head:
        layer_config[lm_head_name] = copy.deepcopy(default_dict)
    if not quant_lm_head and not gguf_name:
        layer_config.pop(lm_head_name, None)

    if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:
        for n, m in model.named_modules():
            if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
                if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
                    layer_config.setdefault(n, copy.deepcopy(default_dict))
                    layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})

    if export_to_awq and not gguf_name:
        # Re-mark after regex/partial layer_config entries are expanded: the same
        # marking already ran at format-resolution time, but raw user entries can
        # overwrite it during expansion.
        from auto_round.export.formats.backends.auto_awq import mark_awq_unservable_layers

        layer_config = mark_awq_unservable_layers(model, layer_config, default_dict)

    if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
        skipped_layers = []
        for n, m in model.named_modules():
            if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
                if m.weight.shape[1] % default_dict["group_size"]:
                    layer_config.setdefault(n, copy.deepcopy(default_dict))
                    layer_config[n].update(
                        {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}
                    )
                    skipped_layers.append(n)
        compressed_skipped_layers = compress_layer_names(skipped_layers)
        if compressed_skipped_layers:
            logger.warning_once(
                f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "
                f"{compressed_skipped_layers}"
            )

    for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):
        if name not in layer_config:
            layer_config[name] = copy.deepcopy(default_dict)
            layer_config[name]["fixed_by_user"] = False
        layer_config[name]["in_blocks"] = True

    has_qlayer_outside_block = False
    for cfg in layer_config.values():
        if "in_blocks" not in cfg:
            cfg["in_blocks"] = False
        if not cfg["in_blocks"] and check_to_quantized(cfg):
            has_qlayer_outside_block = True
    return layer_config, has_qlayer_outside_block, lm_head_name, tie_word_embeddings


def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
    """Collect layer names from safetensor files that are not loaded into the model.

    Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
    instantiated by ``transformers``.  This function discovers them so that regex
    patterns in ``layer_config`` can still match them.

    Returns:
        List of layer names (the path without the ``.weight`` suffix) for weight
        tensors present in the safetensor files but absent from *all_module_names*.
    """
    name_or_path = None
    if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
        name_or_path = model.config.name_or_path
    if not name_or_path:
        return []

    if not os.path.isdir(name_or_path):
        # Resolution is intentionally local-only: model downloads belong to model loading,
        # not to the pure layer-config planning phase.
        return []

    try:
        from safetensors import safe_open
    except ImportError:
        return []

    # Build tensor-name list from the safetensors index or single file
    source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
    source_single_file = os.path.join(name_or_path, "model.safetensors")

    tensor_names: list = []
    if os.path.exists(source_index_file):
        with open(source_index_file) as f:
            src_index = json.load(f)
        tensor_names = list(src_index["weight_map"].keys())
    elif os.path.exists(source_single_file):
        with safe_open(source_single_file, framework="pt", device="cpu") as f:
            tensor_names = list(f.keys())
    else:
        return []

    module_name_set = set(all_module_names)
    extra_layer_names = []
    for tensor_name in tensor_names:
        if not tensor_name.endswith(".weight"):
            continue
        layer_name = tensor_name[: -len(".weight")]
        if layer_name not in module_name_set:
            extra_layer_names.append(layer_name)
    return extra_layer_names


def _resolve_layer_config_presets(
    layer_config,
    model,
    ignore_layers,
    default_scheme,
    default_scale_dtype,
    fill_default_value,
    w8_asym_allowed: bool = False,
) -> tuple[dict, dict, tuple[str, ...], set[str]]:
    """Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,
    bits-inference, and default-filling."""
    from auto_round.schemes import QuantizationScheme, preset_name_to_scheme

    def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
        """Convert config entry into dict and validate keys."""
        if isinstance(item, str):
            config = asdict(preset_name_to_scheme(item.upper()))
        elif isinstance(item, QuantizationScheme):
            config = asdict(item)
        elif isinstance(item, dict):
            # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
            # silently drop it before validation.
            item = {k: v for k, v in item.items() if k != "in_blocks"}
            scheme_name = item.pop("scheme", None)
            config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
            invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
            if invalid:
                raise ValueError(
                    f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
                )
            config.update(item)
        else:
            raise TypeError(
                f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
                f"Expected str, dict, or QuantizationScheme."
            )
        # Clean up
        config = {k: v for k, v in config.items() if v is not None}
        config["fixed_by_user"] = True
        return config

    extra_scheme_keys = ("scale_dtype",)
    scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
    layer_config = copy.deepcopy(layer_config) or {}
    ignore_layer_patterns = set()
    if ignore_layers:
        ignore_layers = ignore_layers.replace(" ", "").split(",")
        # "layers.1" -> "layers.1." so it does not also match "layers.10", but only when modules sit
        # under that name; a layer whose own name ends with a digit (e.g. "fc1") is kept as is.
        module_names = [n for n, _ in model.named_modules()]
        ignore_layers = [
            name + "." if name[-1].isdigit() and any(name + "." in n for n in module_names) else name
            for name in ignore_layers
        ]
        ignore_layer_patterns = set(ignore_layers)

    # 1. ignore_layers -> force 16
    for name in get_fp_layer_names(model, ignore_layers):
        layer_config[name] = {
            "bits": 16,
            "act_bits": 16,
            "data_type": "float",
            "act_data_type": "float",
            "fixed_by_user": True,
        }

    # 1b. Merge predefined model defaults; user entries win per-pattern and per-key.
    from auto_round.special_model_handler import get_predefined_layer_config

    predefined_only = set()
    for _pattern, _overlay in get_predefined_layer_config(model).items():
        _existing = layer_config.get(_pattern)
        if isinstance(_existing, dict):
            for _k, _v in _overlay.items():
                _existing.setdefault(_k, _v)
        elif _existing is None:
            layer_config[_pattern] = dict(_overlay)
            predefined_only.add(_pattern)

    # 2. normalize
    layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
    # Entries added only from predefined model defaults are not user choices.
    for _pattern in predefined_only:
        layer_config[_pattern]["fixed_by_user"] = False
    # entries whose sym was set by the user (dict key or a QuantizationScheme
    # object) rather than inherited from the default scheme below
    explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}

    # 3. infer missing bits
    for cfg in layer_config.values():
        if "data_type" in cfg and "bits" not in cfg:
            if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:
                cfg["bits"] = b
        if "act_data_type" in cfg and "act_bits" not in cfg:
            if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:
                cfg["act_bits"] = b

    # 4. fill defaults
    if isinstance(default_scheme, str):
        default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))
    elif default_scheme is None:
        # Defensive fallback: callers that build/patch layer configs without a resolved
        # scheme (e.g. external integrations invoking configure_layer_config() out of the
        # normal init sequence) would otherwise crash `asdict(None)` below.
        default_dict = asdict(QuantizationScheme())
    else:
        default_dict = asdict(default_scheme)
    default_dict["scale_dtype"] = default_scale_dtype

    # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
    # which should not be filled by default q4km again
    for cfg in layer_config.values():
        for key in scheme_keys:
            if fill_default_value:
                cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
            else:
                if key in extra_scheme_keys:
                    cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
                else:
                    cfg.setdefault(key, None)

    # 5. 8-bit asymmetric int entries are not servable in formats without a
    # W8-asym path (vLLM GPTQ-format is symmetric-only, Marlin zp is 4-bit
    # only). Refuse explicit requests up front; pin entries that would only
    # inherit the global asym. llm_compressor exports (and AR_ALLOW_W8_ASYM)
    # are exempt: compressed-tensors serves W8 asym.
    pinned_w8 = []
    for name, cfg in layer_config.items():
        if cfg.get("data_type") == "int" and cfg.get("bits") == 8 and cfg.get("sym") is False:
            if not w8_asym_allowed:
                if name in explicit_sym:
                    raise ValueError(
                        f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "
                        "which this format cannot serve (vLLM W8 GPTQ-format weights are "
                        "symmetric-only; Marlin supports zero points at 4 bits only). Use a "
                        "symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "
                        "'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."
                    )
                cfg["sym"] = True
                pinned_w8.append(name)
    if pinned_w8:
        logger.info(
            "layer_config: pinned %d 8-bit entry(s) to symmetric (inherited asym is not "
            "servable at 8 bits in this format): %s",
            len(pinned_w8),
            ", ".join(sorted(pinned_w8)),
        )

    return layer_config, default_dict, scheme_keys, ignore_layer_patterns


def _traverse_and_expand_layer_config(
    layer_config, model, supported_types, inner_supported_types, ignore_layer_patterns, scheme_keys, gguf_name
) -> tuple[dict, dict, list[str], tuple]:
    """Steps 5-6 of layer-config resolution: supported-module collection (incl. the
    GGUF embedding-type-detection branch) and regex-config expansion."""
    from auto_round.utils.model import get_module

    # 5. collect supported modules
    embedding_types = (torch.nn.Embedding,)
    if gguf_name:
        if torch.nn.Embedding not in supported_types:
            supported_types = (*supported_types, torch.nn.Embedding)

        # for some Embedding which type() is not torch.nn.Embedding
        # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding
        model_module_name = model.__class__.__module__
        module_cls = sys.modules[model_module_name]
        for name in module_cls.__dict__:
            if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):
                embedding_types = (*embedding_types, getattr(module_cls, name))
        supported_types = (*supported_types, *embedding_types)

    all_supported_layer_names, embedding_layer_names = [], []
    all_module_names = []
    for n, m in model.named_modules():
        all_module_names.append(n)
        if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:
            continue
        all_supported_layer_names.append(n)
        if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
            embedding_layer_names.append(n)

    # Also include layer names from safetensor files not loaded into the model
    # (e.g. MTP layers that transformers does not instantiate).
    safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)

    # 6. expand regex configs; exact layer names always take precedence over patterns.
    regex_config = {}
    exact_names = {name for name in layer_config if name in all_supported_layer_names}
    for name in list(layer_config.keys()):
        if name in all_supported_layer_names:
            continue
        if name in all_module_names:
            m = get_module(model, name)
            if len(list(m.children())) == 0 and type(m) not in supported_types:
                val = layer_config.pop(name)
                if name in ignore_layer_patterns:
                    # Keep unsupported ignore_layers entries so export can serialize
                    # them into regex-based extra_config for loaders like vLLM INC.
                    regex_config[name] = val
                else:
                    logger.warning(
                        f"'{name}' exists in the model but is not a supported quantization target "
                        f"in the current scheme, ignoring its setting in `layer_config`"
                    )
                continue

        regex = re.compile(to_standard_regex(name))
        matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
        safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
        # skip it for mtp layers not loaded in transformers
        if not matched and not safetensor_only_matched:
            # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear
            logger.warning_once(
                f"Layer name or regex '{name}' in layer_config does not match any supported layers. "
                + "Please check for typos or update the regex pattern, ignore it for now"
            )
        val = layer_config.pop(name)
        regex_config[name] = val  # keep regex config
        for match in matched:
            if match not in exact_names:
                layer_config[match] = val

    return layer_config, regex_config, embedding_layer_names, supported_types


def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
    """Identifies and returns layers in the model to exclude from quantization.

    This function processes a comma-separated list of fully precision (FP) layers,
    matches them to the names of layers in the model, and returns a list of such
    layers to exclude from quantization.

    Args:
        model (torch.nn.Module): The model whose layers will be inspected.
        ignore_layers (str): A comma-separated string of layer names to be excluded
            from quantization. Whitespace is ignored in this string.

    Returns:
        list: A list of layer names that match the specified FP layers or are
        subcomponents of those layers.
    """
    from auto_round.utils import SUPPORTED_LAYER_TYPES

    if not ignore_layers:
        return []

    all_layer_names = []
    for n, m in model.named_modules():
        if type(m) in SUPPORTED_LAYER_TYPES:
            all_layer_names.append(n)
    not_to_quantized_layers = []

    for fp_layer in ignore_layers:
        if fp_layer == "":
            continue
        if fp_layer in all_layer_names:
            not_to_quantized_layers.append(fp_layer)
            continue
        for name in all_layer_names:
            if fp_layer in name:
                not_to_quantized_layers.append(name)
    not_to_quantized_layers.extend(ignore_layers)  # keep regex name for later use
    if not_to_quantized_layers:
        logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")
    return not_to_quantized_layers


def resolve_layer_config(
    *,
    model,
    scheme: ResolvedScheme,
    layer_config,
    scale_dtype=None,
    supported_types=None,
    inner_supported_types=None,
    quant_block_list=None,
    ignore_layers: str = "",
    quant_lm_head: bool = False,
    enable_gguf_official_mixed: bool = True,
    is_mllm: bool = False,
    fill_default_value: bool = True,
    format: str | None = None,
) -> LayerConfig:
    """Resolve final per-layer configuration without writing model attributes."""
    layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)
    supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
    inner_supported_types = tuple(
        INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
    )
    scheme_value = scheme.value
    gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)

    from auto_round import envs
    from auto_round.schemes import format_allows_w8_asym

    w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
    resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
        layer_config,
        model,
        ignore_layers,
        scheme_value,
        scale_dtype,
        fill_default_value,
        w8_asym_allowed=w8_asym_allowed,
    )
    resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(
        resolved,
        model,
        supported_types,
        inner_supported_types,
        ignore_patterns,
        scheme_keys,
        gguf_name,
    )
    resolved, has_outside, lm_head_name, tied = apply_layer_config_special_cases(
        resolved,
        model,
        default_dict,
        supported_types,
        inner_supported_types,
        quant_block_list,
        quant_lm_head,
        gguf_name,
        export_to_awq="awq" in (format or ""),
    )
    if gguf_name:
        from auto_round.export.formats.backends.gguf import apply_gguf_layer_defaults

        resolved, _ = apply_gguf_layer_defaults(
            dict(resolved),
            model,
            gguf_name,
            lm_head_name,
            tied,
            embedding_names,
            scale_dtype,
            enable_gguf_official_mixed,
            is_mllm,
            has_outside,
        )
    return freeze_mapping(resolved)


def extract_regex_config(
    *,
    model,
    scheme: ResolvedScheme,
    layer_config,
    scale_dtype=None,
    supported_types=None,
    inner_supported_types=None,
    ignore_layers: str = "",
    fill_default_value: bool = True,
    format: str | None = None,
) -> LayerConfig:
    """Resolve only the regex entries retained for export metadata."""
    layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)
    supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
    inner_supported_types = tuple(
        INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
    )
    scheme_value = scheme.value
    gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
    from auto_round import envs
    from auto_round.schemes import format_allows_w8_asym

    # Same allowance rule as resolve_layer_config: AutoScheme output can carry
    # DP-selected W8-asym entries, which stay untouched only when the export
    # format serves them (llm_compressor) or the opt-in env is set.
    w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
    normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
        layer_config,
        model,
        ignore_layers,
        scheme_value,
        scale_dtype,
        fill_default_value,
        w8_asym_allowed=w8_asym_allowed,
    )
    _, regex_config, _, _ = _traverse_and_expand_layer_config(
        normalized,
        model,
        supported_types,
        inner_supported_types,
        ignore_patterns,
        scheme_keys,
        gguf_name,
    )
    return freeze_mapping(regex_config)


def has_quantized_layer_outside_blocks(layer_config: LayerConfig) -> bool:
    """Derive the block-outside flag from a final layer mapping."""
    return any(
        not config.get("in_blocks", False) and check_to_quantized(dict(config)) for config in layer_config.values()
    )


def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:
    """Apply a resolved plan at the single explicit module-attribute write boundary."""
    scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)
    for module_name, module in model.named_modules():
        # AutoRound only ever writes scheme attributes onto quantization-target
        # layers (plus model-level keys such as ``rotation_config`` on the root
        # module). Some scheme keys share a name with attributes a model class
        # legitimately owns -- e.g. a grouped RMSNorm's own ``group_size`` -- so
        # blindly clearing them from every module would corrupt the model
        # (``AttributeError`` on the norm's next forward). Scope the reset to the
        # same modules that receive the plan below.
        is_quant_target = (
            isinstance(module, (SUPPORTED_LAYER_TYPES, torch.nn.Embedding))
            or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES
        )
        if module_name != "" and not is_quant_target:
            continue
        for key in scheme_keys:
            if module_name == "" and key == "rotation_config":
                continue
            if hasattr(module, key):
                delattr(module, key)

    for layer_name, config in plan.layer_config.items():
        module = get_module(model, layer_name)
        if module is None:
            continue
        for name, value in config.items():
            setattr(module, name, value)