Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

9.9 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import re

import torch

from auto_round.compressors.base import SERIALIZATION_KEYS
from auto_round.utils import matches_any_regex, to_standard_regex

_EXPERT_SEGMENTS = ("experts", "shared_experts")


# TODO: Remove when the issue is fixed
# https://github.com/vllm-project/llm-compressor/issues/3069
def add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:
    """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.

    The generic ``"Linear"`` target does not cover all expert layouts.  Keep the
    safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),
    but trigger for non-standard expert paths and for any routed
    expert structure that lives outside the standard ``mlp.experts`` naming.
    """
    if "RoutedExperts" in targets:
        return targets

    def _projection_name(name: str) -> str:
        # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']
        parts = name.split(".")
        if len(parts) >= 2:
            return parts[-2]
        return parts[-1]

    for name in layer_names:
        lname = name.lower()
        if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:
            return list(targets) + ["RoutedExperts"]

        if ".experts." not in lname:
            continue

        projection = _projection_name(name)
        if projection in {"gate_proj", "up_proj", "down_proj"}:
            continue
        return list(targets) + ["RoutedExperts"]

    return targets


def ignore_fused_experts_for_vllm(qconfig: dict, ignored_layers: list[str], quantized_layers: list[str]) -> dict:
    """Add vLLM-matchable ignore entries for ignored fused expert modules when ``RoutedExperts`` is targeted.

    vLLM resolves a FusedMoE scheme via ``<module>.0.{gate,up,down}_proj``, which fused layer names such as
    ``<module>.w13_weight`` never match, so the global ``RoutedExperts`` target would quantize them.
    """
    groups = qconfig.get("config_groups") or {}
    if not any("RoutedExperts" in (g.get("targets") or []) for g in groups.values()):
        return qconfig

    ignore = list(qconfig.get("ignore") or [])
    for layer_name in dict.fromkeys(ignored_layers):
        if layer_name.startswith("re:"):
            continue
        parts = layer_name.split(".")
        if parts[-1] in _EXPERT_SEGMENTS:
            module = layer_name
        elif len(parts) >= 2 and parts[-2] in _EXPERT_SEGMENTS and not parts[-1].isdigit():
            module = ".".join(parts[:-1])
        else:
            continue
        # A partially quantized module cannot be described as ignored.
        if any(q == module or q.startswith(f"{module}.") for q in quantized_layers):
            continue
        entry = f"re:^{re.escape(module)}\\.\\d+\\.(gate|up|down)_proj$"
        if entry not in ignore:
            ignore.append(entry)
    qconfig["ignore"] = ignore
    return qconfig


def get_checkpoint_name_reverter(model: torch.nn.Module):
    """Return ``fn(module_name) -> name`` mapping a loaded-model module name to the saved checkpoint namespace."""
    from auto_round.utils.common import (
        get_reverse_checkpoint_conversion_mapping,
        get_reverse_weight_transforms,
        revert_checkpoint_conversion_mapping,
        revert_name_with_weight_transforms,
    )

    transforms = get_reverse_weight_transforms(model)
    mapping = get_reverse_checkpoint_conversion_mapping(model) if transforms is None else None
    if transforms is None and not mapping:
        return lambda name: name

    def _revert(name: str) -> str:
        # Renamings are defined on parameter keys, so revert a full weight key.
        key = f"{name}.weight"
        if transforms is not None:
            reverted = revert_name_with_weight_transforms(key, transforms)
        else:
            reverted = revert_checkpoint_conversion_mapping(key, mapping)
        return reverted[: -len(".weight")] if reverted.endswith(".weight") else name

    return _revert


def revert_quantization_config_names(qconfig: dict, revert) -> dict:
    """Rewrite module names in ``ignore`` and group ``targets`` with ``revert`` so they match the saved weights."""

    def _revert_entry(entry):
        if not isinstance(entry, str):
            return entry
        if not entry.startswith("re:"):
            return revert(entry)
        # Only rewrite regexes that wrap a single escaped module name, e.g. ``re:.*model\.layers\.0\.mlp\.gate.*``.
        match = re.fullmatch(r"(re:(?:\.\*)?)(.*?)((?:\.\*)?)", entry)
        plain = re.sub(r"\\(.)", r"\1", match.group(2))
        if not plain or re.escape(plain) != match.group(2):
            return entry
        return f"{match.group(1)}{re.escape(revert(plain))}{match.group(3)}"

    if qconfig.get("ignore"):
        qconfig["ignore"] = list(dict.fromkeys(_revert_entry(e) for e in qconfig["ignore"]))
    for group in (qconfig.get("config_groups") or {}).values():
        if group.get("targets"):
            group["targets"] = [_revert_entry(t) for t in group["targets"]]
    return qconfig


def finalize_llmcompressor_config_names(
    qconfig: dict, model: torch.nn.Module, ignored_layers: list[str], quantized_layers: list[str]
) -> dict:
    """Revert config module names to the checkpoint namespace, then add vLLM ignore entries for fused experts."""
    revert = get_checkpoint_name_reverter(model)
    qconfig = revert_quantization_config_names(qconfig, revert)
    ignored = list(qconfig.get("ignore") or []) + [
        revert(n) for n in ignored_layers + collect_fused_expert_modules(model) if not n.startswith("re:")
    ]
    return ignore_fused_experts_for_vllm(qconfig, ignored, [revert(n) for n in quantized_layers])


def collect_fused_expert_modules(model: torch.nn.Module) -> list[str]:
    """Return expert modules still holding fused 3-D weights, i.e. never unfused into quantized Linear layers."""
    return [
        name
        for name, module in model.named_modules()
        if name.split(".")[-1] in _EXPERT_SEGMENTS and any(p.dim() == 3 for p in module.parameters(recurse=False))
    ]


def generate_ignore_regex_list(regex_config: dict[str, dict], layer_config: dict[str, dict]) -> list[str]:
    """
    Generate ignore regex list for llm_compressor based on regex_config and layer_config.

    Rules:
    1. Any layer in regex_config with bits >= 16 is ignored.
    2. Any layer in layer_config with bits >= 16 is ignored if not already included.
    3. Output regex patterns are normalized for llm_compressor ('re:...' style).

    Args:
        regex_config (Dict[str, Dict]): dynamic quantization config
        layer_config (Dict[str, Dict]): layer-wise quantization config

    Returns:
        List[str]: List of regex patterns to ignore during quantization.
    """
    prefix = "re:"
    ignore_regex: list[str] = []

    # Step 1: Add regex_config keys with bits >= 16
    for key, cfg in regex_config.items():
        bits = cfg.get("bits")
        if bits > 8:
            ignore_regex.append(prefix + to_standard_regex(key))

    # Step 2: Add all full named layer from layer_config if bits >= 16
    for key, cfg in layer_config.items():
        bits = cfg.get("bits")
        if bits > 8:
            ignore_regex.append(key)

    return ignore_regex


# Tuning hyper-parameters worth recording so a llm-compressor checkpoint can be
# traced back to (and re-created with) the AutoRound command that produced it.
AUTOROUND_CONFIG_KEY = "autoround_config"
# Prefer the shared compressor serialization allowlist so new hyper-parameters
# land here automatically. ``autoround_version`` is injected into
# ``serialization_dict`` at export time and is not a SerializedCompressorConfig field.
_AUTOROUND_CONFIG_FIELDS = ("autoround_version",) + tuple(SERIALIZATION_KEYS)


def _to_json_value(value):
    import enum

    if isinstance(value, (bool, int, float, str)):
        return value
    if isinstance(value, torch.dtype):
        return str(value)
    if isinstance(value, torch.device):
        return str(value)
    if isinstance(value, enum.Enum):
        enum_value = value.value
        if isinstance(enum_value, (bool, int, float, str)):
            return enum_value
        return value.name
    if isinstance(value, type):
        return f"{value.__module__}.{value.__qualname__}"
    if isinstance(value, (list, tuple)):
        return [_to_json_value(v) for v in value]
    return str(value)


def add_autoround_config(quantization_config: dict, serialization_dict: dict | None) -> dict:
    """Record AutoRound tuning hyper-parameters in an llm-compressor ``quantization_config``.

    The values are stored under a single nested ``"autoround_config"`` key so that the
    compressed-tensors / vLLM loaders, which only read the keys they know, are unaffected.
    Only runs when ``quantization_config["provider"] == "auto-round"``, so shared export
    helpers do not write AutoRound-branded metadata for other providers.
    """
    if not isinstance(quantization_config, dict) or not serialization_dict:
        return quantization_config
    if quantization_config.get("provider") != "auto-round":
        return quantization_config
    autoround_config = {}
    for key in _AUTOROUND_CONFIG_FIELDS:
        value = serialization_dict.get(key)
        if value is None:
            continue
        autoround_config[key] = _to_json_value(value)
    if autoround_config:
        quantization_config[AUTOROUND_CONFIG_KEY] = autoround_config
    return quantization_config