Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

39.1 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import os
import re
from types import SimpleNamespace

import torch
import torch.nn as nn
from packaging.version import Version
from tqdm import tqdm
from transformers.pytorch_utils import Conv1D

from auto_round.export.formats import BackendDataType
from auto_round.inference.backend import (
    BackendInfos,
    dynamic_import_inference_linear,
    get_highest_priority_backend,
    get_layer_backend,
    process_requirement,
)
from auto_round.inference.utils import _expand_regex_config
from auto_round.logger import logger
from auto_round.modeling.unfused_moe import apply_modeling_patch
from auto_round.schemes import QuantizationScheme
from auto_round.special_model_handler import update_module
from auto_round.utils import (
    SUPPORTED_LAYER_TYPES,
    apply_checkpoint_conversion_mapping,
    check_start_with_block_name,
    check_to_quantized,
    find_matching_blocks,
    get_block_names,
    get_checkpoint_conversion_mapping,
    get_module,
    is_hpex_available,
    is_transformers_version_greater_or_equal_5,
    set_module,
)

supported_devices = ("cpu", "hpu", "xpu", "cuda", "mps")


def flatten_list(nested_list):
    flattened = []
    for item in nested_list:
        if isinstance(item, (list, tuple)):
            flattened.extend(flatten_list(item))
        else:
            flattened.append(item)
    return flattened


def skip_not_convert_modules(model, quantization_config, layer_names, layer_configs):
    modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])
    try:  # transformers new api
        modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)
    except Exception:
        modules_to_not_convert = _get_modules_to_not_convert(model, modules_to_not_convert)
    if modules_to_not_convert:
        for layer_name in layer_names:
            if any(re.search(re.compile(n), layer_name) for n in modules_to_not_convert):
                layer_configs[layer_name] = {"bits": 16, "act_bits": 16}
    return layer_configs


def get_keys_to_not_convert(model):
    r"""
    An utility function to get the key of the module to keep in full precision if any For example for CausalLM modules
    we may want to keep the lm_head in full precision for numerical stability reasons. For other architectures, we want
    to keep the tied weights of the model. The function will return a list of the keys of the modules to not convert in
    int8.

    Parameters:
    model (`torch.nn.Module`):
        Input model
    """
    from copy import deepcopy

    from accelerate.utils import find_tied_parameters

    # Create a copy of the model and tie the weights, then
    # check if it contains tied weights
    tied_model = deepcopy(model)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
    if not hasattr(tied_model, "tie_weights") and not hasattr(model, "get_output_embeddings"):
        return []  # not a LLM model, could be a diffusers model

    tied_model.tie_weights()

    tied_params = find_tied_parameters(tied_model)
    # For compatibility with Accelerate < 0.18
    if isinstance(tied_params, dict):
        tied_keys = list(itertools.chain.from_iterable(tied_params.values())) + list(tied_params.keys())
    else:
        tied_keys = list(itertools.chain.from_iterable(tied_params))
    has_tied_params = len(tied_keys) > 0

    # If there is not tied weights, we want to keep the lm_head(output_embedding) in full precision
    if not has_tied_params:
        output_emb = model.get_output_embeddings()
        if output_emb is not None:
            list_last_module = [name for name, module in model.named_modules() if id(module) == id(output_emb)]
            return list_last_module

    # otherwise, no tied weights, no output embedding defined, simply keep the last module in full precision
    list_modules = list(model.named_parameters())
    list_last_module = [list_modules[-1][0]]
    # add last module together with tied weights
    intersection = set(list_last_module) - set(tied_keys)
    list_untouched = list(set(tied_keys)) + list(intersection)

    # remove ".weight" from the keys
    names_to_remove = [".weight", ".bias"]
    filtered_module_names = []
    for name in list_untouched:
        for name_to_remove in names_to_remove:
            if name_to_remove in name:
                name = name.replace(name_to_remove, "")
        filtered_module_names.append(name)

    return filtered_module_names


def _get_modules_to_not_convert(
    model,
    skip_modules=None,
    keep_in_fp32_modules=None,
    add_default_skips: bool = False,
):
    if skip_modules is None or add_default_skips:
        modules_to_not_convert = get_keys_to_not_convert(model)
    else:
        modules_to_not_convert = []

    if skip_modules is not None:
        modules_to_not_convert.extend(skip_modules)

    if keep_in_fp32_modules is not None:
        modules_to_not_convert.extend(keep_in_fp32_modules)

    return modules_to_not_convert


try:
    from transformers.quantizers.base import HfQuantizer

    get_modules_to_not_convert = HfQuantizer.get_modules_to_not_convert
except Exception:
    get_modules_to_not_convert = _get_modules_to_not_convert


def get_available_devices():
    """
    Returns a list of available devices in the current environment.

    Returns:
        List[str]: A list of device identifiers like "cuda", "hpu", "xpu", "cpu".
    """
    devices = []

    if torch.cuda.is_available():
        devices.append("cuda")

    if is_hpex_available():
        devices.append("hpu")

    if hasattr(torch, "xpu") and torch.xpu.is_available():
        devices.append("xpu")

    if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        devices.append("mps")

    devices.append("cpu")  # Always available

    return devices


def _remap_paths_for_text_model(model, quant_block_list, extra_config):
    """Remap quantization paths when a composite model checkpoint is loaded as its text sub-model.

    Uses Transformers' conversion_mapping rules (WeightRenaming, PrefixChange, etc.)
    to fix path mismatches. Returns (remapped_block_list, remapped_extra_config).
    """
    try:
        from transformers.conversion_mapping import get_checkpoint_conversion_mapping
    except ImportError:
        return quant_block_list, extra_config

    model_type = getattr(getattr(model, "config", None), "model_type", None)
    if model_type is None:
        return quant_block_list, extra_config

    mapping = get_checkpoint_conversion_mapping(model_type)

    # For composite models (e.g. VLMs), the composite model_type may not have a mapping,
    # but the text sub-model type does.
    if not mapping:
        text_config = getattr(getattr(model, "config", None), "text_config", None)
        text_model_type = getattr(text_config, "model_type", None)
        if text_model_type:
            mapping = get_checkpoint_conversion_mapping(text_model_type)

    if not mapping:
        return quant_block_list, extra_config

    # Accept any mapping type that has source_patterns and target_patterns
    renamings = [r for r in mapping if hasattr(r, "source_patterns") and hasattr(r, "target_patterns")]
    if not renamings:
        return quant_block_list, extra_config

    def remap(path):
        for r in renamings:
            for src, tgt in zip(r.source_patterns, r.target_patterns):
                new_path = re.sub(src, tgt, path)
                if new_path != path:
                    return new_path
        return path

    new_block_list = [remap(b) for b in quant_block_list]
    new_extra_config = {remap(k): v for k, v in extra_config.items()} if extra_config else extra_config

    if new_block_list != quant_block_list:
        logger.info(f"Remapped block_name_to_quantize: {quant_block_list} -> {new_block_list}")

    return new_block_list, new_extra_config


def get_layer_config(model, quantization_config):
    """
    get a layer-wise quantization configuration for a given model.

    Args:
        model (torch.nn.Module): The model for which quantization settings are generated.
        quantization_config (object): An object containing quantization parameters, including:
            - bits (int): Default bit width for quantization.
            - group_size (int): Group size for weight quantization.
            - data_type (str, optional): Data type for quantization (default: "int").
            - sym (bool): Whether to use symmetric quantization.
            - quant_block_list (list, optional): Predefined list of blocks to quantize.
            - to_quant_block_names (list or str, optional): Blocks to quantize (if quant_block_list is None).
            - extra_config (dict, optional): Per-layer overrides for quantization settings.
            - modules_in_block_to_quantize (list, optional): Specific modules within a block for quantization.
            - modules_to_not_convert (list, optional): Layers excluded from quantization (AWQ format).

    Returns:
        dict: A dictionary mapping layer names to their quantization configurations, where each layer has:
            - "bits" (int): Bit width for quantization.
            - "group_size" (int): Group size for quantization.
            - "data_type" (str): Data type used for quantization.
            - "sym" (bool): Whether symmetric quantization is applied.
    """
    bits = quantization_config.bits
    group_size = quantization_config.group_size
    data_type = getattr(quantization_config, "data_type", "int")  # Default to "int" if not specified
    sym = quantization_config.sym

    act_bits = getattr(quantization_config, "act_bits", None)
    act_group_size = getattr(quantization_config, "act_group_size", False)
    act_sym = getattr(quantization_config, "act_sym", None)
    act_data_type = getattr(quantization_config, "act_data_type", None)
    act_dynamic = getattr(quantization_config, "act_dynamic", False)

    rotation_config = getattr(quantization_config, "rotation_config", None)

    default_quant_scheme = QuantizationScheme(
        bits=bits,
        group_size=group_size,
        data_type=data_type,
        sym=sym,
        act_bits=act_bits,
        act_group_size=act_group_size,
        act_sym=act_sym,
        act_data_type=act_data_type,
        act_dynamic=act_dynamic,
        rotation_config=rotation_config,
    )

    # Determine the quantization block list
    checkpoint_conversion_mapping = get_checkpoint_conversion_mapping(model)

    # Determine whether to apply the conversion mapping.
    # If the model's module paths match the source patterns of the mapping, the model is
    # a composite model (e.g., VLM loaded via AutoModelForImageTextToText) whose paths are
    # already in checkpoint namespace — remapping would incorrectly alter them.
    # Only when the model is loaded as a text sub-model (e.g., via AutoModelForCausalLM)
    # do its paths differ from checkpoint namespace and require remapping.
    _should_remap = bool(checkpoint_conversion_mapping) and not any(
        re.match(src, name) for name, _ in model.named_modules() for src in checkpoint_conversion_mapping
    )

    quant_block_list = getattr(quantization_config, "quant_block_list", None)
    if quant_block_list is not None:
        # Handle nested list format: [[block1, block2, ...], ...] -> [prefix1, ...]
        if quant_block_list and isinstance(quant_block_list[0], (list, tuple)):
            for i in range(len(quant_block_list)):
                quant_block_list[i] = (
                    apply_checkpoint_conversion_mapping(
                        os.path.commonprefix(quant_block_list[i]).rstrip("."), checkpoint_conversion_mapping
                    )
                    if _should_remap
                    else os.path.commonprefix(quant_block_list[i]).rstrip(".")
                )
    elif quant_block_list is None:
        to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None)  # Prioritize this parameter
        if to_quant_block_names is None:
            to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
        if isinstance(to_quant_block_names, (list, tuple)):
            quant_block_list = flatten_list(to_quant_block_names)
        elif isinstance(to_quant_block_names, str):
            # Generate quant block names based on the given layer names
            quant_block_list = to_quant_block_names.split(",")
        else:
            # Find matching blocks if no explicit names are provided
            all_blocks = get_block_names(model, quant_vision=True)
            quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
            # Speed up the matching
            for i in range(len(quant_block_list)):
                quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")
        if _should_remap:
            for i in range(len(quant_block_list)):
                quant_block_list[i] = apply_checkpoint_conversion_mapping(
                    quant_block_list[i], checkpoint_conversion_mapping
                )

    # Get layer names that will be quantized
    layer_names = []
    for n, m in model.named_modules():
        if type(m) not in SUPPORTED_LAYER_TYPES:
            continue
        if check_start_with_block_name(n, quant_block_list):
            layer_names.append(n)

    # Load extra configuration if available
    extra_config = getattr(quantization_config, "extra_config", {})

    # Remap extra_config keys using conversion mapping (e.g. composite VLM paths to text sub-model paths)
    if _should_remap and extra_config:
        remapped_extra_config = {}
        for key, value in extra_config.items():
            new_key = apply_checkpoint_conversion_mapping(key, checkpoint_conversion_mapping)
            remapped_extra_config[new_key] = value
        extra_config = remapped_extra_config

    # When a composite model (e.g. VLM) is loaded as its text sub-model via AutoModelForCausalLM,
    # block_name_to_quantize may still reference composite-level paths (e.g. "model.language_model.layers")
    # while the actual module paths are "model.layers". Use conversion_mapping to remap if no layers matched.
    if not layer_names and quant_block_list:
        quant_block_list, extra_config = _remap_paths_for_text_model(model, quant_block_list, extra_config)
        for n, m in model.named_modules():
            if type(m) not in SUPPORTED_LAYER_TYPES:
                continue
            if check_start_with_block_name(n, quant_block_list):
                layer_names.append(n)

    # Process GPTQ format: identify modules that should be quantized
    if getattr(quantization_config, "modules_in_block_to_quantize", None):
        modules_in_block_to_quantize = flatten_list(
            quantization_config.modules_in_block_to_quantize
        )  # Flatten the list
        for layer_name in layer_names:
            if not any(re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize):
                extra_config[layer_name] = {"bits": 16}  # Default to 16-bit for unquantized layers

    # Expand GPTQ 'dynamic' config (regex-based)
    dynamic_config = getattr(quantization_config, "dynamic", None)
    from auto_round.export.export_to_autogptq.export import convert_from_autogptq_dynamic

    if dynamic_config and isinstance(dynamic_config, dict):
        extra_config = _expand_regex_config(
            regex_config=convert_from_autogptq_dynamic(dynamic_config),
            base_config=extra_config,
            layer_names=layer_names,
            model=model,
        )

    # AWQ format: exclude specified modules
    extra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)

    # Expand auto_round regex configs (regex-based)
    extra_config = _expand_regex_config(
        regex_config=extra_config, base_config=extra_config, layer_names=layer_names, model=model
    )

    # Merge and deduplicate
    layer_names = list(set(layer_names).union(extra_config.keys()))

    # Build final layer configs
    layer_configs = {}
    quant_scheme_attrs = QuantizationScheme.get_attributes()
    for layer_name in layer_names:
        layer_cfg_dict = {}
        layer_extra = extra_config.get(layer_name, {})
        for attr in quant_scheme_attrs:
            layer_cfg_dict[attr] = layer_extra.get(attr, getattr(default_quant_scheme, attr))
        layer_configs[layer_name] = QuantizationScheme.from_dict(layer_cfg_dict)

    return layer_configs


def get_device(obj: torch.Tensor | nn.Module) -> torch.device:
    if isinstance(obj, torch.Tensor):
        return obj.device
    return next(obj.parameters()).device


def _replace_by_quant_layers(
    module: nn.Module,
    layer_configs: dict,
    backend: str,
    target_device: str,
    packing_format: str,
) -> list:
    """Replaces linear layers in a module with quantized layers according to configs.

    This function iterates over each layer in `layer_configs`, checks if it requires
    quantization, determines the appropriate backend, creates a quantized layer, and
    replaces the original layer in the module.

    Args:
        module (nn.Module): The module containing layers to be quantized.
        layer_configs (dict): Configuration for each layer's quantization.
        backend (str): Default backend for quantization.
        target_device (str): Target device for execution ('cuda', 'cpu', 'hpu', etc.).
        packing_format (str): Packing format for the quantized layers.

    Returns:
        list: List of backends actually used for the layers.
    Raises:
        ValueError: If no compatible backend is found for a layer and `backend` is not "auto".
    """

    used_backends = []
    backend_cache = {}

    for layer_name, config in layer_configs.items():
        if not check_to_quantized(config):
            continue  # Skip layers that do not require quantization

        layer = get_module(module, layer_name)
        in_features, out_features = _get_layer_features(layer)
        if in_features is None:
            continue  # Skip unsupported layer types
        scheme_key = "_".join(f"{k}={v}" for k, v in config.items())
        key = f"{scheme_key}_{in_features}_{out_features}"
        if key in backend_cache:
            layer_backend = backend_cache[key]
        else:
            # Determine backend
            layer_backend = get_layer_backend(target_device, backend, packing_format, config, in_features, out_features)
            logger.trace(f"Got backend {layer_backend} for {layer_name}.")
            backend_cache[key] = layer_backend
            if layer_backend not in used_backends:
                used_backends.append(layer_backend)

        if not layer_backend:
            if backend != "auto":
                raise ValueError(
                    f"Backend {backend} is not compatible with layer {layer_name} with config {config},"
                    f" please set the backend='auto' and retry"
                )
            raise ValueError(f"No compatible backend found for layer {layer_name} with config {config}")

        logger.debug(f"{layer_name}: {layer_backend} backend is used")

        # Create and replace layer
        new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format)
        set_module(module, layer_name, new_layer)

        # Transformers may run model-specific initialization after loading. Mark both the
        # replacement and its direct parent so parent initializers do not access attributes
        # such as ``c_proj.weight`` that packed quantized layers intentionally do not expose.
        new_layer._is_hf_initialized = True
        parent_name, _, _ = layer_name.rpartition(".")
        parent_module = get_module(module, parent_name) if parent_name else module
        parent_module._is_hf_initialized = True
    return used_backends


def _get_layer_features(layer):
    """Extracts input and output feature dimensions for supported layers."""
    if type(layer) == nn.Linear:
        return layer.in_features, layer.out_features
    elif type(layer) == Conv1D:  # TODO: Verify correctness
        return layer.weight.shape[0], layer.weight.shape[1]
    return None, None  # Unsupported layer type


def _import_exllamav2_kernels():
    """Attempts to import ExLlamaV2 kernels for performance optimization."""
    try:
        from exllamav2_kernels import gemm_half_q_half, make_q_matrix  # pylint: disable=E0611, E0401
    except Exception:
        logger.warning_once(
            "AutoGPTQ ExLlamaV2 has not been installed, Please install it using the following command: "
            "`pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@b8b4127`"
        )
        logger.warning_once("try to fallback to other autogptq backends for now")


def _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format=None):
    """Creates a quantized layer using the appropriate class."""
    QuantLinear = dynamic_import_inference_linear(layer_backend, config, packing_format=packing_format)
    bias = layer.bias is not None

    # MLX backend with MLX packing format (native MLX checkpoint)
    if "mlx" in layer_backend and (packing_format is None or "mlx" in packing_format):
        return QuantLinear(
            bits=config["bits"],
            group_size=config["group_size"],
            infeatures=in_features,
            outfeatures=out_features,
            bias=bias,
        )

    if "humming" in layer_backend:
        return QuantLinear(
            bits=config["bits"],
            group_size=config["group_size"],
            infeatures=in_features,
            outfeatures=out_features,
            bias=bias,
            sym=config["sym"],
            weight_dtype=layer.weight.dtype,
        )

    if "auto_round_kernel" in layer_backend:

        return QuantLinear(
            bits=config["bits"],
            group_size=config["group_size"],
            sym=config["sym"],
            in_features=in_features,
            out_features=out_features,
            bias=bias,
            weight_dtype=layer.weight.dtype,
        )
    if (
        "awq" in layer_backend
        and f"{QuantLinear.__module__}.{QuantLinear.__class__.__name__}" == "auto_round_kernel.qlinear.QuantLinearAWQ"
    ):
        return QuantLinear.from_linear(
            layer, config["bits"], config["group_size"], init_only=True, has_zero_points=not config["sym"]
        )
    elif "awq" in layer_backend and "gptqmodel" in layer_backend:
        # gptqmodel AWQ QuantLinear — This matches the approach used
        # by transformers' replace_with_awq_linear().
        return QuantLinear(
            bits=config["bits"],
            group_size=config["group_size"],
            desc_act=False,
            sym=config["sym"],
            in_features=in_features,
            out_features=out_features,
            bias=bias,
            register_buffers=True,
        )
    elif "awq" in layer_backend:
        # autoawq WQLinear_GEMM
        return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True)
    elif "gptqmodel" in layer_backend:
        return QuantLinear(
            bits=config["bits"],
            group_size=config["group_size"],
            desc_act=False,
            sym=config["sym"],
            in_features=in_features,
            out_features=out_features,
            bias=bias,
        )
    elif (
        BackendDataType.FP8_STATIC.value in layer_backend
        or BackendDataType.MXFP8.value in layer_backend
        or BackendDataType.MXFP4.value in layer_backend
        or BackendDataType.NVFP4.value in layer_backend
        or BackendDataType.MXINT4.value in layer_backend
        or layer_backend == "auto_round:torch_nvfp4_e5m3"
        or layer_backend == "auto_round:cute_nvfp4_e5m3"
        or layer_backend == "auto_round:torch_nvfp4_e5m3_gs"
        or layer_backend == "auto_round:cute_nvfp4_e5m3_gs"
        or layer_backend == "auto_round:fake"
    ):
        return QuantLinear.from_original(config, layer)

    # Default quantized layer creation
    return QuantLinear(
        config["bits"],
        config["group_size"],
        in_features,
        out_features,
        bias,
        weight_dtype=layer.weight.dtype,
    )


def infer_target_device(device_map: dict | int | str | None = None) -> str:
    """Infers the target device from a device_map.

    Args:
        device_map (Optional[Union[Dict[Any, Any], int, str]]):
            - If None, defaults to "cpu".
            - If dict, checks values to infer the device type.
            - If int or str, assumes it represents a device.

    Returns:
        str: The inferred target device, e.g., "cpu" or "cuda".
    """
    if device_map is None:
        return "cpu"

    if isinstance(device_map, dict):
        for device in set(device_map.values()):
            if device not in ("cpu", "disk"):
                if isinstance(device, int):
                    return get_available_devices()[0]
                return str(device).split(":")[0]
        return "cpu"

    return get_available_devices()[0]


def convert_gptq_v1_to_v2_format(model: nn.Module):
    """Convert gptq v1 to v2 format to ensure compatible with gptqmodel:exllamav2 backend."""
    import gptqmodel  # pylint: disable=E0401

    if Version(gptqmodel.__version__) >= Version("7.0.0"):
        from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2Linear  # pylint: disable=E0401

        ExllamaV2QuantLinear = ExllamaV2Linear  # pylint: disable=E0401
    else:
        from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2QuantLinear  # pylint: disable=E0401

    for n, m in model.named_modules():
        if isinstance(m, ExllamaV2QuantLinear):
            if hasattr(m, "qzeros") and m.qzeros is not None and m.qzeros.dtype == torch.int32 and m.bits == 4:
                m.qzeros += 0b00010001000100010001000100010001
                logger.warning_once("Converting gptq v1 to v2 format")


def _maybe_convert_gptq_to_mlx(model: nn.Module, used_backends: list[str]) -> None:
    """On macOS with MLX available, convert GPTQ-format QuantLinear layers to MLX QuantLinearMLX.

    This is the MLX equivalent of the ARK post_init step: when an MLX backend was
    selected but the checkpoint layers were materialized in GPTQ packing format, we
    re-pack them into the MLX format so that ``mx.quantized_matmul`` can be used for
    hardware-accelerated inference on Apple Silicon. All conversion logic lives in
    :meth:`QuantLinearMLX.from_gptq`.
    """
    import platform

    if platform.system() != "Darwin":
        return

    # Only run if an MLX-related backend was selected for some layer.
    if not any("mlx" in b for b in used_backends):
        return

    try:
        import mlx.core as mx  # noqa: F401
    except ImportError:
        logger.debug("MLX not installed, skipping GPTQ-to-MLX conversion")
        return

    from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX

    # Collect GPTQ-style layers that need re-packing into MLX format.
    layers = []
    for name, module in model.named_modules():
        if not hasattr(module, "QUANT_TYPE"):
            continue
        if getattr(module, "QUANT_TYPE", None) == "mlx":
            continue
        if not hasattr(module, "qweight"):
            continue
        layers.append((name, module))

    if not layers:
        return

    # Get sym flag from model config (once, outside loop)
    quant_config = getattr(model.config, "quantization_config", {})
    if hasattr(quant_config, "to_dict"):
        quant_config = quant_config.to_dict()
    sym = quant_config.get("sym", False)

    converted = 0
    skipped = 0
    for name, module in tqdm(layers, desc="repacking to MLX format", total=len(layers), leave=True):
        try:
            mlx_layer = QuantLinearMLX.from_gptq(module, sym=sym)
            set_module(model, name, mlx_layer)
            converted += 1

        except Exception as e:
            logger.warning(f"Failed to convert layer '{name}' to MLX format: {e}")
            skipped += 1
            continue

    if converted > 0:
        logger.info(f"Auto-converted {converted} GPTQ layers to MLX format for Apple Silicon acceleration")
    if skipped > 0:
        logger.warning(f"Skipped {skipped} layers during GPTQ-to-MLX conversion")


def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
    """Performs post-initialization for different quantization backends.

    This function handles backend-specific post-init steps, including AutoGPTQ,
    GPTQModel, and ExLLaMAv2 kernels. It also ensures the
    model's data type is compatible with all used backends.

    Args:
        model (torch.nn.Module): The model to initialize.
        used_backends (List[str]): List of backend names used for quantization.

    """
    from auto_round.utils.common import monkey_patch_model

    monkey_patch_model(model)

    need_autogptq_init = False
    need_gptqmodel_init = False
    need_ark_init = False
    need_humming_init = False
    used_gptq_exllamav2 = False
    # Determine which backends require post-init
    for backend in used_backends:
        if backend.startswith("auto_gptq"):
            need_autogptq_init = True
            if backend == "auto_gptq:exllamav2":
                used_gptq_exllamav2 = True
        elif backend.startswith("gptqmodel"):
            need_gptqmodel_init = True
        elif backend.startswith("auto_round_kernel"):
            need_ark_init = True
        elif "humming" in backend:
            need_humming_init = True

    # AutoGPTQ post-init
    if need_autogptq_init:
        from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init  # pylint: disable=E0401

        model = gptq_post_init(model, use_act_order=False)

    # GPTQModel post-init
    if need_gptqmodel_init:
        from gptqmodel import __version__ as gptqmodel_version  # pylint: disable=E0401
        from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init  # pylint: disable=E0401
        from packaging import version

        packing_format = None
        if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
            quant_cfg = model.config.quantization_config
            if hasattr(quant_cfg, "packing_format"):
                packing_format = quant_cfg.packing_format

        if packing_format == "auto_round:gptq":
            # v1: auto_round:gptq; v2: auto_round:gptqmodel
            convert_gptq_v1_to_v2_format(model)  # Handle qzero layers if present
        if version.parse(gptqmodel_version) <= version.parse("5.6.0"):
            model = gptq_post_init(model, use_act_order=False)
        else:
            # for new version of gptqmodel, use validate_once to import kernels
            for n, m in model.named_modules():
                if hasattr(m, "validate_once"):
                    m.validate_once()
            model = gptq_post_init(model, use_act_order=False)

    # ARK post-init
    if need_ark_init:
        message = "repacking to CPU/XPU format"
        layers = []  ## ark post_init  will add one more layer
        for n, m in model.named_modules():
            if hasattr(m, "QUANT_TYPE") and "ark" in m.QUANT_TYPE:
                layers.append(m)

        for layer in tqdm(layers, desc=message, total=len(layers), leave=True):
            layer.post_init()

    # humming post-init: repack the checkpoint layout into the kernel-native one
    # and JIT-compile the selected GEMM.
    if need_humming_init:
        layers = [m for _, m in model.named_modules() if getattr(m, "QUANT_TYPE", "").startswith("humming")]
        for layer in tqdm(layers, desc="repacking to humming format", total=len(layers), leave=True):
            layer.post_init()

    # ExLLaMAv2 kernels
    if used_gptq_exllamav2:
        _import_exllamav2_kernels()

    # On macOS (Apple Silicon), auto-convert GPTQ layers to MLX for faster inference
    _maybe_convert_gptq_to_mlx(model, used_backends)

    # Determine common data type across backends
    data_types = [set(BackendInfos[b].compute_dtype) for b in used_backends]
    common_dtypes = set.intersection(*data_types) if data_types else set()

    # Force model dtype if needed
    model_dtype_name = str(model.dtype).split(".")[-1]
    if common_dtypes and model_dtype_name not in common_dtypes:
        target_dtype = None
        if "float16" in common_dtypes:
            target_dtype = torch.float16
        elif "bfloat16" in common_dtypes:
            target_dtype = torch.bfloat16

        if target_dtype:
            model = model.to(target_dtype)
            logger.warning(f"Forced model to {target_dtype}")

    # Rebuild SpinQuant online rotations after weights are loaded.
    # Buffers were pre-registered in convert_hf_model() and populated by
    # HuggingFace's state_dict loader. Now rebuild online rotations
    # (forward patching + R3 monkeypatch) via the generic dispatch.
    from auto_round.algorithms.transforms import rebuild_rotation_if_needed

    rebuild_rotation_if_needed(model)


def disable_moe_conversion_mapping(model):
    """Disables MoE-specific checkpoint conversion mappings to prevent unintended weight merging."""
    from transformers.conversion_mapping import (
        get_checkpoint_conversion_mapping,
        register_checkpoint_conversion_mapping,
    )
    from transformers.core_model_loading import WeightRenaming

    model_type = getattr(model.config, "model_type", None)
    if model_type is not None:
        conversions = get_checkpoint_conversion_mapping(model_type)
        if conversions is not None:
            # Keep only WeightRenaming, skip WeightConverter (MoE merge operations)
            filtered = [c for c in conversions if isinstance(c, WeightRenaming)]
            register_checkpoint_conversion_mapping(model_type, mapping=filtered, overwrite=True)


def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Module, list]:
    """Converts a HuggingFace model into an AutoRound model by replacing layers with quantized layers.

    This function extracts the quantization configuration from the model and updates its layers
    according to the specified quantization parameters. It supports different backends,
    sets appropriate packing formats, and ensures compatibility with the target device.

    Args:
        model (nn.Module): The HuggingFace model to be converted.
        target_device (str, optional): Device to run the model on.
            One of {"cuda", "cpu", "hpu", "xpu"}. Defaults to "cpu".

    Returns:
        Tuple[nn.Module, list]:
            The converted AutoRound model and a list of used backends.

    Raises:
        NotImplementedError: If the GPTQ model uses an unsupported `g_idx`.
        ValueError: If quantization backend is not properly specified.
    """
    if is_transformers_version_greater_or_equal_5():
        disable_moe_conversion_mapping(model)
    quantization_config = model.config.quantization_config
    config_format = (
        quantization_config.get("format")
        if isinstance(quantization_config, dict)
        else getattr(quantization_config, "format", None)
    )
    if config_format in ("nvfp4-e5m3-pack-quantized", "nvfp4-e5m3-gs-pack-quantized"):
        with_global_scale = config_format == "nvfp4-e5m3-gs-pack-quantized"
        config_dict = quantization_config if isinstance(quantization_config, dict) else quantization_config.to_dict()
        ignored = config_dict.get("ignore") or []
        # TODO: For experimental purpose, will delete when it's not necessary.
        quantization_config = SimpleNamespace(
            quant_method="auto-round",
            packing_format=(
                "auto_round:llm_compressor_nvfp4_e5m3_gs"
                if with_global_scale
                else "auto_round:llm_compressor_nvfp4_e5m3"
            ),
            bits=4,
            group_size=16,
            sym=True,
            data_type="nvfp4_v2_gs" if with_global_scale else "nvfp4_v2",
            act_bits=4,
            act_group_size=16,
            act_sym=True,
            act_data_type="nvfp4_v2_with_static_gs" if with_global_scale else "nvfp4_v2",
            act_dynamic=True,
            extra_config={name: {"bits": 16, "act_bits": 16} for name in ignored},
        )
        model.config.quantization_config = quantization_config

    # Check desc_act + static_groups
    if getattr(quantization_config, "desc_act", False):
        if not getattr(quantization_config, "static_groups", False):
            raise NotImplementedError(
                "This GPTQ model may contain a non-dummy g_idx, " "which is not yet supported by AutoRound."
            )

    # Determine backend
    backend = getattr(quantization_config, "backend", "auto")

    # Determine packing format
    if (
        hasattr(quantization_config, "packing_format") and "auto-round" in quantization_config.quant_method
    ):  # pragma: no cover
        packing_format = quantization_config.packing_format
    elif "gptq" in quantization_config.quant_method:  # pragma: no cover
        packing_format = "auto_round:auto_gptq"
    elif "awq" in quantization_config.quant_method:
        packing_format = "auto_round:auto_awq"
    else:  # pragma: no cover
        packing_format = "auto_round:auto_gptq"
        logger.warning("Quantization backend must be specified. " "Defaulting to 'auto_round:auto_gptq'.")

    if packing_format == "auto":
        packing_format = "auto_round:auto_gptq"
    elif packing_format == "auto_round:awq":  # normalize tricky settings
        packing_format = "auto_round:auto_awq"
    elif packing_format == "auto_round:gptq":
        packing_format = "auto_round:auto_gptq"
    elif packing_format in ("mlx", "auto_round:mlx"):
        pass  # keep as-is for MLX backend selection
    is_applied = apply_modeling_patch(model)
    if not is_applied:
        # Preprocess model before replace layers
        model = update_module(model, cleanup_original=True)

    # Replace layers with quantized versions
    layer_configs = get_layer_config(model, quantization_config)
    used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)
    logger.info(
        "Inference backend selection: requested=%s, packing_format=%s, selected=%s",
        backend,
        packing_format,
        ", ".join(used_backends),
    )

    # Apply rotation hooks (hadamard, spinquant, quarot, etc.) via unified dispatch.
    _has_rotation = getattr(quantization_config, "rotation_config", None) or getattr(
        quantization_config, "spinquant_config", None
    )
    if _has_rotation:
        from auto_round.algorithms.transforms import apply_rotation_hooks_from_config

        model = apply_rotation_hooks_from_config(model, quantization_config)

    # Suggest a better backend if available
    if backend == "auto":
        best_backend = get_highest_priority_backend(
            quantization_config,
            target_device,
            packing_format,
        )
        if best_backend and best_backend not in used_backends:
            requirements = BackendInfos[best_backend].requirements
            process_requirement(requirements, target_device, "warning")

    return model, used_backends