Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

48.6 KB
Raw Blame
# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import platform
from dataclasses import dataclass, field
from importlib import import_module
from typing import TYPE_CHECKING, Any

import torch
from packaging.version import Version
from transformers.utils.versions import require_version

import auto_round_extension.cuda.gptqmodel_marlin
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import get_library_version

BackendInfos = {}

import sys

import cpuinfo

if TYPE_CHECKING:
    from transformers import AutoRoundConfig


def get_cpu_manufacturer():
    cpu_info = cpuinfo.get_cpu_info()
    if "brand_raw" in cpu_info and "intel" in cpu_info["brand_raw"].lower():
        return "intel"
    else:
        return "others"


@dataclass
class BackendInfo:
    """Stores configuration details for various backend formats.

    Attributes:
        device: A list of strings representing the devices the backend supports
            (e.g., 'cpu', 'xpu', 'cuda').
        sym: A list of booleans indicating whether the backend supports symmetric
            quantization for weights (True if symmetric, False if not).
        packing_format: A list of strings representing the packing formats used by the backend
            (e.g., 'ark', 'triton').
        bits: A list of integers specifying the bit-widths supported by the backend
            for weight quantization (e.g., [2, 4, 8]).
        group_size: An optional list of integers specifying the group sizes supported
            for weight quantization. Group size determines how weights are grouped
            during quantization. Defaults to None.
        compute_dtype: An optional list of strings representing the compute data types
            supported by the backend (e.g., 'float32', 'bfloat16'). Defaults to None.
        data_type: An optional list of strings representing the data types
            supported for weight quantization (e.g., 'int', 'nv_fp'). Defaults to None.
        act_bits: An optional list of integers specifying the bit-widths supported
            for activation quantization (e.g., [8, 16]). Defaults to None.
        act_group_size: An optional list of integers specifying the group sizes
            supported for activation quantization. Defaults to None.
        act_sym: An optional list of booleans indicating whether the backend supports
            symmetric quantization for activations (True if symmetric, False if not).
            Defaults to None.
        act_data_type: An optional list of strings representing the data types
            supported for activations (e.g., 'mx_fp_rceil'). Defaults to None.
        act_dynamic: An optional list of booleans indicating whether the backend
            supports dynamic quantization for activations. Defaults to None.
        priority: An integer representing the backend's priority, where higher values
            indicate higher priority. Defaults to 0.
        checkers: A list of check functions (e.g., validation methods)
            used to verify whether the backend supports certain features. Defaults to
            an empty list.
        alias: An optional list of strings representing alternative names for the
            backend. Defaults to None.
        requirements: An optional list of strings specifying the library dependencies
            required by the backend (e.g., 'triton>=2.0'). Defaults to None.
        systems: An optional list of strings specifying the operating systems,(e.g., 'windows', 'linux', 'darwin').
    """

    device: list[str]  # TODO change to tuple
    sym: list[bool]
    packing_format: list[str]
    bits: list[int]
    compute_dtype: list[str] = None
    data_type: list[str] | None = None
    group_size: list[int] | None = None
    act_bits: list[int] | None = None
    act_group_size: list[int] | None = None
    act_sym: list[bool] | None = None
    act_data_type: list[str] | None = None
    act_dynamic: list[bool] | None = None
    priority: int = 0  ##higher is better
    checkers: list[Any] = field(default_factory=list)
    alias: list[str] | None = None
    requirements: list[str] | None = None
    systems: list[str] | None = None


BACKEND_ACT_ATTRS = [
    "act_bits",
    "act_group_size",
    "act_sym",
    "act_data_type",
    "act_dynamic",
]

MX_TENSOR_DATA_TYPES = [
    "mx_fp",
    "mx_fp_rceil",
    "mx_int",
]


def feature_multiply_checker(in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None):
    if out_feature_multiplier is None:
        out_feature_multiplier = in_feature_multiplier
    return in_feature % in_feature_multiplier == 0 and out_feature % out_feature_multiplier == 0


def feature_multiply_checker_group_size(
    in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None
):
    group_size = config["group_size"]
    if out_feature_multiplier is None:
        out_feature_multiplier = in_feature_multiplier
    return (
        in_feature % in_feature_multiplier == 0
        and out_feature % out_feature_multiplier == 0
        and in_feature % group_size == 0
    )


def feature_compatible_multiply_checker(
    in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None
):
    group_size = config["group_size"]
    if out_feature_multiplier is None:
        out_feature_multiplier = in_feature_multiplier
    compatible_flag = in_feature < group_size and (in_feature * out_feature) % group_size == 0
    return (
        in_feature % in_feature_multiplier == 0
        and out_feature % out_feature_multiplier == 0
        and (in_feature % group_size == 0 or compatible_flag)
    )


def in_feature_checker_group_size(in_feature, out_feature, config):
    group_size = config["group_size"]
    return in_feature % group_size == 0


feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)
feature_multiply_checker_16 = functools.partial(feature_multiply_checker, in_feature_multiplier=16)
in_output_feature_multiply_checker_32 = functools.partial(
    feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32
)
in_feature_multiply_checker_32 = functools.partial(
    feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=None
)
exllamav2_feature_checker = functools.partial(
    feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32
)
compatible_exllamav2_feature_checker = functools.partial(
    feature_compatible_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32
)

gptqmodel_marlin_feature_checker = functools.partial(
    feature_multiply_checker_group_size, in_feature_multiplier=1, out_feature_multiplier=64
)

mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)

ark_feature_checker = functools.partial(in_feature_checker_group_size)

# humming repacks weights with ``padded_shape_n % 64 == 0`` / ``padded_shape_k % 32 == 0``.
# humming can pad internally; requiring 32-multiples matches the checkpoint packing constraints.
humming_feature_checker = functools.partial(
    feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32
)


# humming is published only as a git repo (distribution name ``humming-kernels``),
# so it cannot be expressed as a PEP 440 requirement string like the other backends.
HUMMING_INSTALL_SPEC = "git+https://github.com/inclusionAI/humming.git"


def humming_available_checker():
    """Requirement hook: humming needs both the package and a CUDA device.

    Returns ``(is_available, install_spec)``; the spec is fed to ``pip install`` in the
    "better backend found" hint, so it must be the git URL rather than a version spec.
    """
    try:
        from auto_round_extension.humming.qlinear_humming import is_humming_available

        return is_humming_available(), HUMMING_INSTALL_SPEC
    except Exception:  # pragma: no cover - depends on the local environment
        return False, HUMMING_INSTALL_SPEC


def fp8_static_scheme_checker(
    in_feature: int,
    out_feature: int,
    config: QuantizationScheme,
    in_feature_multiplier: int | None = None,
    out_feature_multiplier: int | None = None,
):
    from auto_round.schemes import FP8_STATIC

    return config == FP8_STATIC


def cute_nvfp4_e5m3_checker(in_feature: int, out_feature: int, config: QuantizationScheme) -> bool:
    """Require CuTe DSL before selecting the optional NVFP4 E5M3 CUDA path."""
    del out_feature
    if not in_feature_checker_group_size(in_feature, 0, config):
        return False
    try:
        from auto_round_extension.cuda.cute_nvfp4_e5m3 import is_cute_dsl_available

        return torch.cuda.is_available() and is_cute_dsl_available() and torch.cuda.get_device_capability()[0] >= 8
    except (ImportError, RuntimeError):
        return False


GPTQ_FORMAT = ["auto_round:auto_gptq"]  # zp+-1
GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]
AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
FAKE_FORMAT = ["auto_round:fake", "auto_round:auto_gptq", "auto_round", "auto_round:auto_awq"]
NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]
NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3_gs"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]

# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot
# run on Apple Silicon. Restrict them so the MLX backend wins on Darwin and we
# don't try to load auto-round-lib on macOS.
_NON_DARWIN_SYSTEMS = ["linux", "windows"]

# AutoGPTQ is no longer maintained, supports transformers < 5.0.0
BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT,
    bits=[4],
    priority=3,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    # 16, 384,768 accuracy issue
    group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
    checkers=[compatible_exllamav2_feature_checker],
    alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
    requirements=["torch<2.6.0", "auto-gptq>=0.7.1", "transformers<5.0.0"],
)

BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT,
    bits=[2, 4, 8],
    group_size=None,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    priority=0,
    checkers=[exllamav2_feature_checker],
    alias=["auto_gptq:tritonv2"],
    requirements=["auto-gptq>=0.7.1", "triton>=2.0", "transformers<5.0.0"],
)

BackendInfos["auto_gptq:cuda"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT,
    bits=[2, 3, 4, 8],
    group_size=None,
    priority=1,
    checkers=[exllamav2_feature_checker],
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    alias=["auto_gptq:cuda"],
    requirements=[
        "auto-gptq>=0.7.1",
        "transformers<5.0.0",
    ],
)

# FP8 static quant
# Weight: FP8, per-channel, may be extended to per-tensor in future
# Activation: FP8, per-tensor
BackendInfos["auto_round:torch_fp8_static"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=["auto_round:fp8_static"],
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["fp"],
    bits=[8],
    priority=0,
    checkers=[fp8_static_scheme_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.6.0"],
)

# MXFP8
BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=MX_TENSOR_DATA_TYPES,
    group_size=[32],
    bits=[8],
    act_bits=[8],
    act_group_size=[32],
    act_sym=[True],
    act_data_type=MX_TENSOR_DATA_TYPES,
    act_dynamic=[True],
    priority=0,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.7.0"],
)

# MXFP4
BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=MX_TENSOR_DATA_TYPES,
    group_size=[32],
    bits=[4],
    act_bits=[4],
    act_group_size=[32],
    act_sym=[True],
    act_data_type=MX_TENSOR_DATA_TYPES,
    act_dynamic=[True],
    priority=0,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.7.0"],
)

# MXINT4
BackendInfos["auto_round:torch_mxint4"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=["auto_round"],
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=MX_TENSOR_DATA_TYPES,
    group_size=[32],
    bits=[4],
    act_bits=[4],
    act_group_size=[32],
    act_sym=[True],
    act_data_type=MX_TENSOR_DATA_TYPES,
    act_dynamic=[True],
    priority=0,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.12.0"],
)

# NVFP4

BackendInfos["auto_round:fake"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=FAKE_FORMAT,
    compute_dtype=["float32", "float16", "bfloat16"],
    bits=[1, 2, 3, 4, 5, 6, 7, 8],
    sym=[True, False],
    # Keep fake backend as a fallback path; prefer real kernels when available.
    priority=-1,
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.12.0"],
)

BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["nv_fp"],
    group_size=[16],
    bits=[4],
    act_bits=[4],
    act_group_size=[16],
    act_sym=[True],
    act_data_type=["nv_fp4_with_static_gs"],
    act_dynamic=[True],
    priority=0,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.7.0"],
)

BackendInfos["auto_round:torch_nvfp4_e5m3"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["nvfp4_v2"],
    group_size=[16],
    bits=[4],
    act_bits=[4],
    act_group_size=[16],
    act_sym=[True],
    act_data_type=["nvfp4_v2"],
    act_dynamic=[True],
    priority=3,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.12.0"],
)

BackendInfos["auto_round:cute_nvfp4_e5m3"] = BackendInfo(
    device=["cuda"],
    packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["nvfp4_v2"],
    group_size=[16],
    bits=[4],
    act_bits=[4],
    act_group_size=[16],
    act_sym=[True],
    act_data_type=["nvfp4_v2"],
    act_dynamic=[True],
    priority=6,
    checkers=[cute_nvfp4_e5m3_checker],
    alias=["cute_nvfp4_e5m3"],
    requirements=["auto-round>0.12.0"],
)

BackendInfos["auto_round:torch_nvfp4_e5m3_gs"] = BackendInfo(
    device=["xpu", "cuda", "cpu"],
    packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["nvfp4_v2_gs"],
    group_size=[16],
    bits=[4],
    act_bits=[4],
    act_group_size=[16],
    act_sym=[True],
    act_data_type=["nvfp4_v2_with_static_gs"],
    act_dynamic=[True],
    priority=3,
    checkers=[mxfp_nvfp_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>0.12.0"],
)

BackendInfos["auto_round:cute_nvfp4_e5m3_gs"] = BackendInfo(
    device=["cuda"],
    packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,
    sym=[True],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["nvfp4_v2_gs"],
    group_size=[16],
    bits=[4],
    act_bits=[4],
    act_group_size=[16],
    act_sym=[True],
    act_data_type=["nvfp4_v2_with_static_gs"],
    act_dynamic=[True],
    priority=6,
    checkers=[cute_nvfp4_e5m3_checker],
    alias=["cute_nvfp4_e5m3_gs"],
    requirements=["auto-round>0.12.0"],
)

BackendInfos["auto_round:tritonv2"] = BackendInfo(
    device=["cuda", "xpu"],
    data_type=["int"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT_NO_ZP,
    compute_dtype=["float16", "bfloat16"],
    bits=[2, 4, 8],
    priority=2,
    checkers=[feature_multiply_checker_32],
    alias=["auto_round", "tritonv2", "triton"],
    requirements=["triton>=2.0", "auto-round>=0.5.0"],
)

BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
    device=["cuda", "xpu"],
    sym=[True],
    packing_format=GPTQ_FORMAT,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 4, 8],
    priority=2,
    checkers=[feature_multiply_checker_32],
    alias=["tritonv2", "tritonv2_zp", "triton"],
    requirements=["triton>=2.0", "auto-round>=0.5.0"],
    # systems=["windows", "linux", "darwin"],
)

BackendInfos["auto_round:torch"] = BackendInfo(
    device=["cuda", "xpu", "cpu"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT_NO_ZP,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 3, 4, 5, 6, 7, 8],
    priority=0,
    checkers=[exllamav2_feature_checker],
    alias=["auto_round", "torch"],
    requirements=["auto-round>=0.5.1"],
)


BackendInfos["auto_round:torch_zp"] = BackendInfo(
    device=["cuda", "xpu", "cpu"],
    sym=[True],
    packing_format=GPTQ_FORMAT,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 3, 4, 5, 6, 7, 8],
    priority=0,
    checkers=[exllamav2_feature_checker],
    alias=["torch", "torch_zp"],
    requirements=["auto-round>=0.5.1"],
)

# humming: JIT-compiled CUDA GEMM kernels covering every weight bit-width from
# 2 to 8. This is the only fast path for 5/6/7-bit checkpoints, and it is also
# faster than the Triton kernels for the classic 2/3/4/8-bit ones. Priority is
# set above tritonv2 (2) but below marlin (6) so that existing 4-bit defaults
# keep using marlin where it is available.
BackendInfos["auto_round:humming"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT_NO_ZP,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 3, 4, 5, 6, 7, 8],
    priority=4,
    checkers=[humming_feature_checker],
    alias=["humming"],
    requirements=[humming_available_checker],
)

BackendInfos["auto_round:humming_zp"] = BackendInfo(
    device=["cuda"],
    # asym `auto_round:auto_gptq` stores ``zp - 1``, which humming does not model;
    # only the symmetric variant (where the zero point is implicit) is supported.
    sym=[True],
    packing_format=GPTQ_FORMAT,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 3, 4, 5, 6, 7, 8],
    priority=4,
    checkers=[humming_feature_checker],
    alias=["humming", "humming_zp"],
    requirements=[humming_available_checker],
)

BackendInfos["auto_round:humming_awq"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=AWQ_FORMAT,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    bits=[2, 3, 4, 5, 6, 7, 8],
    priority=4,
    checkers=[humming_feature_checker],
    alias=["humming", "humming_awq"],
    requirements=[humming_available_checker],
)

BackendInfos["gptqmodel:marlin"] = BackendInfo(
    device=["cuda"],
    sym=[True],
    packing_format=GPTQ_FORMAT_NO_ZP,
    bits=[4, 8],
    group_size=[-1, 32, 64, 128],
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    priority=6,
    checkers=[gptqmodel_marlin_feature_checker],
    alias=["marlin", "gptqmodel"],
    requirements=["gptqmodel>=2.0"],
)

BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(
    device=["cuda"],
    sym=[True],
    packing_format=GPTQ_FORMAT,
    bits=[4, 8],
    group_size=[-1, 32, 64, 128],
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    priority=6,
    checkers=[gptqmodel_marlin_feature_checker],
    alias=["marlin", "gptqmodel"],
    requirements=["gptqmodel>=2.0"],
)

BackendInfos["gptqmodel:exllamav2"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT + GPTQ_FORMAT_NO_ZP,
    bits=[4],
    group_size=[-1, 32, 64, 128],  ##16 seems has accuracy issue
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    priority=5,
    checkers=[exllamav2_feature_checker],
    alias=["exllamav2"],
    requirements=["gptqmodel>=2.0"],
)

BackendInfos["gptqmodel:awq_exllamav2"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=[-1, 32, 64, 128],
    priority=5,
    compute_dtype=["float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    checkers=[exllamav2_feature_checker],
    alias=["gptqmodel:awq", "gptqmodel:autoawq", "gptqmodel_awq", "exllamav2"],
    requirements=["gptqmodel>=5.6.0"],
)

BackendInfos["gptqmodel:awq_marlin"] = BackendInfo(
    device=["cuda"],
    sym=[True],
    packing_format=AWQ_FORMAT,
    bits=[4, 8],
    group_size=[-1, 32, 64, 128],
    priority=6,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    checkers=[gptqmodel_marlin_feature_checker],
    alias=["gptqmodel:autoawq_marlin", "gptqmodel_awq_marlin", "marlin"],
    requirements=["gptqmodel>=5.6.0"],
)

BackendInfos["gptqmodel:awq_gemm"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=[-1, 16, 32, 64, 128],
    priority=4,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    checkers=[feature_multiply_checker_16],
    alias=["gptqmodel:autoawq_gemm", "gptqmodel_awq_gemm"],
    requirements=["gptqmodel>=5.6.0"],
)

BackendInfos["gptqmodel:awq_torch"] = BackendInfo(
    device=["cuda", "cpu"],
    sym=[True, False],
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=[-1, 16, 32, 64, 128],
    priority=2,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    checkers=[feature_multiply_checker_16],
    alias=["gptqmodel:autoawq_torch", "gptqmodel_awq_torch", "torch"],
    requirements=["gptqmodel>=5.6.0"],
)

# autoawq backend - deprecated, kept for backward compatibility
BackendInfos["auto_awq:gemm"] = BackendInfo(
    device=["cuda"],
    sym=[True, False],  # Actually it is GEMM
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=None,
    priority=3,
    compute_dtype=["float16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    alias=["auto_awq:gemm", "awq", "awq:gemm", "auto_awq"],
    requirements=["autoawq"],
)

BackendInfos["auto_round_kernel"] = BackendInfo(
    device=["cpu"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT_NO_ZP,
    bits=[2, 4, 8],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
    systems=_NON_DARWIN_SYSTEMS,  # auto-round-lib targets x86; not for Apple Silicon
)

BackendInfos["auto_round_kernel_xpu"] = BackendInfo(
    device=["xpu"],
    sym=[True],
    packing_format=GPTQ_FORMAT_NO_ZP,
    bits=[4, 8],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
)

BackendInfos["auto_round_kernel_zp"] = BackendInfo(
    device=["cpu"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT,
    bits=[2, 4, 8],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
    systems=_NON_DARWIN_SYSTEMS,
)

BackendInfos["auto_round_kernel_zp_xpu"] = BackendInfo(
    device=["xpu"],
    sym=[True],
    packing_format=GPTQ_FORMAT,
    bits=[4, 8],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
)

BackendInfos["auto_round_kernel_awq"] = BackendInfo(
    device=["cpu"],
    sym=[True, False],
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
    systems=_NON_DARWIN_SYSTEMS,
)

BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(
    device=["xpu"],
    sym=[True],
    packing_format=AWQ_FORMAT,
    bits=[4],
    group_size=None,
    priority=6,
    checkers=[ark_feature_checker],
    alias=["ark"],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    requirements=["torch>=2.8.0", "auto-round-lib"],
)

BackendInfos["hpu"] = BackendInfo(
    device=["hpu"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT_NO_ZP,
    bits=[4],
    compute_dtype=["bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    alias=["hpu"],
    priority=0,
)

BackendInfos["hpu_zp"] = BackendInfo(
    device=["hpu"],
    sym=[True, False],
    packing_format=GPTQ_FORMAT,
    bits=[4],
    compute_dtype=["bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    alias=["hpu"],
    priority=0,
)

# MLX Backend for Apple Silicon (M1, M2, M3, etc.)
BackendInfos["mlx"] = BackendInfo(
    device=["cpu", "mps"],  # MLX runs on Apple Silicon (CPU or MPS)
    sym=[True, False],
    packing_format=["mlx"] + GPTQ_FORMAT + GPTQ_FORMAT_NO_ZP,
    bits=[2, 3, 4, 5, 6, 8],
    compute_dtype=["float32", "float16", "bfloat16"],
    data_type=["int"],
    act_bits=WOQ_DEFAULT_ACT_BITS,
    group_size=[-1, 32, 64, 128, 256, 512],
    priority=5,  # High priority for Apple Silicon users
    alias=["mlx"],
    requirements=["mlx>=0.16.0"],
    systems=["darwin"],  # Only on macOS
)


def check_compatible(
    backend_name: str,
    device: str,
    config: dict,
    packing_format: str,
    in_features: int,
    out_features: int,
    check_requirements=True,
):
    """Checks if the given configuration is compatible with the specified backend.

    Args:
        backend_name (str): The name of the backend to check compatibility for.
        device (str): The device on which the backend operates (e.g., 'cuda', 'cpu').
        config(dict): scheme
        packing_format (str): The packing format used by the backend (e.g., 'triton').
        in_features (int): The number of input features for the model layer.
        out_features (int): The number of output features for the model layer.
        check_requirements (bool): Whether check the requirement

    Returns:
        bool: True if the configuration is compatible with the backend, False otherwise.

    Raises:
        KeyError: If the backend_name is not found in BackendInfos.

    Compatibility checks:
    - Device must match one of the backend's supported devices.
    - Bit-width must be supported by the backend.
    - If group_size is required by the backend, it must match.
    - Symmetric or asymmetric quantization must be supported.
    - If the packing format matches exactly, all feature checks must pass.
    - If the packing format does not match, it must be convertible.
    """
    backend = BackendInfos[backend_name]
    # Check if the format is convertible when packing formats differ
    if packing_format in backend.packing_format:
        pass
    else:
        return False
    if backend.systems is not None:
        current_system = platform.system()
        systems = [s.lower() for s in backend.systems]
        if current_system.lower() not in systems:
            return False
    # Check scheme
    for key, value in config.items():
        backend_value = getattr(backend, key, None)
        if backend_value is not None and value not in backend_value:
            return False

    # Check if device is supported by the backend
    if device not in backend.device:
        return False

    for check in backend.checkers:
        if not check(in_features, out_features, config):
            return False

    if check_requirements and backend.requirements is not None:
        for requirement in backend.requirements:
            if isinstance(requirement, str):
                try:
                    require_version(requirement)
                except ImportError:
                    return False
            else:
                res, _ = requirement()
                return res

    return True


def dynamic_import_inference_linear(backend, config, packing_format=None):
    """Dynamically imports and returns the appropriate QuantLinear class based on the given backend.

    This function dynamically loads the correct `QuantLinear` class based on the backend and quantization
    configuration (e.g., ark, marlin, hpu, gptq, awq, mlx). It imports specific modules or raises
    errors if the required packages are not installed or the environment is not set up.

    Args:
        backend (str):
            The backend to be used for quantization (e.g., 'ark', 'marlin', 'hpu', 'gptq', 'awq', 'mlx').
        config (QuantizationScheme):
            The quantization configuration containing parameters like bits, group_size, and sym.

    Returns:
        class:
            The dynamically imported QuantLinear class that corresponds to the given backend configuration.

    Raises:
        ImportError:
            If required modules are missing for a backend (e.g., ark, GPTQ, auto_awq).
    """
    bits, group_size, sym = config["bits"], config["group_size"], config["sym"]

    # MLX backend
    if "mlx" in backend:
        # If loading a GPTQ-format checkpoint, use GPTQ QuantLinear for buffer name compatibility.
        # The layers will be converted to MLX in post_init.
        if packing_format and "mlx" not in packing_format:
            from auto_round_extension.torch.qlinear_torch import QuantLinear

            return QuantLinear
        from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX

        return QuantLinearMLX

    if "torch_fp8_static" in backend:
        return ar_qmodules.WeightFP8ActFP8StaticQuantLinear
    if "torch_mxfp8" in backend:
        return ar_qmodules.MXFP8QuantLinear
    if "torch_mxint4" in backend:
        return ar_qmodules.MXINT4QuantLinear
    if "torch_mxfp4" in backend:
        return ar_qmodules.MXFP4QuantLinear
    if backend == "auto_round:cute_nvfp4_e5m3_gs":
        return ar_qmodules.CuteNVFP4E5M3GSQuantLinear
    if backend == "auto_round:torch_nvfp4_e5m3_gs":
        return ar_qmodules.NVFP4E5M3GSQuantLinear
    if backend == "auto_round:cute_nvfp4_e5m3":
        return ar_qmodules.CuteNVFP4E5M3QuantLinear
    if backend == "auto_round:torch_nvfp4_e5m3":
        return ar_qmodules.NVFP4E5M3QuantLinear
    if "torch_nvfp4" in backend:
        return ar_qmodules.NVFP4QuantLinear
    if "auto_round:fake" in backend:
        return ar_qmodules.FakeActQuantLinear

    if "humming" in backend:
        try:
            import humming  # noqa: F401  # pylint: disable=E0401
        except Exception as e:
            raise ImportError(
                "Please install the humming kernels to use this backend, e.g.: "
                "`pip install git+https://github.com/inclusionAI/humming.git`"
            ) from e
        import auto_round_extension.humming.qlinear_humming as humming_qlinear

        if "awq" in backend:
            return humming_qlinear.QuantLinearAWQ
        elif "zp" in backend:
            return humming_qlinear.QuantLinearGPTQ
        else:  # plain auto_round packing must be checked last
            return humming_qlinear.QuantLinear

    if "auto_round_kernel" in backend or "ark" in backend:
        try:
            import auto_round_kernel as ark  # pylint: disable=E0611, E0401
        except Exception as e:
            raise ImportError("Please install auto-round-lib for CPU/XPU, e.g., using: pip install auto-round-lib")
        import auto_round_kernel.qlinear as qlinear  # pylint: disable=E0611, E0401

        if "zp" in backend:
            return qlinear.QuantLinearGPTQ
        elif "awq" in backend:
            return qlinear.QuantLinearAWQ
        else:  # auto_round must be at the end
            return qlinear.QuantLinear

    if "hpu" in backend:
        try:
            import habana_frameworks.torch.hpu  # pylint: disable=E0401
        except ImportError:
            raise ImportError("Please setup hpu environment before using hpu backend")

        if "zp" in backend:
            from auto_round_extension.hpu.qlinear_hpu_gptq import QuantLinear as QuantLinear_gptq

            return QuantLinear_gptq
        else:  # auto_round must be at the end
            from auto_round_extension.hpu.qlinear_hpu import QuantLinear

            return QuantLinear

    # Handle gptqmodel GPTQ/AWQ backends
    if "gptqmodel" in backend:
        return get_gptqmodel_infer_linear(backend, is_awq="awq" in backend)

    if "gptq" in backend and "gptqmodel" not in backend:
        return get_autogptq_infer_linear(backend, bits, group_size, sym)

    if "awq" in backend:
        # Fallback to autoawq for backward compatibility
        from awq.modules.linear import WQLinear_GEMM  # pylint: disable=E0401

        return WQLinear_GEMM

    if backend == "auto_round:tritonv2":
        from auto_round_extension.triton.qlinear_tritonv2 import QuantLinear

        return QuantLinear

    if backend == "auto_round:tritonv2_zp":
        from auto_round_extension.triton.qlinear_tritonv2_zp import QuantLinear

        return QuantLinear

    if backend == "auto_round:torch":
        from auto_round_extension.torch.qlinear_torch import QuantLinear

        return QuantLinear

    if backend == "auto_round:torch_zp":
        from auto_round_extension.torch.qlinear_torch_zp import QuantLinear

        return QuantLinear

    raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")


def safe_import_gptqmodel():
    """Safely import gptqmodel on CPU to avoid meta device issues."""
    dtype = torch.get_default_dtype()
    if dtype != torch.float32:
        torch.set_default_dtype(torch.float32)
    try:
        # When loaded via the "meta" device, `gptqmodel==6.0.3` raises an error (since the internal
        # loading process within the `transformers` library defaults to the "meta" device mode).
        # Therefore, it is necessary to first switch to the CPU to bypass this error, and then
        # switch back to the original data type once the loading process is complete.
        with torch.device("cpu"):
            import gptqmodel  # pylint: disable=E0401

            return gptqmodel
    finally:
        torch.set_default_dtype(dtype)


def get_gptqmodel_infer_linear(backend, is_awq=False):
    gptqmodel = safe_import_gptqmodel()
    new_version = Version(gptqmodel.__version__) >= Version("7.0.0")
    if is_awq:
        backend_specs = (
            ("marlin", "gptqmodel.nn_modules.qlinear.marlin_awq", "AwqMarlinLinear", "AwqMarlinQuantLinear"),
            (
                "exllamav2",
                "gptqmodel.nn_modules.qlinear.exllamav2_awq",
                "AwqExllamaV2Linear",
                "AwqExllamaV2QuantLinear",
            ),
            ("gemm", "gptqmodel.nn_modules.qlinear.gemm_awq", "AwqGEMMLinear", "AwqGEMMQuantLinear"),
            ("torch", "gptqmodel.nn_modules.qlinear.torch_awq", "AwqTorchLinear", "AwqTorchQuantLinear"),
        )
    else:
        backend_specs = (
            ("marlin", auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer),
            ("exllamav2", "gptqmodel.nn_modules.qlinear.exllamav2", "ExllamaV2Linear", "ExllamaV2QuantLinear"),
            ("tritonv2", "gptqmodel.nn_modules.qlinear.tritonv2", "TritonV2Linear", "TritonV2QuantLinear"),
            ("torch", "gptqmodel.nn_modules.qlinear.torch", "TorchLinear", "TorchQuantLinear"),
        )

    for spec in backend_specs:
        if spec[0] not in backend:
            continue
        if is_awq:
            _, module_path, new_name, legacy_name = spec
            module = import_module(module_path)
            return getattr(module, new_name) if new_version else getattr(module, legacy_name)
        if spec[0] == "marlin":
            return spec[1]()
        _, module_path, new_name, legacy_name = spec
        module = import_module(module_path)
        return getattr(module, new_name) if new_version else getattr(module, legacy_name)

    raise ValueError(f"Unsupported {backend}")


def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
    """Returns the appropriate QuantLinear class based on backend configuration.

    This function selects and dynamically imports the `QuantLinear` class according to the specified backend
    and its features, such as using Triton, ExLlama, Marlin, or Qigen for quantization.

    Args:
        backend (str):
            The backend to be used for quantization (e.g., 'triton', 'qigen', 'marlin', 'exllamav2').
        bits (int, optional):
            The number of bits used for quantization. Default is 4.
        group_size (int, optional):
            The group size for quantization. Default is 128.
        sym (bool, optional):
            Whether symmetric quantization is enabled. Default is False.

    Returns:
        class:
            The dynamically imported QuantLinear class for the given configuration.

    Raises:
        ImportError:
            If required packages or backends are not installed.
    """
    use_triton = False
    disable_exllamav2 = False
    disable_exllamav1 = False
    disable_marlin = True
    use_qigen = False
    use_tritonv2 = False

    # Determine backend configurations based on input string
    if "qigen" in backend:
        use_qigen = True
    elif "triton" in backend:
        use_triton = True
    elif "tritonv2" in backend:
        use_triton = False
        use_tritonv2 = True
    elif "marlin" in backend:
        use_triton = False
        disable_marlin = False
    elif "exllamav2" in backend:
        use_triton = False
        disable_exllamav2 = False
        disable_marlin = True
    elif "exllamav1" in backend:
        use_triton = False
        disable_marlin = True
    elif "cuda" in backend:
        use_triton = False
        disable_marlin = True
        disable_exllamav2 = True
        disable_exllamav1 = True

    from auto_gptq.utils.import_utils import dynamically_import_QuantLinear  # pylint: disable=E0401

    version = get_library_version("auto_gptq")
    from packaging.version import Version

    # Import the appropriate QuantLinear based on the version of auto_gptq
    if Version(version) < Version("0.7.2"):
        QuantLinear = dynamically_import_QuantLinear(
            use_triton=use_triton,
            desc_act=False,
            group_size=group_size,
            bits=bits,
            disable_exllama=disable_exllamav1,
            disable_exllamav2=disable_exllamav2,
            use_qigen=use_qigen,
            disable_marlin=disable_marlin,
        )
    else:
        QuantLinear = dynamically_import_QuantLinear(  # pylint: disable=E1123
            use_triton=use_triton,
            desc_act=False,
            group_size=group_size,
            bits=bits,
            disable_exllama=disable_exllamav1,
            disable_exllamav2=disable_exllamav2,
            use_qigen=use_qigen,
            use_marlin=not disable_marlin,
            use_tritonv2=use_tritonv2,
        )

    return QuantLinear


def find_backend(backend: str, orig_backend: str | None = None):
    """
    Finds the matching backend key based on the target backend name or its aliases.

    Args:
        backend (str): Name or alias of the target backend.
        orig_backend (str, optional): Original backend name to check compatibility. Defaults to None.

    Returns:
        str or None: Matching backend key if found and compatible; otherwise, None.
    """
    logger.trace(f"Finding backend for target: {backend}, original: {orig_backend}")

    matched_keys = [
        key for key, info in BackendInfos.items() if key == backend or (info.alias and backend in info.alias)
    ]

    if not matched_keys:
        return None

    if orig_backend is None:
        return matched_keys[0] if len(matched_keys) >= 1 else None

    orig_info = BackendInfos[orig_backend]

    for key in matched_keys:
        target_info = BackendInfos[key]
        if (
            target_info.packing_format == orig_info.packing_format
            or orig_info.packing_format in target_info.convertible_format
        ):
            return key

    raise ValueError(f"{backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry.")


def get_all_compatible_backend(
    device: str, packing_format: str, config: dict, in_features: int, out_features: int
) -> list[str]:
    # Find compatible backends
    compatible_backends = [
        key
        for key in BackendInfos
        if check_compatible(key, device, config, packing_format, in_features, out_features, check_requirements=False)
    ]

    # Return the first compatible backend or an empty list if none found
    return compatible_backends


def get_layer_backend(
    device: str, backend: str, packing_format: str, config: dict, in_features: int, out_features: int
) -> str:
    """Selects the most suitable backend for the layer based on compatibility and priority.

    This function first checks if the specified backend supports the layer with the provided configuration.
    If not, it iterates through other available backends,
    checking compatibility and returning the one with the highest priority.

    Args:
        device (str):
            The device on which the layer will run, e.g., 'cpu', 'cuda'.
        backend (str):
            The target backend to be used for this layer."auto","triton","gptqmodel", etc,
        packing_format (str):
            The original backend from which packing format information is retrieved.
        config (dict): Layer config.
        in_features (int):
            The number of input features for the layer.
        out_features (int):
            The number of output features for the layer.

    Returns:
        str:
            The selected backend that is compatible with the layer configuration.

    Raises:
        ValueError:
            If the specified backend is not supported.
            If no compatible backend is found for the given layer configuration.
    """

    backends = []
    if backend == "auto":
        backends = BackendInfos.keys()
    else:
        for key, info in BackendInfos.items():
            if backend == key or (info.alias and backend in info.alias):
                backends.append(key)

    # Find and store other compatible backends
    supported_backends = []
    for key in backends:
        if check_compatible(key, device, config, packing_format, in_features, out_features):
            logger.trace(f"Backend {key} is compatible")
            supported_backends.append(key)

    # Raise an error if no compatible backends are found
    if len(supported_backends) == 0:
        supported_backends_need_package = get_all_compatible_backend(
            device, packing_format, config, in_features, out_features
        )

        if len(supported_backends_need_package) > 0:
            supported_backends_need_package = sorted(
                supported_backends_need_package,
                key=lambda support_backend: BackendInfos[support_backend].priority,
                reverse=True,
            )
            backend_info = BackendInfos[supported_backends_need_package[0]]
            process_requirement(backend_info.requirements, target_device=device)

        return ""

    # Sort the compatible backends by priority and return the one with the highest priority
    supported_backends = sorted(
        supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority, reverse=True
    )

    return supported_backends[0]


def get_highest_priority_backend(
    quantization_config: "AutoRoundConfig", device: str, packing_format: str
) -> str | None:
    current_system = platform.system().lower()
    supported_backends = []
    for key, backend in BackendInfos.items():
        # Filter by operating system (e.g. MLX is Darwin-only; ark CPU
        # backends are non-Darwin only).
        if backend.systems is not None:
            if current_system not in [s.lower() for s in backend.systems]:
                continue

        # Check if device is supported by the backend
        if device not in backend.device:
            continue

        # Check if bit-width is supported
        if quantization_config.bits not in backend.bits:
            continue

        # Check if group_size is valid (if required by backend)
        if backend.group_size is not None and quantization_config.group_size not in backend.group_size:
            continue

        # Check if symmetric/asymmetric quantization is supported
        if quantization_config.sym not in backend.sym:
            continue

        # Check if the format is convertible when packing formats differ
        if packing_format in backend.packing_format:
            pass
        else:
            continue

        def _is_act_field_supported(backend, quantization, field_name):
            q_val = getattr(quantization, field_name, None)
            b_val = getattr(backend, field_name, None)
            # Case 1. quantization field is None, assume it is not used, so supported
            # Case 2. backend field is not None and contains the quantization field value
            return (q_val is None) or (b_val is not None and q_val in b_val)

        if not all(_is_act_field_supported(backend, quantization_config, field) for field in BACKEND_ACT_ATTRS):
            continue

        supported_backends.append(key)

    if len(supported_backends) > 0:

        supported_backends = sorted(
            supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority, reverse=True
        )
        return supported_backends[0]
    else:
        return None


def process_requirement(requirements: list, target_device="cuda", logger_level="error"):
    def log(message):
        (logger.warning if logger_level != "error" else logger.error)(message)

    def build_pip_commands(gptq_req, other_reqs):
        commands = []

        if gptq_req:
            commands.append(f"pip install -v {gptq_req} --no-build-isolation")

        if other_reqs:
            other_str = " ".join(other_reqs)
            commands.append(f"pip install {other_str}")

        return commands

    # Filter requirements
    missing_requirements = []
    for req in requirements:
        # Callable requirements (e.g. humming) report their own availability and
        # return the spec to hand to `pip install`.
        if callable(req):
            available, install_spec = req()
            if not available and install_spec:
                missing_requirements.append(install_spec)
            continue
        try:
            require_version(req)
        except Exception:
            missing_requirements.append(req)

    gptq_req = next((f'"{req}"' for req in missing_requirements if "gptqmodel" in req), None)
    other_reqs = [f'"{req}"' for req in missing_requirements if "gptqmodel" not in req]

    pip_cmds = build_pip_commands(gptq_req, other_reqs)
    if not pip_cmds:
        return

    # Instructional messages
    install_instructions = []

    prefix_msg = (
        "Better backend is found, please install all the following requirements to enable it."
        if logger_level != "error"
        else "Inference requires the following libraries. Please install all of them."
    )
    log(prefix_msg)

    for msg in install_instructions:
        log(msg)
        if logger_level == "error" and len(pip_cmds) == 0:
            sys.exit(-1)

    joined_cmds = " and ".join(f"`{cmd}`" for cmd in pip_cmds)
    if joined_cmds:
        log(joined_cmds)
        if logger_level == "error":
            sys.exit(-1)