Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[High Risk]Refine inference code (#840)

Wenhua Cheng162424de4ae2
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 1671d157d..78a8fc9d6 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -545,7 +545,7 @@ def tune(args):
eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
- if autoround.act_bits <= 8 or eval_gguf_model:
+ if (autoround.act_bits <= 8 and formats[-1] == "fake") or eval_gguf_model:
if eval_gguf_model:
# for file in os.listdir(eval_folder):
# gguf_file = file
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index be42b824a..a0df6b004 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -972,11 +972,7 @@ class BaseCompressor(object):
elif "auto_round" in format and (
is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)
):
- logger.warning(
- f"AutoRound supports exporting to format '{format}', "
- "but loading quantized models in this format is not yet supported. "
- "It is currently recommended to export to the 'llm_compressor' format."
- )
+ pass
elif format != "fake":
logger.warning(
"Currently only support to export auto_round format quantized model"
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
index cc717e699..5bbb36e90 100644
--- a/auto_round/inference/auto_quantizer.py
+++ b/auto_round/inference/auto_quantizer.py
@@ -57,7 +57,7 @@ AUTOROUND_MINIMUM_VERSION = version.parse("0.2")
def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:
# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
- try: ##TODO remove it later
+ try: # TODO remove it later
import auto_round
return True, auto_round.__version__
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index ce3e209cf..50ec2b120 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -14,14 +14,13 @@
import functools
from dataclasses import dataclass, field
-from typing import Any, List, Optional
+from typing import Any, Optional
from transformers.utils.versions import require_version
import auto_round_extension.cuda.gptqmodel_marlin
from auto_round import schemes as ar_schemes
from auto_round.experimental import qmodules as ar_qmodules
-from auto_round.export.export_to_autoround import AutoRoundFormat
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import get_library_version
@@ -56,8 +55,6 @@ class BackendInfo:
quantization. Defaults to None.
priority: An integer representing the backend's priority, where higher values
indicate higher priority. Defaults to 0.
- convertable_format: A list of strings specifying the formats that the backend
- can convert from. Defaults to an empty list.
checkers: A list of check functions (e.g., validation methods)
used to verify whether the backend supports certain features. Defaults to
an empty list.
@@ -65,17 +62,16 @@ class BackendInfo:
backend. Defaults to None.
"""
- device: List[str]
- sym: List[bool]
- packing_format: str
- bits: List[int]
- dtype: List[str] = None
- group_size: Optional[List[int]] = None
+ device: list[str] # TODO change to tuple
+ sym: list[bool]
+ packing_format: list[str]
+ bits: list[int]
+ dtype: list[str] = None
+ group_size: Optional[list[int]] = None
priority: int = 0 ##higher is better
- convertable_format: List[str] = field(default_factory=list)
- checkers: List[Any] = field(default_factory=list)
- alias: Optional[List[str]] = None
- requirements: Optional[List[str]] = None
+ checkers: list[Any] = field(default_factory=list)
+ alias: Optional[list[str]] = None
+ requirements: Optional[list[str]] = None
# TODO(Yi): Add more fields for activation dtype, group size, etc.
@@ -156,14 +152,19 @@ def mxfp4_scheme_checker(
return _scheme_checker_common(config, ar_schemes.MXFP4)
+GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1
+GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]
+AWQ_FORMAT = ["auto_round:auto_awq"]
+LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
+
BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
device=["cuda"],
sym=[True, False],
- packing_format="int32_zp",
+ packing_format=GPTQ_FORMAT,
bits=[4],
priority=5,
dtype=["float16"],
- ##16, 384,768 accuracy issue
+ # 16, 384,768 accuracy issue
group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
checkers=[exllamav2_feature_checker],
alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
@@ -173,7 +174,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
device=["cuda"],
sym=[True, False],
- packing_format="int32_zp",
+ packing_format=GPTQ_FORMAT,
bits=[2, 4, 8],
group_size=None,
dtype=["float16"],
@@ -186,51 +187,25 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
BackendInfos["auto_gptq:cuda"] = BackendInfo(
device=["cuda"],
sym=[True, False],
- packing_format="int32_zp",
+ packing_format=GPTQ_FORMAT,
bits=[2, 3, 4, 8],
group_size=None,
priority=1,
checkers=[exllamav2_feature_checker],
- alias=["auto_gptq:cuda"],
dtype=["float16"],
- convertable_format=["int32_zp"],
+ alias=["auto_gptq:cuda"],
requirements=[
"torch<2.6.0",
"auto-gptq>=0.7.1",
],
)
-BackendInfos["auto_round:tritonv2"] = BackendInfo(
- device=["cuda", "xpu"],
- sym=[True, False],
- packing_format="int32",
- dtype=["float16", "bfloat16"],
- bits=[2, 4, 8],
- priority=2,
- checkers=[feature_multiply_checker_32],
- alias=["auto_round", "tritonv2", "triton"],
- requirements=["triton>=2.0", "auto-round>=0.5.0"],
-)
-
-BackendInfos["auto_round:torch"] = BackendInfo(
- device=["cuda", "xpu", "cpu"],
- sym=[True, False],
- packing_format="int32",
- dtype=["float16", "bfloat16"],
- bits=[2, 3, 4, 8],
- priority=0,
- checkers=[exllamav2_feature_checker],
- alias=["auto_round", "torch"],
- requirements=["auto-round>=0.5.1"],
-)
-
# FP8 static quant
# Weight: FP8, per-channel, may be extended to per-tensor in future
# Activation: FP8, per-tensor
-
-BackendInfos["auto_round:fp8_static"] = BackendInfo(
+BackendInfos["auto_round:torch_fp8_static"] = BackendInfo(
device=["xpu", "cuda", "cpu"],
- packing_format="",
+ packing_format=["auto_round:fp8_static"],
sym=[True],
dtype=["float32", "float16", "bfloat16"],
bits=[8],
@@ -241,38 +216,47 @@ BackendInfos["auto_round:fp8_static"] = BackendInfo(
)
# MXFP8
-
-BackendInfos["auto_round:mxfp8"] = BackendInfo(
+BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(
device=["xpu", "cuda", "cpu"],
- packing_format="",
+ packing_format=LLM_COMPRESSOR_FORMAT,
sym=[True],
dtype=["float32", "float16", "bfloat16"],
bits=[8],
priority=0,
checkers=[mxfp8_scheme_checker],
- alias=["torch"],
+ alias=["auto_round", "torch"],
requirements=["auto-round>0.7.0"],
)
# MXFP4
-
-BackendInfos["auto_round:mxfp4"] = BackendInfo(
+BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(
device=["xpu", "cuda", "cpu"],
- packing_format="",
+ packing_format=LLM_COMPRESSOR_FORMAT,
sym=[True],
dtype=["float32", "float16", "bfloat16"],
bits=[4],
priority=0,
checkers=[mxfp4_scheme_checker],
- alias=["auto_round:llm_compressor"],
+ alias=["auto_round", "torch"],
requirements=["auto-round>0.7.0"],
)
+BackendInfos["auto_round:tritonv2"] = BackendInfo(
+ device=["cuda", "xpu"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT_NO_ZP,
+ dtype=["float16", "bfloat16"],
+ bits=[2, 4, 8],
+ priority=2,
+ checkers=[feature_multiply_checker_32],
+ alias=["auto_round", "tritonv2", "triton"],
+ requirements=["triton>=2.0", "auto-round>=0.5.0"],
+)
+
BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
device=["cuda", "xpu"],
- sym=[True], ## asym has accuracys
- # issue
- packing_format="int32_zp",
+ sym=[True],
+ packing_format=GPTQ_FORMAT,
dtype=["float16", "bfloat16"],
bits=[2, 4, 8],
priority=2,
@@ -281,10 +265,23 @@ BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
requirements=["triton>=2.0", "auto-round>=0.5.0"],
)
+BackendInfos["auto_round:torch"] = BackendInfo(
+ device=["cuda", "xpu", "cpu"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT_NO_ZP,
+ dtype=["float16", "bfloat16"],
+ bits=[2, 3, 4, 8],
+ priority=0,
+ checkers=[exllamav2_feature_checker],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>=0.5.1"],
+)
+
+
BackendInfos["auto_round:torch_zp"] = BackendInfo(
device=["cuda", "xpu", "cpu"],
sym=[True],
- packing_format="int32_zp",
+ packing_format=GPTQ_FORMAT,
dtype=["float16", "bfloat16"],
bits=[2, 3, 4, 8],
priority=0,
@@ -296,7 +293,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(
BackendInfos["gptqmodel:marlin"] = BackendInfo(
device=["cuda"],
sym=[True],
- packing_format="int32",
+ packing_format=GPTQ_FORMAT_NO_ZP,
bits=[4, 8],
group_size=[-1, 32, 64, 128],
dtype=["float16", "bfloat16"],
@@ -309,7 +306,7 @@ BackendInfos["gptqmodel:marlin"] = BackendInfo(
BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(
device=["cuda"],
sym=[True],
- packing_format="int32_zp",
+ packing_format=GPTQ_FORMAT,
bits=[4, 8],
group_size=[-1, 32, 64, 128],
dtype=["float16", "bfloat16"],
@@ -322,7 +319,7 @@ BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(
BackendInfos["gptqmodel:exllamav2"] = BackendInfo(
device=["cuda"],
sym=[True, False],
- packing_format="int32",
+ packing_format=GPTQ_FORMAT,
bits=[4],
group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue
dtype=["float16", "bfloat16"],
@@ -334,8 +331,8 @@ BackendInfos["gptqmodel:exllamav2"] = BackendInfo(
BackendInfos["auto_awq:gemm"] = BackendInfo(
device=["cuda"],
- sym=[True, False], ##actually is gemm
- packing_format="awq",
+ sym=[True, False], # Actually it is GEMM
+ packing_format=AWQ_FORMAT,
bits=[4],
group_size=None,
priority=5,
@@ -347,35 +344,34 @@ BackendInfos["auto_awq:gemm"] = BackendInfo(
BackendInfos["qbits"] = BackendInfo(
device=["cpu"],
sym=[True, False],
- packing_format="qbits",
+ packing_format=GPTQ_FORMAT_NO_ZP,
bits=[2, 4, 8],
group_size=None,
priority=1,
checkers=[],
alias=["itrex", "qbits"],
dtype=["float16", "bfloat16"],
- convertable_format=["int32"],
requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
BackendInfos["qbits_zp"] = BackendInfo(
device=["cpu"],
sym=[True, False],
- packing_format="qbits_zp",
+ packing_format=GPTQ_FORMAT,
bits=[2, 4, 8],
group_size=None,
dtype=["float16", "bfloat16"],
priority=1,
checkers=[],
alias=["itrex", "qbits"],
- convertable_format=["int32_zp"],
requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
-BackendInfos["auto_round:qbits_awq"] = BackendInfo(
+
+BackendInfos["qbits_awq"] = BackendInfo(
device=["cpu"],
- sym=[True, False], ## for awq, not robust
- packing_format="awq",
+ sym=[True, False],
+ packing_format=AWQ_FORMAT,
bits=[2, 4, 8],
group_size=None,
dtype=["float16", "bfloat16"],
@@ -384,17 +380,15 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(
alias=["itrex", "qbits"],
requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
-
BackendInfos["ipex_gptq"] = BackendInfo(
device=["cpu", "xpu"],
sym=[True, False],
- packing_format="ipex_gptq",
+ packing_format=GPTQ_FORMAT,
bits=[4],
group_size=None,
priority=5,
checkers=[],
dtype=["float16", "bfloat16"],
- convertable_format=["int32_zp"],
alias=["ipex"],
requirements=["intel-extension-for-pytorch>=2.5"],
)
@@ -402,50 +396,51 @@ BackendInfos["ipex_gptq"] = BackendInfo(
BackendInfos["ipex_awq"] = BackendInfo(
device=["cpu", "xpu"],
sym=[True, False],
- packing_format="ipex_awq",
+ packing_format=AWQ_FORMAT,
bits=[4],
group_size=None,
- priority=1,
- dtype=["float16", "bfloat16"],
+ priority=5,
checkers=[],
+ dtype=["float16", "bfloat16"],
alias=["ipex"],
- convertable_format=["awq"],
- requirements=["intel-extension-for-pytorch>=2.6"],
+ requirements=["intel-extension-for-pytorch>=2.5"],
)
-
BackendInfos["hpu"] = BackendInfo(
device=["hpu"],
sym=[True, False],
- packing_format="hpu",
+ packing_format=GPTQ_FORMAT_NO_ZP,
bits=[4],
dtype=["bfloat16"],
alias=["hpu"],
priority=0,
- convertable_format=["int32"],
)
BackendInfos["hpu_zp"] = BackendInfo(
device=["hpu"],
sym=[True, False],
- packing_format="hpu_zp",
+ packing_format=GPTQ_FORMAT,
bits=[4],
dtype=["bfloat16"],
alias=["hpu"],
priority=0,
- convertable_format=["int32_zp"],
)
-def check_compatible(backend_name, device, config, packing_format, in_features, out_features, check_requirements=True):
+def check_compatible(
+ backend_name: str,
+ device: str,
+ config: dict,
+ packing_format: str,
+ in_features: int,
+ out_features: int,
+ check_requirements=True,
+):
"""Checks if the given configuration is compatible with the specified backend.
Args:
backend_name (str): The name of the backend to check compatibility for.
device (str): The device on which the backend operates (e.g., 'cuda', 'cpu').
- bits (int): The bit-width of the quantization (e.g., 2, 4, 8).
- group_size (Optional[int]): The size of the quantization group. Can be None if
- not required by the backend.
- sym (bool): Whether symmetric quantization is required (True for symmetric).
+ config(dict): scheme
packing_format (str): The packing format used by the backend (e.g., 'triton').
in_features (int): The number of input features for the model layer.
out_features (int): The number of output features for the model layer.
@@ -484,7 +479,7 @@ def check_compatible(backend_name, device, config, packing_format, in_features,
return False
# Check if the format is convertible when packing formats differ
- if packing_format == backend.packing_format or packing_format in backend.convertable_format:
+ if packing_format in backend.packing_format:
pass
else:
return False
@@ -534,11 +529,11 @@ def dynamic_import_inference_linear(backend, config):
"""
bits, group_size, sym = config["bits"], config["group_size"], config["sym"]
- if AutoRoundFormat.FP8_STATIC.value in backend:
+ if "torch_fp8_static" in backend:
return ar_qmodules.WeightFP8ActFP8StaticQuantLinear
- if AutoRoundFormat.MXFP8.value in backend:
+ if "torch_mxfp8" in backend:
return ar_qmodules.MXFP8QuantLinear
- if AutoRoundFormat.MXFP4.value in backend:
+ if "torch_mxfp4" in backend:
return ar_qmodules.MXFP4QuantLinear
if "qbits" in backend:
@@ -734,22 +729,21 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
return QuantLinear
-def find_backend(target_backend: str, orig_backend: str = None):
+def find_backend(backend: str, orig_backend: str = None):
"""
Finds the matching backend key based on the target backend name or its aliases.
Args:
- target_backend (str): Name or alias of the target backend.
+ backend (str): Name or alias of the target backend.
orig_backend (str, optional): Original backend name to check compatibility. Defaults to None.
Returns:
str or None: Matching backend key if found and compatible; otherwise, None.
"""
- logger.trace(f"Finding backend for target: {target_backend}, original: {orig_backend}")
+ logger.trace(f"Finding backend for target: {backend}, original: {orig_backend}")
+
matched_keys = [
- key
- for key, info in BackendInfos.items()
- if key == target_backend or (info.alias and target_backend in info.alias)
+ key for key, info in BackendInfos.items() if key == backend or (info.alias and backend in info.alias)
]
if not matched_keys:
@@ -768,15 +762,12 @@ def find_backend(target_backend: str, orig_backend: str = None):
):
return key
- raise ValueError(
- f"{target_backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry."
- )
+ raise ValueError(f"{backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry.")
-def get_all_compatible_backend(device, backend, orig_backend, config, in_features, out_features):
- # Get packing format from the original backend
- packing_format = BackendInfos[orig_backend].packing_format
-
+def get_all_compatible_backend(
+ device: str, packing_format: str, config: dict, in_features: int, out_features: int
+) -> list[str]:
# Find compatible backends
compatible_backends = [
key
@@ -788,7 +779,9 @@ def get_all_compatible_backend(device, backend, orig_backend, config, in_feature
return compatible_backends
-def get_layer_backend(device, backend, orig_backend, config, in_features, out_features):
+def get_layer_backend(
+ device: str, backend: str, packing_format: str, config: dict, in_features: int, out_features: int
+) -> str:
"""Selects the most suitable backend for the layer based on compatibility and priority.
This function first checks if the specified backend supports the layer with the provided configuration.
@@ -799,15 +792,10 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
device (str):
The device on which the layer will run, e.g., 'cpu', 'cuda'.
backend (str):
- The target backend to be used for this layer.
- orig_backend (str):
+ The target backend to be used for this layer."auto","triton","gptqmodel", etc,
+ packing_format (str):
The original backend from which packing format information is retrieved.
- bits (int):
- The number of bits used for quantization.
- group_size (int):
- The group size for quantization.
- sym (bool):
- Whether symmetric quantization is enabled.
+ config (dict): Layer config.
in_features (int):
The number of input features for the layer.
out_features (int):
@@ -822,17 +810,18 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
If the specified backend is not supported.
If no compatible backend is found for the given layer configuration.
"""
- # Check if the provided backend is in BackendInfos
- backend = find_backend(backend)
- logger.trace(f"Found backend: {backend} for device: {device}, orig_backend: {orig_backend}")
- if backend not in BackendInfos.keys():
- raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")
- packing_format = BackendInfos[orig_backend].packing_format
+ backends = []
+ if backend == "auto":
+ backends = BackendInfos.keys()
+ else:
+ for key in BackendInfos.keys():
+ if backend == key or (BackendInfos[key].alias and backend in BackendInfos[key].alias):
+ backends.append(key)
# Find and store other compatible backends
supported_backends = []
- for key in BackendInfos.keys():
+ for key in backends:
if check_compatible(key, device, config, packing_format, in_features, out_features):
logger.trace(f"Backend {key} is compatible")
supported_backends.append(key)
@@ -840,7 +829,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
# Raise an error if no compatible backends are found
if len(supported_backends) == 0:
supported_backends_need_package = get_all_compatible_backend(
- device, backend, orig_backend, config, in_features, out_features
+ device, packing_format, config, in_features, out_features
)
if len(supported_backends_need_package) > 0:
@@ -852,7 +841,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
backend_info = BackendInfos[supported_backends_need_package[0]]
process_requirement(backend_info.requirements, target_device=device)
- return None
+ return ""
# Sort the compatible backends by priority and return the one with the highest priority
supported_backends = sorted(
@@ -883,7 +872,7 @@ def get_highest_priority_backend(bits, sym, group_size, device, packing_format):
continue
# Check if the format is convertible when packing formats differ
- if packing_format == backend.packing_format or packing_format in backend.convertable_format:
+ if packing_format in backend.packing_format:
pass
else:
continue
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 23c01003d..bebf75df7 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -13,7 +13,6 @@
# limitations under the License.
import os
import re
-from logging import getLogger
from typing import Union
import torch
@@ -25,7 +24,6 @@ from auto_round.export.export_to_autoround import AutoRoundFormat
from auto_round.inference.backend import (
BackendInfos,
dynamic_import_inference_linear,
- find_backend,
get_highest_priority_backend,
get_layer_backend,
process_requirement,
@@ -38,7 +36,6 @@ from auto_round.utils import (
check_to_quantized,
find_matching_blocks,
get_block_names,
- get_layer_names_in_block,
get_module,
is_hpex_available,
set_module,
@@ -175,40 +172,6 @@ def get_available_devices():
return devices
-def parse_target_device_and_backend(target_backend: str):
- """
- Parse the target device and backend from the given target_backend string.
-
- Args:
- target_backend (str): A string specifying the target backend, which may include a device prefix.
-
- Returns:
- tuple: (target_device, backend), where:
- - target_device (str): The identified device (e.g., "cpu", "cuda", "auto").
- - backend (str): The backend string after removing the device prefix.
- """
-
- def remove_device_prefix(backend: str, prefix: str) -> str:
- """Removes the given device prefix from the backend string, if present."""
- return backend[len(prefix) :].lstrip(":") if backend.startswith(prefix) else backend
-
- # Handle "auto" case explicitly
- if target_backend == "auto":
- return "auto", ""
- if target_backend.startswith("auto:"):
- return "auto", target_backend[5:].lstrip(":")
-
- # Define supported devices
-
- # Identify the target device and adjust the backend string accordingly
- for device in supported_devices:
- if target_backend.startswith(device):
- return device, remove_device_prefix(target_backend, device)
-
- # Default case: If no known device is found, assume "auto"
- return "auto", target_backend
-
-
def get_layer_config(model, quantization_config):
"""
get a layer-wise quantization configuration for a given model.
@@ -232,7 +195,6 @@ def get_layer_config(model, quantization_config):
- "group_size" (int): Group size for quantization.
- "data_type" (str): Data type used for quantization.
- "sym" (bool): Whether symmetric quantization is applied.
- - "clip" (bool): Whether weight clipping is enabled.
"""
bits = quantization_config.bits
group_size = quantization_config.group_size
@@ -272,7 +234,7 @@ def get_layer_config(model, quantization_config):
# Find matching blocks if no explicit names are provided
all_blocks = get_block_names(model, quant_vision=True)
quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
- ##speed up the matching
+ # Speed up the matching
for i in range(len(quant_block_list)):
quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")
@@ -314,45 +276,39 @@ def get_layer_config(model, quantization_config):
return layer_configs
-def get_device(obj: Union[torch.Tensor, nn.Module]):
+def get_device(obj: Union[torch.Tensor, nn.Module]) -> torch.device:
if isinstance(obj, torch.Tensor):
return obj.device
return next(obj.parameters()).device
-def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, target_device, orig_backend):
- """
- Replaces linear layers in the given module with quantized layers.
+def _replace_by_quant_layers(
+ module: nn.Module,
+ layer_configs: dict,
+ backend: str,
+ target_device: str,
+ packing_format: str,
+) -> list:
+ """Replaces linear layers in a module with quantized layers according to configs.
+
+ This function iterates over each layer in `layer_configs`, checks if it requires
+ quantization, determines the appropriate backend, creates a quantized layer, and
+ replaces the original layer in the module.
Args:
module (nn.Module): The module containing layers to be quantized.
layer_configs (dict): Configuration for each layer's quantization.
- target_backend (str): Backend for quantization (device and format).
- target_device (str): Device for execution ('cuda', 'cpu', 'hpu').
- orig_backend (str): Original backend of the model.
+ backend (str): Default backend for quantization.
+ target_device (str): Target device for execution ('cuda', 'cpu', 'hpu', etc.).
+ packing_format (str): Packing format for the quantized layers.
Returns:
- dict: Flags indicating which backends were used.
+ list: List of backends actually used for the layers.
+ Raises:
+ ValueError: If no compatible backend is found for a layer and `backend` is not "auto".
"""
used_backends = []
- must_use_target_backend = False
- if target_backend:
- must_use_target_backend = True
- layer_backend = target_backend
- layer_backend_must = find_backend(layer_backend, orig_backend)
- used_backends.append(layer_backend_must)
- if layer_backend_must is None:
- raise ValueError(f"{target_backend} is not compatible, please change backend to `auto` and retry")
- devices = BackendInfos[layer_backend_must].device
- if target_device not in devices:
- raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")
- ##check requirements
- requirements = BackendInfos[layer_backend_must].requirements
- process_requirement(requirements, target_device)
-
- target_backend = target_backend or orig_backend # Default to original backend if not specified
-
backend_cache = {}
for layer_name, config in layer_configs.items():
@@ -363,27 +319,27 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, t
in_features, out_features = _get_layer_features(layer)
if in_features is None:
continue # Skip unsupported layer types
-
- if must_use_target_backend:
- layer_backend = layer_backend_must
+ scheme_key = "_".join(f"{k}={v}" for k, v in config.items())
+ key = f"{scheme_key}_{in_features}_{out_features}"
+ if key in backend_cache:
+ layer_backend = backend_cache[key]
else:
- key = f"{config['bits']}_{config['group_size']}_{config['sym']}_{in_features}_{out_features}"
- if key in backend_cache:
- layer_backend = backend_cache[key]
- if layer_backend not in used_backends:
- used_backends.append(layer_backend)
- else:
- # Determine backend
- layer_backend = _get_layer_backend(
- target_device, target_backend, orig_backend, config, in_features, out_features
- )
- logger.trace(f"Got backend {layer_backend} for {layer_name}.")
- backend_cache[key] = layer_backend
+ # Determine backend
+ layer_backend = get_layer_backend(target_device, backend, packing_format, config, in_features, out_features)
+ logger.trace(f"Got backend {layer_backend} for {layer_name}.")
+ backend_cache[key] = layer_backend
+ if layer_backend not in used_backends:
+ used_backends.append(layer_backend)
if not layer_backend:
+ if backend != "auto":
+ raise ValueError(
+ f"Backend {backend} is not compatible with layer {layer_name} with config {config},"
+ f" please set the backend='auto' and retry"
+ )
raise ValueError(f"No compatible backend found for layer {layer_name} with config {config}")
- logger.debug(f"{layer_name}: {layer_backend} backend is used") # TODO delete
+ logger.debug(f"{layer_name}: {layer_backend} backend is used")
# Create and replace layer
new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)
@@ -401,21 +357,6 @@ def _get_layer_features(layer):
return None, None # Unsupported layer type
-def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features, out_features):
- """Determines the best backend for a given layer."""
-
- backend = get_layer_backend(
- target_device,
- target_backend,
- orig_backend,
- config,
- in_features,
- out_features,
- )
-
- return backend
-
-
def _import_exllamav2_kernels():
"""Attempts to import ExLlamaV2 kernels for performance optimization."""
try:
@@ -458,64 +399,85 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)
or AutoRoundFormat.MXFP4.value in layer_backend
):
return QuantLinear.from_original(config, layer)
+
# Default quantized layer creation
- try:
- return QuantLinear(
- config["bits"],
- config["group_size"],
- in_features,
- out_features,
- bias,
- weight_dtype=layer.weight.dtype,
- clip=config["clip"],
- )
- except: # Handle cases where `clip` is not a valid argument
- return QuantLinear(
- config["bits"], config["group_size"], in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
+ return QuantLinear(
+ config["bits"],
+ config["group_size"],
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ )
-def infer_target_device(device_map=None):
+def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:
+ """Infers the target device from a device_map.
+
+ Args:
+ device_map (Optional[Union[Dict[Any, Any], int, str]]):
+ - If None, defaults to "cpu".
+ - If dict, checks values to infer the device type.
+ - If int or str, assumes it represents a device.
+
+ Returns:
+ str: The inferred target device, e.g., "cpu" or "cuda".
+ """
if device_map is None:
- target_device = "cpu"
- elif isinstance(device_map, dict):
- devices = set(device_map.values())
- target_device = "cpu"
- for device in devices:
- if device != "cpu" and device != "disk":
+ return "cpu"
+
+ if isinstance(device_map, dict):
+ for device in set(device_map.values()):
+ if device not in ("cpu", "disk"):
if isinstance(device, int):
- target_device = get_available_devices()[0]
- else:
- target_device = str(device).split(":")[0]
- else:
- target_device = get_available_devices()[0]
- return target_device
+ return get_available_devices()[0]
+ return str(device).split(":")[0]
+ return "cpu"
+
+ return get_available_devices()[0]
-def post_init(model, used_backends):
+def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
+ """Performs post-initialization for different quantization backends.
+
+ This function handles backend-specific post-init steps, including AutoGPTQ,
+ GPTQModel, IPEX/ITREx layers, and ExLLaMAv2 kernels. It also ensures the
+ model's data type is compatible with all used backends.
+
+ Args:
+ model (torch.nn.Module): The model to initialize.
+ used_backends (List[str]): List of backend names used for quantization.
+
+ """
need_autogptq_init = False
need_gptqmodel_init = False
need_ipex_itrex_init = False
used_gptq_exllamav2 = False
- for used_backend in used_backends:
- if used_backend.startswith("auto_gptq"):
+
+ # Determine which backends require post-init
+ for backend in used_backends:
+ if backend.startswith("auto_gptq"):
need_autogptq_init = True
- if used_backend == "auto_gptq:exllamav2":
+ if backend == "auto_gptq:exllamav2":
used_gptq_exllamav2 = True
- elif used_backend.startswith("gptqmodel"):
+ elif backend.startswith("gptqmodel"):
need_gptqmodel_init = True
- elif used_backend.startswith("ipex"):
- need_ipex_itrex_init = True
- elif used_backend.startswith("qbit"):
+ elif backend.startswith(("ipex", "qbit")):
need_ipex_itrex_init = True
+
+ # AutoGPTQ post-init
if need_autogptq_init:
from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init # pylint: disable=E0401
model = gptq_post_init(model, use_act_order=False)
+
+ # GPTQModel post-init
if need_gptqmodel_init:
from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401
model = gptq_post_init(model, use_act_order=False)
+
+ # IPEX/ITREx post-init
if need_ipex_itrex_init:
message = "repacking to CPU/XPU format"
layers = [] ## ipex post_init will add one more layer
@@ -526,98 +488,94 @@ def post_init(model, used_backends):
for layer in tqdm(layers, desc=message, total=len(layers), leave=True):
layer.post_init()
+ # ExLLaMAv2 kernels
if used_gptq_exllamav2:
_import_exllamav2_kernels()
- ## convert datatype
- data_types = []
- for used_backend in used_backends:
- backend = BackendInfos[used_backend]
- data_types.append(backend.dtype)
- common = set(data_types[0])
- for l in data_types[1:]:
- common &= set(l)
- common = list(common)
- if len(common) > 0 and str(model.dtype).split(".")[-1] not in common:
- if common[0] == "float16":
- model = model.to(torch.float16)
- logger.warning("force model to float16")
- elif common[0] == "bfloat16":
- model = model.to(torch.bfloat16)
- logger.warning("force model to bfloat16")
-
-
-def convert_hf_model(model: nn.Module, target_device="cpu"):
- """Converts the given model to an AutoRound model by replacing its layers with quantized layers.
-
- This method extracts the quantization configuration from the model and adjusts its layers
- according to the specified quantization parameters. It supports different backends and
- ensures that the model's data type is compatible with the selected hardware.
+ # Determine common data type across backends
+ data_types = [set(BackendInfos[b].dtype) for b in used_backends]
+ common_dtypes = set.intersection(*data_types) if data_types else set()
+
+ # Force model dtype if needed
+ model_dtype_name = str(model.dtype).split(".")[-1]
+ if common_dtypes and model_dtype_name not in common_dtypes:
+ target_dtype = None
+ if "float16" in common_dtypes:
+ target_dtype = torch.float16
+ elif "bfloat16" in common_dtypes:
+ target_dtype = torch.bfloat16
+
+ if target_dtype:
+ model = model.to(target_dtype)
+ logger.warning(f"Forced model to {target_dtype}")
+
+
+def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Module, list]:
+ """Converts a HuggingFace model into an AutoRound model by replacing layers with quantized layers.
+
+ This function extracts the quantization configuration from the model and updates its layers
+ according to the specified quantization parameters. It supports different backends,
+ sets appropriate packing formats, and ensures compatibility with the target device.
Args:
- model (nn.Module):
- The model to be converted into an AutoRound model.
+ model (nn.Module): The HuggingFace model to be converted.
+ target_device (str, optional): Device to run the model on.
+ One of {"cuda", "cpu", "hpu", "xpu"}. Defaults to "cpu".
Returns:
- nn.Module:
- The converted AutoRound model with quantized layers.
+ Tuple[nn.Module, list]:
+ The converted AutoRound model and a list of used backends.
Raises:
- ValueError:
- If the quantization backend is not specified in the configuration.
+ NotImplementedError: If the GPTQ model uses an unsupported `g_idx`.
+ ValueError: If quantization backend is not properly specified.
"""
-
quantization_config = model.config.quantization_config
- if hasattr(quantization_config, "desc_act") and quantization_config.desc_act:
- ##check static_group
- if (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (
- not hasattr(quantization_config, "static_groups")
- ):
+ # Check desc_act + static_groups
+ if getattr(quantization_config, "desc_act", False):
+ if not getattr(quantization_config, "static_groups", False):
raise NotImplementedError(
- "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound"
+ "This GPTQ model may contain a non-dummy g_idx, " "which is not yet supported by AutoRound."
)
- if hasattr(quantization_config, "backend"):
- backend = quantization_config.backend
- else:
- backend = "auto"
- ##target_backend could be None
- _, backend = parse_target_device_and_backend(backend)
+ # Determine backend
+ backend = getattr(quantization_config, "backend", "auto")
+ # Determine packing format
if (
hasattr(quantization_config, "packing_format") and "auto-round" in quantization_config.quant_method
): # pragma: no cover
packing_format = quantization_config.packing_format
elif "gptq" in quantization_config.quant_method: # pragma: no cover
- packing_format = "auto_gptq"
+ packing_format = "auto_round:auto_gptq"
elif "awq" in quantization_config.quant_method:
- packing_format = "auto_awq"
+ packing_format = "auto_round:auto_awq"
else: # pragma: no cover
- packing_format = "auto_gptq"
- logger.warning("quantization backend must be specified. Set it to 'auto_gptq' by default.")
+ packing_format = "auto_round:auto_gptq"
+ logger.warning("Quantization backend must be specified. " "Defaulting to 'auto_round:auto_gptq'.")
+
if packing_format == "auto":
- packing_format = "auto_gptq"
+ packing_format = "auto_round:auto_gptq"
+ elif packing_format == "auto_round:awq": # normalize tricky settings
+ packing_format = "auto_round:auto_awq"
+ elif packing_format == "auto_round:gptq":
+ packing_format = "auto_round:auto_gptq"
+ # Replace layers with quantized versions
layer_configs = get_layer_config(model, quantization_config)
- if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
- packing_format = packing_format[len("auto_round:") :]
-
- orig_backend = find_backend(packing_format)
-
- if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
- backend = backend[len("auto_round:") :]
+ used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)
- used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)
- if backend == "auto" or backend == "":
- best_backend = get_highest_priority_backend(
+ # Suggest a better backend if available
+ if backend == "auto":
+ best_backend = get_highest_priority_backend( # TODO add activation scheme
quantization_config.bits,
quantization_config.sym,
quantization_config.group_size,
target_device,
- BackendInfos[orig_backend].packing_format,
+ packing_format,
)
- if best_backend is not None and best_backend not in used_backends:
+ if best_backend and best_backend not in used_backends:
requirements = BackendInfos[best_backend].requirements
process_requirement(requirements, target_device, "warning")
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index ab804b0a8..909ff5ffd 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -32,7 +32,6 @@ class QuantizationScheme:
act_dynamic: Optional[bool] = None
super_bits: Optional[int] = None
super_group_size: Optional[int] = None
- clip: Optional[bool] = False
@classmethod
def from_dict(cls, config: dict):