PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[High Risk]Refine inference code (#840)
Wenhua Cheng162424de4ae2
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 1671d157d..78a8fc9d6 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -545,7 +545,7 @@ def tune(args):eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")- if autoround.act_bits <= 8 or eval_gguf_model:+ if (autoround.act_bits <= 8 and formats[-1] == "fake") or eval_gguf_model:if eval_gguf_model:# for file in os.listdir(eval_folder):# gguf_file = filediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex be42b824a..a0df6b004 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -972,11 +972,7 @@ class BaseCompressor(object):elif "auto_round" in format and (is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)):- logger.warning(- f"AutoRound supports exporting to format '{format}', "- "but loading quantized models in this format is not yet supported. "- "It is currently recommended to export to the 'llm_compressor' format."- )+ passelif format != "fake":logger.warning("Currently only support to export auto_round format quantized model"diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex cc717e699..5bbb36e90 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -57,7 +57,7 @@ AUTOROUND_MINIMUM_VERSION = version.parse("0.2")def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version- try: ##TODO remove it later+ try: # TODO remove it laterimport auto_roundreturn True, auto_round.__version__diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex ce3e209cf..50ec2b120 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -14,14 +14,13 @@import functoolsfrom dataclasses import dataclass, field-from typing import Any, List, Optional+from typing import Any, Optionalfrom transformers.utils.versions import require_versionimport auto_round_extension.cuda.gptqmodel_marlinfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodules-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import get_library_version@@ -56,8 +55,6 @@ class BackendInfo:quantization. Defaults to None.priority: An integer representing the backend's priority, where higher valuesindicate higher priority. Defaults to 0.- convertable_format: A list of strings specifying the formats that the backend- can convert from. Defaults to an empty list.checkers: A list of check functions (e.g., validation methods)used to verify whether the backend supports certain features. Defaults toan empty list.@@ -65,17 +62,16 @@ class BackendInfo:backend. Defaults to None."""- device: List[str]- sym: List[bool]- packing_format: str- bits: List[int]- dtype: List[str] = None- group_size: Optional[List[int]] = None+ device: list[str] # TODO change to tuple+ sym: list[bool]+ packing_format: list[str]+ bits: list[int]+ dtype: list[str] = None+ group_size: Optional[list[int]] = Nonepriority: int = 0 ##higher is better- convertable_format: List[str] = field(default_factory=list)- checkers: List[Any] = field(default_factory=list)- alias: Optional[List[str]] = None- requirements: Optional[List[str]] = None+ checkers: list[Any] = field(default_factory=list)+ alias: Optional[list[str]] = None+ requirements: Optional[list[str]] = None# TODO(Yi): Add more fields for activation dtype, group size, etc.@@ -156,14 +152,19 @@ def mxfp4_scheme_checker(return _scheme_checker_common(config, ar_schemes.MXFP4)+GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1+GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]+AWQ_FORMAT = ["auto_round:auto_awq"]+LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]+BackendInfos["auto_gptq:exllamav2"] = BackendInfo(device=["cuda"],sym=[True, False],- packing_format="int32_zp",+ packing_format=GPTQ_FORMAT,bits=[4],priority=5,dtype=["float16"],- ##16, 384,768 accuracy issue+ # 16, 384,768 accuracy issuegroup_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],checkers=[exllamav2_feature_checker],alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],@@ -173,7 +174,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(BackendInfos["auto_gptq:tritonv2"] = BackendInfo(device=["cuda"],sym=[True, False],- packing_format="int32_zp",+ packing_format=GPTQ_FORMAT,bits=[2, 4, 8],group_size=None,dtype=["float16"],@@ -186,51 +187,25 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(BackendInfos["auto_gptq:cuda"] = BackendInfo(device=["cuda"],sym=[True, False],- packing_format="int32_zp",+ packing_format=GPTQ_FORMAT,bits=[2, 3, 4, 8],group_size=None,priority=1,checkers=[exllamav2_feature_checker],- alias=["auto_gptq:cuda"],dtype=["float16"],- convertable_format=["int32_zp"],+ alias=["auto_gptq:cuda"],requirements=["torch<2.6.0","auto-gptq>=0.7.1",],)-BackendInfos["auto_round:tritonv2"] = BackendInfo(- device=["cuda", "xpu"],- sym=[True, False],- packing_format="int32",- dtype=["float16", "bfloat16"],- bits=[2, 4, 8],- priority=2,- checkers=[feature_multiply_checker_32],- alias=["auto_round", "tritonv2", "triton"],- requirements=["triton>=2.0", "auto-round>=0.5.0"],-)--BackendInfos["auto_round:torch"] = BackendInfo(- device=["cuda", "xpu", "cpu"],- sym=[True, False],- packing_format="int32",- dtype=["float16", "bfloat16"],- bits=[2, 3, 4, 8],- priority=0,- checkers=[exllamav2_feature_checker],- alias=["auto_round", "torch"],- requirements=["auto-round>=0.5.1"],-)-# FP8 static quant# Weight: FP8, per-channel, may be extended to per-tensor in future# Activation: FP8, per-tensor--BackendInfos["auto_round:fp8_static"] = BackendInfo(+BackendInfos["auto_round:torch_fp8_static"] = BackendInfo(device=["xpu", "cuda", "cpu"],- packing_format="",+ packing_format=["auto_round:fp8_static"],sym=[True],dtype=["float32", "float16", "bfloat16"],bits=[8],@@ -241,38 +216,47 @@ BackendInfos["auto_round:fp8_static"] = BackendInfo()# MXFP8--BackendInfos["auto_round:mxfp8"] = BackendInfo(+BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(device=["xpu", "cuda", "cpu"],- packing_format="",+ packing_format=LLM_COMPRESSOR_FORMAT,sym=[True],dtype=["float32", "float16", "bfloat16"],bits=[8],priority=0,checkers=[mxfp8_scheme_checker],- alias=["torch"],+ alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)# MXFP4--BackendInfos["auto_round:mxfp4"] = BackendInfo(+BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(device=["xpu", "cuda", "cpu"],- packing_format="",+ packing_format=LLM_COMPRESSOR_FORMAT,sym=[True],dtype=["float32", "float16", "bfloat16"],bits=[4],priority=0,checkers=[mxfp4_scheme_checker],- alias=["auto_round:llm_compressor"],+ alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)+BackendInfos["auto_round:tritonv2"] = BackendInfo(+ device=["cuda", "xpu"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT_NO_ZP,+ dtype=["float16", "bfloat16"],+ bits=[2, 4, 8],+ priority=2,+ checkers=[feature_multiply_checker_32],+ alias=["auto_round", "tritonv2", "triton"],+ requirements=["triton>=2.0", "auto-round>=0.5.0"],+)+BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(device=["cuda", "xpu"],- sym=[True], ## asym has accuracys- # issue- packing_format="int32_zp",+ sym=[True],+ packing_format=GPTQ_FORMAT,dtype=["float16", "bfloat16"],bits=[2, 4, 8],priority=2,@@ -281,10 +265,23 @@ BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(requirements=["triton>=2.0", "auto-round>=0.5.0"],)+BackendInfos["auto_round:torch"] = BackendInfo(+ device=["cuda", "xpu", "cpu"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT_NO_ZP,+ dtype=["float16", "bfloat16"],+ bits=[2, 3, 4, 8],+ priority=0,+ checkers=[exllamav2_feature_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>=0.5.1"],+)++BackendInfos["auto_round:torch_zp"] = BackendInfo(device=["cuda", "xpu", "cpu"],sym=[True],- packing_format="int32_zp",+ packing_format=GPTQ_FORMAT,dtype=["float16", "bfloat16"],bits=[2, 3, 4, 8],priority=0,@@ -296,7 +293,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(BackendInfos["gptqmodel:marlin"] = BackendInfo(device=["cuda"],sym=[True],- packing_format="int32",+ packing_format=GPTQ_FORMAT_NO_ZP,bits=[4, 8],group_size=[-1, 32, 64, 128],dtype=["float16", "bfloat16"],@@ -309,7 +306,7 @@ BackendInfos["gptqmodel:marlin"] = BackendInfo(BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(device=["cuda"],sym=[True],- packing_format="int32_zp",+ packing_format=GPTQ_FORMAT,bits=[4, 8],group_size=[-1, 32, 64, 128],dtype=["float16", "bfloat16"],@@ -322,7 +319,7 @@ BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(BackendInfos["gptqmodel:exllamav2"] = BackendInfo(device=["cuda"],sym=[True, False],- packing_format="int32",+ packing_format=GPTQ_FORMAT,bits=[4],group_size=[-1, 32, 64, 128], ##16 seems has accuracy issuedtype=["float16", "bfloat16"],@@ -334,8 +331,8 @@ BackendInfos["gptqmodel:exllamav2"] = BackendInfo(BackendInfos["auto_awq:gemm"] = BackendInfo(device=["cuda"],- sym=[True, False], ##actually is gemm- packing_format="awq",+ sym=[True, False], # Actually it is GEMM+ packing_format=AWQ_FORMAT,bits=[4],group_size=None,priority=5,@@ -347,35 +344,34 @@ BackendInfos["auto_awq:gemm"] = BackendInfo(BackendInfos["qbits"] = BackendInfo(device=["cpu"],sym=[True, False],- packing_format="qbits",+ packing_format=GPTQ_FORMAT_NO_ZP,bits=[2, 4, 8],group_size=None,priority=1,checkers=[],alias=["itrex", "qbits"],dtype=["float16", "bfloat16"],- convertable_format=["int32"],requirements=["torch<2.7.0", "intel-extension-for-transformers"],)BackendInfos["qbits_zp"] = BackendInfo(device=["cpu"],sym=[True, False],- packing_format="qbits_zp",+ packing_format=GPTQ_FORMAT,bits=[2, 4, 8],group_size=None,dtype=["float16", "bfloat16"],priority=1,checkers=[],alias=["itrex", "qbits"],- convertable_format=["int32_zp"],requirements=["torch<2.7.0", "intel-extension-for-transformers"],)-BackendInfos["auto_round:qbits_awq"] = BackendInfo(++BackendInfos["qbits_awq"] = BackendInfo(device=["cpu"],- sym=[True, False], ## for awq, not robust- packing_format="awq",+ sym=[True, False],+ packing_format=AWQ_FORMAT,bits=[2, 4, 8],group_size=None,dtype=["float16", "bfloat16"],@@ -384,17 +380,15 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(alias=["itrex", "qbits"],requirements=["torch<2.7.0", "intel-extension-for-transformers"],)-BackendInfos["ipex_gptq"] = BackendInfo(device=["cpu", "xpu"],sym=[True, False],- packing_format="ipex_gptq",+ packing_format=GPTQ_FORMAT,bits=[4],group_size=None,priority=5,checkers=[],dtype=["float16", "bfloat16"],- convertable_format=["int32_zp"],alias=["ipex"],requirements=["intel-extension-for-pytorch>=2.5"],)@@ -402,50 +396,51 @@ BackendInfos["ipex_gptq"] = BackendInfo(BackendInfos["ipex_awq"] = BackendInfo(device=["cpu", "xpu"],sym=[True, False],- packing_format="ipex_awq",+ packing_format=AWQ_FORMAT,bits=[4],group_size=None,- priority=1,- dtype=["float16", "bfloat16"],+ priority=5,checkers=[],+ dtype=["float16", "bfloat16"],alias=["ipex"],- convertable_format=["awq"],- requirements=["intel-extension-for-pytorch>=2.6"],+ requirements=["intel-extension-for-pytorch>=2.5"],)-BackendInfos["hpu"] = BackendInfo(device=["hpu"],sym=[True, False],- packing_format="hpu",+ packing_format=GPTQ_FORMAT_NO_ZP,bits=[4],dtype=["bfloat16"],alias=["hpu"],priority=0,- convertable_format=["int32"],)BackendInfos["hpu_zp"] = BackendInfo(device=["hpu"],sym=[True, False],- packing_format="hpu_zp",+ packing_format=GPTQ_FORMAT,bits=[4],dtype=["bfloat16"],alias=["hpu"],priority=0,- convertable_format=["int32_zp"],)-def check_compatible(backend_name, device, config, packing_format, in_features, out_features, check_requirements=True):+def check_compatible(+ backend_name: str,+ device: str,+ config: dict,+ packing_format: str,+ in_features: int,+ out_features: int,+ check_requirements=True,+):"""Checks if the given configuration is compatible with the specified backend.Args:backend_name (str): The name of the backend to check compatibility for.device (str): The device on which the backend operates (e.g., 'cuda', 'cpu').- bits (int): The bit-width of the quantization (e.g., 2, 4, 8).- group_size (Optional[int]): The size of the quantization group. Can be None if- not required by the backend.- sym (bool): Whether symmetric quantization is required (True for symmetric).+ config(dict): schemepacking_format (str): The packing format used by the backend (e.g., 'triton').in_features (int): The number of input features for the model layer.out_features (int): The number of output features for the model layer.@@ -484,7 +479,7 @@ def check_compatible(backend_name, device, config, packing_format, in_features,return False# Check if the format is convertible when packing formats differ- if packing_format == backend.packing_format or packing_format in backend.convertable_format:+ if packing_format in backend.packing_format:passelse:return False@@ -534,11 +529,11 @@ def dynamic_import_inference_linear(backend, config):"""bits, group_size, sym = config["bits"], config["group_size"], config["sym"]- if AutoRoundFormat.FP8_STATIC.value in backend:+ if "torch_fp8_static" in backend:return ar_qmodules.WeightFP8ActFP8StaticQuantLinear- if AutoRoundFormat.MXFP8.value in backend:+ if "torch_mxfp8" in backend:return ar_qmodules.MXFP8QuantLinear- if AutoRoundFormat.MXFP4.value in backend:+ if "torch_mxfp4" in backend:return ar_qmodules.MXFP4QuantLinearif "qbits" in backend:@@ -734,22 +729,21 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):return QuantLinear-def find_backend(target_backend: str, orig_backend: str = None):+def find_backend(backend: str, orig_backend: str = None):"""Finds the matching backend key based on the target backend name or its aliases.Args:- target_backend (str): Name or alias of the target backend.+ backend (str): Name or alias of the target backend.orig_backend (str, optional): Original backend name to check compatibility. Defaults to None.Returns:str or None: Matching backend key if found and compatible; otherwise, None."""- logger.trace(f"Finding backend for target: {target_backend}, original: {orig_backend}")+ logger.trace(f"Finding backend for target: {backend}, original: {orig_backend}")+matched_keys = [- key- for key, info in BackendInfos.items()- if key == target_backend or (info.alias and target_backend in info.alias)+ key for key, info in BackendInfos.items() if key == backend or (info.alias and backend in info.alias)]if not matched_keys:@@ -768,15 +762,12 @@ def find_backend(target_backend: str, orig_backend: str = None):):return key- raise ValueError(- f"{target_backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry."- )+ raise ValueError(f"{backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry.")-def get_all_compatible_backend(device, backend, orig_backend, config, in_features, out_features):- # Get packing format from the original backend- packing_format = BackendInfos[orig_backend].packing_format-+def get_all_compatible_backend(+ device: str, packing_format: str, config: dict, in_features: int, out_features: int+) -> list[str]:# Find compatible backendscompatible_backends = [key@@ -788,7 +779,9 @@ def get_all_compatible_backend(device, backend, orig_backend, config, in_featurereturn compatible_backends-def get_layer_backend(device, backend, orig_backend, config, in_features, out_features):+def get_layer_backend(+ device: str, backend: str, packing_format: str, config: dict, in_features: int, out_features: int+) -> str:"""Selects the most suitable backend for the layer based on compatibility and priority.This function first checks if the specified backend supports the layer with the provided configuration.@@ -799,15 +792,10 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fedevice (str):The device on which the layer will run, e.g., 'cpu', 'cuda'.backend (str):- The target backend to be used for this layer.- orig_backend (str):+ The target backend to be used for this layer."auto","triton","gptqmodel", etc,+ packing_format (str):The original backend from which packing format information is retrieved.- bits (int):- The number of bits used for quantization.- group_size (int):- The group size for quantization.- sym (bool):- Whether symmetric quantization is enabled.+ config (dict): Layer config.in_features (int):The number of input features for the layer.out_features (int):@@ -822,17 +810,18 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_feIf the specified backend is not supported.If no compatible backend is found for the given layer configuration."""- # Check if the provided backend is in BackendInfos- backend = find_backend(backend)- logger.trace(f"Found backend: {backend} for device: {device}, orig_backend: {orig_backend}")- if backend not in BackendInfos.keys():- raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")- packing_format = BackendInfos[orig_backend].packing_format+ backends = []+ if backend == "auto":+ backends = BackendInfos.keys()+ else:+ for key in BackendInfos.keys():+ if backend == key or (BackendInfos[key].alias and backend in BackendInfos[key].alias):+ backends.append(key)# Find and store other compatible backendssupported_backends = []- for key in BackendInfos.keys():+ for key in backends:if check_compatible(key, device, config, packing_format, in_features, out_features):logger.trace(f"Backend {key} is compatible")supported_backends.append(key)@@ -840,7 +829,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe# Raise an error if no compatible backends are foundif len(supported_backends) == 0:supported_backends_need_package = get_all_compatible_backend(- device, backend, orig_backend, config, in_features, out_features+ device, packing_format, config, in_features, out_features)if len(supported_backends_need_package) > 0:@@ -852,7 +841,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_febackend_info = BackendInfos[supported_backends_need_package[0]]process_requirement(backend_info.requirements, target_device=device)- return None+ return ""# Sort the compatible backends by priority and return the one with the highest prioritysupported_backends = sorted(@@ -883,7 +872,7 @@ def get_highest_priority_backend(bits, sym, group_size, device, packing_format):continue# Check if the format is convertible when packing formats differ- if packing_format == backend.packing_format or packing_format in backend.convertable_format:+ if packing_format in backend.packing_format:passelse:continuediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 23c01003d..bebf75df7 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -13,7 +13,6 @@# limitations under the License.import osimport re-from logging import getLoggerfrom typing import Unionimport torch@@ -25,7 +24,6 @@ from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.inference.backend import (BackendInfos,dynamic_import_inference_linear,- find_backend,get_highest_priority_backend,get_layer_backend,process_requirement,@@ -38,7 +36,6 @@ from auto_round.utils import (check_to_quantized,find_matching_blocks,get_block_names,- get_layer_names_in_block,get_module,is_hpex_available,set_module,@@ -175,40 +172,6 @@ def get_available_devices():return devices-def parse_target_device_and_backend(target_backend: str):- """- Parse the target device and backend from the given target_backend string.-- Args:- target_backend (str): A string specifying the target backend, which may include a device prefix.-- Returns:- tuple: (target_device, backend), where:- - target_device (str): The identified device (e.g., "cpu", "cuda", "auto").- - backend (str): The backend string after removing the device prefix.- """-- def remove_device_prefix(backend: str, prefix: str) -> str:- """Removes the given device prefix from the backend string, if present."""- return backend[len(prefix) :].lstrip(":") if backend.startswith(prefix) else backend-- # Handle "auto" case explicitly- if target_backend == "auto":- return "auto", ""- if target_backend.startswith("auto:"):- return "auto", target_backend[5:].lstrip(":")-- # Define supported devices-- # Identify the target device and adjust the backend string accordingly- for device in supported_devices:- if target_backend.startswith(device):- return device, remove_device_prefix(target_backend, device)-- # Default case: If no known device is found, assume "auto"- return "auto", target_backend--def get_layer_config(model, quantization_config):"""get a layer-wise quantization configuration for a given model.@@ -232,7 +195,6 @@ def get_layer_config(model, quantization_config):- "group_size" (int): Group size for quantization.- "data_type" (str): Data type used for quantization.- "sym" (bool): Whether symmetric quantization is applied.- - "clip" (bool): Whether weight clipping is enabled."""bits = quantization_config.bitsgroup_size = quantization_config.group_size@@ -272,7 +234,7 @@ def get_layer_config(model, quantization_config):# Find matching blocks if no explicit names are providedall_blocks = get_block_names(model, quant_vision=True)quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)- ##speed up the matching+ # Speed up the matchingfor i in range(len(quant_block_list)):quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")@@ -314,45 +276,39 @@ def get_layer_config(model, quantization_config):return layer_configs-def get_device(obj: Union[torch.Tensor, nn.Module]):+def get_device(obj: Union[torch.Tensor, nn.Module]) -> torch.device:if isinstance(obj, torch.Tensor):return obj.devicereturn next(obj.parameters()).device-def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, target_device, orig_backend):- """- Replaces linear layers in the given module with quantized layers.+def _replace_by_quant_layers(+ module: nn.Module,+ layer_configs: dict,+ backend: str,+ target_device: str,+ packing_format: str,+) -> list:+ """Replaces linear layers in a module with quantized layers according to configs.++ This function iterates over each layer in `layer_configs`, checks if it requires+ quantization, determines the appropriate backend, creates a quantized layer, and+ replaces the original layer in the module.Args:module (nn.Module): The module containing layers to be quantized.layer_configs (dict): Configuration for each layer's quantization.- target_backend (str): Backend for quantization (device and format).- target_device (str): Device for execution ('cuda', 'cpu', 'hpu').- orig_backend (str): Original backend of the model.+ backend (str): Default backend for quantization.+ target_device (str): Target device for execution ('cuda', 'cpu', 'hpu', etc.).+ packing_format (str): Packing format for the quantized layers.Returns:- dict: Flags indicating which backends were used.+ list: List of backends actually used for the layers.+ Raises:+ ValueError: If no compatible backend is found for a layer and `backend` is not "auto"."""used_backends = []- must_use_target_backend = False- if target_backend:- must_use_target_backend = True- layer_backend = target_backend- layer_backend_must = find_backend(layer_backend, orig_backend)- used_backends.append(layer_backend_must)- if layer_backend_must is None:- raise ValueError(f"{target_backend} is not compatible, please change backend to `auto` and retry")- devices = BackendInfos[layer_backend_must].device- if target_device not in devices:- raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")- ##check requirements- requirements = BackendInfos[layer_backend_must].requirements- process_requirement(requirements, target_device)-- target_backend = target_backend or orig_backend # Default to original backend if not specified-backend_cache = {}for layer_name, config in layer_configs.items():@@ -363,27 +319,27 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, tin_features, out_features = _get_layer_features(layer)if in_features is None:continue # Skip unsupported layer types-- if must_use_target_backend:- layer_backend = layer_backend_must+ scheme_key = "_".join(f"{k}={v}" for k, v in config.items())+ key = f"{scheme_key}_{in_features}_{out_features}"+ if key in backend_cache:+ layer_backend = backend_cache[key]else:- key = f"{config['bits']}_{config['group_size']}_{config['sym']}_{in_features}_{out_features}"- if key in backend_cache:- layer_backend = backend_cache[key]- if layer_backend not in used_backends:- used_backends.append(layer_backend)- else:- # Determine backend- layer_backend = _get_layer_backend(- target_device, target_backend, orig_backend, config, in_features, out_features- )- logger.trace(f"Got backend {layer_backend} for {layer_name}.")- backend_cache[key] = layer_backend+ # Determine backend+ layer_backend = get_layer_backend(target_device, backend, packing_format, config, in_features, out_features)+ logger.trace(f"Got backend {layer_backend} for {layer_name}.")+ backend_cache[key] = layer_backend+ if layer_backend not in used_backends:+ used_backends.append(layer_backend)if not layer_backend:+ if backend != "auto":+ raise ValueError(+ f"Backend {backend} is not compatible with layer {layer_name} with config {config},"+ f" please set the backend='auto' and retry"+ )raise ValueError(f"No compatible backend found for layer {layer_name} with config {config}")- logger.debug(f"{layer_name}: {layer_backend} backend is used") # TODO delete+ logger.debug(f"{layer_name}: {layer_backend} backend is used")# Create and replace layernew_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)@@ -401,21 +357,6 @@ def _get_layer_features(layer):return None, None # Unsupported layer type-def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features, out_features):- """Determines the best backend for a given layer."""-- backend = get_layer_backend(- target_device,- target_backend,- orig_backend,- config,- in_features,- out_features,- )-- return backend--def _import_exllamav2_kernels():"""Attempts to import ExLlamaV2 kernels for performance optimization."""try:@@ -458,64 +399,85 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)or AutoRoundFormat.MXFP4.value in layer_backend):return QuantLinear.from_original(config, layer)+# Default quantized layer creation- try:- return QuantLinear(- config["bits"],- config["group_size"],- in_features,- out_features,- bias,- weight_dtype=layer.weight.dtype,- clip=config["clip"],- )- except: # Handle cases where `clip` is not a valid argument- return QuantLinear(- config["bits"], config["group_size"], in_features, out_features, bias, weight_dtype=layer.weight.dtype- )+ return QuantLinear(+ config["bits"],+ config["group_size"],+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ )-def infer_target_device(device_map=None):+def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:+ """Infers the target device from a device_map.++ Args:+ device_map (Optional[Union[Dict[Any, Any], int, str]]):+ - If None, defaults to "cpu".+ - If dict, checks values to infer the device type.+ - If int or str, assumes it represents a device.++ Returns:+ str: The inferred target device, e.g., "cpu" or "cuda".+ """if device_map is None:- target_device = "cpu"- elif isinstance(device_map, dict):- devices = set(device_map.values())- target_device = "cpu"- for device in devices:- if device != "cpu" and device != "disk":+ return "cpu"++ if isinstance(device_map, dict):+ for device in set(device_map.values()):+ if device not in ("cpu", "disk"):if isinstance(device, int):- target_device = get_available_devices()[0]- else:- target_device = str(device).split(":")[0]- else:- target_device = get_available_devices()[0]- return target_device+ return get_available_devices()[0]+ return str(device).split(":")[0]+ return "cpu"++ return get_available_devices()[0]-def post_init(model, used_backends):+def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:+ """Performs post-initialization for different quantization backends.++ This function handles backend-specific post-init steps, including AutoGPTQ,+ GPTQModel, IPEX/ITREx layers, and ExLLaMAv2 kernels. It also ensures the+ model's data type is compatible with all used backends.++ Args:+ model (torch.nn.Module): The model to initialize.+ used_backends (List[str]): List of backend names used for quantization.++ """need_autogptq_init = Falseneed_gptqmodel_init = Falseneed_ipex_itrex_init = Falseused_gptq_exllamav2 = False- for used_backend in used_backends:- if used_backend.startswith("auto_gptq"):++ # Determine which backends require post-init+ for backend in used_backends:+ if backend.startswith("auto_gptq"):need_autogptq_init = True- if used_backend == "auto_gptq:exllamav2":+ if backend == "auto_gptq:exllamav2":used_gptq_exllamav2 = True- elif used_backend.startswith("gptqmodel"):+ elif backend.startswith("gptqmodel"):need_gptqmodel_init = True- elif used_backend.startswith("ipex"):- need_ipex_itrex_init = True- elif used_backend.startswith("qbit"):+ elif backend.startswith(("ipex", "qbit")):need_ipex_itrex_init = True++ # AutoGPTQ post-initif need_autogptq_init:from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init # pylint: disable=E0401model = gptq_post_init(model, use_act_order=False)++ # GPTQModel post-initif need_gptqmodel_init:from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401model = gptq_post_init(model, use_act_order=False)++ # IPEX/ITREx post-initif need_ipex_itrex_init:message = "repacking to CPU/XPU format"layers = [] ## ipex post_init will add one more layer@@ -526,98 +488,94 @@ def post_init(model, used_backends):for layer in tqdm(layers, desc=message, total=len(layers), leave=True):layer.post_init()+ # ExLLaMAv2 kernelsif used_gptq_exllamav2:_import_exllamav2_kernels()- ## convert datatype- data_types = []- for used_backend in used_backends:- backend = BackendInfos[used_backend]- data_types.append(backend.dtype)- common = set(data_types[0])- for l in data_types[1:]:- common &= set(l)- common = list(common)- if len(common) > 0 and str(model.dtype).split(".")[-1] not in common:- if common[0] == "float16":- model = model.to(torch.float16)- logger.warning("force model to float16")- elif common[0] == "bfloat16":- model = model.to(torch.bfloat16)- logger.warning("force model to bfloat16")---def convert_hf_model(model: nn.Module, target_device="cpu"):- """Converts the given model to an AutoRound model by replacing its layers with quantized layers.-- This method extracts the quantization configuration from the model and adjusts its layers- according to the specified quantization parameters. It supports different backends and- ensures that the model's data type is compatible with the selected hardware.+ # Determine common data type across backends+ data_types = [set(BackendInfos[b].dtype) for b in used_backends]+ common_dtypes = set.intersection(*data_types) if data_types else set()++ # Force model dtype if needed+ model_dtype_name = str(model.dtype).split(".")[-1]+ if common_dtypes and model_dtype_name not in common_dtypes:+ target_dtype = None+ if "float16" in common_dtypes:+ target_dtype = torch.float16+ elif "bfloat16" in common_dtypes:+ target_dtype = torch.bfloat16++ if target_dtype:+ model = model.to(target_dtype)+ logger.warning(f"Forced model to {target_dtype}")+++def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Module, list]:+ """Converts a HuggingFace model into an AutoRound model by replacing layers with quantized layers.++ This function extracts the quantization configuration from the model and updates its layers+ according to the specified quantization parameters. It supports different backends,+ sets appropriate packing formats, and ensures compatibility with the target device.Args:- model (nn.Module):- The model to be converted into an AutoRound model.+ model (nn.Module): The HuggingFace model to be converted.+ target_device (str, optional): Device to run the model on.+ One of {"cuda", "cpu", "hpu", "xpu"}. Defaults to "cpu".Returns:- nn.Module:- The converted AutoRound model with quantized layers.+ Tuple[nn.Module, list]:+ The converted AutoRound model and a list of used backends.Raises:- ValueError:- If the quantization backend is not specified in the configuration.+ NotImplementedError: If the GPTQ model uses an unsupported `g_idx`.+ ValueError: If quantization backend is not properly specified."""-quantization_config = model.config.quantization_config- if hasattr(quantization_config, "desc_act") and quantization_config.desc_act:- ##check static_group- if (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (- not hasattr(quantization_config, "static_groups")- ):+ # Check desc_act + static_groups+ if getattr(quantization_config, "desc_act", False):+ if not getattr(quantization_config, "static_groups", False):raise NotImplementedError(- "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound"+ "This GPTQ model may contain a non-dummy g_idx, " "which is not yet supported by AutoRound.")- if hasattr(quantization_config, "backend"):- backend = quantization_config.backend- else:- backend = "auto"- ##target_backend could be None- _, backend = parse_target_device_and_backend(backend)+ # Determine backend+ backend = getattr(quantization_config, "backend", "auto")+ # Determine packing formatif (hasattr(quantization_config, "packing_format") and "auto-round" in quantization_config.quant_method): # pragma: no coverpacking_format = quantization_config.packing_formatelif "gptq" in quantization_config.quant_method: # pragma: no cover- packing_format = "auto_gptq"+ packing_format = "auto_round:auto_gptq"elif "awq" in quantization_config.quant_method:- packing_format = "auto_awq"+ packing_format = "auto_round:auto_awq"else: # pragma: no cover- packing_format = "auto_gptq"- logger.warning("quantization backend must be specified. Set it to 'auto_gptq' by default.")+ packing_format = "auto_round:auto_gptq"+ logger.warning("Quantization backend must be specified. " "Defaulting to 'auto_round:auto_gptq'.")+if packing_format == "auto":- packing_format = "auto_gptq"+ packing_format = "auto_round:auto_gptq"+ elif packing_format == "auto_round:awq": # normalize tricky settings+ packing_format = "auto_round:auto_awq"+ elif packing_format == "auto_round:gptq":+ packing_format = "auto_round:auto_gptq"+ # Replace layers with quantized versionslayer_configs = get_layer_config(model, quantization_config)- if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):- packing_format = packing_format[len("auto_round:") :]-- orig_backend = find_backend(packing_format)-- if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):- backend = backend[len("auto_round:") :]+ used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)- used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)- if backend == "auto" or backend == "":- best_backend = get_highest_priority_backend(+ # Suggest a better backend if available+ if backend == "auto":+ best_backend = get_highest_priority_backend( # TODO add activation schemequantization_config.bits,quantization_config.sym,quantization_config.group_size,target_device,- BackendInfos[orig_backend].packing_format,+ packing_format,)- if best_backend is not None and best_backend not in used_backends:+ if best_backend and best_backend not in used_backends:requirements = BackendInfos[best_backend].requirementsprocess_requirement(requirements, target_device, "warning")diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex ab804b0a8..909ff5ffd 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -32,7 +32,6 @@ class QuantizationScheme:act_dynamic: Optional[bool] = Nonesuper_bits: Optional[int] = Nonesuper_group_size: Optional[int] = None- clip: Optional[bool] = False@classmethoddef from_dict(cls, config: dict):