PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support loading for static quant weight fp8 act fp8 (#730)
Yi Liu09e4d312771a
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 03e012578..607ade4a9 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -19,6 +19,7 @@ import sysimport timeimport tracebackfrom dataclasses import asdict, fields+from enum import Enumfrom typing import Any, Callable, Unionimport accelerate@@ -30,6 +31,7 @@ from transformers import set_seedfrom auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_blockfrom auto_round.schemes import QuantizationScheme, preset_name_to_scheme@@ -857,8 +859,8 @@ class AutoRound(object):format = "auto_round:auto_awq"elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):format = f"auto_round:{self.data_type}"- elif is_wfp8afp8(self): # staic wfp8afp8- format = "auto_round:fp8"+ elif is_static_wfp8afp8(self): # staic wfp8afp8+ format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8format = "auto_round:fp8"elif self.act_bits < 16:@@ -956,10 +958,10 @@ class AutoRound(object):)format = "fake"else:- if not (format == "auto_round" or format == "auto_round:fp8"):+ if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):logger.warning(f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"- " change format to auto_round"+ f" change format {format} to auto_round")format = "auto_round"if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":diff --git a/auto_round/experimental/qmodules/base.py b/auto_round/experimental/qmodules/base.pynew file mode 100644index 000000000..8b7a9c138--- /dev/null+++ b/auto_round/experimental/qmodules/base.py@@ -0,0 +1,54 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from abc import ABC, abstractmethod+from typing import Optional, Union++import torch++__all__ = ["QModuleBase"]+++class QModuleBase(torch.nn.Module):+ """+ Base class used to describe the weight creation and forward pass+ of different quantization schemes supported by Auto-Round.+ The design is inspired by vLLM's CompressedTensorsScheme:+ https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py++ """++ def __init__(self):+ super().__init__()++ @classmethod+ @abstractmethod+ def from_original(cls, config, original_layer: torch.nn.Module):+ raise NotImplementedError++ @classmethod+ @abstractmethod+ def get_min_capability(cls) -> int:+ """+ Get minimum device capability.+ """+ raise NotImplementedError++ @abstractmethod+ def process_weights_after_loading(self, layer: torch.nn.Module):+ """+ Called after weight loading is complete for any cleanup that+ needs to occur.+ """+ raise NotImplementedErrordiff --git a/auto_round/experimental/qmodules/fp8_static.py b/auto_round/experimental/qmodules/fp8_static.pynew file mode 100644index 000000000..e7c55086d--- /dev/null+++ b/auto_round/experimental/qmodules/fp8_static.py@@ -0,0 +1,122 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++from typing import Optional, Union++import torch++from auto_round.experimental.qmodules.base import QModuleBase+from auto_round.utils import logger++__all__ = ["WeightFP8ActFP8StaticQuantLinear"]+++def _quant_tensor_to_fp8_with_scale(tensor: torch.Tensor, scale: torch.Tensor):+ FULL_RANGE = torch.finfo(torch.float8_e4m3fn).max+ qtensor = tensor / scale+ clipped_qtensor = torch.clamp(qtensor, -FULL_RANGE, FULL_RANGE)+ clipped_qtensor_fp8 = clipped_qtensor.to(torch.float8_e4m3fn)+ return scale, clipped_qtensor_fp8+++class WeightFP8ActFP8StaticQuantLinear(QModuleBase):+ hp_dtype = torch.bfloat16+ fp8_dtype = torch.float8_e4m3fn++ def __init__(+ self,+ in_features,+ out_features,+ weight: Optional[torch.Tensor] = None,+ weight_scale: Optional[torch.Tensor] = None,+ bias: Union[torch.Tensor, bool, None] = None,+ input_scale: Optional[torch.Tensor] = None,+ dtype=torch.bfloat16,+ ):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ init_weight = torch.zeros((out_features, in_features), dtype=dtype) if weight is None else weight+ self.weight = torch.nn.Parameter(init_weight, requires_grad=False)+ self.dtype = dtype+ if bias is not None:+ if isinstance(bias, bool):+ bias = torch.zeros((out_features,), dtype=dtype)+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)+ init_weight_scale = torch.empty((out_features), dtype=dtype) if weight_scale is None else weight_scale+ self.register_buffer("weight_scale", init_weight_scale.to(dtype))++ init_input_scale = torch.zeros((1), dtype=dtype) if input_scale is None else input_scale+ self.register_buffer("input_scale", init_input_scale.to(dtype))+ self.pre_dequantized = False++ @classmethod+ def get_min_capability(cls) -> int:+ """+ Get minimum device capability.+ """+ # TODO: correct that config once we add fp8 op support.+ logger.warning_once("FP8 ops are not yet supported. Using capability 0.")+ return 0++ def process_weights_after_loading(self, layer: torch.nn.Module):+ pass++ @classmethod+ def from_original(cls, config, original_layer):+ """+ Create an `WeightFP8ActFP8StaticQuantLinear` layer from an original linear layer.+ """+ logger.warning_once(+ "FP8 static quantization is still in experimental stage, the inference speed might be slow."+ )+ device = original_layer.weight.device+ with torch.device(device):+ qdq_linear = cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ bias=original_layer.bias,+ )+ return qdq_linear++ def dequant_weight_online(self):+ if self.pre_dequantized:+ return self.weight+ qdq_weight = self.weight.to(self.dtype) * self.weight_scale.unsqueeze(1)+ return qdq_weight++ def pre_dequantize(self):+ if self.pre_dequantized:+ return+ dequant_weight = self.dequant_weight_online()+ del self.weight+ del self.weight_scale+ self.weight = torch.nn.Parameter(dequant_weight, requires_grad=False)+ self.pre_dequantized = True++ def qdq_input(self, bf16_input: torch.Tensor):+ input_scale, input_fp8 = _quant_tensor_to_fp8_with_scale(bf16_input, self.input_scale.data)+ qdq_input_bf16 = input_fp8.to(self.dtype) * input_scale+ return qdq_input_bf16++ @torch.no_grad()+ def forward(self, bf16_input: torch.Tensor) -> torch.Tensor:++ qdq_input = self.qdq_input(bf16_input)+ qdq_weight = self.dequant_weight_online()+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)+ return outdiff --git a/auto_round/export/export_to_autoround/__init__.py b/auto_round/export/export_to_autoround/__init__.pyindex 0c036d831..6cdcd5aed 100644--- a/auto_round/export/export_to_autoround/__init__.py+++ b/auto_round/export/export_to_autoround/__init__.py@@ -12,4 +12,4 @@# See the License for the specific language governing permissions and# limitations under the License.-from .export import save_quantized_as_autoround+from .export import save_quantized_as_autoround, AutoRoundFormatdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 1f6f94005..ffd10036c 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -18,6 +18,7 @@ import inspectimport jsonimport osfrom concurrent.futures import ThreadPoolExecutor+from enum import Enumimport threadpoolctl as tctlimport torch@@ -43,6 +44,12 @@ from auto_round.utils import ()+class AutoRoundFormat(str, Enum):+ # Weight: FP8, per-channel, may be extended to per-tensor in future+ # Activation: FP8, per-tensor+ TORCH_FP8_STATIC = "fp8_static"++def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):"""Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.@@ -152,7 +159,7 @@ def pack_layer(layer_name, model, backend, device=None):return pack_layer(layer_name, model, backend, device)- if backend == "auto_round:fp8":+ if backend == "auto_round:fp8" or backend == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}":from auto_round.export.export_to_autoround.export_to_fp8 import pack_layerreturn pack_layer(layer_name, model, backend, device)@@ -268,9 +275,14 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exfrom auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoroundreturn save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)+ from auto_round.autoround import AutoRoundFormat##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source- if (kwargs.get("sym") is None or kwargs.get("sym")) and ("gptq" not in backend and "awq" not in backend):+ if (+ (kwargs.get("sym") is None or kwargs.get("sym"))+ and ("gptq" not in backend and "awq" not in backend)+ and (AutoRoundFormat.TORCH_FP8_STATIC.value not in backend)+ ):backend = backend.replace("auto_round", "auto_round:auto_gptq")model = kwargs["model"]diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 1569e8005..1868ee14c 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -19,6 +19,8 @@ from typing import Any, List, Optionalfrom transformers.utils.versions import require_versionimport auto_round_extension.cuda.gptqmodel_marlin+from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.schemes import QuantizationSchemefrom auto_round.utils import get_library_version, loggerBackendInfos = {}@@ -53,7 +55,7 @@ class BackendInfo:indicate higher priority. Defaults to 0.convertable_format: A list of strings specifying the formats that the backendcan convert from. Defaults to an empty list.- feature_checks: A list of feature check functions (e.g., validation methods)+ checkers: A list of check functions (e.g., validation methods)used to verify whether the backend supports certain features. Defaults toan empty list.alias: An optional list of strings representing alternative names for the@@ -68,20 +70,22 @@ class BackendInfo:group_size: Optional[List[int]] = Nonepriority: int = 0 ##higher is betterconvertable_format: List[str] = field(default_factory=list)- feature_checks: List[Any] = field(default_factory=list)+ checkers: List[Any] = field(default_factory=list)alias: Optional[List[str]] = Nonerequirements: Optional[List[str]] = None+ # TODO(Yi): Add more fields for activation dtype, group size, etc.-def feature_multiply_checker(in_feature, out_feature, group_size, in_feature_multiplier, out_feature_multiplier=None):+def feature_multiply_checker(in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None):if out_feature_multiplier is None:out_feature_multiplier = in_feature_multiplierreturn in_feature % in_feature_multiplier == 0 and out_feature % out_feature_multiplier == 0def feature_multiply_checker_group_size(- in_feature, out_feature, group_size, in_feature_multiplier, out_feature_multiplier=None+ in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None):+ group_size = config["group_size"]if out_feature_multiplier is None:out_feature_multiplier = in_feature_multiplierreturn (@@ -96,14 +100,27 @@ in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32)-exllamav2_feature_check = functools.partial(+exllamav2_feature_checker = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32)-gptqmodel_marlin_feature_check = functools.partial(+gptqmodel_marlin_feature_checker = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=1, out_feature_multiplier=64)++def fp8_static_scheme_checker(+ in_feature: int,+ out_feature: int,+ config: QuantizationScheme,+ in_feature_multiplier: Optional[int] = None,+ out_feature_multiplier: Optional[int] = None,+):+ from auto_round.schemes import FP8_STATIC++ return config == FP8_STATIC++BackendInfos["auto_gptq:exllamav2"] = BackendInfo(device=["cuda"],sym=[True, False],@@ -113,7 +130,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(dtype=["float16"],##16, 384,768 accuracy issuegroup_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],)@@ -126,7 +143,7 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(group_size=None,dtype=["float16"],priority=0,- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["auto_gptq:tritonv2"],requirements=["torch<2.6.0", "auto-gptq>=0.7.1", "triton>=2.0"],)@@ -138,7 +155,7 @@ BackendInfos["auto_gptq:cuda"] = BackendInfo(bits=[2, 3, 4, 8],group_size=None,priority=1,- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["auto_gptq:cuda"],dtype=["float16"],convertable_format=["int32_zp"],@@ -155,7 +172,7 @@ BackendInfos["auto_round:tritonv2"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 4, 8],priority=2,- feature_checks=[feature_multiply_checker_32],+ checkers=[feature_multiply_checker_32],alias=["auto_round", "tritonv2", "triton"],requirements=["triton>=2.0", "auto-round>=0.5.0"],)@@ -167,11 +184,27 @@ BackendInfos["auto_round:torch"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 3, 4, 8],priority=0,- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["auto_round", "torch"],requirements=["auto-round>=0.5.1"],)+# FP8 static quant+# Weight: FP8, per-channel, may be extended to per-tensor in future+# Activation: FP8, per-tensor++BackendInfos["auto_round:fp8_static"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format="",+ sym=[True],+ dtype=["float32", "float16", "bfloat16"],+ bits=[8],+ priority=0,+ checkers=[fp8_static_scheme_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.6.0"],+)+BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(device=["cuda", "xpu"],sym=[True], ## asym has accuracys@@ -180,7 +213,7 @@ BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 4, 8],priority=2,- feature_checks=[feature_multiply_checker_32],+ checkers=[feature_multiply_checker_32],alias=["tritonv2", "tritonv2_zp", "triton"],requirements=["triton>=2.0", "auto-round>=0.5.0"],)@@ -192,7 +225,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 3, 4, 8],priority=0,- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["torch", "torch_zp"],requirements=["auto-round>=0.5.1"],)@@ -205,7 +238,7 @@ BackendInfos["gptqmodel:marlin"] = BackendInfo(group_size=[-1, 32, 64, 128],dtype=["float16", "bfloat16"],priority=6,- feature_checks=[gptqmodel_marlin_feature_check],+ checkers=[gptqmodel_marlin_feature_checker],alias=["marlin", "gptqmodel"],requirements=["gptqmodel>=2.0"],)@@ -218,7 +251,7 @@ BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(group_size=[-1, 32, 64, 128],dtype=["float16", "bfloat16"],priority=6,- feature_checks=[gptqmodel_marlin_feature_check],+ checkers=[gptqmodel_marlin_feature_checker],alias=["marlin", "gptqmodel"],requirements=["gptqmodel>=2.0"],)@@ -231,7 +264,7 @@ BackendInfos["gptqmodel:exllamav2"] = BackendInfo(group_size=[-1, 32, 64, 128], ##16 seems has accuracy issuedtype=["float16", "bfloat16"],priority=5,- feature_checks=[exllamav2_feature_check],+ checkers=[exllamav2_feature_checker],alias=["exllamav2"],requirements=["gptqmodel>=2.0"],)@@ -255,7 +288,7 @@ BackendInfos["qbits"] = BackendInfo(bits=[2, 4, 8],group_size=None,priority=1,- feature_checks=[],+ checkers=[],alias=["itrex", "qbits"],dtype=["float16", "bfloat16"],convertable_format=["int32"],@@ -270,7 +303,7 @@ BackendInfos["qbits_zp"] = BackendInfo(group_size=None,dtype=["float16", "bfloat16"],priority=1,- feature_checks=[],+ checkers=[],alias=["itrex", "qbits"],convertable_format=["int32_zp"],requirements=["torch<2.7.0", "intel-extension-for-transformers"],@@ -284,7 +317,7 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(group_size=None,dtype=["float16", "bfloat16"],priority=1,- feature_checks=[],+ checkers=[],alias=["itrex", "qbits"],requirements=["torch<2.7.0", "intel-extension-for-transformers"],)@@ -296,7 +329,7 @@ BackendInfos["ipex_gptq"] = BackendInfo(bits=[4],group_size=None,priority=5,- feature_checks=[],+ checkers=[],dtype=["float16", "bfloat16"],convertable_format=["int32_zp"],alias=["ipex"],@@ -311,7 +344,7 @@ BackendInfos["ipex_awq"] = BackendInfo(group_size=None,priority=1,dtype=["float16", "bfloat16"],- feature_checks=[],+ checkers=[],alias=["ipex"],convertable_format=["awq"],requirements=["intel-extension-for-pytorch>=2.6"],@@ -340,9 +373,7 @@ BackendInfos["hpu_zp"] = BackendInfo()-def check_compatible(- backend_name, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=True-):+def check_compatible(backend_name, device, config, packing_format, in_features, out_features, check_requirements=True):"""Checks if the given configuration is compatible with the specified backend.Args:@@ -372,7 +403,7 @@ def check_compatible(- If the packing format does not match, it must be convertible."""backend = BackendInfos[backend_name]-+ bits, group_size, sym = config["bits"], config["group_size"], config["sym"]# Check if device is supported by the backendif device not in backend.device:return False@@ -395,8 +426,8 @@ def check_compatible(else:return False- for check in backend.feature_checks:- if not check(in_features, out_features, group_size):+ for check in backend.checkers:+ if not check(in_features, out_features, config):return Falseif check_requirements and backend.requirements is not None:@@ -413,7 +444,7 @@ def check_compatible(return True-def dynamic_import_inference_linear(backend, bits, group_size, sym):+def dynamic_import_inference_linear(backend, config):"""Dynamically imports and returns the appropriate QuantLinear class based on the given backend.This function dynamically loads the correct `QuantLinear` class based on the backend and quantization@@ -438,6 +469,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):ImportError:If required modules are missing for a backend (e.g., Intel Extension, GPTQ, auto_awq)."""+ bits, group_size, sym = config["bits"], config["group_size"], config["sym"]++ if AutoRoundFormat.TORCH_FP8_STATIC.value in backend:+ from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear++ return WeightFP8ActFP8StaticQuantLinear+if "qbits" in backend:try:from intel_extension_for_transformers import qbits # pylint: disable=E0401@@ -662,7 +700,7 @@ def find_backend(target_backend: str, orig_backend: str = None):)-def get_all_compatible_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):+def get_all_compatible_backend(device, backend, orig_backend, config, in_features, out_features):# Get packing format from the original backendpacking_format = BackendInfos[orig_backend].packing_format@@ -670,16 +708,14 @@ def get_all_compatible_backend(device, backend, orig_backend, bits, group_size,compatible_backends = [keyfor key in BackendInfos.keys()- if check_compatible(- key, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=False- )+ if check_compatible(key, device, config, packing_format, in_features, out_features, check_requirements=False)]# Return the first compatible backend or an empty list if none foundreturn compatible_backends-def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):+def get_layer_backend(device, backend, orig_backend, config, in_features, out_features):"""Selects the most suitable backend for the layer based on compatibility and priority.This function first checks if the specified backend supports the layer with the provided configuration.@@ -715,6 +751,7 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f"""# Check if the provided backend is in BackendInfosbackend = find_backend(backend)+if backend not in BackendInfos.keys():raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")@@ -723,13 +760,13 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f# Find and store other compatible backendssupported_backends = []for key in BackendInfos.keys():- if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features):+ if check_compatible(key, device, config, packing_format, in_features, out_features):supported_backends.append(key)# Raise an error if no compatible backends are foundif len(supported_backends) == 0:supported_backends_need_package = get_all_compatible_backend(- device, backend, orig_backend, bits, group_size, sym, in_features, out_features+ device, backend, orig_backend, config, in_features, out_features)if len(supported_backends_need_package) > 0:@@ -825,6 +862,7 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="# Instructional messagesinstall_instructions = []+for cmd in pip_cmds:if "intel-extension-for-pytorch" in cmd and target_device == "xpu":install_instructions.append(diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 3454e422d..fcfb83b4c 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -21,6 +21,7 @@ import torch.nn as nnfrom tqdm import tqdmfrom transformers.pytorch_utils import Conv1D+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.inference.backend import (BackendInfos,dynamic_import_inference_linear,@@ -29,6 +30,7 @@ from auto_round.inference.backend import (get_layer_backend,process_requirement,)+from auto_round.schemes import QuantizationSchemefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -61,7 +63,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_conftry: # transformers new apimodules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)except:- modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)+ modules_to_not_convert = _get_modules_to_not_convert(model, modules_to_not_convert)if modules_to_not_convert:for layer_name in layer_names:if any([re.search(re.compile(n), layer_name) for n in modules_to_not_convert]):@@ -238,6 +240,24 @@ def get_layer_config(model, quantization_config):data_type = getattr(quantization_config, "data_type", "int") # Default to "int" if not specifiedsym = quantization_config.sym+ act_bits = getattr(quantization_config, "act_bits", None)+ act_group_size = getattr(quantization_config, "act_group_size", False)+ act_sym = getattr(quantization_config, "act_sym", None)+ act_data_type = getattr(quantization_config, "act_data_type", None)+ act_dynamic = getattr(quantization_config, "act_dynamic", False)++ default_quant_scheme = QuantizationScheme(+ bits=bits,+ group_size=group_size,+ data_type=data_type,+ sym=sym,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ )+# Determine the quantization block listquant_block_list = getattr(quantization_config, "quant_block_list", None)if quant_block_list is None:@@ -284,17 +304,14 @@ def get_layer_config(model, quantization_config):layer_names = list(set(layer_names).union(extra_config.keys()))# Construct final layer configuration- layer_configs = {- layer_name: {- "bits": extra_config.get(layer_name, {}).get("bits", bits),- "group_size": extra_config.get(layer_name, {}).get("group_size", group_size),- "data_type": extra_config.get(layer_name, {}).get("data_type", data_type),- "sym": extra_config.get(layer_name, {}).get("sym", sym),- "clip": extra_config.get(layer_name, {}).get("clip", False),- }- for layer_name in layer_names- }-+ layer_configs = {}+ quant_scheme_attrs = QuantizationScheme.get_attributes()+ for layer_name in layer_names:+ layer_config = {}+ layer_extra_config = extra_config.get(layer_name, {})+ for scheme_attr in quant_scheme_attrs:+ layer_config[scheme_attr] = layer_extra_config.get(scheme_attr, getattr(default_quant_scheme, scheme_attr))+ layer_configs[layer_name] = QuantizationScheme.from_dict(layer_config)return layer_configs@@ -391,9 +408,7 @@ def _get_layer_backend(target_device, target_backend, orig_backend, config, in_ftarget_device,target_backend,orig_backend,- config["bits"],- config["group_size"],- config["sym"],+ config,in_features,out_features,)@@ -415,7 +430,7 @@ def _import_exllamav2_kernels():def _create_quant_layer(layer, layer_backend, config, in_features, out_features):"""Creates a quantized layer using the appropriate class."""- QuantLinear = dynamic_import_inference_linear(layer_backend, config["bits"], config["group_size"], config["sym"])+ QuantLinear = dynamic_import_inference_linear(layer_backend, config)bias = layer.bias is not None# Special handling for AWQ layers@@ -437,6 +452,8 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)out_features=out_features,bias=bias,)+ elif AutoRoundFormat.TORCH_FP8_STATIC.value in layer_backend:+ return QuantLinear.from_original(config, layer)# Default quantized layer creationtry:return QuantLinear(@@ -561,7 +578,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):backend = quantization_config.backendelse:backend = "auto"-##target_backend could be None_, backend = parse_target_device_and_backend(backend)@@ -588,7 +604,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):backend = backend[len("auto_round:") :]used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)-if backend == "auto" or backend == "":best_backend = get_highest_priority_backend(quantization_config.bits,diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex e9211959d..053a77523 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -13,8 +13,8 @@# limitations under the License.import copyfrom copy import deepcopy-from dataclasses import dataclass-from typing import Optional+from dataclasses import dataclass, fields+from typing import Generator, List, Optional__all__ = ["QuantizationScheme", "preset_name_to_scheme"]@@ -32,11 +32,52 @@ class QuantizationScheme:act_dynamic: Optional[bool] = Nonesuper_bits: Optional[int] = Nonesuper_group_size: Optional[int] = None+ clip: Optional[bool] = False@classmethoddef from_dict(cls, config: dict):return cls(**config)+ @classmethod+ def get_attributes(cls: "QuantizationScheme") -> List[str]:+ return [field.name for field in fields(cls)]++ def __getitem__(self, key: str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ return getattr(self, key)++ def __setitem__(self, key: str, value: None | int | str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ setattr(self, key, value)++ def items(self):+ return ((field, getattr(self, field)) for field in self.get_attributes())++ def keys(self):+ return self.get_attributes()++ def values(self):+ return (getattr(self, field) for field in self.get_attributes())++ def get(self, key: str, default=None):+ if key not in self.get_attributes():+ return default+ res = getattr(self, key)+ # In case the attribute is explicitly set to None, return default+ if res is None:+ return default+ return getattr(self, key)++ def __eq__(self, other: "QuantizationScheme") -> bool:+ if not isinstance(other, QuantizationScheme):+ return False+ for field in self.get_attributes():+ if getattr(self, field) != getattr(other, field):+ return False+ return True+def preset_name_to_scheme(name: str) -> QuantizationScheme:"""Get a QuantizationScheme instance from a preset scheme name."""@@ -154,6 +195,7 @@ FP8_STATIC = QuantizationScheme.from_dict("act_group_size": 0,"act_data_type": "fp","act_dynamic": False,+ "act_sym": True,})diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 4d00e570f..efac5bdfc 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -34,6 +34,7 @@ from packaging import versionfrom torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType+from auto_round.schemes import QuantizationSchemeSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")@@ -107,9 +108,17 @@ def infer_bits_by_data_type(data_type: str):return None-@lru_cache(None)-def warning_once(self, msg: str):- self.warning(msg)+@lru_cache(maxsize=None)+def warning_once(self, msg, *args, **kwargs):+ """+ Log a warning message only once per unique message/arguments combination.++ Args:+ msg: The warning message format string+ *args: Variable positional arguments for message formatting+ **kwargs: Variable keyword arguments for message formatting and logging options+ """+ self.warning(msg, *args, **kwargs)class AutoRoundFormatter(logging.Formatter):@@ -517,7 +526,7 @@ def check_to_quantized(config):bool: True if the configuration is valid for quantization (bits <= 8),False otherwise."""- if isinstance(config, dict):+ if isinstance(config, (dict, QuantizationScheme)):bits = int(config.get("bits", 16))act_bits = int(config.get("act_bits", 16))elif hasattr(config, "orig_layer"):@@ -2552,6 +2561,12 @@ def is_nv_fp(backend):return BackendDataType.NV_FP in backend+def _is_weight_fp8_activation_static_fp8(+ bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool+) -> bool:+ return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic++def is_wfp8afp8(ar):if (("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex e0a856f3e..aef11a30e 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -241,6 +241,33 @@ class TestAutoRound(unittest.TestCase):self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)+ if static_kv_dtype is None:+ with torch.no_grad():+ import transformers++ model = transformers.AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ low_cpu_mem_usage=True,+ trust_remote_code=True,+ )+ model.eval()+ assert (+ model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__+ == "WeightFP8ActFP8StaticQuantLinear"+ ), f"Expected WeightFP8ActFP8StaticQuantLinear, got {model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__}"+ tokenizer = transformers.AutoTokenizer.from_pretrained(quantized_model_path)+ prompt = "AI is "+ encode = tokenizer.encode(prompt, return_tensors="pt")+ with torch.no_grad():+ output_tokens = model.generate(+ encode,+ max_length=10,+ )+ output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)+ print(f"Prompt: {prompt}")+ print(f"Output: {output}")+ assert output is not None, "Output should not be None"if static_kv_dtype == "fp8":self.assertIn("model.decoder.layers.8.self_attn.k_scale", f.keys())