Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support loading for static quant weight fp8 act fp8 (#730)

Yi Liu09e4d312771a
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 03e012578..607ade4a9 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -19,6 +19,7 @@ import sys
import time
import traceback
from dataclasses import asdict, fields
+from enum import Enum
from typing import Any, Callable, Union
import accelerate
@@ -30,6 +31,7 @@ from transformers import set_seed
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.export.export_to_autoround import AutoRoundFormat
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
@@ -857,8 +859,8 @@ class AutoRound(object):
format = "auto_round:auto_awq"
elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
format = f"auto_round:{self.data_type}"
- elif is_wfp8afp8(self): # staic wfp8afp8
- format = "auto_round:fp8"
+ elif is_static_wfp8afp8(self): # staic wfp8afp8
+ format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"
elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8
format = "auto_round:fp8"
elif self.act_bits < 16:
@@ -956,10 +958,10 @@ class AutoRound(object):
)
format = "fake"
else:
- if not (format == "auto_round" or format == "auto_round:fp8"):
+ if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):
logger.warning(
f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"
- " change format to auto_round"
+ f" change format {format} to auto_round"
)
format = "auto_round"
if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":
diff --git a/auto_round/experimental/qmodules/base.py b/auto_round/experimental/qmodules/base.py
new file mode 100644
index 000000000..8b7a9c138
--- /dev/null
+++ b/auto_round/experimental/qmodules/base.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import torch
+
+__all__ = ["QModuleBase"]
+
+
+class QModuleBase(torch.nn.Module):
+ """
+ Base class used to describe the weight creation and forward pass
+ of different quantization schemes supported by Auto-Round.
+ The design is inspired by vLLM's CompressedTensorsScheme:
+ https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+
+ """
+
+ def __init__(self):
+ super().__init__()
+
+ @classmethod
+ @abstractmethod
+ def from_original(cls, config, original_layer: torch.nn.Module):
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def get_min_capability(cls) -> int:
+ """
+ Get minimum device capability.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def process_weights_after_loading(self, layer: torch.nn.Module):
+ """
+ Called after weight loading is complete for any cleanup that
+ needs to occur.
+ """
+ raise NotImplementedError
diff --git a/auto_round/experimental/qmodules/fp8_static.py b/auto_round/experimental/qmodules/fp8_static.py
new file mode 100644
index 000000000..e7c55086d
--- /dev/null
+++ b/auto_round/experimental/qmodules/fp8_static.py
@@ -0,0 +1,122 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Optional, Union
+
+import torch
+
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.utils import logger
+
+__all__ = ["WeightFP8ActFP8StaticQuantLinear"]
+
+
+def _quant_tensor_to_fp8_with_scale(tensor: torch.Tensor, scale: torch.Tensor):
+ FULL_RANGE = torch.finfo(torch.float8_e4m3fn).max
+ qtensor = tensor / scale
+ clipped_qtensor = torch.clamp(qtensor, -FULL_RANGE, FULL_RANGE)
+ clipped_qtensor_fp8 = clipped_qtensor.to(torch.float8_e4m3fn)
+ return scale, clipped_qtensor_fp8
+
+
+class WeightFP8ActFP8StaticQuantLinear(QModuleBase):
+ hp_dtype = torch.bfloat16
+ fp8_dtype = torch.float8_e4m3fn
+
+ def __init__(
+ self,
+ in_features,
+ out_features,
+ weight: Optional[torch.Tensor] = None,
+ weight_scale: Optional[torch.Tensor] = None,
+ bias: Union[torch.Tensor, bool, None] = None,
+ input_scale: Optional[torch.Tensor] = None,
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ init_weight = torch.zeros((out_features, in_features), dtype=dtype) if weight is None else weight
+ self.weight = torch.nn.Parameter(init_weight, requires_grad=False)
+ self.dtype = dtype
+ if bias is not None:
+ if isinstance(bias, bool):
+ bias = torch.zeros((out_features,), dtype=dtype)
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+ init_weight_scale = torch.empty((out_features), dtype=dtype) if weight_scale is None else weight_scale
+ self.register_buffer("weight_scale", init_weight_scale.to(dtype))
+
+ init_input_scale = torch.zeros((1), dtype=dtype) if input_scale is None else input_scale
+ self.register_buffer("input_scale", init_input_scale.to(dtype))
+ self.pre_dequantized = False
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ """
+ Get minimum device capability.
+ """
+ # TODO: correct that config once we add fp8 op support.
+ logger.warning_once("FP8 ops are not yet supported. Using capability 0.")
+ return 0
+
+ def process_weights_after_loading(self, layer: torch.nn.Module):
+ pass
+
+ @classmethod
+ def from_original(cls, config, original_layer):
+ """
+ Create an `WeightFP8ActFP8StaticQuantLinear` layer from an original linear layer.
+ """
+ logger.warning_once(
+ "FP8 static quantization is still in experimental stage, the inference speed might be slow."
+ )
+ device = original_layer.weight.device
+ with torch.device(device):
+ qdq_linear = cls(
+ in_features=original_layer.in_features,
+ out_features=original_layer.out_features,
+ bias=original_layer.bias,
+ )
+ return qdq_linear
+
+ def dequant_weight_online(self):
+ if self.pre_dequantized:
+ return self.weight
+ qdq_weight = self.weight.to(self.dtype) * self.weight_scale.unsqueeze(1)
+ return qdq_weight
+
+ def pre_dequantize(self):
+ if self.pre_dequantized:
+ return
+ dequant_weight = self.dequant_weight_online()
+ del self.weight
+ del self.weight_scale
+ self.weight = torch.nn.Parameter(dequant_weight, requires_grad=False)
+ self.pre_dequantized = True
+
+ def qdq_input(self, bf16_input: torch.Tensor):
+ input_scale, input_fp8 = _quant_tensor_to_fp8_with_scale(bf16_input, self.input_scale.data)
+ qdq_input_bf16 = input_fp8.to(self.dtype) * input_scale
+ return qdq_input_bf16
+
+ @torch.no_grad()
+ def forward(self, bf16_input: torch.Tensor) -> torch.Tensor:
+
+ qdq_input = self.qdq_input(bf16_input)
+ qdq_weight = self.dequant_weight_online()
+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)
+ return out
diff --git a/auto_round/export/export_to_autoround/__init__.py b/auto_round/export/export_to_autoround/__init__.py
index 0c036d831..6cdcd5aed 100644
--- a/auto_round/export/export_to_autoround/__init__.py
+++ b/auto_round/export/export_to_autoround/__init__.py
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .export import save_quantized_as_autoround
+from .export import save_quantized_as_autoround, AutoRoundFormat
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 1f6f94005..ffd10036c 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -18,6 +18,7 @@ import inspect
import json
import os
from concurrent.futures import ThreadPoolExecutor
+from enum import Enum
import threadpoolctl as tctl
import torch
@@ -43,6 +44,12 @@ from auto_round.utils import (
)
+class AutoRoundFormat(str, Enum):
+ # Weight: FP8, per-channel, may be extended to per-tensor in future
+ # Activation: FP8, per-tensor
+ TORCH_FP8_STATIC = "fp8_static"
+
+
def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):
"""
Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.
@@ -152,7 +159,7 @@ def pack_layer(layer_name, model, backend, device=None):
return pack_layer(layer_name, model, backend, device)
- if backend == "auto_round:fp8":
+ if backend == "auto_round:fp8" or backend == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}":
from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
return pack_layer(layer_name, model, backend, device)
@@ -268,9 +275,14 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)
+ from auto_round.autoround import AutoRoundFormat
##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source
- if (kwargs.get("sym") is None or kwargs.get("sym")) and ("gptq" not in backend and "awq" not in backend):
+ if (
+ (kwargs.get("sym") is None or kwargs.get("sym"))
+ and ("gptq" not in backend and "awq" not in backend)
+ and (AutoRoundFormat.TORCH_FP8_STATIC.value not in backend)
+ ):
backend = backend.replace("auto_round", "auto_round:auto_gptq")
model = kwargs["model"]
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 1569e8005..1868ee14c 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -19,6 +19,8 @@ from typing import Any, List, Optional
from transformers.utils.versions import require_version
import auto_round_extension.cuda.gptqmodel_marlin
+from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.schemes import QuantizationScheme
from auto_round.utils import get_library_version, logger
BackendInfos = {}
@@ -53,7 +55,7 @@ class BackendInfo:
indicate higher priority. Defaults to 0.
convertable_format: A list of strings specifying the formats that the backend
can convert from. Defaults to an empty list.
- feature_checks: A list of feature check functions (e.g., validation methods)
+ checkers: A list of check functions (e.g., validation methods)
used to verify whether the backend supports certain features. Defaults to
an empty list.
alias: An optional list of strings representing alternative names for the
@@ -68,20 +70,22 @@ class BackendInfo:
group_size: Optional[List[int]] = None
priority: int = 0 ##higher is better
convertable_format: List[str] = field(default_factory=list)
- feature_checks: List[Any] = field(default_factory=list)
+ checkers: List[Any] = field(default_factory=list)
alias: Optional[List[str]] = None
requirements: Optional[List[str]] = None
+ # TODO(Yi): Add more fields for activation dtype, group size, etc.
-def feature_multiply_checker(in_feature, out_feature, group_size, in_feature_multiplier, out_feature_multiplier=None):
+def feature_multiply_checker(in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None):
if out_feature_multiplier is None:
out_feature_multiplier = in_feature_multiplier
return in_feature % in_feature_multiplier == 0 and out_feature % out_feature_multiplier == 0
def feature_multiply_checker_group_size(
- in_feature, out_feature, group_size, in_feature_multiplier, out_feature_multiplier=None
+ in_feature, out_feature, config, in_feature_multiplier, out_feature_multiplier=None
):
+ group_size = config["group_size"]
if out_feature_multiplier is None:
out_feature_multiplier = in_feature_multiplier
return (
@@ -96,14 +100,27 @@ in_output_feature_multiply_checker_32 = functools.partial(
feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32
)
-exllamav2_feature_check = functools.partial(
+exllamav2_feature_checker = functools.partial(
feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32
)
-gptqmodel_marlin_feature_check = functools.partial(
+gptqmodel_marlin_feature_checker = functools.partial(
feature_multiply_checker_group_size, in_feature_multiplier=1, out_feature_multiplier=64
)
+
+def fp8_static_scheme_checker(
+ in_feature: int,
+ out_feature: int,
+ config: QuantizationScheme,
+ in_feature_multiplier: Optional[int] = None,
+ out_feature_multiplier: Optional[int] = None,
+):
+ from auto_round.schemes import FP8_STATIC
+
+ return config == FP8_STATIC
+
+
BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
device=["cuda"],
sym=[True, False],
@@ -113,7 +130,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
dtype=["float16"],
##16, 384,768 accuracy issue
group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],
)
@@ -126,7 +143,7 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
group_size=None,
dtype=["float16"],
priority=0,
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["auto_gptq:tritonv2"],
requirements=["torch<2.6.0", "auto-gptq>=0.7.1", "triton>=2.0"],
)
@@ -138,7 +155,7 @@ BackendInfos["auto_gptq:cuda"] = BackendInfo(
bits=[2, 3, 4, 8],
group_size=None,
priority=1,
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["auto_gptq:cuda"],
dtype=["float16"],
convertable_format=["int32_zp"],
@@ -155,7 +172,7 @@ BackendInfos["auto_round:tritonv2"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 4, 8],
priority=2,
- feature_checks=[feature_multiply_checker_32],
+ checkers=[feature_multiply_checker_32],
alias=["auto_round", "tritonv2", "triton"],
requirements=["triton>=2.0", "auto-round>=0.5.0"],
)
@@ -167,11 +184,27 @@ BackendInfos["auto_round:torch"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 3, 4, 8],
priority=0,
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["auto_round", "torch"],
requirements=["auto-round>=0.5.1"],
)
+# FP8 static quant
+# Weight: FP8, per-channel, may be extended to per-tensor in future
+# Activation: FP8, per-tensor
+
+BackendInfos["auto_round:fp8_static"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format="",
+ sym=[True],
+ dtype=["float32", "float16", "bfloat16"],
+ bits=[8],
+ priority=0,
+ checkers=[fp8_static_scheme_checker],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>0.6.0"],
+)
+
BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
device=["cuda", "xpu"],
sym=[True], ## asym has accuracys
@@ -180,7 +213,7 @@ BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 4, 8],
priority=2,
- feature_checks=[feature_multiply_checker_32],
+ checkers=[feature_multiply_checker_32],
alias=["tritonv2", "tritonv2_zp", "triton"],
requirements=["triton>=2.0", "auto-round>=0.5.0"],
)
@@ -192,7 +225,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 3, 4, 8],
priority=0,
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["torch", "torch_zp"],
requirements=["auto-round>=0.5.1"],
)
@@ -205,7 +238,7 @@ BackendInfos["gptqmodel:marlin"] = BackendInfo(
group_size=[-1, 32, 64, 128],
dtype=["float16", "bfloat16"],
priority=6,
- feature_checks=[gptqmodel_marlin_feature_check],
+ checkers=[gptqmodel_marlin_feature_checker],
alias=["marlin", "gptqmodel"],
requirements=["gptqmodel>=2.0"],
)
@@ -218,7 +251,7 @@ BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(
group_size=[-1, 32, 64, 128],
dtype=["float16", "bfloat16"],
priority=6,
- feature_checks=[gptqmodel_marlin_feature_check],
+ checkers=[gptqmodel_marlin_feature_checker],
alias=["marlin", "gptqmodel"],
requirements=["gptqmodel>=2.0"],
)
@@ -231,7 +264,7 @@ BackendInfos["gptqmodel:exllamav2"] = BackendInfo(
group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue
dtype=["float16", "bfloat16"],
priority=5,
- feature_checks=[exllamav2_feature_check],
+ checkers=[exllamav2_feature_checker],
alias=["exllamav2"],
requirements=["gptqmodel>=2.0"],
)
@@ -255,7 +288,7 @@ BackendInfos["qbits"] = BackendInfo(
bits=[2, 4, 8],
group_size=None,
priority=1,
- feature_checks=[],
+ checkers=[],
alias=["itrex", "qbits"],
dtype=["float16", "bfloat16"],
convertable_format=["int32"],
@@ -270,7 +303,7 @@ BackendInfos["qbits_zp"] = BackendInfo(
group_size=None,
dtype=["float16", "bfloat16"],
priority=1,
- feature_checks=[],
+ checkers=[],
alias=["itrex", "qbits"],
convertable_format=["int32_zp"],
requirements=["torch<2.7.0", "intel-extension-for-transformers"],
@@ -284,7 +317,7 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(
group_size=None,
dtype=["float16", "bfloat16"],
priority=1,
- feature_checks=[],
+ checkers=[],
alias=["itrex", "qbits"],
requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
@@ -296,7 +329,7 @@ BackendInfos["ipex_gptq"] = BackendInfo(
bits=[4],
group_size=None,
priority=5,
- feature_checks=[],
+ checkers=[],
dtype=["float16", "bfloat16"],
convertable_format=["int32_zp"],
alias=["ipex"],
@@ -311,7 +344,7 @@ BackendInfos["ipex_awq"] = BackendInfo(
group_size=None,
priority=1,
dtype=["float16", "bfloat16"],
- feature_checks=[],
+ checkers=[],
alias=["ipex"],
convertable_format=["awq"],
requirements=["intel-extension-for-pytorch>=2.6"],
@@ -340,9 +373,7 @@ BackendInfos["hpu_zp"] = BackendInfo(
)
-def check_compatible(
- backend_name, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=True
-):
+def check_compatible(backend_name, device, config, packing_format, in_features, out_features, check_requirements=True):
"""Checks if the given configuration is compatible with the specified backend.
Args:
@@ -372,7 +403,7 @@ def check_compatible(
- If the packing format does not match, it must be convertible.
"""
backend = BackendInfos[backend_name]
-
+ bits, group_size, sym = config["bits"], config["group_size"], config["sym"]
# Check if device is supported by the backend
if device not in backend.device:
return False
@@ -395,8 +426,8 @@ def check_compatible(
else:
return False
- for check in backend.feature_checks:
- if not check(in_features, out_features, group_size):
+ for check in backend.checkers:
+ if not check(in_features, out_features, config):
return False
if check_requirements and backend.requirements is not None:
@@ -413,7 +444,7 @@ def check_compatible(
return True
-def dynamic_import_inference_linear(backend, bits, group_size, sym):
+def dynamic_import_inference_linear(backend, config):
"""Dynamically imports and returns the appropriate QuantLinear class based on the given backend.
This function dynamically loads the correct `QuantLinear` class based on the backend and quantization
@@ -438,6 +469,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
ImportError:
If required modules are missing for a backend (e.g., Intel Extension, GPTQ, auto_awq).
"""
+ bits, group_size, sym = config["bits"], config["group_size"], config["sym"]
+
+ if AutoRoundFormat.TORCH_FP8_STATIC.value in backend:
+ from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
+
+ return WeightFP8ActFP8StaticQuantLinear
+
if "qbits" in backend:
try:
from intel_extension_for_transformers import qbits # pylint: disable=E0401
@@ -662,7 +700,7 @@ def find_backend(target_backend: str, orig_backend: str = None):
)
-def get_all_compatible_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):
+def get_all_compatible_backend(device, backend, orig_backend, config, in_features, out_features):
# Get packing format from the original backend
packing_format = BackendInfos[orig_backend].packing_format
@@ -670,16 +708,14 @@ def get_all_compatible_backend(device, backend, orig_backend, bits, group_size,
compatible_backends = [
key
for key in BackendInfos.keys()
- if check_compatible(
- key, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=False
- )
+ if check_compatible(key, device, config, packing_format, in_features, out_features, check_requirements=False)
]
# Return the first compatible backend or an empty list if none found
return compatible_backends
-def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):
+def get_layer_backend(device, backend, orig_backend, config, in_features, out_features):
"""Selects the most suitable backend for the layer based on compatibility and priority.
This function first checks if the specified backend supports the layer with the provided configuration.
@@ -715,6 +751,7 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
"""
# Check if the provided backend is in BackendInfos
backend = find_backend(backend)
+
if backend not in BackendInfos.keys():
raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")
@@ -723,13 +760,13 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
# Find and store other compatible backends
supported_backends = []
for key in BackendInfos.keys():
- if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features):
+ if check_compatible(key, device, config, packing_format, in_features, out_features):
supported_backends.append(key)
# Raise an error if no compatible backends are found
if len(supported_backends) == 0:
supported_backends_need_package = get_all_compatible_backend(
- device, backend, orig_backend, bits, group_size, sym, in_features, out_features
+ device, backend, orig_backend, config, in_features, out_features
)
if len(supported_backends_need_package) > 0:
@@ -825,6 +862,7 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
# Instructional messages
install_instructions = []
+
for cmd in pip_cmds:
if "intel-extension-for-pytorch" in cmd and target_device == "xpu":
install_instructions.append(
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 3454e422d..fcfb83b4c 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -21,6 +21,7 @@ import torch.nn as nn
from tqdm import tqdm
from transformers.pytorch_utils import Conv1D
+from auto_round.export.export_to_autoround import AutoRoundFormat
from auto_round.inference.backend import (
BackendInfos,
dynamic_import_inference_linear,
@@ -29,6 +30,7 @@ from auto_round.inference.backend import (
get_layer_backend,
process_requirement,
)
+from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -61,7 +63,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_conf
try: # transformers new api
modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)
except:
- modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)
+ modules_to_not_convert = _get_modules_to_not_convert(model, modules_to_not_convert)
if modules_to_not_convert:
for layer_name in layer_names:
if any([re.search(re.compile(n), layer_name) for n in modules_to_not_convert]):
@@ -238,6 +240,24 @@ def get_layer_config(model, quantization_config):
data_type = getattr(quantization_config, "data_type", "int") # Default to "int" if not specified
sym = quantization_config.sym
+ act_bits = getattr(quantization_config, "act_bits", None)
+ act_group_size = getattr(quantization_config, "act_group_size", False)
+ act_sym = getattr(quantization_config, "act_sym", None)
+ act_data_type = getattr(quantization_config, "act_data_type", None)
+ act_dynamic = getattr(quantization_config, "act_dynamic", False)
+
+ default_quant_scheme = QuantizationScheme(
+ bits=bits,
+ group_size=group_size,
+ data_type=data_type,
+ sym=sym,
+ act_bits=act_bits,
+ act_group_size=act_group_size,
+ act_sym=act_sym,
+ act_data_type=act_data_type,
+ act_dynamic=act_dynamic,
+ )
+
# Determine the quantization block list
quant_block_list = getattr(quantization_config, "quant_block_list", None)
if quant_block_list is None:
@@ -284,17 +304,14 @@ def get_layer_config(model, quantization_config):
layer_names = list(set(layer_names).union(extra_config.keys()))
# Construct final layer configuration
- layer_configs = {
- layer_name: {
- "bits": extra_config.get(layer_name, {}).get("bits", bits),
- "group_size": extra_config.get(layer_name, {}).get("group_size", group_size),
- "data_type": extra_config.get(layer_name, {}).get("data_type", data_type),
- "sym": extra_config.get(layer_name, {}).get("sym", sym),
- "clip": extra_config.get(layer_name, {}).get("clip", False),
- }
- for layer_name in layer_names
- }
-
+ layer_configs = {}
+ quant_scheme_attrs = QuantizationScheme.get_attributes()
+ for layer_name in layer_names:
+ layer_config = {}
+ layer_extra_config = extra_config.get(layer_name, {})
+ for scheme_attr in quant_scheme_attrs:
+ layer_config[scheme_attr] = layer_extra_config.get(scheme_attr, getattr(default_quant_scheme, scheme_attr))
+ layer_configs[layer_name] = QuantizationScheme.from_dict(layer_config)
return layer_configs
@@ -391,9 +408,7 @@ def _get_layer_backend(target_device, target_backend, orig_backend, config, in_f
target_device,
target_backend,
orig_backend,
- config["bits"],
- config["group_size"],
- config["sym"],
+ config,
in_features,
out_features,
)
@@ -415,7 +430,7 @@ def _import_exllamav2_kernels():
def _create_quant_layer(layer, layer_backend, config, in_features, out_features):
"""Creates a quantized layer using the appropriate class."""
- QuantLinear = dynamic_import_inference_linear(layer_backend, config["bits"], config["group_size"], config["sym"])
+ QuantLinear = dynamic_import_inference_linear(layer_backend, config)
bias = layer.bias is not None
# Special handling for AWQ layers
@@ -437,6 +452,8 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)
out_features=out_features,
bias=bias,
)
+ elif AutoRoundFormat.TORCH_FP8_STATIC.value in layer_backend:
+ return QuantLinear.from_original(config, layer)
# Default quantized layer creation
try:
return QuantLinear(
@@ -561,7 +578,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
backend = quantization_config.backend
else:
backend = "auto"
-
##target_backend could be None
_, backend = parse_target_device_and_backend(backend)
@@ -588,7 +604,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
backend = backend[len("auto_round:") :]
used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)
-
if backend == "auto" or backend == "":
best_backend = get_highest_priority_backend(
quantization_config.bits,
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index e9211959d..053a77523 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -13,8 +13,8 @@
# limitations under the License.
import copy
from copy import deepcopy
-from dataclasses import dataclass
-from typing import Optional
+from dataclasses import dataclass, fields
+from typing import Generator, List, Optional
__all__ = ["QuantizationScheme", "preset_name_to_scheme"]
@@ -32,11 +32,52 @@ class QuantizationScheme:
act_dynamic: Optional[bool] = None
super_bits: Optional[int] = None
super_group_size: Optional[int] = None
+ clip: Optional[bool] = False
@classmethod
def from_dict(cls, config: dict):
return cls(**config)
+ @classmethod
+ def get_attributes(cls: "QuantizationScheme") -> List[str]:
+ return [field.name for field in fields(cls)]
+
+ def __getitem__(self, key: str):
+ if key not in self.get_attributes():
+ raise KeyError(f"{key} is not a valid attribute")
+ return getattr(self, key)
+
+ def __setitem__(self, key: str, value: None | int | str):
+ if key not in self.get_attributes():
+ raise KeyError(f"{key} is not a valid attribute")
+ setattr(self, key, value)
+
+ def items(self):
+ return ((field, getattr(self, field)) for field in self.get_attributes())
+
+ def keys(self):
+ return self.get_attributes()
+
+ def values(self):
+ return (getattr(self, field) for field in self.get_attributes())
+
+ def get(self, key: str, default=None):
+ if key not in self.get_attributes():
+ return default
+ res = getattr(self, key)
+ # In case the attribute is explicitly set to None, return default
+ if res is None:
+ return default
+ return getattr(self, key)
+
+ def __eq__(self, other: "QuantizationScheme") -> bool:
+ if not isinstance(other, QuantizationScheme):
+ return False
+ for field in self.get_attributes():
+ if getattr(self, field) != getattr(other, field):
+ return False
+ return True
+
def preset_name_to_scheme(name: str) -> QuantizationScheme:
"""Get a QuantizationScheme instance from a preset scheme name."""
@@ -154,6 +195,7 @@ FP8_STATIC = QuantizationScheme.from_dict(
"act_group_size": 0,
"act_data_type": "fp",
"act_dynamic": False,
+ "act_sym": True,
}
)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 4d00e570f..efac5bdfc 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -34,6 +34,7 @@ from packaging import version
from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
+from auto_round.schemes import QuantizationScheme
SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
@@ -107,9 +108,17 @@ def infer_bits_by_data_type(data_type: str):
return None
-@lru_cache(None)
-def warning_once(self, msg: str):
- self.warning(msg)
+@lru_cache(maxsize=None)
+def warning_once(self, msg, *args, **kwargs):
+ """
+ Log a warning message only once per unique message/arguments combination.
+
+ Args:
+ msg: The warning message format string
+ *args: Variable positional arguments for message formatting
+ **kwargs: Variable keyword arguments for message formatting and logging options
+ """
+ self.warning(msg, *args, **kwargs)
class AutoRoundFormatter(logging.Formatter):
@@ -517,7 +526,7 @@ def check_to_quantized(config):
bool: True if the configuration is valid for quantization (bits <= 8),
False otherwise.
"""
- if isinstance(config, dict):
+ if isinstance(config, (dict, QuantizationScheme)):
bits = int(config.get("bits", 16))
act_bits = int(config.get("act_bits", 16))
elif hasattr(config, "orig_layer"):
@@ -2552,6 +2561,12 @@ def is_nv_fp(backend):
return BackendDataType.NV_FP in backend
+def _is_weight_fp8_activation_static_fp8(
+ bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool
+) -> bool:
+ return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic
+
+
def is_wfp8afp8(ar):
if (
("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index e0a856f3e..aef11a30e 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -241,6 +241,33 @@ class TestAutoRound(unittest.TestCase):
self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())
self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.input_scale").shape, torch.Size([1]))
self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)
+ if static_kv_dtype is None:
+ with torch.no_grad():
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ low_cpu_mem_usage=True,
+ trust_remote_code=True,
+ )
+ model.eval()
+ assert (
+ model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__
+ == "WeightFP8ActFP8StaticQuantLinear"
+ ), f"Expected WeightFP8ActFP8StaticQuantLinear, got {model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__}"
+ tokenizer = transformers.AutoTokenizer.from_pretrained(quantized_model_path)
+ prompt = "AI is "
+ encode = tokenizer.encode(prompt, return_tensors="pt")
+ with torch.no_grad():
+ output_tokens = model.generate(
+ encode,
+ max_length=10,
+ )
+ output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+ print(f"Prompt: {prompt}")
+ print(f"Output: {output}")
+ assert output is not None, "Output should not be None"
if static_kv_dtype == "fp8":
self.assertIn("model.decoder.layers.8.self_attn.k_scale", f.keys())