Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support block-wise fp8 quant (#1487)

Wang, Mengnie6600591706f
diff --git a/README.md b/README.md
index d38161bfc..348d1dcd7 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,7 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]
## 🆕 What's New
+* [2026/03] **Block-wise FP8** quantization is available via `--scheme FP8_BLOCK --iters 0 --disable_opt_rtn`.
* [2026/03] MTP layer quantization has been supported in [PR](https://github.com/intel/auto-round/pull/1526)
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 2254aafa8..5721a9501 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -288,7 +288,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
scheme = self.add_argument_group("Scheme Arguments")
scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")
- scheme.add_argument("--group_size", default=None, type=int, help="Group size for weight quantization.")
+ scheme.add_argument(
+ "--group_size",
+ default=None,
+ type=lambda s: int(s) if s.isdigit() else tuple([int(x.strip()) for x in s.split(",")]),
+ help="Group size for weight quantization.",
+ )
scheme.add_argument("--asym", action="store_true", help="Use asymmetric quantization instead of symmetric.")
scheme.add_argument(
"--data_type",
@@ -724,13 +729,17 @@ def tune(args):
elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:
export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")
else:
- if autoround.group_size <= 0:
- if "fp" in autoround.act_data_type:
- suffix = f"afp{autoround.act_bits}"
- else:
- suffix = f"a{autoround.act_bits}"
+ if isinstance(autoround.group_size, tuple):
+ assert len(autoround.group_size) == 2, f"Only support 2D group_size, but get {autoround.group_size}"
+ suffix = f"g{autoround.group_size[0]}x{autoround.group_size[1]}"
else:
- suffix = f"g{autoround.group_size}"
+ if autoround.group_size <= 0:
+ if "fp" in autoround.act_data_type:
+ suffix = f"afp{autoround.act_bits}"
+ else:
+ suffix = f"a{autoround.act_bits}"
+ else:
+ suffix = f"g{autoround.group_size}"
export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")
# ======================= Quantize and save model =======================
diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.py
index dd188896a..8a45b443e 100644
--- a/auto_round/alg_ext.py
+++ b/auto_round/alg_ext.py
@@ -31,7 +31,7 @@ from auto_round.data_type.nvfp import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, ref_nvfp
from auto_round.data_type.utils import floor_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.logger import logger
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal, set_module
-from auto_round.wrapper import NORM_MAPPING, WrapperLinear, reshape_and_pad_tensor
+from auto_round.wrapper import NORM_MAPPING, WrapperLinear
__all__ = ["wrapper_autoround"]
@@ -335,7 +335,7 @@ class WrapperLinearV2(WrapperLinear):
super()._init_tuning_params_and_quant_func()
orig_weight = getattr(self.orig_layer, "get_weight", lambda: self.orig_layer.weight)()
- weight_reshape = reshape_and_pad_tensor(orig_weight.data, self.orig_layer.group_size)
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, self.orig_layer.group_size)
if hasattr(self.orig_layer, "imatrix"): # MOE model may have no imatrix
imatrix = self.orig_layer.imatrix.reshape(1, -1)
imatrix = reshape_pad_tensor_by_group_size(imatrix, self.orig_layer.group_size, val=1e-5)[0].view(1, -1)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4c3abe4ba..8b161be73 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -45,7 +45,7 @@ class AutoRound:
tokenizer: Tokenizer used to prepare input text for calibration/tuning.
platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
bits (int): Weight quantization bits.
- group_size (int): Per-group size for weight quantization.
+ group_size (int or tuple): Per-group size for weight quantization.
sym (bool): Whether to use symmetric weight quantization.
layer_config (dict): Per-layer quantization configuration.
nsamples (int): Number of calibration samples.
@@ -55,7 +55,7 @@ class AutoRound:
SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "extra_config", "enable_adam")
bits: int | None
- group_size: int | None
+ group_size: int | tuple | None
sym: bool | None
data_type: str | None
act_bits: int | None
@@ -118,7 +118,7 @@ class AutoRound:
low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.
bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int, optional): Weight quantization group size. Defaults to 128.
+ group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.
sym (bool, optional): Symmetric weight quantization. Defaults to True.
data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
act_bits (int, optional): Activation quantization bits. Defaults to 16.
@@ -253,7 +253,7 @@ class AutoRoundLLM(LLMCompressor):
tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int, optional): Weight quantization group size. Defaults to 128.
+ group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.
sym (bool, optional): Symmetric weight quantization. Defaults to True.
layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
batch_size (int, optional): Calibration batch size. Defaults to 8.
@@ -307,7 +307,7 @@ class AutoRoundLLM(LLMCompressor):
"""
bits: int | None
- group_size: int | None
+ group_size: int | tuple | None
sym: bool | None
data_type: str | None
act_bits: int | None
@@ -354,7 +354,7 @@ class AutoRoundAdam(AdamCompressor):
platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
+ group_size (int or tuple): Size of the quantization group (default is 128).
sym (bool): Whether sym to be used (default is True).
layer_config (dict): Configuration for weight quantization (default is None).
batch_size (int): Batch size for training (default is 8).
@@ -396,7 +396,7 @@ class AutoRoundAdam(AdamCompressor):
"""
bits: int | None
- group_size: int | None
+ group_size: int | tuple | None
sym: bool | None
data_type: str | None
act_bits: int | None
@@ -446,7 +446,7 @@ class AutoRoundMLLM(MLLMCompressor):
decode the data that groups several modalities (among text, vision and audio).
image_processor: Image processor for special model like llava.
bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
+ group_size (int or tuple): Size of the quantization group (default is 128).
sym (bool): Whether sym to be used (default is True).
layer_config (dict): Configuration for weight quantization (default is None).
batch_size (int): Batch size for training (default is 8).
@@ -485,7 +485,7 @@ class AutoRoundMLLM(MLLMCompressor):
"""
bits: int | None
- group_size: int | None
+ group_size: int | tuple | None
sym: bool | None
data_type: str | None
act_bits: int | None
@@ -552,7 +552,7 @@ class AutoRoundDiffusion(DiffusionCompressor):
"""
bits: int | None
- group_size: int | None
+ group_size: int | tuple | None
sym: bool | None
data_type: str | None
act_bits: int | None
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 9d7184e40..663186f3f 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -44,6 +44,8 @@ from auto_round.compressors.utils import (
get_shared_keys,
infer_bits_by_data_type,
init_cache,
+ is_block_wfp8,
+ is_dynamic_afp8,
is_dynamic_wint8aint8,
is_mx_fp,
is_nv_fp,
@@ -457,7 +459,7 @@ class BaseCompressor(object):
self.bits >= 8
and self.act_bits >= 8
and self.iters == 0
- and self.data_type == "int"
+ and self.data_type in ["int", "fp"]
and disable_opt_rtn is None
):
logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")
@@ -468,6 +470,9 @@ class BaseCompressor(object):
)
disable_opt_rtn = False
+ if self.iters > 0 and is_block_wfp8(self):
+ logger.warning("RTN is recommended since it shows even better accuracy for block-wise fp8 quantization.")
+
# Important Note! This is not very robust, do NOT rely on it to do high risky thing
self.is_moe_model = is_moe_model(self.model)
@@ -840,10 +845,14 @@ class BaseCompressor(object):
raise ValueError("`bits` must be positive")
if self.act_bits <= 0:
raise ValueError("`act_bits` must be positive")
- if not (self.group_size == -1 or self.group_size >= 0):
- raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
- if not (self.act_group_size == -1 or self.act_group_size >= 0):
+ if not isinstance(self.group_size, tuple) and not (self.group_size == -1 or self.group_size >= 0):
+ raise ValueError(
+ "`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer or a tuple of length 2"
+ )
+ if isinstance(self.act_group_size, tuple) or not (self.act_group_size == -1 or self.act_group_size >= 0):
raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
+ if isinstance(self.group_size, tuple) and len(self.group_size) != 2:
+ raise ValueError("`group_size` must be a tuple of length 2")
if self.batch_size <= 0:
raise ValueError("`batch_size` must be positive")
if self.iters < 0:
@@ -861,6 +870,7 @@ class BaseCompressor(object):
and not is_mx_fp(self.act_data_type)
and not is_dynamic_wint8aint8(self)
and not is_static_wfp8afp8(self.act_data_type)
+ and not is_dynamic_afp8(self)
):
logger.warning(
"activation quantization is an experimental feature with limited support and a complex API. "
@@ -873,6 +883,12 @@ class BaseCompressor(object):
if is_nv_fp(self.data_type) and (self.group_size != 16):
logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")
+ if isinstance(self.group_size, tuple):
+ if not is_block_wfp8(self):
+ raise NotImplementedError("only support block-wise quantization for fp8 weight quantization.")
+ if not is_dynamic_afp8(self):
+ raise NotImplementedError("only support dynamic fp8 activation for fp8 weight quantization.")
+
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
if self.iters > 0: # GGUF should log this warning, but we don't know the format here
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 1aef5bd7c..bda959e37 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -99,6 +99,19 @@ def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:
return False
+def is_dynamic_afp8(ar_or_format: Callable) -> bool:
+ return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8
+
+
+def is_block_wfp8(ar_or_format: Callable) -> bool:
+ return (
+ isinstance(ar_or_format.group_size, tuple)
+ and len(ar_or_format.group_size) == 2
+ and ar_or_format.data_type.startswith("fp")
+ and ar_or_format.bits == 8
+ )
+
+
def block_forward(
block: torch.nn.Module,
input_ids: torch.Tensor,
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.py
index ed446f314..58c746470 100644
--- a/auto_round/data_type/__init__.py
+++ b/auto_round/data_type/__init__.py
@@ -17,6 +17,10 @@ import auto_round.data_type.mxfp
import auto_round.data_type.fp8
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
import auto_round.data_type.w4fp8
-from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scales
+from auto_round.data_type.utils import (
+ get_quant_func,
+ reshape_pad_tensor_by_group_size,
+ update_fused_layer_global_scales,
+)
import auto_round.data_type.nvfp
import auto_round.data_type.gguf
diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.py
index fe108c3af..ea7a6e99e 100644
--- a/auto_round/data_type/fp8.py
+++ b/auto_round/data_type/fp8.py
@@ -25,6 +25,51 @@ from auto_round.data_type.utils import (
from auto_round.utils import is_gaudi2, logger
+@register_dtype(("block_fp8_sym", "block_fp8", "block_fp8_e4m3"))
+def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, **kwargs):
+ """Symmetric quantization using block float8 format.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.to(tensor.device)
+ if isinstance(v, torch.Tensor):
+ v = v.to(tensor.device)
+ if tensor_max is None:
+ max_tensor = tensor.abs().amax(dim=(-2, -1)) * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ scale = max_tensor / info.max
+ assert len(scale.shape) == 2, f"Only support 2D group_size, but get {len(scale.shape)}"
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = torch.clip(scale, min=min_scaling_factor)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+
+ fp8_res = tensor / scale.unsqueeze(-1).unsqueeze(-1) + v
+ fp8_res = torch.clip(fp8_res, info.min, info.max)
+ fp8_res = float8_e4m3fn_ste(fp8_res)
+ qdq_res = fp8_res * scale.unsqueeze(-1).unsqueeze(-1)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))
def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
"""Symmetric quantization using float8 format.
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 465ea0b87..a8e3eb218 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -14,7 +14,8 @@
import math
from functools import lru_cache
-from typing import List
+from math import ceil
+from typing import List, Union
import torch
from torch.nn import Linear, Module
@@ -24,7 +25,7 @@ from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
from auto_round.utils import check_to_quantized, logger
-def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: float = 0.0):
+def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):
"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.
This function adjusts the
@@ -35,7 +36,7 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: f
Args:
data (torch.Tensor): The input tensor to be reshaped and padded.
- group_size (int): The size of the groups that the tensor should be reshaped into.
+ group_size (int or tuple): The size of the groups that the tensor should be reshaped into.
Returns:
torch.Tensor: The reshaped and padded tensor, if necessary.
@@ -44,6 +45,14 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: f
"""
orig_shape = data.shape
pad_len = 0
+ if isinstance(group_size, tuple):
+ assert len(group_size) == 2, f"Only support 2D group_size, but get {len(group_size)}"
+ M, N = group_size
+ pad_len_m = ceil(orig_shape[0] / M) * M - orig_shape[0]
+ pad_len_n = ceil(orig_shape[1] / N) * N - orig_shape[1]
+ data_new = torch.nn.functional.pad(data, (0, pad_len_n, 0, pad_len_m))
+ data_new = data_new.view(data_new.shape[0] // M, M, data_new.shape[1] // N, N).permute(0, 2, 1, 3)
+ return data_new, orig_shape, (pad_len_m, pad_len_n)
if group_size == 0:
data = data.reshape(1, -1)
return data, orig_shape, pad_len
@@ -55,13 +64,13 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: f
data = data.reshape(-1, group_size)
return data, orig_shape, pad_len
else:
- pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]
+ pad_len = ceil(data.shape[1] / group_size) * group_size - data.shape[1]
data_new = torch.nn.functional.pad(data, (0, pad_len), value=val)
data_new = data_new.reshape(-1, group_size)
return data_new, orig_shape, pad_len
-def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):
+def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: Union[int, list]):
"""Reverts the tensor to its original shape by removing padding.
This function removes the padding added during reshaping and returns the tensor to
@@ -70,11 +79,15 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):
Args:
data (torch.Tensor): The reshaped and possibly padded tensor.
orig_shape (tuple): The original shape of the tensor before reshaping.
- pad_len (int): The length of the padding to be removed.
+ pad_len (int or tuple): The length of the padding to be removed.
Returns:
torch.Tensor: The tensor restored to its original shape.
"""
+ if isinstance(pad_len, tuple):
+ assert len(pad_len) == 2, f"Only support 2D group_size, but get {len(pad_len)}"
+ data = data.permute(0, 2, 1, 3).reshape(orig_shape[0] + pad_len[0], orig_shape[1] + pad_len[1])
+ return data[: data.shape[0] - pad_len[0], : data.shape[1] - pad_len[1]]
if pad_len == 0:
return data.reshape(orig_shape)
else:
@@ -88,7 +101,7 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):
return data_new
-def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> tuple[callable, str]:
+def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None) -> tuple[callable, str]:
"""Retrieve the quantization function based on data type, bit width, and symmetry.
This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE
@@ -100,6 +113,7 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> t
bits (int): The bit width for the quantization (e.g., 2,4,8).
sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).
disable_opt_rtn(bool): whether to disable optimized rtn.
+ group_size (tuple): The block size for weight quantization (e.g., (128, 128)).
Returns:
function: The quantization function corresponding to the specified parameters.
@@ -125,6 +139,21 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> t
if data_type in QUANT_FUNC_WITH_DTYPE:
return QUANT_FUNC_WITH_DTYPE[data_type], data_type
+ if group_size is not None and isinstance(group_size, tuple):
+ block_data_type = "block_" + dtype
+ data_types = [
+ block_data_type,
+ pad_bits(block_data_type),
+ pad_sym(block_data_type),
+ pad_sym(pad_bits(block_data_type)),
+ ]
+
+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+
+ for data_type in data_types:
+ if data_type in QUANT_FUNC_WITH_DTYPE:
+ return QUANT_FUNC_WITH_DTYPE[data_type], data_type
+
data_types = [dtype, pad_bits(dtype), pad_sym(dtype), pad_sym(pad_bits(dtype))]
for data_type in data_types:
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index 70c7c71b1..fadebba5d 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -73,6 +73,32 @@ class FP8QLinear(torch.nn.Module):
self.register_buffer("input_scale", input_scale.to(dtype))
+class FP8BlockQLinear(torch.nn.Module):
+
+ def __init__(
+ self,
+ in_features,
+ out_features,
+ weight,
+ weight_scale,
+ bias=None,
+ weight_zp=None,
+ input_scale=None,
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = torch.nn.Parameter(weight, requires_grad=False)
+
+ if bias is not None:
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+
+ self.register_buffer("weight_scale_inv", weight_scale.to(dtype))
+
+
def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
"""
Packs a model layer for quantization based on its type and configuration.
@@ -102,7 +128,7 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
return
orig_device = layer.weight.device
- scale = layer.scale.view(-1)
+ scale = layer.scale if isinstance(layer.group_size, tuple) else layer.scale.view(-1)
zp = layer.zp
weight = layer.weight
weight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)
@@ -114,9 +140,15 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
if zp is not None:
if isinstance(zp, torch.Tensor):
zp = zp.to(packing_device)
- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp
+ if isinstance(layer.group_size, tuple):
+ q_weight = weight.to(packing_device) / scale.unsqueeze(-1).unsqueeze(-1).to(packing_device) + zp
+ else:
+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp
else:
- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
+ if isinstance(layer.group_size, tuple):
+ q_weight = weight.to(packing_device) / scale.unsqueeze(-1).unsqueeze(-1).to(packing_device)
+ else:
+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)
q_weight = torch.clamp(q_weight, info.min, info.max)
q_weight = q_weight.to(torch_dtype)
@@ -130,7 +162,8 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
in_features = layer.weight.shape[0]
out_features = layer.weight.shape[1]
bias = layer.bias
- my_linear = FP8QLinear(
+ linear_cls = FP8BlockQLinear if isinstance(layer.group_size, tuple) else FP8QLinear
+ my_linear = linear_cls(
in_features,
out_features,
weight=q_weight,
@@ -140,7 +173,12 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
input_scale=act_scale,
dtype=model.dtype,
)
- if unsqueeze and len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:
+ if (
+ unsqueeze
+ and isinstance(linear_cls, FP8QLinear)
+ and len(my_linear.weight_scale.shape)
+ and my_linear.weight_scale.shape[0] != 1
+ ):
my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)
my_linear.to(orig_device)
@@ -158,6 +196,7 @@ def save_quantized_as_autoround(
backend: str = None,
device: Union[str, torch.device] = "cpu",
serialization_dict: dict = None,
+ quant_method: str = "auto-round",
**kwargs,
):
safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]
@@ -165,7 +204,7 @@ def save_quantized_as_autoround(
model = copy.deepcopy(model.to("cpu"))
quantization_config = serialization_dict
quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
- quantization_config["quant_method"] = "auto-round"
+ quantization_config["quant_method"] = quant_method
if backend:
quantization_config["packing_format"] = backend
if "e5m2" in serialization_dict.get("data_type", "fp8"):
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 8df22b93d..e5fe46abf 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -32,24 +32,55 @@ from auto_round.utils import (
from auto_round.wrapper import WrapperWALayer
+def _get_weight_scheme_strategy(group_size):
+ if group_size == 0:
+ return "tensor"
+ if group_size == -1:
+ return "channel"
+ if isinstance(group_size, tuple):
+ return "block"
+ if isinstance(group_size, int):
+ return "group"
+ return None
+
+
+def _get_act_scheme_strategy(group_size):
+ if group_size == 0:
+ return "tensor"
+ if group_size == -1:
+ return "token"
+ if isinstance(group_size, int):
+ return "group"
+ return None
+
+
+def _get_scheme_type(data_type):
+ if "int" in data_type:
+ return "int"
+ if "fp" in data_type or "float" in data_type:
+ return "float"
+ raise NotImplementedError("only support `int` and `fp` data type")
+
+
def construct_ct_scheme(layer):
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme # pylint: disable=E0401
weights_args = QuantizationArgs(
num_bits=layer.bits,
- type=layer.data_type.split("_")[-2], # int_sym, rtn_int_sym
+ type=_get_scheme_type(layer.data_type),
symmetric=layer.sym,
dynamic=False,
- group_size=None,
- strategy="tensor" if layer.act_group_size == 0 else "channel" if layer.act_group_size == -1 else None,
+ group_size=layer.group_size if _get_weight_scheme_strategy(layer.group_size) == "group" else None,
+ strategy=_get_weight_scheme_strategy(layer.group_size),
+ block_structure=layer.group_size if _get_weight_scheme_strategy(layer.group_size) == "block" else None,
)
activations_args = QuantizationArgs(
num_bits=layer.act_bits,
- type=layer.act_data_type.split("_")[-2], # int_sym, rtn_int_sym
+ type=_get_scheme_type(layer.act_data_type),
symmetric=layer.act_sym,
dynamic=layer.act_dynamic,
- group_size=None,
- strategy="tensor" if layer.act_group_size == 0 else "token" if layer.act_group_size == -1 else None,
+ group_size=layer.act_group_size if _get_act_scheme_strategy(layer.act_group_size) == "group" else None,
+ strategy=_get_act_scheme_strategy(layer.act_group_size),
)
scheme = QuantizationScheme(
targets=[layer.__class__.__name__],
@@ -59,8 +90,16 @@ def construct_ct_scheme(layer):
return scheme
+def _get_quant_format(model):
+ for n, m in model.named_modules():
+ if hasattr(m, "quantization_scheme") and hasattr(m.quantization_scheme, "format"):
+ return m.quantization_scheme.format
+ return None
+
+
def pack_layer(name, model, device=None):
- from compressed_tensors.compressors import IntQuantizationCompressor # pylint: disable=E0401
+ from compressed_tensors.compressors import NaiveQuantizationCompressor # pylint: disable=E0401
+ from compressed_tensors.config.format import set_per_module_format # pylint: disable=E0401
from compressed_tensors.quantization import QuantizationStatus # pylint: disable=E0401
from compressed_tensors.utils import delete_offload_parameter, register_offload_parameter # pylint: disable=E0401
@@ -96,7 +135,7 @@ def pack_layer(name, model, device=None):
setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))
delattr(layer, "scale")
- compressor = IntQuantizationCompressor()
+ compressor = NaiveQuantizationCompressor()
q_state_dict = compressor.compress(layer.state_dict(), names_to_scheme={"": scheme}, show_progress=False)
# remove any existing parameters
@@ -109,6 +148,7 @@ def pack_layer(name, model, device=None):
register_offload_parameter(layer, name, param, device)
layer.quantization_status = QuantizationStatus.COMPRESSED
+ set_per_module_format(layer)
@torch.no_grad()
@@ -166,12 +206,14 @@ def save_quantized_as_llmcompressor(
for n, m in model.named_modules():
pack_layer(n, model, device)
+ quant_format = _get_quant_format(model)
+ quantization_config = QuantizationConfig.from_pretrained(model, format=quant_format)
+ model.config.quantization_config = quantization_config.to_dict()
+
if output_dir is None:
return model
- quantization_config = QuantizationConfig.from_pretrained(model)
# save model.config, model.state_dict()
- model.config.quantization_config = quantization_config.to_dict()
model.config.save_pretrained(output_dir)
save_model(model, output_dir, safe_serialization=safe_serialization)
diff --git a/auto_round/formats.py b/auto_round/formats.py
index e6fc7f56f..d6f40362a 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -27,6 +27,8 @@ import torch
import transformers
from auto_round.compressors.utils import (
+ is_block_wfp8,
+ is_dynamic_afp8,
is_dynamic_wint8aint8,
is_mx_fp,
is_nv_fp,
@@ -65,6 +67,7 @@ class AutoRoundExportFormat(str, Enum):
MX_FP_RCEIL = "mx_fp_rceil"
NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"
INT8_W8A8 = "int8_w8a8"
+ FP8_BLOCK = "fp8_block"
if TYPE_CHECKING:
@@ -90,6 +93,11 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):
f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake"
)
formats = tmp_format_name.split(",")
+ if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):
+ logger.warning(
+ "`auto_round` format can't be used for deploying block-wise fp8 quantization now, use `fp8` instead."
+ )
+ formats = ["fp8" if "auto_round" in f.lower() else f for f in formats]
return formats
@@ -235,7 +243,10 @@ class OutputFormat(ABC):
w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8
act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8
- if w_fp8 or act_fp8:
+ is_block_dynamic_fp8 = (
+ self.format_name in ["fp8", "auto_round:fp8"] and isinstance(ar.group_size, tuple) and ar.act_dynamic
+ )
+ if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:
error_msg = (
f"is only supported to export auto_round or llm_compressor format,"
f" but got {self.format_name}, please check."
@@ -245,7 +256,7 @@ class OutputFormat(ABC):
logger.error(error_msg)
sys.exit(-1)
- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):
+ if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic) and not is_block_dynamic_fp8:
logger.warning(
f"{self.format_name} format not support for current activation quantization configuration,"
" reset to fake format and save."
@@ -328,7 +339,7 @@ class FakeFormat(OutputFormat):
@OutputFormat.register("llm_compressor")
class LLMCompressorFormat(OutputFormat):
- support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC", "INT8_W8A8"]
+ support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC", "INT8_W8A8", "FP8_BLOCK"]
format_name = "llm_compressor"
def __init__(self, format, ar):
@@ -347,6 +358,8 @@ class LLMCompressorFormat(OutputFormat):
check_compressed_tensors_supported()
self.backend = LLMCompressorFormat(ar.data_type, ar)
+ elif is_dynamic_afp8(ar) and is_block_wfp8(ar):
+ self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_BLOCK.value, ar)
elif is_static_wfp8afp8(ar):
self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)
if ar.act_group_size != 0:
@@ -384,6 +397,21 @@ class LLMCompressorFormat(OutputFormat):
error_logs.append(f"super_bits={scheme.super_bits}")
if scheme.super_group_size:
error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if isinstance(scheme.group_size, tuple):
+ if scheme.bits != 8:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.data_type != "fp":
+ error_logs.append(f"data_type={scheme.data_type}")
+ if len(scheme.group_size) != 2:
+ error_logs.append(f"group_size={scheme.group_size}")
+ if not scheme.act_dynamic:
+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")
+ if not isinstance(scheme.act_group_size, int):
+ error_logs.append(f"act_group_size={scheme.act_group_size}")
+ if scheme.act_bits != 8:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if scheme.act_data_type != "fp":
+ error_logs.append(f"act_data_type={scheme.act_data_type}")
if error_logs:
raise ValueError(
f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "
@@ -399,6 +427,8 @@ class LLMCompressorFormat(OutputFormat):
if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):
if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):
return None
+ elif is_dynamic_afp8(ar) and is_block_wfp8(ar):
+ return None
else:
bits, group_size, sym, act_bits = 8, -1, True, 8
assert (
@@ -430,6 +460,10 @@ class LLMCompressorFormat(OutputFormat):
elif re.search(f"{AutoRoundExportFormat.INT8_W8A8.value}", self.output_format):
from auto_round.export.export_to_llmcompressor.export import pack_layer
+ return pack_layer(layer_name, model, device=device)
+ elif re.search(f"{AutoRoundExportFormat.FP8_BLOCK.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
return pack_layer(layer_name, model, device=device)
## passed as no other llm_compressor format is supported yet
logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")
@@ -943,10 +977,84 @@ class GGUFFormat(OutputFormat):
)
+class FP8Format(OutputFormat):
+ support_schemes = ["FP8_BLOCK"]
+ format_name = "fp8"
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits != 8:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.data_type != "fp":
+ error_logs.append(f"data_type={scheme.data_type}")
+ if not isinstance(scheme.group_size, tuple):
+ error_logs.append(f"group_size={scheme.group_size}")
+ if not scheme.act_dynamic:
+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")
+ if not isinstance(scheme.act_group_size, int):
+ error_logs.append(f"act_group_size={scheme.act_group_size}")
+ if scheme.act_bits != 8:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if scheme.act_data_type != "fp":
+ error_logs.append(f"act_data_type={scheme.act_data_type}")
+ if error_logs:
+ raise ValueError(
+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
+
+ pack_layer(layer_name, model, self.output_format, device=device)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
+
+ backend = self.get_backend_name()
+
+ # weight_block_size & ignored_layers are required by fp8 format, skip them in auto_round:fp8 format
+ if isinstance(serialization_dict["group_size"], tuple) and "auto_round" not in backend:
+ serialization_dict["weight_block_size"] = serialization_dict["group_size"]
+
+ ignored_layers = []
+ for layer_name, cfg in layer_config.items():
+ if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:
+ ignored_layers.append(layer_name)
+ if len(ignored_layers) > 0:
+ serialization_dict["ignored_layers"] = ignored_layers
+
+ return save_quantized_as_autoround(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ serialization_dict=serialization_dict,
+ quant_method=backend,
+ **kwargs,
+ )
+
+
@OutputFormat.register("auto_round")
@OutputFormat.register("auto_round:auto_awq")
@OutputFormat.register("auto_round:llm_compressor")
@OutputFormat.register("auto_round:gptqmodel", "auto_round:auto_gptq")
[email protected]("auto_round:fp8")
class AutoRoundFormat(OutputFormat):
support_schemes = [
"W4A16",
@@ -962,6 +1070,7 @@ class AutoRoundFormat(OutputFormat):
"W2A16G32",
"FP8_STATIC",
"BF16",
+ "FP8_BLOCK",
]
format_name = "auto_round"
@@ -987,6 +1096,8 @@ class AutoRoundFormat(OutputFormat):
self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)
elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8
self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)
+ elif ar.data_type.startswith("fp") and ar.bits == 8 and isinstance(ar.group_size, tuple):
+ self.backend = AutoRoundFormat("auto_round:fp8", ar)
elif ar.act_bits < 16:
raise ValueError(
"AutoRound format does not support exporting "
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 9ac3a4a9d..c48b86ad7 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -244,6 +244,18 @@ FPW8A16 = QuantizationScheme.from_dict(
}
)
+FP8_BLOCK = QuantizationScheme.from_dict(
+ {
+ "bits": 8,
+ "group_size": (128, 128),
+ "data_type": "fp",
+ "act_bits": 8,
+ "act_group_size": 128,
+ "act_data_type": "fp",
+ "act_dynamic": True,
+ "act_sym": True,
+ }
+)
# FP8 = asdict(QuantArgs.from_dict({
# "bits": 8,
@@ -309,6 +321,7 @@ PRESET_SCHEMES = {
"BF16": BF16,
"W4A16_MIXED": W4A16,
"INT8_W8A8": INT8_W8A8,
+ "FP8_BLOCK": FP8_BLOCK,
}
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 126a23432..8a6c414fe 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -237,6 +237,8 @@ class SupportedFormats:
"auto_round:llm_compressor",
"fake",
"llm_compressor",
+ "fp8",
+ "auto_round:fp8",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 89f903827..2478db548 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from math import ceil
+
import torch
import transformers
from torch.functional import F
from auto_round.compressors.utils import is_nv_fp
-from auto_round.data_type import get_quant_func
+from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -37,44 +39,24 @@ def get_scale_shape(weight, group_size):
Args:
weight (torch.Tensor): The weight tensor of the layer.
- group_size (int): The size of the groups for quantization.
+ group_size (int or tuple): The size of the groups for quantization.
Returns:
The shape of the scale tensor to be used for quantization.
"""
+ if isinstance(group_size, tuple):
+ assert len(weight.shape) == len(group_size), f"Expected group_size is {len(weight.shape)}D but get {group_size}"
+ return (weight.shape[0] // group_size[0], weight.shape[1] // group_size[1])
if group_size == 0:
return 1
elif group_size == -1 or weight.shape[1] < group_size:
shape = weight.shape[0]
else:
- shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)
+ shape = weight.shape[0] * ceil(weight.shape[1] / group_size)
return shape
-def reshape_and_pad_tensor(v, group_size=-1):
- """Reshapes the tensor based on the group size.
-
- Args:
- v (torch.Tensor): The input tensor to be reshaped.
- group_size (int, optional): The number of elements to group together.
-
- Returns:
- torch.Tensor: The reshaped tensor. If padding is applied, the padded tensor is returned.
- """
- if group_size == 0:
- return v.reshape(1, -1)
- if group_size == -1 or v.shape[1] < group_size:
- return v
- if v.shape[1] % group_size == 0:
- v = v.reshape(-1, group_size)
- else:
- pad_len = (v.shape[1] + group_size - 1) // group_size * group_size - v.shape[1]
- v = torch.nn.functional.pad(v, (0, pad_len))
- v = v.reshape(-1, group_size)
- return v
-
-
class WrapperLinear(torch.nn.Module):
"""A wrapper for linear/conv1d layers to enable quantization and tuning.
@@ -162,10 +144,19 @@ class WrapperLinear(torch.nn.Module):
orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
orig_weight = orig_weight.t()
- weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)
+
if self.enable_round_tuning:
- self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)
- self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)
+ self.weight_min = (
+ torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)
+ if isinstance(orig_layer.group_size, tuple)
+ else torch.clamp(weight_reshape.min(1)[0], max=0)
+ )
+ self.weight_max = (
+ torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)
+ if isinstance(orig_layer.group_size, tuple)
+ else torch.clamp(weight_reshape.max(1)[0], min=0)
+ )
else:
self.weight_min = None
self.weight_max = None
@@ -178,7 +169,7 @@ class WrapperLinear(torch.nn.Module):
self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
self.weight_quant_func, self.data_type = get_quant_func(
- orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn
+ orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn, orig_layer.group_size
)
if self.enable_torch_compile:
self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
@@ -346,14 +337,17 @@ class WrapperLinear(torch.nn.Module):
name = "w_" + key
setattr(self.orig_layer, name, attr_dict[key].to("cpu"))
- if isinstance(scale, dict):
- _set_dict_attr(scale, "scale")
- elif scale is None:
- self.orig_layer.scale = None
- elif scale.numel() > 1:
- self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
+ if not isinstance(self.orig_layer.group_size, tuple):
+ if isinstance(scale, dict):
+ _set_dict_attr(scale, "scale")
+ elif scale is None:
+ self.orig_layer.scale = None
+ elif scale.numel() > 1:
+ self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
+ else:
+ self.orig_layer.scale = scale.view(-1).to("cpu")
else:
- self.orig_layer.scale = scale.view(-1).to("cpu")
+ self.orig_layer.scale = scale.to("cpu")
if zp is not None:
if isinstance(zp, dict):
@@ -555,9 +549,9 @@ class WrapperLayerNorm(torch.nn.Module):
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
- reshape_and_pad_tensor(
+ reshape_pad_tensor_by_group_size(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
- ),
+ )[0],
requires_grad=True,
)
self.params = {"v": self.v}
@@ -606,9 +600,9 @@ class WrapperLlamaNorm(torch.nn.Module):
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
- reshape_and_pad_tensor(
+ reshape_pad_tensor_by_group_size(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
- ),
+ )[0],
requires_grad=True,
)
self.params = {"v": self.v}
diff --git a/docs/fp8_block_acc.md b/docs/fp8_block_acc.md
new file mode 100644
index 000000000..51564460c
--- /dev/null
+++ b/docs/fp8_block_acc.md
@@ -0,0 +1,34 @@
+Below accuracy results are got by lm-eval.
+
+cmd:
+
+```bash
+# model quantization
+auto-round --model model_name_or_path --scheme FP8_BLOCK --iters 0 --format fp8 # RTN
+auto-round --model model_name_or_path --scheme FP8_BLOCK --format fp8 # Tuning
+
+
+# accuracy evaluation
+# --apply_chat_template --fewshot_as_multiturn are required for gsm8k_llama,mmlu_llama,mmlu_pro_llama tasks
+# `add_bos_token=true` is only required in model_args for llama evaluation
+lm_eval --model vllm --model_args pretrained=model_path,add_bos_token=true,tensor_parallel_size=1,gpu_memory_utilization=0.8,max_model_len=8192 --tasks tasks --batch_size 128
+```
+
+## LLaMa-3-8B-Instruct
+
+|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k_llama|mmlu_llama|mmlu_pro_llama|Avg. |
+|:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:----------|:---------|:-------------|:-----|
+|BF16 |0.5341 |0.8232 |0.8428|0.5805 |0.7258 |0.3540 |0.7758|0.3709 |0.7151 |0.6111 |0.6737 |0.3778 |0.6145|
+|FP8_BLOCK RTN |0.5239 |0.8182 |0.8483|0.5780 |0.7235 |0.3520 |0.7840|0.3647 |0.7040 |0.6171 |0.6681 |0.3738 |0.6130|
+|FP8_BLOCK Tuning|0.5213 |0.8047 |0.8376|0.5834 |0.7099 |0.3480 |0.7764|0.3684 |0.7245 |0.6224 |0.6654 |0.3690 |0.6109|
+
+
+## Qwen3-8B
+
+|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k |mmlu |mmlu_pro|Avg. |
+|:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:-----|:-----|:-------|:-----|
+|BF16 |0.5589 |0.8329 |0.8661|0.5719 |0.6501 |0.3140 |0.7688|0.3647 |0.6772 |0.8726|0.7297|0.6218 |0.6524|
+|FP8_BLOCK RTN |0.5444 |0.8350 |0.8679|0.5704 |0.6466 |0.3060 |0.7644|0.3611 |0.6914 |0.8741|0.7268|0.6242 |0.6510|
+|FP8_BLOCK Tuning|0.5486 |0.8308 |0.8639|0.5691 |0.6513 |0.3080 |0.7622|0.3586 |0.6882 |0.8673|0.7272|0.6139 |0.6491|
+
+
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index dde5006f8..2f67600cb 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -156,11 +156,12 @@ adopted within the community, **only 4-bits quantization is supported**. Please
| Format | Supported Schemes |
|:---|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `BF16` |
+| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16` |
| **auto_awq** | W4A16, BF16 |
| **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 |
-| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC` |
+| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK |
| **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 |
+| **fp8** | FP8_BLOCK |
| **fake** | `all schemes (only for research)` |
### Hardware Compatibility
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 3a4801a25..5aee7303b 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -146,11 +146,12 @@ AutoRound 支持多种量化配置:
| 格式 | 支持的量化方案 |
|:-------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`BF16` |
+| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16` |
| **auto_awq** | W4A16、BF16 |
| **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |
-| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |
+| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |
| **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |
+| **fp8** | FP8_BLOCK |
| **fake** | `所有方案(仅用于研究场景)` |
### 硬件兼容性
diff --git a/test/test_cpu/quantization/test_block_fp.py b/test/test_cpu/quantization/test_block_fp.py
new file mode 100644
index 000000000..30b62e321
--- /dev/null
+++ b/test/test_cpu/quantization/test_block_fp.py
@@ -0,0 +1,152 @@
+import shutil
+from math import ceil
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+from auto_round.data_type.fp8 import quant_block_fp_sym
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+
+from ...helpers import get_model_path
+
+
+class TestAutoRoundBlockFP:
+ @classmethod
+ def setup_class(self):
+ self.model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
+ self.save_dir = "./saved"
+
+ @classmethod
+ def teardown_class(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_invalid_scheme(self):
+ model_name = self.model_name
+
+ with pytest.raises(ValueError):
+ scheme = {
+ "bits": 8,
+ "group_size": (128, 128),
+ "data_type": "int",
+ "act_bits": 16,
+ }
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+
+ with pytest.raises(NotImplementedError):
+ scheme = {
+ "bits": 8,
+ "group_size": (128, 128),
+ "data_type": "fp",
+ "act_bits": 8,
+ "act_data_type": "fp",
+ "act_group_size": 128,
+ "act_dynamic": False,
+ }
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+
+ with pytest.raises(ValueError):
+ scheme = {
+ "bits": 8,
+ "group_size": (128, 128),
+ "data_type": "fp",
+ "act_bits": 8,
+ "act_data_type": "fp",
+ "act_group_size": (128, 128),
+ "act_dynamic": True,
+ }
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+
+ def test_block_fp8_quant(self):
+ data = torch.randn(256, 240)
+ group_size = (128, 128)
+ reshaped_data, orig_shape, pad_len = reshape_pad_tensor_by_group_size(data, group_size)
+ assert list(reshaped_data.shape) == [2, 2, 128, 128]
+ assert list(orig_shape) == [256, 240]
+ assert pad_len == (0, 16)
+
+ qdq_data, scale, _ = quant_block_fp_sym(data)
+ M = ceil(data.shape[0] / 128)
+ N = ceil(data.shape[1] / 128)
+ scale_ref = torch.zeros(M, N)
+
+ max_val = torch.finfo(torch.float8_e4m3fn).max
+ for i in range(M):
+ for j in range(N):
+ scale_ref[i, j] = data[i * 128 : (i + 1) * 128, j * 128 : (j + 1) * 128].abs().max() / max_val
+ assert (scale == scale_ref).all()
+
+ def test_fp8_block_autoround_format(self):
+ model_name = self.model_name
+
+ scheme = "FP8_BLOCK"
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale_inv")
+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn
+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]
+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"
+ assert compressed_model.config.quantization_config["group_size"] == (128, 128)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_fp8_block_fp8_format(self):
+ model_name = self.model_name
+
+ scheme = "FP8_BLOCK"
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")
+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale_inv")
+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn
+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]
+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"
+ assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_fp8_block_llm_compressor_format(self):
+ model_name = self.model_name
+
+ scheme = "FP8_BLOCK"
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale")
+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn
+ assert list(tmp_layer.weight_scale.shape) == [12, 12]
+ assert compressed_model.config.quantization_config["quant_method"] == "compressed-tensors"
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
diff --git a/test/test_cuda/quantization/test_block_fp.py b/test/test_cuda/quantization/test_block_fp.py
new file mode 100644
index 000000000..6ee0ababb
--- /dev/null
+++ b/test/test_cuda/quantization/test_block_fp.py
@@ -0,0 +1,64 @@
+import shutil
+from math import ceil
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+from auto_round.data_type.fp8 import quant_block_fp_sym
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+
+from ...helpers import evaluate_accuracy, get_model_path
+
+
+class TestAutoRoundBlockFP:
+ @classmethod
+ def setup_class(self):
+ self.model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
+ self.save_dir = "./saved"
+
+ @classmethod
+ def teardown_class(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_fp8_block_fp8_format(self):
+ model_name = self.model_name
+
+ scheme = "FP8_BLOCK"
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")
+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale_inv")
+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn
+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]
+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"
+ assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
+ evaluate_accuracy(quantized_model_path, threshold=0.55, batch_size=32, limit=100)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_fp8_block_llm_compressor_format(self):
+ model_name = self.model_name
+
+ scheme = "FP8_BLOCK"
+ autoround = AutoRound(
+ model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale")
+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn
+ assert list(tmp_layer.weight_scale.shape) == [12, 12]
+ assert compressed_model.config.quantization_config["quant_method"] == "compressed-tensors"
+ evaluate_accuracy(quantized_model_path, threshold=0.55, batch_size=32, limit=100)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)