PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support block-wise fp8 quant (#1487)
Wang, Mengnie6600591706f
diff --git a/README.md b/README.mdindex d38161bfc..348d1dcd7 100644--- a/README.md+++ b/README.md@@ -36,6 +36,7 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]## 🆕 What's New+* [2026/03] **Block-wise FP8** quantization is available via `--scheme FP8_BLOCK --iters 0 --disable_opt_rtn`.* [2026/03] MTP layer quantization has been supported in [PR](https://github.com/intel/auto-round/pull/1526)diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 2254aafa8..5721a9501 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -288,7 +288,12 @@ class BasicArgumentParser(argparse.ArgumentParser):scheme = self.add_argument_group("Scheme Arguments")scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")- scheme.add_argument("--group_size", default=None, type=int, help="Group size for weight quantization.")+ scheme.add_argument(+ "--group_size",+ default=None,+ type=lambda s: int(s) if s.isdigit() else tuple([int(x.strip()) for x in s.split(",")]),+ help="Group size for weight quantization.",+ )scheme.add_argument("--asym", action="store_true", help="Use asymmetric quantization instead of symmetric.")scheme.add_argument("--data_type",@@ -724,13 +729,17 @@ def tune(args):elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")else:- if autoround.group_size <= 0:- if "fp" in autoround.act_data_type:- suffix = f"afp{autoround.act_bits}"- else:- suffix = f"a{autoround.act_bits}"+ if isinstance(autoround.group_size, tuple):+ assert len(autoround.group_size) == 2, f"Only support 2D group_size, but get {autoround.group_size}"+ suffix = f"g{autoround.group_size[0]}x{autoround.group_size[1]}"else:- suffix = f"g{autoround.group_size}"+ if autoround.group_size <= 0:+ if "fp" in autoround.act_data_type:+ suffix = f"afp{autoround.act_bits}"+ else:+ suffix = f"a{autoround.act_bits}"+ else:+ suffix = f"g{autoround.group_size}"export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")# ======================= Quantize and save model =======================diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.pyindex dd188896a..8a45b443e 100644--- a/auto_round/alg_ext.py+++ b/auto_round/alg_ext.py@@ -31,7 +31,7 @@ from auto_round.data_type.nvfp import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, ref_nvfpfrom auto_round.data_type.utils import floor_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.logger import loggerfrom auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal, set_module-from auto_round.wrapper import NORM_MAPPING, WrapperLinear, reshape_and_pad_tensor+from auto_round.wrapper import NORM_MAPPING, WrapperLinear__all__ = ["wrapper_autoround"]@@ -335,7 +335,7 @@ class WrapperLinearV2(WrapperLinear):super()._init_tuning_params_and_quant_func()orig_weight = getattr(self.orig_layer, "get_weight", lambda: self.orig_layer.weight)()- weight_reshape = reshape_and_pad_tensor(orig_weight.data, self.orig_layer.group_size)+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, self.orig_layer.group_size)if hasattr(self.orig_layer, "imatrix"): # MOE model may have no imatriximatrix = self.orig_layer.imatrix.reshape(1, -1)imatrix = reshape_pad_tensor_by_group_size(imatrix, self.orig_layer.group_size, val=1e-5)[0].view(1, -1)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4c3abe4ba..8b161be73 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -45,7 +45,7 @@ class AutoRound:tokenizer: Tokenizer used to prepare input text for calibration/tuning.platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]bits (int): Weight quantization bits.- group_size (int): Per-group size for weight quantization.+ group_size (int or tuple): Per-group size for weight quantization.sym (bool): Whether to use symmetric weight quantization.layer_config (dict): Per-layer quantization configuration.nsamples (int): Number of calibration samples.@@ -55,7 +55,7 @@ class AutoRound:SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "extra_config", "enable_adam")bits: int | None- group_size: int | None+ group_size: int | tuple | Nonesym: bool | Nonedata_type: str | Noneact_bits: int | None@@ -118,7 +118,7 @@ class AutoRound:low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.bits (int, optional): Weight quantization bits. Defaults to 4.- group_size (int, optional): Weight quantization group size. Defaults to 128.+ group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.sym (bool, optional): Symmetric weight quantization. Defaults to True.data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".act_bits (int, optional): Activation quantization bits. Defaults to 16.@@ -253,7 +253,7 @@ class AutoRoundLLM(LLMCompressor):tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int, optional): Weight quantization bits. Defaults to 4.- group_size (int, optional): Weight quantization group size. Defaults to 128.+ group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.sym (bool, optional): Symmetric weight quantization. Defaults to True.layer_config (dict, optional): Layer-wise quantization config. Defaults to None.batch_size (int, optional): Calibration batch size. Defaults to 8.@@ -307,7 +307,7 @@ class AutoRoundLLM(LLMCompressor):"""bits: int | None- group_size: int | None+ group_size: int | tuple | Nonesym: bool | Nonedata_type: str | Noneact_bits: int | None@@ -354,7 +354,7 @@ class AutoRoundAdam(AdamCompressor):platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int): Number of bits for quantization (default is 4).- group_size (int): Size of the quantization group (default is 128).+ group_size (int or tuple): Size of the quantization group (default is 128).sym (bool): Whether sym to be used (default is True).layer_config (dict): Configuration for weight quantization (default is None).batch_size (int): Batch size for training (default is 8).@@ -396,7 +396,7 @@ class AutoRoundAdam(AdamCompressor):"""bits: int | None- group_size: int | None+ group_size: int | tuple | Nonesym: bool | Nonedata_type: str | Noneact_bits: int | None@@ -446,7 +446,7 @@ class AutoRoundMLLM(MLLMCompressor):decode the data that groups several modalities (among text, vision and audio).image_processor: Image processor for special model like llava.bits (int): Number of bits for quantization (default is 4).- group_size (int): Size of the quantization group (default is 128).+ group_size (int or tuple): Size of the quantization group (default is 128).sym (bool): Whether sym to be used (default is True).layer_config (dict): Configuration for weight quantization (default is None).batch_size (int): Batch size for training (default is 8).@@ -485,7 +485,7 @@ class AutoRoundMLLM(MLLMCompressor):"""bits: int | None- group_size: int | None+ group_size: int | tuple | Nonesym: bool | Nonedata_type: str | Noneact_bits: int | None@@ -552,7 +552,7 @@ class AutoRoundDiffusion(DiffusionCompressor):"""bits: int | None- group_size: int | None+ group_size: int | tuple | Nonesym: bool | Nonedata_type: str | Noneact_bits: int | Nonediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 9d7184e40..663186f3f 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -44,6 +44,8 @@ from auto_round.compressors.utils import (get_shared_keys,infer_bits_by_data_type,init_cache,+ is_block_wfp8,+ is_dynamic_afp8,is_dynamic_wint8aint8,is_mx_fp,is_nv_fp,@@ -457,7 +459,7 @@ class BaseCompressor(object):self.bits >= 8and self.act_bits >= 8and self.iters == 0- and self.data_type == "int"+ and self.data_type in ["int", "fp"]and disable_opt_rtn is None):logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")@@ -468,6 +470,9 @@ class BaseCompressor(object):)disable_opt_rtn = False+ if self.iters > 0 and is_block_wfp8(self):+ logger.warning("RTN is recommended since it shows even better accuracy for block-wise fp8 quantization.")+# Important Note! This is not very robust, do NOT rely on it to do high risky thingself.is_moe_model = is_moe_model(self.model)@@ -840,10 +845,14 @@ class BaseCompressor(object):raise ValueError("`bits` must be positive")if self.act_bits <= 0:raise ValueError("`act_bits` must be positive")- if not (self.group_size == -1 or self.group_size >= 0):- raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")- if not (self.act_group_size == -1 or self.act_group_size >= 0):+ if not isinstance(self.group_size, tuple) and not (self.group_size == -1 or self.group_size >= 0):+ raise ValueError(+ "`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer or a tuple of length 2"+ )+ if isinstance(self.act_group_size, tuple) or not (self.act_group_size == -1 or self.act_group_size >= 0):raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")+ if isinstance(self.group_size, tuple) and len(self.group_size) != 2:+ raise ValueError("`group_size` must be a tuple of length 2")if self.batch_size <= 0:raise ValueError("`batch_size` must be positive")if self.iters < 0:@@ -861,6 +870,7 @@ class BaseCompressor(object):and not is_mx_fp(self.act_data_type)and not is_dynamic_wint8aint8(self)and not is_static_wfp8afp8(self.act_data_type)+ and not is_dynamic_afp8(self)):logger.warning("activation quantization is an experimental feature with limited support and a complex API. "@@ -873,6 +883,12 @@ class BaseCompressor(object):if is_nv_fp(self.data_type) and (self.group_size != 16):logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")+ if isinstance(self.group_size, tuple):+ if not is_block_wfp8(self):+ raise NotImplementedError("only support block-wise quantization for fp8 weight quantization.")+ if not is_dynamic_afp8(self):+ raise NotImplementedError("only support dynamic fp8 activation for fp8 weight quantization.")+if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:if self.iters > 0: # GGUF should log this warning, but we don't know the format herediff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 1aef5bd7c..bda959e37 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -99,6 +99,19 @@ def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:return False+def is_dynamic_afp8(ar_or_format: Callable) -> bool:+ return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8+++def is_block_wfp8(ar_or_format: Callable) -> bool:+ return (+ isinstance(ar_or_format.group_size, tuple)+ and len(ar_or_format.group_size) == 2+ and ar_or_format.data_type.startswith("fp")+ and ar_or_format.bits == 8+ )++def block_forward(block: torch.nn.Module,input_ids: torch.Tensor,diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex ed446f314..58c746470 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -17,6 +17,10 @@ import auto_round.data_type.mxfpimport auto_round.data_type.fp8from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEimport auto_round.data_type.w4fp8-from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scales+from auto_round.data_type.utils import (+ get_quant_func,+ reshape_pad_tensor_by_group_size,+ update_fused_layer_global_scales,+)import auto_round.data_type.nvfpimport auto_round.data_type.ggufdiff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex fe108c3af..ea7a6e99e 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -25,6 +25,51 @@ from auto_round.data_type.utils import (from auto_round.utils import is_gaudi2, logger+@register_dtype(("block_fp8_sym", "block_fp8", "block_fp8_e4m3"))+def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, **kwargs):+ """Symmetric quantization using block float8 format.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.to(tensor.device)+ if isinstance(v, torch.Tensor):+ v = v.to(tensor.device)+ if tensor_max is None:+ max_tensor = tensor.abs().amax(dim=(-2, -1)) * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ scale = max_tensor / info.max+ assert len(scale.shape) == 2, f"Only support 2D group_size, but get {len(scale.shape)}"+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = torch.clip(scale, min=min_scaling_factor)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)++ fp8_res = tensor / scale.unsqueeze(-1).unsqueeze(-1) + v+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ fp8_res = float8_e4m3fn_ste(fp8_res)+ qdq_res = fp8_res * scale.unsqueeze(-1).unsqueeze(-1)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 465ea0b87..a8e3eb218 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -14,7 +14,8 @@import mathfrom functools import lru_cache-from typing import List+from math import ceil+from typing import List, Unionimport torchfrom torch.nn import Linear, Module@@ -24,7 +25,7 @@ from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import check_to_quantized, logger-def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: float = 0.0):+def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.This function adjusts the@@ -35,7 +36,7 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: fArgs:data (torch.Tensor): The input tensor to be reshaped and padded.- group_size (int): The size of the groups that the tensor should be reshaped into.+ group_size (int or tuple): The size of the groups that the tensor should be reshaped into.Returns:torch.Tensor: The reshaped and padded tensor, if necessary.@@ -44,6 +45,14 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: f"""orig_shape = data.shapepad_len = 0+ if isinstance(group_size, tuple):+ assert len(group_size) == 2, f"Only support 2D group_size, but get {len(group_size)}"+ M, N = group_size+ pad_len_m = ceil(orig_shape[0] / M) * M - orig_shape[0]+ pad_len_n = ceil(orig_shape[1] / N) * N - orig_shape[1]+ data_new = torch.nn.functional.pad(data, (0, pad_len_n, 0, pad_len_m))+ data_new = data_new.view(data_new.shape[0] // M, M, data_new.shape[1] // N, N).permute(0, 2, 1, 3)+ return data_new, orig_shape, (pad_len_m, pad_len_n)if group_size == 0:data = data.reshape(1, -1)return data, orig_shape, pad_len@@ -55,13 +64,13 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: fdata = data.reshape(-1, group_size)return data, orig_shape, pad_lenelse:- pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]+ pad_len = ceil(data.shape[1] / group_size) * group_size - data.shape[1]data_new = torch.nn.functional.pad(data, (0, pad_len), value=val)data_new = data_new.reshape(-1, group_size)return data_new, orig_shape, pad_len-def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):+def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: Union[int, list]):"""Reverts the tensor to its original shape by removing padding.This function removes the padding added during reshaping and returns the tensor to@@ -70,11 +79,15 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):Args:data (torch.Tensor): The reshaped and possibly padded tensor.orig_shape (tuple): The original shape of the tensor before reshaping.- pad_len (int): The length of the padding to be removed.+ pad_len (int or tuple): The length of the padding to be removed.Returns:torch.Tensor: The tensor restored to its original shape."""+ if isinstance(pad_len, tuple):+ assert len(pad_len) == 2, f"Only support 2D group_size, but get {len(pad_len)}"+ data = data.permute(0, 2, 1, 3).reshape(orig_shape[0] + pad_len[0], orig_shape[1] + pad_len[1])+ return data[: data.shape[0] - pad_len[0], : data.shape[1] - pad_len[1]]if pad_len == 0:return data.reshape(orig_shape)else:@@ -88,7 +101,7 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):return data_new-def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> tuple[callable, str]:+def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None) -> tuple[callable, str]:"""Retrieve the quantization function based on data type, bit width, and symmetry.This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE@@ -100,6 +113,7 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> tbits (int): The bit width for the quantization (e.g., 2,4,8).sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).disable_opt_rtn(bool): whether to disable optimized rtn.+ group_size (tuple): The block size for weight quantization (e.g., (128, 128)).Returns:function: The quantization function corresponding to the specified parameters.@@ -125,6 +139,21 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False) -> tif data_type in QUANT_FUNC_WITH_DTYPE:return QUANT_FUNC_WITH_DTYPE[data_type], data_type+ if group_size is not None and isinstance(group_size, tuple):+ block_data_type = "block_" + dtype+ data_types = [+ block_data_type,+ pad_bits(block_data_type),+ pad_sym(block_data_type),+ pad_sym(pad_bits(block_data_type)),+ ]++ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE++ for data_type in data_types:+ if data_type in QUANT_FUNC_WITH_DTYPE:+ return QUANT_FUNC_WITH_DTYPE[data_type], data_type+data_types = [dtype, pad_bits(dtype), pad_sym(dtype), pad_sym(pad_bits(dtype))]for data_type in data_types:from auto_round.data_type import QUANT_FUNC_WITH_DTYPEdiff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 70c7c71b1..fadebba5d 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -73,6 +73,32 @@ class FP8QLinear(torch.nn.Module):self.register_buffer("input_scale", input_scale.to(dtype))+class FP8BlockQLinear(torch.nn.Module):++ def __init__(+ self,+ in_features,+ out_features,+ weight,+ weight_scale,+ bias=None,+ weight_zp=None,+ input_scale=None,+ dtype=torch.bfloat16,+ ):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ self.weight = torch.nn.Parameter(weight, requires_grad=False)++ if bias is not None:+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)++ self.register_buffer("weight_scale_inv", weight_scale.to(dtype))++def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):"""Packs a model layer for quantization based on its type and configuration.@@ -102,7 +128,7 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):returnorig_device = layer.weight.device- scale = layer.scale.view(-1)+ scale = layer.scale if isinstance(layer.group_size, tuple) else layer.scale.view(-1)zp = layer.zpweight = layer.weightweight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)@@ -114,9 +140,15 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):if zp is not None:if isinstance(zp, torch.Tensor):zp = zp.to(packing_device)- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp+ if isinstance(layer.group_size, tuple):+ q_weight = weight.to(packing_device) / scale.unsqueeze(-1).unsqueeze(-1).to(packing_device) + zp+ else:+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zpelse:- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)+ if isinstance(layer.group_size, tuple):+ q_weight = weight.to(packing_device) / scale.unsqueeze(-1).unsqueeze(-1).to(packing_device)+ else:+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)q_weight = torch.clamp(q_weight, info.min, info.max)q_weight = q_weight.to(torch_dtype)@@ -130,7 +162,8 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):in_features = layer.weight.shape[0]out_features = layer.weight.shape[1]bias = layer.bias- my_linear = FP8QLinear(+ linear_cls = FP8BlockQLinear if isinstance(layer.group_size, tuple) else FP8QLinear+ my_linear = linear_cls(in_features,out_features,weight=q_weight,@@ -140,7 +173,12 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):input_scale=act_scale,dtype=model.dtype,)- if unsqueeze and len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:+ if (+ unsqueeze+ and isinstance(linear_cls, FP8QLinear)+ and len(my_linear.weight_scale.shape)+ and my_linear.weight_scale.shape[0] != 1+ ):my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)my_linear.to(orig_device)@@ -158,6 +196,7 @@ def save_quantized_as_autoround(backend: str = None,device: Union[str, torch.device] = "cpu",serialization_dict: dict = None,+ quant_method: str = "auto-round",**kwargs,):safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]@@ -165,7 +204,7 @@ def save_quantized_as_autoround(model = copy.deepcopy(model.to("cpu"))quantization_config = serialization_dictquantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)- quantization_config["quant_method"] = "auto-round"+ quantization_config["quant_method"] = quant_methodif backend:quantization_config["packing_format"] = backendif "e5m2" in serialization_dict.get("data_type", "fp8"):diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 8df22b93d..e5fe46abf 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -32,24 +32,55 @@ from auto_round.utils import (from auto_round.wrapper import WrapperWALayer+def _get_weight_scheme_strategy(group_size):+ if group_size == 0:+ return "tensor"+ if group_size == -1:+ return "channel"+ if isinstance(group_size, tuple):+ return "block"+ if isinstance(group_size, int):+ return "group"+ return None+++def _get_act_scheme_strategy(group_size):+ if group_size == 0:+ return "tensor"+ if group_size == -1:+ return "token"+ if isinstance(group_size, int):+ return "group"+ return None+++def _get_scheme_type(data_type):+ if "int" in data_type:+ return "int"+ if "fp" in data_type or "float" in data_type:+ return "float"+ raise NotImplementedError("only support `int` and `fp` data type")++def construct_ct_scheme(layer):from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme # pylint: disable=E0401weights_args = QuantizationArgs(num_bits=layer.bits,- type=layer.data_type.split("_")[-2], # int_sym, rtn_int_sym+ type=_get_scheme_type(layer.data_type),symmetric=layer.sym,dynamic=False,- group_size=None,- strategy="tensor" if layer.act_group_size == 0 else "channel" if layer.act_group_size == -1 else None,+ group_size=layer.group_size if _get_weight_scheme_strategy(layer.group_size) == "group" else None,+ strategy=_get_weight_scheme_strategy(layer.group_size),+ block_structure=layer.group_size if _get_weight_scheme_strategy(layer.group_size) == "block" else None,)activations_args = QuantizationArgs(num_bits=layer.act_bits,- type=layer.act_data_type.split("_")[-2], # int_sym, rtn_int_sym+ type=_get_scheme_type(layer.act_data_type),symmetric=layer.act_sym,dynamic=layer.act_dynamic,- group_size=None,- strategy="tensor" if layer.act_group_size == 0 else "token" if layer.act_group_size == -1 else None,+ group_size=layer.act_group_size if _get_act_scheme_strategy(layer.act_group_size) == "group" else None,+ strategy=_get_act_scheme_strategy(layer.act_group_size),)scheme = QuantizationScheme(targets=[layer.__class__.__name__],@@ -59,8 +90,16 @@ def construct_ct_scheme(layer):return scheme+def _get_quant_format(model):+ for n, m in model.named_modules():+ if hasattr(m, "quantization_scheme") and hasattr(m.quantization_scheme, "format"):+ return m.quantization_scheme.format+ return None++def pack_layer(name, model, device=None):- from compressed_tensors.compressors import IntQuantizationCompressor # pylint: disable=E0401+ from compressed_tensors.compressors import NaiveQuantizationCompressor # pylint: disable=E0401+ from compressed_tensors.config.format import set_per_module_format # pylint: disable=E0401from compressed_tensors.quantization import QuantizationStatus # pylint: disable=E0401from compressed_tensors.utils import delete_offload_parameter, register_offload_parameter # pylint: disable=E0401@@ -96,7 +135,7 @@ def pack_layer(name, model, device=None):setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))delattr(layer, "scale")- compressor = IntQuantizationCompressor()+ compressor = NaiveQuantizationCompressor()q_state_dict = compressor.compress(layer.state_dict(), names_to_scheme={"": scheme}, show_progress=False)# remove any existing parameters@@ -109,6 +148,7 @@ def pack_layer(name, model, device=None):register_offload_parameter(layer, name, param, device)layer.quantization_status = QuantizationStatus.COMPRESSED+ set_per_module_format(layer)@torch.no_grad()@@ -166,12 +206,14 @@ def save_quantized_as_llmcompressor(for n, m in model.named_modules():pack_layer(n, model, device)+ quant_format = _get_quant_format(model)+ quantization_config = QuantizationConfig.from_pretrained(model, format=quant_format)+ model.config.quantization_config = quantization_config.to_dict()+if output_dir is None:return model- quantization_config = QuantizationConfig.from_pretrained(model)# save model.config, model.state_dict()- model.config.quantization_config = quantization_config.to_dict()model.config.save_pretrained(output_dir)save_model(model, output_dir, safe_serialization=safe_serialization)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex e6fc7f56f..d6f40362a 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -27,6 +27,8 @@ import torchimport transformersfrom auto_round.compressors.utils import (+ is_block_wfp8,+ is_dynamic_afp8,is_dynamic_wint8aint8,is_mx_fp,is_nv_fp,@@ -65,6 +67,7 @@ class AutoRoundExportFormat(str, Enum):MX_FP_RCEIL = "mx_fp_rceil"NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"INT8_W8A8 = "int8_w8a8"+ FP8_BLOCK = "fp8_block"if TYPE_CHECKING:@@ -90,6 +93,11 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake")formats = tmp_format_name.split(",")+ if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):+ logger.warning(+ "`auto_round` format can't be used for deploying block-wise fp8 quantization now, use `fp8` instead."+ )+ formats = ["fp8" if "auto_round" in f.lower() else f for f in formats]return formats@@ -235,7 +243,10 @@ class OutputFormat(ABC):w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8- if w_fp8 or act_fp8:+ is_block_dynamic_fp8 = (+ self.format_name in ["fp8", "auto_round:fp8"] and isinstance(ar.group_size, tuple) and ar.act_dynamic+ )+ if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:error_msg = (f"is only supported to export auto_round or llm_compressor format,"f" but got {self.format_name}, please check."@@ -245,7 +256,7 @@ class OutputFormat(ABC):logger.error(error_msg)sys.exit(-1)- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):+ if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic) and not is_block_dynamic_fp8:logger.warning(f"{self.format_name} format not support for current activation quantization configuration,"" reset to fake format and save."@@ -328,7 +339,7 @@ class FakeFormat(OutputFormat):@OutputFormat.register("llm_compressor")class LLMCompressorFormat(OutputFormat):- support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC", "INT8_W8A8"]+ support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC", "INT8_W8A8", "FP8_BLOCK"]format_name = "llm_compressor"def __init__(self, format, ar):@@ -347,6 +358,8 @@ class LLMCompressorFormat(OutputFormat):check_compressed_tensors_supported()self.backend = LLMCompressorFormat(ar.data_type, ar)+ elif is_dynamic_afp8(ar) and is_block_wfp8(ar):+ self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_BLOCK.value, ar)elif is_static_wfp8afp8(ar):self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)if ar.act_group_size != 0:@@ -384,6 +397,21 @@ class LLMCompressorFormat(OutputFormat):error_logs.append(f"super_bits={scheme.super_bits}")if scheme.super_group_size:error_logs.append(f"super_group_size={scheme.super_group_size}")+ if isinstance(scheme.group_size, tuple):+ if scheme.bits != 8:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.data_type != "fp":+ error_logs.append(f"data_type={scheme.data_type}")+ if len(scheme.group_size) != 2:+ error_logs.append(f"group_size={scheme.group_size}")+ if not scheme.act_dynamic:+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")+ if not isinstance(scheme.act_group_size, int):+ error_logs.append(f"act_group_size={scheme.act_group_size}")+ if scheme.act_bits != 8:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if scheme.act_data_type != "fp":+ error_logs.append(f"act_data_type={scheme.act_data_type}")if error_logs:raise ValueError(f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "@@ -399,6 +427,8 @@ class LLMCompressorFormat(OutputFormat):if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):return None+ elif is_dynamic_afp8(ar) and is_block_wfp8(ar):+ return Noneelse:bits, group_size, sym, act_bits = 8, -1, True, 8assert (@@ -430,6 +460,10 @@ class LLMCompressorFormat(OutputFormat):elif re.search(f"{AutoRoundExportFormat.INT8_W8A8.value}", self.output_format):from auto_round.export.export_to_llmcompressor.export import pack_layer+ return pack_layer(layer_name, model, device=device)+ elif re.search(f"{AutoRoundExportFormat.FP8_BLOCK.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export import pack_layer+return pack_layer(layer_name, model, device=device)## passed as no other llm_compressor format is supported yetlogger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")@@ -943,10 +977,84 @@ class GGUFFormat(OutputFormat):)[email protected]("fp8")+class FP8Format(OutputFormat):+ support_schemes = ["FP8_BLOCK"]+ format_name = "fp8"++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits != 8:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.data_type != "fp":+ error_logs.append(f"data_type={scheme.data_type}")+ if not isinstance(scheme.group_size, tuple):+ error_logs.append(f"group_size={scheme.group_size}")+ if not scheme.act_dynamic:+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")+ if not isinstance(scheme.act_group_size, int):+ error_logs.append(f"act_group_size={scheme.act_group_size}")+ if scheme.act_bits != 8:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if scheme.act_data_type != "fp":+ error_logs.append(f"act_data_type={scheme.act_data_type}")+ if error_logs:+ raise ValueError(+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer++ pack_layer(layer_name, model, self.output_format, device=device)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround++ backend = self.get_backend_name()++ # weight_block_size & ignored_layers are required by fp8 format, skip them in auto_round:fp8 format+ if isinstance(serialization_dict["group_size"], tuple) and "auto_round" not in backend:+ serialization_dict["weight_block_size"] = serialization_dict["group_size"]++ ignored_layers = []+ for layer_name, cfg in layer_config.items():+ if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:+ ignored_layers.append(layer_name)+ if len(ignored_layers) > 0:+ serialization_dict["ignored_layers"] = ignored_layers++ return save_quantized_as_autoround(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ serialization_dict=serialization_dict,+ quant_method=backend,+ **kwargs,+ )++@OutputFormat.register("auto_round")@OutputFormat.register("auto_round:auto_awq")@OutputFormat.register("auto_round:llm_compressor")@OutputFormat.register("auto_round:gptqmodel", "auto_round:auto_gptq")[email protected]("auto_round:fp8")class AutoRoundFormat(OutputFormat):support_schemes = ["W4A16",@@ -962,6 +1070,7 @@ class AutoRoundFormat(OutputFormat):"W2A16G32","FP8_STATIC","BF16",+ "FP8_BLOCK",]format_name = "auto_round"@@ -987,6 +1096,8 @@ class AutoRoundFormat(OutputFormat):self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)+ elif ar.data_type.startswith("fp") and ar.bits == 8 and isinstance(ar.group_size, tuple):+ self.backend = AutoRoundFormat("auto_round:fp8", ar)elif ar.act_bits < 16:raise ValueError("AutoRound format does not support exporting "diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 9ac3a4a9d..c48b86ad7 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -244,6 +244,18 @@ FPW8A16 = QuantizationScheme.from_dict(})+FP8_BLOCK = QuantizationScheme.from_dict(+ {+ "bits": 8,+ "group_size": (128, 128),+ "data_type": "fp",+ "act_bits": 8,+ "act_group_size": 128,+ "act_data_type": "fp",+ "act_dynamic": True,+ "act_sym": True,+ }+)# FP8 = asdict(QuantArgs.from_dict({# "bits": 8,@@ -309,6 +321,7 @@ PRESET_SCHEMES = {"BF16": BF16,"W4A16_MIXED": W4A16,"INT8_W8A8": INT8_W8A8,+ "FP8_BLOCK": FP8_BLOCK,}from auto_round.export.export_to_gguf.config import GGUF_CONFIGdiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 126a23432..8a6c414fe 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -237,6 +237,8 @@ class SupportedFormats:"auto_round:llm_compressor","fake","llm_compressor",+ "fp8",+ "auto_round:fp8",)self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))self._support_list = self._support_format + self._gguf_formatdiff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 89f903827..2478db548 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -12,12 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.+from math import ceil+import torchimport transformersfrom torch.functional import Ffrom auto_round.compressors.utils import is_nv_fp-from auto_round.data_type import get_quant_func+from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_sizefrom auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -37,44 +39,24 @@ def get_scale_shape(weight, group_size):Args:weight (torch.Tensor): The weight tensor of the layer.- group_size (int): The size of the groups for quantization.+ group_size (int or tuple): The size of the groups for quantization.Returns:The shape of the scale tensor to be used for quantization."""+ if isinstance(group_size, tuple):+ assert len(weight.shape) == len(group_size), f"Expected group_size is {len(weight.shape)}D but get {group_size}"+ return (weight.shape[0] // group_size[0], weight.shape[1] // group_size[1])if group_size == 0:return 1elif group_size == -1 or weight.shape[1] < group_size:shape = weight.shape[0]else:- shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)+ shape = weight.shape[0] * ceil(weight.shape[1] / group_size)return shape-def reshape_and_pad_tensor(v, group_size=-1):- """Reshapes the tensor based on the group size.-- Args:- v (torch.Tensor): The input tensor to be reshaped.- group_size (int, optional): The number of elements to group together.-- Returns:- torch.Tensor: The reshaped tensor. If padding is applied, the padded tensor is returned.- """- if group_size == 0:- return v.reshape(1, -1)- if group_size == -1 or v.shape[1] < group_size:- return v- if v.shape[1] % group_size == 0:- v = v.reshape(-1, group_size)- else:- pad_len = (v.shape[1] + group_size - 1) // group_size * group_size - v.shape[1]- v = torch.nn.functional.pad(v, (0, pad_len))- v = v.reshape(-1, group_size)- return v--class WrapperLinear(torch.nn.Module):"""A wrapper for linear/conv1d layers to enable quantization and tuning.@@ -162,10 +144,19 @@ class WrapperLinear(torch.nn.Module):orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:orig_weight = orig_weight.t()- weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)+if self.enable_round_tuning:- self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)- self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)+ self.weight_min = (+ torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)+ if isinstance(orig_layer.group_size, tuple)+ else torch.clamp(weight_reshape.min(1)[0], max=0)+ )+ self.weight_max = (+ torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)+ if isinstance(orig_layer.group_size, tuple)+ else torch.clamp(weight_reshape.max(1)[0], min=0)+ )else:self.weight_min = Noneself.weight_max = None@@ -178,7 +169,7 @@ class WrapperLinear(torch.nn.Module):self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))self.weight_quant_func, self.data_type = get_quant_func(- orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn+ orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn, orig_layer.group_size)if self.enable_torch_compile:self.weight_quant_func = compile_func(self.weight_quant_func, self.device)@@ -346,14 +337,17 @@ class WrapperLinear(torch.nn.Module):name = "w_" + keysetattr(self.orig_layer, name, attr_dict[key].to("cpu"))- if isinstance(scale, dict):- _set_dict_attr(scale, "scale")- elif scale is None:- self.orig_layer.scale = None- elif scale.numel() > 1:- self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")+ if not isinstance(self.orig_layer.group_size, tuple):+ if isinstance(scale, dict):+ _set_dict_attr(scale, "scale")+ elif scale is None:+ self.orig_layer.scale = None+ elif scale.numel() > 1:+ self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")+ else:+ self.orig_layer.scale = scale.view(-1).to("cpu")else:- self.orig_layer.scale = scale.view(-1).to("cpu")+ self.orig_layer.scale = scale.to("cpu")if zp is not None:if isinstance(zp, dict):@@ -555,9 +549,9 @@ class WrapperLayerNorm(torch.nn.Module):weight_dtype = torch.float32self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(- reshape_and_pad_tensor(+ reshape_pad_tensor_by_group_size(torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size- ),+ )[0],requires_grad=True,)self.params = {"v": self.v}@@ -606,9 +600,9 @@ class WrapperLlamaNorm(torch.nn.Module):weight_dtype = torch.float32self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(- reshape_and_pad_tensor(+ reshape_pad_tensor_by_group_size(torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size- ),+ )[0],requires_grad=True,)self.params = {"v": self.v}diff --git a/docs/fp8_block_acc.md b/docs/fp8_block_acc.mdnew file mode 100644index 000000000..51564460c--- /dev/null+++ b/docs/fp8_block_acc.md@@ -0,0 +1,34 @@+Below accuracy results are got by lm-eval.++cmd:++```bash+# model quantization+auto-round --model model_name_or_path --scheme FP8_BLOCK --iters 0 --format fp8 # RTN+auto-round --model model_name_or_path --scheme FP8_BLOCK --format fp8 # Tuning+++# accuracy evaluation+# --apply_chat_template --fewshot_as_multiturn are required for gsm8k_llama,mmlu_llama,mmlu_pro_llama tasks+# `add_bos_token=true` is only required in model_args for llama evaluation+lm_eval --model vllm --model_args pretrained=model_path,add_bos_token=true,tensor_parallel_size=1,gpu_memory_utilization=0.8,max_model_len=8192 --tasks tasks --batch_size 128+```++## LLaMa-3-8B-Instruct++|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k_llama|mmlu_llama|mmlu_pro_llama|Avg. |+|:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:----------|:---------|:-------------|:-----|+|BF16 |0.5341 |0.8232 |0.8428|0.5805 |0.7258 |0.3540 |0.7758|0.3709 |0.7151 |0.6111 |0.6737 |0.3778 |0.6145|+|FP8_BLOCK RTN |0.5239 |0.8182 |0.8483|0.5780 |0.7235 |0.3520 |0.7840|0.3647 |0.7040 |0.6171 |0.6681 |0.3738 |0.6130|+|FP8_BLOCK Tuning|0.5213 |0.8047 |0.8376|0.5834 |0.7099 |0.3480 |0.7764|0.3684 |0.7245 |0.6224 |0.6654 |0.3690 |0.6109|+++## Qwen3-8B++|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k |mmlu |mmlu_pro|Avg. |+|:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:-----|:-----|:-------|:-----|+|BF16 |0.5589 |0.8329 |0.8661|0.5719 |0.6501 |0.3140 |0.7688|0.3647 |0.6772 |0.8726|0.7297|0.6218 |0.6524|+|FP8_BLOCK RTN |0.5444 |0.8350 |0.8679|0.5704 |0.6466 |0.3060 |0.7644|0.3611 |0.6914 |0.8741|0.7268|0.6242 |0.6510|+|FP8_BLOCK Tuning|0.5486 |0.8308 |0.8639|0.5691 |0.6513 |0.3080 |0.7622|0.3586 |0.6882 |0.8673|0.7272|0.6139 |0.6491|++diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex dde5006f8..2f67600cb 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -156,11 +156,12 @@ adopted within the community, **only 4-bits quantization is supported**. Please| Format | Supported Schemes ||:---|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `BF16` |+| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16` || **auto_awq** | W4A16, BF16 || **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 |-| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC` |+| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK || **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 |+| **fp8** | FP8_BLOCK || **fake** | `all schemes (only for research)` |### Hardware Compatibilitydiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 3a4801a25..5aee7303b 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -146,11 +146,12 @@ AutoRound 支持多种量化配置:| 格式 | 支持的量化方案 ||:-------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`BF16` |+| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16` || **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |-| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |+| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC || **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |+| **fp8** | FP8_BLOCK || **fake** | `所有方案(仅用于研究场景)` |### 硬件兼容性diff --git a/test/test_cpu/quantization/test_block_fp.py b/test/test_cpu/quantization/test_block_fp.pynew file mode 100644index 000000000..30b62e321--- /dev/null+++ b/test/test_cpu/quantization/test_block_fp.py@@ -0,0 +1,152 @@+import shutil+from math import ceil++import pytest+import torch++from auto_round import AutoRound+from auto_round.data_type.fp8 import quant_block_fp_sym+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad++from ...helpers import get_model_path+++class TestAutoRoundBlockFP:+ @classmethod+ def setup_class(self):+ self.model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")+ self.save_dir = "./saved"++ @classmethod+ def teardown_class(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_invalid_scheme(self):+ model_name = self.model_name++ with pytest.raises(ValueError):+ scheme = {+ "bits": 8,+ "group_size": (128, 128),+ "data_type": "int",+ "act_bits": 16,+ }+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )++ with pytest.raises(NotImplementedError):+ scheme = {+ "bits": 8,+ "group_size": (128, 128),+ "data_type": "fp",+ "act_bits": 8,+ "act_data_type": "fp",+ "act_group_size": 128,+ "act_dynamic": False,+ }+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )++ with pytest.raises(ValueError):+ scheme = {+ "bits": 8,+ "group_size": (128, 128),+ "data_type": "fp",+ "act_bits": 8,+ "act_data_type": "fp",+ "act_group_size": (128, 128),+ "act_dynamic": True,+ }+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )++ def test_block_fp8_quant(self):+ data = torch.randn(256, 240)+ group_size = (128, 128)+ reshaped_data, orig_shape, pad_len = reshape_pad_tensor_by_group_size(data, group_size)+ assert list(reshaped_data.shape) == [2, 2, 128, 128]+ assert list(orig_shape) == [256, 240]+ assert pad_len == (0, 16)++ qdq_data, scale, _ = quant_block_fp_sym(data)+ M = ceil(data.shape[0] / 128)+ N = ceil(data.shape[1] / 128)+ scale_ref = torch.zeros(M, N)++ max_val = torch.finfo(torch.float8_e4m3fn).max+ for i in range(M):+ for j in range(N):+ scale_ref[i, j] = data[i * 128 : (i + 1) * 128, j * 128 : (j + 1) * 128].abs().max() / max_val+ assert (scale == scale_ref).all()++ def test_fp8_block_autoround_format(self):+ model_name = self.model_name++ scheme = "FP8_BLOCK"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale_inv")+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"+ assert compressed_model.config.quantization_config["group_size"] == (128, 128)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_fp8_block_fp8_format(self):+ model_name = self.model_name++ scheme = "FP8_BLOCK"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale_inv")+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"+ assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_fp8_block_llm_compressor_format(self):+ model_name = self.model_name++ scheme = "FP8_BLOCK"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale")+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn+ assert list(tmp_layer.weight_scale.shape) == [12, 12]+ assert compressed_model.config.quantization_config["quant_method"] == "compressed-tensors"+ shutil.rmtree(quantized_model_path, ignore_errors=True)diff --git a/test/test_cuda/quantization/test_block_fp.py b/test/test_cuda/quantization/test_block_fp.pynew file mode 100644index 000000000..6ee0ababb--- /dev/null+++ b/test/test_cuda/quantization/test_block_fp.py@@ -0,0 +1,64 @@+import shutil+from math import ceil++import pytest+import torch++from auto_round import AutoRound+from auto_round.data_type.fp8 import quant_block_fp_sym+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad++from ...helpers import evaluate_accuracy, get_model_path+++class TestAutoRoundBlockFP:+ @classmethod+ def setup_class(self):+ self.model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")+ self.save_dir = "./saved"++ @classmethod+ def teardown_class(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_fp8_block_fp8_format(self):+ model_name = self.model_name++ scheme = "FP8_BLOCK"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale_inv")+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn+ assert list(tmp_layer.weight_scale_inv.shape) == [12, 12]+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"+ assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)+ evaluate_accuracy(quantized_model_path, threshold=0.55, batch_size=32, limit=100)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_fp8_block_llm_compressor_format(self):+ model_name = self.model_name++ scheme = "FP8_BLOCK"+ autoround = AutoRound(+ model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ tmp_layer = compressed_model.model.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale")+ assert tmp_layer.weight.dtype is torch.float8_e4m3fn+ assert list(tmp_layer.weight_scale.shape) == [12, 12]+ assert compressed_model.config.quantization_config["quant_method"] == "compressed-tensors"+ evaluate_accuracy(quantized_model_path, threshold=0.55, batch_size=32, limit=100)+ shutil.rmtree(quantized_model_path, ignore_errors=True)