PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add rtn quant func (#1564)
Wang, Mengni1ff8d028f44f
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex ea99fa511..27e956d1d 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -33,8 +33,8 @@ from auto_round.auto_scheme.utils import ()from auto_round.calib_dataset import get_dataloaderfrom auto_round.data_type.gguf import (- quant_tensor_gguf_asym_dq,- quant_tensor_gguf_sym_dq,+ quant_tensor_gguf_opt_rtn_asym_dq,+ quant_tensor_gguf_opt_rtn_sym_dq,search_gguf_scale_min_asym,search_gguf_scale_min_sym,)@@ -260,7 +260,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)- qdq_w, _, _ = quant_tensor_gguf_asym_dq(+ qdq_w, _, _ = quant_tensor_gguf_opt_rtn_asym_dq(tensor=tensor,bits=bits,scale_dtype=scale_dtype,@@ -275,7 +275,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)scale, d_scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=1)tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)- qdq_w, _, _ = quant_tensor_gguf_sym_dq(+ qdq_w, _, _ = quant_tensor_gguf_opt_rtn_sym_dq(tensor=tensor, bits=bits, scale_dtype=scale_dtype, imatrix=imatrix, scale=scale, d_scale=d_scale)else:diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 421396571..864cc069d 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1258,6 +1258,7 @@ class BaseCompressor(object):enable_round_tuning=False,enable_torch_compile=self.enable_torch_compile,disable_opt_rtn=disable_opt_rtn,+ enable_rtn=self.iters == 0,)m = m.unwrapper({})except torch.OutOfMemoryError:@@ -1273,6 +1274,7 @@ class BaseCompressor(object):enable_norm_bias_tuning=False,enable_round_tuning=False,enable_torch_compile=self.enable_torch_compile,+ enable_rtn=self.iters == 0,)m = m.unwrapper({})except Exception as e:@@ -1936,6 +1938,7 @@ class BaseCompressor(object):enable_torch_compile=self.enable_torch_compile,device=self.device,disable_opt_rtn=self.disable_opt_rtn,+ enable_rtn=self.iters == 0,)new_layer = wrapper_layer.unwrapper({})set_module(self.model, layer_name, new_layer)@@ -2704,6 +2707,7 @@ class BaseCompressor(object):enable_minmax_tuning=self.enable_minmax_tuning,enable_torch_compile=self.enable_torch_compile,device=device,+ enable_rtn=self.iters == 0,).to(device)round_params = []minmax_params = []@@ -3015,6 +3019,7 @@ class BaseCompressor(object):self.enable_norm_bias_tuning,enable_torch_compile=self.enable_torch_compile,device=device,+ enable_rtn=self.iters == 0,)# Call this before quantization and after applying the block wrapper.if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse.diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex ea7a6e99e..a45042172 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -22,7 +22,7 @@ from auto_round.data_type.utils import (reshape_pad_tensor_by_group_size,revert_tensor_by_pad,)-from auto_round.utils import is_gaudi2, logger+from auto_round.utils import is_gaudi2, is_hpex_available, logger@register_dtype(("block_fp8_sym", "block_fp8", "block_fp8_e4m3"))@@ -60,7 +60,6 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,scale = torch.clip(scale, min=min_scaling_factor)if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)-fp8_res = tensor / scale.unsqueeze(-1).unsqueeze(-1) + vfp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e4m3fn_ste(fp8_res)@@ -70,6 +69,47 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,return qdq_res, scale, None+@register_dtype(("rtn_block_fp8_sym", "rtn_block_fp8", "rtn_block_fp8_e4m3"))+def quant_rtn_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, **kwargs):+ """Symmetric quantization using block float8 format.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.to(tensor.device)+ if isinstance(v, torch.Tensor):+ v = v.to(tensor.device)+ if tensor_max is None:+ max_tensor = tensor.abs().amax(dim=(-2, -1)) * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = max_tensor.div_(info.max).clamp_(min=min_scaling_factor)+ assert len(scale.shape) == 2, f"Only support 2D group_size, but get {len(scale.shape)}"+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ tensor.div_(scale.unsqueeze(-1).unsqueeze(-1)).add_(v).clamp_(info.min, info.max)+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale.unsqueeze(-1).unsqueeze(-1))+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.@@ -116,6 +156,48 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **return qdq_res, scale, None+@register_dtype(("rtn_fp8_sym", "rtn_fp8", "rtn_fp8_e4m3"))+def quant_rtn_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.to(tensor.device)+ if isinstance(v, torch.Tensor):+ v = v.to(tensor.device)+ if tensor_max is None: ##dynamic per-token+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = max_tensor.float().div_(info.max).clamp_(min=min_scaling_factor).unsqueeze_(dim=-1)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype("fp8_e5m2")def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.@@ -158,6 +240,44 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, *return qdq_res, scale, None+@register_dtype("rtn_fp8_e5m2")+def quant_rtn_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e5m2)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor_max is None: ##dynamic per-token+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = max_tensor.float().div_(info.max).clamp_(min=min_scaling_factor).unsqueeze_(dim=-1)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)+ qdq_res = tensor.to(torch.float8_e5m2).to(orig_dtype).mul_(scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype("fp8_unit_scale")def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.@@ -191,6 +311,37 @@ def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1,return qdq_res, scale, None+@register_dtype("rtn_fp8_unit_scale")+def quant_rtn_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = torch.ones((1), device=tensor.device)+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype("fp8_e5m2_unit_scale")def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.@@ -224,6 +375,37 @@ def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_sizereturn qdq_res, scale, None+@register_dtype("rtn_fp8_e5m2_unit_scale")+def quant_rtn_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e5m2)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = torch.ones((1), device=tensor.device)+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)+ qdq_res = tensor.to(torch.float8_e5m2).to(orig_dtype).mul_(scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None++@register_dtype("fp8_gaudi3_sym")def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):"""Symmetric quantization using float8 format.@@ -268,6 +450,53 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):return qdq_res, scale, None+@register_dtype("rtn_fp8_gaudi3_sym")+def quant_rtn_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ orig_shape = tensor.shape+ fp8_max = torch.finfo(torch.float8_e4m3fn).max+ orig_dtype = tensor.dtype++ if tensor_max is None: ##dynamic per-te+ tensor = tensor.reshape(-1, orig_shape[-1])+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ scale = max_tensor.to(torch.float32) / fp8_max+ min_scaling_factor = float(1.0 / (fp8_max * 512.0)) ##copy from vllm+ scale = torch.clip(scale, min=min_scaling_factor)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = scale.unsqueeze(dim=-1)+ fp8_res = tensor / scale+ fp8_res = torch.clip(fp8_res, -fp8_max, fp8_max)+ fp8_res = (+ torch.ops.hpu.cast_to_fp8_v2(tensor, 1.0, False, False, torch.float8_e4m3fn)[0].to(orig_dtype)+ if is_hpex_available()+ else tensor.to(torch.float8_e4m3fn).to(orig_dtype)+ )+ qdq_res = fp8_res * scale+ qdq_res = qdq_res.to(orig_dtype).reshape(orig_shape)+ return qdq_res, scale, None++if is_gaudi2():@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))@@ -318,3 +547,52 @@ if is_gaudi2():qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)qdq_res = qdq_res.to(orig_dtype)return qdq_res, scale, None++ @register_dtype(("rtn_fp8_sym", "rtn_fp8", "rtn_fp8_e4m3"))+ def quant_rtn_fp8_sym(+ tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs+ ): # pylint: disable=E0102+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ logger.warning_once("Using float8_e4m3fnuz G2.")+ # !!! USE float8_e4m3fnuz for Gaudi2+ info = torch.finfo(torch.float8_e4m3fnuz)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.to(tensor.device)+ if isinstance(v, torch.Tensor):+ v = v.to(tensor.device)+ if tensor_max is None: ##dynamic per-token+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ scale = max_tensor.to(torch.float32) / info.max+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = torch.clip(scale, min=min_scaling_factor)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = scale.unsqueeze(dim=-1)+ fp8_res = tensor / scale + v+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ fp8_res2 = torch.ops.hpu.cast_to_fp8_v2(fp8_res, 1.0, False, False, torch.float8_e4m3fn)[0].to(orig_dtype)+ qdq_res = fp8_res2 * scale+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, Nonediff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 27a97f010..91feb7460 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -84,6 +84,65 @@ def quant_tensor_sym_dq(return qdq_result, {"scale": scale, "d_scale": d_scale}, zp+@register_dtype("rtn_int_sym_dq")+def quant_tensor_rtn_sym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ **kwargs,+):+ """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp community++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype++ maxq = 2 ** (bits - 1)+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max++ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130+ wmax_abs = wmax_tmp * max_scale+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)+ scale = (max_v / maxq).to(scale_dtype)+ scale = scale.view(-1, super_group_size)+ # conduct double quant+ scale, d_scale = double_quant_tensor_sym(scale, super_bits)++ scale = scale.view(-1, 1)+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ tensor.mul_(get_reciprocal(scale)).add_(v).round_().add_(zp).clamp_(0, 2**bits - 1).sub_(zp).mul_(scale)+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)+ return tensor, {"scale": scale, "d_scale": d_scale}, zp++@register_dtype("int_asym_float_zp")def quant_tensor_asym_float_zp(tensor,@@ -141,6 +200,62 @@ def quant_tensor_asym_float_zp(return qdq_result, scale, zp+@register_dtype("rtn_int_asym_float_zp")+def quant_tensor_rtn_asym_float_zp(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs,+):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = 2**bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ zp = -wmin / scale # pylint: disable=E1130+ scale = scale.unsqueeze(dim=-1)+ zp = zp.unsqueeze(dim=-1)+ tensor.div_(scale).add_(v).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)+ return tensor, scale, zp++## the values should be positivedef double_quant_tensor(tensor, bits):tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision@@ -318,6 +433,76 @@ def quant_tensor_asym_dq(return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}+@register_dtype("rtn_int_asym_dq")+def quant_tensor_rtn_asym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=8,+ super_bits=6,+ **kwargs,+):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = 2**bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ scale = scale.view(-1, super_group_size)+ wmin = -wmin # pylint: disable=E1130+ wmin = wmin.view(-1, super_group_size)++ ##conduct double quant+ scale, d_scale = double_quant_tensor(scale, super_bits)+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)++ scale = scale.view(-1, 1)+ scale = torch.clamp(scale, q_scale_thresh)+ d_scale = torch.clamp(d_scale, q_scale_thresh)+ d_wmin = torch.clamp(d_wmin, q_scale_thresh)+ wmin = wmin.view(-1, 1)++ tensor.add_(wmin).div_(scale).add_(v).round_().clamp_(0, maxq).mul_(scale).sub_(wmin)+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)++ return tensor, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}++def _imatrix_handle_zero(imatrix: Union[torch.Tensor, float], weight: torch.Tensor, bits: int):if not isinstance(imatrix, torch.Tensor):return imatrix@@ -442,8 +627,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrireturn scale, wmin, d_scale, d_wmin-@register_dtype("rtn_int_asym_dq")-def quant_tensor_gguf_asym_dq(+@register_dtype("opt_rtn_int_asym_dq")+def quant_tensor_gguf_opt_rtn_asym_dq(tensor: torch.Tensor,bits: int = 4,scale_dtype=torch.float16,@@ -483,11 +668,8 @@ def quant_tensor_gguf_asym_dq()inverse_scale = get_reciprocal(scale)- tensor = tensor + wmin- tensor = (tensor.mul_(inverse_scale)).round_().clamp_(0, maxq)- tensor = tensor.mul_(scale)- tensor = tensor.sub_(wmin).to(orig_dtype)- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ tensor.add_(wmin).mul_(inverse_scale).round_().clamp_(0, maxq).mul_(scale).sub_(wmin)+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)return tensor, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}@@ -659,8 +841,8 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num):return scale-@register_dtype("rtn_int_sym_dq")-def quant_tensor_gguf_sym_dq(+@register_dtype("opt_rtn_int_sym_dq")+def quant_tensor_gguf_opt_rtn_sym_dq(tensor,bits=3,imatrix=None,@@ -717,8 +899,7 @@ def quant_tensor_gguf_sym_dq(inverse_scale = get_reciprocal(scale)# int_w = round_ste(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxq# qdq_result = (scale * (int_w - zp)).to(orig_dtype)- tensor = tensor.mul_(inverse_scale).round_().clamp_(-maxq, maxq - 1)- tensor = tensor.mul_(scale).to(orig_dtype)- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ tensor.mul_(inverse_scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)return tensor, {"scale": scale, "d_scale": d_scale}, maxqdiff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 960a7fc08..8d6155deb 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -79,8 +79,8 @@ def search_scales(data: torch.Tensor, bits: int, qw: Union[None, torch.Tensor, freturn scales-@register_dtype("rtn_int_sym")-def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5, imatrix=None, **kwargs):+@register_dtype("opt_rtn_int_sym")+def quant_tensor_opt_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5, imatrix=None, **kwargs):"""Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp communityArgs:@@ -96,6 +96,7 @@ def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5from auto_round.data_type.gguf import _imatrix_handle_zerotensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtypemaxq = 2 ** (bits - 1)if imatrix is None:imatrix = 1.0@@ -109,9 +110,61 @@ def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5scale = search_scales(tensor, bits, qw=imatrix)scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))- int_w = tensor.div(scale).round_().clamp_(-maxq, maxq - 1)- qdq_result = (int_w.mul_(scale)).to(tensor.dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ tensor.div_(scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, scale, maxq+++@register_dtype("rtn_int_sym")+def quant_tensor_rtn_sym(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):+ """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = 2 ** (bits - 1)+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max++ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130+ wmax_abs = wmax_tmp * max_scale+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)+ scale = (max_v / maxq).to(scale_dtype)+ scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))+ scale = scale.unsqueeze(dim=-1)+ tensor.div_(scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)return qdq_result, scale, maxq@@ -226,6 +279,61 @@ def quant_tensor_asym(return qdq_result, scale, zp+@register_dtype("rtn_int_asym")+def quant_tensor_rtn_asym(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = 2**bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale.clamp_(min=q_scale_thresh)+ zp = (-wmin).div_(scale).round_().unsqueeze_(dim=-1)+ scale.unsqueeze_(dim=-1)+ tensor.div_(scale).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, scale, zp++@register_dtype("int_sym_gptq")def quant_tensor_sym_gptq(tensor,@@ -290,6 +398,68 @@ def quant_tensor_sym_gptq(return qdq_result, scale, zp+@register_dtype("rtn_int_sym_gptq")+def quant_tensor_rtn_sym_gptq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = 2**bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp++ wmax_new = torch.max(wmin.abs(), wmax)+ tmp = wmin < 0+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur+ if torch.any(tmp):+ wmin_new[tmp] = -wmax_new[tmp]++ scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)+ scale.clamp_(min=q_scale_thresh).unsqueeze_(dim=-1)+ zp = torch.full_like(scale, (maxq + 1) / 2)++ tensor.div_(scale).add_(v).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, scale, zp++def quant_tensor_asym_wo_round(tensor,bits=4,diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 9a7315a11..4051814ee 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -186,10 +186,111 @@ def quant_mx_rceil(return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None+def quant_rtn_mx(+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs+):+ """Quantize the given tensor using the specified parameters.++ This function performs quantization on the `tensor` tensor according to the+ given bit width (`bits`), data type (`data_type`), and additional parameters.+ The quantization process involves scaling the tensor values and adjusting+ the exponent and mantissa to fit within the specified format.++ Args:+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.+ bits (int): The bit width to be used for quantization.+ group_size (int): The group size of sharing scale and exponent.+ data_type (str): The data type for quantization (e.g., 'mx_fp4').+ v (float): A value used for adjusting the tensors.+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor++ Returns:+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).++ Raises:+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)+ if isinstance(max_scale, torch.Tensor):+ max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)+ else:+ max_val *= max_scale++ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))+ scale_emax = 2.0 ** float(8 - 1) - 1+ shared_exp = shared_exp.floor_().sub_(emax).clamp_(-scale_emax, scale_emax)++ scale = torch.pow(2.0, shared_exp.float())+ tensor.div_(scale).add_(v).clamp_(-max_norm, max_norm)+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)+ tensor.mul_(scale)+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None+++def quant_rtn_mx_rceil(+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs+):+ """Quantize the given tensor using the specified parameters.++ This function performs quantization on the `tensor` tensor according to the+ given bit width (`bits`), data type (`data_type`), and additional parameters.+ The quantization process involves scaling the tensor values and adjusting+ the exponent and mantissa to fit within the specified format.++ Args:+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.+ bits (int): The bit width to be used for quantization.+ group_size (int): The group size of sharing scale and exponent.+ data_type (str): The data type for quantization (e.g., 'mx_fp4').+ v (float): A value used for adjusting the tensors.+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor++ Returns:+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).++ Raises:+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)+ if isinstance(max_scale, torch.Tensor):+ max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)+ else:+ max_val *= max_scale++ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), (torch.log2(max_val / max_norm).ceil_()))+ scale_emax = 2.0 ** float(8 - 1) - 1+ shared_exp.clamp_(-scale_emax, scale_emax)++ scale = torch.pow(2.0, shared_exp.float())+ tensor.div_(scale).add_(v).clamp_(-max_norm, max_norm)+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)+ tensor.mul_(scale)+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None++for key in MXFP_FORMAT_CACHE.keys():QUANT_FUNC_WITH_DTYPE[key] = quant_mxQUANT_FUNC_WITH_DTYPE[key + "_rceil"] = quant_mx_rceil+ QUANT_FUNC_WITH_DTYPE["rtn_" + key] = quant_rtn_mx+ QUANT_FUNC_WITH_DTYPE["rtn_" + key + "_rceil"] = quant_rtn_mx_rceilQUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceil+QUANT_FUNC_WITH_DTYPE["rtn_mx_fp_rceil"] = quant_rtn_mx_rceilif __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 60cee640a..9f236fdbf 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -63,6 +63,20 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):return global_scale+def ref_nvfp4_quant_rtn(x, global_scale, block_size=16, v=0, scale_coeff=1.0):+ assert global_scale.dtype == torch.float32+ assert x.ndim == 2+ m, n = x.shape+ if isinstance(scale_coeff, torch.Tensor):+ scale_coeff = scale_coeff.view(-1, 1).to(x.device)+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * scale_coeff+ scale = vec_max.mul_(get_reciprocal(FLOAT4_E2M1_MAX)).mul_(global_scale).clamp_(FLOAT8_E4M3_MIN, FLOAT8_E4M3_MAX)+ scale = scale.to(torch.float8_e4m3fn).float()+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))+ clipped_x = x.float().mul_(output_scale).add_(v).clamp_(-6.0, 6.0)+ return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale++def ref_nvfp4_quant(x, global_scale, block_size=16, v=0, scale_coeff=1.0):assert global_scale.dtype == torch.float32assert x.ndim == 2@@ -92,6 +106,19 @@ def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0,return qdq_res.to(orig_dtype), scale, None+@register_dtype("rtn_nv_fp4")+def rtn_nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if global_scale is None:+ tensor_max = tensor.abs().max().to(torch.float32)+ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_E4M3_MAX).mul_(FLOAT4_E2M1_MAX)+ global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)+ qdq_res, scale = ref_nvfp4_quant_rtn(tensor, global_scale, group_size, v, scale_coeff=max_scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None++@register_dtype("nv_fp4_with_static_gs")def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):if tensor is None or tensor.numel() == 0:@@ -115,6 +142,29 @@ def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, *return qdq_res.to(orig_dtype), scale, None+@register_dtype("rtn_nv_fp4_with_static_gs")+def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):+ if tensor is None or tensor.numel() == 0:+ return tensor, None, None+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor_max is None:+ tensor_max = tensor.abs().max().to(torch.float32)+ else:+ if not isinstance(tensor_max, torch.Tensor):+ tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)+ else:+ tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32)+ if tensor_max.numel() != 1:+ tensor_max = tensor_max.abs().max()++ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_E4M3_MAX).mul_(FLOAT4_E2M1_MAX)+ global_scale = global_scale.to(tensor.device)+ qdq_res, scale = ref_nvfp4_quant_rtn(tensor, global_scale, group_size, v)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None++FLOAT8_UE5M3_MAX = 114688@@ -206,6 +256,20 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale+def ref_fp4_quant_rtn(x, global_scale, block_size=16, v=0, max_scale=1.0):+ assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32+ assert x.ndim == 2+ m, n = x.shape+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.unsqueeze(dim=-1).to(x.device)+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale+ scale = vec_max.mul_(get_reciprocal(FLOAT4_E2M1_MAX)).mul_(global_scale).clamp_(0, FLOAT8_UE5M3_MAX)+ scale = cast_to_ue5m3(scale).float()+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))+ clipped_x = x.float().mul_(output_scale).add_(v).clamp_(-6.0, 6.0)+ return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale++@register_dtype("fp4_v2_with_global_scale")def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16@@ -224,6 +288,24 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=Nonereturn qdq_res.to(orig_dtype), scale, None+@register_dtype("rtn_fp4_v2_with_global_scale")+def rtn_fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):+ assert group_size == 32 or group_size == 16+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor_max is None:+ tensor_max = tensor.abs().max().to(torch.float32)+ elif tensor_max is not None:+ if not isinstance(tensor_max, torch.Tensor):+ tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)+ if tensor_max.numel() != 1:+ tensor_max = tensor.abs().max().to(torch.float32)+ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_UE5M3_MAX).mul_(FLOAT4_E2M1_MAX)+ qdq_res, scale = ref_fp4_quant_rtn(tensor, global_scale, group_size, v)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None++@register_dtype("fp4_v2")def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16@@ -235,6 +317,17 @@ def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):return qdq_res.to(orig_dtype), scale, None+@register_dtype("rtn_fp4_v2")+def rtn_fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):+ assert group_size == 32 or group_size == 16+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ global_scale = 1.0+ qdq_res, scale = ref_fp4_quant_rtn(tensor, global_scale, group_size, v, max_scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None++if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])data1 = cast_to_fp4(data)diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex a8e3eb218..d82773c10 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -101,7 +101,9 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: Union[ireturn data_new-def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None) -> tuple[callable, str]:+def get_quant_func(+ dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None, enable_rtn=False+) -> tuple[callable, str]:"""Retrieve the quantization function based on data type, bit width, and symmetry.This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE@@ -114,6 +116,7 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, grousym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).disable_opt_rtn(bool): whether to disable optimized rtn.group_size (tuple): The block size for weight quantization (e.g., (128, 128)).+ enable_rtn (bool): whether to use rtn.Returns:function: The quantization function corresponding to the specified parameters.@@ -131,8 +134,13 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, groureturn data_type + str(bits)if not disable_opt_rtn:- rtn_data_type = "rtn_" + dtype- data_types = [rtn_data_type, pad_bits(rtn_data_type), pad_sym(rtn_data_type), pad_sym(pad_bits(rtn_data_type))]+ opt_rtn_data_type = "opt_rtn_" + dtype+ data_types = [+ opt_rtn_data_type,+ pad_bits(opt_rtn_data_type),+ pad_sym(opt_rtn_data_type),+ pad_sym(pad_bits(opt_rtn_data_type)),+ ]for data_type in data_types:from auto_round.data_type import QUANT_FUNC_WITH_DTYPE@@ -140,19 +148,10 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, groureturn QUANT_FUNC_WITH_DTYPE[data_type], data_typeif group_size is not None and isinstance(group_size, tuple):- block_data_type = "block_" + dtype- data_types = [- block_data_type,- pad_bits(block_data_type),- pad_sym(block_data_type),- pad_sym(pad_bits(block_data_type)),- ]+ dtype = "block_" + dtype- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-- for data_type in data_types:- if data_type in QUANT_FUNC_WITH_DTYPE:- return QUANT_FUNC_WITH_DTYPE[data_type], data_type+ if enable_rtn:+ dtype = "rtn_" + dtypedata_types = [dtype, pad_bits(dtype), pad_sym(dtype), pad_sym(pad_bits(dtype))]for data_type in data_types:diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.pyindex b8e987d4c..8ebd5a592 100644--- a/auto_round/data_type/w4fp8.py+++ b/auto_round/data_type/w4fp8.py@@ -14,6 +14,7 @@import torch+from auto_round.data_type.int import quant_tensor_rtn_sym, quant_tensor_symfrom auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func@@ -56,7 +57,6 @@ def progressive_quant_fp8_int4(##convert to bf16fp8_res_using_16bit = fp8_res.to(tensor.dtype)##convert to int4- from auto_round.data_type.int import quant_tensor_symqdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_sym(fp8_res_using_16bit,@@ -72,3 +72,57 @@ def progressive_quant_fp8_int4(bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scalereturn qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4+++@register_dtype("rtn_fp8_to_int_sym")+def progressive_quant_rtn_fp8_int4(+ tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, q_scale_thresh=1e-5, **kwargs+):+ """Two-stage quantization: quantize tensor to fp8 by per tensor, then quantize fp8 to w4g128++ This method first quantizes the input tensor into float8 format and then performs+ a secondary quantization to int4 with grouping.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ bits (int, optional): Bit precision for secondary quantization. Defaults to 4.+ group_size (int, optional): Group size for int4 quantization. Defaults to -1 (no grouping).+ v (float, optional): Optional parameter for variance tuning. Defaults to 0.+ min_scale (float, optional): Minimum scaling factor for int4 quantization. Defaults to 1.0.+ max_scale (float, optional): Maximum scaling factor for int4 quantization. Defaults to 1.0.+ q_scale_thresh (float, optional): Threshold for scaling. Defaults to 1e-5.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Combined scaling factor (torch.Tensor).+ - Placeholder for zp (None).+ """++ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor_max = torch.max(torch.abs(tensor)).to(torch.float32)+ scale = tensor_max.to(torch.float32) / info.max+ min_scaling_factor = 1.0 / (info.max * 512.0) ##copy from vllm+ bf16_to_fp8_scale = tensor_max.div_(info.max).clamp_(min=min_scaling_factor)+ fp8_res = tensor.div_(bf16_to_fp8_scale).clamp_(info.min, info.max).to(torch.float8_e4m3fn)++ ##convert to bf16+ fp8_res_using_16bit = fp8_res.to(orig_dtype)+ ##convert to int4++ qdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_rtn_sym(+ fp8_res_using_16bit,+ bits=bits,+ group_size=group_size,+ v=v,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=torch.bfloat16,+ q_scale_thresh=q_scale_thresh,+ )+ qdq_tensor = qdq_int4_tensor * bf16_to_fp8_scale++ bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale+ return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex c64066932..d860192ec 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -652,10 +652,12 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i)else:- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dqblocks.reshape(blocks.shape[0], -1)- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix)+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(+ blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix+ )scales, d_scale = scales["scale"], scales["d_scale"]mins, d_wmin = mins["wmin"], mins["d_wmin"]blocks = blocks.reshape((nb, QK_K // 16, 16))@@ -702,10 +704,10 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,all_L = blocks.mul_(reverse_qdq_scale.unsqueeze(-1)).round_().clamp_(-4, 3).add_(4).to(torch.uint8)q_scales_offset = (qdq_scale * inverse_dq_scale).round_().clamp_(-32, 31).add_(32)else:- from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_sym_dqblocks = blocks.reshape(blocks.shape[0], -1)- blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)+ blocks, scales, _ = quant_tensor_gguf_opt_rtn_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)scales, d_scale = scales["scale"], scales["d_scale"]blocks = blocks.reshape((nb, QK_K // 16, 16))@@ -780,10 +782,12 @@ def q4_k_quant_block()else:- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dqblocks.reshape(blocks.shape[0], -1)- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(+ blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix+ )scales, d_scale = scales["scale"], scales["d_scale"]mins, d_wmin = mins["wmin"], mins["d_wmin"]@@ -875,10 +879,12 @@ def q5_k_quant_block(.to(torch.uint8))else:- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dqblocks.reshape(blocks.shape[0], -1)- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(+ blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix+ )scales, d_scale = scales["scale"], scales["d_scale"]mins, d_wmin = mins["wmin"], mins["d_wmin"]@@ -948,10 +954,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,blocks[replace_ids].div(d_tmp[replace_ids]).reshape(-1, 1).add_(32).round_().clamp_(0, 63).to(torch.uint8))else:- from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_sym_dqblocks = blocks.reshape(blocks.shape[0], -1)- blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)+ blocks, scales, _ = quant_tensor_gguf_opt_rtn_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)scales, d_scale = scales["scale"], scales["d_scale"]blocks = blocks.reshape((nb, QK_K // 16, 16))diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 2478db548..892b20eb4 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -80,6 +80,7 @@ class WrapperLinear(torch.nn.Module):enable_round_tuning=True,enable_torch_compile=False,disable_opt_rtn=True,+ enable_rtn=False,**kwargs,):"""Initializes the WrapperLinear module.@@ -93,6 +94,7 @@ class WrapperLinear(torch.nn.Module):super(WrapperLinear, self).__init__()self.orig_layer = orig_layerself.disable_opt_rtn = disable_opt_rtn+ self.enable_rtn = enable_rtnself.output_device = deviceself.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else deviceself.enable_minmax_tuning = enable_minmax_tuning@@ -169,15 +171,25 @@ class WrapperLinear(torch.nn.Module):self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))self.weight_quant_func, self.data_type = get_quant_func(- orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn, orig_layer.group_size+ orig_layer.data_type,+ orig_layer.bits,+ orig_layer.sym,+ self.disable_opt_rtn,+ orig_layer.group_size,+ self.enable_rtn,)if self.enable_torch_compile:self.weight_quant_func = compile_func(self.weight_quant_func, self.device)if self.enable_act_quant:self.act_quant_func, self.act_data_type = get_quant_func(- orig_layer.act_data_type, orig_layer.act_bits, orig_layer.act_sym, self.disable_opt_rtn+ orig_layer.act_data_type,+ orig_layer.act_bits,+ orig_layer.act_sym,+ self.disable_opt_rtn,+ enable_rtn=self.enable_rtn,)+if self.enable_torch_compile:self.act_quant_func = compile_func(self.act_quant_func, self.device)self._init_params("act_max_scale", p_dtype, (1), 1.0, not orig_layer.act_dynamic)