Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add rtn quant func (#1564)

Wang, Mengni1ff8d028f44f
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index ea99fa511..27e956d1d 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -33,8 +33,8 @@ from auto_round.auto_scheme.utils import (
)
from auto_round.calib_dataset import get_dataloader
from auto_round.data_type.gguf import (
- quant_tensor_gguf_asym_dq,
- quant_tensor_gguf_sym_dq,
+ quant_tensor_gguf_opt_rtn_asym_dq,
+ quant_tensor_gguf_opt_rtn_sym_dq,
search_gguf_scale_min_asym,
search_gguf_scale_min_sym,
)
@@ -260,7 +260,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
- qdq_w, _, _ = quant_tensor_gguf_asym_dq(
+ qdq_w, _, _ = quant_tensor_gguf_opt_rtn_asym_dq(
tensor=tensor,
bits=bits,
scale_dtype=scale_dtype,
@@ -275,7 +275,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
scale, d_scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=1)
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
- qdq_w, _, _ = quant_tensor_gguf_sym_dq(
+ qdq_w, _, _ = quant_tensor_gguf_opt_rtn_sym_dq(
tensor=tensor, bits=bits, scale_dtype=scale_dtype, imatrix=imatrix, scale=scale, d_scale=d_scale
)
else:
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 421396571..864cc069d 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1258,6 +1258,7 @@ class BaseCompressor(object):
enable_round_tuning=False,
enable_torch_compile=self.enable_torch_compile,
disable_opt_rtn=disable_opt_rtn,
+ enable_rtn=self.iters == 0,
)
m = m.unwrapper({})
except torch.OutOfMemoryError:
@@ -1273,6 +1274,7 @@ class BaseCompressor(object):
enable_norm_bias_tuning=False,
enable_round_tuning=False,
enable_torch_compile=self.enable_torch_compile,
+ enable_rtn=self.iters == 0,
)
m = m.unwrapper({})
except Exception as e:
@@ -1936,6 +1938,7 @@ class BaseCompressor(object):
enable_torch_compile=self.enable_torch_compile,
device=self.device,
disable_opt_rtn=self.disable_opt_rtn,
+ enable_rtn=self.iters == 0,
)
new_layer = wrapper_layer.unwrapper({})
set_module(self.model, layer_name, new_layer)
@@ -2704,6 +2707,7 @@ class BaseCompressor(object):
enable_minmax_tuning=self.enable_minmax_tuning,
enable_torch_compile=self.enable_torch_compile,
device=device,
+ enable_rtn=self.iters == 0,
).to(device)
round_params = []
minmax_params = []
@@ -3015,6 +3019,7 @@ class BaseCompressor(object):
self.enable_norm_bias_tuning,
enable_torch_compile=self.enable_torch_compile,
device=device,
+ enable_rtn=self.iters == 0,
)
# Call this before quantization and after applying the block wrapper.
if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse.
diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.py
index ea7a6e99e..a45042172 100644
--- a/auto_round/data_type/fp8.py
+++ b/auto_round/data_type/fp8.py
@@ -22,7 +22,7 @@ from auto_round.data_type.utils import (
reshape_pad_tensor_by_group_size,
revert_tensor_by_pad,
)
-from auto_round.utils import is_gaudi2, logger
+from auto_round.utils import is_gaudi2, is_hpex_available, logger
@register_dtype(("block_fp8_sym", "block_fp8", "block_fp8_e4m3"))
@@ -60,7 +60,6 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,
scale = torch.clip(scale, min=min_scaling_factor)
if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
tensor = tensor.to(torch.bfloat16)
-
fp8_res = tensor / scale.unsqueeze(-1).unsqueeze(-1) + v
fp8_res = torch.clip(fp8_res, info.min, info.max)
fp8_res = float8_e4m3fn_ste(fp8_res)
@@ -70,6 +69,47 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,
return qdq_res, scale, None
+@register_dtype(("rtn_block_fp8_sym", "rtn_block_fp8", "rtn_block_fp8_e4m3"))
+def quant_rtn_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, **kwargs):
+ """Symmetric quantization using block float8 format.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.to(tensor.device)
+ if isinstance(v, torch.Tensor):
+ v = v.to(tensor.device)
+ if tensor_max is None:
+ max_tensor = tensor.abs().amax(dim=(-2, -1)) * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = max_tensor.div_(info.max).clamp_(min=min_scaling_factor)
+ assert len(scale.shape) == 2, f"Only support 2D group_size, but get {len(scale.shape)}"
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ tensor.div_(scale.unsqueeze(-1).unsqueeze(-1)).add_(v).clamp_(info.min, info.max)
+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale.unsqueeze(-1).unsqueeze(-1))
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))
def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
"""Symmetric quantization using float8 format.
@@ -116,6 +156,48 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **
return qdq_res, scale, None
+@register_dtype(("rtn_fp8_sym", "rtn_fp8", "rtn_fp8_e4m3"))
+def quant_rtn_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.to(tensor.device)
+ if isinstance(v, torch.Tensor):
+ v = v.to(tensor.device)
+ if tensor_max is None: ##dynamic per-token
+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = max_tensor.float().div_(info.max).clamp_(min=min_scaling_factor).unsqueeze_(dim=-1)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)
+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype("fp8_e5m2")
def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
"""Symmetric quantization using float8 format.
@@ -158,6 +240,44 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, *
return qdq_res, scale, None
+@register_dtype("rtn_fp8_e5m2")
+def quant_rtn_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e5m2)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor_max is None: ##dynamic per-token
+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = max_tensor.float().div_(info.max).clamp_(min=min_scaling_factor).unsqueeze_(dim=-1)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)
+ qdq_res = tensor.to(torch.float8_e5m2).to(orig_dtype).mul_(scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype("fp8_unit_scale")
def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
"""Symmetric quantization using float8 format.
@@ -191,6 +311,37 @@ def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1,
return qdq_res, scale, None
+@register_dtype("rtn_fp8_unit_scale")
+def quant_rtn_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = torch.ones((1), device=tensor.device)
+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)
+ qdq_res = tensor.to(torch.float8_e4m3fn).to(orig_dtype).mul_(scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype("fp8_e5m2_unit_scale")
def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
"""Symmetric quantization using float8 format.
@@ -224,6 +375,37 @@ def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size
return qdq_res, scale, None
+@register_dtype("rtn_fp8_e5m2_unit_scale")
+def quant_rtn_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e5m2)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = torch.ones((1), device=tensor.device)
+ tensor.div_(scale).add_(v).clamp_(info.min, info.max)
+ qdq_res = tensor.to(torch.float8_e5m2).to(orig_dtype).mul_(scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
@register_dtype("fp8_gaudi3_sym")
def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):
"""Symmetric quantization using float8 format.
@@ -268,6 +450,53 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):
return qdq_res, scale, None
+@register_dtype("rtn_fp8_gaudi3_sym")
+def quant_rtn_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ orig_shape = tensor.shape
+ fp8_max = torch.finfo(torch.float8_e4m3fn).max
+ orig_dtype = tensor.dtype
+
+ if tensor_max is None: ##dynamic per-te
+ tensor = tensor.reshape(-1, orig_shape[-1])
+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ scale = max_tensor.to(torch.float32) / fp8_max
+ min_scaling_factor = float(1.0 / (fp8_max * 512.0)) ##copy from vllm
+ scale = torch.clip(scale, min=min_scaling_factor)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = scale.unsqueeze(dim=-1)
+ fp8_res = tensor / scale
+ fp8_res = torch.clip(fp8_res, -fp8_max, fp8_max)
+ fp8_res = (
+ torch.ops.hpu.cast_to_fp8_v2(tensor, 1.0, False, False, torch.float8_e4m3fn)[0].to(orig_dtype)
+ if is_hpex_available()
+ else tensor.to(torch.float8_e4m3fn).to(orig_dtype)
+ )
+ qdq_res = fp8_res * scale
+ qdq_res = qdq_res.to(orig_dtype).reshape(orig_shape)
+ return qdq_res, scale, None
+
+
if is_gaudi2():
@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))
@@ -318,3 +547,52 @@ if is_gaudi2():
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
qdq_res = qdq_res.to(orig_dtype)
return qdq_res, scale, None
+
+ @register_dtype(("rtn_fp8_sym", "rtn_fp8", "rtn_fp8_e4m3"))
+ def quant_rtn_fp8_sym(
+ tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs
+ ): # pylint: disable=E0102
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ logger.warning_once("Using float8_e4m3fnuz G2.")
+ # !!! USE float8_e4m3fnuz for Gaudi2
+ info = torch.finfo(torch.float8_e4m3fnuz)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.to(tensor.device)
+ if isinstance(v, torch.Tensor):
+ v = v.to(tensor.device)
+ if tensor_max is None: ##dynamic per-token
+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ scale = max_tensor.to(torch.float32) / info.max
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = torch.clip(scale, min=min_scaling_factor)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = scale.unsqueeze(dim=-1)
+ fp8_res = tensor / scale + v
+ fp8_res = torch.clip(fp8_res, info.min, info.max)
+ fp8_res2 = torch.ops.hpu.cast_to_fp8_v2(fp8_res, 1.0, False, False, torch.float8_e4m3fn)[0].to(orig_dtype)
+ qdq_res = fp8_res2 * scale
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 27a97f010..91feb7460 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -84,6 +84,65 @@ def quant_tensor_sym_dq(
return qdq_result, {"scale": scale, "d_scale": d_scale}, zp
+@register_dtype("rtn_int_sym_dq")
+def quant_tensor_rtn_sym_dq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ super_group_size=16,
+ super_bits=6,
+ **kwargs,
+):
+ """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp community
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+
+ maxq = 2 ** (bits - 1)
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+
+ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130
+ wmax_abs = wmax_tmp * max_scale
+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
+ scale = (max_v / maxq).to(scale_dtype)
+ scale = scale.view(-1, super_group_size)
+ # conduct double quant
+ scale, d_scale = double_quant_tensor_sym(scale, super_bits)
+
+ scale = scale.view(-1, 1)
+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130
+ tensor.mul_(get_reciprocal(scale)).add_(v).round_().add_(zp).clamp_(0, 2**bits - 1).sub_(zp).mul_(scale)
+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+ return tensor, {"scale": scale, "d_scale": d_scale}, zp
+
+
@register_dtype("int_asym_float_zp")
def quant_tensor_asym_float_zp(
tensor,
@@ -141,6 +200,62 @@ def quant_tensor_asym_float_zp(
return qdq_result, scale, zp
+@register_dtype("rtn_int_asym_float_zp")
+def quant_tensor_rtn_asym_float_zp(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ **kwargs,
+):
+ """Quantize and de-quantize tensor asymmetrically.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ maxq = 2**bits - 1
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+ if isinstance(min_scale, torch.Tensor):
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
+ else:
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
+ zp = -wmin / scale # pylint: disable=E1130
+ scale = scale.unsqueeze(dim=-1)
+ zp = zp.unsqueeze(dim=-1)
+ tensor.div_(scale).add_(v).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)
+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+ return tensor, scale, zp
+
+
## the values should be positive
def double_quant_tensor(tensor, bits):
tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision
@@ -318,6 +433,76 @@ def quant_tensor_asym_dq(
return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
+@register_dtype("rtn_int_asym_dq")
+def quant_tensor_rtn_asym_dq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ super_group_size=8,
+ super_bits=6,
+ **kwargs,
+):
+ """Quantize and de-quantize tensor asymmetrically.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ maxq = 2**bits - 1
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+ if isinstance(min_scale, torch.Tensor):
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
+ else:
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
+ scale = scale.view(-1, super_group_size)
+ wmin = -wmin # pylint: disable=E1130
+ wmin = wmin.view(-1, super_group_size)
+
+ ##conduct double quant
+ scale, d_scale = double_quant_tensor(scale, super_bits)
+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)
+
+ scale = scale.view(-1, 1)
+ scale = torch.clamp(scale, q_scale_thresh)
+ d_scale = torch.clamp(d_scale, q_scale_thresh)
+ d_wmin = torch.clamp(d_wmin, q_scale_thresh)
+ wmin = wmin.view(-1, 1)
+
+ tensor.add_(wmin).div_(scale).add_(v).round_().clamp_(0, maxq).mul_(scale).sub_(wmin)
+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+
+ return tensor, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
+
+
def _imatrix_handle_zero(imatrix: Union[torch.Tensor, float], weight: torch.Tensor, bits: int):
if not isinstance(imatrix, torch.Tensor):
return imatrix
@@ -442,8 +627,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
return scale, wmin, d_scale, d_wmin
-@register_dtype("rtn_int_asym_dq")
-def quant_tensor_gguf_asym_dq(
+@register_dtype("opt_rtn_int_asym_dq")
+def quant_tensor_gguf_opt_rtn_asym_dq(
tensor: torch.Tensor,
bits: int = 4,
scale_dtype=torch.float16,
@@ -483,11 +668,8 @@ def quant_tensor_gguf_asym_dq(
)
inverse_scale = get_reciprocal(scale)
- tensor = tensor + wmin
- tensor = (tensor.mul_(inverse_scale)).round_().clamp_(0, maxq)
- tensor = tensor.mul_(scale)
- tensor = tensor.sub_(wmin).to(orig_dtype)
- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+ tensor.add_(wmin).mul_(inverse_scale).round_().clamp_(0, maxq).mul_(scale).sub_(wmin)
+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
return tensor, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
@@ -659,8 +841,8 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num):
return scale
-@register_dtype("rtn_int_sym_dq")
-def quant_tensor_gguf_sym_dq(
+@register_dtype("opt_rtn_int_sym_dq")
+def quant_tensor_gguf_opt_rtn_sym_dq(
tensor,
bits=3,
imatrix=None,
@@ -717,8 +899,7 @@ def quant_tensor_gguf_sym_dq(
inverse_scale = get_reciprocal(scale)
# int_w = round_ste(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxq
# qdq_result = (scale * (int_w - zp)).to(orig_dtype)
- tensor = tensor.mul_(inverse_scale).round_().clamp_(-maxq, maxq - 1)
- tensor = tensor.mul_(scale).to(orig_dtype)
- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+ tensor.mul_(inverse_scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)
+ tensor = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
return tensor, {"scale": scale, "d_scale": d_scale}, maxq
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 960a7fc08..8d6155deb 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -79,8 +79,8 @@ def search_scales(data: torch.Tensor, bits: int, qw: Union[None, torch.Tensor, f
return scales
-@register_dtype("rtn_int_sym")
-def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5, imatrix=None, **kwargs):
+@register_dtype("opt_rtn_int_sym")
+def quant_tensor_opt_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5, imatrix=None, **kwargs):
"""Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community
Args:
@@ -96,6 +96,7 @@ def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5
from auto_round.data_type.gguf import _imatrix_handle_zero
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
maxq = 2 ** (bits - 1)
if imatrix is None:
imatrix = 1.0
@@ -109,9 +110,61 @@ def quant_tensor_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=1e-5
scale = search_scales(tensor, bits, qw=imatrix)
scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))
- int_w = tensor.div(scale).round_().clamp_(-maxq, maxq - 1)
- qdq_result = (int_w.mul_(scale)).to(tensor.dtype)
- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ tensor.div_(scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)
+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, scale, maxq
+
+
+@register_dtype("rtn_int_sym")
+def quant_tensor_rtn_sym(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ **kwargs
+):
+ """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ maxq = 2 ** (bits - 1)
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+
+ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130
+ wmax_abs = wmax_tmp * max_scale
+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
+ scale = (max_v / maxq).to(scale_dtype)
+ scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))
+ scale = scale.unsqueeze(dim=-1)
+ tensor.div_(scale).round_().clamp_(-maxq, maxq - 1).mul_(scale)
+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
return qdq_result, scale, maxq
@@ -226,6 +279,61 @@ def quant_tensor_asym(
return qdq_result, scale, zp
+@register_dtype("rtn_int_asym")
+def quant_tensor_rtn_asym(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ **kwargs
+):
+ """Quantize and de-quantize tensor asymmetrically.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ maxq = 2**bits - 1
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+ if isinstance(min_scale, torch.Tensor):
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
+ else:
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale.clamp_(min=q_scale_thresh)
+ zp = (-wmin).div_(scale).round_().unsqueeze_(dim=-1)
+ scale.unsqueeze_(dim=-1)
+ tensor.div_(scale).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)
+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, scale, zp
+
+
@register_dtype("int_sym_gptq")
def quant_tensor_sym_gptq(
tensor,
@@ -290,6 +398,68 @@ def quant_tensor_sym_gptq(
return qdq_result, scale, zp
+@register_dtype("rtn_int_sym_gptq")
+def quant_tensor_rtn_sym_gptq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ **kwargs
+):
+ """Quantize and de-quantize tensor asymmetrically.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ maxq = 2**bits - 1
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+ if isinstance(min_scale, torch.Tensor):
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
+ else:
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+
+ wmax_new = torch.max(wmin.abs(), wmax)
+ tmp = wmin < 0
+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
+ if torch.any(tmp):
+ wmin_new[tmp] = -wmax_new[tmp]
+
+ scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
+ scale.clamp_(min=q_scale_thresh).unsqueeze_(dim=-1)
+ zp = torch.full_like(scale, (maxq + 1) / 2)
+
+ tensor.div_(scale).add_(v).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)
+ qdq_result = revert_tensor_by_pad(tensor.to(orig_dtype), orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, scale, zp
+
+
def quant_tensor_asym_wo_round(
tensor,
bits=4,
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 9a7315a11..4051814ee 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -186,10 +186,111 @@ def quant_mx_rceil(
return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None
+def quant_rtn_mx(
+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs
+):
+ """Quantize the given tensor using the specified parameters.
+
+ This function performs quantization on the `tensor` tensor according to the
+ given bit width (`bits`), data type (`data_type`), and additional parameters.
+ The quantization process involves scaling the tensor values and adjusting
+ the exponent and mantissa to fit within the specified format.
+
+ Args:
+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.
+ bits (int): The bit width to be used for quantization.
+ group_size (int): The group size of sharing scale and exponent.
+ data_type (str): The data type for quantization (e.g., 'mx_fp4').
+ v (float): A value used for adjusting the tensors.
+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.
+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor
+
+ Returns:
+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).
+
+ Raises:
+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)
+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]
+ orig_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
+ if isinstance(max_scale, torch.Tensor):
+ max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)
+ else:
+ max_val *= max_scale
+
+ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))
+ scale_emax = 2.0 ** float(8 - 1) - 1
+ shared_exp = shared_exp.floor_().sub_(emax).clamp_(-scale_emax, scale_emax)
+
+ scale = torch.pow(2.0, shared_exp.float())
+ tensor.div_(scale).add_(v).clamp_(-max_norm, max_norm)
+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)
+ tensor.mul_(scale)
+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+ return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None
+
+
+def quant_rtn_mx_rceil(
+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs
+):
+ """Quantize the given tensor using the specified parameters.
+
+ This function performs quantization on the `tensor` tensor according to the
+ given bit width (`bits`), data type (`data_type`), and additional parameters.
+ The quantization process involves scaling the tensor values and adjusting
+ the exponent and mantissa to fit within the specified format.
+
+ Args:
+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.
+ bits (int): The bit width to be used for quantization.
+ group_size (int): The group size of sharing scale and exponent.
+ data_type (str): The data type for quantization (e.g., 'mx_fp4').
+ v (float): A value used for adjusting the tensors.
+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.
+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor
+
+ Returns:
+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).
+
+ Raises:
+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)
+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]
+ orig_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
+ if isinstance(max_scale, torch.Tensor):
+ max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)
+ else:
+ max_val *= max_scale
+
+ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), (torch.log2(max_val / max_norm).ceil_()))
+ scale_emax = 2.0 ** float(8 - 1) - 1
+ shared_exp.clamp_(-scale_emax, scale_emax)
+
+ scale = torch.pow(2.0, shared_exp.float())
+ tensor.div_(scale).add_(v).clamp_(-max_norm, max_norm)
+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)
+ tensor.mul_(scale)
+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+ return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None
+
+
for key in MXFP_FORMAT_CACHE.keys():
QUANT_FUNC_WITH_DTYPE[key] = quant_mx
QUANT_FUNC_WITH_DTYPE[key + "_rceil"] = quant_mx_rceil
+ QUANT_FUNC_WITH_DTYPE["rtn_" + key] = quant_rtn_mx
+ QUANT_FUNC_WITH_DTYPE["rtn_" + key + "_rceil"] = quant_rtn_mx_rceil
QUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceil
+QUANT_FUNC_WITH_DTYPE["rtn_mx_fp_rceil"] = quant_rtn_mx_rceil
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 60cee640a..9f236fdbf 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -63,6 +63,20 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):
return global_scale
+def ref_nvfp4_quant_rtn(x, global_scale, block_size=16, v=0, scale_coeff=1.0):
+ assert global_scale.dtype == torch.float32
+ assert x.ndim == 2
+ m, n = x.shape
+ if isinstance(scale_coeff, torch.Tensor):
+ scale_coeff = scale_coeff.view(-1, 1).to(x.device)
+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * scale_coeff
+ scale = vec_max.mul_(get_reciprocal(FLOAT4_E2M1_MAX)).mul_(global_scale).clamp_(FLOAT8_E4M3_MIN, FLOAT8_E4M3_MAX)
+ scale = scale.to(torch.float8_e4m3fn).float()
+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
+ clipped_x = x.float().mul_(output_scale).add_(v).clamp_(-6.0, 6.0)
+ return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+
+
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0, scale_coeff=1.0):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
@@ -92,6 +106,19 @@ def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0,
return qdq_res.to(orig_dtype), scale, None
+@register_dtype("rtn_nv_fp4")
+def rtn_nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if global_scale is None:
+ tensor_max = tensor.abs().max().to(torch.float32)
+ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_E4M3_MAX).mul_(FLOAT4_E2M1_MAX)
+ global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)
+ qdq_res, scale = ref_nvfp4_quant_rtn(tensor, global_scale, group_size, v, scale_coeff=max_scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
@register_dtype("nv_fp4_with_static_gs")
def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):
if tensor is None or tensor.numel() == 0:
@@ -115,6 +142,29 @@ def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, *
return qdq_res.to(orig_dtype), scale, None
+@register_dtype("rtn_nv_fp4_with_static_gs")
+def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):
+ if tensor is None or tensor.numel() == 0:
+ return tensor, None, None
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor_max is None:
+ tensor_max = tensor.abs().max().to(torch.float32)
+ else:
+ if not isinstance(tensor_max, torch.Tensor):
+ tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)
+ else:
+ tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32)
+ if tensor_max.numel() != 1:
+ tensor_max = tensor_max.abs().max()
+
+ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_E4M3_MAX).mul_(FLOAT4_E2M1_MAX)
+ global_scale = global_scale.to(tensor.device)
+ qdq_res, scale = ref_nvfp4_quant_rtn(tensor, global_scale, group_size, v)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
FLOAT8_UE5M3_MAX = 114688
@@ -206,6 +256,20 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+def ref_fp4_quant_rtn(x, global_scale, block_size=16, v=0, max_scale=1.0):
+ assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32
+ assert x.ndim == 2
+ m, n = x.shape
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.unsqueeze(dim=-1).to(x.device)
+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale
+ scale = vec_max.mul_(get_reciprocal(FLOAT4_E2M1_MAX)).mul_(global_scale).clamp_(0, FLOAT8_UE5M3_MAX)
+ scale = cast_to_ue5m3(scale).float()
+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
+ clipped_x = x.float().mul_(output_scale).add_(v).clamp_(-6.0, 6.0)
+ return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+
+
@register_dtype("fp4_v2_with_global_scale")
def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
@@ -224,6 +288,24 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None
return qdq_res.to(orig_dtype), scale, None
+@register_dtype("rtn_fp4_v2_with_global_scale")
+def rtn_fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):
+ assert group_size == 32 or group_size == 16
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor_max is None:
+ tensor_max = tensor.abs().max().to(torch.float32)
+ elif tensor_max is not None:
+ if not isinstance(tensor_max, torch.Tensor):
+ tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)
+ if tensor_max.numel() != 1:
+ tensor_max = tensor.abs().max().to(torch.float32)
+ global_scale = get_reciprocal(tensor_max).mul_(FLOAT8_UE5M3_MAX).mul_(FLOAT4_E2M1_MAX)
+ qdq_res, scale = ref_fp4_quant_rtn(tensor, global_scale, group_size, v)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
@register_dtype("fp4_v2")
def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
@@ -235,6 +317,17 @@ def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
return qdq_res.to(orig_dtype), scale, None
+@register_dtype("rtn_fp4_v2")
+def rtn_fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
+ assert group_size == 32 or group_size == 16
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ global_scale = 1.0
+ qdq_res, scale = ref_fp4_quant_rtn(tensor, global_scale, group_size, v, max_scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])
data1 = cast_to_fp4(data)
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index a8e3eb218..d82773c10 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -101,7 +101,9 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: Union[i
return data_new
-def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None) -> tuple[callable, str]:
+def get_quant_func(
+ dtype: str, bits: int, sym: bool, disable_opt_rtn=False, group_size=None, enable_rtn=False
+) -> tuple[callable, str]:
"""Retrieve the quantization function based on data type, bit width, and symmetry.
This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE
@@ -114,6 +116,7 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, grou
sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).
disable_opt_rtn(bool): whether to disable optimized rtn.
group_size (tuple): The block size for weight quantization (e.g., (128, 128)).
+ enable_rtn (bool): whether to use rtn.
Returns:
function: The quantization function corresponding to the specified parameters.
@@ -131,8 +134,13 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, grou
return data_type + str(bits)
if not disable_opt_rtn:
- rtn_data_type = "rtn_" + dtype
- data_types = [rtn_data_type, pad_bits(rtn_data_type), pad_sym(rtn_data_type), pad_sym(pad_bits(rtn_data_type))]
+ opt_rtn_data_type = "opt_rtn_" + dtype
+ data_types = [
+ opt_rtn_data_type,
+ pad_bits(opt_rtn_data_type),
+ pad_sym(opt_rtn_data_type),
+ pad_sym(pad_bits(opt_rtn_data_type)),
+ ]
for data_type in data_types:
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
@@ -140,19 +148,10 @@ def get_quant_func(dtype: str, bits: int, sym: bool, disable_opt_rtn=False, grou
return QUANT_FUNC_WITH_DTYPE[data_type], data_type
if group_size is not None and isinstance(group_size, tuple):
- block_data_type = "block_" + dtype
- data_types = [
- block_data_type,
- pad_bits(block_data_type),
- pad_sym(block_data_type),
- pad_sym(pad_bits(block_data_type)),
- ]
+ dtype = "block_" + dtype
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-
- for data_type in data_types:
- if data_type in QUANT_FUNC_WITH_DTYPE:
- return QUANT_FUNC_WITH_DTYPE[data_type], data_type
+ if enable_rtn:
+ dtype = "rtn_" + dtype
data_types = [dtype, pad_bits(dtype), pad_sym(dtype), pad_sym(pad_bits(dtype))]
for data_type in data_types:
diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.py
index b8e987d4c..8ebd5a592 100644
--- a/auto_round/data_type/w4fp8.py
+++ b/auto_round/data_type/w4fp8.py
@@ -14,6 +14,7 @@
import torch
+from auto_round.data_type.int import quant_tensor_rtn_sym, quant_tensor_sym
from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func
@@ -56,7 +57,6 @@ def progressive_quant_fp8_int4(
##convert to bf16
fp8_res_using_16bit = fp8_res.to(tensor.dtype)
##convert to int4
- from auto_round.data_type.int import quant_tensor_sym
qdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_sym(
fp8_res_using_16bit,
@@ -72,3 +72,57 @@ def progressive_quant_fp8_int4(
bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale
return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4
+
+
+@register_dtype("rtn_fp8_to_int_sym")
+def progressive_quant_rtn_fp8_int4(
+ tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, q_scale_thresh=1e-5, **kwargs
+):
+ """Two-stage quantization: quantize tensor to fp8 by per tensor, then quantize fp8 to w4g128
+
+ This method first quantizes the input tensor into float8 format and then performs
+ a secondary quantization to int4 with grouping.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ bits (int, optional): Bit precision for secondary quantization. Defaults to 4.
+ group_size (int, optional): Group size for int4 quantization. Defaults to -1 (no grouping).
+ v (float, optional): Optional parameter for variance tuning. Defaults to 0.
+ min_scale (float, optional): Minimum scaling factor for int4 quantization. Defaults to 1.0.
+ max_scale (float, optional): Maximum scaling factor for int4 quantization. Defaults to 1.0.
+ q_scale_thresh (float, optional): Threshold for scaling. Defaults to 1e-5.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Combined scaling factor (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor_max = torch.max(torch.abs(tensor)).to(torch.float32)
+ scale = tensor_max.to(torch.float32) / info.max
+ min_scaling_factor = 1.0 / (info.max * 512.0) ##copy from vllm
+ bf16_to_fp8_scale = tensor_max.div_(info.max).clamp_(min=min_scaling_factor)
+ fp8_res = tensor.div_(bf16_to_fp8_scale).clamp_(info.min, info.max).to(torch.float8_e4m3fn)
+
+ ##convert to bf16
+ fp8_res_using_16bit = fp8_res.to(orig_dtype)
+ ##convert to int4
+
+ qdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_rtn_sym(
+ fp8_res_using_16bit,
+ bits=bits,
+ group_size=group_size,
+ v=v,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=torch.bfloat16,
+ q_scale_thresh=q_scale_thresh,
+ )
+ qdq_tensor = qdq_int4_tensor * bf16_to_fp8_scale
+
+ bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale
+ return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index c64066932..d860192ec 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -652,10 +652,12 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i
)
else:
- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dq
blocks.reshape(blocks.shape[0], -1)
- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(
+ blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix
+ )
scales, d_scale = scales["scale"], scales["d_scale"]
mins, d_wmin = mins["wmin"], mins["d_wmin"]
blocks = blocks.reshape((nb, QK_K // 16, 16))
@@ -702,10 +704,10 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
all_L = blocks.mul_(reverse_qdq_scale.unsqueeze(-1)).round_().clamp_(-4, 3).add_(4).to(torch.uint8)
q_scales_offset = (qdq_scale * inverse_dq_scale).round_().clamp_(-32, 31).add_(32)
else:
- from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq
+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_sym_dq
blocks = blocks.reshape(blocks.shape[0], -1)
- blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, _ = quant_tensor_gguf_opt_rtn_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)
scales, d_scale = scales["scale"], scales["d_scale"]
blocks = blocks.reshape((nb, QK_K // 16, 16))
@@ -780,10 +782,12 @@ def q4_k_quant_block(
)
else:
- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dq
blocks.reshape(blocks.shape[0], -1)
- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(
+ blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix
+ )
scales, d_scale = scales["scale"], scales["d_scale"]
mins, d_wmin = mins["wmin"], mins["d_wmin"]
@@ -875,10 +879,12 @@ def q5_k_quant_block(
.to(torch.uint8)
)
else:
- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_asym_dq
blocks.reshape(blocks.shape[0], -1)
- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, mins = quant_tensor_gguf_opt_rtn_asym_dq(
+ blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix
+ )
scales, d_scale = scales["scale"], scales["d_scale"]
mins, d_wmin = mins["wmin"], mins["d_wmin"]
@@ -948,10 +954,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
blocks[replace_ids].div(d_tmp[replace_ids]).reshape(-1, 1).add_(32).round_().clamp_(0, 63).to(torch.uint8)
)
else:
- from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq
+ from auto_round.data_type.gguf import quant_tensor_gguf_opt_rtn_sym_dq
blocks = blocks.reshape(blocks.shape[0], -1)
- blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, _ = quant_tensor_gguf_opt_rtn_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)
scales, d_scale = scales["scale"], scales["d_scale"]
blocks = blocks.reshape((nb, QK_K // 16, 16))
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 2478db548..892b20eb4 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -80,6 +80,7 @@ class WrapperLinear(torch.nn.Module):
enable_round_tuning=True,
enable_torch_compile=False,
disable_opt_rtn=True,
+ enable_rtn=False,
**kwargs,
):
"""Initializes the WrapperLinear module.
@@ -93,6 +94,7 @@ class WrapperLinear(torch.nn.Module):
super(WrapperLinear, self).__init__()
self.orig_layer = orig_layer
self.disable_opt_rtn = disable_opt_rtn
+ self.enable_rtn = enable_rtn
self.output_device = device
self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.enable_minmax_tuning = enable_minmax_tuning
@@ -169,15 +171,25 @@ class WrapperLinear(torch.nn.Module):
self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
self.weight_quant_func, self.data_type = get_quant_func(
- orig_layer.data_type, orig_layer.bits, orig_layer.sym, self.disable_opt_rtn, orig_layer.group_size
+ orig_layer.data_type,
+ orig_layer.bits,
+ orig_layer.sym,
+ self.disable_opt_rtn,
+ orig_layer.group_size,
+ self.enable_rtn,
)
if self.enable_torch_compile:
self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
if self.enable_act_quant:
self.act_quant_func, self.act_data_type = get_quant_func(
- orig_layer.act_data_type, orig_layer.act_bits, orig_layer.act_sym, self.disable_opt_rtn
+ orig_layer.act_data_type,
+ orig_layer.act_bits,
+ orig_layer.act_sym,
+ self.disable_opt_rtn,
+ enable_rtn=self.enable_rtn,
)
+
if self.enable_torch_compile:
self.act_quant_func = compile_func(self.act_quant_func, self.device)
self._init_params("act_max_scale", p_dtype, (1), 1.0, not orig_layer.act_dynamic)