Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

change to even rounding for mantissa of mx_fp (#277)

wenhuach2198a9c755f9a7
diff --git a/README.md b/README.md
index 8c6e556c8..e9ee5d9a5 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ of models. AutoRound adopts sign gradient descent to fine-tune rounding values a
steps,
which competes impressively against recent methods without introducing any additional inference overhead and keeping low
tuning cost. The below
-image presents an overview of AutoRound. Check out our paper on [arxiv](https://arxiv.org/pdf/2309.05516v4) for more
+image presents an overview of AutoRound. Check out our paper on [arxiv](https://arxiv.org/pdf/2309.05516) for more
details and visit [low_bit_open_llm_leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard) for
more accuracy data and recipes across various models.
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index c95db837c..f2ddfbde0 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -38,7 +38,7 @@ FP32_EXPONENT_BIAS = 127
FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)
-def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
+def quant_mx(tensor, bits, data_type, v, max_scale, mantissa_rounding="even", **kwargs):
"""Quantize the given tensor using the specified parameters.
This function performs quantization on the `tensor` tensor according to the
@@ -52,6 +52,7 @@ def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
data_type (str): The data type for quantization (e.g., 'mx_fp4').
v (float): A value used for adjusting the tensors.
max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.
+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor
Returns:
tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).
@@ -74,7 +75,7 @@ def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan
shared_exp[shared_exp < -scale_emax] = -scale_emax
tensor = tensor / (2 ** shared_exp)
- is_mx_fp4 = data_type == "mx_fp4" or ("mx_fp" in data_type and bits==4)
+ is_mx_fp4 = data_type == "mx_fp4" or ("mx_fp" in data_type and bits == 4)
multiply = 2 if is_mx_fp4 else 1 ## 2 is a tricky setting
tensor = tensor + v * multiply
if ebits != 0:
@@ -89,7 +90,16 @@ def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
# Scale up so appropriate number of mbits are in the integer portion of the number
tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))
- tensor = torch.sign(tensor) * round_ste(torch.abs(tensor)) ##adopt round-to-floor which we found is much better
+ if mantissa_rounding == "even":
+ abs_tensor = torch.abs(tensor)
+ mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)
+ tensor = torch.sign(tensor) * (floor_ste(abs_tensor + 0.5) - mask_tensor)
+ elif mantissa_rounding == "nearest":
+ tensor = round_ste(tensor)
+ elif mantissa_rounding == "floor":
+ tensor = floor_ste(tensor)
+ else:
+ raise ValueError("mantissa_rounding only supports even, nearest or floor.")
max_mantissa = 2 ** (mbits - 1) - 1
tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)