Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add initial support of mxfp4 (#187)

wenhuach21d48d040498be
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.py
new file mode 100644
index 000000000..2a68bcdb5
--- /dev/null
+++ b/auto_round/data_type/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import auto_round.data_type.int
+import auto_round.data_type.mxfp
+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.utils import get_quant_func
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
new file mode 100644
index 000000000..726a2b9c8
--- /dev/null
+++ b/auto_round/data_type/int.py
@@ -0,0 +1,122 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from .utils import floor_ste, round_ste
+from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE
+
+
+@register_dtype("int_asym")
+def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
+ weight_min=None, weight_max=None, q_scale_thresh=0.0,**kwargs):
+ """Quantizes and dequantizes weight asymmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2 ** num_bits - 1)
+ if isinstance(min_scale, torch.Tensor):
+ if weight_min is None or weight_max is None:
+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
+ else:
+ wmin_tmp = weight_min
+ wmax_tmp = weight_max
+ wmin_tmp = wmin_tmp * min_scale
+ wmax_tmp = wmax_tmp * max_scale
+ wmax = torch.maximum(wmax_tmp, wmin_tmp)
+ wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ else:
+ if weight_min is None or weight_max is None:
+ wmin = torch.clamp(weight.min(1)[0], max=0)
+ wmax = torch.clamp(weight.max(1)[0], min=0)
+ else:
+ wmin = weight_min
+ wmax = weight_max
+
+ tmp = (wmin == 0) & (wmax == 0)
+ wmin[tmp] = -1
+ wmax[tmp] = +1
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
+ zp = round_ste(-wmin / scale) # pylint: disable=E1130
+ scale = scale.unsqueeze(dim=-1)
+ zp = zp.unsqueeze(dim=-1)
+ int_w = round_ste(weight / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ return scale * (q - zp), scale, zp
+
+@register_dtype("int_sym")
+def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
+ weight_max=None, q_scale_thresh=0.0,**kargs):
+ """Quantizes and dequantizes weight symmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2 ** num_bits - 1)
+ if isinstance(min_scale, torch.Tensor):
+ if weight_min is None or weight_max is None:
+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
+ else:
+ wmin_tmp = weight_min
+ wmax_tmp = weight_max
+ wmin_tmp = wmin_tmp * min_scale
+ wmax_tmp = wmax_tmp * max_scale
+ wmax = torch.maximum(wmax_tmp, wmin_tmp)
+ wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ else:
+ if weight_min is None or weight_max is None:
+ wmin = torch.clamp(weight.min(1)[0], max=0)
+ wmax = torch.clamp(weight.max(1)[0], min=0)
+ else:
+ wmin = weight_min
+ wmax = weight_max
+
+ wmax_new = torch.max(wmin.abs(), wmax)
+ tmp = wmin < 0
+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
+ if torch.any(tmp):
+ wmin_new[tmp] = -wmax_new[tmp]
+
+ tmp = (wmin_new == 0) & (wmax_new == 0)
+ wmin_new[tmp] = -1
+ wmax_new[tmp] = +1
+ scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
+ scale = scale.unsqueeze(dim=-1)
+ zp = torch.full_like(scale, (maxq + 1) / 2)
+
+ int_w = round_ste(weight / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ return scale * (q - zp), scale, zp
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
new file mode 100644
index 000000000..efadf0c2b
--- /dev/null
+++ b/auto_round/data_type/mxfp.py
@@ -0,0 +1,103 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from .utils import floor_ste, round_ste
+from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE
+
+MXFP_FORMAT_CACHE = {
+ # data type: ebits, mbits, emax, max_norm, min_norm
+ "mx_int8": (0, 8, 0, 1.984375, 0),
+ "mx_int4": (0, 4, 0, 1.75, 0),
+ "mx_int2": (0, 2, 0, 1.0, 0),
+ "mx_fp8e5m2": (5, 4, 15, 57344.0, 6.103515625e-05),
+ "mx_fp8e4m3": (4, 5, 8, 448.0, 0.015625),
+ "mx_fp6e3m2": (3, 4, 4, 28.0, 0.25),
+ "mx_fp6e2m3": (2, 5, 2, 7.5, 1.0),
+ "mx_fp4": (2, 3, 2, 6.0, 1.0),
+ "mx_fp4e2m1": (2, 3, 2, 6.0, 1.0),
+ "mx_float16": (5, 12, 15, 65504.0, 6.103515625e-05),
+ "mx_fp16": (5, 12, 15, 65504.0, 6.103515625e-05),
+ "mx_bfloat16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),
+ "mx_bf16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),
+}
+
+FP32_EXPONENT_BIAS = 127
+FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)
+
+
+def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
+ """Quantize the given tensor using the specified parameters.
+
+ This function performs quantization on the `tensor` tensor according to the
+ given bit width (`bits`), data type (`data_type`), and additional parameters.
+ The quantization process involves scaling the tensor values and adjusting
+ the exponent and mantissa to fit within the specified format.
+
+ Args:
+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.
+ bits (int): The bit width to be used for quantization.
+ data_type (str): The data type for quantization (e.g., 'mx_fp4').
+ v (float): A value used for adjusting the tensors.
+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.
+
+ Returns:
+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).
+
+ Raises:
+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.
+ """
+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]
+ shared_exp, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
+ if isinstance(max_scale, torch.Tensor):
+ shared_exp *= (max_scale.unsqueeze(dim=-1))
+ else:
+ shared_exp *= max_scale
+ scale_emax = 2 ** (8 - 1) - 1
+ shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
+ shared_exp[shared_exp == torch.inf] = scale_emax + emax
+ shared_exp[shared_exp == -torch.inf] = -scale_emax + emax
+ shared_exp = (shared_exp - emax)
+ shared_exp = floor_ste(shared_exp)
+ shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan
+ shared_exp[shared_exp < -scale_emax] = -scale_emax
+ tensor = tensor / (2 ** shared_exp)
+ multiply = 2 if data_type == "mx_fp4" else 1 ## 2 is a tricky setting
+ tensor = tensor + v * multiply
+ if ebits != 0:
+ private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))
+
+ # The minimum representable exponent for 8 exp bits is -126
+ min_exp = -(2 ** (ebits - 1)) + 2
+ private_exp = private_exp.clip(min=min_exp)
+ else:
+ private_exp = None
+
+ # Scale up so appropriate number of mbits are in the integer portion of the number
+ tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))
+
+ tensor = torch.sign(tensor) * round_ste(torch.abs(tensor)) ##adopt round-to-floor which we found is much better
+ max_mantissa = 2 ** (mbits - 1) - 1
+ tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)
+
+ # Undo scaling
+ tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)
+
+ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
+ tensor = tensor * (2 ** shared_exp)
+ return tensor, shared_exp, None
+
+
+for key in MXFP_FORMAT_CACHE.keys():
+ QUANT_FUNC_WITH_DTYPE[key] = quant_mx
diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.py
new file mode 100644
index 000000000..1a3b63289
--- /dev/null
+++ b/auto_round/data_type/register.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+QUANT_FUNC_WITH_DTYPE = {}
+
+
+def register_dtype(name):
+ """Class decorator to register a EXPORT subclass to the registry.
+
+ Decorator function used before a Pattern subclass.
+
+ Args:
+ cls (class): The subclass of register.
+ name: A string. Define the export type.
+
+ Returns:
+ cls: The class of register.
+ """
+
+ def register(dtype):
+ QUANT_FUNC_WITH_DTYPE[name] = dtype
+ return dtype
+
+ return register
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
new file mode 100644
index 000000000..594a78151
--- /dev/null
+++ b/auto_round/data_type/utils.py
@@ -0,0 +1,87 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
+
+
+def get_quant_func(dtype, bits, sym):
+ """Retrieve the quantization function based on data type, bit width, and symmetry.
+
+ This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE
+ dictionary based on the provided data type (`dtype`), bit width (`bits`), and whether
+ the quantization is symmetric (`sym`). If the function does not exist, it asserts False.
+
+ Args:
+ dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').
+ bits (int): The bit width for the quantization (e.g., 2,4,8).
+ sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).
+
+ Returns:
+ function: The quantization function corresponding to the specified parameters.
+ """
+ key = dtype
+ if key in QUANT_FUNC_WITH_DTYPE.keys():
+ return QUANT_FUNC_WITH_DTYPE[key], key
+
+ if sym:
+ key = dtype + "_sym"
+ else:
+ key = dtype + "_asym"
+
+ if key in QUANT_FUNC_WITH_DTYPE.keys():
+ return QUANT_FUNC_WITH_DTYPE[key], key
+
+ ##need to add bits
+ if sym:
+ key = dtype + str(bits) + "_sym"
+ else:
+ key = dtype + str(bits) + "_asym"
+
+ if key in QUANT_FUNC_WITH_DTYPE.keys():
+ return QUANT_FUNC_WITH_DTYPE[key], key
+
+ if sym:
+ key = dtype + str(bits)
+ else:
+ key = dtype + str(bits)
+
+ if key in QUANT_FUNC_WITH_DTYPE.keys():
+ return QUANT_FUNC_WITH_DTYPE[key], key
+
+ assert False, f"{dtype} is not supported"
+
+
+def round_ste(x: torch.Tensor):
+ """Straight-Through Estimator for rounding.
+
+ Args:
+ x: torch.Tensor
+
+ Returns:
+ torch.Tensor
+ """
+ return (x.round() - x).detach() + x
+
+
+def floor_ste(x: torch.Tensor):
+ """Straight-Through Estimator for floor.
+
+ Args:
+ x: torch.Tensor
+
+ Returns:
+ torch.Tensor
+ """
+ return (x.floor() - x).detach() + x
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 89ec18a08..cb5eddcfb 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -14,7 +14,7 @@
import torch
import transformers
-
+from auto_round.data_type import get_quant_func
from .utils import (
check_to_quantized,
get_scale_shape,
@@ -22,149 +22,6 @@ from .utils import (
)
-def round_ste(x: torch.Tensor):
- """Straight-Through Estimator for rounding.
- This function is adapted from omniquant.
-
- Args:
- x: torch.Tensor
-
- Returns:
- torch.Tensor
- """
- return (x.round() - x).detach() + x
-
-
-def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- weight_min=None, weight_max=None, q_scale_thresh=0.0):
- """Quantizes and dequantizes weight asymmetrically.
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- maxq = torch.tensor(2 ** num_bits - 1)
- if isinstance(min_scale, torch.Tensor):
- if weight_min is None or weight_max is None:
- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin_tmp = weight_min
- wmax_tmp = weight_max
- wmin_tmp = wmin_tmp * min_scale
- wmax_tmp = wmax_tmp * max_scale
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
- else:
- if weight_min is None or weight_max is None:
- wmin = torch.clamp(weight.min(1)[0], max=0)
- wmax = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin = weight_min
- wmax = weight_max
-
- tmp = (wmin == 0) & (wmax == 0)
- wmin[tmp] = -1
- wmax[tmp] = +1
- scale = ((wmax - wmin) / maxq).to(scale_dtype)
- scale = torch.clamp(scale, min=q_scale_thresh)
- zp = round_ste(-wmin / scale) # pylint: disable=E1130
- scale = scale.unsqueeze(dim=-1)
- zp = zp.unsqueeze(dim=-1)
- int_w = round_ste(weight / scale + v)
- q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
-
-
-def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
- weight_max=None, q_scale_thresh=0.0):
- """Quantizes and dequantizes weight symmetrically.
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- maxq = torch.tensor(2 ** num_bits - 1)
- if isinstance(min_scale, torch.Tensor):
- if weight_min is None or weight_max is None:
- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin_tmp = weight_min
- wmax_tmp = weight_max
- wmin_tmp = wmin_tmp * min_scale
- wmax_tmp = wmax_tmp * max_scale
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
- else:
- if weight_min is None or weight_max is None:
- wmin = torch.clamp(weight.min(1)[0], max=0)
- wmax = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin = weight_min
- wmax = weight_max
-
- wmax_new = torch.max(wmin.abs(), wmax)
- tmp = wmin < 0
- wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
- if torch.any(tmp):
- wmin_new[tmp] = -wmax_new[tmp]
-
- tmp = (wmin_new == 0) & (wmax_new == 0)
- wmin_new[tmp] = -1
- wmax_new[tmp] = +1
- scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
- scale = torch.clamp(scale, min=q_scale_thresh)
- scale = scale.unsqueeze(dim=-1)
- zp = torch.full_like(scale, (maxq + 1) / 2)
-
- int_w = round_ste(weight / scale + v)
- q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
-
-
-def quant_tensor_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,
- weight_max=None, q_scale_thresh=0.0):
- """Quantizes and dequantizes weight symmetrically or asymmetrically .
-
- Args:
- weight: Tensor containing the weight to be quantized
- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
- sym: Sym or asym
- v: Rounding value perturbation
- min_scale: Minimum scale coefficient for weight
- max_scale: Maximum scale coefficient for weight
- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
-
- Returns:
- Quantized and dequantized weight, scale, zero-point
- """
- assert num_bits > 0, "num_bits should be larger than 0"
- if sym:
- return quant_tensor_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,
- q_scale_thresh)
- else:
- return quant_tensor_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,
- q_scale_thresh)
-
-
def reshape_tensor(v, group_size=-1):
"""Reshapes the tensor based on the group size.
@@ -186,8 +43,8 @@ def reshape_tensor(v, group_size=-1):
def quant_tensor(
- data, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- weight_min=None, weight_max=None, q_scale_thresh=0.0
+ quant_func, data, num_bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
+ weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs,
):
"""Quantizes and dequantizes weight, handing the group size issue .
@@ -209,17 +66,17 @@ def quant_tensor(
if len(data.shape) > 2:
data = data.reshape(-1, orig_shape[-1])
if group_size == -1 or data.shape[1] < group_size:
- data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,
- scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
- q_scale_thresh=q_scale_thresh)
+ data, scale, zp = quant_func(data, num_bits, v=v, min_scale=min_scale, max_scale=max_scale,
+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
+ q_scale_thresh=q_scale_thresh, **kwargs)
data = data.reshape(orig_shape)
return data, scale, zp
if data.shape[1] % group_size == 0:
data = data.reshape(-1, group_size)
- data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,
- scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
- q_scale_thresh=q_scale_thresh)
+ data, scale, zp = quant_func(data, num_bits, v=v, min_scale=min_scale, max_scale=max_scale,
+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,
+ q_scale_thresh=q_scale_thresh, **kwargs)
data = data.reshape(orig_shape)
return data, scale, zp
@@ -228,9 +85,9 @@ def quant_tensor(
pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]
data_new = torch.nn.functional.pad(data, (0, pad_len))
data_new = data_new.reshape(-1, group_size)
- data_new, scale, zp = quant_tensor_actor(data_new, num_bits, sym=sym, v=v, min_scale=min_scale,
- max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,
- weight_max=weight_max, q_scale_thresh=q_scale_thresh)
+ data_new, scale, zp = quant_func(data_new, num_bits, v=v, min_scale=min_scale,
+ max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,
+ weight_max=weight_max, q_scale_thresh=q_scale_thresh, **kwargs)
data_new = data_new.reshape(tmp_shape[0], -1)
data_new = data_new[:, :-pad_len]
data_new = data_new.reshape(orig_shape)
@@ -241,10 +98,14 @@ class WrapperWALayer(torch.nn.Module):
def __init__(self, orig_layer):
super(WrapperWALayer, self).__init__()
self.orig_layer = orig_layer
+ self.act_quant_func = self.orig_layer.act_quant_func
def forward(self, x):
- x, _, _ = quant_tensor(x, self.orig_layer.act_bits, self.orig_layer.group_size, self.orig_layer.sym,
- scale_dtype=self.orig_layer.scale_dtype, q_scale_thresh=self.orig_layer.q_scale_thresh)
+ x, _, _ = quant_tensor(self.orig_layer.act_quant_func, x, self.orig_layer.act_bits,
+ self.orig_layer.group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.orig_layer.q_scale_thresh,
+ data_type=self.orig_layer.data_type)
return self.orig_layer.forward(x)
@@ -273,11 +134,16 @@ class WrapperLinear(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.scale_dtype = self.orig_layer.scale_dtype
self.sym = self.orig_layer.sym
+ self.data_type = self.orig_layer.data_type
+ self.weight_quant_func, self.data_type = get_quant_func(self.data_type, self.num_bits, self.sym)
self.act_bits = self.orig_layer.act_bits
self.act_group_size = self.orig_layer.act_group_size
self.act_sym = self.orig_layer.act_sym
self.act_dynamic = self.orig_layer.act_dynamic
self.act_quant = self.act_bits <= 8
+ if self.act_quant:
+ self.act_quant_func, _ = get_quant_func(self.data_type, self.act_bits, self.act_sym)
+
self.q_scale_thresh = 1e-5
weight_dtype = torch.float32
@@ -321,8 +187,10 @@ class WrapperLinear(torch.nn.Module):
if self.orig_layer.weight.device.type == 'meta':
self.orig_layer.to(self.device)
- qdq_weight, scale, zp = quant_tensor(self.orig_layer.weight, self.num_bits, self.group_size, self.sym, v,
- min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max)
+ qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.orig_layer.weight, self.num_bits,
+ self.group_size, v,
+ min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max,
+ data_type=self.data_type)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
@@ -331,10 +199,12 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.scale = scale.to("cpu")
self.orig_layer.zp = zp.to("cpu") if zp is not None else None
self.orig_layer.q_scale_thresh = self.q_scale_thresh
+ self.orig_layer.data_type = self.data_type
if hasattr(self.orig_layer, 'update'):
self.orig_layer.update()
self.orig_layer.to('meta')
if self.act_quant:
+ self.orig_layer.act_quant_func = self.act_quant_func
wrapper_layer = WrapperWALayer(self.orig_layer)
return wrapper_layer
return self.orig_layer
@@ -355,12 +225,15 @@ class WrapperLinear(torch.nn.Module):
weight = self.orig_layer.get_weight().to(self.device)
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, 0, 1.0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, 0, 1.0))
- weight_q, _, _ = quant_tensor(weight, self.num_bits, self.group_size, self.sym, self.value, self.min_scale,
- self.max_scale, self.scale_dtype, self.weight_min, self.weight_max)
+ weight_q, _, _ = quant_tensor(self.weight_quant_func, weight, self.num_bits, self.group_size, self.value,
+ self.min_scale,
+ self.max_scale, self.scale_dtype, self.weight_min, self.weight_max,
+ data_type=self.data_type)
weight_q = weight_q.to(weight.dtype)
if self.act_quant:
- x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,
- scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)
+ x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,
+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh,
+ data_type=self.data_type)
# pylint: disable=not-callable
bias = self.orig_layer.bias
if bias is not None and bias.device.type == 'meta':
@@ -397,11 +270,16 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.sym = self.orig_layer.sym
self.scale_dtype = self.orig_layer.scale_dtype
+ self.data_type = self.orig_layer.data_type
self.act_bits = self.orig_layer.act_bits
self.act_group_size = self.orig_layer.act_group_size
self.act_sym = self.orig_layer.act_sym
self.act_dynamic = self.orig_layer.act_dynamic
self.act_quant = self.act_bits <= 8
+ self.weight_quant_func, self.data_type = get_quant_func(self.data_type, self.num_bits, self.sym)
+ if self.act_quant:
+ self.act_quant_func, _ = get_quant_func(self.data_type, self.act_bits, self.act_sym)
+
self.q_scale_thresh = 1e-5
weight_dtype = torch.float32
self.device = device
@@ -445,8 +323,10 @@ class WrapperTransformerConv1d(torch.nn.Module):
"""
min_scale.clamp_(0, 1.0)
max_scale.clamp_(0, 1.0)
- qdq_weight, scale, zp = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale,
- max_scale, self.scale_dtype, self.weight_min, self.weight_max)
+ qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.weight_t, self.num_bits, self.group_size, v,
+ min_scale,
+ max_scale, self.scale_dtype, self.weight_min, self.weight_max,
+ data_type=self.data_type)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
@@ -457,7 +337,9 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.orig_layer.scale = scale.to("cpu")
self.orig_layer.zp = zp.to("cpu")
self.orig_layer.q_scale_thresh = self.q_scale_thresh
+ self.orig_layer.data_type = self.data_type
if self.act_quant:
+ self.orig_layer.act_quant_func = self.act_quant_func
wrapper_layer = WrapperWALayer(self.orig_layer)
return wrapper_layer
if hasattr(self.orig_layer, 'update'):
@@ -477,14 +359,15 @@ class WrapperTransformerConv1d(torch.nn.Module):
with torch.no_grad():
self.min_scale.clamp_(0, 1.0)
self.max_scale.clamp_(0, 1.0)
- weight_q, _, _ = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, self.value,
+ weight_q, _, _ = quant_tensor(self.weight_quant_func, self.weight_t, self.num_bits, self.group_size, self.value,
self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,
- self.weight_max)
+ self.weight_max, data_type=self.data_type)
weight_q = weight_q.to(self.weight_t.dtype)
size_out = x.size()[:-1] + (self.orig_layer.nf,)
if self.act_quant:
- x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,
- scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)
+ x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,
+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh,
+ data_type=self.data_type)
x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())
x = x.view(*size_out)
return x
diff --git a/auto_round/utils.py b/auto_round/utils.py
index aae086a75..43f1043b1 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -82,6 +82,7 @@ auto_gptq = LazyImport("auto_gptq")
htcore = LazyImport("habana_frameworks.torch.core")
+
def is_optimum_habana_available():
from transformers.utils.import_utils import is_optimum_available
diff --git a/auto_round/version.py b/auto_round/version.py
index b62be5f3b..c7c0b0e2a 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -14,4 +14,4 @@
"""Intel® auto-round: An open-source Python library
supporting popular model weight only compression based on signround."""
-__version__ = "0.2.1.dev"
+__version__ = "0.3.0.dev"
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 5f74de672..942a51063 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -87,6 +87,10 @@ if __name__ == '__main__':
help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'."
"default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
+ parser.add_argument("--data_type", default='int',
+ help="data type for tuning, 'int', 'mx_fp' and etc.")
+
+
parser.add_argument("--scale_dtype", default='fp16',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
@@ -118,6 +122,7 @@ if __name__ == '__main__':
parser.add_argument("--low_cpu_mem_mode", default=0, type=int,
help="choose low cpu memory mode, 1 for block-wise, 2 for layer-wise, others means not use low cpu memory.")
+
parser.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
@@ -144,12 +149,21 @@ if __name__ == '__main__':
tasks = args.tasks
use_eval_legacy = False
+ if "gpu" in args.deployment_device and args.sym is False:
+ print("warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")
+
if "marlin" in args.deployment_device and args.sym is False:
assert False, "marlin backend only supports sym quantization, please set --sym"
if args.act_bits <= 8 and args.deployment_device != "fake":
assert False, "only support fake mode for activation quantization currently"
+ if "mx_fp" in args.data_type and args.deployment_device != "fake":
+ assert False, "only support fake mode for mx_fp data type currently"
+
+ if "mx_fp" in args.data_type and args.group_size != 32:
+ print("warning, mx_fp should only support group_size of 32 in real deployment")
+
model_name = args.model_name
if model_name[-1] == "/":
model_name = model_name[:-1]
@@ -281,7 +295,7 @@ if __name__ == '__main__':
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, layer_config=layer_config,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
- low_cpu_mem_usage=low_cpu_mem_usage)
+ low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type)
model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 8b8e9a8e2..0d454ed59 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -34,12 +34,12 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("runs", ignore_errors=True)
def test_remove_whole_block(self):
- layer_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},
- "model.decoder.layers.0.self_attn.v_proj": {"data_type": "float"},
- "model.decoder.layers.0.self_attn.q_proj": {"data_type": "float"},
- "model.decoder.layers.0.self_attn.out_proj": {"data_type": "float"},
- "model.decoder.layers.0.fc1": {"data_type": "float"},
- "model.decoder.layers.0.fc2": {"data_type": "float"},
+ layer_config={"model.decoder.layers.0.self_attn.k_proj":{"bits":32},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},
+ "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},
+ "model.decoder.layers.0.fc1": {"bits": 32},
+ "model.decoder.layers.0.fc2": {"bits": 32},
}
bits, group_size, sym = 4, 128, False
autoround = AutoRound(
@@ -55,6 +55,21 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_mx_fp4(self):
+ bits, group_size, sym = 4, 32, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ data_type="mx_fp4"
+ )
+ autoround.quantize()
+
def test_default(self):
bits, group_size, sym = 4, 128, False