PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add initial support of mxfp4 (#187)
wenhuach21d48d040498be
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pynew file mode 100644index 000000000..2a68bcdb5--- /dev/null+++ b/auto_round/data_type/__init__.py@@ -0,0 +1,18 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import auto_round.data_type.int+import auto_round.data_type.mxfp+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.utils import get_quant_funcdiff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pynew file mode 100644index 000000000..726a2b9c8--- /dev/null+++ b/auto_round/data_type/int.py@@ -0,0 +1,122 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+from .utils import floor_ste, round_ste+from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE+++@register_dtype("int_asym")+def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None, q_scale_thresh=0.0,**kwargs):+ """Quantizes and dequantizes weight asymmetrically.++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ maxq = torch.tensor(2 ** num_bits - 1)+ if isinstance(min_scale, torch.Tensor):+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_max+ wmin_tmp = wmin_tmp * min_scale+ wmax_tmp = wmax_tmp * max_scale+ wmax = torch.maximum(wmax_tmp, wmin_tmp)+ wmin = torch.minimum(wmax_tmp, wmin_tmp)+ else:+ if weight_min is None or weight_max is None:+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin = weight_min+ wmax = weight_max++ tmp = (wmin == 0) & (wmax == 0)+ wmin[tmp] = -1+ wmax[tmp] = +1+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ zp = round_ste(-wmin / scale) # pylint: disable=E1130+ scale = scale.unsqueeze(dim=-1)+ zp = zp.unsqueeze(dim=-1)+ int_w = round_ste(weight / scale + v)+ q = torch.clamp(int_w + zp, 0, maxq)+ return scale * (q - zp), scale, zp++@register_dtype("int_sym")+def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,+ weight_max=None, q_scale_thresh=0.0,**kargs):+ """Quantizes and dequantizes weight symmetrically.++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ maxq = torch.tensor(2 ** num_bits - 1)+ if isinstance(min_scale, torch.Tensor):+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_max+ wmin_tmp = wmin_tmp * min_scale+ wmax_tmp = wmax_tmp * max_scale+ wmax = torch.maximum(wmax_tmp, wmin_tmp)+ wmin = torch.minimum(wmax_tmp, wmin_tmp)+ else:+ if weight_min is None or weight_max is None:+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin = weight_min+ wmax = weight_max++ wmax_new = torch.max(wmin.abs(), wmax)+ tmp = wmin < 0+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur+ if torch.any(tmp):+ wmin_new[tmp] = -wmax_new[tmp]++ tmp = (wmin_new == 0) & (wmax_new == 0)+ wmin_new[tmp] = -1+ wmax_new[tmp] = +1+ scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ scale = scale.unsqueeze(dim=-1)+ zp = torch.full_like(scale, (maxq + 1) / 2)++ int_w = round_ste(weight / scale + v)+ q = torch.clamp(int_w + zp, 0, maxq)+ return scale * (q - zp), scale, zpdiff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pynew file mode 100644index 000000000..efadf0c2b--- /dev/null+++ b/auto_round/data_type/mxfp.py@@ -0,0 +1,103 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+from .utils import floor_ste, round_ste+from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE++MXFP_FORMAT_CACHE = {+ # data type: ebits, mbits, emax, max_norm, min_norm+ "mx_int8": (0, 8, 0, 1.984375, 0),+ "mx_int4": (0, 4, 0, 1.75, 0),+ "mx_int2": (0, 2, 0, 1.0, 0),+ "mx_fp8e5m2": (5, 4, 15, 57344.0, 6.103515625e-05),+ "mx_fp8e4m3": (4, 5, 8, 448.0, 0.015625),+ "mx_fp6e3m2": (3, 4, 4, 28.0, 0.25),+ "mx_fp6e2m3": (2, 5, 2, 7.5, 1.0),+ "mx_fp4": (2, 3, 2, 6.0, 1.0),+ "mx_fp4e2m1": (2, 3, 2, 6.0, 1.0),+ "mx_float16": (5, 12, 15, 65504.0, 6.103515625e-05),+ "mx_fp16": (5, 12, 15, 65504.0, 6.103515625e-05),+ "mx_bfloat16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),+ "mx_bf16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),+}++FP32_EXPONENT_BIAS = 127+FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)+++def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):+ """Quantize the given tensor using the specified parameters.++ This function performs quantization on the `tensor` tensor according to the+ given bit width (`bits`), data type (`data_type`), and additional parameters.+ The quantization process involves scaling the tensor values and adjusting+ the exponent and mantissa to fit within the specified format.++ Args:+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.+ bits (int): The bit width to be used for quantization.+ data_type (str): The data type for quantization (e.g., 'mx_fp4').+ v (float): A value used for adjusting the tensors.+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.++ Returns:+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).++ Raises:+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.+ """+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]+ shared_exp, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)+ if isinstance(max_scale, torch.Tensor):+ shared_exp *= (max_scale.unsqueeze(dim=-1))+ else:+ shared_exp *= max_scale+ scale_emax = 2 ** (8 - 1) - 1+ shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp[shared_exp == torch.inf] = scale_emax + emax+ shared_exp[shared_exp == -torch.inf] = -scale_emax + emax+ shared_exp = (shared_exp - emax)+ shared_exp = floor_ste(shared_exp)+ shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan+ shared_exp[shared_exp < -scale_emax] = -scale_emax+ tensor = tensor / (2 ** shared_exp)+ multiply = 2 if data_type == "mx_fp4" else 1 ## 2 is a tricky setting+ tensor = tensor + v * multiply+ if ebits != 0:+ private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))++ # The minimum representable exponent for 8 exp bits is -126+ min_exp = -(2 ** (ebits - 1)) + 2+ private_exp = private_exp.clip(min=min_exp)+ else:+ private_exp = None++ # Scale up so appropriate number of mbits are in the integer portion of the number+ tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))++ tensor = torch.sign(tensor) * round_ste(torch.abs(tensor)) ##adopt round-to-floor which we found is much better+ max_mantissa = 2 ** (mbits - 1) - 1+ tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)++ # Undo scaling+ tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)++ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)+ tensor = tensor * (2 ** shared_exp)+ return tensor, shared_exp, None+++for key in MXFP_FORMAT_CACHE.keys():+ QUANT_FUNC_WITH_DTYPE[key] = quant_mxdiff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pynew file mode 100644index 000000000..1a3b63289--- /dev/null+++ b/auto_round/data_type/register.py@@ -0,0 +1,36 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++QUANT_FUNC_WITH_DTYPE = {}+++def register_dtype(name):+ """Class decorator to register a EXPORT subclass to the registry.++ Decorator function used before a Pattern subclass.++ Args:+ cls (class): The subclass of register.+ name: A string. Define the export type.++ Returns:+ cls: The class of register.+ """++ def register(dtype):+ QUANT_FUNC_WITH_DTYPE[name] = dtype+ return dtype++ return registerdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pynew file mode 100644index 000000000..594a78151--- /dev/null+++ b/auto_round/data_type/utils.py@@ -0,0 +1,87 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE+++def get_quant_func(dtype, bits, sym):+ """Retrieve the quantization function based on data type, bit width, and symmetry.++ This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE+ dictionary based on the provided data type (`dtype`), bit width (`bits`), and whether+ the quantization is symmetric (`sym`). If the function does not exist, it asserts False.++ Args:+ dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').+ bits (int): The bit width for the quantization (e.g., 2,4,8).+ sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).++ Returns:+ function: The quantization function corresponding to the specified parameters.+ """+ key = dtype+ if key in QUANT_FUNC_WITH_DTYPE.keys():+ return QUANT_FUNC_WITH_DTYPE[key], key++ if sym:+ key = dtype + "_sym"+ else:+ key = dtype + "_asym"++ if key in QUANT_FUNC_WITH_DTYPE.keys():+ return QUANT_FUNC_WITH_DTYPE[key], key++ ##need to add bits+ if sym:+ key = dtype + str(bits) + "_sym"+ else:+ key = dtype + str(bits) + "_asym"++ if key in QUANT_FUNC_WITH_DTYPE.keys():+ return QUANT_FUNC_WITH_DTYPE[key], key++ if sym:+ key = dtype + str(bits)+ else:+ key = dtype + str(bits)++ if key in QUANT_FUNC_WITH_DTYPE.keys():+ return QUANT_FUNC_WITH_DTYPE[key], key++ assert False, f"{dtype} is not supported"+++def round_ste(x: torch.Tensor):+ """Straight-Through Estimator for rounding.++ Args:+ x: torch.Tensor++ Returns:+ torch.Tensor+ """+ return (x.round() - x).detach() + x+++def floor_ste(x: torch.Tensor):+ """Straight-Through Estimator for floor.++ Args:+ x: torch.Tensor++ Returns:+ torch.Tensor+ """+ return (x.floor() - x).detach() + xdiff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 89ec18a08..cb5eddcfb 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -14,7 +14,7 @@import torchimport transformers-+from auto_round.data_type import get_quant_funcfrom .utils import (check_to_quantized,get_scale_shape,@@ -22,149 +22,6 @@ from .utils import ()-def round_ste(x: torch.Tensor):- """Straight-Through Estimator for rounding.- This function is adapted from omniquant.-- Args:- x: torch.Tensor-- Returns:- torch.Tensor- """- return (x.round() - x).detach() + x---def quant_tensor_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- weight_min=None, weight_max=None, q_scale_thresh=0.0):- """Quantizes and dequantizes weight asymmetrically.-- Args:- weight: Tensor containing the weight to be quantized- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- v: Rounding value perturbation- min_scale: Minimum scale coefficient for weight- max_scale: Maximum scale coefficient for weight- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.-- Returns:- Quantized and dequantized weight, scale, zero-point- """- maxq = torch.tensor(2 ** num_bits - 1)- if isinstance(min_scale, torch.Tensor):- if weight_min is None or weight_max is None:- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- else:- wmin_tmp = weight_min- wmax_tmp = weight_max- wmin_tmp = wmin_tmp * min_scale- wmax_tmp = wmax_tmp * max_scale- wmax = torch.maximum(wmax_tmp, wmin_tmp)- wmin = torch.minimum(wmax_tmp, wmin_tmp)- else:- if weight_min is None or weight_max is None:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)- else:- wmin = weight_min- wmax = weight_max-- tmp = (wmin == 0) & (wmax == 0)- wmin[tmp] = -1- wmax[tmp] = +1- scale = ((wmax - wmin) / maxq).to(scale_dtype)- scale = torch.clamp(scale, min=q_scale_thresh)- zp = round_ste(-wmin / scale) # pylint: disable=E1130- scale = scale.unsqueeze(dim=-1)- zp = zp.unsqueeze(dim=-1)- int_w = round_ste(weight / scale + v)- q = torch.clamp(int_w + zp, 0, maxq)- return scale * (q - zp), scale, zp---def quant_tensor_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,- weight_max=None, q_scale_thresh=0.0):- """Quantizes and dequantizes weight symmetrically.-- Args:- weight: Tensor containing the weight to be quantized- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- v: Rounding value perturbation- min_scale: Minimum scale coefficient for weight- max_scale: Maximum scale coefficient for weight- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.-- Returns:- Quantized and dequantized weight, scale, zero-point- """- maxq = torch.tensor(2 ** num_bits - 1)- if isinstance(min_scale, torch.Tensor):- if weight_min is None or weight_max is None:- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- else:- wmin_tmp = weight_min- wmax_tmp = weight_max- wmin_tmp = wmin_tmp * min_scale- wmax_tmp = wmax_tmp * max_scale- wmax = torch.maximum(wmax_tmp, wmin_tmp)- wmin = torch.minimum(wmax_tmp, wmin_tmp)- else:- if weight_min is None or weight_max is None:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)- else:- wmin = weight_min- wmax = weight_max-- wmax_new = torch.max(wmin.abs(), wmax)- tmp = wmin < 0- wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur- if torch.any(tmp):- wmin_new[tmp] = -wmax_new[tmp]-- tmp = (wmin_new == 0) & (wmax_new == 0)- wmin_new[tmp] = -1- wmax_new[tmp] = +1- scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)- scale = torch.clamp(scale, min=q_scale_thresh)- scale = scale.unsqueeze(dim=-1)- zp = torch.full_like(scale, (maxq + 1) / 2)-- int_w = round_ste(weight / scale + v)- q = torch.clamp(int_w + zp, 0, maxq)- return scale * (q - zp), scale, zp---def quant_tensor_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,- weight_max=None, q_scale_thresh=0.0):- """Quantizes and dequantizes weight symmetrically or asymmetrically .-- Args:- weight: Tensor containing the weight to be quantized- num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- sym: Sym or asym- v: Rounding value perturbation- min_scale: Minimum scale coefficient for weight- max_scale: Maximum scale coefficient for weight- weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.- weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.-- Returns:- Quantized and dequantized weight, scale, zero-point- """- assert num_bits > 0, "num_bits should be larger than 0"- if sym:- return quant_tensor_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,- q_scale_thresh)- else:- return quant_tensor_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max,- q_scale_thresh)--def reshape_tensor(v, group_size=-1):"""Reshapes the tensor based on the group size.@@ -186,8 +43,8 @@ def reshape_tensor(v, group_size=-1):def quant_tensor(- data, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- weight_min=None, weight_max=None, q_scale_thresh=0.0+ quant_func, data, num_bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs,):"""Quantizes and dequantizes weight, handing the group size issue .@@ -209,17 +66,17 @@ def quant_tensor(if len(data.shape) > 2:data = data.reshape(-1, orig_shape[-1])if group_size == -1 or data.shape[1] < group_size:- data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,- scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,- q_scale_thresh=q_scale_thresh)+ data, scale, zp = quant_func(data, num_bits, v=v, min_scale=min_scale, max_scale=max_scale,+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,+ q_scale_thresh=q_scale_thresh, **kwargs)data = data.reshape(orig_shape)return data, scale, zpif data.shape[1] % group_size == 0:data = data.reshape(-1, group_size)- data, scale, zp = quant_tensor_actor(data, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale,- scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,- q_scale_thresh=q_scale_thresh)+ data, scale, zp = quant_func(data, num_bits, v=v, min_scale=min_scale, max_scale=max_scale,+ scale_dtype=scale_dtype, weight_min=weight_min, weight_max=weight_max,+ q_scale_thresh=q_scale_thresh, **kwargs)data = data.reshape(orig_shape)return data, scale, zp@@ -228,9 +85,9 @@ def quant_tensor(pad_len = (data.shape[1] + group_size - 1) // group_size * group_size - data.shape[1]data_new = torch.nn.functional.pad(data, (0, pad_len))data_new = data_new.reshape(-1, group_size)- data_new, scale, zp = quant_tensor_actor(data_new, num_bits, sym=sym, v=v, min_scale=min_scale,- max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,- weight_max=weight_max, q_scale_thresh=q_scale_thresh)+ data_new, scale, zp = quant_func(data_new, num_bits, v=v, min_scale=min_scale,+ max_scale=max_scale, scale_dtype=scale_dtype, weight_min=weight_min,+ weight_max=weight_max, q_scale_thresh=q_scale_thresh, **kwargs)data_new = data_new.reshape(tmp_shape[0], -1)data_new = data_new[:, :-pad_len]data_new = data_new.reshape(orig_shape)@@ -241,10 +98,14 @@ class WrapperWALayer(torch.nn.Module):def __init__(self, orig_layer):super(WrapperWALayer, self).__init__()self.orig_layer = orig_layer+ self.act_quant_func = self.orig_layer.act_quant_funcdef forward(self, x):- x, _, _ = quant_tensor(x, self.orig_layer.act_bits, self.orig_layer.group_size, self.orig_layer.sym,- scale_dtype=self.orig_layer.scale_dtype, q_scale_thresh=self.orig_layer.q_scale_thresh)+ x, _, _ = quant_tensor(self.orig_layer.act_quant_func, x, self.orig_layer.act_bits,+ self.orig_layer.group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.data_type)return self.orig_layer.forward(x)@@ -273,11 +134,16 @@ class WrapperLinear(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.scale_dtype = self.orig_layer.scale_dtypeself.sym = self.orig_layer.sym+ self.data_type = self.orig_layer.data_type+ self.weight_quant_func, self.data_type = get_quant_func(self.data_type, self.num_bits, self.sym)self.act_bits = self.orig_layer.act_bitsself.act_group_size = self.orig_layer.act_group_sizeself.act_sym = self.orig_layer.act_symself.act_dynamic = self.orig_layer.act_dynamicself.act_quant = self.act_bits <= 8+ if self.act_quant:+ self.act_quant_func, _ = get_quant_func(self.data_type, self.act_bits, self.act_sym)+self.q_scale_thresh = 1e-5weight_dtype = torch.float32@@ -321,8 +187,10 @@ class WrapperLinear(torch.nn.Module):if self.orig_layer.weight.device.type == 'meta':self.orig_layer.to(self.device)- qdq_weight, scale, zp = quant_tensor(self.orig_layer.weight, self.num_bits, self.group_size, self.sym, v,- min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max)+ qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.orig_layer.weight, self.num_bits,+ self.group_size, v,+ min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max,+ data_type=self.data_type)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)@@ -331,10 +199,12 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu") if zp is not None else Noneself.orig_layer.q_scale_thresh = self.q_scale_thresh+ self.orig_layer.data_type = self.data_typeif hasattr(self.orig_layer, 'update'):self.orig_layer.update()self.orig_layer.to('meta')if self.act_quant:+ self.orig_layer.act_quant_func = self.act_quant_funcwrapper_layer = WrapperWALayer(self.orig_layer)return wrapper_layerreturn self.orig_layer@@ -355,12 +225,15 @@ class WrapperLinear(torch.nn.Module):weight = self.orig_layer.get_weight().to(self.device)self.min_scale.data.copy_(torch.clamp(self.min_scale.data, 0, 1.0))self.max_scale.data.copy_(torch.clamp(self.max_scale.data, 0, 1.0))- weight_q, _, _ = quant_tensor(weight, self.num_bits, self.group_size, self.sym, self.value, self.min_scale,- self.max_scale, self.scale_dtype, self.weight_min, self.weight_max)+ weight_q, _, _ = quant_tensor(self.weight_quant_func, weight, self.num_bits, self.group_size, self.value,+ self.min_scale,+ self.max_scale, self.scale_dtype, self.weight_min, self.weight_max,+ data_type=self.data_type)weight_q = weight_q.to(weight.dtype)if self.act_quant:- x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,- scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)+ x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh,+ data_type=self.data_type)# pylint: disable=not-callablebias = self.orig_layer.biasif bias is not None and bias.device.type == 'meta':@@ -397,11 +270,16 @@ class WrapperTransformerConv1d(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.sym = self.orig_layer.symself.scale_dtype = self.orig_layer.scale_dtype+ self.data_type = self.orig_layer.data_typeself.act_bits = self.orig_layer.act_bitsself.act_group_size = self.orig_layer.act_group_sizeself.act_sym = self.orig_layer.act_symself.act_dynamic = self.orig_layer.act_dynamicself.act_quant = self.act_bits <= 8+ self.weight_quant_func, self.data_type = get_quant_func(self.data_type, self.num_bits, self.sym)+ if self.act_quant:+ self.act_quant_func, _ = get_quant_func(self.data_type, self.act_bits, self.act_sym)+self.q_scale_thresh = 1e-5weight_dtype = torch.float32self.device = device@@ -445,8 +323,10 @@ class WrapperTransformerConv1d(torch.nn.Module):"""min_scale.clamp_(0, 1.0)max_scale.clamp_(0, 1.0)- qdq_weight, scale, zp = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale,- max_scale, self.scale_dtype, self.weight_min, self.weight_max)+ qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.weight_t, self.num_bits, self.group_size, v,+ min_scale,+ max_scale, self.scale_dtype, self.weight_min, self.weight_max,+ data_type=self.data_type)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)@@ -457,7 +337,9 @@ class WrapperTransformerConv1d(torch.nn.Module):self.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu")self.orig_layer.q_scale_thresh = self.q_scale_thresh+ self.orig_layer.data_type = self.data_typeif self.act_quant:+ self.orig_layer.act_quant_func = self.act_quant_funcwrapper_layer = WrapperWALayer(self.orig_layer)return wrapper_layerif hasattr(self.orig_layer, 'update'):@@ -477,14 +359,15 @@ class WrapperTransformerConv1d(torch.nn.Module):with torch.no_grad():self.min_scale.clamp_(0, 1.0)self.max_scale.clamp_(0, 1.0)- weight_q, _, _ = quant_tensor(self.weight_t, self.num_bits, self.group_size, self.sym, self.value,+ weight_q, _, _ = quant_tensor(self.weight_quant_func, self.weight_t, self.num_bits, self.group_size, self.value,self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,- self.weight_max)+ self.weight_max, data_type=self.data_type)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)if self.act_quant:- x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,- scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)+ x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,+ scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh,+ data_type=self.data_type)x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())x = x.view(*size_out)return xdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex aae086a75..43f1043b1 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -82,6 +82,7 @@ auto_gptq = LazyImport("auto_gptq")htcore = LazyImport("habana_frameworks.torch.core")+def is_optimum_habana_available():from transformers.utils.import_utils import is_optimum_availablediff --git a/auto_round/version.py b/auto_round/version.pyindex b62be5f3b..c7c0b0e2a 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -14,4 +14,4 @@"""Intel® auto-round: An open-source Python librarysupporting popular model weight only compression based on signround."""-__version__ = "0.2.1.dev"+__version__ = "0.3.0.dev"diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 5f74de672..942a51063 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -87,6 +87,10 @@ if __name__ == '__main__':help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")+ parser.add_argument("--data_type", default='int',+ help="data type for tuning, 'int', 'mx_fp' and etc.")++parser.add_argument("--scale_dtype", default='fp16',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")@@ -118,6 +122,7 @@ if __name__ == '__main__':parser.add_argument("--low_cpu_mem_mode", default=0, type=int,help="choose low cpu memory mode, 1 for block-wise, 2 for layer-wise, others means not use low cpu memory.")+parser.add_argument("--model_dtype", default=None, type=str,help="force to convert the dtype, some backends supports fp16 dtype better")@@ -144,12 +149,21 @@ if __name__ == '__main__':tasks = args.tasksuse_eval_legacy = False+ if "gpu" in args.deployment_device and args.sym is False:+ print("warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")+if "marlin" in args.deployment_device and args.sym is False:assert False, "marlin backend only supports sym quantization, please set --sym"if args.act_bits <= 8 and args.deployment_device != "fake":assert False, "only support fake mode for activation quantization currently"+ if "mx_fp" in args.data_type and args.deployment_device != "fake":+ assert False, "only support fake mode for mx_fp data type currently"++ if "mx_fp" in args.data_type and args.group_size != 32:+ print("warning, mx_fp should only support group_size of 32 in real deployment")+model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]@@ -281,7 +295,7 @@ if __name__ == '__main__':seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, layer_config=layer_config,enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,- low_cpu_mem_usage=low_cpu_mem_usage)+ low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type)model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 8b8e9a8e2..0d454ed59 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -34,12 +34,12 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("runs", ignore_errors=True)def test_remove_whole_block(self):- layer_config={"model.decoder.layers.0.self_attn.k_proj":{"data_type":"float"},- "model.decoder.layers.0.self_attn.v_proj": {"data_type": "float"},- "model.decoder.layers.0.self_attn.q_proj": {"data_type": "float"},- "model.decoder.layers.0.self_attn.out_proj": {"data_type": "float"},- "model.decoder.layers.0.fc1": {"data_type": "float"},- "model.decoder.layers.0.fc2": {"data_type": "float"},+ layer_config={"model.decoder.layers.0.self_attn.k_proj":{"bits":32},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},+ "model.decoder.layers.0.fc1": {"bits": 32},+ "model.decoder.layers.0.fc2": {"bits": 32},}bits, group_size, sym = 4, 128, Falseautoround = AutoRound(@@ -55,6 +55,21 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_mx_fp4(self):+ bits, group_size, sym = 4, 32, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ data_type="mx_fp4"+ )+ autoround.quantize()+def test_default(self):bits, group_size, sym = 4, 128, False