PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix critic bug of mxfp4 (#451)
Wenhua Chengabe69c54aa4a
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex fb27c3faa..48add5a8c 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -13,7 +13,7 @@# limitations under the License.import torch-from .utils import floor_ste, round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.utils import floor_ste, round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPEMXFP_FORMAT_CACHE = {@@ -39,8 +39,43 @@ FP32_EXPONENT_BIAS = 127FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)+def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):+ if ebits != 0:+ private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))++ # The minimum representable exponent for 8 exp bits is -126+ min_exp = -(2 ** (ebits - 1)) + 2+ private_exp = private_exp.clip(min=min_exp)+ else:+ private_exp = None++ # Scale up so appropriate number of mbits are in the integer portion of the number+ tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))++ if mantissa_rounding == "even":+ abs_tensor = torch.abs(tensor)+ mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)+ tensor = torch.sign(tensor) * (floor_ste(abs_tensor + 0.5) - mask_tensor)+ elif mantissa_rounding == "nearest":+ tensor = round_ste(tensor)+ elif mantissa_rounding == "floor":+ tensor = floor_ste(tensor)+ else:+ raise ValueError("mantissa_rounding only supports even, nearest or floor.")++ ##the clamp is False in official code+ # max_mantissa = 2 ** (mbits - 1) - 1 ## this is incorrect+ # tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)++ # Undo scaling+ tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)++ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)+ return tensor++def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,- mantissa_rounding="even", data_type="mx_fp", **kwargs):+ mantissa_rounding="even", data_type="mx_fp", **kwargs):"""Quantize the given tensor using the specified parameters.This function performs quantization on the `tensor` tensor according to the@@ -71,12 +106,17 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,shared_exp *= (max_scale.unsqueeze(dim=-1))else:shared_exp *= max_scale- scale_emax = 2 ** (8 - 1) - 1+shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp = floor_ste(shared_exp)+ # if flush_fp32_subnorms:+ # tensor = tensor * (shared_exp > -FP32_EXPONENT_BIAS).type(tensor.dtype)++ scale_emax = 2 ** (8 - 1) - 1shared_exp[shared_exp == torch.inf] = scale_emax + emaxshared_exp[shared_exp == -torch.inf] = -scale_emax + emaxshared_exp = (shared_exp - emax)- shared_exp = floor_ste(shared_exp)+shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nanshared_exp[shared_exp < -scale_emax] = -scale_emaxif (shared_exp.dtype == torch.float16 and (torch.any(shared_exp > 15) or torch.any(shared_exp < -24))) or (@@ -85,35 +125,8 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,shared_exp = shared_exp.to(torch.float32)tensor = tensor / (2 ** shared_exp)tensor = tensor + v- if ebits != 0:- private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)- # The minimum representable exponent for 8 exp bits is -126- min_exp = -(2 ** (ebits - 1)) + 2- private_exp = private_exp.clip(min=min_exp)- else:- private_exp = None-- # Scale up so appropriate number of mbits are in the integer portion of the number- tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))-- if mantissa_rounding == "even":- abs_tensor = torch.abs(tensor)- mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)- tensor = torch.sign(tensor) * (floor_ste(abs_tensor + 0.5) - mask_tensor)- elif mantissa_rounding == "nearest":- tensor = round_ste(tensor)- elif mantissa_rounding == "floor":- tensor = floor_ste(tensor)- else:- raise ValueError("mantissa_rounding only supports even, nearest or floor.")- max_mantissa = 2 ** (mbits - 1) - 1- tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)-- # Undo scaling- tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)-- tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)tensor = tensor * (2 ** shared_exp)tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None@@ -121,3 +134,15 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,for key in MXFP_FORMAT_CACHE.keys():QUANT_FUNC_WITH_DTYPE[key] = quant_mx++if __name__ == "__main__":+ data = torch.tensor([0.0, 0.25, 0.4,0.75, 1.25,1.4, 1.75, 2.5, 2.9,3.5, 5.0, 5.1])+ data1 = quant_element(data, 2, 3, 6.0)+ gt = torch.tensor([0.0,0.0,0.5,1.0,1.0,1.5,2.0,2.0,3.0,4.0,4.0,6.0])+ assert(torch.sum(torch.abs(data1-gt))<1e-6)++ data_neg = data*-1+ data2 = quant_element(data_neg, 2, 3, 6.0)+ assert(torch.sum(torch.abs(data2-gt*-1))<1e-6)++