PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
speedup mxfp tuning and fix nvfp bug (#647)
Wenhua Chengfddcb5d8932b
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex cf4c24568..2c2192efe 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -42,7 +42,6 @@ FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):if ebits != 0:private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))-# The minimum representable exponent for 8 exp bits is -126min_exp = -(2 ** (ebits - 1)) + 2private_exp = private_exp.clip(min=min_exp)@@ -51,7 +50,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):# Scale up so appropriate number of mbits are in the integer portion of the numbertensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))-if mantissa_rounding == "even":abs_tensor = torch.abs(tensor)mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)@@ -63,10 +61,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):else:raise ValueError("mantissa_rounding only supports even, nearest or floor.")- ##the clamp is False in official code- # max_mantissa = 2 ** (mbits - 1) - 1 ## this is incorrect- # tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)-# Undo scalingtensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)@@ -74,6 +68,7 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):return tensordef quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,mantissa_rounding="even", data_type="mx_fp", **kwargs):"""Quantize the given tensor using the specified parameters.@@ -101,34 +96,25 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)shared_exp, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)if isinstance(max_scale, torch.Tensor):shared_exp *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)else:shared_exp *= max_scale- shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp = torch.where(shared_exp == 0, torch.ones_like(shared_exp), torch.log2(shared_exp))shared_exp = floor_ste(shared_exp)- # if flush_fp32_subnorms:- # tensor = tensor * (shared_exp > -FP32_EXPONENT_BIAS).type(tensor.dtype)-scale_emax = 2 ** (8 - 1) - 1- shared_exp[shared_exp == torch.inf] = scale_emax + emax- shared_exp[shared_exp == -torch.inf] = -scale_emax + emax- shared_exp = (shared_exp - emax)-- shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan- shared_exp[shared_exp < -scale_emax] = -scale_emax- if (shared_exp.dtype == torch.float16 and (torch.any(shared_exp > 15) or torch.any(shared_exp < -24))) or (- shared_exp.dtype == torch.bfloat16 and torch.any((shared_exp < -126))):- tensor = tensor.to(torch.float32)- shared_exp = shared_exp.to(torch.float32)- tensor = tensor / (2 ** shared_exp)- tensor = tensor + v+ shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)++ scale = torch.pow(2, shared_exp)+ tensor = tensor / scale + vtensor = torch.clamp(tensor, min=-max_norm, max=max_norm)tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)- tensor = tensor * (2 ** shared_exp)+ tensor = tensor * scaletensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None@@ -137,13 +123,11 @@ for key in MXFP_FORMAT_CACHE.keys():QUANT_FUNC_WITH_DTYPE[key] = quant_mxif __name__ == "__main__":- data = torch.tensor([0.0, 0.25, 0.4,0.75, 1.25,1.4, 1.75, 2.5, 2.9,3.5, 5.0, 5.1])+ data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])data1 = quant_element(data, 2, 3, 6.0)- gt = torch.tensor([0.0,0.0,0.5,1.0,1.0,1.5,2.0,2.0,3.0,4.0,4.0,6.0])- assert(torch.sum(torch.abs(data1-gt))<1e-6)+ gt = torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0, 1.5, 2.0, 2.0, 3.0, 4.0, 4.0, 6.0])+ assert (torch.sum(torch.abs(data1 - gt)) < 1e-6)- data_neg = data*-1+ data_neg = data * -1data2 = quant_element(data_neg, 2, 3, 6.0)- assert(torch.sum(torch.abs(data2-gt*-1))<1e-6)--+ assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 82dcbcf85..76c4eb92d 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -16,7 +16,7 @@ import torchfrom auto_round.data_type.fp8 import float8_e4m3fn_stefrom auto_round.data_type.register import register_dtype-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger# taken from@@ -43,7 +43,7 @@ def cast_to_fp4_ste(x):def get_reciprocal(x):if isinstance(x, torch.Tensor):- return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)+ return torch.where(x == 0, torch.zeros_like(x, dtype=x.dtype), 1.0 / x)elif isinstance(x, (float, int)):return 0.0 if x == 0 else 1.0 / xelse:@@ -54,27 +54,27 @@ FLOAT4_E2M1_MAX = 6.0FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).maxFLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min+def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):assert global_scale.dtype == torch.float32assert x.ndim == 2m, n = x.shapevec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))- scale = float8_e4m3fn_ste(scale).to(torch.float32)scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)+ scale = float8_e4m3fn_ste(scale).to(torch.float32) ##e4m3 does not support torch compileoutput_scale = get_reciprocal(scale * get_reciprocal(global_scale))-scaled_x = x.to(torch.float32) * output_scale + vclipped_x = torch.clamp(scaled_x, -6.0, 6.0)return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale@register_dtype("nv_fp4")-def full_quant(tensor, bits=4, group_size=16, v=0, **kwargs):+def nv_fp4(tensor, bits=4, group_size=16, v=0, **kwargs):orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)tensor_amax = tensor.abs().max().to(torch.float32)- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)return qdq_res.to(orig_dtype), output_scale, None@@ -155,9 +155,52 @@ def cast_to_ue5m3_ste(x):return fp4+def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):+ assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32+ assert x.ndim == 2+ m, n = x.shape+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.unsqueeze(dim=-1).to(x.device)+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale+ scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))+ scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)+ scale = cast_to_ue5m3_ste(scale).to(torch.float32)+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))++ scaled_x = x.to(torch.float32) * output_scale + v+ clipped_x = torch.clamp(scaled_x, -6.0, 6.0)+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale+++@register_dtype("fp4_v2_with_global_scale")+def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, max_scale=1.0, **kwargs):+ assert (group_size == 32 or group_size == 16)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor_amax = tensor.abs().max().to(torch.float32)+ global_scale = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)+ qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), output_scale, None+++@register_dtype("fp4_v2")+def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):+ assert (group_size == 32 or group_size == 16)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ global_scale = 1.0+ qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), output_scale, None++if __name__ == "__main__":+test = torch.tensor(- [0.0, 2 ** (-17), (2 ** -14) * 0.875, 2 ** -14, 2 ** -13, 2 ** -6,+ [0.0, 1e-38, 2 ** (-17), (2 ** -14) * 0.875, 2 ** -14, 2 ** -13, 2 ** -6,1e-6, 2.7657e-05, 0.1, 1.0, 3.14, 1000.0,114688,1e10],diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 82f8645d6..5536c48c4 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -28,6 +28,7 @@ def simple_evaluate_user_model(batch_size: Optional[int] = 1,max_batch_size: Optional[int] = 64,eval_model_dtype = "auto",+ add_bos_token: bool = False,**kwargs):hflm = HFLM(@@ -35,7 +36,8 @@ def simple_evaluate_user_model(tokenizer=tokenizer,batch_size=batch_size,max_batch_size=max_batch_size,- dtype=eval_model_dtype)+ dtype=eval_model_dtype,+ add_bos_token=add_bos_token)return lm_simple_evaluate(model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex ac1bb5cbe..a7fd648ed 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -581,13 +581,18 @@ def tune(args):args.eval_bs = 16from auto_round.eval.evaluation import simple_evaluate_user_modelst = time.time()+ add_bos_token=False+ if "llama" in args.model.lower():+ add_bos_token=Trueres = simple_evaluate_user_model(model,tokenizer,tasks=tasks,batch_size=args.eval_bs,device=device_str,- eval_model_dtype=eval_model_dtype)+ eval_model_dtype=eval_model_dtype,+ add_bos_token=add_bos_token+ )print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))else: