Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

speedup mxfp tuning and fix nvfp bug (#647)

Wenhua Chengfddcb5d8932b
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index cf4c24568..2c2192efe 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -42,7 +42,6 @@ FP32_MIN_NORMAL = 2 ** (-FP32_EXPONENT_BIAS + 1)
def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
if ebits != 0:
private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))
-
# The minimum representable exponent for 8 exp bits is -126
min_exp = -(2 ** (ebits - 1)) + 2
private_exp = private_exp.clip(min=min_exp)
@@ -51,7 +50,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
# Scale up so appropriate number of mbits are in the integer portion of the number
tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))
-
if mantissa_rounding == "even":
abs_tensor = torch.abs(tensor)
mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)
@@ -63,10 +61,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
else:
raise ValueError("mantissa_rounding only supports even, nearest or floor.")
- ##the clamp is False in official code
- # max_mantissa = 2 ** (mbits - 1) - 1 ## this is incorrect
- # tensor = torch.clamp(tensor, -max_mantissa, max_mantissa)
-
# Undo scaling
tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)
@@ -74,6 +68,7 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
return tensor
def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,
mantissa_rounding="even", data_type="mx_fp", **kwargs):
"""Quantize the given tensor using the specified parameters.
@@ -101,34 +96,25 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]
orig_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
shared_exp, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
if isinstance(max_scale, torch.Tensor):
shared_exp *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)
else:
shared_exp *= max_scale
- shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
+ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
+ shared_exp = torch.where(shared_exp == 0, torch.ones_like(shared_exp), torch.log2(shared_exp))
shared_exp = floor_ste(shared_exp)
- # if flush_fp32_subnorms:
- # tensor = tensor * (shared_exp > -FP32_EXPONENT_BIAS).type(tensor.dtype)
-
scale_emax = 2 ** (8 - 1) - 1
- shared_exp[shared_exp == torch.inf] = scale_emax + emax
- shared_exp[shared_exp == -torch.inf] = -scale_emax + emax
- shared_exp = (shared_exp - emax)
-
- shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan
- shared_exp[shared_exp < -scale_emax] = -scale_emax
- if (shared_exp.dtype == torch.float16 and (torch.any(shared_exp > 15) or torch.any(shared_exp < -24))) or (
- shared_exp.dtype == torch.bfloat16 and torch.any((shared_exp < -126))):
- tensor = tensor.to(torch.float32)
- shared_exp = shared_exp.to(torch.float32)
- tensor = tensor / (2 ** shared_exp)
- tensor = tensor + v
+ shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)
+
+ scale = torch.pow(2, shared_exp)
+ tensor = tensor / scale + v
tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)
- tensor = tensor * (2 ** shared_exp)
+ tensor = tensor * scale
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None
@@ -137,13 +123,11 @@ for key in MXFP_FORMAT_CACHE.keys():
QUANT_FUNC_WITH_DTYPE[key] = quant_mx
if __name__ == "__main__":
- data = torch.tensor([0.0, 0.25, 0.4,0.75, 1.25,1.4, 1.75, 2.5, 2.9,3.5, 5.0, 5.1])
+ data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])
data1 = quant_element(data, 2, 3, 6.0)
- gt = torch.tensor([0.0,0.0,0.5,1.0,1.0,1.5,2.0,2.0,3.0,4.0,4.0,6.0])
- assert(torch.sum(torch.abs(data1-gt))<1e-6)
+ gt = torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0, 1.5, 2.0, 2.0, 3.0, 4.0, 4.0, 6.0])
+ assert (torch.sum(torch.abs(data1 - gt)) < 1e-6)
- data_neg = data*-1
+ data_neg = data * -1
data2 = quant_element(data_neg, 2, 3, 6.0)
- assert(torch.sum(torch.abs(data2-gt*-1))<1e-6)
-
-
+ assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 82dcbcf85..76c4eb92d 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -16,7 +16,7 @@ import torch
from auto_round.data_type.fp8 import float8_e4m3fn_ste
from auto_round.data_type.register import register_dtype
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger
# taken from
@@ -43,7 +43,7 @@ def cast_to_fp4_ste(x):
def get_reciprocal(x):
if isinstance(x, torch.Tensor):
- return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
+ return torch.where(x == 0, torch.zeros_like(x, dtype=x.dtype), 1.0 / x)
elif isinstance(x, (float, int)):
return 0.0 if x == 0 else 1.0 / x
else:
@@ -54,27 +54,27 @@ FLOAT4_E2M1_MAX = 6.0
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min
+
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
m, n = x.shape
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
- scale = float8_e4m3fn_ste(scale).to(torch.float32)
scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)
+ scale = float8_e4m3fn_ste(scale).to(torch.float32) ##e4m3 does not support torch compile
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
-
scaled_x = x.to(torch.float32) * output_scale + v
clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale
@register_dtype("nv_fp4")
-def full_quant(tensor, bits=4, group_size=16, v=0, **kwargs):
+def nv_fp4(tensor, bits=4, group_size=16, v=0, **kwargs):
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
tensor_amax = tensor.abs().max().to(torch.float32)
- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
return qdq_res.to(orig_dtype), output_scale, None
@@ -155,9 +155,52 @@ def cast_to_ue5m3_ste(x):
return fp4
+def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
+ assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32
+ assert x.ndim == 2
+ m, n = x.shape
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.unsqueeze(dim=-1).to(x.device)
+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale
+ scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
+ scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)
+ scale = cast_to_ue5m3_ste(scale).to(torch.float32)
+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
+
+ scaled_x = x.to(torch.float32) * output_scale + v
+ clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale
+
+
+@register_dtype("fp4_v2_with_global_scale")
+def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, max_scale=1.0, **kwargs):
+ assert (group_size == 32 or group_size == 16)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ tensor_amax = tensor.abs().max().to(torch.float32)
+ global_scale = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
+ qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), output_scale, None
+
+
+@register_dtype("fp4_v2")
+def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
+ assert (group_size == 32 or group_size == 16)
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ global_scale = 1.0
+ qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), output_scale, None
+
+
if __name__ == "__main__":
+
test = torch.tensor(
- [0.0, 2 ** (-17), (2 ** -14) * 0.875, 2 ** -14, 2 ** -13, 2 ** -6,
+ [0.0, 1e-38, 2 ** (-17), (2 ** -14) * 0.875, 2 ** -14, 2 ** -13, 2 ** -6,
1e-6, 2.7657e-05, 0.1, 1.0, 3.14, 1000.0,
114688,
1e10],
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 82f8645d6..5536c48c4 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -28,6 +28,7 @@ def simple_evaluate_user_model(
batch_size: Optional[int] = 1,
max_batch_size: Optional[int] = 64,
eval_model_dtype = "auto",
+ add_bos_token: bool = False,
**kwargs
):
hflm = HFLM(
@@ -35,7 +36,8 @@ def simple_evaluate_user_model(
tokenizer=tokenizer,
batch_size=batch_size,
max_batch_size=max_batch_size,
- dtype=eval_model_dtype)
+ dtype=eval_model_dtype,
+ add_bos_token=add_bos_token)
return lm_simple_evaluate(
model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index ac1bb5cbe..a7fd648ed 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -581,13 +581,18 @@ def tune(args):
args.eval_bs = 16
from auto_round.eval.evaluation import simple_evaluate_user_model
st = time.time()
+ add_bos_token=False
+ if "llama" in args.model.lower():
+ add_bos_token=True
res = simple_evaluate_user_model(
model,
tokenizer,
tasks=tasks,
batch_size=args.eval_bs,
device=device_str,
- eval_model_dtype=eval_model_dtype)
+ eval_model_dtype=eval_model_dtype,
+ add_bos_token=add_bos_token
+ )
print(make_table(res))
print("evaluation running time=%ds" % (time.time() - st))
else: