PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix fp8 accuracy drop issue (#1611)
Wang, Mengnib78453cd3cfa
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex eb7f6ffcc..1db653783 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -474,7 +474,10 @@ class BaseCompressor(object):disable_opt_rtn = Falseif self.iters > 0 and is_block_wfp8(self):- logger.warning("RTN is recommended since it shows even better accuracy for block-wise fp8 quantization.")+ logger.warning(+ "RTN is recommended as it achieves accuracy comparable to tuning for block-wise FP8 quantization "+ "while being significantly faster. You can set `--iters 0 --disable_opt_rtn` to enable RTN mode."+ )# Important Note! This is not very robust, do NOT rely on it to do high risky thingself.is_moe_model = is_moe_model(self.model)diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex ea7a6e99e..11f4594c2 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -26,7 +26,7 @@ from auto_round.utils import is_gaudi2, logger@register_dtype(("block_fp8_sym", "block_fp8", "block_fp8_e4m3"))-def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, **kwargs):+def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128, 128), v=0, tensor_min=None, **kwargs):"""Symmetric quantization using block float8 format.Args:@@ -51,9 +51,18 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,if tensor_max is None:max_tensor = tensor.abs().amax(dim=(-2, -1)) * max_scaleelif isinstance(tensor_max, torch.Tensor):- max_tensor = tensor_max.to(tensor.device) * max_scale+ max_tensor = (+ tensor_max.to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(tensor_max.abs(), tensor_min.abs()).to(tensor.device) * max_scale+ )else:- max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ max_tensor = (+ torch.tensor(tensor_max).to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(torch.tensor(tensor_max).abs(), torch.tensor(tensor_min).abs()).to(tensor.device)+ * max_scale+ )scale = max_tensor / info.maxassert len(scale.shape) == 2, f"Only support 2D group_size, but get {len(scale.shape)}"min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm@@ -71,7 +80,7 @@ def quant_block_fp_sym(tensor, max_scale=1.0, tensor_max=None, group_size=(128,@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))-def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, tensor_min=None, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -98,9 +107,18 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **if tensor_max is None: ##dynamic per-tokenmax_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):- max_tensor = tensor_max.to(tensor.device) * max_scale+ max_tensor = (+ tensor_max.to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(tensor_max.abs(), tensor_min.abs()).to(tensor.device) * max_scale+ )else:- max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ max_tensor = (+ torch.tensor(tensor_max).to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(torch.tensor(tensor_max).abs(), torch.tensor(tensor_min).abs()).to(tensor.device)+ * max_scale+ )scale = max_tensor.to(torch.float32) / info.maxmin_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllmscale = torch.clip(scale, min=min_scaling_factor)@@ -117,7 +135,7 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **@register_dtype("fp8_e5m2")-def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):+def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, tensor_min=None, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -140,9 +158,18 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, *if tensor_max is None: ##dynamic per-tokenmax_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):- max_tensor = tensor_max.to(tensor.device) * max_scale+ max_tensor = (+ tensor_max.to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(tensor_max.abs(), tensor_min.abs()).to(tensor.device) * max_scale+ )else:- max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ max_tensor = (+ torch.tensor(tensor_max).to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(torch.tensor(tensor_max).abs(), torch.tensor(tensor_min).abs()).to(tensor.device)+ * max_scale+ )scale = max_tensor.to(torch.float32) / info.maxmin_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllmscale = torch.clip(scale, min=min_scaling_factor)@@ -225,7 +252,7 @@ def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size@register_dtype("fp8_gaudi3_sym")-def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):+def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, tensor_min=None, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -250,9 +277,19 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):tensor = tensor.reshape(-1, orig_shape[-1])max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):- max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scale+ max_tensor = (+ tensor_max.to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(tensor_max.clone().detach().abs(), tensor_min.clone().detach().abs()).to(tensor.device)+ * max_scale+ )else:- max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ max_tensor = (+ torch.tensor(tensor_max).to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(torch.tensor(tensor_max).abs(), torch.tensor(tensor_min).abs()).to(tensor.device)+ * max_scale+ )scale = max_tensor.to(torch.float32) / fp8_maxmin_scaling_factor = float(1.0 / (fp8_max * 512.0)) ##copy from vllmscale = torch.clip(scale, min=min_scaling_factor)@@ -271,7 +308,9 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):if is_gaudi2():@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))- def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs): # pylint: disable=E0102+ def quant_fp8_sym(+ tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, tensor_min=None, **kwargs+ ): # pylint: disable=E0102"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -300,9 +339,18 @@ if is_gaudi2():if tensor_max is None: ##dynamic per-tokenmax_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):- max_tensor = tensor_max.to(tensor.device) * max_scale+ max_tensor = (+ tensor_max.to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(tensor_max.abs(), tensor_min.abs()).to(tensor.device) * max_scale+ )else:- max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ max_tensor = (+ torch.tensor(tensor_max).to(tensor.device) * max_scale+ if tensor_min is None+ else torch.maximum(torch.tensor(tensor_max).abs(), torch.tensor(tensor_min).abs()).to(tensor.device)+ * max_scale+ )scale = max_tensor.to(torch.float32) / info.maxmin_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllmscale = torch.clip(scale, min=min_scaling_factor)diff --git a/docs/fp8_block_acc.md b/docs/fp8_block_acc.mdindex 51564460c..a89e526bd 100644--- a/docs/fp8_block_acc.md+++ b/docs/fp8_block_acc.md@@ -18,17 +18,17 @@ lm_eval --model vllm --model_args pretrained=model_path,add_bos_token=true,tenso|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k_llama|mmlu_llama|mmlu_pro_llama|Avg. ||:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:----------|:---------|:-------------|:-----|-|BF16 |0.5341 |0.8232 |0.8428|0.5805 |0.7258 |0.3540 |0.7758|0.3709 |0.7151 |0.6111 |0.6737 |0.3778 |0.6145|-|FP8_BLOCK RTN |0.5239 |0.8182 |0.8483|0.5780 |0.7235 |0.3520 |0.7840|0.3647 |0.7040 |0.6171 |0.6681 |0.3738 |0.6130|-|FP8_BLOCK Tuning|0.5213 |0.8047 |0.8376|0.5834 |0.7099 |0.3480 |0.7764|0.3684 |0.7245 |0.6224 |0.6654 |0.3690 |0.6109|+|BF16 |0.5358 |0.8232 |0.8459|0.5795 |0.7266 |0.3540 |0.7786|0.3721 |0.7135 |0.7369 |0.6741 |0.4334 |0.6311|+|FP8_BLOCK RTN |0.5316 |0.8173 |0.8468|0.5787 |0.7256 |0.3560 |0.7824|0.3684 |0.7103 |0.7369 |0.6661 |0.4358 |0.6297|+|FP8_BLOCK Tuning|0.5401 |0.8228 |0.8459|0.5786 |0.7275 |0.3480 |0.7769|0.3684 |0.7103 |0.7437 |0.6727 |0.4315 |0.6305|## Qwen3-8B|Scheme |arc_challenge|arc_easy|boolq |hellaswag|lambada_openai|openbookqa|piqa |truthfulqa_mc1|winogrande|gsm8k |mmlu |mmlu_pro|Avg. ||:---------------|:------------|:-------|:-----|:--------|:-------------|:---------|:-----|:-------------|:---------|:-----|:-----|:-------|:-----|-|BF16 |0.5589 |0.8329 |0.8661|0.5719 |0.6501 |0.3140 |0.7688|0.3647 |0.6772 |0.8726|0.7297|0.6218 |0.6524|-|FP8_BLOCK RTN |0.5444 |0.8350 |0.8679|0.5704 |0.6466 |0.3060 |0.7644|0.3611 |0.6914 |0.8741|0.7268|0.6242 |0.6510|-|FP8_BLOCK Tuning|0.5486 |0.8308 |0.8639|0.5691 |0.6513 |0.3080 |0.7622|0.3586 |0.6882 |0.8673|0.7272|0.6139 |0.6491|+|BF16 |0.5580 |0.8342 |0.8667|0.5712 |0.6517 |0.3120 |0.7688|0.3647 |0.6788 |0.8726|0.7292|0.6214 |0.6524|+|FP8_BLOCK RTN |0.5538 |0.8350 |0.8691|0.5713 |0.6468 |0.3140 |0.7661|0.3721 |0.6780 |0.8704|0.7269|0.6204 |0.6520|+|FP8_BLOCK Tuning|0.5520 |0.8384 |0.8703|0.5710 |0.6519 |0.3260 |0.7671|0.3635 |0.6725 |0.8681|0.7302|0.6168 |0.6523|