PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine nvfp4 algorithm (#790)
Wenhua Cheng536810e1a05d
diff --git a/README.md b/README.mdindex 434ad93bb..95ba7435b 100644--- a/README.md+++ b/README.md@@ -35,7 +35,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).all bits other than 3 bits. Examplemodels: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar). **A more advanced algorithm** tailored for specific configurations may be available in- v0.6.1.+ v0.6.2.[2025/05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please referto [Intel/DeepSeek-R1-0528-int2-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-ar), [Intel/DeepSeek-R1-0528-int4-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-ar)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b98a6de15..176e57516 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -32,7 +32,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_block-from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme+from auto_round.schemes import QuantizationScheme, preset_name_to_schemefrom auto_round.sign_sgd import SignSGDfrom auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (@@ -1513,8 +1513,6 @@ class AutoRound(object):if (is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)) or is_static_wfp8afp8(self):- from auto_round.utils import set_amax_for_all_moe_layers-# enable moe experts act_max automatic generation for Linearset_amax_for_all_moe_layers(block, attr_name="act_max")# Normalize imatrix and quantize layers@@ -2024,11 +2022,11 @@ class AutoRound(object):continuetry:if isinstance(data_new, torch.Tensor):- self.model(data_new)+ self.model(data_new, use_cache=False)elif isinstance(data_new, tuple) or isinstance(data_new, list):- self.model(*data_new)+ self.model(*data_new, use_cache=False)else:- self.model(**data_new)+ self.model(**data_new, use_cache=False)except NotImplementedError:passexcept RuntimeError as error:@@ -2730,8 +2728,6 @@ class AutoRound(object):unwrapper_block(block, best_params)if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):- from auto_round.utils import set_amax_for_all_moe_layers-# enable moe experts act_max automatic generation for WrapperWALayerset_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex cec9ee894..12e8e4bfb 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -68,11 +68,13 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):@torch.compile()-def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):+def ref_nvfp4_quant(x, global_scale, block_size=16, v=0, scale_coeff=1.0):assert global_scale.dtype == torch.float32assert x.ndim == 2m, n = x.shape- vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)+ if isinstance(scale_coeff, torch.Tensor):+ scale_coeff = scale_coeff.view(-1, 1).to(x.device)+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * scale_coeffscale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)scale = float8_e4m3fn_ste(scale).to(torch.float32) ##e4m3 does not support torch compile@@ -83,14 +85,14 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):@register_dtype("nv_fp4")-def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, **kwargs):+def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if global_scale is None:tensor_max = tensor.abs().max().to(torch.float32)global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)- qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v, scale_coeff=max_scale)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)return qdq_res.to(orig_dtype), scale, None