Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine nvfp4 algorithm (#790)

Wenhua Cheng536810e1a05d
diff --git a/README.md b/README.md
index 434ad93bb..95ba7435b 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
all bits other than 3 bits. Example
models: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)
and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar). **A more advanced algorithm** tailored for specific configurations may be available in
- v0.6.1.
+ v0.6.2.
[2025/05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [Intel/DeepSeek-R1-0528-int2-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-ar), [Intel/DeepSeek-R1-0528-int4-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-ar)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b98a6de15..176e57516 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -32,7 +32,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
-from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
from auto_round.sign_sgd import SignSGD
from auto_round.special_model_handler import _handle_moe_model
from auto_round.utils import (
@@ -1513,8 +1513,6 @@ class AutoRound(object):
if (
is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)
) or is_static_wfp8afp8(self):
- from auto_round.utils import set_amax_for_all_moe_layers
-
# enable moe experts act_max automatic generation for Linear
set_amax_for_all_moe_layers(block, attr_name="act_max")
# Normalize imatrix and quantize layers
@@ -2024,11 +2022,11 @@ class AutoRound(object):
continue
try:
if isinstance(data_new, torch.Tensor):
- self.model(data_new)
+ self.model(data_new, use_cache=False)
elif isinstance(data_new, tuple) or isinstance(data_new, list):
- self.model(*data_new)
+ self.model(*data_new, use_cache=False)
else:
- self.model(**data_new)
+ self.model(**data_new, use_cache=False)
except NotImplementedError:
pass
except RuntimeError as error:
@@ -2730,8 +2728,6 @@ class AutoRound(object):
unwrapper_block(block, best_params)
if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):
- from auto_round.utils import set_amax_for_all_moe_layers
-
# enable moe experts act_max automatic generation for WrapperWALayer
set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index cec9ee894..12e8e4bfb 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -68,11 +68,13 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):
@torch.compile()
-def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
+def ref_nvfp4_quant(x, global_scale, block_size=16, v=0, scale_coeff=1.0):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
m, n = x.shape
- vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
+ if isinstance(scale_coeff, torch.Tensor):
+ scale_coeff = scale_coeff.view(-1, 1).to(x.device)
+ vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * scale_coeff
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)
scale = float8_e4m3fn_ste(scale).to(torch.float32) ##e4m3 does not support torch compile
@@ -83,14 +85,14 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
@register_dtype("nv_fp4")
-def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, **kwargs):
+def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
if global_scale is None:
tensor_max = tensor.abs().max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)
- qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v, scale_coeff=max_scale)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
return qdq_res.to(orig_dtype), scale, None