Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

bug fix: fix NVFP4 evaluation with input_ global_scale handling and testing (#2374)

Xin He27daf7f8e952
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index cb4e2efca..1b3414422 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -102,23 +102,24 @@ def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0,
@register_dtype("nv_fp4_with_static_gs")
-def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):
+def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, global_scale=None, **kwargs):
if tensor is None or tensor.numel() == 0:
return tensor, None, None
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- if tensor_max is None:
- tensor_max = tensor.to(torch.float32).abs().max()
- else:
- if not isinstance(tensor_max, torch.Tensor):
+ if global_scale is None:
+ if tensor_max is None:
+ tensor_max = tensor.to(torch.float32).abs().max()
+ elif not isinstance(tensor_max, torch.Tensor):
tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)
else:
tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32)
- if tensor_max.numel() != 1:
- tensor_max = tensor_max.to(torch.float32).abs().max()
-
- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
- global_scale = global_scale.to(tensor.device)
+ if tensor_max.numel() != 1:
+ tensor_max = tensor_max.abs().max()
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
+ elif not isinstance(global_scale, torch.Tensor):
+ global_scale = torch.tensor(global_scale, device=tensor.device, dtype=torch.float32)
+ global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)
qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
return qdq_res.to(orig_dtype), scale, None
@@ -570,7 +571,7 @@ class _NVFPActivationQuantizer:
observed = activation.detach().float().abs().max().unsqueeze(0)
return observed if current is None else torch.maximum(current.to(observed.device), observed)
- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):
if self.requires_calibration and observed_max is None:
raise ValueError(f"{self.data_type} activation requires observed_max")
primitive = {
@@ -582,11 +583,17 @@ class _NVFPActivationQuantizer:
kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}
if self.requires_calibration:
kwargs["tensor_max"] = observed_max
+ if global_scale is not None:
+ kwargs["global_scale"] = global_scale
return primitive(activation, **kwargs)
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):
quantized, _, _ = self.qdq_with_scale(
- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
+ activation,
+ observed_max=observed_max,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ global_scale=global_scale,
)
return quantized
diff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.py
index 0089a2555..e7419e91d 100644
--- a/auto_round/experimental/qmodules/fake.py
+++ b/auto_round/experimental/qmodules/fake.py
@@ -18,7 +18,7 @@ import torch
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental.qmodules.base import QModuleBase
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, is_nv_fp
__all__ = ["FakeActQuantLinear"]
@@ -46,6 +46,8 @@ class FakeActQuantLinear(QModuleBase):
self.register_parameter("bias", None)
else:
self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ if is_nv_fp(config.act_data_type or ""):
+ self.register_buffer("input_global_scale", torch.ones(1, dtype=torch.float32))
@classmethod
def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):
@@ -78,6 +80,7 @@ class FakeActQuantLinear(QModuleBase):
tensor=activation,
bits=self.config.act_bits,
group_size=self.config.act_group_size,
+ global_scale=getattr(self, "input_global_scale", None),
)
return qdq_activation.to(activation.dtype)
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
index 3c516199b..f873411b7 100644
--- a/auto_round/export/formats/backends/fake.py
+++ b/auto_round/export/formats/backends/fake.py
@@ -21,7 +21,7 @@ import torch
from auto_round.export.formats.base import OutputFormat
from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, is_nv_fp
from auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device
@@ -80,6 +80,18 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:
logger.warning("Failed to normalize pytorch checkpoint keys for %s: %s", file_path, exc)
+def _materialize_fake_input_global_scales(model: torch.nn.Module) -> None:
+ """Persist calibrated NVFP activation scales in fake-format checkpoints."""
+ for layer in model.modules():
+ if not is_nv_fp(getattr(layer, "act_data_type", "") or "") or "input_global_scale" in layer._buffers:
+ continue
+ input_global_scale = getattr(layer, "input_global_scale", None)
+ if not isinstance(input_global_scale, torch.Tensor):
+ continue
+ delattr(layer, "input_global_scale")
+ layer.register_buffer("input_global_scale", input_global_scale.to(torch.float32))
+
+
@OutputFormat.register("fake")
class FakeFormat(OutputFormat):
support_schemes = None
@@ -134,6 +146,8 @@ class FakeFormat(OutputFormat):
if not has_meta_device:
model = model.to("cpu")
+ if has_fake_act_quant:
+ _materialize_fake_input_global_scales(model)
model.save_pretrained(output_dir)
else:
from auto_round.export.utils import save_config_artifact
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 2c9193902..6b70865f2 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -518,12 +518,10 @@ class WrapperWALayer(torch.nn.Module):
max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
- x = self.activation_quantizer.qdq(
- x,
- observed_max=act_max,
- min_scale=min_scale,
- max_scale=max_scale,
- )
+ quant_kwargs = {"observed_max": act_max, "min_scale": min_scale, "max_scale": max_scale}
+ if hasattr(self.orig_layer, "input_global_scale"):
+ quant_kwargs["global_scale"] = self.orig_layer.input_global_scale
+ x = self.activation_quantizer.qdq(x, **quant_kwargs)
# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)
return self.orig_layer.forward(x)
diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.py
index a70d1e3ab..5cce40ef2 100644
--- a/test/unit/common/data_type/test_nvfp.py
+++ b/test/unit/common/data_type/test_nvfp.py
@@ -396,6 +396,16 @@ class TestNvFp4WithStaticGs:
q, s, z = nv_fp4_with_static_gs(t, tensor_max=tm)
assert q.shape == t.shape
+ def test_explicit_global_scale_is_used(self):
+ t = torch.tensor([[0.25, -1.0] * 8], dtype=torch.float32)
+ global_scale = torch.ones(1, dtype=torch.float32)
+
+ q, scale, _ = nv_fp4_with_static_gs(t, global_scale=global_scale)
+ expected_q, expected_scale = ref_nvfp4_quant(t, global_scale, block_size=16)
+
+ torch.testing.assert_close(q, expected_q)
+ torch.testing.assert_close(scale, expected_scale)
+
def test_empty_tensor(self):
t = torch.empty(0, 16, dtype=torch.bfloat16)
q, s, z = nv_fp4_with_static_gs(t)
diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.py
index 798ac9767..96a285442 100644
--- a/test/unit/test_cpu/export/test_fake_format.py
+++ b/test/unit/test_cpu/export/test_fake_format.py
@@ -19,6 +19,7 @@ from types import SimpleNamespace
import torch
from transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLM
+import auto_round.experimental.qmodules.fake as fake_qmodule
from auto_round.experimental.qmodules.fake import FakeActQuantLinear
from auto_round.export.formats.backends.fake import (
_normalize_state_dict_keys,
@@ -37,6 +38,55 @@ def _assert_has_act_hook(layer):
assert callable(layer.qdq_input)
+def test_fake_nvfp4_qdq_uses_saved_input_global_scale(monkeypatch):
+ captured_kwargs = {}
+
+ def quant_func(**kwargs):
+ captured_kwargs.update(kwargs)
+ return kwargs["tensor"], None, None
+
+ monkeypatch.setattr(fake_qmodule, "get_quant_func", lambda **kwargs: (quant_func, None))
+ layer = FakeActQuantLinear(16, 4, PRESET_SCHEMES["NVFP4"], dtype=torch.float32)
+ layer.input_global_scale.fill_(1.5)
+
+ layer.qdq_input(torch.randn(2, 16))
+
+ assert captured_kwargs["global_scale"] is layer.input_global_scale
+
+
+def test_fake_evaluation_wrapper_uses_saved_input_global_scale():
+ captured_kwargs = {}
+ linear = torch.nn.Linear(16, 4)
+ linear.act_max_scale = torch.ones(1)
+ linear.act_min_scale = torch.ones(1)
+ linear.input_global_scale = torch.tensor([1.5], dtype=torch.float32)
+
+ class ActivationQuantizer:
+ def qdq(self, activation, **kwargs):
+ captured_kwargs.update(kwargs)
+ return activation
+
+ wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())
+
+ wrapper(torch.randn(2, 16))
+
+ assert captured_kwargs["global_scale"] is linear.input_global_scale
+
+
+def test_fake_evaluation_wrapper_does_not_pass_global_scale_to_other_quantizers():
+ linear = torch.nn.Linear(16, 4)
+ linear.act_max_scale = torch.ones(1)
+ linear.act_min_scale = torch.ones(1)
+
+ class ActivationQuantizer:
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ return activation
+
+ wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())
+
+ wrapper(torch.randn(2, 16))
+
+
class _WrappedLinear(WrapperWALayer):
def __init__(self, linear):
torch.nn.Module.__init__(self)
@@ -67,10 +117,12 @@ class _SaveableModel(torch.nn.Module):
def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
model = _SaveableModel()
+ model.linear.orig_layer.act_data_type = "nv_fp4_with_static_gs"
+ model.linear.orig_layer.input_global_scale = torch.tensor([1.5], dtype=torch.float32)
expected_weight = model.linear.orig_layer.weight.detach().clone()
output_dir = str(tmp_path / "fake_model")
- saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
+ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4"], SimpleNamespace(mllm=False)).save_quantized(
output_dir=output_dir,
model=model,
inplace=False,
@@ -78,19 +130,20 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
"bits": 4,
"group_size": 16,
"sym": True,
- "data_type": "nvfp4_v2",
+ "data_type": "nv_fp",
"act_bits": 4,
"act_group_size": 16,
"act_sym": True,
- "act_data_type": "nvfp4_v2",
+ "act_data_type": "nv_fp4_with_static_gs",
"to_quant_block_names": ["block"],
"supported_types": [torch.nn.Linear],
},
)
state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)
- assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale"}
+ assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale", "linear.input_global_scale"}
assert torch.equal(state_dict["linear.weight"], expected_weight)
+ assert torch.equal(state_dict["linear.input_global_scale"], torch.tensor([1.5], dtype=torch.float32))
# Save-time should keep in-memory wrappers unchanged; replacement happens on load.
assert hasattr(saved_model.linear, "orig_layer")
with open(os.path.join(output_dir, "config.json")) as config_file:
@@ -103,8 +156,12 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
loaded_model = _TinyLoadModel(SimpleNamespace(**quantization_config))
loaded_model, used_backends = convert_hf_model(loaded_model, target_device="cpu")
+ loaded_model.block.linear.load_state_dict(
+ {"input_global_scale": state_dict["linear.input_global_scale"]}, strict=False
+ )
assert used_backends == ["auto_round:fake"]
_assert_has_act_hook(loaded_model.block.linear)
+ assert torch.equal(loaded_model.block.linear.input_global_scale, torch.tensor([1.5], dtype=torch.float32))
roundtrip_activation = torch.randn(2, 3, 16)
assert not torch.equal(loaded_model.block.linear.qdq_input(roundtrip_activation), roundtrip_activation)