PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
bug fix: fix NVFP4 evaluation with input_ global_scale handling and testing (#2374)
Xin He27daf7f8e952
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex cb4e2efca..1b3414422 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -102,23 +102,24 @@ def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0,@register_dtype("nv_fp4_with_static_gs")-def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):+def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, global_scale=None, **kwargs):if tensor is None or tensor.numel() == 0:return tensor, None, Noneorig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- if tensor_max is None:- tensor_max = tensor.to(torch.float32).abs().max()- else:- if not isinstance(tensor_max, torch.Tensor):+ if global_scale is None:+ if tensor_max is None:+ tensor_max = tensor.to(torch.float32).abs().max()+ elif not isinstance(tensor_max, torch.Tensor):tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)else:tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32)- if tensor_max.numel() != 1:- tensor_max = tensor_max.to(torch.float32).abs().max()-- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- global_scale = global_scale.to(tensor.device)+ if tensor_max.numel() != 1:+ tensor_max = tensor_max.abs().max()+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)+ elif not isinstance(global_scale, torch.Tensor):+ global_scale = torch.tensor(global_scale, device=tensor.device, dtype=torch.float32)+ global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)return qdq_res.to(orig_dtype), scale, None@@ -570,7 +571,7 @@ class _NVFPActivationQuantizer:observed = activation.detach().float().abs().max().unsqueeze(0)return observed if current is None else torch.maximum(current.to(observed.device), observed)- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):if self.requires_calibration and observed_max is None:raise ValueError(f"{self.data_type} activation requires observed_max")primitive = {@@ -582,11 +583,17 @@ class _NVFPActivationQuantizer:kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}if self.requires_calibration:kwargs["tensor_max"] = observed_max+ if global_scale is not None:+ kwargs["global_scale"] = global_scalereturn primitive(activation, **kwargs)- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):quantized, _, _ = self.qdq_with_scale(- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale+ activation,+ observed_max=observed_max,+ min_scale=min_scale,+ max_scale=max_scale,+ global_scale=global_scale,)return quantizeddiff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.pyindex 0089a2555..e7419e91d 100644--- a/auto_round/experimental/qmodules/fake.py+++ b/auto_round/experimental/qmodules/fake.py@@ -18,7 +18,7 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental.qmodules.base import QModuleBase-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, is_nv_fp__all__ = ["FakeActQuantLinear"]@@ -46,6 +46,8 @@ class FakeActQuantLinear(QModuleBase):self.register_parameter("bias", None)else:self.bias = torch.nn.Parameter(bias, requires_grad=False)+ if is_nv_fp(config.act_data_type or ""):+ self.register_buffer("input_global_scale", torch.ones(1, dtype=torch.float32))@classmethoddef from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):@@ -78,6 +80,7 @@ class FakeActQuantLinear(QModuleBase):tensor=activation,bits=self.config.act_bits,group_size=self.config.act_group_size,+ global_scale=getattr(self, "input_global_scale", None),)return qdq_activation.to(activation.dtype)diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pyindex 3c516199b..f873411b7 100644--- a/auto_round/export/formats/backends/fake.py+++ b/auto_round/export/formats/backends/fake.py@@ -21,7 +21,7 @@ import torchfrom auto_round.export.formats.base import OutputFormatfrom auto_round.logger import logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, is_nv_fpfrom auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device@@ -80,6 +80,18 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:logger.warning("Failed to normalize pytorch checkpoint keys for %s: %s", file_path, exc)+def _materialize_fake_input_global_scales(model: torch.nn.Module) -> None:+ """Persist calibrated NVFP activation scales in fake-format checkpoints."""+ for layer in model.modules():+ if not is_nv_fp(getattr(layer, "act_data_type", "") or "") or "input_global_scale" in layer._buffers:+ continue+ input_global_scale = getattr(layer, "input_global_scale", None)+ if not isinstance(input_global_scale, torch.Tensor):+ continue+ delattr(layer, "input_global_scale")+ layer.register_buffer("input_global_scale", input_global_scale.to(torch.float32))++@OutputFormat.register("fake")class FakeFormat(OutputFormat):support_schemes = None@@ -134,6 +146,8 @@ class FakeFormat(OutputFormat):if not has_meta_device:model = model.to("cpu")+ if has_fake_act_quant:+ _materialize_fake_input_global_scales(model)model.save_pretrained(output_dir)else:from auto_round.export.utils import save_config_artifactdiff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 2c9193902..6b70865f2 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -518,12 +518,10 @@ class WrapperWALayer(torch.nn.Module):max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scalemin_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale- x = self.activation_quantizer.qdq(- x,- observed_max=act_max,- min_scale=min_scale,- max_scale=max_scale,- )+ quant_kwargs = {"observed_max": act_max, "min_scale": min_scale, "max_scale": max_scale}+ if hasattr(self.orig_layer, "input_global_scale"):+ quant_kwargs["global_scale"] = self.orig_layer.input_global_scale+ x = self.activation_quantizer.qdq(x, **quant_kwargs)# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)return self.orig_layer.forward(x)diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.pyindex a70d1e3ab..5cce40ef2 100644--- a/test/unit/common/data_type/test_nvfp.py+++ b/test/unit/common/data_type/test_nvfp.py@@ -396,6 +396,16 @@ class TestNvFp4WithStaticGs:q, s, z = nv_fp4_with_static_gs(t, tensor_max=tm)assert q.shape == t.shape+ def test_explicit_global_scale_is_used(self):+ t = torch.tensor([[0.25, -1.0] * 8], dtype=torch.float32)+ global_scale = torch.ones(1, dtype=torch.float32)++ q, scale, _ = nv_fp4_with_static_gs(t, global_scale=global_scale)+ expected_q, expected_scale = ref_nvfp4_quant(t, global_scale, block_size=16)++ torch.testing.assert_close(q, expected_q)+ torch.testing.assert_close(scale, expected_scale)+def test_empty_tensor(self):t = torch.empty(0, 16, dtype=torch.bfloat16)q, s, z = nv_fp4_with_static_gs(t)diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.pyindex 798ac9767..96a285442 100644--- a/test/unit/test_cpu/export/test_fake_format.py+++ b/test/unit/test_cpu/export/test_fake_format.py@@ -19,6 +19,7 @@ from types import SimpleNamespaceimport torchfrom transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLM+import auto_round.experimental.qmodules.fake as fake_qmodulefrom auto_round.experimental.qmodules.fake import FakeActQuantLinearfrom auto_round.export.formats.backends.fake import (_normalize_state_dict_keys,@@ -37,6 +38,55 @@ def _assert_has_act_hook(layer):assert callable(layer.qdq_input)+def test_fake_nvfp4_qdq_uses_saved_input_global_scale(monkeypatch):+ captured_kwargs = {}++ def quant_func(**kwargs):+ captured_kwargs.update(kwargs)+ return kwargs["tensor"], None, None++ monkeypatch.setattr(fake_qmodule, "get_quant_func", lambda **kwargs: (quant_func, None))+ layer = FakeActQuantLinear(16, 4, PRESET_SCHEMES["NVFP4"], dtype=torch.float32)+ layer.input_global_scale.fill_(1.5)++ layer.qdq_input(torch.randn(2, 16))++ assert captured_kwargs["global_scale"] is layer.input_global_scale+++def test_fake_evaluation_wrapper_uses_saved_input_global_scale():+ captured_kwargs = {}+ linear = torch.nn.Linear(16, 4)+ linear.act_max_scale = torch.ones(1)+ linear.act_min_scale = torch.ones(1)+ linear.input_global_scale = torch.tensor([1.5], dtype=torch.float32)++ class ActivationQuantizer:+ def qdq(self, activation, **kwargs):+ captured_kwargs.update(kwargs)+ return activation++ wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())++ wrapper(torch.randn(2, 16))++ assert captured_kwargs["global_scale"] is linear.input_global_scale+++def test_fake_evaluation_wrapper_does_not_pass_global_scale_to_other_quantizers():+ linear = torch.nn.Linear(16, 4)+ linear.act_max_scale = torch.ones(1)+ linear.act_min_scale = torch.ones(1)++ class ActivationQuantizer:+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ return activation++ wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())++ wrapper(torch.randn(2, 16))++class _WrappedLinear(WrapperWALayer):def __init__(self, linear):torch.nn.Module.__init__(self)@@ -67,10 +117,12 @@ class _SaveableModel(torch.nn.Module):def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):model = _SaveableModel()+ model.linear.orig_layer.act_data_type = "nv_fp4_with_static_gs"+ model.linear.orig_layer.input_global_scale = torch.tensor([1.5], dtype=torch.float32)expected_weight = model.linear.orig_layer.weight.detach().clone()output_dir = str(tmp_path / "fake_model")- saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(+ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4"], SimpleNamespace(mllm=False)).save_quantized(output_dir=output_dir,model=model,inplace=False,@@ -78,19 +130,20 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):"bits": 4,"group_size": 16,"sym": True,- "data_type": "nvfp4_v2",+ "data_type": "nv_fp","act_bits": 4,"act_group_size": 16,"act_sym": True,- "act_data_type": "nvfp4_v2",+ "act_data_type": "nv_fp4_with_static_gs","to_quant_block_names": ["block"],"supported_types": [torch.nn.Linear],},)state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)- assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale"}+ assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale", "linear.input_global_scale"}assert torch.equal(state_dict["linear.weight"], expected_weight)+ assert torch.equal(state_dict["linear.input_global_scale"], torch.tensor([1.5], dtype=torch.float32))# Save-time should keep in-memory wrappers unchanged; replacement happens on load.assert hasattr(saved_model.linear, "orig_layer")with open(os.path.join(output_dir, "config.json")) as config_file:@@ -103,8 +156,12 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):loaded_model = _TinyLoadModel(SimpleNamespace(**quantization_config))loaded_model, used_backends = convert_hf_model(loaded_model, target_device="cpu")+ loaded_model.block.linear.load_state_dict(+ {"input_global_scale": state_dict["linear.input_global_scale"]}, strict=False+ )assert used_backends == ["auto_round:fake"]_assert_has_act_hook(loaded_model.block.linear)+ assert torch.equal(loaded_model.block.linear.input_global_scale, torch.tensor([1.5], dtype=torch.float32))roundtrip_activation = torch.randn(2, 3, 16)assert not torch.equal(loaded_model.block.linear.qdq_input(roundtrip_activation), roundtrip_activation)