Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

NVFP4 Loading support (#839)

Yi Liu075133771f0e
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
index f95615eba..0179dd646 100644
--- a/auto_round/experimental/qmodules/__init__.py
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -13,4 +13,5 @@
# limitations under the License.
from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear
+from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/nvfp4.py b/auto_round/experimental/qmodules/nvfp4.py
new file mode 100644
index 000000000..88ea577e2
--- /dev/null
+++ b/auto_round/experimental/qmodules/nvfp4.py
@@ -0,0 +1,199 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Optional, Union
+
+import torch
+
+from auto_round.data_type.nvfp import get_reciprocal, ref_nvfp4_quant
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+__all__ = ["NVFP4QuantLinear"]
+
+
+# Adapted from auto_round/data_type/nvfp.py
+def _nv_fp4_with_static_gs(
+ tensor: torch.Tensor, global_scale: torch.Tensor, bits: int = 4, group_size: int = 16
+) -> tuple[torch.Tensor, torch.Tensor, None]:
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+
+ global_scale = global_scale.to(tensor.device)
+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v=0)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
+def _nvfp4_qdq(tensor: torch.Tensor, config: QuantizationScheme, global_scale: torch.Tensor) -> torch.Tensor:
+ qdq_tensor, scales, _ = _nv_fp4_with_static_gs(
+ tensor=tensor, global_scale=global_scale, bits=config.act_bits, group_size=config.act_group_size
+ )
+ return qdq_tensor
+
+
+class NVFP4QuantLinear(QModuleBase):
+ """
+ Quantized linear layer using NVFP4 quantization scheme.
+ """
+
+ SUPPORTED_COMPUTE_DTYPE = [torch.bfloat16, torch.float16, torch.float32]
+
+ def __init__(
+ self,
+ in_features: int,
+ out_features: int,
+ config: QuantizationScheme,
+ weight: Optional[torch.Tensor] = None,
+ weight_scale: Optional[torch.Tensor] = None,
+ bias: Union[torch.Tensor, bool, None] = None,
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.group_size = 16
+ self.config = config
+ self.dtype = dtype
+ self.pre_dequantized = False
+
+ # Validate dtype
+ assert (
+ dtype in self.SUPPORTED_COMPUTE_DTYPE
+ ), f"Expected dtype to be one of {self.SUPPORTED_COMPUTE_DTYPE}, but got {dtype}."
+
+ # check group size
+ assert self.group_size == config.group_size, f"Group size mismatch: {self.group_size} vs {config.group_size}"
+ assert (
+ self.group_size == config.act_group_size
+ ), f"Group size mismatch: {self.group_size} vs {config.act_group_size}"
+
+ # Initialize weights
+ init_weight = self.initialize_weights(weight)
+ self.register_buffer("weight_packed", init_weight)
+
+ # Initialize bias
+ if bias is not None:
+ if isinstance(bias, bool):
+ bias = torch.zeros((out_features,), dtype=dtype)
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+
+ # Initialize weight scale
+ init_weight_scale = (
+ torch.empty((out_features, in_features // self.group_size), dtype=torch.float8_e4m3fn)
+ if weight_scale is None
+ else weight_scale
+ )
+ self.register_buffer("weight_scale", init_weight_scale)
+
+ self.register_buffer(
+ "weight_global_scale",
+ torch.zeros(
+ (1,),
+ dtype=torch.float32,
+ ),
+ )
+ self.register_buffer(
+ "input_global_scale",
+ torch.zeros(
+ (1,),
+ dtype=torch.float32,
+ ),
+ )
+
+ @staticmethod
+ def _convert_global_scale_to_float32(state_dict: dict[str, torch.Tensor], name: str):
+ if name not in state_dict or state_dict[name].dtype == torch.float32:
+ return
+ original_scale = state_dict[name]
+ state_dict[name] = original_scale.to(torch.float32)
+ logger.warning_once("Forcing global scale to float32 for better precision.")
+
+ def load_state_dict(self, state_dict, strict=True, assign=False):
+ self._convert_global_scale_to_float32(state_dict, "weight_global_scale")
+ self._convert_global_scale_to_float32(state_dict, "input_global_scale")
+ return super().load_state_dict(state_dict, strict, assign)
+
+ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:
+ """
+ Initialize weights.
+ """
+ weight_dtype = torch.uint8
+ weight_in_features = self.in_features // 2
+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ """
+ Get minimum device capability.
+ """
+ logger.warning_once("NVFP4 quantization is still in experimental stage, the inference speed might be slow.")
+ return 0
+
+ def _dequant_nvfp4_tensor(
+ self, packed_data: torch.Tensor, scale: torch.Tensor, target_dtype: torch.dtype = torch.float32
+ ) -> torch.Tensor:
+ weight_global_scale = self.weight_global_scale
+ unpacked_data = self.unpack_data(packed_data)
+ unpacked_data = unpacked_data.to(target_dtype) * get_reciprocal(weight_global_scale).to(target_dtype)
+ scale_float = scale.to(target_dtype)
+ original_shape = unpacked_data.shape
+ unpacked_data = unpacked_data.reshape(-1, self.group_size)
+ scale_float = scale_float.reshape(-1, 1)
+ data_dequant = unpacked_data * scale_float
+ data_dequant = data_dequant.reshape(original_shape)
+ return data_dequant
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ dq_weight = self._dequant_nvfp4_tensor(self.weight_packed, self.weight_scale)
+ return dq_weight
+
+ def qdq_input(self, activation: torch.Tensor):
+ original_dtype = activation.dtype
+ temp_qdq_act = _nvfp4_qdq(activation.to(torch.float32), self.config, self.input_global_scale)
+ return temp_qdq_act.to(original_dtype)
+
+ @torch.inference_mode()
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
+ qdq_input = self.qdq_input(input)
+ qdq_weight = self.dequant_weight_online()
+ qdq_weight = qdq_weight.to(qdq_input.dtype)
+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)
+ return out
+
+ @classmethod
+ def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):
+ """
+ Create an `NVFPQuantLinear` layer from an original linear layer.
+ """
+ logger.warning_once("NVFP4 quantization is still in experimental stage, the inference speed might be slow.")
+ qdq_linear = cls(
+ in_features=original_layer.in_features,
+ out_features=original_layer.out_features,
+ config=config,
+ bias=original_layer.bias,
+ dtype=original_layer.weight.dtype,
+ )
+ return qdq_linear
+
+ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:
+ m, half_n = packed_data.shape
+ unpacked_data = unpack_fp4_from_uint8(packed_data, m, half_n * 2, dtype=self.dtype)
+ return unpacked_data
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 1e4036b69..231a5438f 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -51,6 +51,7 @@ class AutoRoundFormat(str, Enum):
FP8_STATIC = "fp8_static"
MXFP8 = "mxfp8"
MXFP4 = "mxfp4"
+ NVFP4 = "nvfp4"
FP8 = "fp8"
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index 6a285c5ae..d536e1a4d 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -145,6 +145,7 @@ class QuantLinear(nn.Module):
scales = scales.to(device)
if self.is_nv:
assert global_scale is not None and global_scale.numel() == 1
+ global_scale = global_scale.reshape([1])
global_scale = global_scale.to(device)
scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(
scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)
@@ -173,7 +174,8 @@ class QuantLinear(nn.Module):
self.weight_global_scale = global_scale.to(torch.float32).to(device)
if input_global_scale is not None:
- self.input_global_scale = input_global_scale.to(torch.float32).to(device)
+ # TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.
+ self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])
return
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 50ec2b120..a8a22534d 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -95,6 +95,7 @@ def feature_multiply_checker_group_size(
feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)
+feature_multiply_checker_16 = functools.partial(feature_multiply_checker, in_feature_multiplier=16)
in_output_feature_multiply_checker_32 = functools.partial(
feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32
)
@@ -152,6 +153,16 @@ def mxfp4_scheme_checker(
return _scheme_checker_common(config, ar_schemes.MXFP4)
+def nvfp4_scheme_checker(
+ in_feature: int,
+ out_feature: int,
+ config: QuantizationScheme,
+ in_feature_multiplier: Optional[int] = None,
+ out_feature_multiplier: Optional[int] = None,
+):
+ return _scheme_checker_common(config, ar_schemes.NVFP4)
+
+
GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1
GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]
AWQ_FORMAT = ["auto_round:auto_awq"]
@@ -221,9 +232,10 @@ BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(
packing_format=LLM_COMPRESSOR_FORMAT,
sym=[True],
dtype=["float32", "float16", "bfloat16"],
+ group_size=[32],
bits=[8],
priority=0,
- checkers=[mxfp8_scheme_checker],
+ checkers=[mxfp8_scheme_checker, feature_multiply_checker_32],
alias=["auto_round", "torch"],
requirements=["auto-round>0.7.0"],
)
@@ -234,9 +246,25 @@ BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(
packing_format=LLM_COMPRESSOR_FORMAT,
sym=[True],
dtype=["float32", "float16", "bfloat16"],
+ group_size=[32],
+ bits=[4],
+ priority=0,
+ checkers=[mxfp4_scheme_checker, feature_multiply_checker_32],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>0.7.0"],
+)
+
+# NVFP4
+
+BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format=LLM_COMPRESSOR_FORMAT,
+ sym=[True],
+ dtype=["float32", "float16", "bfloat16"],
+ group_size=[16],
bits=[4],
priority=0,
- checkers=[mxfp4_scheme_checker],
+ checkers=[nvfp4_scheme_checker, feature_multiply_checker_16],
alias=["auto_round", "torch"],
requirements=["auto-round>0.7.0"],
)
@@ -535,6 +563,8 @@ def dynamic_import_inference_linear(backend, config):
return ar_qmodules.MXFP8QuantLinear
if "torch_mxfp4" in backend:
return ar_qmodules.MXFP4QuantLinear
+ if "torch_nvfp4" in backend:
+ return ar_qmodules.NVFP4QuantLinear
if "qbits" in backend:
try:
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index bebf75df7..3be1d7939 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -397,6 +397,7 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)
AutoRoundFormat.FP8_STATIC.value in layer_backend
or AutoRoundFormat.MXFP8.value in layer_backend
or AutoRoundFormat.MXFP4.value in layer_backend
+ or AutoRoundFormat.NVFP4.value in layer_backend
):
return QuantLinear.from_original(config, layer)
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 909ff5ffd..a5c5975c9 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -166,6 +166,7 @@ NVFP4 = QuantizationScheme.from_dict(
"act_bits": 4,
"act_data_type": "nv_fp4_with_static_gs",
"act_group_size": 16,
+ "act_sym": True,
}
)
diff --git a/test/test_cpu/test_nvfp4_quant_linear.py b/test/test_cpu/test_nvfp4_quant_linear.py
new file mode 100644
index 000000000..0a42f009a
--- /dev/null
+++ b/test/test_cpu/test_nvfp4_quant_linear.py
@@ -0,0 +1,119 @@
+import pytest
+import torch
+
+from auto_round.data_type.nvfp import calculate_gparam
+from auto_round.data_type.utils import get_quant_func
+from auto_round.experimental import qmodules as ar_qmodules
+from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear
+from auto_round.schemes import PRESET_SCHEMES
+
+QMODULE_MAPPING = {
+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
+}
+
+
[email protected](autouse=True)
+def fixed_seed():
+ """Ensure deterministic RNG for every test."""
+ seed = 42
+ print("\nSetting fixed random seed for test:", seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ yield
+ # (Optional) cleanup or reset after test
+
+
[email protected]("scheme", [AutoRoundFormat.NVFP4.value])
+def test_nvfp4_quantlinear_from_original_and_forward(scheme):
+ """
+ Test NVFP4 quantization schemes by creating quantized layers
+ from an original torch.nn.Linear layer and validating their forward pass.
+ """
+
+ # Set random seed for reproducibility
+ torch.manual_seed(42)
+
+ # Define layer dimensions
+ in_features = 64
+ out_features = 512
+
+ # Create an original torch.nn.Linear layer
+ original_layer = torch.nn.Linear(in_features, out_features, bias=False)
+
+ # Select the quantization scheme
+ config = PRESET_SCHEMES[scheme.upper()]
+
+ # Define weight scale shape
+ weight_scale_shape = (out_features, in_features // config.group_size)
+
+ # Quantize the weights using the quantization function
+ weight_qdq_func, _ = get_quant_func(dtype=config.data_type, bits=config.bits, sym=True)
+
+ weight_global_scale = calculate_gparam(original_layer.weight, config.group_size)
+ weight_qdq, weight_scale, _ = weight_qdq_func(
+ original_layer.weight, bits=config.bits, group_size=config.group_size, global_scale=weight_global_scale
+ )
+
+ # Generate a random input tensor
+ input_tensor = torch.randn((4, in_features), dtype=torch.float32)
+ input_global_scale = calculate_gparam(input_tensor, config.act_group_size)
+
+ weight_scale = weight_scale.reshape(weight_scale_shape)
+
+ # Pack the weights using the QuantLinear class
+ kwargs = {"act_bits": config.act_bits}
+ nvfp4_lin = _FPLinear(
+ bits=config.bits,
+ group_size=config.group_size,
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=original_layer.bias is not None,
+ data_type=config.data_type,
+ **kwargs,
+ )
+
+ nvfp4_lin.pack(
+ linear=original_layer,
+ scales=weight_scale,
+ global_scale=weight_global_scale,
+ input_global_scale=input_global_scale,
+ )
+
+ # Create an NVFP4QuantLinear layer from the original layer
+ QuantLinearClass = QMODULE_MAPPING[scheme]
+ nvfp4_layer = QuantLinearClass.from_original(
+ config=config,
+ original_layer=original_layer,
+ )
+
+ # Copy the packed weights and scales to the quantized layer
+ packed_weight = nvfp4_lin.weight_packed if config.bits == 4 else nvfp4_lin.weight
+ nvfp4_layer.weight_packed.data.copy_(packed_weight)
+ nvfp4_layer.weight_scale.data.copy_(nvfp4_lin.weight_scale)
+ nvfp4_layer.weight_global_scale.data.copy_(nvfp4_lin.weight_global_scale)
+ nvfp4_layer.input_global_scale.data.copy_(nvfp4_lin.input_global_scale)
+
+ # Validate layer attributes
+ assert nvfp4_layer.in_features == original_layer.in_features
+ assert nvfp4_layer.out_features == original_layer.out_features
+
+ # Perform a forward pass with both layers
+ original_output = original_layer(input_tensor)
+ nvfp4_output = nvfp4_layer(input_tensor)
+
+ # Compute the difference between the outputs
+ diff = nvfp4_output - original_output
+ # Note: Remove NaN values, as we might get NaN when casting scales to FP8
+
+ diff = diff[~torch.isnan(diff)]
+ diff_amax = diff.abs().max()
+
+ # Print the maximum difference for debugging
+ print(f"Scheme: {scheme}, Max Difference: {diff_amax}")
+
+ # Assert that the outputs are close within a tolerance
+ assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"
diff --git a/test/test_cuda/test_mx_quant.py b/test/test_cuda/test_mxfp_and_nvfp_quant.py
similarity index 92%
rename from test/test_cuda/test_mx_quant.py
rename to test/test_cuda/test_mxfp_and_nvfp_quant.py
index acf58d25c..d15cde5be 100644
--- a/test/test_cuda/test_mx_quant.py
+++ b/test/test_cuda/test_mxfp_and_nvfp_quant.py
@@ -11,10 +11,11 @@ from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround import AutoRoundFormat
from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp
-mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]
+testing_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value, AutoRoundFormat.NVFP4.value]
QMODULE_MAPPING = {
AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
}
@@ -26,7 +27,7 @@ def has_module(model: torch.nn.Module, target_module_type: torch.nn.Module) -> b
return False
[email protected]("scheme", mx_schemes)
[email protected]("scheme", testing_schemes)
@torch.inference_mode()
def test_e2e_quant_and_infer(scheme):
# Use a temporary directory for saving the quantized model