PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
NVFP4 Loading support (#839)
Yi Liu075133771f0e
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pyindex f95615eba..0179dd646 100644--- a/auto_round/experimental/qmodules/__init__.py+++ b/auto_round/experimental/qmodules/__init__.py@@ -13,4 +13,5 @@# limitations under the License.from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear+from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinearfrom auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/nvfp4.py b/auto_round/experimental/qmodules/nvfp4.pynew file mode 100644index 000000000..88ea577e2--- /dev/null+++ b/auto_round/experimental/qmodules/nvfp4.py@@ -0,0 +1,199 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++from typing import Optional, Union++import torch++from auto_round.data_type.nvfp import get_reciprocal, ref_nvfp4_quant+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.experimental.qmodules.base import QModuleBase+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++__all__ = ["NVFP4QuantLinear"]+++# Adapted from auto_round/data_type/nvfp.py+def _nv_fp4_with_static_gs(+ tensor: torch.Tensor, global_scale: torch.Tensor, bits: int = 4, group_size: int = 16+) -> tuple[torch.Tensor, torch.Tensor, None]:+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)++ global_scale = global_scale.to(tensor.device)+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v=0)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None+++def _nvfp4_qdq(tensor: torch.Tensor, config: QuantizationScheme, global_scale: torch.Tensor) -> torch.Tensor:+ qdq_tensor, scales, _ = _nv_fp4_with_static_gs(+ tensor=tensor, global_scale=global_scale, bits=config.act_bits, group_size=config.act_group_size+ )+ return qdq_tensor+++class NVFP4QuantLinear(QModuleBase):+ """+ Quantized linear layer using NVFP4 quantization scheme.+ """++ SUPPORTED_COMPUTE_DTYPE = [torch.bfloat16, torch.float16, torch.float32]++ def __init__(+ self,+ in_features: int,+ out_features: int,+ config: QuantizationScheme,+ weight: Optional[torch.Tensor] = None,+ weight_scale: Optional[torch.Tensor] = None,+ bias: Union[torch.Tensor, bool, None] = None,+ dtype=torch.bfloat16,+ ):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ self.group_size = 16+ self.config = config+ self.dtype = dtype+ self.pre_dequantized = False++ # Validate dtype+ assert (+ dtype in self.SUPPORTED_COMPUTE_DTYPE+ ), f"Expected dtype to be one of {self.SUPPORTED_COMPUTE_DTYPE}, but got {dtype}."++ # check group size+ assert self.group_size == config.group_size, f"Group size mismatch: {self.group_size} vs {config.group_size}"+ assert (+ self.group_size == config.act_group_size+ ), f"Group size mismatch: {self.group_size} vs {config.act_group_size}"++ # Initialize weights+ init_weight = self.initialize_weights(weight)+ self.register_buffer("weight_packed", init_weight)++ # Initialize bias+ if bias is not None:+ if isinstance(bias, bool):+ bias = torch.zeros((out_features,), dtype=dtype)+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)++ # Initialize weight scale+ init_weight_scale = (+ torch.empty((out_features, in_features // self.group_size), dtype=torch.float8_e4m3fn)+ if weight_scale is None+ else weight_scale+ )+ self.register_buffer("weight_scale", init_weight_scale)++ self.register_buffer(+ "weight_global_scale",+ torch.zeros(+ (1,),+ dtype=torch.float32,+ ),+ )+ self.register_buffer(+ "input_global_scale",+ torch.zeros(+ (1,),+ dtype=torch.float32,+ ),+ )++ @staticmethod+ def _convert_global_scale_to_float32(state_dict: dict[str, torch.Tensor], name: str):+ if name not in state_dict or state_dict[name].dtype == torch.float32:+ return+ original_scale = state_dict[name]+ state_dict[name] = original_scale.to(torch.float32)+ logger.warning_once("Forcing global scale to float32 for better precision.")++ def load_state_dict(self, state_dict, strict=True, assign=False):+ self._convert_global_scale_to_float32(state_dict, "weight_global_scale")+ self._convert_global_scale_to_float32(state_dict, "input_global_scale")+ return super().load_state_dict(state_dict, strict, assign)++ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ """+ Initialize weights.+ """+ weight_dtype = torch.uint8+ weight_in_features = self.in_features // 2+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight++ @classmethod+ def get_min_capability(cls) -> int:+ """+ Get minimum device capability.+ """+ logger.warning_once("NVFP4 quantization is still in experimental stage, the inference speed might be slow.")+ return 0++ def _dequant_nvfp4_tensor(+ self, packed_data: torch.Tensor, scale: torch.Tensor, target_dtype: torch.dtype = torch.float32+ ) -> torch.Tensor:+ weight_global_scale = self.weight_global_scale+ unpacked_data = self.unpack_data(packed_data)+ unpacked_data = unpacked_data.to(target_dtype) * get_reciprocal(weight_global_scale).to(target_dtype)+ scale_float = scale.to(target_dtype)+ original_shape = unpacked_data.shape+ unpacked_data = unpacked_data.reshape(-1, self.group_size)+ scale_float = scale_float.reshape(-1, 1)+ data_dequant = unpacked_data * scale_float+ data_dequant = data_dequant.reshape(original_shape)+ return data_dequant++ def dequant_weight_online(self) -> torch.Tensor:+ dq_weight = self._dequant_nvfp4_tensor(self.weight_packed, self.weight_scale)+ return dq_weight++ def qdq_input(self, activation: torch.Tensor):+ original_dtype = activation.dtype+ temp_qdq_act = _nvfp4_qdq(activation.to(torch.float32), self.config, self.input_global_scale)+ return temp_qdq_act.to(original_dtype)++ @torch.inference_mode()+ def forward(self, input: torch.Tensor) -> torch.Tensor:+ qdq_input = self.qdq_input(input)+ qdq_weight = self.dequant_weight_online()+ qdq_weight = qdq_weight.to(qdq_input.dtype)+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)+ return out++ @classmethod+ def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ """+ Create an `NVFPQuantLinear` layer from an original linear layer.+ """+ logger.warning_once("NVFP4 quantization is still in experimental stage, the inference speed might be slow.")+ qdq_linear = cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ config=config,+ bias=original_layer.bias,+ dtype=original_layer.weight.dtype,+ )+ return qdq_linear++ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:+ m, half_n = packed_data.shape+ unpacked_data = unpack_fp4_from_uint8(packed_data, m, half_n * 2, dtype=self.dtype)+ return unpacked_datadiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 1e4036b69..231a5438f 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -51,6 +51,7 @@ class AutoRoundFormat(str, Enum):FP8_STATIC = "fp8_static"MXFP8 = "mxfp8"MXFP4 = "mxfp4"+ NVFP4 = "nvfp4"FP8 = "fp8"diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 6a285c5ae..d536e1a4d 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -145,6 +145,7 @@ class QuantLinear(nn.Module):scales = scales.to(device)if self.is_nv:assert global_scale is not None and global_scale.numel() == 1+ global_scale = global_scale.reshape([1])global_scale = global_scale.to(device)scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)@@ -173,7 +174,8 @@ class QuantLinear(nn.Module):self.weight_global_scale = global_scale.to(torch.float32).to(device)if input_global_scale is not None:- self.input_global_scale = input_global_scale.to(torch.float32).to(device)+ # TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.+ self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])returndiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 50ec2b120..a8a22534d 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -95,6 +95,7 @@ def feature_multiply_checker_group_size(feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)+feature_multiply_checker_16 = functools.partial(feature_multiply_checker, in_feature_multiplier=16)in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32)@@ -152,6 +153,16 @@ def mxfp4_scheme_checker(return _scheme_checker_common(config, ar_schemes.MXFP4)+def nvfp4_scheme_checker(+ in_feature: int,+ out_feature: int,+ config: QuantizationScheme,+ in_feature_multiplier: Optional[int] = None,+ out_feature_multiplier: Optional[int] = None,+):+ return _scheme_checker_common(config, ar_schemes.NVFP4)++GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]AWQ_FORMAT = ["auto_round:auto_awq"]@@ -221,9 +232,10 @@ BackendInfos["auto_round:torch_mxfp8"] = BackendInfo(packing_format=LLM_COMPRESSOR_FORMAT,sym=[True],dtype=["float32", "float16", "bfloat16"],+ group_size=[32],bits=[8],priority=0,- checkers=[mxfp8_scheme_checker],+ checkers=[mxfp8_scheme_checker, feature_multiply_checker_32],alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)@@ -234,9 +246,25 @@ BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(packing_format=LLM_COMPRESSOR_FORMAT,sym=[True],dtype=["float32", "float16", "bfloat16"],+ group_size=[32],+ bits=[4],+ priority=0,+ checkers=[mxfp4_scheme_checker, feature_multiply_checker_32],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.7.0"],+)++# NVFP4++BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format=LLM_COMPRESSOR_FORMAT,+ sym=[True],+ dtype=["float32", "float16", "bfloat16"],+ group_size=[16],bits=[4],priority=0,- checkers=[mxfp4_scheme_checker],+ checkers=[nvfp4_scheme_checker, feature_multiply_checker_16],alias=["auto_round", "torch"],requirements=["auto-round>0.7.0"],)@@ -535,6 +563,8 @@ def dynamic_import_inference_linear(backend, config):return ar_qmodules.MXFP8QuantLinearif "torch_mxfp4" in backend:return ar_qmodules.MXFP4QuantLinear+ if "torch_nvfp4" in backend:+ return ar_qmodules.NVFP4QuantLinearif "qbits" in backend:try:diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex bebf75df7..3be1d7939 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -397,6 +397,7 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)AutoRoundFormat.FP8_STATIC.value in layer_backendor AutoRoundFormat.MXFP8.value in layer_backendor AutoRoundFormat.MXFP4.value in layer_backend+ or AutoRoundFormat.NVFP4.value in layer_backend):return QuantLinear.from_original(config, layer)diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 909ff5ffd..a5c5975c9 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -166,6 +166,7 @@ NVFP4 = QuantizationScheme.from_dict("act_bits": 4,"act_data_type": "nv_fp4_with_static_gs","act_group_size": 16,+ "act_sym": True,})diff --git a/test/test_cpu/test_nvfp4_quant_linear.py b/test/test_cpu/test_nvfp4_quant_linear.pynew file mode 100644index 000000000..0a42f009a--- /dev/null+++ b/test/test_cpu/test_nvfp4_quant_linear.py@@ -0,0 +1,119 @@+import pytest+import torch++from auto_round.data_type.nvfp import calculate_gparam+from auto_round.data_type.utils import get_quant_func+from auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear+from auto_round.schemes import PRESET_SCHEMES++QMODULE_MAPPING = {+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+}++[email protected](autouse=True)+def fixed_seed():+ """Ensure deterministic RNG for every test."""+ seed = 42+ print("\nSetting fixed random seed for test:", seed)+ torch.manual_seed(seed)+ torch.cuda.manual_seed_all(seed)+ torch.backends.cudnn.deterministic = True+ torch.backends.cudnn.benchmark = False+ yield+ # (Optional) cleanup or reset after test++[email protected]("scheme", [AutoRoundFormat.NVFP4.value])[email protected]_mode()+def test_nvfp4_quantlinear_from_original_and_forward(scheme):+ """+ Test NVFP4 quantization schemes by creating quantized layers+ from an original torch.nn.Linear layer and validating their forward pass.+ """++ # Set random seed for reproducibility+ torch.manual_seed(42)++ # Define layer dimensions+ in_features = 64+ out_features = 512++ # Create an original torch.nn.Linear layer+ original_layer = torch.nn.Linear(in_features, out_features, bias=False)++ # Select the quantization scheme+ config = PRESET_SCHEMES[scheme.upper()]++ # Define weight scale shape+ weight_scale_shape = (out_features, in_features // config.group_size)++ # Quantize the weights using the quantization function+ weight_qdq_func, _ = get_quant_func(dtype=config.data_type, bits=config.bits, sym=True)++ weight_global_scale = calculate_gparam(original_layer.weight, config.group_size)+ weight_qdq, weight_scale, _ = weight_qdq_func(+ original_layer.weight, bits=config.bits, group_size=config.group_size, global_scale=weight_global_scale+ )++ # Generate a random input tensor+ input_tensor = torch.randn((4, in_features), dtype=torch.float32)+ input_global_scale = calculate_gparam(input_tensor, config.act_group_size)++ weight_scale = weight_scale.reshape(weight_scale_shape)++ # Pack the weights using the QuantLinear class+ kwargs = {"act_bits": config.act_bits}+ nvfp4_lin = _FPLinear(+ bits=config.bits,+ group_size=config.group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=original_layer.bias is not None,+ data_type=config.data_type,+ **kwargs,+ )++ nvfp4_lin.pack(+ linear=original_layer,+ scales=weight_scale,+ global_scale=weight_global_scale,+ input_global_scale=input_global_scale,+ )++ # Create an NVFP4QuantLinear layer from the original layer+ QuantLinearClass = QMODULE_MAPPING[scheme]+ nvfp4_layer = QuantLinearClass.from_original(+ config=config,+ original_layer=original_layer,+ )++ # Copy the packed weights and scales to the quantized layer+ packed_weight = nvfp4_lin.weight_packed if config.bits == 4 else nvfp4_lin.weight+ nvfp4_layer.weight_packed.data.copy_(packed_weight)+ nvfp4_layer.weight_scale.data.copy_(nvfp4_lin.weight_scale)+ nvfp4_layer.weight_global_scale.data.copy_(nvfp4_lin.weight_global_scale)+ nvfp4_layer.input_global_scale.data.copy_(nvfp4_lin.input_global_scale)++ # Validate layer attributes+ assert nvfp4_layer.in_features == original_layer.in_features+ assert nvfp4_layer.out_features == original_layer.out_features++ # Perform a forward pass with both layers+ original_output = original_layer(input_tensor)+ nvfp4_output = nvfp4_layer(input_tensor)++ # Compute the difference between the outputs+ diff = nvfp4_output - original_output+ # Note: Remove NaN values, as we might get NaN when casting scales to FP8++ diff = diff[~torch.isnan(diff)]+ diff_amax = diff.abs().max()++ # Print the maximum difference for debugging+ print(f"Scheme: {scheme}, Max Difference: {diff_amax}")++ # Assert that the outputs are close within a tolerance+ assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"diff --git a/test/test_cuda/test_mx_quant.py b/test/test_cuda/test_mxfp_and_nvfp_quant.pysimilarity index 92%rename from test/test_cuda/test_mx_quant.pyrename to test/test_cuda/test_mxfp_and_nvfp_quant.pyindex acf58d25c..d15cde5be 100644--- a/test/test_cuda/test_mx_quant.py+++ b/test/test_cuda/test_mxfp_and_nvfp_quant.py@@ -11,10 +11,11 @@ from auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp-mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]+testing_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value, AutoRoundFormat.NVFP4.value]QMODULE_MAPPING = {AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}@@ -26,7 +27,7 @@ def has_module(model: torch.nn.Module, target_module_type: torch.nn.Module) -> breturn False[email protected]("scheme", mx_schemes)[email protected]("scheme", testing_schemes)@torch.inference_mode()def test_e2e_quant_and_infer(scheme):# Use a temporary directory for saving the quantized model