PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
MXFP4 and MXFP8 loading support (#832)
Yi Liuf44d1f7a4f6e
diff --git a/auto_round/experimental/__init__.py b/auto_round/experimental/__init__.pynew file mode 100644index 000000000..0d2740cdb--- /dev/null+++ b/auto_round/experimental/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pynew file mode 100644index 000000000..f95615eba--- /dev/null+++ b/auto_round/experimental/qmodules/__init__.py@@ -0,0 +1,16 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear+from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.pynew file mode 100644index 000000000..24ab8b534--- /dev/null+++ b/auto_round/experimental/qmodules/fp4_utils.py@@ -0,0 +1,98 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch++kE2M1ToFloat = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32)+++def unpack_fp4_from_uint8(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ """+ Unpacks uint8 values into FP4. Each uint8 contains two FP4 values+ (low nibble first). The 4-bit indices are mapped to FP4 values using kE2M1ToFloat.+ """+ if a.device.type == "cuda":+ return _unpack_fp4_from_uint8_cuda(a, m, n, dtype)+ else:+ return _unpack_fp4_from_uint8_cpu(a, m, n, dtype)+++def _unpack_fp4_from_uint8_cpu(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ return _unpack_fp4_from_uint8(a, m, n, dtype)++[email protected](fullgraph=True, dynamic=True)+def _unpack_fp4_from_uint8_cuda(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ return _unpack_fp4_from_uint8(a, m, n, dtype)+++# reference: : https://github.com/vllm-project/vllm/pull/16362+def _unpack_fp4_from_uint8(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ """+ Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values+ (i.e. first four bits correspond to one fp4 value, last four correspond to a+ consecutive fp4 value). The bits represent an index, which are mapped to an fp4+ value.++ :param a: tensor to unpack+ :param m: original dim 0 size of the unpacked tensor+ :param n: original dim 1 size of the unpacked tensor+ :param dtype: dense dtype to cast the unpacked tensor to+ """+ assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"++ # Vectorized nibble processing+ a_flat = a.flatten()+ high = (a_flat & 0xF0) >> 4 # Upper nibbles+ low = a_flat & 0x0F # Lower nibbles++ # Combine nibbles for batch processing+ combined = torch.stack((low, high), dim=1).flatten()++ # Vectorized sign and magnitude extraction+ signs = (combined & 0x08).to(torch.bool) # Sign bits+ abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices++ # Device-aware lookup and sign application+ kE2M1 = kE2M1ToFloat.to(device=a.device)+ values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)++ # Reshape to final form+ return values.reshape(m, n).to(dtype=dtype)diff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.pynew file mode 100644index 000000000..f06076173--- /dev/null+++ b/auto_round/experimental/qmodules/mx.py@@ -0,0 +1,211 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++from typing import Optional, Union++import torch++from auto_round.data_type.utils import get_quant_func+from auto_round.experimental.qmodules.base import QModuleBase+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++__all__ = ["MXFP4QuantLinear", "MXFP8QuantLinear"]++SUPPORTED_HIGHER_DTYPE = [torch.bfloat16, torch.float16, torch.float32]+E8M0_EXPONENT_BIAS = 127+++def _mx_qdq(tensor: torch.Tensor, config: QuantizationScheme):+ qdq_func, _ = get_quant_func(dtype=config.act_data_type, bits=config.act_bits, sym=True)+ qdq_tensor, shared_exp, _ = qdq_func(tensor=tensor, bits=config.act_bits, group_size=config.act_group_size)+ return qdq_tensor+++# https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337+def get_fp_scale(scale_e8m0):+ scale_e8m0 = scale_e8m0.view(torch.uint8)+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS+ two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)+ # TODO(later): handle this for float16 if we decide to support float16+ s_fp = torch.pow(two, s_offset)++ return s_fp+++class MXQuantLinearBase(QModuleBase):+ """+ Base class for quantized linear layers using MXFP quantization schemes.+ """++ def __init__(+ self,+ in_features,+ out_features,+ config: QuantizationScheme,+ weight: Optional[torch.Tensor] = None,+ weight_scale: Optional[torch.Tensor] = None,+ bias: Union[torch.Tensor, bool, None] = None,+ dtype=torch.bfloat16,+ ):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ self.group_size = 32+ self.config = config+ self.dtype = dtype+ self.pre_dequantized = False++ # Validate dtype+ assert (+ dtype in SUPPORTED_HIGHER_DTYPE+ ), f"Expected dtype to be one of {SUPPORTED_HIGHER_DTYPE}, but got {dtype}."++ # Initialize weights+ init_weight = self.initialize_weights(weight)+ self.register_buffer(self.weight_name, init_weight)++ # Initialize bias+ if bias is not None:+ if isinstance(bias, bool):+ bias = torch.zeros((out_features,), dtype=dtype)+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)++ # Initialize weight scale+ init_weight_scale = (+ torch.empty((out_features, in_features // self.group_size), dtype=torch.uint8)+ if weight_scale is None+ else weight_scale+ )+ self.register_buffer("weight_scale", init_weight_scale)++ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ """+ Initialize weights. This method should be overridden by subclasses.+ """+ raise NotImplementedError("Subclasses must implement `initialize_weights`.")++ @classmethod+ def get_min_capability(cls) -> int:+ """+ Get minimum device capability.+ """+ logger.warning_once("MXFP quantization is still in experimental stage, the inference speed might be slow.")+ return 0++ def dequant_mx_tensor(+ self, packed_data: torch.Tensor, scale: torch.Tensor, target_dtype: torch.dtype = torch.float32+ ) -> torch.Tensor:+ scale_float = self._get_float_scale(scale).to(target_dtype)+ unpacked_data = self.unpack_data(packed_data)+ original_shape = unpacked_data.shape+ unpacked_data = unpacked_data.reshape(-1, self.group_size)+ scale_float = scale_float.reshape(-1, 1)+ data_float = unpacked_data.to(target_dtype)+ data_dequant = data_float * scale_float+ data_dequant = data_dequant.reshape(original_shape)+ return data_dequant++ def dequant_weight_online(self):+ if self.pre_dequantized:+ return self.weight+ dq_weight = self.dequant_mx_tensor(self.weight, self.weight_scale)+ return dq_weight++ def pre_dequantize(self):+ if self.pre_dequantized:+ return+ dequant_weight = self.dequant_weight_online()+ delattr(self, self.weight_name)+ del self.weight_scale+ self.weight = torch.nn.Parameter(dequant_weight, requires_grad=False)+ self.pre_dequantized = True++ def qdq_input(self, activation: torch.Tensor):+ return _mx_qdq(activation, self.config)++ @classmethod+ def _get_float_scale(cls, scale_e8m0: torch.Tensor) -> torch.Tensor:+ return get_fp_scale(scale_e8m0)++ @torch.inference_mode()+ def forward(self, input: torch.Tensor) -> torch.Tensor:+ qdq_input = self.qdq_input(input)+ qdq_weight = self.dequant_weight_online()+ qdq_weight = qdq_weight.to(qdq_input.dtype)+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)+ return out++ @classmethod+ def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ """+ Create an `MXQuantLinear` layer from an original linear layer.+ """+ logger.warning_once("MXFP quantization is still in experimental stage, the inference speed might be slow.")+ qdq_linear = cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ config=config,+ bias=original_layer.bias,+ dtype=original_layer.weight.dtype,+ )+ return qdq_linear+++class MXFP4QuantLinear(MXQuantLinearBase):+ """+ Quantized linear layer using the MXFP4 quantization scheme.+ """++ def __init__(self, *args, **kwargs):+ self.weight_name = "weight_packed"+ super().__init__(*args, **kwargs)++ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ weight_dtype = torch.uint8+ weight_in_features = self.in_features // 2+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight++ def dequant_weight_online(self) -> torch.Tensor:+ if self.pre_dequantized:+ return self.weight+ dq_weight = self.dequant_mx_tensor(self.weight_packed, self.weight_scale)+ return dq_weight++ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:+ m, half_n = packed_data.shape+ unpacked_data = unpack_fp4_from_uint8(packed_data, m, half_n * 2, dtype=self.dtype)+ return unpacked_data+++class MXFP8QuantLinear(MXQuantLinearBase):+ """+ Quantized linear layer using the MXFP8 quantization scheme.+ """++ def __init__(self, *args, **kwargs):+ self.weight_name = "weight"+ super().__init__(*args, **kwargs)++ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ weight_dtype = torch.float8_e4m3fn+ weight_in_features = self.in_features+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight++ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:+ return packed_data.to(self.dtype)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 7f134276b..12ed39bd8 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -49,6 +49,8 @@ class AutoRoundFormat(str, Enum):# Weight: FP8, per-channel, may be extended to per-tensor in future# Activation: FP8, per-tensorFP8_STATIC = "fp8_static"+ MXFP8 = "mxfp8"+ MXFP4 = "mxfp4"FP8 = "fp8"diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 2c74e575e..6a285c5ae 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -90,15 +90,17 @@ class QuantLinear(nn.Module):self.act_bits = kwargs.get("act_bits", None)weight_name = "weight" if self.bits == 8 and self.is_mx else "weight_packed"+ weight_infeatures = infeatures if self.bits == 8 else infeatures // 2+ weight_dtype = torch.float8_e4m3fn if self.bits == 8 else torch.uint8## TODO check the dtype of weight_packed and weight_scaleself.register_buffer(weight_name,- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ torch.zeros((outfeatures, weight_infeatures), dtype=weight_dtype),)self.register_buffer("weight_scale",torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),+ (outfeatures, math.ceil(infeatures / self.group_size)),dtype=torch.float16, ## TODO update to correct scale dtype for different bits),)@@ -156,11 +158,13 @@ class QuantLinear(nn.Module):final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)else:final_scale = scales.to(torch.float8_e4m3fn)+self.weight_scale = final_scale# self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODOif self.bits == 8:compress_dtype = torch.float8_e4m3fnself.weight = scaled_tensor.to(compress_dtype)+else:compress_dtype = torch.uint8self.weight_packed = pack_fp4_to_uint8(scaled_tensor)diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 139a98120..ce3e209cf 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -19,9 +19,12 @@ from typing import Any, List, Optionalfrom transformers.utils.versions import require_versionimport auto_round_extension.cuda.gptqmodel_marlin+from auto_round import schemes as ar_schemes+from auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme-from auto_round.utils import get_library_version, logger+from auto_round.utils import get_library_versionBackendInfos = {}@@ -121,6 +124,38 @@ def fp8_static_scheme_checker(return config == FP8_STATIC+def _scheme_checker_common(config1: QuantizationScheme, config2: QuantizationScheme):+ SCHEME_CHECK_ATTRS = ["bits", "group_size", "sym", "data_type", "act_bits", "act_group_size", "act_sym"]++ for attr in SCHEME_CHECK_ATTRS:+ if getattr(config1, attr) != getattr(config2, attr):+ logger.debug(+ f"Scheme check failed on attribute {attr}: {getattr(config1, attr)} != {getattr(config2, attr)}"+ )+ return False+ return True+++def mxfp8_scheme_checker(+ in_feature: int,+ out_feature: int,+ config: QuantizationScheme,+ in_feature_multiplier: Optional[int] = None,+ out_feature_multiplier: Optional[int] = None,+):+ return _scheme_checker_common(config, ar_schemes.MXFP8)+++def mxfp4_scheme_checker(+ in_feature: int,+ out_feature: int,+ config: QuantizationScheme,+ in_feature_multiplier: Optional[int] = None,+ out_feature_multiplier: Optional[int] = None,+):+ return _scheme_checker_common(config, ar_schemes.MXFP4)++BackendInfos["auto_gptq:exllamav2"] = BackendInfo(device=["cuda"],sym=[True, False],@@ -205,6 +240,34 @@ BackendInfos["auto_round:fp8_static"] = BackendInfo(requirements=["auto-round>0.6.0"],)+# MXFP8++BackendInfos["auto_round:mxfp8"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format="",+ sym=[True],+ dtype=["float32", "float16", "bfloat16"],+ bits=[8],+ priority=0,+ checkers=[mxfp8_scheme_checker],+ alias=["torch"],+ requirements=["auto-round>0.7.0"],+)++# MXFP4++BackendInfos["auto_round:mxfp4"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format="",+ sym=[True],+ dtype=["float32", "float16", "bfloat16"],+ bits=[4],+ priority=0,+ checkers=[mxfp4_scheme_checker],+ alias=["auto_round:llm_compressor"],+ requirements=["auto-round>0.7.0"],+)+BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(device=["cuda", "xpu"],sym=[True], ## asym has accuracys@@ -472,9 +535,11 @@ def dynamic_import_inference_linear(backend, config):bits, group_size, sym = config["bits"], config["group_size"], config["sym"]if AutoRoundFormat.FP8_STATIC.value in backend:- from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear-- return WeightFP8ActFP8StaticQuantLinear+ return ar_qmodules.WeightFP8ActFP8StaticQuantLinear+ if AutoRoundFormat.MXFP8.value in backend:+ return ar_qmodules.MXFP8QuantLinear+ if AutoRoundFormat.MXFP4.value in backend:+ return ar_qmodules.MXFP4QuantLinearif "qbits" in backend:try:@@ -680,6 +745,7 @@ def find_backend(target_backend: str, orig_backend: str = None):Returns:str or None: Matching backend key if found and compatible; otherwise, None."""+ logger.trace(f"Finding backend for target: {target_backend}, original: {orig_backend}")matched_keys = [keyfor key, info in BackendInfos.items()@@ -758,7 +824,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe"""# Check if the provided backend is in BackendInfosbackend = find_backend(backend)-+ logger.trace(f"Found backend: {backend} for device: {device}, orig_backend: {orig_backend}")if backend not in BackendInfos.keys():raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")@@ -768,6 +834,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fesupported_backends = []for key in BackendInfos.keys():if check_compatible(key, device, config, packing_format, in_features, out_features):+ logger.trace(f"Backend {key} is compatible")supported_backends.append(key)# Raise an error if no compatible backends are founddiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 827b1744c..c29b30482 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -30,6 +30,7 @@ from auto_round.inference.backend import (get_layer_backend,process_requirement,)+from auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -43,8 +44,6 @@ from auto_round.utils import (set_module,)-logger = getLogger(__name__)-supported_devices = ("cpu", "hpu", "xpu", "cuda")@@ -378,6 +377,7 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, tlayer_backend = _get_layer_backend(target_device, target_backend, orig_backend, config, in_features, out_features)+ logger.trace(f"Got backend {layer_backend} for {layer_name}.")backend_cache[key] = layer_backendif not layer_backend:@@ -452,7 +452,11 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)out_features=out_features,bias=bias,)- elif AutoRoundFormat.FP8_STATIC.value in layer_backend:+ elif (+ AutoRoundFormat.FP8_STATIC.value in layer_backend+ or AutoRoundFormat.MXFP8.value in layer_backend+ or AutoRoundFormat.MXFP4.value in layer_backend+ ):return QuantLinear.from_original(config, layer)# Default quantized layer creationtry:@@ -598,6 +602,7 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):layer_configs = get_layer_config(model, quantization_config)if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):packing_format = packing_format[len("auto_round:") :]+orig_backend = find_backend(packing_format)if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 053a77523..ab804b0a8 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -143,6 +143,7 @@ MXFP4 = QuantizationScheme.from_dict("act_bits": 4,"act_data_type": "mx_fp_rceil","act_group_size": 32,+ "act_sym": True,})@@ -154,6 +155,7 @@ MXFP8 = QuantizationScheme.from_dict("act_bits": 8,"act_data_type": "mx_fp_rceil","act_group_size": 32,+ "act_sym": True,})diff --git a/test/test_cpu/test_mx_quant_linear.py b/test/test_cpu/test_mx_quant_linear.pynew file mode 100644index 000000000..e8e18c3bb--- /dev/null+++ b/test/test_cpu/test_mx_quant_linear.py@@ -0,0 +1,109 @@+import pytest+import torch++from auto_round.data_type.utils import get_quant_func+from auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear+from auto_round.schemes import PRESET_SCHEMES++mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]+QMODULE_MAPPING = {+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+}++[email protected](autouse=True)+def fixed_seed():+ """Ensure deterministic RNG for every test."""+ seed = 42+ print("\nSetting fixed random seed for test:", seed)+ torch.manual_seed(seed)+ torch.cuda.manual_seed_all(seed)+ torch.backends.cudnn.deterministic = True+ torch.backends.cudnn.benchmark = False+ yield+ # (Optional) cleanup or reset after test++[email protected]("scheme", mx_schemes)[email protected]_mode()+def test_mxquantlinear_from_original_and_forward(scheme):+ """+ Test MXFP4 and MXFP8 quantization schemes by creating quantized layers+ from an original torch.nn.Linear layer and validating their forward pass.+ """+ # Set random seed for reproducibility+ torch.manual_seed(42)++ # Define layer dimensions+ in_features = 64+ out_features = 512++ # Create an original torch.nn.Linear layer+ original_layer = torch.nn.Linear(in_features, out_features, bias=False)++ # Select the quantization scheme+ config = PRESET_SCHEMES[scheme.upper()]++ # Define weight scale shape+ weight_scale_shape = (out_features, in_features // config.group_size)++ # Quantize the weights using the quantization function+ qdq_func, _ = get_quant_func(dtype=config.act_data_type, bits=config.act_bits, sym=True)+ qdq_weight, shared_exp, _ = qdq_func(+ tensor=original_layer.weight, bits=config.act_bits, group_size=config.act_group_size+ )+ shared_exp = shared_exp.reshape(weight_scale_shape)++ # Pack the weights using the QuantLinear class+ mxfp_lin = _MXFPLinear(+ bits=config.bits,+ group_size=config.group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=original_layer.bias is not None,+ data_type=config.data_type,+ )+ mxfp_lin.pack(linear=original_layer, scales=shared_exp)++ # Create an MXQuantLinear layer from the original layer+ QuantLinearClass = QMODULE_MAPPING[scheme]+ mxfp_layer = QuantLinearClass.from_original(+ config=config,+ original_layer=original_layer,+ )++ # Copy the packed weights and scales to the quantized layer+ packed_weight = mxfp_lin.weight_packed if config.bits == 4 else mxfp_lin.weight+ if config.bits == 4:+ mxfp_layer.weight_packed.data.copy_(packed_weight)+ elif config.bits == 8:+ mxfp_layer.weight.data.copy_(packed_weight)+ else:+ raise ValueError("Only 4-bit and 8-bit quantization are supported.")+ mxfp_layer.weight_scale.data.copy_(mxfp_lin.weight_scale)++ # Validate layer attributes+ assert mxfp_layer.in_features == original_layer.in_features+ assert mxfp_layer.out_features == original_layer.out_features++ # Generate a random input tensor+ input_tensor = torch.randn((4, in_features), dtype=torch.float32)++ # Perform a forward pass with both layers+ original_output = original_layer(input_tensor)+ mx_output = mxfp_layer(input_tensor)++ # Compute the difference between the outputs+ diff = mx_output - original_output+ # Note: Remove NaN values, as we might get NaN when casting scales to FP8+ diff = diff[~torch.isnan(diff)]+ diff_amax = diff.abs().max()++ # Print the maximum difference for debugging+ print(f"Scheme: {scheme}, Max Difference: {diff_amax}")++ # Assert that the outputs are close within a tolerance+ assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"diff --git a/test/test_cuda/test_mx_quant.py b/test/test_cuda/test_mx_quant.pynew file mode 100644index 000000000..acf58d25c--- /dev/null+++ b/test/test_cuda/test_mx_quant.py@@ -0,0 +1,82 @@+import shutil+import tempfile++import pytest+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round import schemes as ar_schemes+from auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp++mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]+QMODULE_MAPPING = {+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+}+++def has_module(model: torch.nn.Module, target_module_type: torch.nn.Module) -> bool:+ """Check if the model contains a specific module type."""+ for _, module in model.named_modules():+ if isinstance(module, target_module_type):+ return True+ return False++[email protected]("scheme", mx_schemes)[email protected]_mode()+def test_e2e_quant_and_infer(scheme):+ # Use a temporary directory for saving the quantized model+ with tempfile.TemporaryDirectory() as temp_dir:+ model_name = "Qwen/Qwen2.5-0.5B-Instruct"++ # Load the tokenizer and model+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ device_map="cpu",+ torch_dtype="auto",+ trust_remote_code=True,+ )++ # Initialize AutoRound for quantization+ autoround = AutoRound(+ model,+ tokenizer,+ scheme=scheme,+ iters=0,+ nsamples=2,+ )++ # Quantize and save the model to the temporary directory+ quantized_model_path = f"{temp_dir}/tmp_autoround_{scheme}"+ autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)++ # Perform inference with the quantized model+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ )+ model.eval()+ assert has_module(model, QMODULE_MAPPING[scheme]), f"Expected {QMODULE_MAPPING[scheme].__name__} in the model."++ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ prompt = "Ai is "++ # Tokenize the input prompt+ encode = tokenizer.encode(prompt, return_tensors="pt")++ # Generate output tokens+ output_tokens = model.generate(+ encode,+ max_length=30,+ )+ output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)++ # Print and validate the output+ print(f"Prompt: {prompt}")+ print(f"Output: {output}")+ assert output is not None, "Output should not be None"