Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

MXFP4 and MXFP8 loading support (#832)

Yi Liuf44d1f7a4f6e
diff --git a/auto_round/experimental/__init__.py b/auto_round/experimental/__init__.py
new file mode 100644
index 000000000..0d2740cdb
--- /dev/null
+++ b/auto_round/experimental/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
new file mode 100644
index 000000000..f95615eba
--- /dev/null
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear
+from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.py
new file mode 100644
index 000000000..24ab8b534
--- /dev/null
+++ b/auto_round/experimental/qmodules/fp4_utils.py
@@ -0,0 +1,98 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+
+kE2M1ToFloat = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32)
+
+
+def unpack_fp4_from_uint8(
+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
+) -> torch.Tensor:
+ """
+ Unpacks uint8 values into FP4. Each uint8 contains two FP4 values
+ (low nibble first). The 4-bit indices are mapped to FP4 values using kE2M1ToFloat.
+ """
+ if a.device.type == "cuda":
+ return _unpack_fp4_from_uint8_cuda(a, m, n, dtype)
+ else:
+ return _unpack_fp4_from_uint8_cpu(a, m, n, dtype)
+
+
+def _unpack_fp4_from_uint8_cpu(
+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
+) -> torch.Tensor:
+ return _unpack_fp4_from_uint8(a, m, n, dtype)
+
+
[email protected](fullgraph=True, dynamic=True)
+def _unpack_fp4_from_uint8_cuda(
+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
+) -> torch.Tensor:
+ return _unpack_fp4_from_uint8(a, m, n, dtype)
+
+
+# reference: : https://github.com/vllm-project/vllm/pull/16362
+def _unpack_fp4_from_uint8(
+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
+) -> torch.Tensor:
+ """
+ Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values
+ (i.e. first four bits correspond to one fp4 value, last four correspond to a
+ consecutive fp4 value). The bits represent an index, which are mapped to an fp4
+ value.
+
+ :param a: tensor to unpack
+ :param m: original dim 0 size of the unpacked tensor
+ :param n: original dim 1 size of the unpacked tensor
+ :param dtype: dense dtype to cast the unpacked tensor to
+ """
+ assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"
+
+ # Vectorized nibble processing
+ a_flat = a.flatten()
+ high = (a_flat & 0xF0) >> 4 # Upper nibbles
+ low = a_flat & 0x0F # Lower nibbles
+
+ # Combine nibbles for batch processing
+ combined = torch.stack((low, high), dim=1).flatten()
+
+ # Vectorized sign and magnitude extraction
+ signs = (combined & 0x08).to(torch.bool) # Sign bits
+ abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices
+
+ # Device-aware lookup and sign application
+ kE2M1 = kE2M1ToFloat.to(device=a.device)
+ values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)
+
+ # Reshape to final form
+ return values.reshape(m, n).to(dtype=dtype)
diff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.py
new file mode 100644
index 000000000..f06076173
--- /dev/null
+++ b/auto_round/experimental/qmodules/mx.py
@@ -0,0 +1,211 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Optional, Union
+
+import torch
+
+from auto_round.data_type.utils import get_quant_func
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+__all__ = ["MXFP4QuantLinear", "MXFP8QuantLinear"]
+
+SUPPORTED_HIGHER_DTYPE = [torch.bfloat16, torch.float16, torch.float32]
+E8M0_EXPONENT_BIAS = 127
+
+
+def _mx_qdq(tensor: torch.Tensor, config: QuantizationScheme):
+ qdq_func, _ = get_quant_func(dtype=config.act_data_type, bits=config.act_bits, sym=True)
+ qdq_tensor, shared_exp, _ = qdq_func(tensor=tensor, bits=config.act_bits, group_size=config.act_group_size)
+ return qdq_tensor
+
+
+# https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337
+def get_fp_scale(scale_e8m0):
+ scale_e8m0 = scale_e8m0.view(torch.uint8)
+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
+ two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
+ # TODO(later): handle this for float16 if we decide to support float16
+ s_fp = torch.pow(two, s_offset)
+
+ return s_fp
+
+
+class MXQuantLinearBase(QModuleBase):
+ """
+ Base class for quantized linear layers using MXFP quantization schemes.
+ """
+
+ def __init__(
+ self,
+ in_features,
+ out_features,
+ config: QuantizationScheme,
+ weight: Optional[torch.Tensor] = None,
+ weight_scale: Optional[torch.Tensor] = None,
+ bias: Union[torch.Tensor, bool, None] = None,
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.group_size = 32
+ self.config = config
+ self.dtype = dtype
+ self.pre_dequantized = False
+
+ # Validate dtype
+ assert (
+ dtype in SUPPORTED_HIGHER_DTYPE
+ ), f"Expected dtype to be one of {SUPPORTED_HIGHER_DTYPE}, but got {dtype}."
+
+ # Initialize weights
+ init_weight = self.initialize_weights(weight)
+ self.register_buffer(self.weight_name, init_weight)
+
+ # Initialize bias
+ if bias is not None:
+ if isinstance(bias, bool):
+ bias = torch.zeros((out_features,), dtype=dtype)
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+
+ # Initialize weight scale
+ init_weight_scale = (
+ torch.empty((out_features, in_features // self.group_size), dtype=torch.uint8)
+ if weight_scale is None
+ else weight_scale
+ )
+ self.register_buffer("weight_scale", init_weight_scale)
+
+ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:
+ """
+ Initialize weights. This method should be overridden by subclasses.
+ """
+ raise NotImplementedError("Subclasses must implement `initialize_weights`.")
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ """
+ Get minimum device capability.
+ """
+ logger.warning_once("MXFP quantization is still in experimental stage, the inference speed might be slow.")
+ return 0
+
+ def dequant_mx_tensor(
+ self, packed_data: torch.Tensor, scale: torch.Tensor, target_dtype: torch.dtype = torch.float32
+ ) -> torch.Tensor:
+ scale_float = self._get_float_scale(scale).to(target_dtype)
+ unpacked_data = self.unpack_data(packed_data)
+ original_shape = unpacked_data.shape
+ unpacked_data = unpacked_data.reshape(-1, self.group_size)
+ scale_float = scale_float.reshape(-1, 1)
+ data_float = unpacked_data.to(target_dtype)
+ data_dequant = data_float * scale_float
+ data_dequant = data_dequant.reshape(original_shape)
+ return data_dequant
+
+ def dequant_weight_online(self):
+ if self.pre_dequantized:
+ return self.weight
+ dq_weight = self.dequant_mx_tensor(self.weight, self.weight_scale)
+ return dq_weight
+
+ def pre_dequantize(self):
+ if self.pre_dequantized:
+ return
+ dequant_weight = self.dequant_weight_online()
+ delattr(self, self.weight_name)
+ del self.weight_scale
+ self.weight = torch.nn.Parameter(dequant_weight, requires_grad=False)
+ self.pre_dequantized = True
+
+ def qdq_input(self, activation: torch.Tensor):
+ return _mx_qdq(activation, self.config)
+
+ @classmethod
+ def _get_float_scale(cls, scale_e8m0: torch.Tensor) -> torch.Tensor:
+ return get_fp_scale(scale_e8m0)
+
+ @torch.inference_mode()
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
+ qdq_input = self.qdq_input(input)
+ qdq_weight = self.dequant_weight_online()
+ qdq_weight = qdq_weight.to(qdq_input.dtype)
+ out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)
+ return out
+
+ @classmethod
+ def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):
+ """
+ Create an `MXQuantLinear` layer from an original linear layer.
+ """
+ logger.warning_once("MXFP quantization is still in experimental stage, the inference speed might be slow.")
+ qdq_linear = cls(
+ in_features=original_layer.in_features,
+ out_features=original_layer.out_features,
+ config=config,
+ bias=original_layer.bias,
+ dtype=original_layer.weight.dtype,
+ )
+ return qdq_linear
+
+
+class MXFP4QuantLinear(MXQuantLinearBase):
+ """
+ Quantized linear layer using the MXFP4 quantization scheme.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.weight_name = "weight_packed"
+ super().__init__(*args, **kwargs)
+
+ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:
+ weight_dtype = torch.uint8
+ weight_in_features = self.in_features // 2
+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ if self.pre_dequantized:
+ return self.weight
+ dq_weight = self.dequant_mx_tensor(self.weight_packed, self.weight_scale)
+ return dq_weight
+
+ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:
+ m, half_n = packed_data.shape
+ unpacked_data = unpack_fp4_from_uint8(packed_data, m, half_n * 2, dtype=self.dtype)
+ return unpacked_data
+
+
+class MXFP8QuantLinear(MXQuantLinearBase):
+ """
+ Quantized linear layer using the MXFP8 quantization scheme.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.weight_name = "weight"
+ super().__init__(*args, **kwargs)
+
+ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:
+ weight_dtype = torch.float8_e4m3fn
+ weight_in_features = self.in_features
+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight
+
+ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:
+ return packed_data.to(self.dtype)
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 7f134276b..12ed39bd8 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -49,6 +49,8 @@ class AutoRoundFormat(str, Enum):
# Weight: FP8, per-channel, may be extended to per-tensor in future
# Activation: FP8, per-tensor
FP8_STATIC = "fp8_static"
+ MXFP8 = "mxfp8"
+ MXFP4 = "mxfp4"
FP8 = "fp8"
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index 2c74e575e..6a285c5ae 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -90,15 +90,17 @@ class QuantLinear(nn.Module):
self.act_bits = kwargs.get("act_bits", None)
weight_name = "weight" if self.bits == 8 and self.is_mx else "weight_packed"
+ weight_infeatures = infeatures if self.bits == 8 else infeatures // 2
+ weight_dtype = torch.float8_e4m3fn if self.bits == 8 else torch.uint8
## TODO check the dtype of weight_packed and weight_scale
self.register_buffer(
weight_name,
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ torch.zeros((outfeatures, weight_infeatures), dtype=weight_dtype),
)
self.register_buffer(
"weight_scale",
torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
+ (outfeatures, math.ceil(infeatures / self.group_size)),
dtype=torch.float16, ## TODO update to correct scale dtype for different bits
),
)
@@ -156,11 +158,13 @@ class QuantLinear(nn.Module):
final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
else:
final_scale = scales.to(torch.float8_e4m3fn)
+
self.weight_scale = final_scale
# self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO
if self.bits == 8:
compress_dtype = torch.float8_e4m3fn
self.weight = scaled_tensor.to(compress_dtype)
+
else:
compress_dtype = torch.uint8
self.weight_packed = pack_fp4_to_uint8(scaled_tensor)
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 139a98120..ce3e209cf 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -19,9 +19,12 @@ from typing import Any, List, Optional
from transformers.utils.versions import require_version
import auto_round_extension.cuda.gptqmodel_marlin
+from auto_round import schemes as ar_schemes
+from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
-from auto_round.utils import get_library_version, logger
+from auto_round.utils import get_library_version
BackendInfos = {}
@@ -121,6 +124,38 @@ def fp8_static_scheme_checker(
return config == FP8_STATIC
+def _scheme_checker_common(config1: QuantizationScheme, config2: QuantizationScheme):
+ SCHEME_CHECK_ATTRS = ["bits", "group_size", "sym", "data_type", "act_bits", "act_group_size", "act_sym"]
+
+ for attr in SCHEME_CHECK_ATTRS:
+ if getattr(config1, attr) != getattr(config2, attr):
+ logger.debug(
+ f"Scheme check failed on attribute {attr}: {getattr(config1, attr)} != {getattr(config2, attr)}"
+ )
+ return False
+ return True
+
+
+def mxfp8_scheme_checker(
+ in_feature: int,
+ out_feature: int,
+ config: QuantizationScheme,
+ in_feature_multiplier: Optional[int] = None,
+ out_feature_multiplier: Optional[int] = None,
+):
+ return _scheme_checker_common(config, ar_schemes.MXFP8)
+
+
+def mxfp4_scheme_checker(
+ in_feature: int,
+ out_feature: int,
+ config: QuantizationScheme,
+ in_feature_multiplier: Optional[int] = None,
+ out_feature_multiplier: Optional[int] = None,
+):
+ return _scheme_checker_common(config, ar_schemes.MXFP4)
+
+
BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
device=["cuda"],
sym=[True, False],
@@ -205,6 +240,34 @@ BackendInfos["auto_round:fp8_static"] = BackendInfo(
requirements=["auto-round>0.6.0"],
)
+# MXFP8
+
+BackendInfos["auto_round:mxfp8"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format="",
+ sym=[True],
+ dtype=["float32", "float16", "bfloat16"],
+ bits=[8],
+ priority=0,
+ checkers=[mxfp8_scheme_checker],
+ alias=["torch"],
+ requirements=["auto-round>0.7.0"],
+)
+
+# MXFP4
+
+BackendInfos["auto_round:mxfp4"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format="",
+ sym=[True],
+ dtype=["float32", "float16", "bfloat16"],
+ bits=[4],
+ priority=0,
+ checkers=[mxfp4_scheme_checker],
+ alias=["auto_round:llm_compressor"],
+ requirements=["auto-round>0.7.0"],
+)
+
BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(
device=["cuda", "xpu"],
sym=[True], ## asym has accuracys
@@ -472,9 +535,11 @@ def dynamic_import_inference_linear(backend, config):
bits, group_size, sym = config["bits"], config["group_size"], config["sym"]
if AutoRoundFormat.FP8_STATIC.value in backend:
- from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
-
- return WeightFP8ActFP8StaticQuantLinear
+ return ar_qmodules.WeightFP8ActFP8StaticQuantLinear
+ if AutoRoundFormat.MXFP8.value in backend:
+ return ar_qmodules.MXFP8QuantLinear
+ if AutoRoundFormat.MXFP4.value in backend:
+ return ar_qmodules.MXFP4QuantLinear
if "qbits" in backend:
try:
@@ -680,6 +745,7 @@ def find_backend(target_backend: str, orig_backend: str = None):
Returns:
str or None: Matching backend key if found and compatible; otherwise, None.
"""
+ logger.trace(f"Finding backend for target: {target_backend}, original: {orig_backend}")
matched_keys = [
key
for key, info in BackendInfos.items()
@@ -758,7 +824,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
"""
# Check if the provided backend is in BackendInfos
backend = find_backend(backend)
-
+ logger.trace(f"Found backend: {backend} for device: {device}, orig_backend: {orig_backend}")
if backend not in BackendInfos.keys():
raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")
@@ -768,6 +834,7 @@ def get_layer_backend(device, backend, orig_backend, config, in_features, out_fe
supported_backends = []
for key in BackendInfos.keys():
if check_compatible(key, device, config, packing_format, in_features, out_features):
+ logger.trace(f"Backend {key} is compatible")
supported_backends.append(key)
# Raise an error if no compatible backends are found
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 827b1744c..c29b30482 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -30,6 +30,7 @@ from auto_round.inference.backend import (
get_layer_backend,
process_requirement,
)
+from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -43,8 +44,6 @@ from auto_round.utils import (
set_module,
)
-logger = getLogger(__name__)
-
supported_devices = ("cpu", "hpu", "xpu", "cuda")
@@ -378,6 +377,7 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, t
layer_backend = _get_layer_backend(
target_device, target_backend, orig_backend, config, in_features, out_features
)
+ logger.trace(f"Got backend {layer_backend} for {layer_name}.")
backend_cache[key] = layer_backend
if not layer_backend:
@@ -452,7 +452,11 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)
out_features=out_features,
bias=bias,
)
- elif AutoRoundFormat.FP8_STATIC.value in layer_backend:
+ elif (
+ AutoRoundFormat.FP8_STATIC.value in layer_backend
+ or AutoRoundFormat.MXFP8.value in layer_backend
+ or AutoRoundFormat.MXFP4.value in layer_backend
+ ):
return QuantLinear.from_original(config, layer)
# Default quantized layer creation
try:
@@ -598,6 +602,7 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
layer_configs = get_layer_config(model, quantization_config)
if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
packing_format = packing_format[len("auto_round:") :]
+
orig_backend = find_backend(packing_format)
if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 053a77523..ab804b0a8 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -143,6 +143,7 @@ MXFP4 = QuantizationScheme.from_dict(
"act_bits": 4,
"act_data_type": "mx_fp_rceil",
"act_group_size": 32,
+ "act_sym": True,
}
)
@@ -154,6 +155,7 @@ MXFP8 = QuantizationScheme.from_dict(
"act_bits": 8,
"act_data_type": "mx_fp_rceil",
"act_group_size": 32,
+ "act_sym": True,
}
)
diff --git a/test/test_cpu/test_mx_quant_linear.py b/test/test_cpu/test_mx_quant_linear.py
new file mode 100644
index 000000000..e8e18c3bb
--- /dev/null
+++ b/test/test_cpu/test_mx_quant_linear.py
@@ -0,0 +1,109 @@
+import pytest
+import torch
+
+from auto_round.data_type.utils import get_quant_func
+from auto_round.experimental import qmodules as ar_qmodules
+from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear
+from auto_round.schemes import PRESET_SCHEMES
+
+mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]
+QMODULE_MAPPING = {
+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+}
+
+
[email protected](autouse=True)
+def fixed_seed():
+ """Ensure deterministic RNG for every test."""
+ seed = 42
+ print("\nSetting fixed random seed for test:", seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ yield
+ # (Optional) cleanup or reset after test
+
+
[email protected]("scheme", mx_schemes)
+def test_mxquantlinear_from_original_and_forward(scheme):
+ """
+ Test MXFP4 and MXFP8 quantization schemes by creating quantized layers
+ from an original torch.nn.Linear layer and validating their forward pass.
+ """
+ # Set random seed for reproducibility
+ torch.manual_seed(42)
+
+ # Define layer dimensions
+ in_features = 64
+ out_features = 512
+
+ # Create an original torch.nn.Linear layer
+ original_layer = torch.nn.Linear(in_features, out_features, bias=False)
+
+ # Select the quantization scheme
+ config = PRESET_SCHEMES[scheme.upper()]
+
+ # Define weight scale shape
+ weight_scale_shape = (out_features, in_features // config.group_size)
+
+ # Quantize the weights using the quantization function
+ qdq_func, _ = get_quant_func(dtype=config.act_data_type, bits=config.act_bits, sym=True)
+ qdq_weight, shared_exp, _ = qdq_func(
+ tensor=original_layer.weight, bits=config.act_bits, group_size=config.act_group_size
+ )
+ shared_exp = shared_exp.reshape(weight_scale_shape)
+
+ # Pack the weights using the QuantLinear class
+ mxfp_lin = _MXFPLinear(
+ bits=config.bits,
+ group_size=config.group_size,
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=original_layer.bias is not None,
+ data_type=config.data_type,
+ )
+ mxfp_lin.pack(linear=original_layer, scales=shared_exp)
+
+ # Create an MXQuantLinear layer from the original layer
+ QuantLinearClass = QMODULE_MAPPING[scheme]
+ mxfp_layer = QuantLinearClass.from_original(
+ config=config,
+ original_layer=original_layer,
+ )
+
+ # Copy the packed weights and scales to the quantized layer
+ packed_weight = mxfp_lin.weight_packed if config.bits == 4 else mxfp_lin.weight
+ if config.bits == 4:
+ mxfp_layer.weight_packed.data.copy_(packed_weight)
+ elif config.bits == 8:
+ mxfp_layer.weight.data.copy_(packed_weight)
+ else:
+ raise ValueError("Only 4-bit and 8-bit quantization are supported.")
+ mxfp_layer.weight_scale.data.copy_(mxfp_lin.weight_scale)
+
+ # Validate layer attributes
+ assert mxfp_layer.in_features == original_layer.in_features
+ assert mxfp_layer.out_features == original_layer.out_features
+
+ # Generate a random input tensor
+ input_tensor = torch.randn((4, in_features), dtype=torch.float32)
+
+ # Perform a forward pass with both layers
+ original_output = original_layer(input_tensor)
+ mx_output = mxfp_layer(input_tensor)
+
+ # Compute the difference between the outputs
+ diff = mx_output - original_output
+ # Note: Remove NaN values, as we might get NaN when casting scales to FP8
+ diff = diff[~torch.isnan(diff)]
+ diff_amax = diff.abs().max()
+
+ # Print the maximum difference for debugging
+ print(f"Scheme: {scheme}, Max Difference: {diff_amax}")
+
+ # Assert that the outputs are close within a tolerance
+ assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"
diff --git a/test/test_cuda/test_mx_quant.py b/test/test_cuda/test_mx_quant.py
new file mode 100644
index 000000000..acf58d25c
--- /dev/null
+++ b/test/test_cuda/test_mx_quant.py
@@ -0,0 +1,82 @@
+import shutil
+import tempfile
+
+import pytest
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round import schemes as ar_schemes
+from auto_round.experimental import qmodules as ar_qmodules
+from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp
+
+mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]
+QMODULE_MAPPING = {
+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+}
+
+
+def has_module(model: torch.nn.Module, target_module_type: torch.nn.Module) -> bool:
+ """Check if the model contains a specific module type."""
+ for _, module in model.named_modules():
+ if isinstance(module, target_module_type):
+ return True
+ return False
+
+
[email protected]("scheme", mx_schemes)
+def test_e2e_quant_and_infer(scheme):
+ # Use a temporary directory for saving the quantized model
+ with tempfile.TemporaryDirectory() as temp_dir:
+ model_name = "Qwen/Qwen2.5-0.5B-Instruct"
+
+ # Load the tokenizer and model
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ device_map="cpu",
+ torch_dtype="auto",
+ trust_remote_code=True,
+ )
+
+ # Initialize AutoRound for quantization
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme=scheme,
+ iters=0,
+ nsamples=2,
+ )
+
+ # Quantize and save the model to the temporary directory
+ quantized_model_path = f"{temp_dir}/tmp_autoround_{scheme}"
+ autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)
+
+ # Perform inference with the quantized model
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ )
+ model.eval()
+ assert has_module(model, QMODULE_MAPPING[scheme]), f"Expected {QMODULE_MAPPING[scheme].__name__} in the model."
+
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ prompt = "Ai is "
+
+ # Tokenize the input prompt
+ encode = tokenizer.encode(prompt, return_tensors="pt")
+
+ # Generate output tokens
+ output_tokens = model.generate(
+ encode,
+ max_length=30,
+ )
+ output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+
+ # Print and validate the output
+ print(f"Prompt: {prompt}")
+ print(f"Output: {output}")
+ assert output is not None, "Output should not be None"