PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support MXINT4 scheme (#1666)
Wang, Mengnic817d4959db0
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex ab4da0b68..264524b75 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -756,7 +756,11 @@ def tune(args):suffix = f"a{autoround.act_bits}"else:suffix = f"g{autoround.group_size}"- prefix = autoround.data_type.lower().replace("_", "") if "int" not in autoround.data_type else ""+ prefix = (+ autoround.data_type.lower().replace("_", "")+ if "int" not in autoround.data_type or "mx" in autoround.data_type+ else ""+ )export_dir = os.path.join(args.output_dir,model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{autoround.bits}{suffix}",diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 578bba570..8240c8304 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -35,6 +35,7 @@ class BackendDataType(str, Enum):STANDARD_FP = "fp"MX_FP = "mx_fp"NV_FP = "nv_fp"+ MX_INT = "mx_int"def is_standard_fp(backend):@@ -47,6 +48,11 @@ def is_mx_fp(backend):return BackendDataType.MX_FP in backend+def is_mx_int(backend):+ backend = backend.lower()+ return BackendDataType.MX_INT in backend++def is_nv_fp(backend):backend = backend.lower()return BackendDataType.NV_FP in backenddiff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pyindex 3862e0293..df20d4afa 100644--- a/auto_round/experimental/qmodules/__init__.py+++ b/auto_round/experimental/qmodules/__init__.py@@ -12,6 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear, HadamardMXFP4QuantLinear+from auto_round.experimental.qmodules.mx import (+ MXFP4QuantLinear,+ MXFP8QuantLinear,+ MXINT4QuantLinear,+ HadamardMXFP4QuantLinear,+)from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinearfrom auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.pyindex b5bc3e939..ff93c34ff 100644--- a/auto_round/experimental/qmodules/mx.py+++ b/auto_round/experimental/qmodules/mx.py@@ -20,10 +20,11 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental.qmodules.base import QModuleBasefrom auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8+from auto_round.experimental.qmodules.mxint4_utils import unpack_int4_from_uint8from auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme-__all__ = ["MXFP4QuantLinear", "MXFP8QuantLinear"]+__all__ = ["MXFP4QuantLinear", "MXFP8QuantLinear", "MXINT4QuantLinear"]SUPPORTED_HIGHER_DTYPE = [torch.bfloat16, torch.float16, torch.float32]E8M0_EXPONENT_BIAS = 127@@ -196,6 +197,47 @@ class MXFP4QuantLinear(MXQuantLinearBase):return unpacked_data+class MXINT4QuantLinear(MXQuantLinearBase):+ """+ Quantized linear layer using the MXINT4 quantization scheme.+ """++ def __init__(self, *args, **kwargs):+ self.weight_name = "weight_packed"+ super().__init__(*args, **kwargs)++ def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ weight_dtype = torch.uint8+ weight_in_features = self.in_features // 2+ return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight++ def dequant_weight_online(self) -> torch.Tensor:+ if self.pre_dequantized:+ return self.weight+ dq_weight = self.dequant_mx_tensor(self.weight_packed, self.weight_scale)+ return dq_weight++ def unpack_data(self, packed_data: torch.Tensor) -> torch.Tensor:+ m, half_n = packed_data.shape+ unpacked_data = unpack_int4_from_uint8(packed_data, m, half_n * 2, dtype=self.dtype)+ return unpacked_data++ @classmethod+ def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ """+ Create an `MXQuantLinear` layer from an original linear layer.+ """+ logger.warning_once("MXINT quantization is still in experimental stage, the inference speed might be slow.")+ qdq_linear = cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ config=config,+ bias=original_layer.bias,+ dtype=original_layer.weight.dtype,+ )+ return qdq_linear++class HadamardMXFP4QuantLinear(MXFP4QuantLinear):"""Quantized linear layer using the MXFP4 quantization scheme.diff --git a/auto_round/experimental/qmodules/mxint4_utils.py b/auto_round/experimental/qmodules/mxint4_utils.pynew file mode 100644index 000000000..b772f54c1--- /dev/null+++ b/auto_round/experimental/qmodules/mxint4_utils.py@@ -0,0 +1,108 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch++_DEVICE_E0M4_TENSORS = {}++# Constants for INT4 values+_E0M4_VALUES = [0.0, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75]+++def get_e0m4_tensor(device):+ """Get device-specific E0M4 lookup tensor, creating it if needed."""+ device_str = str(device)+ if device_str not in _DEVICE_E0M4_TENSORS:+ _DEVICE_E0M4_TENSORS[device_str] = torch.tensor(_E0M4_VALUES, dtype=torch.float32, device=device)+ return _DEVICE_E0M4_TENSORS[device_str]+++def unpack_int4_from_uint8(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ """+ Unpacks uint8 values into int4. Each uint8 contains two int4 values+ (low nibble first). The 4-bit indices are mapped to int4 values using kE0M4ToFloat.+ """+ if a.device.type == "cuda":+ return _unpack_int4_from_uint8_cuda(a, m, n, dtype)+ else:+ return _unpack_int4_from_uint8_cpu(a, m, n, dtype)+++def _unpack_int4_from_uint8_cpu(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ return _unpack_int4_from_uint8(a, m, n, dtype)+++# @torch.compile(fullgraph=True, dynamic=True)+def _unpack_int4_from_uint8_cuda(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ return _unpack_int4_from_uint8(a, m, n, dtype)+++def _unpack_int4_from_uint8(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ """+ Unpacks uint8 values into int4. Each uint8 consists of two int4 values+ (i.e. first four bits correspond to one int4 value, last four correspond to a+ consecutive int4 value). The bits represent an index, which are mapped to an int4+ value.++ :param a: tensor to unpack+ :param m: original dim 0 size of the unpacked tensor+ :param n: original dim 1 size of the unpacked tensor+ :param dtype: dense dtype to cast the unpacked tensor to+ """+ assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"++ # Vectorized nibble processing+ a_flat = a.flatten()+ high = (a_flat & 0xF0) >> 4 # Upper nibbles+ low = a_flat & 0x0F # Lower nibbles++ # Combine nibbles for batch processing+ combined = torch.stack((low, high), dim=1).flatten()++ # Vectorized sign and magnitude extraction+ signs = (combined & 0x08).to(torch.bool) # Sign bits+ abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices++ # Device-aware lookup and sign application+ kE0M4 = get_e0m4_tensor(device=a.device)+ values = kE0M4[abs_vals] * torch.where(signs, -1.0, 1.0)++ # Reshape to final form+ return values.reshape(m, n).to(dtype=dtype)diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pysimilarity index 97%rename from auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyrename to auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex 502c49676..81ffb3e51 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -44,7 +44,8 @@ from auto_round.utils import ()from auto_round.wrapper import WrapperWALayer-from .qlinear_fp import QuantLinear+from .qlinear_fp import QuantLinear as FpQuantLinear+from .qlinear_int import QuantLinear as IntQuantLinear__all__ = ["pack_layer",@@ -94,7 +95,8 @@ def pack_layer(name, model, backend, device=None):bias = layer.bias is not None##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?- qlayer = QuantLinear( ##pylint: disable=E1123+ linear_func = FpQuantLinear if "fp" in data_type else IntQuantLinear+ qlayer = linear_func( ##pylint: disable=E1123bits,group_size,in_features,diff --git a/auto_round/export/export_to_autoround/qlinear_int.py b/auto_round/export/export_to_autoround/qlinear_int.pynew file mode 100644index 000000000..f1e6d581e--- /dev/null+++ b/auto_round/export/export_to_autoround/qlinear_int.py@@ -0,0 +1,202 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math++import numpy as np+import torch+import torch.nn as nn+import transformers++import auto_round.envs as envs+from auto_round.compressors.utils import BackendDataType+from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.utils import get_packing_device, logger++# from auto_round.utils import get_weight_compress_dtype+E8M0_EXPONENT_BIAS = 127+E8M0_EXPONENT_NAN_VAL = 255++__all__ = ["QuantLinear"]++FLOAT_TO_E0M4 = [+ 0.0,+ 0.25,+ 0.5,+ 0.75,+ 1.0,+ 1.25,+ 1.5,+ 1.75,+]+++class QuantLinear(nn.Module):+ """+ MXINT quantized linear layer.+ """++ QUANT_TYPE = "MXINT"++ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, data_type="mx_int4", **kwargs):+ super().__init__()+ if bits not in [4]:+ raise NotImplementedError("Only 4 bits are supported.")+ if group_size != 32:+ raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_INT} data type.")+ if infeatures % group_size != 0:+ raise NotImplementedError(+ f"in_feature must be divisible by {group_size} for {BackendDataType.MX_INT} data type."+ )+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.data_type = data_type+ self.sym = kwargs.get("sym", True)+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1+ self.act_bits = kwargs.get("act_bits", None)++ weight_name = "weight_packed"+ weight_infeatures = infeatures if self.bits == 8 else infeatures // 2+ weight_dtype = torch.uint8+ ## TODO check the dtype of weight_packed and weight_scale+ self.register_buffer(+ weight_name,+ torch.zeros((outfeatures, weight_infeatures), dtype=weight_dtype),+ )+ self.register_buffer(+ "weight_scale",+ torch.zeros(+ (outfeatures, math.ceil(infeatures / self.group_size)),+ dtype=torch.float16, ## TODO update to correct scale dtype for different bits+ ),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None):+ device = get_packing_device(device)+ if getattr(linear, "bias", None) is not None:+ self.bias = linear.bias.detach().to(torch.float16)++ W = linear.weight.data.detach().to(device)+ if type(linear) == nn.Conv2d:+ W = W.flatten(1)+ if type(linear) == transformers.pytorch_utils.Conv1D:+ W = W.t()++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)+ scales = scales.to(device)+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)++ self.weight_scale = final_scale+ compress_dtype = torch.uint8+ self.weight_packed = pack_int4_to_uint8(scaled_tensor)+++def pack_int4_to_uint8(scaled_tensor: torch.Tensor):+ if scaled_tensor.device.type == "cuda":+ return pack_int4_to_uint8_cuda(scaled_tensor)+ else:+ return pack_int4_to_uint8_cpu(scaled_tensor)+++# The torch.compile with dynamic=True is incompatible with multiple threads+# https://github.com/pytorch/pytorch/issues/126024+def pack_int4_to_uint8_cpu(x: torch.Tensor) -> torch.Tensor:+ return _pack_int4_to_uint8(x)+++# Adapted from https://github.com/neuralmagic/compressed-tensors/pull/400+++def _get_packing_fn():+ if envs.AR_ENABLE_COMPILE_PACKING:+ logger.warning_once(+ "Compiled INT4 to UINT8 packing may be incompatible with multi-threading."+ " Disable it by setting AR_ENABLE_COMPILE_PACKING=0"+ )+ return torch.compile(fullgraph=True, dynamic=True)(_pack_int4_to_uint8)+ else:+ return torch.compiler.disable()(_pack_int4_to_uint8)+++def pack_int4_to_uint8_cuda(x: torch.Tensor) -> torch.Tensor:+ """+ Packs a tensor with values in the int4 range into uint8.++ :param x: tensor to pack+ returns: a packed tensor in uint8+ """+ pack_fn = _get_packing_fn()+ return pack_fn(x)+++def _pack_int4_to_uint8(x: torch.Tensor) -> torch.Tensor:++ m, n = x.shape+ device = x.device++ # Create lookup table for INT4 values to indices+ # Map the absolute values to 0-7 indices+ kE0M4 = torch.tensor(FLOAT_TO_E0M4, device=device, dtype=x.dtype)++ # Find closest valid INT4 value index for each element+ abs_x = torch.abs(x)+ abs_diff_x = torch.abs(abs_x.unsqueeze(-1) - kE0M4) # [m, n, 8]+ abs_indices = torch.argmin(abs_diff_x, dim=-1) # [m, n]++ # Apply sign bit (bit 3) to get final 4-bit representation+ indices = abs_indices + (torch.signbit(x).to(torch.long) << 3)++ # Reshape to prepare for packing pairs of values+ indices = indices.reshape(-1)++ # Handle odd length by padding if necessary+ if indices.numel() % 2 != 0:+ indices = torch.cat([indices, torch.zeros(1, dtype=torch.long, device=device)])++ # Reshape to pair consecutive elements+ indices = indices.reshape(-1, 2)++ # Pack pairs of 4-bit values into 8-bit values+ packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)++ return packed.reshape(m, n // 2)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 3bd8a4679..1f6d2dbc1 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -31,6 +31,7 @@ from auto_round.compressors.utils import (is_dynamic_afp8,is_dynamic_wint8aint8,is_mx_fp,+ is_mx_int,is_nv_fp,is_standard_fp,is_static_wfp8afp8,@@ -70,6 +71,8 @@ class AutoRoundExportFormat(str, Enum):NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"INT8_W8A8 = "int8_w8a8"FP8_BLOCK = "fp8_block"+ MXINT4 = "mxint4"+ MX_INT = "mx_int"WINT_A16 = "wint_a16"@@ -1090,6 +1093,7 @@ class AutoRoundFormat(OutputFormat):"FP8_STATIC","BF16","FP8_BLOCK",+ "MXINT4",]format_name = "auto_round"@@ -1098,7 +1102,7 @@ class AutoRoundFormat(OutputFormat):self.backend = Noneif format == "auto_round":- if ar.sym and "int" in ar.data_type:+ if ar.sym and "int" in ar.data_type and "mx" not in ar.data_type:self.backend = AutoGPTQFormat("auto_round:auto_gptq", ar)elif ar.bits == 4 and not ar.sym and "int" in ar.data_type:if ar.layer_config is None:@@ -1111,6 +1115,8 @@ class AutoRoundFormat(OutputFormat):self.backend = AutoAWQFormat("auto_round:auto_awq", ar)elif is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):self.backend = AutoRoundFormat(ar.data_type, ar)+ elif is_mx_int(ar.data_type) and ar.bits == 4: # only add mx_int4 now+ self.backend = AutoRoundFormat(ar.data_type, ar)elif is_static_wfp8afp8(ar): # static wfp8afp8self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8@@ -1168,7 +1174,11 @@ class AutoRoundFormat(OutputFormat):f"auto_round:{AutoRoundExportFormat.MX_FP_RCEIL.value}",f"auto_round:{AutoRoundExportFormat.NV_FP4_WITH_STATIC_GS.value}",]:- from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import pack_layer+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer++ pack_func = pack_layer+ elif self.output_format in [f"auto_round:{AutoRoundExportFormat.MX_INT.value}"]:+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layerpack_func = pack_layerelif self.output_format in [@@ -1209,7 +1219,7 @@ class AutoRoundFormat(OutputFormat):)backend = self.get_backend_name()if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", backend):- from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import save_quantized_as_fp+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fpbackend = "auto_round:llm_compressor"export_func = save_quantized_as_fp@@ -1218,6 +1228,11 @@ class AutoRoundFormat(OutputFormat):backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else Noneexport_func = save_quantized_as_autoround+ elif re.search(f"{AutoRoundExportFormat.MX_INT.value}", backend):+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp++ backend = "auto_round"+ export_func = save_quantized_as_fpelse:from auto_round.export.export_to_autoround.export import save_quantized_as_autorounddiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex c5b10c4dc..ee472ea82 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -114,6 +114,7 @@ BACKEND_ACT_ATTRS = [MX_TENSOR_DATA_TYPES = ["mx_fp","mx_fp_rceil",+ "mx_int",]@@ -303,6 +304,26 @@ BackendInfos["auto_round:torch_mxfp4"] = BackendInfo(requirements=["auto-round>0.7.0"],)+# MXINT4+BackendInfos["auto_round:torch_mxint4"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format=["auto_round"],+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=MX_TENSOR_DATA_TYPES,+ group_size=[32],+ bits=[4],+ act_bits=[4],+ act_group_size=[32],+ act_sym=[True],+ act_data_type=MX_TENSOR_DATA_TYPES,+ act_dynamic=[True],+ priority=0,+ checkers=[mxfp_nvfp_feature_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.12.0"],+)+# NVFP4BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(@@ -326,6 +347,7 @@ BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(BackendInfos["auto_round:tritonv2"] = BackendInfo(device=["cuda", "xpu"],+ data_type=["int"],sym=[True, False],packing_format=GPTQ_FORMAT_NO_ZP,compute_dtype=["float16", "bfloat16"],@@ -774,6 +796,8 @@ def dynamic_import_inference_linear(backend, config):return ar_qmodules.WeightFP8ActFP8StaticQuantLinearif "torch_mxfp8" in backend:return ar_qmodules.MXFP8QuantLinear+ if "torch_mxint4" in backend:+ return ar_qmodules.MXINT4QuantLinearif "torch_mxfp4" in backend:hadamard_config = getattr(config, "hadamard_config", None)if hadamard_config is not None and hadamard_config:diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex a5b9096b3..9f2d6fca3 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -447,6 +447,7 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)or AutoRoundExportFormat.MXFP8.value in layer_backendor AutoRoundExportFormat.MXFP4.value in layer_backendor AutoRoundExportFormat.NVFP4.value in layer_backend+ or AutoRoundExportFormat.MXINT4.value in layer_backend):return QuantLinear.from_original(config, layer)diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 5318b1fec..2e0641554 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -228,6 +228,18 @@ MXFP8_RCEIL = QuantizationScheme.from_dict(})+MXINT4 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 32,+ "data_type": "mx_int",+ "act_bits": 4,+ "act_data_type": "mx_int",+ "act_group_size": 32,+ "act_sym": True,+ "act_dynamic": True,+ }+)NVFP4 = QuantizationScheme.from_dict({@@ -330,6 +342,7 @@ PRESET_SCHEMES = {"W4A16_MIXED": W4A16,"INT8_W8A8": INT8_W8A8,"FP8_BLOCK": FP8_BLOCK,+ "MXINT4": MXINT4,}from auto_round.export.export_to_gguf.config import GGUF_CONFIGdiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex a076e9acb..0327f4dc1 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -125,6 +125,7 @@ AutoRound supports several Schemes:- **Mixed Bits Weight only**- **NVFP4**(Experimental feature, recommend exporting to `llm_compressor` format.data_type nvfp4,act_data_type nvfp4,static_global_scale,group_size 16)- **MXFP4**(**Research feature, no real kernel**, Standard MXFP4, data_type mxfp,act_data_type mxfp,bits 4, act_bits 4, group_size 32)+- **MXINT4**(**Research feature, no real kernel**, Standard MXINT4, data_type mxint,act_data_type mxint,bits 4, act_bits 4, group_size 32)- **MXFP4_RCEIL**(**Research feature,no real kernel**, NVIDIA's variant, data_type mxfp,act_data_type mxfp_rceil,bits 4, act_bits 4, group_size 32)- **MXFP8**(**Research feature, no real kernel**, data_type mxfp,act_data_type mxfp_rceil,group_size 32)- **FPW8A16**(**Research feature, no real kernel**, data_type fp8,group_size 0->per tensor )@@ -157,7 +158,7 @@ adopted within the community, **only 4-bits quantization is supported**. Please| Format | Supported Schemes ||:---|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16` |+| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` || **auto_awq** | W4A16, BF16 || **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 || **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK |diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex b7cd57f64..d535718a6 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -119,6 +119,7 @@ AutoRound 支持多种量化配置:- **混合bit**: (实验性功能)请使用 AutoScheme 接口或者使用 API 中的 `layer_config` 参数自己自定义- **NVFP4**(实验性功能)推荐导出为`llm_compressor`格式,参数:data_type=nvfp4, act_data_type=nvfp4, static_global_scale, group_size=16- **MXFP4**(研究性功能,暂无实际内核):标准 MXFP4 量化,参数:data_type=mxfp, act_data_type=mxfp, bits=4, act_bits=4, group_size=32+- **MXINT4**(研究性功能,暂无实际内核):标准 MXINT4 量化,参数:data_type=mxint, act_data_type=mxint, bits=4, act_bits=4, group_size=32- **MXFP4_RCEIL**(研究性功能,暂无实际内核):NVIDIA变体,参数:data_type=mxfp, act_data_type=mxfp_rceil, bits=4, act_bits=4, group_size=32- **MXFP8**(研究性功能,暂无实际内核),参数:data_type=mxfp, act_data_type=mxfp_rceil, group_size=32- **FPW8A16**(研究性功能,暂无实际内核),参数:data_type=fp8, group_size=0 -> 每张量@@ -147,7 +148,7 @@ AutoRound 支持多种量化配置:| 格式 | 支持的量化方案 ||:-------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16` |+| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` || **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 || **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |diff --git a/test/test_cpu/quantization/test_mx_quant_linear.py b/test/test_cpu/quantization/test_mx_quant_linear.pyindex c2e9a3c00..1ec5cb729 100644--- a/test/test_cpu/quantization/test_mx_quant_linear.py+++ b/test/test_cpu/quantization/test_mx_quant_linear.py@@ -4,13 +4,16 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear+from auto_round.export.export_to_autoround.qlinear_int import QuantLinear as _MXINTLinearfrom auto_round.formats import AutoRoundExportFormatfrom auto_round.schemes import PRESET_SCHEMESmx_schemes = [AutoRoundExportFormat.MXFP8.value, AutoRoundExportFormat.MXFP4.value]+mx_int_schemes = [AutoRoundExportFormat.MXINT4.value]QMODULE_MAPPING = {AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundExportFormat.MXINT4.value: ar_qmodules.MXINT4QuantLinear,}@@ -107,3 +110,86 @@ def test_mxquantlinear_from_original_and_forward(scheme):# Assert that the outputs are close within a toleranceassert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"++[email protected]("scheme", mx_int_schemes)[email protected]_mode()+def test_mxint_quantlinear_from_original_and_forward(scheme):+ """+ Test MXINT4 quantization schemes by creating quantized layers+ from an original torch.nn.Linear layer and validating their forward pass.+ """+ # Set random seed for reproducibility+ torch.manual_seed(42)++ # Define layer dimensions+ in_features = 64+ out_features = 512++ # Create an original torch.nn.Linear layer+ original_layer = torch.nn.Linear(in_features, out_features, bias=False)++ # Select the quantization scheme+ config = PRESET_SCHEMES[scheme.upper()]++ # Define weight scale shape+ weight_scale_shape = (out_features, in_features // config.group_size)++ # Quantize the weights using the quantization function+ qdq_func, _ = get_quant_func(dtype=config.data_type, bits=config.bits, sym=config.sym)+ qdq_weight, shared_exp, _ = qdq_func(+ tensor=original_layer.weight,+ bits=config.bits,+ group_size=config.group_size,+ data_type=config.data_type + str(config.bits),+ )+ shared_exp = shared_exp.reshape(weight_scale_shape)++ # Pack the weights using the QuantLinear class+ mxint_lin = _MXINTLinear(+ bits=config.bits,+ group_size=config.group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=original_layer.bias is not None,+ data_type=config.data_type,+ )+ mxint_lin.pack(linear=original_layer, scales=shared_exp)++ # Create an MXQuantLinear layer from the original layer+ QuantLinearClass = QMODULE_MAPPING[scheme]+ mxint_layer = QuantLinearClass.from_original(+ config=config,+ original_layer=original_layer,+ )++ # Copy the packed weights and scales to the quantized layer+ packed_weight = mxint_lin.weight_packed+ if config.bits == 4:+ mxint_layer.weight_packed.data.copy_(packed_weight)+ else:+ raise ValueError("Only 4-bit quantization are supported.")+ mxint_layer.weight_scale.data.copy_(mxint_lin.weight_scale)++ # Validate layer attributes+ assert mxint_layer.in_features == original_layer.in_features+ assert mxint_layer.out_features == original_layer.out_features++ # Generate a random input tensor+ input_tensor = torch.randn((4, in_features), dtype=torch.float32)++ # Perform a forward pass with both layers+ original_output = original_layer(input_tensor)+ mx_output = mxint_layer(input_tensor)++ # Compute the difference between the outputs+ diff = mx_output - original_output+ # Note: Remove NaN values, as we might get NaN when casting scales to FP8+ diff = diff[~torch.isnan(diff)]+ diff_amax = diff.abs().max()++ # Print the maximum difference for debugging+ print(f"Scheme: {scheme}, Max Difference: {diff_amax}")++ # Assert that the outputs are close within a tolerance+ assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"diff --git a/test/test_cpu/quantization/test_mxfp_save_load.py b/test/test_cpu/quantization/test_mxfp_save_load.pyindex 5e12edc68..25e5a2428 100644--- a/test/test_cpu/quantization/test_mxfp_save_load.py+++ b/test/test_cpu/quantization/test_mxfp_save_load.py@@ -28,11 +28,12 @@ SCHEMES_MAPPING = {AutoRoundExportFormat.MXFP8.value: ar_schemes.MXFP8,AutoRoundExportFormat.MXFP4.value: ar_schemes.MXFP4,}+MX_TENSOR_DATA_TYPES_FP = [i for i in MX_TENSOR_DATA_TYPES if "int" not in i]@pytest.mark.parametrize("scheme_name", testing_scheme_name_lst)[email protected]("weight_data_type", MX_TENSOR_DATA_TYPES)[email protected]("act_data_type", MX_TENSOR_DATA_TYPES)[email protected]("weight_data_type", MX_TENSOR_DATA_TYPES_FP)[email protected]("act_data_type", MX_TENSOR_DATA_TYPES_FP)@torch.inference_mode()def test_e2e_quant_and_load(scheme_name, weight_data_type, act_data_type):# Use a temporary directory for saving the quantized model