Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support hadamard transform for mxfp4 with rtn or autoround method (#1515)

lkka49550bc5e9c
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 8b161be73..9343130b2 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -89,6 +89,7 @@ class AutoRound:
enable_alg_ext: bool = False,
disable_opt_rtn: bool | None = None,
low_cpu_mem_usage: bool = True,
+ transform_config: dict | None = None,
**kwargs,
) -> BaseCompressor:
"""Initialize AutoRound with quantization and tuning configuration.
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 5f6f73c6f..8ff815fef 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -150,6 +150,7 @@ SERIALIZATION_KEYS = (
"super_bits",
"super_group_size",
"to_quant_block_names",
+ "transform_config",
)
@@ -551,6 +552,8 @@ class BaseCompressor(object):
except (ImportError, ModuleNotFoundError):
logger.error("algorithm extension import error, fallback to default mode")
+ self.transform_config = kwargs.pop("transform_config", {})
+
def _gen_auto_scheme(self) -> dict[str, dict]:
if self.mllm:
logger.info("AutoScheme is not yet supported for multimodal LLMs.")
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
index 0179dd646..cf805645b 100644
--- a/auto_round/experimental/qmodules/__init__.py
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -12,6 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear
+from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear, TransformMXFP4QuantLinear
from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.py
index f06076173..a5109b441 100644
--- a/auto_round/experimental/qmodules/mx.py
+++ b/auto_round/experimental/qmodules/mx.py
@@ -68,6 +68,7 @@ class MXQuantLinearBase(QModuleBase):
self.config = config
self.dtype = dtype
self.pre_dequantized = False
+ self.pre_dequantized_input = False
# Validate dtype
assert (
@@ -145,10 +146,12 @@ class MXQuantLinearBase(QModuleBase):
@torch.inference_mode()
def forward(self, input: torch.Tensor) -> torch.Tensor:
- qdq_input = self.qdq_input(input)
+ if not self.pre_dequantized_input:
+ input = self.qdq_input(input)
+
qdq_weight = self.dequant_weight_online()
- qdq_weight = qdq_weight.to(qdq_input.dtype)
- out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)
+ qdq_weight = qdq_weight.to(input.dtype)
+ out = torch.nn.functional.linear(input, qdq_weight, self.bias)
return out
@classmethod
@@ -193,6 +196,25 @@ class MXFP4QuantLinear(MXQuantLinearBase):
return unpacked_data
+class TransformMXFP4QuantLinear(MXFP4QuantLinear):
+ """
+ Quantized linear layer using the MXFP4 quantization scheme.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.weight_name = "weight_packed"
+ super().__init__(*args, **kwargs)
+ self.enable_transform = True
+ self.register_buffer(
+ "transform_matrix",
+ torch.empty(
+ self.group_size,
+ self.group_size,
+ dtype=self.dtype,
+ ),
+ )
+
+
class MXFP8QuantLinear(MXQuantLinearBase):
"""
Quantized linear layer using the MXFP8 quantization scheme.
diff --git a/auto_round/experimental/transform/__init__.py b/auto_round/experimental/transform/__init__.py
new file mode 100644
index 000000000..0b3bf2eae
--- /dev/null
+++ b/auto_round/experimental/transform/__init__.py
@@ -0,0 +1,2 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/apply.py b/auto_round/experimental/transform/apply.py
new file mode 100644
index 000000000..dd5289dcc
--- /dev/null
+++ b/auto_round/experimental/transform/apply.py
@@ -0,0 +1,158 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import torch
+import tqdm
+
+from auto_round.experimental.qmodules.mx import MXQuantLinearBase
+from auto_round.experimental.transform.transform_config import TransformConfig
+from auto_round.experimental.transform.transforms import build_transform
+
+__all__ = ["apply_transform"]
+
+
+def apply_transform(model: torch.nn.Module, config: TransformConfig, use_tqdm=True, desc=None):
+ """
+ Apply a transform config to a model. Add weight transforms and
+ activation transforms are attached as submodules and trigger via pytorch hooks
+
+ :param model: model to apply config to
+ :param config: transform config to apply
+ """
+
+ modules_config = [
+ (name, module, config)
+ for name, module in model.named_modules()
+ if isinstance(module, torch.nn.Linear) or isinstance(module, MXQuantLinearBase)
+ ]
+
+ desc = f"Applying {config.transform_type} transforms" if desc is None else desc
+ for name, module, config in tqdm.tqdm(modules_config, desc=desc, disable=(not use_tqdm)):
+ if "lm_head" in name:
+ continue
+ _apply_to_module(model, module, config)
+
+ # attach config to model for compression/serialization
+ setattr(model, "transform_config", config)
+
+ return model
+
+
+def _apply_to_module(
+ model: torch.nn.Module,
+ module: torch.nn.Module,
+ config: TransformConfig,
+):
+ """
+ Create transforms and apply them to the module
+
+ :param model: model which module belongs to
+ :param module: target module to apply transforms to
+ """
+
+ # create transform as submodule
+ transform_name = "transform_matrix"
+
+ if config.location == "input":
+ from auto_round.experimental.transform.triton.utils import is_triton_kernel_available
+
+ # activation needs transpose
+ inp_transform = build_transform(
+ **config.dict(),
+ inverse=True,
+ device="cpu",
+ precision=module.dtype,
+ )
+
+ if config.transform_type != "random_hadamard":
+ transform_weight = inp_transform.weight
+ else:
+ transform_weight = None
+
+ if is_triton_kernel_available():
+ from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper
+
+ def input_hook(self, args):
+ input = args[0]
+ # transform(input)
+ orig_shape = input.shape
+ x_flat = input.contiguous().flatten(end_dim=-2)
+ qdq_input, _ = mxfp4_forward_kernel_wrapper(
+ x_flat,
+ (
+ transform_weight if transform_weight is not None else self.transform_matrix.T
+ ), # this matrix from w_transform, needs transpose
+ )
+ return qdq_input.reshape(orig_shape)
+
+ # for fused transform + quantization kernel
+ module.pre_dequantized_input = True
+ module.register_forward_pre_hook(input_hook, prepend=True)
+ else:
+
+ from auto_round.experimental.transform.utils.matrix import _multihead_matmul
+
+ def input_hook(self, args):
+ input = args[0]
+
+ ori_shape = input.shape
+
+ if transform_weight is not None:
+ input = input.view(-1, transform_weight.shape[0])
+ return _multihead_matmul(input, transform_weight.to(input.device)).view(ori_shape)
+ else:
+ input = input.view(-1, self.transform_matrix.shape[0])
+ return _multihead_matmul(input, self.transform_matrix.T).view(ori_shape)
+
+ # for fused transform + quantization kernel
+ module.pre_dequantized_input = False
+ module.register_forward_pre_hook(input_hook, prepend=True)
+
+ elif config.location == "weight":
+ # eagerly apply transformation to weight
+ # fuse transform into weight
+ assert hasattr(module, "weight")
+
+ w_transform = build_transform(
+ **config.dict(),
+ device=module.weight.device,
+ precision=module.weight.dtype,
+ )
+
+ # need save random hadamard matrix needed when inference
+ if config.transform_type == "random_hadamard":
+ module.register_module(transform_name, w_transform)
+ # for saving transform weight
+ from auto_round.experimental.transform.patch_modules import patch_quantlinear
+
+ patch_quantlinear()
+
+ if config.need_calibration:
+ # for training, the weight changes with every forward pass
+ # for autoround tuning: patch wrapper linear qdq_weight func
+ from auto_round.experimental.transform.patch_modules import (
+ patch_wrapperlinear_to_apply_transform,
+ patch_wrapperwalayer_forward_to_apply_transform,
+ )
+
+ inp_transform = build_transform(
+ **config.dict(),
+ location="input",
+ inverse=True,
+ device=module.weight.device,
+ precision=module.weight.dtype,
+ )
+
+ patch_wrapperlinear_to_apply_transform(w_transform, inp_transform)
+ patch_wrapperwalayer_forward_to_apply_transform(inp_transform)
+
+ else:
+ # transform is no longer needed (unfusing is not supported)
+ # delattr(module, transform_name)
+ # fuse transform into weight
+ with torch.no_grad():
+ getattr(module, "weight").copy_(w_transform(module.weight).to(module.weight.device))
+
+ else:
+ # TODO: apply transform to output/q/k
+ raise NotImplementedError()
diff --git a/auto_round/experimental/transform/patch_modules.py b/auto_round/experimental/transform/patch_modules.py
new file mode 100644
index 000000000..cf15b46e0
--- /dev/null
+++ b/auto_round/experimental/transform/patch_modules.py
@@ -0,0 +1,210 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import torch
+import transformers
+
+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8
+from auto_round.wrapper import WrapperLinear, WrapperWALayer
+
+
+def patch_wrapperlinear_to_apply_transform(w_transform, inp_transform):
+ """
+ Globally monkey-patch WrapperLinear._qdq_weight and WrapperLinear._qdq_act so that it applies
+ a weight and activation transform before quantization.
+
+ e.g. by apply_transform() before wrapper_block().
+ """
+
+ if getattr(WrapperLinear, "_hadamard_patched", False):
+ return
+
+ orig_qdq_weight = WrapperLinear._qdq_weight
+
+ def _qdq_weight_patched(self, value, min_scale, max_scale):
+ """
+ # If no transform attached, fall back to original behavior
+ if not hasattr(self.orig_layer, transform_attr):
+ return orig_qdq_weight(self, value, min_scale, max_scale)
+ """
+
+ if self.orig_layer.bits >= 16:
+ # keep original behavior for >=16bit to avoid changing semantics unexpectedly
+ return orig_qdq_weight(self, value, min_scale, max_scale)
+
+ min_scale.data.clamp_(0, 1.0)
+ max_scale.data.clamp_(0, 1.0)
+
+ weight = self.orig_layer.weight
+ if weight.device.type == "meta":
+ weight = self.orig_layer.get_weight().to(self.device)
+
+ is_conv1d = type(self.orig_layer) == transformers.pytorch_utils.Conv1D
+ if is_conv1d:
+ weight = weight.t()
+
+ weight = weight.to(self.device)
+
+ weight_t = w_transform(weight)
+
+ quant_kwargs = {}
+ if hasattr(self.orig_layer, "super_bits"):
+ quant_kwargs["super_bits"] = self.orig_layer.super_bits
+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
+
+ weight_q, scale, zp = self.weight_quant_func(
+ weight_t,
+ bits=self.orig_layer.bits,
+ group_size=self.orig_layer.group_size,
+ v=value,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.orig_layer.scale_dtype,
+ tensor_min=self.weight_min,
+ tensor_max=self.weight_max,
+ data_type=self.data_type,
+ q_scale_thresh=self.q_scale_thresh,
+ imatrix=self.orig_layer.imatrix.to(self.device) if hasattr(self.orig_layer, "imatrix") else None,
+ global_scale=getattr(self, "weight_global_scale", None),
+ **quant_kwargs,
+ )
+
+ weight_q = weight_q.to(dtype=weight.dtype)
+
+ if is_conv1d:
+ weight_q = weight_q.t()
+
+ return weight_q, scale, zp
+
+ def _qdq_act_patched(self, x, act_max_scale, act_max=None):
+
+ # transform = getattr(self.orig_layer, transform_attr)
+ x = inp_transform(x)
+ act_max_scale.data.clamp_(0, 1.0)
+ x, scale, zp = self.act_quant_func(
+ x,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.q_scale_thresh,
+ data_type=self.act_data_type,
+ max_scale=act_max_scale,
+ tensor_max=act_max,
+ global_scale=getattr(self, "input_global_scale", None),
+ )
+ return x, scale, zp
+
+ WrapperLinear._qdq_weight = _qdq_weight_patched
+ WrapperLinear._qdq_act = _qdq_act_patched
+ WrapperLinear._hadamard_patched = True
+
+
+def patch_wrapperwalayer_forward_to_apply_transform(inp_transform):
+ """
+ Globally monkey-patch WrapperWALayer.forward so that it applies
+ a activation transform before quantization.
+
+ e.g. by apply_transform() before wrapper_block().
+ """
+
+ if getattr(WrapperWALayer, "_hadamard_forward_patched", False):
+ return
+
+ orig_forward = WrapperWALayer.forward
+
+ def _forward_patched(self, x):
+ """
+ # If no transform attached, fall back to original behavior
+ if not hasattr(self.orig_layer, transform_attr):
+ return orig_forward(self, x)
+ """
+
+ act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
+
+ # transform = getattr(self.orig_layer, transform_attr)
+ x = inp_transform(x)
+
+ x, _, _ = self.orig_layer.act_quant_func(
+ x,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.orig_layer.q_scale_thresh,
+ data_type=self.orig_layer.act_data_type,
+ tensor_max=act_max,
+ )
+ return self.orig_layer.forward(x)
+
+ WrapperWALayer.forward = _forward_patched
+ WrapperWALayer._hadamard_forward_patched = True
+
+
+def patch_quantlinear():
+ """ """
+
+ if getattr(QuantLinear, "_pack_patched", False):
+ return
+
+ from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+ from auto_round.utils import get_packing_device
+
+ E8M0_EXPONENT_BIAS = 127
+ E8M0_EXPONENT_NAN_VAL = 255
+
+ def _pack_patched(
+ self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None
+ ):
+ device = get_packing_device(device)
+ if getattr(linear, "bias", None) is not None:
+ self.bias = linear.bias.detach().to(torch.float16)
+
+ W = linear.weight.data.detach().to(device)
+ if type(linear) == torch.nn.Conv2d:
+ W = W.flatten(1)
+ if type(linear) == transformers.pytorch_utils.Conv1D:
+ W = W.t()
+
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)
+ scales = scales.to(device)
+ if self.is_nv:
+ assert global_scale is not None and global_scale.numel() == 1
+ global_scale = global_scale.reshape([1])
+ global_scale = global_scale.to(device)
+ scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(
+ scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)
+ )
+ scaled_tensor.clamp_(-6.0, 6.0)
+ scaled_tensor = cast_to_fp4(scaled_tensor)
+ else:
+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))
+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
+ if self.is_mx:
+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
+ else:
+ final_scale = scales.to(torch.float8_e4m3fn)
+
+ self.weight_scale = final_scale
+ # self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO
+ if self.bits == 8:
+ compress_dtype = torch.float8_e4m3fn
+ self.weight = scaled_tensor.to(compress_dtype)
+
+ else:
+ compress_dtype = torch.uint8
+ self.weight_packed = pack_fp4_to_uint8(scaled_tensor)
+
+ if global_scale is not None:
+ self.weight_global_scale = global_scale.to(torch.float32).to(device)
+
+ if input_global_scale is not None:
+ # TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.
+ self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])
+
+ # add transform weight
+ transform = getattr(linear, "transform_matrix")
+ self.register_buffer("transform_matrix", transform.weight.to(device))
+ return
+
+ QuantLinear.pack = _pack_patched
+ QuantLinear._pack_patched = True
diff --git a/auto_round/experimental/transform/transform_config.py b/auto_round/experimental/transform/transform_config.py
new file mode 100644
index 000000000..91713666b
--- /dev/null
+++ b/auto_round/experimental/transform/transform_config.py
@@ -0,0 +1,43 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field, field_validator
+
+__all__ = ["TransformConfig"]
+
+
+class TransformConfig(BaseModel):
+ """
+ Configuration of transforms to be applied to a model. This config is to be
+ serialized within a model's `config.json` file
+ """
+
+ # required, currently only supports mxfp4
+ quant_scheme: str = Field(..., description="Quantization scheme. Currently supports 'MXFP4/MXFP8'.")
+
+ transform_block_size: int = Field(default=32)
+
+ transform_type: str = Field(default="hadamard")
+
+ location: str = Field(default="weight", exclude=True)
+
+ # apply transform inside modules for nvfp4, autoround tuning etc.
+ need_calibration: bool = Field(default=False, exclude=True)
+
+ # for random hadamard transform
+ random_seed: bool = Field(default=False, exclude=True)
+
+ @field_validator("quant_scheme")
+ @classmethod
+ def validate_quant_scheme(cls, v: str) -> str:
+ if v not in ["MXFP4", "MXFP8"]:
+ raise ValueError(f"Unsupported quant_scheme: {v}. Currently 'mxfp4/mxfp8' are supported.")
+ return v
+
+ @field_validator("transform_type")
+ @classmethod
+ def validate_transform_type(cls, v: str) -> str:
+ allowed = {"hadamard", "random_hadamard"}
+ if v not in allowed:
+ raise ValueError(f"Unsupported transform_type: {v}. Supported values: {sorted(allowed)}")
+ return v
diff --git a/auto_round/experimental/transform/transforms.py b/auto_round/experimental/transform/transforms.py
new file mode 100644
index 000000000..38bb1acb7
--- /dev/null
+++ b/auto_round/experimental/transform/transforms.py
@@ -0,0 +1,130 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import math
+from typing import Any, Callable, Dict
+
+import torch
+import torch.nn as nn
+
+from auto_round.experimental.transform.utils.hadamard import deterministic_hadamard_matrix, random_hadamard_matrix
+from auto_round.experimental.transform.utils.matrix import apply_transform_weight
+
+
+def filter_kwarg_dict(fn_or_method: Callable, kwarg_dict: Dict[str, Any]) -> Dict[str, Any]:
+ fn_or_method_keys = inspect.signature(fn_or_method).parameters.keys()
+ return {k: v for k, v in kwarg_dict.items() if k in fn_or_method_keys}
+
+
+class IdentityTransform(nn.Module):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__()
+
+ def forward(self, x: torch.Tensor):
+ return x
+
+ def remove_parametrizations(self) -> None:
+ pass
+
+
+class HadamardTransform(nn.Module):
+
+ def __init__(
+ self,
+ transform_block_size: int = 32,
+ device: torch.device = None,
+ precision: torch.dtype = None,
+ location: str = "weight",
+ module_type: type[torch.nn.Module] = torch.nn.Linear,
+ inverse: bool = False,
+ ):
+ super().__init__()
+ self.size = transform_block_size
+ self.scale = 1 / math.sqrt(self.size)
+ self.location = location
+ self.module_type = module_type
+ self.inverse = inverse
+ self.weight = self._create_weight(self.size, device, precision)
+
+ def _create_weight(
+ self,
+ size: int,
+ device: torch.device = None,
+ precision: torch.dtype = None,
+ ) -> torch.nn.Parameter:
+ data = deterministic_hadamard_matrix(size, precision, device) * self.scale
+ # TODO: implement SpinQuant, which rotation matrix is learnable
+ return nn.Parameter(data, requires_grad=False)
+
+ def forward(self, x: torch.Tensor):
+ # Hadamard transform is it own inverse
+ ori_shape = x.shape
+ x = x.view(-1, self.size)
+ return (
+ (
+ apply_transform_weight(
+ self.weight,
+ x.to(dtype=self.weight.dtype),
+ self.location,
+ self.module_type,
+ )
+ )
+ .to(x.dtype)
+ .view(ori_shape)
+ )
+
+
+class RandomHadamardTransform(HadamardTransform):
+ def __init__(
+ self,
+ *args,
+ seed: int | None = None,
+ generator: torch.Generator | None = None,
+ **kwargs,
+ ):
+ if generator is not None:
+ self.generator = generator
+ else:
+ self.generator = torch.Generator()
+ if seed is not None:
+ self.generator.manual_seed(seed)
+ super().__init__(*args, **kwargs)
+
+ def _create_weight(
+ self,
+ size: int,
+ device: torch.device = None,
+ precision: torch.dtype = None,
+ ) -> torch.nn.Parameter:
+ data = random_hadamard_matrix(size, precision, device, self.generator) * self.scale
+ # activation needs transpose
+ if self.inverse:
+ data = data.T
+ # data = deterministic_hadamard_matrix(size, precision, device) * self.scale
+ # TODO: implement SpinQuant, which rotation matrix is learnable
+ return nn.Parameter(data, requires_grad=False)
+
+
+TRANSFORMS = {
+ "identity": IdentityTransform,
+ "hadamard": HadamardTransform,
+ "random_hadamard": RandomHadamardTransform,
+}
+
+
+def build_transform(transform_type: str, **transform_kwargs):
+ transform = TRANSFORMS[transform_type]
+ return transform(**filter_kwarg_dict(transform.__init__, transform_kwargs))
diff --git a/auto_round/experimental/transform/triton/__init__.py b/auto_round/experimental/transform/triton/__init__.py
new file mode 100644
index 000000000..0b3bf2eae
--- /dev/null
+++ b/auto_round/experimental/transform/triton/__init__.py
@@ -0,0 +1,2 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/triton/mxfp4.py b/auto_round/experimental/transform/triton/mxfp4.py
new file mode 100644
index 000000000..c26413248
--- /dev/null
+++ b/auto_round/experimental/transform/triton/mxfp4.py
@@ -0,0 +1,192 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Refer code here:
+# https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py
+
+import torch
+import triton # pylint: disable=E0401
+import triton.language as tl # pylint: disable=E0401
+
+
+ configs=[
+ triton.Config({"BLOCK_SIZE": 32 * 32}),
+ triton.Config({"BLOCK_SIZE": 64 * 32}),
+ triton.Config({"BLOCK_SIZE": 128 * 32}),
+ triton.Config({"BLOCK_SIZE": 256 * 32}),
+ triton.Config({"BLOCK_SIZE": 512 * 32}),
+ ],
+ key=[],
+)
+def mxfp4_forward_kernel(
+ x_ptr,
+ hadamard_matrix_ptr,
+ output_ptr,
+ clip_mask_ptr,
+ n_elements: tl.constexpr,
+ hadamard_dim: tl.constexpr,
+ group_size: tl.constexpr,
+ gaussian_scale: tl.constexpr,
+ quest: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+):
+ offsets_hadamard = tl.arange(0, hadamard_dim * hadamard_dim)
+ hadamard_matrix = tl.load(hadamard_matrix_ptr + offsets_hadamard).reshape(hadamard_dim, hadamard_dim)
+
+ # load x
+ pid = tl.program_id(0)
+ start_idx = pid * BLOCK_SIZE
+ offsets = start_idx + tl.arange(0, BLOCK_SIZE)
+ mask = offsets < n_elements
+ x_flat = tl.load(x_ptr + offsets, mask=mask)
+
+ # hadamard transform
+ x = tl.reshape(x_flat, (BLOCK_SIZE // hadamard_dim, hadamard_dim))
+ x_had = tl.dot(x, hadamard_matrix)
+
+ # group
+ x_had_grouped = tl.reshape(x_had, (BLOCK_SIZE // group_size, group_size))
+
+ # scale
+ # quest=True: per-group Gaussian-based scale = gaussian_scale * std
+ # quest=False: per-group max-abs-based scale, adjusted to FP4 range
+ if quest:
+ mean_squared = tl.sum(x_had_grouped * x_had_grouped, axis=-1, keep_dims=True) / group_size
+ mean = tl.sum(x_had_grouped, axis=-1, keep_dims=True) / group_size
+ std = tl.sqrt(mean_squared - mean * mean)
+ scales = gaussian_scale * std + 1e-8
+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)))
+ x_had_scaled = x_had_grouped / shared_exps
+ else:
+ scales = tl.max(tl.abs(x_had_grouped), axis=-1, keep_dims=True)
+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)) - 2) / (3 / 4)
+ x_had_scaled = x_had_grouped / shared_exps
+
+ # quantize
+ # Map abs(x) to FP4 levels {0, 0.5, 1, 1.5, 2, 3, 4, 6}
+ x_had_scaled_abs = tl.abs(x_had_scaled)
+ x_had_scaled_sign = tl.where(
+ x_had_scaled > 0,
+ 1,
+ -1,
+ )
+
+ x_fp4 = (
+ tl.where(
+ x_had_scaled_abs > 5,
+ 6,
+ tl.where(
+ x_had_scaled_abs > 3.5,
+ 4,
+ tl.where(
+ x_had_scaled_abs > 2.5,
+ 3,
+ tl.where(
+ x_had_scaled_abs > 1.75,
+ 2,
+ tl.where(
+ x_had_scaled_abs > 1.25,
+ 1.5,
+ tl.where(
+ x_had_scaled_abs > 0.75,
+ 1,
+ tl.where(
+ x_had_scaled_abs > 0.25,
+ 0.5,
+ 0,
+ ),
+ ),
+ ),
+ ),
+ ),
+ ),
+ )
+ * x_had_scaled_sign
+ )
+ if clip_mask_ptr is not None:
+ tl.store(
+ clip_mask_ptr + offsets,
+ tl.reshape(x_had_scaled_abs < 6, (BLOCK_SIZE,)),
+ mask=mask,
+ )
+
+ # dequantize
+ x_dequantized = x_fp4 * shared_exps
+
+ # Reshape back to flat form for storage
+ x_dequantized_flat = tl.reshape(x_dequantized, (BLOCK_SIZE,))
+
+ # store
+ tl.store(output_ptr + offsets, x_dequantized_flat, mask=mask)
+
+
+def mxfp4_forward_kernel_wrapper(
+ x,
+ hadamard_matrix,
+ return_clip_mask=False,
+ quest=False,
+ gaussian_scale=3 / 4,
+):
+ """
+ Refer code here:
+ https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py
+ Apply Hadamard transform + group-wise FP4 quantize/dequantize on x.
+
+ Note:
+ The output is still in the Hadamard-transformed space (no inverse Hadamard is applied).
+ """
+ # Pick a device — we require CUDA
+ device = x.device
+ if device.type != "cuda":
+ raise RuntimeError(
+ f"mxfp4_forward_kernel_wrapper requires a CUDA tensor for 'x', "
+ f"but got device '{device.type}'. Please move inputs to CUDA before calling."
+ )
+
+ # Ensure hadamard_matrix is on the same CUDA device
+ if hadamard_matrix.device != device:
+ hadamard_matrix = hadamard_matrix.to(device)
+
+ # Make sure inputs are contiguous
+ x = x.contiguous()
+ hadamard_matrix = hadamard_matrix.contiguous()
+
+ # Create output tensors on CUDA
+ output = torch.empty_like(x, device=device)
+ if return_clip_mask:
+ clip_mask = torch.empty_like(x, dtype=torch.bool, device=device).contiguous()
+ else:
+ clip_mask = None
+
+ # Get total number of elements and calculate grid for launching the kernel
+ n_elements = x.numel()
+ grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)
+
+ # Launch kernel – no need for `with torch.device(...)`
+ mxfp4_forward_kernel[grid](
+ x_ptr=x,
+ hadamard_matrix_ptr=hadamard_matrix,
+ output_ptr=output,
+ clip_mask_ptr=clip_mask,
+ n_elements=n_elements,
+ hadamard_dim=hadamard_matrix.shape[-1],
+ group_size=32,
+ gaussian_scale=gaussian_scale,
+ quest=quest,
+ )
+
+ return output, clip_mask
diff --git a/auto_round/experimental/transform/triton/utils.py b/auto_round/experimental/transform/triton/utils.py
new file mode 100644
index 000000000..8aa5da743
--- /dev/null
+++ b/auto_round/experimental/transform/triton/utils.py
@@ -0,0 +1,24 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import torch
+
+
+def is_triton_kernel_available() -> bool:
+ """
+ Best-effort check for whether Triton kernel path can be used.
+ """
+ try:
+ import triton # pylint: disable=E0401
+ except Exception:
+ return False
+
+ if not torch.cuda.is_available():
+ return False
+
+ try:
+ from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper # pylint: disable=E0401
+ except Exception:
+ return False
+
+ return True
diff --git a/auto_round/experimental/transform/utils/__init__.py b/auto_round/experimental/transform/utils/__init__.py
new file mode 100644
index 000000000..0b3bf2eae
--- /dev/null
+++ b/auto_round/experimental/transform/utils/__init__.py
@@ -0,0 +1,2 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/utils/hadamard.py b/auto_round/experimental/transform/utils/hadamard.py
new file mode 100644
index 000000000..f1bbec2dd
--- /dev/null
+++ b/auto_round/experimental/transform/utils/hadamard.py
@@ -0,0 +1,151 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import math
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+
+REPO_PATH = Path(__file__).parent / "hadamards.safetensors"
+
+
+__all__ = ["random_hadamard_matrix", "deterministic_hadamard_matrix", "is_pow2"]
+
+
+# note that hadamard matrix multiplication reuses the code from
+# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/hadamard.py
+
+
+def deterministic_hadamard_matrix(
+ size: int,
+ dtype: torch.dtype = torch.bfloat16,
+ device: torch.device = torch.device("cpu"),
+) -> torch.Tensor:
+ """
+ Construct an n-by-n Hadamard matrix, using Sylvester's construction.
+ `n` must be a power of 2.
+
+ Adapted from https://github.com/scipy/scipy/blob/v1.15.2/scipy/linalg/_special_matrices.py # noqa: E501
+
+ :param size: order of the matrix, must be a power of 2
+ :param dtype: data type of matrix
+ :param device: device to construct matrix on
+ :return: hadamard matrix of size `size`
+ """
+ if size <= 0:
+ raise ValueError("Cannot construct deterministic hadamard of size <= 0")
+
+ log2 = int(math.log2(size))
+ if size != 2**log2:
+ raise ValueError("Cannot construct deterministic hadamard of size != 2^n")
+
+ H = torch.tensor([[1]], dtype=dtype, device=device)
+
+ # Sylvester's construction
+ for _ in range(log2):
+ H = torch.vstack((torch.hstack((H, H)), torch.hstack((H, -H))))
+
+ return H
+
+
+def random_hadamard_matrix(
+ size: int,
+ dtype: torch.dtype = torch.bfloat16,
+ device: torch.device = torch.device("cpu"),
+ gen: torch.Generator | None = None,
+) -> torch.Tensor:
+ """
+ Produces a randomly generated Hadamard matrix. Differs from
+ `deterministic_hadamard_matrix` in that this function supports non powers of 2
+ and randomization using a seeded generator
+
+ Adapted from https://github.com/facebookresearch/SpinQuant/blob/main/utils/hadamard_utils.py # noqa: E501
+ Known matrices were retrieved from N. J. A. Sloane's Library of Hadamard Matrices
+ http://www.neilsloane.com/hadamard/ # noqa: E501
+
+ :param size: The dimension of the hamadard matrix
+ :param dtype: data type of matrix
+ :param device: device to construct matrix on
+ :param gen: Optional generator random values
+ :return: randomly generated hadamard matrix
+ """
+ Q = torch.randint(low=0, high=2, size=(size,), generator=gen, dtype=dtype) # cpu
+ Q = Q.to(device=device)
+ Q = Q * 2 - 1
+ Q = torch.diag(Q)
+ return _matmul_hadU(Q)
+
+
+def is_pow2(n: int) -> bool:
+ """
+ Check if a number is a power of 2
+
+ :param n: number to check
+ :return: True iff `n` is a power of 2
+ """
+ return n > 0 and (n & (n - 1) == 0)
+
+
+def _fetch_hadamard_divisor(
+ n: int,
+ dtype: torch.dtype,
+ device: torch.device = torch.device("cpu"),
+ file_path: str = REPO_PATH,
+) -> torch.Tensor | None:
+ """
+ Fetch a known hadamard matrix from the given file path. The returned matrix will
+ be of of size `k` such that `n / k` is a power of two. Return None if no such
+ matrix exists.
+
+ Note: This function reopens the safetensors file every time it is called.
+ This is technically inefficient, but a very small runtime cost and simpler
+ than forcing callers to manage the file open context
+
+ :param n: size of known hadamard matrix
+ :param dtype: data type to move fetched hadamard to
+ :param device: device to move fetched hadamard to
+ :return: a known hadamard matrix of size `n` if one exists, else None
+ """
+ open_device = torch.device("cpu") if device.type == "meta" else device
+ with safe_open(file_path, framework="pt", device=str(open_device)) as file:
+ divisors = sorted((int(key) for key in file.keys()), reverse=True)
+ for divisor in divisors:
+ if n % divisor == 0 and is_pow2(n // divisor):
+ return file.get_tensor(str(divisor)).to(dtype=dtype, device=device)
+
+ return None
+
+
+def _matmul_hadU(X: torch.Tensor) -> torch.Tensor:
+ size = X.size(0)
+ dtype = X.dtype
+ device = X.device
+
+ # Check if we have the determined hadamard matrix
+ hadK = _fetch_hadamard_divisor(size, dtype, device=device)
+ if hadK is None:
+ raise ValueError(f"Cannot construct random hadamard matrix of size {size}")
+ K = hadK.size(0)
+
+ # Reshape diag matrix with randomized -1/+1
+ input = X.clone().view(-1, size, 1)
+ output = input.clone()
+ while input.shape[1] > K:
+ input = input.view(input.shape[0], input.shape[1] // 2, 2, input.shape[2])
+ output = output.view(input.shape)
+ output[:, :, 0, :] = input[:, :, 0, :] + input[:, :, 1, :]
+ output[:, :, 1, :] = input[:, :, 0, :] - input[:, :, 1, :]
+ output = output.view(input.shape[0], input.shape[1], -1)
+ (input, output) = (output, input)
+ assert input.shape[1] == K
+ del output
+
+ # Do not explicitly repeat - OOM
+ # input = torch.bmm(
+ # hadK.repeat(len(input), 1, 1).to(input.device).to(input.dtype), input)
+ # Use bcast instead
+ input = hadK.view(1, K, K).to(input) @ input
+
+ # normalize
+ return input.view(X.shape)
diff --git a/auto_round/experimental/transform/utils/hadamards.safetensors b/auto_round/experimental/transform/utils/hadamards.safetensors
new file mode 100644
index 000000000..9624e0086
Binary files /dev/null and b/auto_round/experimental/transform/utils/hadamards.safetensors differ
diff --git a/auto_round/experimental/transform/utils/matrix.py b/auto_round/experimental/transform/utils/matrix.py
new file mode 100644
index 000000000..46d684e80
--- /dev/null
+++ b/auto_round/experimental/transform/utils/matrix.py
@@ -0,0 +1,98 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import torch
+
+__all__ = ["apply_transform_weight"]
+
+# note that apply_transform_weight reuses some code from
+# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/matrix.py
+
+
+def apply_transform_weight(
+ transform_weight: torch.Tensor,
+ value: torch.Tensor,
+ location: str,
+ module_type: type[torch.nn.Module],
+) -> torch.Tensor:
+ """
+ Using the transform location, apply the transform_weight to the
+ given value wrt linear weights. For more info on input and output transforms,
+ see `TransformLocation`
+
+ The following explains how weights should be applied to values according to location
+
+ let x be input activation
+ W be weight,
+ yh, xh, Wh be transformed output, input, weight
+
+ note that
+ y = (x W.T) // torch.nn.Linear
+
+ Choose values for yh, xh, and Wh which incorporate matrix transforms
+
+ let V, Vi be transform matrices on input side
+ U, Ui be transform matrices on output side
+
+ pick xh = (x V)
+ Wh = (U.T W Vi.T)
+ yh = (y U)
+
+ The following shows that `yh = (xh) (Wh).T` for the chosen values of yh, xh, and Wh
+
+ (xh) (Wh).T = (x V) (U.T W Vi.T).T
+ = (x V) (Vi W.T U) // transpose matrix product identity
+ = (x W.T) U
+ = y U
+ = yh
+
+ :param transform_weight: transform weight to apply
+ :param value: value to apply transform_weight to
+ :param location: determines how weight should be applied
+ :param model_type: result of type(module), passed in to determine application of
+ weight transform
+ :return: value after transform_weight has been applied
+ """
+
+ if location == "input":
+ return _multihead_matmul(value, transform_weight)
+
+ if module_type == torch.nn.Linear:
+ return _multihead_matmul(value, transform_weight.T)
+
+
+def _multihead_matmul(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
+ """
+ Performs A @ B for last two dims of two matrices A and B that possibly
+ have different shapes, as is the case in multi-headed dimension. If
+ shapes are different, this is equivalent to converting the last two dims
+ of the smaller matrix into a block-diagonal matrix with the same shape as
+ the last two dims of the larger matrix.
+
+ E.g. if A is half the size of B, this function will perform
+ [[A ] @ B
+ [ A]]
+
+ If B is a third of the size of A, this function will perform
+ A @ [[B ]
+ [ B ]
+ [ B]]
+
+ This function will error out if the shapes are not evenly divisible
+
+ :param A: left-hand tensor
+ :param B: right-hand tensor
+ :return: result
+ """
+ if A.shape[-1] > B.shape[-2]:
+ head_dim = B.shape[-2]
+ num_heads = A.shape[-1] // head_dim
+ A = A.unflatten(-1, (num_heads, head_dim))
+ return (A @ B).flatten(-2, -1)
+ elif A.shape[-1] < B.shape[-2]:
+ head_dim = A.shape[-1]
+ num_heads = B.shape[-2] // head_dim
+ B = B.unflatten(-2, (num_heads, head_dim))
+ return (A @ B).flatten(-3, -2)
+ else:
+ return A @ B
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index fb347d34d..9cf5fe961 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -775,6 +775,10 @@ def dynamic_import_inference_linear(backend, config):
if "torch_mxfp8" in backend:
return ar_qmodules.MXFP8QuantLinear
if "torch_mxfp4" in backend:
+ transform_config = getattr(config, "transform_config", None)
+ if transform_config is not None and transform_config:
+ if transform_config["transform_type"] == "random_hadamard":
+ return ar_qmodules.TransformMXFP4QuantLinear
return ar_qmodules.MXFP4QuantLinear
if "torch_nvfp4" in backend:
return ar_qmodules.NVFP4QuantLinear
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 901b71dd3..f55ca7c50 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -211,6 +211,8 @@ def get_layer_config(model, quantization_config):
act_data_type = getattr(quantization_config, "act_data_type", None)
act_dynamic = getattr(quantization_config, "act_dynamic", False)
+ transform_config = getattr(quantization_config, "transform_config", None)
+
default_quant_scheme = QuantizationScheme(
bits=bits,
group_size=group_size,
@@ -221,6 +223,7 @@ def get_layer_config(model, quantization_config):
act_sym=act_sym,
act_data_type=act_data_type,
act_dynamic=act_dynamic,
+ transform_config=transform_config,
)
# Determine the quantization block list
@@ -668,6 +671,20 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
layer_configs = get_layer_config(model, quantization_config)
used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)
+ transform_config = getattr(quantization_config, "transform_config", None)
+ if transform_config is not None and transform_config:
+ from auto_round.experimental.transform.apply import apply_transform
+ from auto_round.experimental.transform.transform_config import TransformConfig
+
+ # apply forward hook
+ act_transform_config = TransformConfig(
+ quant_scheme=transform_config["quant_scheme"],
+ transform_block_size=transform_config["transform_block_size"],
+ transform_type=transform_config["transform_type"],
+ location="input",
+ ) # apply to activation
+ model = apply_transform(model, act_transform_config, desc="Register pre forward hook for transform")
+
# Suggest a better backend if available
if backend == "auto":
best_backend = get_highest_priority_backend(
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index c48b86ad7..2480e859e 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -36,6 +36,7 @@ class QuantizationScheme:
act_dynamic: Optional[bool] = None
super_bits: Optional[int] = None
super_group_size: Optional[int] = None
+ transform_config: Optional[dict] = None
@classmethod
def from_dict(cls, config: dict):
diff --git a/requirements.txt b/requirements.txt
index b84214f51..b7abf7181 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ py-cpuinfo
threadpoolctl
torch
tqdm
-transformers>=4.38
\ No newline at end of file
+transformers>=4.38
+pydantic
diff --git a/test/test_cuda/transform/__init__.py b/test/test_cuda/transform/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/test_cuda/transform/test_mxfp4_transform.py b/test/test_cuda/transform/test_mxfp4_transform.py
new file mode 100644
index 000000000..8bbf52aaf
--- /dev/null
+++ b/test/test_cuda/transform/test_mxfp4_transform.py
@@ -0,0 +1,68 @@
+import copy
+import shutil
+
+import pytest
+import torch
+import transformers
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+from ...helpers import get_model_path, save_tiny_model
+
+
+class TestAutoRound:
+ save_dir = "./saved"
+
+ @pytest.fixture(autouse=True, scope="class")
+ def setup_and_teardown_class(self):
+ # ===== SETUP (setup_class) =====
+ print("[Setup] Running before any test in class")
+
+ # Yield to hand control to the test methods
+ yield
+
+ # ===== TEARDOWN (teardown_class) =====
+ print("[Teardown] Running after all tests in class")
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_transform_mxfp4_quant_infer(self):
+ model_name = get_model_path("qwen/Qwen3-0.6B")
+ scheme = "MXFP4"
+
+ from auto_round.utils import llm_load_model
+
+ model, tokenizer = llm_load_model(
+ model_name,
+ platform="hf",
+ device="cpu", # always load cpu first
+ model_dtype=None,
+ trust_remote_code=True,
+ )
+
+ from auto_round.experimental.transform.apply import apply_transform
+ from auto_round.experimental.transform.transform_config import TransformConfig
+
+ transform_config = TransformConfig(quant_scheme="MXFP4")
+ model = apply_transform(
+ model,
+ transform_config,
+ )
+
+ ar = AutoRound(
+ model=model,
+ iters=0,
+ seqlen=2,
+ scheme=scheme,
+ transform_config=transform_config.dict(),
+ )
+ compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ tokenizer.save_pretrained(self.save_dir)
+
+ model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", device_map="cuda")
+ tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
+ from ...helpers import generate_prompt
+
+ generate_prompt(model, tokenizer)