PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support hadamard transform for mxfp4 with rtn or autoround method (#1515)
lkka49550bc5e9c
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 8b161be73..9343130b2 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -89,6 +89,7 @@ class AutoRound:enable_alg_ext: bool = False,disable_opt_rtn: bool | None = None,low_cpu_mem_usage: bool = True,+ transform_config: dict | None = None,**kwargs,) -> BaseCompressor:"""Initialize AutoRound with quantization and tuning configuration.diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5f6f73c6f..8ff815fef 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -150,6 +150,7 @@ SERIALIZATION_KEYS = ("super_bits","super_group_size","to_quant_block_names",+ "transform_config",)@@ -551,6 +552,8 @@ class BaseCompressor(object):except (ImportError, ModuleNotFoundError):logger.error("algorithm extension import error, fallback to default mode")+ self.transform_config = kwargs.pop("transform_config", {})+def _gen_auto_scheme(self) -> dict[str, dict]:if self.mllm:logger.info("AutoScheme is not yet supported for multimodal LLMs.")diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pyindex 0179dd646..cf805645b 100644--- a/auto_round/experimental/qmodules/__init__.py+++ b/auto_round/experimental/qmodules/__init__.py@@ -12,6 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear+from auto_round.experimental.qmodules.mx import MXFP4QuantLinear, MXFP8QuantLinear, TransformMXFP4QuantLinearfrom auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinearfrom auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.pyindex f06076173..a5109b441 100644--- a/auto_round/experimental/qmodules/mx.py+++ b/auto_round/experimental/qmodules/mx.py@@ -68,6 +68,7 @@ class MXQuantLinearBase(QModuleBase):self.config = configself.dtype = dtypeself.pre_dequantized = False+ self.pre_dequantized_input = False# Validate dtypeassert (@@ -145,10 +146,12 @@ class MXQuantLinearBase(QModuleBase):@torch.inference_mode()def forward(self, input: torch.Tensor) -> torch.Tensor:- qdq_input = self.qdq_input(input)+ if not self.pre_dequantized_input:+ input = self.qdq_input(input)+qdq_weight = self.dequant_weight_online()- qdq_weight = qdq_weight.to(qdq_input.dtype)- out = torch.nn.functional.linear(qdq_input, qdq_weight, self.bias)+ qdq_weight = qdq_weight.to(input.dtype)+ out = torch.nn.functional.linear(input, qdq_weight, self.bias)return out@classmethod@@ -193,6 +196,25 @@ class MXFP4QuantLinear(MXQuantLinearBase):return unpacked_data+class TransformMXFP4QuantLinear(MXFP4QuantLinear):+ """+ Quantized linear layer using the MXFP4 quantization scheme.+ """++ def __init__(self, *args, **kwargs):+ self.weight_name = "weight_packed"+ super().__init__(*args, **kwargs)+ self.enable_transform = True+ self.register_buffer(+ "transform_matrix",+ torch.empty(+ self.group_size,+ self.group_size,+ dtype=self.dtype,+ ),+ )++class MXFP8QuantLinear(MXQuantLinearBase):"""Quantized linear layer using the MXFP8 quantization scheme.diff --git a/auto_round/experimental/transform/__init__.py b/auto_round/experimental/transform/__init__.pynew file mode 100644index 000000000..0b3bf2eae--- /dev/null+++ b/auto_round/experimental/transform/__init__.py@@ -0,0 +1,2 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round/experimental/transform/apply.py b/auto_round/experimental/transform/apply.pynew file mode 100644index 000000000..dd5289dcc--- /dev/null+++ b/auto_round/experimental/transform/apply.py@@ -0,0 +1,158 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import torch+import tqdm++from auto_round.experimental.qmodules.mx import MXQuantLinearBase+from auto_round.experimental.transform.transform_config import TransformConfig+from auto_round.experimental.transform.transforms import build_transform++__all__ = ["apply_transform"]+++def apply_transform(model: torch.nn.Module, config: TransformConfig, use_tqdm=True, desc=None):+ """+ Apply a transform config to a model. Add weight transforms and+ activation transforms are attached as submodules and trigger via pytorch hooks++ :param model: model to apply config to+ :param config: transform config to apply+ """++ modules_config = [+ (name, module, config)+ for name, module in model.named_modules()+ if isinstance(module, torch.nn.Linear) or isinstance(module, MXQuantLinearBase)+ ]++ desc = f"Applying {config.transform_type} transforms" if desc is None else desc+ for name, module, config in tqdm.tqdm(modules_config, desc=desc, disable=(not use_tqdm)):+ if "lm_head" in name:+ continue+ _apply_to_module(model, module, config)++ # attach config to model for compression/serialization+ setattr(model, "transform_config", config)++ return model+++def _apply_to_module(+ model: torch.nn.Module,+ module: torch.nn.Module,+ config: TransformConfig,+):+ """+ Create transforms and apply them to the module++ :param model: model which module belongs to+ :param module: target module to apply transforms to+ """++ # create transform as submodule+ transform_name = "transform_matrix"++ if config.location == "input":+ from auto_round.experimental.transform.triton.utils import is_triton_kernel_available++ # activation needs transpose+ inp_transform = build_transform(+ **config.dict(),+ inverse=True,+ device="cpu",+ precision=module.dtype,+ )++ if config.transform_type != "random_hadamard":+ transform_weight = inp_transform.weight+ else:+ transform_weight = None++ if is_triton_kernel_available():+ from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper++ def input_hook(self, args):+ input = args[0]+ # transform(input)+ orig_shape = input.shape+ x_flat = input.contiguous().flatten(end_dim=-2)+ qdq_input, _ = mxfp4_forward_kernel_wrapper(+ x_flat,+ (+ transform_weight if transform_weight is not None else self.transform_matrix.T+ ), # this matrix from w_transform, needs transpose+ )+ return qdq_input.reshape(orig_shape)++ # for fused transform + quantization kernel+ module.pre_dequantized_input = True+ module.register_forward_pre_hook(input_hook, prepend=True)+ else:++ from auto_round.experimental.transform.utils.matrix import _multihead_matmul++ def input_hook(self, args):+ input = args[0]++ ori_shape = input.shape++ if transform_weight is not None:+ input = input.view(-1, transform_weight.shape[0])+ return _multihead_matmul(input, transform_weight.to(input.device)).view(ori_shape)+ else:+ input = input.view(-1, self.transform_matrix.shape[0])+ return _multihead_matmul(input, self.transform_matrix.T).view(ori_shape)++ # for fused transform + quantization kernel+ module.pre_dequantized_input = False+ module.register_forward_pre_hook(input_hook, prepend=True)++ elif config.location == "weight":+ # eagerly apply transformation to weight+ # fuse transform into weight+ assert hasattr(module, "weight")++ w_transform = build_transform(+ **config.dict(),+ device=module.weight.device,+ precision=module.weight.dtype,+ )++ # need save random hadamard matrix needed when inference+ if config.transform_type == "random_hadamard":+ module.register_module(transform_name, w_transform)+ # for saving transform weight+ from auto_round.experimental.transform.patch_modules import patch_quantlinear++ patch_quantlinear()++ if config.need_calibration:+ # for training, the weight changes with every forward pass+ # for autoround tuning: patch wrapper linear qdq_weight func+ from auto_round.experimental.transform.patch_modules import (+ patch_wrapperlinear_to_apply_transform,+ patch_wrapperwalayer_forward_to_apply_transform,+ )++ inp_transform = build_transform(+ **config.dict(),+ location="input",+ inverse=True,+ device=module.weight.device,+ precision=module.weight.dtype,+ )++ patch_wrapperlinear_to_apply_transform(w_transform, inp_transform)+ patch_wrapperwalayer_forward_to_apply_transform(inp_transform)++ else:+ # transform is no longer needed (unfusing is not supported)+ # delattr(module, transform_name)+ # fuse transform into weight+ with torch.no_grad():+ getattr(module, "weight").copy_(w_transform(module.weight).to(module.weight.device))++ else:+ # TODO: apply transform to output/q/k+ raise NotImplementedError()diff --git a/auto_round/experimental/transform/patch_modules.py b/auto_round/experimental/transform/patch_modules.pynew file mode 100644index 000000000..cf15b46e0--- /dev/null+++ b/auto_round/experimental/transform/patch_modules.py@@ -0,0 +1,210 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import torch+import transformers++from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8+from auto_round.wrapper import WrapperLinear, WrapperWALayer+++def patch_wrapperlinear_to_apply_transform(w_transform, inp_transform):+ """+ Globally monkey-patch WrapperLinear._qdq_weight and WrapperLinear._qdq_act so that it applies+ a weight and activation transform before quantization.++ e.g. by apply_transform() before wrapper_block().+ """++ if getattr(WrapperLinear, "_hadamard_patched", False):+ return++ orig_qdq_weight = WrapperLinear._qdq_weight++ def _qdq_weight_patched(self, value, min_scale, max_scale):+ """+ # If no transform attached, fall back to original behavior+ if not hasattr(self.orig_layer, transform_attr):+ return orig_qdq_weight(self, value, min_scale, max_scale)+ """++ if self.orig_layer.bits >= 16:+ # keep original behavior for >=16bit to avoid changing semantics unexpectedly+ return orig_qdq_weight(self, value, min_scale, max_scale)++ min_scale.data.clamp_(0, 1.0)+ max_scale.data.clamp_(0, 1.0)++ weight = self.orig_layer.weight+ if weight.device.type == "meta":+ weight = self.orig_layer.get_weight().to(self.device)++ is_conv1d = type(self.orig_layer) == transformers.pytorch_utils.Conv1D+ if is_conv1d:+ weight = weight.t()++ weight = weight.to(self.device)++ weight_t = w_transform(weight)++ quant_kwargs = {}+ if hasattr(self.orig_layer, "super_bits"):+ quant_kwargs["super_bits"] = self.orig_layer.super_bits+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_size++ weight_q, scale, zp = self.weight_quant_func(+ weight_t,+ bits=self.orig_layer.bits,+ group_size=self.orig_layer.group_size,+ v=value,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.orig_layer.scale_dtype,+ tensor_min=self.weight_min,+ tensor_max=self.weight_max,+ data_type=self.data_type,+ q_scale_thresh=self.q_scale_thresh,+ imatrix=self.orig_layer.imatrix.to(self.device) if hasattr(self.orig_layer, "imatrix") else None,+ global_scale=getattr(self, "weight_global_scale", None),+ **quant_kwargs,+ )++ weight_q = weight_q.to(dtype=weight.dtype)++ if is_conv1d:+ weight_q = weight_q.t()++ return weight_q, scale, zp++ def _qdq_act_patched(self, x, act_max_scale, act_max=None):++ # transform = getattr(self.orig_layer, transform_attr)+ x = inp_transform(x)+ act_max_scale.data.clamp_(0, 1.0)+ x, scale, zp = self.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.q_scale_thresh,+ data_type=self.act_data_type,+ max_scale=act_max_scale,+ tensor_max=act_max,+ global_scale=getattr(self, "input_global_scale", None),+ )+ return x, scale, zp++ WrapperLinear._qdq_weight = _qdq_weight_patched+ WrapperLinear._qdq_act = _qdq_act_patched+ WrapperLinear._hadamard_patched = True+++def patch_wrapperwalayer_forward_to_apply_transform(inp_transform):+ """+ Globally monkey-patch WrapperWALayer.forward so that it applies+ a activation transform before quantization.++ e.g. by apply_transform() before wrapper_block().+ """++ if getattr(WrapperWALayer, "_hadamard_forward_patched", False):+ return++ orig_forward = WrapperWALayer.forward++ def _forward_patched(self, x):+ """+ # If no transform attached, fall back to original behavior+ if not hasattr(self.orig_layer, transform_attr):+ return orig_forward(self, x)+ """++ act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None++ # transform = getattr(self.orig_layer, transform_attr)+ x = inp_transform(x)++ x, _, _ = self.orig_layer.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ tensor_max=act_max,+ )+ return self.orig_layer.forward(x)++ WrapperWALayer.forward = _forward_patched+ WrapperWALayer._hadamard_forward_patched = True+++def patch_quantlinear():+ """ """++ if getattr(QuantLinear, "_pack_patched", False):+ return++ from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+ from auto_round.utils import get_packing_device++ E8M0_EXPONENT_BIAS = 127+ E8M0_EXPONENT_NAN_VAL = 255++ def _pack_patched(+ self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None+ ):+ device = get_packing_device(device)+ if getattr(linear, "bias", None) is not None:+ self.bias = linear.bias.detach().to(torch.float16)++ W = linear.weight.data.detach().to(device)+ if type(linear) == torch.nn.Conv2d:+ W = W.flatten(1)+ if type(linear) == transformers.pytorch_utils.Conv1D:+ W = W.t()++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)+ scales = scales.to(device)+ if self.is_nv:+ assert global_scale is not None and global_scale.numel() == 1+ global_scale = global_scale.reshape([1])+ global_scale = global_scale.to(device)+ scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(+ scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)+ )+ scaled_tensor.clamp_(-6.0, 6.0)+ scaled_tensor = cast_to_fp4(scaled_tensor)+ else:+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)+ if self.is_mx:+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)+ else:+ final_scale = scales.to(torch.float8_e4m3fn)++ self.weight_scale = final_scale+ # self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO+ if self.bits == 8:+ compress_dtype = torch.float8_e4m3fn+ self.weight = scaled_tensor.to(compress_dtype)++ else:+ compress_dtype = torch.uint8+ self.weight_packed = pack_fp4_to_uint8(scaled_tensor)++ if global_scale is not None:+ self.weight_global_scale = global_scale.to(torch.float32).to(device)++ if input_global_scale is not None:+ # TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.+ self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])++ # add transform weight+ transform = getattr(linear, "transform_matrix")+ self.register_buffer("transform_matrix", transform.weight.to(device))+ return++ QuantLinear.pack = _pack_patched+ QuantLinear._pack_patched = Truediff --git a/auto_round/experimental/transform/transform_config.py b/auto_round/experimental/transform/transform_config.pynew file mode 100644index 000000000..91713666b--- /dev/null+++ b/auto_round/experimental/transform/transform_config.py@@ -0,0 +1,43 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from pydantic import BaseModel, Field, field_validator++__all__ = ["TransformConfig"]+++class TransformConfig(BaseModel):+ """+ Configuration of transforms to be applied to a model. This config is to be+ serialized within a model's `config.json` file+ """++ # required, currently only supports mxfp4+ quant_scheme: str = Field(..., description="Quantization scheme. Currently supports 'MXFP4/MXFP8'.")++ transform_block_size: int = Field(default=32)++ transform_type: str = Field(default="hadamard")++ location: str = Field(default="weight", exclude=True)++ # apply transform inside modules for nvfp4, autoround tuning etc.+ need_calibration: bool = Field(default=False, exclude=True)++ # for random hadamard transform+ random_seed: bool = Field(default=False, exclude=True)++ @field_validator("quant_scheme")+ @classmethod+ def validate_quant_scheme(cls, v: str) -> str:+ if v not in ["MXFP4", "MXFP8"]:+ raise ValueError(f"Unsupported quant_scheme: {v}. Currently 'mxfp4/mxfp8' are supported.")+ return v++ @field_validator("transform_type")+ @classmethod+ def validate_transform_type(cls, v: str) -> str:+ allowed = {"hadamard", "random_hadamard"}+ if v not in allowed:+ raise ValueError(f"Unsupported transform_type: {v}. Supported values: {sorted(allowed)}")+ return vdiff --git a/auto_round/experimental/transform/transforms.py b/auto_round/experimental/transform/transforms.pynew file mode 100644index 000000000..38bb1acb7--- /dev/null+++ b/auto_round/experimental/transform/transforms.py@@ -0,0 +1,130 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import inspect+import math+from typing import Any, Callable, Dict++import torch+import torch.nn as nn++from auto_round.experimental.transform.utils.hadamard import deterministic_hadamard_matrix, random_hadamard_matrix+from auto_round.experimental.transform.utils.matrix import apply_transform_weight+++def filter_kwarg_dict(fn_or_method: Callable, kwarg_dict: Dict[str, Any]) -> Dict[str, Any]:+ fn_or_method_keys = inspect.signature(fn_or_method).parameters.keys()+ return {k: v for k, v in kwarg_dict.items() if k in fn_or_method_keys}+++class IdentityTransform(nn.Module):++ def __init__(self, *args, **kwargs):+ super().__init__()++ def forward(self, x: torch.Tensor):+ return x++ def remove_parametrizations(self) -> None:+ pass+++class HadamardTransform(nn.Module):++ def __init__(+ self,+ transform_block_size: int = 32,+ device: torch.device = None,+ precision: torch.dtype = None,+ location: str = "weight",+ module_type: type[torch.nn.Module] = torch.nn.Linear,+ inverse: bool = False,+ ):+ super().__init__()+ self.size = transform_block_size+ self.scale = 1 / math.sqrt(self.size)+ self.location = location+ self.module_type = module_type+ self.inverse = inverse+ self.weight = self._create_weight(self.size, device, precision)++ def _create_weight(+ self,+ size: int,+ device: torch.device = None,+ precision: torch.dtype = None,+ ) -> torch.nn.Parameter:+ data = deterministic_hadamard_matrix(size, precision, device) * self.scale+ # TODO: implement SpinQuant, which rotation matrix is learnable+ return nn.Parameter(data, requires_grad=False)++ def forward(self, x: torch.Tensor):+ # Hadamard transform is it own inverse+ ori_shape = x.shape+ x = x.view(-1, self.size)+ return (+ (+ apply_transform_weight(+ self.weight,+ x.to(dtype=self.weight.dtype),+ self.location,+ self.module_type,+ )+ )+ .to(x.dtype)+ .view(ori_shape)+ )+++class RandomHadamardTransform(HadamardTransform):+ def __init__(+ self,+ *args,+ seed: int | None = None,+ generator: torch.Generator | None = None,+ **kwargs,+ ):+ if generator is not None:+ self.generator = generator+ else:+ self.generator = torch.Generator()+ if seed is not None:+ self.generator.manual_seed(seed)+ super().__init__(*args, **kwargs)++ def _create_weight(+ self,+ size: int,+ device: torch.device = None,+ precision: torch.dtype = None,+ ) -> torch.nn.Parameter:+ data = random_hadamard_matrix(size, precision, device, self.generator) * self.scale+ # activation needs transpose+ if self.inverse:+ data = data.T+ # data = deterministic_hadamard_matrix(size, precision, device) * self.scale+ # TODO: implement SpinQuant, which rotation matrix is learnable+ return nn.Parameter(data, requires_grad=False)+++TRANSFORMS = {+ "identity": IdentityTransform,+ "hadamard": HadamardTransform,+ "random_hadamard": RandomHadamardTransform,+}+++def build_transform(transform_type: str, **transform_kwargs):+ transform = TRANSFORMS[transform_type]+ return transform(**filter_kwarg_dict(transform.__init__, transform_kwargs))diff --git a/auto_round/experimental/transform/triton/__init__.py b/auto_round/experimental/transform/triton/__init__.pynew file mode 100644index 000000000..0b3bf2eae--- /dev/null+++ b/auto_round/experimental/transform/triton/__init__.py@@ -0,0 +1,2 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round/experimental/transform/triton/mxfp4.py b/auto_round/experimental/transform/triton/mxfp4.pynew file mode 100644index 000000000..c26413248--- /dev/null+++ b/auto_round/experimental/transform/triton/mxfp4.py@@ -0,0 +1,192 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Refer code here:+# https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py++import torch+import triton # pylint: disable=E0401+import triton.language as tl # pylint: disable=E0401+++ configs=[+ triton.Config({"BLOCK_SIZE": 32 * 32}),+ triton.Config({"BLOCK_SIZE": 64 * 32}),+ triton.Config({"BLOCK_SIZE": 128 * 32}),+ triton.Config({"BLOCK_SIZE": 256 * 32}),+ triton.Config({"BLOCK_SIZE": 512 * 32}),+ ],+ key=[],+)+def mxfp4_forward_kernel(+ x_ptr,+ hadamard_matrix_ptr,+ output_ptr,+ clip_mask_ptr,+ n_elements: tl.constexpr,+ hadamard_dim: tl.constexpr,+ group_size: tl.constexpr,+ gaussian_scale: tl.constexpr,+ quest: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+):+ offsets_hadamard = tl.arange(0, hadamard_dim * hadamard_dim)+ hadamard_matrix = tl.load(hadamard_matrix_ptr + offsets_hadamard).reshape(hadamard_dim, hadamard_dim)++ # load x+ pid = tl.program_id(0)+ start_idx = pid * BLOCK_SIZE+ offsets = start_idx + tl.arange(0, BLOCK_SIZE)+ mask = offsets < n_elements+ x_flat = tl.load(x_ptr + offsets, mask=mask)++ # hadamard transform+ x = tl.reshape(x_flat, (BLOCK_SIZE // hadamard_dim, hadamard_dim))+ x_had = tl.dot(x, hadamard_matrix)++ # group+ x_had_grouped = tl.reshape(x_had, (BLOCK_SIZE // group_size, group_size))++ # scale+ # quest=True: per-group Gaussian-based scale = gaussian_scale * std+ # quest=False: per-group max-abs-based scale, adjusted to FP4 range+ if quest:+ mean_squared = tl.sum(x_had_grouped * x_had_grouped, axis=-1, keep_dims=True) / group_size+ mean = tl.sum(x_had_grouped, axis=-1, keep_dims=True) / group_size+ std = tl.sqrt(mean_squared - mean * mean)+ scales = gaussian_scale * std + 1e-8+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)))+ x_had_scaled = x_had_grouped / shared_exps+ else:+ scales = tl.max(tl.abs(x_had_grouped), axis=-1, keep_dims=True)+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)) - 2) / (3 / 4)+ x_had_scaled = x_had_grouped / shared_exps++ # quantize+ # Map abs(x) to FP4 levels {0, 0.5, 1, 1.5, 2, 3, 4, 6}+ x_had_scaled_abs = tl.abs(x_had_scaled)+ x_had_scaled_sign = tl.where(+ x_had_scaled > 0,+ 1,+ -1,+ )++ x_fp4 = (+ tl.where(+ x_had_scaled_abs > 5,+ 6,+ tl.where(+ x_had_scaled_abs > 3.5,+ 4,+ tl.where(+ x_had_scaled_abs > 2.5,+ 3,+ tl.where(+ x_had_scaled_abs > 1.75,+ 2,+ tl.where(+ x_had_scaled_abs > 1.25,+ 1.5,+ tl.where(+ x_had_scaled_abs > 0.75,+ 1,+ tl.where(+ x_had_scaled_abs > 0.25,+ 0.5,+ 0,+ ),+ ),+ ),+ ),+ ),+ ),+ )+ * x_had_scaled_sign+ )+ if clip_mask_ptr is not None:+ tl.store(+ clip_mask_ptr + offsets,+ tl.reshape(x_had_scaled_abs < 6, (BLOCK_SIZE,)),+ mask=mask,+ )++ # dequantize+ x_dequantized = x_fp4 * shared_exps++ # Reshape back to flat form for storage+ x_dequantized_flat = tl.reshape(x_dequantized, (BLOCK_SIZE,))++ # store+ tl.store(output_ptr + offsets, x_dequantized_flat, mask=mask)+++def mxfp4_forward_kernel_wrapper(+ x,+ hadamard_matrix,+ return_clip_mask=False,+ quest=False,+ gaussian_scale=3 / 4,+):+ """+ Refer code here:+ https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py+ Apply Hadamard transform + group-wise FP4 quantize/dequantize on x.++ Note:+ The output is still in the Hadamard-transformed space (no inverse Hadamard is applied).+ """+ # Pick a device — we require CUDA+ device = x.device+ if device.type != "cuda":+ raise RuntimeError(+ f"mxfp4_forward_kernel_wrapper requires a CUDA tensor for 'x', "+ f"but got device '{device.type}'. Please move inputs to CUDA before calling."+ )++ # Ensure hadamard_matrix is on the same CUDA device+ if hadamard_matrix.device != device:+ hadamard_matrix = hadamard_matrix.to(device)++ # Make sure inputs are contiguous+ x = x.contiguous()+ hadamard_matrix = hadamard_matrix.contiguous()++ # Create output tensors on CUDA+ output = torch.empty_like(x, device=device)+ if return_clip_mask:+ clip_mask = torch.empty_like(x, dtype=torch.bool, device=device).contiguous()+ else:+ clip_mask = None++ # Get total number of elements and calculate grid for launching the kernel+ n_elements = x.numel()+ grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)++ # Launch kernel – no need for `with torch.device(...)`+ mxfp4_forward_kernel[grid](+ x_ptr=x,+ hadamard_matrix_ptr=hadamard_matrix,+ output_ptr=output,+ clip_mask_ptr=clip_mask,+ n_elements=n_elements,+ hadamard_dim=hadamard_matrix.shape[-1],+ group_size=32,+ gaussian_scale=gaussian_scale,+ quest=quest,+ )++ return output, clip_maskdiff --git a/auto_round/experimental/transform/triton/utils.py b/auto_round/experimental/transform/triton/utils.pynew file mode 100644index 000000000..8aa5da743--- /dev/null+++ b/auto_round/experimental/transform/triton/utils.py@@ -0,0 +1,24 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import torch+++def is_triton_kernel_available() -> bool:+ """+ Best-effort check for whether Triton kernel path can be used.+ """+ try:+ import triton # pylint: disable=E0401+ except Exception:+ return False++ if not torch.cuda.is_available():+ return False++ try:+ from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper # pylint: disable=E0401+ except Exception:+ return False++ return Truediff --git a/auto_round/experimental/transform/utils/__init__.py b/auto_round/experimental/transform/utils/__init__.pynew file mode 100644index 000000000..0b3bf2eae--- /dev/null+++ b/auto_round/experimental/transform/utils/__init__.py@@ -0,0 +1,2 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round/experimental/transform/utils/hadamard.py b/auto_round/experimental/transform/utils/hadamard.pynew file mode 100644index 000000000..f1bbec2dd--- /dev/null+++ b/auto_round/experimental/transform/utils/hadamard.py@@ -0,0 +1,151 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import math+from pathlib import Path++import torch+from safetensors import safe_open++REPO_PATH = Path(__file__).parent / "hadamards.safetensors"+++__all__ = ["random_hadamard_matrix", "deterministic_hadamard_matrix", "is_pow2"]+++# note that hadamard matrix multiplication reuses the code from+# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/hadamard.py+++def deterministic_hadamard_matrix(+ size: int,+ dtype: torch.dtype = torch.bfloat16,+ device: torch.device = torch.device("cpu"),+) -> torch.Tensor:+ """+ Construct an n-by-n Hadamard matrix, using Sylvester's construction.+ `n` must be a power of 2.++ Adapted from https://github.com/scipy/scipy/blob/v1.15.2/scipy/linalg/_special_matrices.py # noqa: E501++ :param size: order of the matrix, must be a power of 2+ :param dtype: data type of matrix+ :param device: device to construct matrix on+ :return: hadamard matrix of size `size`+ """+ if size <= 0:+ raise ValueError("Cannot construct deterministic hadamard of size <= 0")++ log2 = int(math.log2(size))+ if size != 2**log2:+ raise ValueError("Cannot construct deterministic hadamard of size != 2^n")++ H = torch.tensor([[1]], dtype=dtype, device=device)++ # Sylvester's construction+ for _ in range(log2):+ H = torch.vstack((torch.hstack((H, H)), torch.hstack((H, -H))))++ return H+++def random_hadamard_matrix(+ size: int,+ dtype: torch.dtype = torch.bfloat16,+ device: torch.device = torch.device("cpu"),+ gen: torch.Generator | None = None,+) -> torch.Tensor:+ """+ Produces a randomly generated Hadamard matrix. Differs from+ `deterministic_hadamard_matrix` in that this function supports non powers of 2+ and randomization using a seeded generator++ Adapted from https://github.com/facebookresearch/SpinQuant/blob/main/utils/hadamard_utils.py # noqa: E501+ Known matrices were retrieved from N. J. A. Sloane's Library of Hadamard Matrices+ http://www.neilsloane.com/hadamard/ # noqa: E501++ :param size: The dimension of the hamadard matrix+ :param dtype: data type of matrix+ :param device: device to construct matrix on+ :param gen: Optional generator random values+ :return: randomly generated hadamard matrix+ """+ Q = torch.randint(low=0, high=2, size=(size,), generator=gen, dtype=dtype) # cpu+ Q = Q.to(device=device)+ Q = Q * 2 - 1+ Q = torch.diag(Q)+ return _matmul_hadU(Q)+++def is_pow2(n: int) -> bool:+ """+ Check if a number is a power of 2++ :param n: number to check+ :return: True iff `n` is a power of 2+ """+ return n > 0 and (n & (n - 1) == 0)+++def _fetch_hadamard_divisor(+ n: int,+ dtype: torch.dtype,+ device: torch.device = torch.device("cpu"),+ file_path: str = REPO_PATH,+) -> torch.Tensor | None:+ """+ Fetch a known hadamard matrix from the given file path. The returned matrix will+ be of of size `k` such that `n / k` is a power of two. Return None if no such+ matrix exists.++ Note: This function reopens the safetensors file every time it is called.+ This is technically inefficient, but a very small runtime cost and simpler+ than forcing callers to manage the file open context++ :param n: size of known hadamard matrix+ :param dtype: data type to move fetched hadamard to+ :param device: device to move fetched hadamard to+ :return: a known hadamard matrix of size `n` if one exists, else None+ """+ open_device = torch.device("cpu") if device.type == "meta" else device+ with safe_open(file_path, framework="pt", device=str(open_device)) as file:+ divisors = sorted((int(key) for key in file.keys()), reverse=True)+ for divisor in divisors:+ if n % divisor == 0 and is_pow2(n // divisor):+ return file.get_tensor(str(divisor)).to(dtype=dtype, device=device)++ return None+++def _matmul_hadU(X: torch.Tensor) -> torch.Tensor:+ size = X.size(0)+ dtype = X.dtype+ device = X.device++ # Check if we have the determined hadamard matrix+ hadK = _fetch_hadamard_divisor(size, dtype, device=device)+ if hadK is None:+ raise ValueError(f"Cannot construct random hadamard matrix of size {size}")+ K = hadK.size(0)++ # Reshape diag matrix with randomized -1/+1+ input = X.clone().view(-1, size, 1)+ output = input.clone()+ while input.shape[1] > K:+ input = input.view(input.shape[0], input.shape[1] // 2, 2, input.shape[2])+ output = output.view(input.shape)+ output[:, :, 0, :] = input[:, :, 0, :] + input[:, :, 1, :]+ output[:, :, 1, :] = input[:, :, 0, :] - input[:, :, 1, :]+ output = output.view(input.shape[0], input.shape[1], -1)+ (input, output) = (output, input)+ assert input.shape[1] == K+ del output++ # Do not explicitly repeat - OOM+ # input = torch.bmm(+ # hadK.repeat(len(input), 1, 1).to(input.device).to(input.dtype), input)+ # Use bcast instead+ input = hadK.view(1, K, K).to(input) @ input++ # normalize+ return input.view(X.shape)diff --git a/auto_round/experimental/transform/utils/hadamards.safetensors b/auto_round/experimental/transform/utils/hadamards.safetensorsnew file mode 100644index 000000000..9624e0086Binary files /dev/null and b/auto_round/experimental/transform/utils/hadamards.safetensors differdiff --git a/auto_round/experimental/transform/utils/matrix.py b/auto_round/experimental/transform/utils/matrix.pynew file mode 100644index 000000000..46d684e80--- /dev/null+++ b/auto_round/experimental/transform/utils/matrix.py@@ -0,0 +1,98 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import torch++__all__ = ["apply_transform_weight"]++# note that apply_transform_weight reuses some code from+# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/matrix.py+++def apply_transform_weight(+ transform_weight: torch.Tensor,+ value: torch.Tensor,+ location: str,+ module_type: type[torch.nn.Module],+) -> torch.Tensor:+ """+ Using the transform location, apply the transform_weight to the+ given value wrt linear weights. For more info on input and output transforms,+ see `TransformLocation`++ The following explains how weights should be applied to values according to location++ let x be input activation+ W be weight,+ yh, xh, Wh be transformed output, input, weight++ note that+ y = (x W.T) // torch.nn.Linear++ Choose values for yh, xh, and Wh which incorporate matrix transforms++ let V, Vi be transform matrices on input side+ U, Ui be transform matrices on output side++ pick xh = (x V)+ Wh = (U.T W Vi.T)+ yh = (y U)++ The following shows that `yh = (xh) (Wh).T` for the chosen values of yh, xh, and Wh++ (xh) (Wh).T = (x V) (U.T W Vi.T).T+ = (x V) (Vi W.T U) // transpose matrix product identity+ = (x W.T) U+ = y U+ = yh++ :param transform_weight: transform weight to apply+ :param value: value to apply transform_weight to+ :param location: determines how weight should be applied+ :param model_type: result of type(module), passed in to determine application of+ weight transform+ :return: value after transform_weight has been applied+ """++ if location == "input":+ return _multihead_matmul(value, transform_weight)++ if module_type == torch.nn.Linear:+ return _multihead_matmul(value, transform_weight.T)+++def _multihead_matmul(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:+ """+ Performs A @ B for last two dims of two matrices A and B that possibly+ have different shapes, as is the case in multi-headed dimension. If+ shapes are different, this is equivalent to converting the last two dims+ of the smaller matrix into a block-diagonal matrix with the same shape as+ the last two dims of the larger matrix.++ E.g. if A is half the size of B, this function will perform+ [[A ] @ B+ [ A]]++ If B is a third of the size of A, this function will perform+ A @ [[B ]+ [ B ]+ [ B]]++ This function will error out if the shapes are not evenly divisible++ :param A: left-hand tensor+ :param B: right-hand tensor+ :return: result+ """+ if A.shape[-1] > B.shape[-2]:+ head_dim = B.shape[-2]+ num_heads = A.shape[-1] // head_dim+ A = A.unflatten(-1, (num_heads, head_dim))+ return (A @ B).flatten(-2, -1)+ elif A.shape[-1] < B.shape[-2]:+ head_dim = A.shape[-1]+ num_heads = B.shape[-2] // head_dim+ B = B.unflatten(-2, (num_heads, head_dim))+ return (A @ B).flatten(-3, -2)+ else:+ return A @ Bdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex fb347d34d..9cf5fe961 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -775,6 +775,10 @@ def dynamic_import_inference_linear(backend, config):if "torch_mxfp8" in backend:return ar_qmodules.MXFP8QuantLinearif "torch_mxfp4" in backend:+ transform_config = getattr(config, "transform_config", None)+ if transform_config is not None and transform_config:+ if transform_config["transform_type"] == "random_hadamard":+ return ar_qmodules.TransformMXFP4QuantLinearreturn ar_qmodules.MXFP4QuantLinearif "torch_nvfp4" in backend:return ar_qmodules.NVFP4QuantLineardiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 901b71dd3..f55ca7c50 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -211,6 +211,8 @@ def get_layer_config(model, quantization_config):act_data_type = getattr(quantization_config, "act_data_type", None)act_dynamic = getattr(quantization_config, "act_dynamic", False)+ transform_config = getattr(quantization_config, "transform_config", None)+default_quant_scheme = QuantizationScheme(bits=bits,group_size=group_size,@@ -221,6 +223,7 @@ def get_layer_config(model, quantization_config):act_sym=act_sym,act_data_type=act_data_type,act_dynamic=act_dynamic,+ transform_config=transform_config,)# Determine the quantization block list@@ -668,6 +671,20 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mlayer_configs = get_layer_config(model, quantization_config)used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)+ transform_config = getattr(quantization_config, "transform_config", None)+ if transform_config is not None and transform_config:+ from auto_round.experimental.transform.apply import apply_transform+ from auto_round.experimental.transform.transform_config import TransformConfig++ # apply forward hook+ act_transform_config = TransformConfig(+ quant_scheme=transform_config["quant_scheme"],+ transform_block_size=transform_config["transform_block_size"],+ transform_type=transform_config["transform_type"],+ location="input",+ ) # apply to activation+ model = apply_transform(model, act_transform_config, desc="Register pre forward hook for transform")+# Suggest a better backend if availableif backend == "auto":best_backend = get_highest_priority_backend(diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex c48b86ad7..2480e859e 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -36,6 +36,7 @@ class QuantizationScheme:act_dynamic: Optional[bool] = Nonesuper_bits: Optional[int] = Nonesuper_group_size: Optional[int] = None+ transform_config: Optional[dict] = None@classmethoddef from_dict(cls, config: dict):diff --git a/requirements.txt b/requirements.txtindex b84214f51..b7abf7181 100644--- a/requirements.txt+++ b/requirements.txt@@ -6,4 +6,5 @@ py-cpuinfothreadpoolctltorchtqdm-transformers>=4.38\ No newline at end of file+transformers>=4.38+pydanticdiff --git a/test/test_cuda/transform/__init__.py b/test/test_cuda/transform/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/test/test_cuda/transform/test_mxfp4_transform.py b/test/test_cuda/transform/test_mxfp4_transform.pynew file mode 100644index 000000000..8bbf52aaf--- /dev/null+++ b/test/test_cuda/transform/test_mxfp4_transform.py@@ -0,0 +1,68 @@+import copy+import shutil++import pytest+import torch+import transformers+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound++from ...helpers import get_model_path, save_tiny_model+++class TestAutoRound:+ save_dir = "./saved"++ @pytest.fixture(autouse=True, scope="class")+ def setup_and_teardown_class(self):+ # ===== SETUP (setup_class) =====+ print("[Setup] Running before any test in class")++ # Yield to hand control to the test methods+ yield++ # ===== TEARDOWN (teardown_class) =====+ print("[Teardown] Running after all tests in class")+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_transform_mxfp4_quant_infer(self):+ model_name = get_model_path("qwen/Qwen3-0.6B")+ scheme = "MXFP4"++ from auto_round.utils import llm_load_model++ model, tokenizer = llm_load_model(+ model_name,+ platform="hf",+ device="cpu", # always load cpu first+ model_dtype=None,+ trust_remote_code=True,+ )++ from auto_round.experimental.transform.apply import apply_transform+ from auto_round.experimental.transform.transform_config import TransformConfig++ transform_config = TransformConfig(quant_scheme="MXFP4")+ model = apply_transform(+ model,+ transform_config,+ )++ ar = AutoRound(+ model=model,+ iters=0,+ seqlen=2,+ scheme=scheme,+ transform_config=transform_config.dict(),+ )+ compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ tokenizer.save_pretrained(self.save_dir)++ model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", device_map="cuda")+ tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ from ...helpers import generate_prompt++ generate_prompt(model, tokenizer)