Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix hadamard transform weight dtype, using float32 as default and in-place transformed weight. (#1665)

lkk43748dd9fe7f
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 395e116a8..75bc46c0c 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -67,6 +67,7 @@ from auto_round.schemes import (
_handle_special_schemes,
get_gguf_scheme,
preset_name_to_scheme,
+ scheme_to_preset_name,
)
from auto_round.sign_sgd import SignSGD
from auto_round.special_model_handler import get_predefined_ignore_layers, update_module
@@ -560,16 +561,12 @@ class BaseCompressor(object):
# apply hadamard transform
if hadamard_config:
+ logger.info("Applying Hadamard transform to the model.")
from auto_round.experimental.transform.apply import apply_hadamard_transform
- from auto_round.experimental.utils import check_supported_schemes, normalize_hadamard_config
+ from auto_round.experimental.utils import normalize_hadamard_config
- check_supported_schemes(self.scheme)
-
- self.model = apply_hadamard_transform(
- self.model, hadamard_config, need_calibration=True if self.iters > 0 else False
- )
-
- self.hadamard_config = normalize_hadamard_config(hadamard_config)
+ self.hadamard_config = normalize_hadamard_config(hadamard_config, self.data_type)
+ self.model = apply_hadamard_transform(self.model, self.hadamard_config, data_type=self.data_type)
def _gen_auto_scheme(self) -> dict[str, dict]:
if self.mllm:
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
index df20d4afa..4f5ff8e82 100644
--- a/auto_round/experimental/qmodules/__init__.py
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
from auto_round.experimental.qmodules.mx import (
MXFP4QuantLinear,
MXFP8QuantLinear,
MXINT4QuantLinear,
- HadamardMXFP4QuantLinear,
)
+
from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.py
index ff93c34ff..23d05872c 100644
--- a/auto_round/experimental/qmodules/mx.py
+++ b/auto_round/experimental/qmodules/mx.py
@@ -96,6 +96,19 @@ class MXQuantLinearBase(QModuleBase):
)
self.register_buffer("weight_scale", init_weight_scale)
+ hadamard_config = getattr(config, "hadamard_config", None)
+ # TODO: remove the limit: hadamard_config["hadamard_type"] == "random_hadamard"
+ if hadamard_config is not None and hadamard_config["hadamard_type"] == "random_hadamard":
+ self.enable_transform = True
+ self.register_buffer(
+ "hadamard_matrix",
+ torch.empty(
+ self.group_size,
+ self.group_size,
+ dtype=self.dtype,
+ ),
+ )
+
def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:
"""
Initialize weights. This method should be overridden by subclasses.
@@ -238,25 +251,6 @@ class MXINT4QuantLinear(MXQuantLinearBase):
return qdq_linear
-class HadamardMXFP4QuantLinear(MXFP4QuantLinear):
- """
- Quantized linear layer using the MXFP4 quantization scheme.
- """
-
- def __init__(self, *args, **kwargs):
- self.weight_name = "weight_packed"
- super().__init__(*args, **kwargs)
- self.enable_transform = True
- self.register_buffer(
- "hadamard_matrix",
- torch.empty(
- self.group_size,
- self.group_size,
- dtype=self.dtype,
- ),
- )
-
-
class MXFP8QuantLinear(MXQuantLinearBase):
"""
Quantized linear layer using the MXFP8 quantization scheme.
diff --git a/auto_round/experimental/qmodules/nvfp4.py b/auto_round/experimental/qmodules/nvfp4.py
index 81aea8b54..4350f0567 100644
--- a/auto_round/experimental/qmodules/nvfp4.py
+++ b/auto_round/experimental/qmodules/nvfp4.py
@@ -119,6 +119,19 @@ class NVFP4QuantLinear(QModuleBase):
),
)
+ hadamard_config = getattr(config, "hadamard_config", None)
+ # TODO: remove the limit: hadamard_config["hadamard_type"] == "random_hadamard"
+ if hadamard_config is not None and hadamard_config["hadamard_type"] == "random_hadamard":
+ self.enable_transform = True
+ self.register_buffer(
+ "hadamard_matrix",
+ torch.empty(
+ self.group_size,
+ self.group_size,
+ dtype=self.dtype,
+ ),
+ )
+
@staticmethod
def _convert_global_scale_to_float32(state_dict: dict[str, torch.Tensor], name: str):
if name not in state_dict or state_dict[name].dtype == torch.float32:
diff --git a/auto_round/experimental/transform/apply.py b/auto_round/experimental/transform/apply.py
index 6980d75e4..4ae74f5fb 100644
--- a/auto_round/experimental/transform/apply.py
+++ b/auto_round/experimental/transform/apply.py
@@ -4,7 +4,7 @@
import torch
import tqdm
-from auto_round.experimental.qmodules.mx import MXQuantLinearBase
+from auto_round.experimental.qmodules.base import QModuleBase
from auto_round.experimental.transform.hadamard_config import HadamardConfig
from auto_round.experimental.transform.hadamards import build_hadamard_transform
from auto_round.experimental.utils import is_triton_kernel_available, normalize_hadamard_config
@@ -15,10 +15,10 @@ __all__ = ["apply_hadamard_transform"]
def apply_hadamard_transform(
model: torch.nn.Module,
config: str | dict | HadamardConfig | None,
- need_calibration: bool = False,
location: str = "weight",
use_tqdm=True,
desc=None,
+ data_type="mx_fp",
):
"""
Apply a transform configuration to a model.
@@ -29,7 +29,6 @@ def apply_hadamard_transform(
:param model: Model to which the transform configuration will be applied.
:param config: Transform configuration to apply. Supported values are:
* ``str``: A named/preset transform configuration. In this case,
- ``scheme`` is typically required so that the preset can be
resolved to a concrete quantization/transform configuration.
* ``dict``: A raw configuration mapping that will be normalized
(via :func:`normalize_hadamard_config`) and then passed to
@@ -39,13 +38,9 @@ def apply_hadamard_transform(
normalization.
* ``None``: Uses the default behavior of
:func:`_normalize_hadamard_config` (for example, inferring a
- configuration from ``scheme`` or other project defaults), if
+ configuration from ``data_type`` or other project defaults), if
supported.
- :param scheme: Optional quantization/transform scheme identifier used
- when ``config`` is a ``str`` (and, if supported, when it is
- ``None``) to determine which concrete configuration to build.
- Ignored when ``config`` is already a ``dict`` or
- :class:`TransformConfig`.
+ :param data_type: quantization data type.
:param use_tqdm: If ``True``, wrap the per-module application in a
tqdm progress bar.
:param desc: Optional description string to show in the tqdm progress
@@ -53,21 +48,21 @@ def apply_hadamard_transform(
``config.transform_type``.
"""
- config = normalize_hadamard_config(config)
+ config = normalize_hadamard_config(config, data_type)
if not isinstance(config, HadamardConfig):
config = HadamardConfig(**config)
modules_config = [
(name, module, config)
for name, module in model.named_modules()
- if isinstance(module, torch.nn.Linear) or isinstance(module, MXQuantLinearBase)
+ if isinstance(module, torch.nn.Linear) or isinstance(module, QModuleBase)
]
desc = f"Applying {config.hadamard_type} transforms" if desc is None else desc
for name, module, config in tqdm.tqdm(modules_config, desc=desc, disable=(not use_tqdm)):
if "lm_head" in name:
continue
- _apply_to_module(model, module, config, need_calibration, location)
+ _apply_to_module(model, module, config, location, data_type)
# attach config to model for compression/serialization
setattr(model, "hadamard_config", config)
@@ -79,8 +74,8 @@ def _apply_to_module(
model: torch.nn.Module,
module: torch.nn.Module,
config: HadamardConfig,
- need_calibration: bool = False,
location: str = "weight",
+ data_type: str = "mx_fp",
):
"""
Create transforms and apply them to the module
@@ -96,11 +91,11 @@ def _apply_to_module(
# activation needs transpose
input_hadamard_transform = build_hadamard_transform(
- **config.dict(),
+ **config.model_dump(),
location="input",
inverse=True,
device="cpu",
- precision=module.dtype,
+ precision=module.dtype, # for online activation, the transform dtype maybe bfloat16/float16.
)
if config.hadamard_type != "random_hadamard":
@@ -108,21 +103,24 @@ def _apply_to_module(
else:
hadamard_weight = None
- if is_triton_kernel_available():
+ if is_triton_kernel_available(data_type):
from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper
def input_hook(self, args):
input = args[0]
# transform(input)
orig_shape = input.shape
+ orig_dtype = input.dtype
x_flat = input.contiguous().flatten(end_dim=-2)
qdq_input, _ = mxfp4_forward_kernel_wrapper(
x_flat,
(
- hadamard_weight if hadamard_weight is not None else self.hadamard_matrix.T
+ hadamard_weight.to(orig_dtype)
+ if hadamard_weight is not None
+ else self.hadamard_matrix.T.to(orig_dtype)
), # this matrix from w_transform, needs transpose
)
- return qdq_input.reshape(orig_shape)
+ return qdq_input.reshape(orig_shape).to(orig_dtype)
# for fused transform + quantization kernel
module.pre_dequantized_input = True
@@ -135,13 +133,20 @@ def _apply_to_module(
input = args[0]
ori_shape = input.shape
+ orig_dtype = input.dtype
if hadamard_weight is not None:
input = input.view(-1, hadamard_weight.shape[0])
- return _multihead_matmul(input, hadamard_weight.to(input.device)).view(ori_shape)
+ return (
+ (_multihead_matmul(input, hadamard_weight.to(input.device).to(orig_dtype)))
+ .view(ori_shape)
+ .to(orig_dtype)
+ )
else:
input = input.view(-1, self.hadamard_matrix.shape[0])
- return _multihead_matmul(input, self.hadamard_matrix.T).view(ori_shape)
+ return (
+ (_multihead_matmul(input, self.hadamard_matrix.T.to(orig_dtype))).view(ori_shape).to(orig_dtype)
+ )
# for fused transform + quantization kernel
module.pre_dequantized_input = False
@@ -153,45 +158,35 @@ def _apply_to_module(
assert hasattr(module, "weight")
weight_hadamard_transform = build_hadamard_transform(
- **config.dict(),
+ **config.model_dump(),
location="weight",
device=module.weight.device,
- precision=module.weight.dtype,
)
# need save random hadamard matrix needed when inference
if config.hadamard_type == "random_hadamard":
- module.register_module(config.hadamard_type, weight_hadamard_transform)
# for saving transform weight
from auto_round.experimental.transform.patch_modules import patch_quantlinear
- patch_quantlinear(config.hadamard_type)
+ patch_quantlinear(weight_hadamard_transform)
- if need_calibration:
- # for training, the weight changes with every forward pass
- # for autoround tuning: patch wrapper linear qdq_weight func
- from auto_round.experimental.transform.patch_modules import (
- patch_wrapperlinear_to_apply_transform,
- patch_wrapperwalayer_forward_to_apply_transform,
- )
-
- input_hadamard_transform = build_hadamard_transform(
- **config.dict(),
- location="input",
- inverse=True,
- device=module.weight.device,
- precision=module.weight.dtype,
- )
+ # for autoround tuning: weight not tuning
+ # for rtn: weight transformed before saving
+ from auto_round.experimental.transform.patch_modules import (
+ patch_wrapperlinear_to_apply_transform,
+ patch_wrapperwalayer_forward_to_apply_transform,
+ )
- patch_wrapperlinear_to_apply_transform(weight_hadamard_transform, input_hadamard_transform)
- patch_wrapperwalayer_forward_to_apply_transform(input_hadamard_transform)
+ input_hadamard_transform = build_hadamard_transform(
+ **config.model_dump(),
+ location="input",
+ inverse=True,
+ device=module.weight.device,
+ precision=module.weight.dtype, # for online activation, the transform dtype maybe bfloat16/float16.
+ )
- else:
- # transform is no longer needed (unfusing is not supported)
- # delattr(module, transform_name)
- # fuse transform into weight
- with torch.no_grad():
- getattr(module, "weight").copy_(weight_hadamard_transform(module.weight).to(module.weight.device))
+ patch_wrapperlinear_to_apply_transform(weight_hadamard_transform, input_hadamard_transform)
+ patch_wrapperwalayer_forward_to_apply_transform(input_hadamard_transform)
else:
# TODO: apply transform to output/q/k
diff --git a/auto_round/experimental/transform/hadamards.py b/auto_round/experimental/transform/hadamards.py
index 712232a9a..42c47818d 100644
--- a/auto_round/experimental/transform/hadamards.py
+++ b/auto_round/experimental/transform/hadamards.py
@@ -34,11 +34,25 @@ class HadamardTransform(nn.Module):
self,
block_size: int = 32,
device: torch.device = None,
- precision: torch.dtype = None,
+ precision: torch.dtype = torch.float32,
location: str = "weight",
module_type: type[torch.nn.Module] = torch.nn.Linear,
inverse: bool = False,
):
+ """Initialize a Hadamard transform module.
+
+ Args:
+ block_size: Size of each Hadamard block. The input tensor is reshaped
+ to ``(-1, block_size)`` before applying the transform.
+ device: Device on which to create the Hadamard matrix.
+ precision: Data type used for the Hadamard matrix weights, using float32 as default.
+ location: Target location used by ``apply_transform_weight`` when
+ applying the transform.
+ module_type: Module type associated with the transform application,
+ typically ``torch.nn.Linear``.
+ inverse: Whether to build the inverse form of the transform.
+ """
+
super().__init__()
self.size = block_size
self.scale = 1 / math.sqrt(self.size)
@@ -51,7 +65,7 @@ class HadamardTransform(nn.Module):
self,
size: int,
device: torch.device = None,
- precision: torch.dtype = None,
+ precision: torch.dtype = torch.float32,
) -> torch.nn.Parameter:
data = deterministic_hadamard_matrix(size, precision, device) * self.scale
# TODO: implement SpinQuant, which rotation matrix is learnable
@@ -78,10 +92,14 @@ class HadamardTransform(nn.Module):
class RandomHadamardTransform(HadamardTransform):
def __init__(
self,
- *args,
+ block_size: int = 32,
+ device: torch.device = None,
+ precision: torch.dtype = None,
+ location: str = "weight",
+ module_type: type[torch.nn.Module] = torch.nn.Linear,
+ inverse: bool = False,
seed: int | None = None,
generator: torch.Generator | None = None,
- **kwargs,
):
if generator is not None:
self.generator = generator
@@ -89,7 +107,15 @@ class RandomHadamardTransform(HadamardTransform):
self.generator = torch.Generator()
if seed is not None:
self.generator.manual_seed(seed)
- super().__init__(*args, **kwargs)
+
+ super().__init__(
+ block_size=block_size,
+ device=device,
+ precision=precision,
+ location=location,
+ module_type=module_type,
+ inverse=inverse,
+ )
def _create_weight(
self,
diff --git a/auto_round/experimental/transform/patch_modules.py b/auto_round/experimental/transform/patch_modules.py
index 934ebea9d..34cb2f1ce 100644
--- a/auto_round/experimental/transform/patch_modules.py
+++ b/auto_round/experimental/transform/patch_modules.py
@@ -32,67 +32,30 @@ def patch_wrapperlinear_to_apply_transform(w_transform, inp_transform):
# keep original behavior for >=16bit to avoid changing semantics unexpectedly
return orig_qdq_weight(self, value, min_scale, max_scale)
- min_scale.data.clamp_(0, 1.0)
- max_scale.data.clamp_(0, 1.0)
+ if getattr(self, "applied_weight_hadamard", None) is None:
+ with torch.no_grad():
+ weight = self.orig_layer.weight
+ if weight.device.type == "meta":
+ weight = self.orig_layer.get_weight().to(self.device)
- weight = self.orig_layer.weight
- if weight.device.type == "meta":
- weight = self.orig_layer.get_weight().to(self.device)
+ is_conv1d = type(self.orig_layer) == transformers.pytorch_utils.Conv1D
+ if is_conv1d:
+ weight = weight.t().continuous()
+ new_weight = w_transform(weight).to(self.device)
+ if is_conv1d:
+ new_weight = weight.t().continuous()
+ self.orig_layer.weight.data.copy_(new_weight)
+ self.applied_weight_hadamard = True
- is_conv1d = type(self.orig_layer) == transformers.pytorch_utils.Conv1D
- if is_conv1d:
- weight = weight.t()
+ return orig_qdq_weight(self, value, min_scale, max_scale)
- weight = weight.to(self.device)
-
- weight_t = w_transform(weight)
-
- quant_kwargs = {}
- if hasattr(self.orig_layer, "super_bits"):
- quant_kwargs["super_bits"] = self.orig_layer.super_bits
- quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
-
- weight_q, scale, zp = self.weight_quant_func(
- weight_t,
- bits=self.orig_layer.bits,
- group_size=self.orig_layer.group_size,
- v=value,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.orig_layer.scale_dtype,
- tensor_min=self.weight_min,
- tensor_max=self.weight_max,
- data_type=self.data_type,
- q_scale_thresh=self.q_scale_thresh,
- imatrix=self.orig_layer.imatrix.to(self.device) if hasattr(self.orig_layer, "imatrix") else None,
- global_scale=getattr(self, "weight_global_scale", None),
- **quant_kwargs,
- )
-
- weight_q = weight_q.to(dtype=weight.dtype)
-
- if is_conv1d:
- weight_q = weight_q.t()
-
- return weight_q, scale, zp
+ orig_qdq_act = WrapperLinear._qdq_act
def _qdq_act_patched(self, x, act_max_scale, act_max=None):
- # transform = getattr(self.orig_layer, transform_attr)
x = inp_transform(x)
- act_max_scale.data.clamp_(0, 1.0)
- x, scale, zp = self.act_quant_func(
- x,
- bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.act_group_size,
- scale_dtype=self.orig_layer.scale_dtype,
- q_scale_thresh=self.q_scale_thresh,
- data_type=self.act_data_type,
- max_scale=act_max_scale,
- tensor_max=act_max,
- global_scale=getattr(self, "input_global_scale", None),
- )
- return x, scale, zp
+
+ return orig_qdq_act(self, x, act_max_scale, act_max)
WrapperLinear._qdq_weight = _qdq_weight_patched
WrapperLinear._qdq_act = _qdq_act_patched
@@ -139,7 +102,7 @@ def patch_wrapperwalayer_forward_to_apply_transform(inp_transform):
WrapperWALayer._hadamard_forward_patched = True
-def patch_quantlinear(hadamard_type):
+def patch_quantlinear(w_transform):
""" """
if getattr(QuantLinear, "_pack_patched", False):
@@ -202,8 +165,7 @@ def patch_quantlinear(hadamard_type):
self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])
# add transform weight
- transform = getattr(linear, hadamard_type)
- self.register_buffer("hadamard_matrix", transform.weight.to(device))
+ self.register_buffer("hadamard_matrix", w_transform.weight.to(device))
return
QuantLinear.pack = _pack_patched
diff --git a/auto_round/experimental/transform/triton/mxfp4.py b/auto_round/experimental/transform/triton/mxfp4.py
index c26413248..8028c167b 100644
--- a/auto_round/experimental/transform/triton/mxfp4.py
+++ b/auto_round/experimental/transform/triton/mxfp4.py
@@ -161,6 +161,11 @@ def mxfp4_forward_kernel_wrapper(
if hadamard_matrix.device != device:
hadamard_matrix = hadamard_matrix.to(device)
+ dtype = hadamard_matrix.dtype
+
+ if x.dtype != dtype:
+ x = x.to(dtype)
+
# Make sure inputs are contiguous
x = x.contiguous()
hadamard_matrix = hadamard_matrix.contiguous()
diff --git a/auto_round/experimental/transform/utils/hadamard.py b/auto_round/experimental/transform/utils/hadamard.py
index 5ec6bccbd..320ae1832 100644
--- a/auto_round/experimental/transform/utils/hadamard.py
+++ b/auto_round/experimental/transform/utils/hadamard.py
@@ -70,8 +70,8 @@ def random_hadamard_matrix(
:param gen: Optional generator random values
:return: randomly generated hadamard matrix
"""
- Q = torch.randint(low=0, high=2, size=(size,), generator=gen, dtype=dtype) # cpu
- Q = Q.to(device=device)
+ Q = torch.randint(low=0, high=2, size=(size,), generator=gen, device="cpu") # cpu
+ Q = Q.to(device=device, dtype=dtype)
Q = Q * 2 - 1
Q = torch.diag(Q)
return _matmul_hadU(Q)
diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.py
index 39a7ff135..8527dbe00 100644
--- a/auto_round/experimental/utils.py
+++ b/auto_round/experimental/utils.py
@@ -16,11 +16,12 @@ from typing import Any
import torch
+from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.experimental.transform.hadamard_config import HadamardConfig
from auto_round.experimental.transform.hadamards import HADAMARDS
from auto_round.utils import logger
-SUPPORTED_QUANTIZATION_SCHEMES = ["MXFP4"]
+SUPPORTED_QUANTIZATION_SCHEMES = ["MXFP8", "MXFP4", "NVFP4"]
def per_tensor_fp8_qdq(
@@ -114,10 +115,12 @@ def clean_model_parameters_and_buffers_(model: torch.nn.Module, name_tuple: tupl
_clean_param_or_buff_if_exists(module, name_tuple)
-def is_triton_kernel_available() -> bool:
+def is_triton_kernel_available(data_type: str) -> bool:
"""
Best-effort check for whether Triton kernel path can be used.
"""
+ if is_nv_fp(data_type):
+ return False
try:
import triton # pylint: disable=E0401
except Exception:
@@ -134,60 +137,104 @@ def is_triton_kernel_available() -> bool:
return True
-def normalize_hadamard_config(hadamard_config: str | dict | HadamardConfig | None) -> dict[str, Any]:
+def normalize_hadamard_config(hadamard_config: str | dict | HadamardConfig | None, data_type: str) -> dict[str, Any]:
"""
Normalize and validate `hadamard_config`.
Supported input types:
- - None -> {}
- - dict -> validated via HadamardConfig
+ - None -> {}
+ - dict -> validated via HadamardConfig
- HadamardConfig -> validated & converted to dict
- - str -> shorthand for `transform_type` in TRANSFORMS keys
-
- On any validation failure, raises ValueError/TypeError.
+ - str -> shorthand for `hadamard_type` in HADAMARDS keys
+
+ Additional behavior:
+ - If block_size is not set by user:
+ - mx_fp -> default block_size to 32
+ - nv_fp -> default block_size to 16
+ - other data types -> emit a warning
+ - If block_size is set but does not match the recommended value:
+ - mx_fp expects 32
+ - nv_fp expects 16
+ - emit a warning
"""
+
+ def _apply_data_type_block_size(cfg_dict: dict[str, Any], block_size_explicitly_set: bool) -> dict[str, Any]:
+ block_size = cfg_dict.get("block_size")
+
+ if not block_size_explicitly_set or block_size is None:
+ if is_mx_fp(data_type):
+ cfg_dict["block_size"] = 32
+ logger.warning("block_size is not set for data_type 'mx_fp'; defaulting to 32.")
+ elif is_nv_fp(data_type):
+ cfg_dict["block_size"] = 16
+ logger.warning("block_size is not set for data_type 'nv_fp'; defaulting to 16.")
+ else:
+ logger.warning(
+ f"block_size is not set and cannot be inferred for data_type {data_type!r}; "
+ "please set block_size explicitly in hadamard_config if needed."
+ )
+ else:
+ if is_mx_fp(data_type) and block_size != 32:
+ logger.warning(f"data_type is 'mx_fp' but block_size={block_size}; recommended value is 32.")
+ elif is_nv_fp(data_type) and block_size != 16:
+ logger.warning(f"data_type is 'nv_fp' but block_size={block_size}; recommended value is 16.")
+
+ return cfg_dict
+
# 1) None -> {}
if hadamard_config is None:
return {}
- # 2) Already a HadamardConfig instance
+ # 2) HadamardConfig instance
if isinstance(hadamard_config, HadamardConfig):
- # Ensure it passes its own validation and convert to dict
- cfg = HadamardConfig.model_validate(hadamard_config).model_dump()
- return cfg
+ raw_cfg_dict = hadamard_config.model_dump(exclude_unset=True)
+ block_size_explicitly_set = "block_size" in raw_cfg_dict
- # 3) dict -> validate via HadamardConfig
+ cfg_dict = dict(raw_cfg_dict)
+ cfg_dict = _apply_data_type_block_size(cfg_dict, block_size_explicitly_set)
+
+ try:
+ return HadamardConfig.model_validate(cfg_dict).model_dump()
+ except Exception as e:
+ raise ValueError(f"Invalid HadamardConfig: {e}") from e
+
+ # 3) dict
if isinstance(hadamard_config, dict):
+ block_size_explicitly_set = "block_size" in hadamard_config
+
+ cfg_dict = dict(hadamard_config)
+ cfg_dict = _apply_data_type_block_size(cfg_dict, block_size_explicitly_set)
+
try:
- cfg = HadamardConfig.model_validate(hadamard_config).model_dump()
+ return HadamardConfig.model_validate(cfg_dict).model_dump()
except Exception as e:
raise ValueError(f"Invalid hadamard_config dict: {e}") from e
- return cfg
- # 4) str -> shorthand for transform_type
+ # 4) str -> shorthand for hadamard_type
if isinstance(hadamard_config, str):
key = hadamard_config.strip()
if not key:
return {}
if key == "default":
- cfg = HadamardConfig()
- return cfg.model_dump()
+ cfg_dict = {}
+ cfg_dict = _apply_data_type_block_size(cfg_dict, block_size_explicitly_set=False)
+ try:
+ return HadamardConfig.model_validate(cfg_dict).model_dump()
+ except Exception as e:
+ raise ValueError(f"Invalid default hadamard_config after data_type adjustment: {e}") from e
if key not in HADAMARDS:
- raise ValueError(
- f"Invalid hadamard_config string: {key!r}. " f"Expected one of {sorted(HADAMARDS.keys())}."
- )
+ raise ValueError(f"Invalid hadamard_config string: {key!r}. Expected one of {sorted(HADAMARDS.keys())}.")
cfg_dict = {"hadamard_type": key}
+ cfg_dict = _apply_data_type_block_size(cfg_dict, block_size_explicitly_set=False)
try:
- cfg = HadamardConfig.model_validate(cfg_dict).model_dump()
+ return HadamardConfig.model_validate(cfg_dict).model_dump()
except Exception as e:
raise ValueError(f"hadamard_config built from string {key!r} is invalid for HadamardConfig: {e}") from e
- return cfg
-
raise TypeError(
"hadamard_config must be one of: None, dict, HadamardConfig, or str " f"(got {type(hadamard_config).__name__})"
)
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index ee472ea82..b36234c8c 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -799,10 +799,6 @@ def dynamic_import_inference_linear(backend, config):
if "torch_mxint4" in backend:
return ar_qmodules.MXINT4QuantLinear
if "torch_mxfp4" in backend:
- hadamard_config = getattr(config, "hadamard_config", None)
- if hadamard_config is not None and hadamard_config:
- if hadamard_config["hadamard_type"] == "random_hadamard":
- return ar_qmodules.HadamardMXFP4QuantLinear
return ar_qmodules.MXFP4QuantLinear
if "torch_nvfp4" in backend:
return ar_qmodules.NVFP4QuantLinear
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index ab03e701e..e769e1eca 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -691,7 +691,11 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
hadamard_type=hadamard_config["hadamard_type"],
) # apply to activation
model = apply_hadamard_transform(
- model, act_hadamard_config, location="input", desc="Register pre forward hook for hadamard transform"
+ model,
+ act_hadamard_config,
+ location="input",
+ desc="Register pre forward hook for hadamard transform",
+ data_type=quantization_config.data_type,
)
# Suggest a better backend if available
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 2e0641554..20d238134 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -109,6 +109,18 @@ def preset_name_to_scheme(name: str) -> QuantizationScheme:
return scheme_args
+def scheme_to_preset_name(scheme: Union[str, QuantizationScheme]) -> str:
+ """Get preset scheme name from a QuantizationScheme instance."""
+ if isinstance(scheme, str):
+ name = scheme.upper()
+ return name if name in PRESET_SCHEMES else ""
+
+ for key, val in PRESET_SCHEMES.items():
+ if val == scheme:
+ return key
+ return ""
+
+
def is_preset_scheme(name: str) -> bool:
"""Check if the given name is a preset scheme name."""
return name.upper() in PRESET_SCHEMES