Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Revert "refactor: decouple datatype quantization from algorithms (#2406)

Heng Guo86e97adf13cd
diff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.py
index cfc921152..b3fd72535 100644
--- a/auto_round/algorithms/base.py
+++ b/auto_round/algorithms/base.py
@@ -127,10 +127,6 @@ class BaseAlgorithm:
def scheme(self) -> "QuantizationScheme | None":
return self.__run_ctx.scheme if self.__run_ctx is not None else None
- @property
- def run_context(self) -> "QuantizationRunContext | None":
- return self.__run_ctx
-
# ── Derived convenience properties ────────────────────────────────────────
@property
def model(self) -> "torch.nn.Module | None":
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 4ea01e8d8..725ead4fc 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -42,9 +42,8 @@ from auto_round.algorithms.config_resolver import (
split_quantization_configs,
)
from auto_round.algorithms.utils import _has_nvfp4_layer
-from auto_round.data_type.base import canonical_data_type, prepare_data_type_block
from auto_round.logger import logger
-from auto_round.utils import check_to_quantized, clear_memory
+from auto_round.utils import clear_memory
from auto_round.utils.device_manager import device_manager
if TYPE_CHECKING: # avoid circular imports at runtime
@@ -211,6 +210,10 @@ class AlgorithmComposer:
", ".join(blockers),
)
+ if _has_nvfp4_layer(orchestrator):
+ can_compile_block_forward = False
+ logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")
+
# Bind compressor-level infrastructure (set before _build_quantizer is called).
self.block_forward = (
BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)
@@ -242,15 +245,31 @@ class AlgorithmComposer:
Returns a list of hook handles that the caller must remove when done.
"""
- from auto_round.data_type.base import cache_activation_quantizer
+ from auto_round.compressors.utils import is_nv_fp
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+
+ is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)
def collect_act_max(module, input, output):
input = input[0] if isinstance(input, (tuple, list)) else input
if input.numel() == 0:
return
- quantizer = cache_activation_quantizer(module)
- if quantizer is not None:
- module.act_max = quantizer.observe(input, getattr(module, "act_max", None))
+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fp
+ input, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)
+ act_max = torch.max(torch.abs(input), dim=-1).values
+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:
+ module.act_max = act_max
+ if is_module_act_nv_fp:
+ max_val = act_max.max()
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ return
+ act_max = act_max.to(module.act_max.device)
+ if is_module_act_nv_fp:
+ max_val = torch.max(act_max.max(), module.act_max.max())
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ module.act_max = torch.max(act_max, module.act_max)
def should_collect(name, module):
from auto_round.compressors.utils import check_need_act_calibration
@@ -259,8 +278,8 @@ class AlgorithmComposer:
if isinstance(module, tuple(SUPPORTED_LAYER_TYPES)):
return (
hasattr(module, "act_dynamic")
+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)
and check_to_quantized(module)
- and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False))
)
if hasattr(module, "bits"):
act_dynamic = getattr(module, "act_dynamic", True)
@@ -268,8 +287,8 @@ class AlgorithmComposer:
act_bits = getattr(module, "act_bits", 16)
return (
module.bits <= 8
+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
and check_to_quantized(module)
- and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False))
)
return False
@@ -311,19 +330,39 @@ class AlgorithmComposer:
q_inputs: Optional list of quantized input tensors; used instead of
``fp_inputs`` when provided.
"""
- from auto_round.data_type.base import cache_activation_quantizer
+ from auto_round.compressors.utils import is_nv_fp
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
target_input = q_inputs or fp_inputs
- quantizer = cache_activation_quantizer(layer)
- if quantizer is None:
- return
+ act_group_size = getattr(layer, "act_group_size")
+ if act_group_size is None:
+ act_group_size = layer.group_size
+ act_data_type = getattr(layer, "act_data_type")
+ if act_data_type is None:
+ act_data_type = layer.data_type
+ is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False
for inp in target_input:
if isinstance(inp, (tuple, list)):
inp = inp[0]
if inp.numel() == 0:
continue
- layer.act_max = quantizer.observe(inp, getattr(layer, "act_max", None))
+ inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)
+ act_max = torch.max(torch.abs(inp), dim=-1).values
+
+ if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:
+ layer.act_max = act_max
+ if is_act_nv_fp_flag:
+ max_val = act_max.max()
+ layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ continue
+
+ act_max = act_max.to(layer.act_max.device)
+ if is_act_nv_fp_flag:
+ max_val = torch.max(act_max.max(), layer.act_max.max())
+ layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ layer.act_max = torch.max(act_max, layer.act_max)
def need_quanted_input(self):
for preprocessor in self.preprocessors:
@@ -432,26 +471,12 @@ class AlgorithmComposer:
act_data_type = self.scheme.act_data_type if self.scheme else data_type
if act_data_type is not None or not act_dynamic:
from auto_round.compressors.utils import is_nv_fp
+ from auto_round.data_type.utils import update_block_global_scale_if_needed
from auto_round.utils import set_amax_for_all_moe_layers
if is_nv_fp(act_data_type) or not act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
-
- runtimes_by_datatype = {}
- for name, module in block.named_modules():
- if not check_to_quantized(module):
- continue
- module_data_type = getattr(module, "data_type", data_type)
- try:
- canonical_id = canonical_data_type(module_data_type)
- except LookupError:
- continue
- runtimes_by_datatype.setdefault(canonical_id, {})[name] = {
- "data_type": module_data_type,
- "group_size": getattr(module, "group_size", group_size),
- }
- for canonical_id, runtimes in runtimes_by_datatype.items():
- prepare_data_type_block(canonical_id, block, runtimes)
+ update_block_global_scale_if_needed(block, data_type, group_size)
if q_inputs is not None and fp_inputs is not q_inputs:
clear_memory(fp_inputs)
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index fe463fe20..70b3299e4 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
from auto_round.algorithms.base import BaseAlgorithm
from auto_round.algorithms.quantization.config import QuantizationConfig
-from auto_round.data_type.base import create_quantizer
+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.logger import logger
from auto_round.utils import (
check_to_quantized,
@@ -57,49 +57,6 @@ class BaseQuantizer(BaseAlgorithm):
def can_compile_block_forward(self):
return True
- def build_weight_qdq(self, *, weight: torch.Tensor, default_qdq, parameters: dict[str, torch.Tensor]):
- """Return the per-layer weight QDQ callable used for tuning and export.
-
- Algorithms that need different QDQ behavior may override this method.
- The returned callable accepts ``weight`` and keyword ``materialize=False``
- and returns ``WeightQuantizationResult``. Bind settings in a closure and
- add trainable tensors to ``parameters`` before returning. This per-layer
- dictionary is also used by the optimizer and best-parameter restoration.
- Read its entries at call time; do not replace the dictionary.
-
- Args:
- weight: Logical [out_features, in_features] weights on the tuning
- device, including transposed Conv1D weights. Use only to create
- parameters with the right shape/device; do not capture this
- tensor in the returned closure.
- default_qdq: Datatype function accepting weight, materialize=False,
- and explicit datatype parameters. This builder binds the current
- parameter dictionary to that function. Existing weight parameters
- are datatype-dependent: value is a rounding offset, while
- min_scale/max_scale adjust clipping bounds where supported.
- parameters: Per-layer tensors collected for optimization. Add a
- torch.nn.Parameter to train it; fixed settings belong in the
- closure. Use distinct names that do not collide with wrapper
- attributes. Read entries on each call to see restored best values.
-
- Returns:
- Callable qdq(weight, *, materialize=False). With materialize=True,
- its WeightQuantizationResult must contain scale/zero-point/metadata
- required by the chosen datatype's exporter. Preserve autograd during
- training.
-
- To customize, override this method and use ``weight`` to initialize
- any new entries in ``parameters``. In the returned function, read those
- entries and transform the input or explicitly pass the datatype options
- you need. The implementation below is the default binding template.
- """
-
- def qdq(weight, *, materialize=False):
- """Use current parameters during training and restored best values at export."""
- return default_qdq(weight, materialize=materialize, **parameters)
-
- return qdq
-
# ── Calibration hook registration ─────────────────────────────────────────
def register_fp_input_forward_hooks(self, block: torch.nn.Module) -> list:
"""Register hooks that fire during the reference (FP-input) block forward.
@@ -137,22 +94,53 @@ class BaseQuantizer(BaseAlgorithm):
if not check_to_quantized(module):
continue
is_quantized = True
+ bits = getattr(module, "bits", None)
+ group_size = getattr(module, "group_size", None)
+ sym = getattr(module, "sym", None)
+ data_type = getattr(module, "data_type", None)
super_bits = getattr(module, "super_bits", None)
super_group_size = getattr(module, "super_group_size", None)
+ scale_dtype = self.scale_dtype
+ quant_dtype = data_type
+ if quant_dtype not in QUANT_FUNC_WITH_DTYPE:
+ quant_dtype = f"{quant_dtype}_{'sym' if sym else 'asym'}"
+ if not hasattr(self, "iters") or self.iters <= 0: # pylint: disable=E1101
+ tmp_dtype = "rtn_" + quant_dtype
+ if tmp_dtype in QUANT_FUNC_WITH_DTYPE:
+ quant_dtype = tmp_dtype
+ quant_func = QUANT_FUNC_WITH_DTYPE[quant_dtype]
# float32 is used in RTN scale search; avoids caching a bf16 copy.
weight_dtype = torch.float32 if super_group_size is not None else module.weight.dtype
- quantizer = create_quantizer(module, disable_opt_rtn=True)
+ quant_kwargs = {
+ "bits": bits,
+ "group_size": group_size,
+ "super_bits": super_bits,
+ "super_group_size": super_group_size,
+ "scale_dtype": scale_dtype,
+ }
try:
- weight = module.weight.to(dtype=weight_dtype, device=device_manager.device)
- quantizer.initialize(weight)
- quantizer.write_back(module, weight)
+ weight, scale, zp = quant_func(
+ module.weight.to(dtype=weight_dtype, device=device_manager.device),
+ **quant_kwargs,
+ )
except torch.OutOfMemoryError:
- logger.error(traceback.format_exc())
- logger.warning("falling back to CPU")
- weight = module.weight.to("cpu")
- quantizer.initialize(weight)
- quantizer.write_back(module, weight)
- del weight, quantizer
+ cuda_error_msg = traceback.format_exc()
+ try:
+ logger.error(cuda_error_msg)
+ logger.warning("falling back to CPU")
+ weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)
+ except Exception:
+ raise
+ module.weight.data.copy_(weight.cpu())
+ for param_name, val in zip(["scale", "zp"], [scale, zp]):
+ if isinstance(val, dict):
+ for k, v in val.items():
+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
+ elif isinstance(val, torch.Tensor):
+ setattr(module, param_name, val.cpu())
+ else:
+ setattr(module, param_name, val)
+ del weight, scale, zp
clear_memory()
return is_quantized
@@ -244,7 +232,6 @@ class BaseQuantizer(BaseAlgorithm):
enable_torch_compile=self.compress_context.enable_torch_compile,
disable_opt_rtn=disable_opt_rtn,
iters=0,
- weight_qdq_builder=self.build_weight_qdq,
)
layer = layer.unwrapper({})
except torch.OutOfMemoryError:
@@ -261,7 +248,6 @@ class BaseQuantizer(BaseAlgorithm):
enable_round_tuning=False,
enable_torch_compile=self.compress_context.enable_torch_compile,
iters=0,
- weight_qdq_builder=self.build_weight_qdq,
)
layer = layer.unwrapper({})
except Exception:
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 278240c87..116e5b2e9 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -374,7 +374,6 @@ class SignRoundQuantizer(BaseQuantizer):
self.enable_norm_bias_tuning,
enable_torch_compile=self.compress_context.enable_torch_compile,
device=device,
- weight_qdq_builder=self.build_weight_qdq,
)
round_params = []
@@ -504,6 +503,9 @@ class SignRoundQuantizer(BaseQuantizer):
cache_budget,
device,
)
+ if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):
+ for n, m in block.named_modules():
+ m.cur_iter = i
total_loss = 0
global_indices = index_sampler.next_batch()
if valid_token_mask:
@@ -664,7 +666,6 @@ class SignRoundQuantizer(BaseQuantizer):
enable_minmax_tuning=self.enable_minmax_tuning,
enable_torch_compile=self.compress_context.enable_torch_compile,
device=device,
- weight_qdq_builder=self.build_weight_qdq,
).to(device)
round_params = []
minmax_params = []
@@ -672,7 +673,7 @@ class SignRoundQuantizer(BaseQuantizer):
if "min" in key or "max" in key:
minmax_params.append(wrapper_linear.params[key])
else:
- round_params.append(wrapper_linear.params[key])
+ round_params.append(wrapper_linear.value)
if len(round_params) + len(minmax_params) <= 0:
dump_info = f"quantized {layer_name}"
logger.info(dump_info)
diff --git a/auto_round/algorithms/quantization/sign_roundv2/__init__.py b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
index b567f1ca6..4eb376f53 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/__init__.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
@@ -12,8 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer
+from auto_round.algorithms.quantization.sign_roundv2.quantizer import (
+ SignRoundDQWrapperLinear,
+ SignRoundOptimizedWrapperLinear,
+ SignRoundV2Quantizer,
+)
__all__ = [
+ "SignRoundDQWrapperLinear",
+ "SignRoundOptimizedWrapperLinear",
"SignRoundV2Quantizer",
]
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
index 4a7cabc4e..eb9c977c6 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -13,20 +13,309 @@
# limitations under the License.
from contextlib import nullcontext
+from functools import partial
from typing import TYPE_CHECKING, Callable, Union
import torch
+import transformers
from torch import autocast
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
-from auto_round.algorithms.registry import register_pipeline_member
-from auto_round.data_type.base import canonical_data_type
-from auto_round.logger import logger
-from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
if TYPE_CHECKING:
from auto_round.algorithms.composer import AlgorithmComposer
+from auto_round.algorithms.registry import register_pipeline_member
+from auto_round.data_type.gguf import (
+ double_quant_tensor_sym_rtn,
+ quant_tensor_gguf_asym_dq,
+ quant_tensor_gguf_sym_dq,
+ search_gguf_scale_min_asym,
+ search_gguf_scale_min_sym,
+)
+from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym
+from auto_round.data_type.utils import (
+ get_optimized_quant_func,
+ reshape_imatrix_for_weight,
+ reshape_pad_tensor_by_group_size,
+ revert_tensor_by_pad,
+ round_ste,
+ search_optimized_init_scale,
+)
+from auto_round.logger import logger
+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal
+from auto_round.wrapper import WrapperLinear, wrapper_block
+
+
+def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):
+ """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
+ maxq = int(2.0**bits) - 1
+ inverse_scale = get_reciprocal(scale)
+ int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
+ qdq = (scale * int_w - wmin).to(orig_dtype)
+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq
+
+
+def _dq_sym_qdq(tensor, scale, bits, v=0):
+ """Pure sym double-quant qdq math given precomputed scale (compilable)."""
+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
+
+ group_size = 16
+ super_group_size = 16
+ maxq = int(2.0 ** (bits - 1))
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ orig_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
+ ggml_type = f"q{bits}_k"
+ block_size, _ = GGML_QUANT_SIZES[ggml_type]
+ n_blocks = tensor.nelement() // block_size
+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+ if isinstance(v, torch.Tensor):
+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+ else:
+ v_r = v
+ zp = torch.full_like(scale, maxq)
+ inverse_scale = get_reciprocal(scale)
+ int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq
+ qdq = (scale * (int_w - zp)).to(orig_dtype)
+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq
+
+
+def _named_wrapper_block(wrapper_cls, name: str):
+ wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)
+ wrapped.__name__ = name
+ return wrapped
+
+
+def _dq_asym_group_size(bits: int) -> int:
+ """Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""
+ return 16 if bits == 2 else 32
+
+
+class SignRoundOptimizedWrapperLinear(WrapperLinear):
+ minmax_scale_bound = (0.0, 2.0)
+
+ def _init_tuning_params_and_quant_func(self):
+ super()._init_tuning_params_and_quant_func()
+
+ layer = self.orig_layer
+ data_type = layer.data_type
+ weight_reshape = self._prepare_init_scale_weight()
+ imatrix = reshape_imatrix_for_weight(getattr(layer, "imatrix", None), weight_reshape, layer.group_size)
+
+ self.init_scale = search_optimized_init_scale(
+ weight_reshape, data_type, layer.bits, imatrix, self.q_scale_thresh
+ )
+ self.weight_quant_func = get_optimized_quant_func(data_type)
+ if self.init_scale is None or self.weight_quant_func is None:
+ raise ValueError(
+ f"SignRound optimized path does not support data_type={data_type!r}; "
+ "expected a symmetric int / mx / nv type."
+ )
+
+ self.data_type = data_type
+ if hasattr(layer, "imatrix"):
+ del layer.imatrix
+ if self.enable_torch_compile:
+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
+
+ def _prepare_init_scale_weight(self) -> torch.Tensor:
+ """Return the group-reshaped weight that seeds the init-scale search.
+
+ Reproduces the layout the quant func sees at tuning time: the (optionally
+ transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in
+ ``clip_as_init`` mode and stored a per-group clip range on the layer, the
+ weight is clamped to that range first so the searched ``init_scale``
+ already reflects the AWQ clip (``max_scale`` then tunes a coefficient on
+ top of it).
+ """
+ layer = self.orig_layer
+ weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight
+ if isinstance(layer, transformers.pytorch_utils.Conv1D):
+ weight = weight.t()
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)
+
+ clip_min = getattr(layer, "awq_clip_min", None)
+ clip_max = getattr(layer, "awq_clip_max", None)
+ if clip_max is not None:
+ clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
+ clip_min = (
+ -clip_max
+ if clip_min is None
+ else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
+ )
+ if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:
+ weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)
+ else:
+ logger.warning_once(
+ "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "
+ "grouped weight shape %s."
+ % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))
+ )
+ return weight_reshape
+
+
+class SignRoundDQWrapperLinear(WrapperLinear):
+ minmax_scale_bound = (0.5, 1.5)
+
+ def __init__(self, *args, **kwargs) -> None:
+ if "enable_minmax_tuning" in kwargs:
+ logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")
+ kwargs["enable_minmax_tuning"] = False
+ super().__init__(*args, **kwargs)
+ self.prev_scale = None
+ self.prev_wmin = None
+ self.prev_d_scale = None
+ self.prev_d_wmin = None
+
+ def _init_tuning_params_and_quant_func(self):
+ super()._init_tuning_params_and_quant_func()
+ # The double-quant search path is data-dependent and kept un-compiled,
+ # while ``weight_quant_func`` is the compilable pure-math half.
+ self._is_dq_path = False
+ self._dq_kind = None
+ self.search_func = None
+ if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:
+ self._is_dq_path = True
+ if self.orig_layer.data_type == "int_asym_dq":
+ self.search_func = search_gguf_scale_min_asym
+ self.weight_quant_func = _dq_asym_qdq
+ self._dq_kind = "asym"
+ else:
+ self.search_func = search_gguf_scale_min_sym
+ self.weight_quant_func = _dq_sym_qdq
+ self._dq_kind = "sym"
+ elif self.orig_layer.sym:
+ self.weight_quant_func = quant_tensor_sym
+ else:
+ self.weight_quant_func = quant_tensor_asym
+ self.data_type = self.orig_layer.data_type
+ if self.enable_act_quant:
+ self.act_quant_func = (
+ quant_tensor_gguf_asym_dq
+ if self.orig_layer.act_data_type == "int_asym_dq"
+ else quant_tensor_gguf_sym_dq
+ )
+ if self.enable_torch_compile:
+ self.act_quant_func = compile_func(self.act_quant_func, self.device)
+ self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)
+ if self.enable_torch_compile:
+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
+
+ @torch.no_grad()
+ def _run_search(self, weight, v):
+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
+
+ bits = self.orig_layer.bits
+ scale_dtype = self.orig_layer.scale_dtype
+ imatrix = getattr(self.orig_layer, "imatrix", None)
+
+ if self._dq_kind == "asym":
+ group_size = _dq_asym_group_size(bits)
+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
+ v_r = v
+ if isinstance(v, torch.Tensor):
+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
+ scale, wmin, d_scale, d_wmin = self.search_func(
+ t,
+ bits=bits,
+ scale_dtype=scale_dtype,
+ imatrix=imatrix,
+ split_num=1,
+ v=v_r,
+ )
+ return {
+ "scale": scale.clone(),
+ "wmin": wmin.clone(),
+ "d_scale": d_scale.clone(),
+ "d_wmin": d_wmin.clone(),
+ }
+
+ group_size = 16
+ super_group_size = 16
+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
+ ggml_type = f"q{bits}_k"
+ block_size, _ = GGML_QUANT_SIZES[ggml_type]
+ n_blocks = t.nelement() // block_size
+ t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+ v_r = v
+ if isinstance(v, torch.Tensor):
+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+ super_bits = 6 if bits == 3 else 8
+ scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)
+ scale = scale.to(scale_dtype)
+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
+ scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
+ scale = scale.unsqueeze(-1)
+ return {"scale": scale.clone(), "d_scale": d_scale.clone()}
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ if not self._is_dq_path:
+ return super()._qdq_weight(value, min_scale, max_scale)
+
+ if self.orig_layer.bits >= 16:
+ return self.orig_layer.weight, None, None
+ min_bound, max_bound = self.minmax_scale_bound
+ min_scale.data.clamp_(min_bound, max_bound)
+ max_scale.data.clamp_(min_bound, max_bound)
+ weight = self.orig_layer.weight
+ if weight.device.type == "meta":
+ weight = self.orig_layer.get_weight().to(self.device)
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
+ weight = weight.t()
+
+ iter_v = getattr(self, "cur_iter", 0)
+ need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)
+ if need_search:
+ params = self._run_search(weight, value)
+ self.prev_scale = params["scale"]
+ self.prev_d_scale = params["d_scale"]
+ if self._dq_kind == "asym":
+ self.prev_wmin = params["wmin"]
+ self.prev_d_wmin = params["d_wmin"]
+ else:
+ params = {
+ "scale": self.prev_scale.detach(),
+ "d_scale": self.prev_d_scale.detach(),
+ }
+ if self._dq_kind == "asym":
+ params["wmin"] = self.prev_wmin.detach()
+ params["d_wmin"] = self.prev_d_wmin.detach()
+
+ bits = self.orig_layer.bits
+ if self._dq_kind == "asym":
+ group_size = _dq_asym_group_size(bits)
+ weight_q = self.weight_quant_func(
+ weight,
+ params["scale"],
+ params["wmin"],
+ bits,
+ group_size,
+ v=value,
+ )
+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
+ zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}
+ else:
+ weight_q = self.weight_quant_func(
+ weight,
+ params["scale"],
+ bits,
+ v=value,
+ )
+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
+ zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))
+
+ weight_q = weight_q.to(weight.dtype)
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
+ weight_q = weight_q.t()
+ return weight_q, scale_out, zp_out
@register_pipeline_member(SignRoundV2Config)
@@ -42,10 +331,18 @@ class SignRoundV2Quantizer(SignRoundQuantizer):
"""Model-level setup: initialise scheme-dependent state once before block iteration."""
super().prepare_run(composer=composer)
- canonical_dtype = canonical_data_type(self.scheme.data_type)
- is_optimized_family = canonical_dtype.startswith(("int_", "mx_", "nv_", "nvfp"))
- if self.scheme.sym and self.scheme.super_group_size is None and is_optimized_family:
- if self.scheme.bits > 2 and canonical_dtype.startswith("int_"):
+ if (
+ self.scheme.sym
+ and self.scheme.super_group_size is None
+ and (
+ self.scheme.data_type.startswith("int")
+ or self.scheme.data_type.startswith("mx")
+ or self.scheme.data_type.startswith("nv")
+ )
+ ):
+ if self.scheme.bits > 2 and not (
+ self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")
+ ):
logger.warning_once(
"algorithm extension has only undergone limited validation on "
"W2A16,INT4, MXFP4 and NVFP4; use with caution."
@@ -54,9 +351,10 @@ class SignRoundV2Quantizer(SignRoundQuantizer):
self._use_outlier_suppressed_loss = True
else:
self._use_outlier_suppressed_loss = False
- # QDQ ownership stays with the generic wrapper. Datatype adapters
- # select optimized and double-quant behavior from the layer request;
- # SignRoundV2 only owns its loss policy.
+ self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")
+
+ if self.scheme.data_type.endswith("dq"):
+ self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")
def can_compile_block_forward(self):
return False
diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.py
index d3822c2f6..5ef7e1d71 100644
--- a/auto_round/algorithms/quantization/utils.py
+++ b/auto_round/algorithms/quantization/utils.py
@@ -14,7 +14,8 @@
import torch
-from auto_round.data_type.base import cache_activation_quantizer
+from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
@@ -25,15 +26,28 @@ def register_act_max_hooks(quantizer, model):
input = input[0]
if input.numel() == 0:
return
- activation_quantizer = cache_activation_quantizer(module)
- if activation_quantizer is not None:
- module.act_max = activation_quantizer.observe(input, getattr(module, "act_max", None))
+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp
+ input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)
+ act_max = torch.max(torch.abs(input), dim=-1).values
+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:
+ module.act_max = act_max
+ if is_module_act_nv_fp:
+ max_val = act_max.max()
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ act_max = act_max.to(module.act_max.device)
+ if is_module_act_nv_fp:
+ max_val = torch.max(act_max.max(), module.act_max.max())
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ module.act_max = torch.max(act_max, module.act_max)
hook_handles = []
if isinstance(model, SUPPORTED_LAYER_TYPES):
if (
hasattr(model, "act_dynamic")
- and getattr(cache_activation_quantizer(model), "requires_calibration", False)
+ and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)
and check_to_quantized(model)
):
hook_handles.append(model.register_forward_hook(get_act_max_hook))
@@ -42,7 +56,7 @@ def register_act_max_hooks(quantizer, model):
for name, module in model.named_modules():
if (
hasattr(module, "act_dynamic")
- and getattr(cache_activation_quantizer(module), "requires_calibration", False)
+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)
and check_to_quantized(module)
):
hook_handles.append(module.register_forward_hook(get_act_max_hook))
@@ -50,14 +64,12 @@ def register_act_max_hooks(quantizer, model):
if name in quantizer.layer_config:
config = quantizer.layer_config[name]
- module.act_dynamic = config.get("act_dynamic", True)
- module.act_data_type = config.get("act_data_type", "int_sym")
- module.act_bits = config.get("act_bits", 16)
- module.act_group_size = config.get("act_group_size", quantizer.act_group_size)
- module.act_sym = config.get("act_sym", True)
+ act_dynamic = config.get("act_dynamic", True)
+ act_data_type = config.get("act_data_type", None)
+ act_bits = config.get("act_bits", 16)
if (
config["bits"] <= 8
- and getattr(cache_activation_quantizer(module), "requires_calibration", False)
+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
and check_to_quantized(config)
):
hook_handles.append(module.register_forward_hook(get_act_max_hook))
diff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.py
index 40c839419..2c8332e5e 100644
--- a/auto_round/algorithms/transforms/awq/base.py
+++ b/auto_round/algorithms/transforms/awq/base.py
@@ -730,7 +730,7 @@ class AWQTransform(BasePreprocessor):
# the grid-search loop. Normal AWQ flow requires one mapping to have
# compatible quant params, but keeping this per-layer avoids hidden
# coupling to the first layer and makes direct calls robust.
- bl_quantizers = {bl: self._qdq_tool.resolve_quantizer(bl_params[bl]) for bl in mapping.balance_layers}
+ bl_quant_funcs = {bl: self._qdq_tool.resolve_quant_funcs(bl_params[bl]) for bl in mapping.balance_layers}
best_error = float("inf")
best_scales = None
@@ -751,11 +751,12 @@ class AWQTransform(BasePreprocessor):
# de-smoothed result back, so the parent forward below sees the
# weights the layer would actually compute with.
for bl in mapping.balance_layers:
- quantizer = bl_quantizers[bl]
+ quant_func, opt_quant_func = bl_quant_funcs[bl]
w_qdq = self._qdq_tool.qdq(
orig_state[bl] * scales_view,
bl_params[bl],
- quantizer=quantizer,
+ quant_func=quant_func,
+ opt_quant_func=opt_quant_func,
imatrix=getattr(bl, "imatrix", None),
)
bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)
@@ -766,12 +767,13 @@ class AWQTransform(BasePreprocessor):
else:
total_loss = 0.0
for bl in mapping.balance_layers:
- quantizer = bl_quantizers[bl]
+ quant_func, opt_quant_func = bl_quant_funcs[bl]
w_orig = orig_weights[bl].to(device)
w_qdq = self._qdq_tool.qdq(
w_orig * scales_view,
bl_params[bl],
- quantizer=quantizer,
+ quant_func=quant_func,
+ opt_quant_func=opt_quant_func,
imatrix=getattr(bl, "imatrix", None),
)
total_loss += (w_orig - w_qdq / scales_view).pow(2).sum().item()
@@ -1030,8 +1032,8 @@ class AWQTransform(BasePreprocessor):
# group size and drop super-block (double-quant) params, which the clip
# path does not apply.
clip_params = {**params, "group_size": gs, "super_bits": None, "super_group_size": None}
- quantizer = self._qdq_tool.resolve_quantizer(clip_params)
- if quantizer is None:
+ quant_func, _ = self._qdq_tool.resolve_quant_funcs(clip_params)
+ if quant_func is None:
return None
feat = input_feat.to(device).reshape(-1, in_features)
@@ -1070,7 +1072,7 @@ class AWQTransform(BasePreprocessor):
max_val = org_max_val * shrink
cur_w = torch.clamp(w_b, min_val, max_val)
cur_w_flat = cur_w.reshape(cur_w.shape[0], n_group * gs)
- q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quantizer=quantizer).reshape(cur_w.shape)
+ q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quant_func=quant_func).reshape(cur_w.shape)
cur_out = (feat * q_w).sum(dim=-1)
err = (cur_out - org_out).pow(2).mean(dim=1).view(min_errs.shape)
improved = err < min_errs
diff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.py
index e0bb05ed5..2a51956a2 100644
--- a/auto_round/algorithms/transforms/awq/qdq.py
+++ b/auto_round/algorithms/transforms/awq/qdq.py
@@ -28,15 +28,19 @@ Usage (the same unified QDQ serves both the scale grid-search and clip-search)::
tool.configure(compressor) # opt_rtn / v2 flags
params = tool.resolve_params(layer) # per-layer scheme
- quantizer = tool.resolve_quantizer(params) # dispatch once, reuse in loop
- w_qdq = tool.qdq(weight, params, quantizer=quantizer)
+ qf, opt_qf = tool.resolve_quant_funcs(params) # dispatch once, reuse in loop
+ w_qdq = tool.qdq(weight, params, quant_func=qf, opt_quant_func=opt_qf)
"""
from __future__ import annotations
import torch
-from auto_round.data_type.base import create_quantizer
+from auto_round.data_type.utils import (
+ compute_optimized_init_scale,
+ get_optimized_quant_func,
+ get_quant_func,
+)
class QDQTool:
@@ -94,7 +98,7 @@ class QDQTool:
"""Resolve the per-layer weight-quant params (``layer_config`` + fallbacks).
Single source of every scheme field the QDQ needs, so callers pass the
- returned dict straight to :meth:`resolve_quantizer` and :meth:`qdq`.
+ returned dict straight to :meth:`resolve_quant_funcs` and :meth:`qdq`.
``super_bits`` / ``super_group_size`` are the GGUF double-quant
super-block params and are ``None`` for non-GGUF schemes.
"""
@@ -109,12 +113,25 @@ class QDQTool:
"super_group_size": cfg.get("super_group_size", None),
}
- def resolve_quantizer(self, params: dict):
- """Create one configured quantizer to reuse throughout a weight search."""
- return create_quantizer(
- {**params, "scale_dtype": torch.float32},
+ def resolve_quant_funcs(self, params: dict):
+ """Dispatch ``(quant_func, opt_quant_func)`` for ``params``.
+
+ ``opt_quant_func`` is non-``None`` only when the SignRoundV2 optimized
+ init-scale path applies (it depends solely on ``data_type`` / ``sym``).
+ Hoisting this out of a search loop avoids repeated dispatch.
+ """
+ quant_func, _ = get_quant_func(
+ params["data_type"],
+ params["bits"],
+ params["sym"],
disable_opt_rtn=params["disable_opt_rtn"],
+ group_size=params["group_size"],
+ iters=0,
)
+ opt_quant_func = None
+ if self.use_v2_scale_search and params["sym"]:
+ opt_quant_func = get_optimized_quant_func(params["data_type"])
+ return quant_func, opt_quant_func
# ── the unified QDQ for AWQ search ──────────────────────
@torch.no_grad()
@@ -123,11 +140,48 @@ class QDQTool:
weight: torch.Tensor,
params: dict,
*,
- quantizer=None,
+ quant_func=None,
+ opt_quant_func=None,
imatrix: torch.Tensor | None = None,
) -> torch.Tensor:
- """Simulate quantization for a search candidate without changing stored weights."""
- if quantizer is None:
- quantizer = self.resolve_quantizer(params)
- quantizer.initialize(weight, imatrix=imatrix)
- return quantizer.quantize(weight)
+ """Quantize-dequantize ``weight`` under the resolved ``params``.
+
+ The single QDQ entry point for every scheme: ``bits`` / ``group_size`` /
+ ``sym`` / ``data_type`` plus the optional GGUF super-block params drive
+ the dispatched ``quant_func``; a non-``None`` ``opt_quant_func`` enables
+ the SignRoundV2 optimized init-scale path for this weight. Used for
+ both the scale grid-search and the clip-search -- reference/loss only,
+ it never mutates stored weights.
+ """
+ if quant_func is None:
+ quant_func, opt_quant_func = self.resolve_quant_funcs(params)
+ if quant_func is None:
+ raise RuntimeError(
+ "QDQTool: no quantization function resolved for "
+ f"data_type={params['data_type']}, bits={params['bits']}, "
+ f"sym={params['sym']}, group_size={params['group_size']}."
+ )
+
+ quant_kwargs = {
+ "bits": params["bits"],
+ "group_size": params["group_size"],
+ "data_type": params["data_type"],
+ "sym": params["sym"],
+ }
+ if params.get("super_bits") is not None:
+ quant_kwargs["super_bits"] = params["super_bits"]
+ if params.get("super_group_size") is not None:
+ quant_kwargs["super_group_size"] = params["super_group_size"]
+
+ active_quant_func = quant_func
+ if opt_quant_func is not None:
+ init_scale = compute_optimized_init_scale(
+ weight, params["data_type"], params["bits"], params["group_size"], imatrix=imatrix
+ )
+ if init_scale is not None:
+ quant_kwargs["init_scale"] = init_scale
+ quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)
+ active_quant_func = opt_quant_func
+
+ qdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)
+ return qdq_weight
diff --git a/auto_round/algorithms/transforms/hadamard/patch.py b/auto_round/algorithms/transforms/hadamard/patch.py
index 6971cd304..e3f5f9f95 100644
--- a/auto_round/algorithms/transforms/hadamard/patch.py
+++ b/auto_round/algorithms/transforms/hadamard/patch.py
@@ -24,8 +24,6 @@ Each patch is idempotent: calling it twice has no effect.
from __future__ import annotations
-import math
-
import torch
import transformers
@@ -77,8 +75,6 @@ def patch_wrapperlinear_to_apply_transform(
if is_conv1d:
new_weight = new_weight.t().contiguous()
self.orig_layer.weight.data.copy_(new_weight)
- logical_weight = new_weight.t() if is_conv1d else new_weight
- self.weight_quantizer.refresh(logical_weight, imatrix=getattr(self.orig_layer, "imatrix", None))
self.applied_weight_hadamard = True
return _orig_qdq_weight(self, value, min_scale, max_scale)
@@ -110,18 +106,16 @@ def patch_wrapperwalayer_forward_to_apply_transform(
_orig_forward = WrapperWALayer.forward
def _forward_patched(self, x):
- x = inp_transform(x)
- import auto_round.envs as envs
-
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
- act_scale = envs.AR_ACT_SCALE
- max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
- min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
- x = self.activation_quantizer.qdq(
+ x = inp_transform(x)
+ x, _, _ = self.orig_layer.act_quant_func(
x,
- observed_max=act_max,
- min_scale=min_scale,
- max_scale=max_scale,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.orig_layer.q_scale_thresh,
+ data_type=self.orig_layer.act_data_type,
+ tensor_max=act_max,
)
return self.orig_layer.forward(x)
diff --git a/auto_round/algorithms/transforms/svdquant/residual.py b/auto_round/algorithms/transforms/svdquant/residual.py
index aa3515b3e..461f6763b 100644
--- a/auto_round/algorithms/transforms/svdquant/residual.py
+++ b/auto_round/algorithms/transforms/svdquant/residual.py
@@ -26,7 +26,7 @@ from dataclasses import dataclass
import torch
-from auto_round.data_type.base import create_quantizer, quantize_activation
+from auto_round.data_type.utils import get_quant_func
_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})
_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})
@@ -108,12 +108,16 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.
requested_dtype = values["data_type"]
if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:
requested_dtype = f"{requested_dtype}_rceil"
- quantizer = create_quantizer(
- {**values, "data_type": requested_dtype, "scale_dtype": tensor.dtype},
+ quant_func, resolved_dtype = get_quant_func(
+ dtype=requested_dtype,
+ bits=values["bits"],
+ sym=values["sym"],
disable_opt_rtn=True,
+ group_size=values["group_size"],
+ iters=0,
)
- quantizer.initialize(tensor)
- resolved_base_dtype = requested_dtype.removesuffix("_rceil")
+ logical_dtype = resolved_dtype.removeprefix("rtn_")
+ resolved_base_dtype = logical_dtype.removesuffix("_rceil")
if (
resolved_base_dtype in _MXFP4_ALIASES
and values["bits"] == 4
@@ -128,7 +132,12 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.
f"got group_size={values['group_size']!r}."
)
- qdq = quantizer.quantize(tensor)
+ qdq, _, _ = quant_func(
+ tensor=tensor,
+ bits=values["bits"],
+ group_size=values["group_size"],
+ data_type=logical_dtype,
+ )
if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:
raise ValueError(
f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "
@@ -153,13 +162,7 @@ def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch
@torch.inference_mode()
def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:
"""Apply deployment-compatible dynamic activation quantize-dequantize."""
- values = _validate_scheme_values(scheme)
- qdq = quantize_activation(activation, values)
- if qdq.shape != activation.shape or qdq.dtype != activation.dtype or qdq.device != activation.device:
- raise ValueError("Activation RTN QDQ must preserve shape, dtype, and device.")
- if not torch.isfinite(qdq).all():
- raise ValueError("Activation RTN QDQ produced non-finite values.")
- return qdq
+ return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")
@torch.inference_mode()
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 8d383e311..8d23ff152 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -44,7 +44,13 @@ from auto_round.auto_scheme.utils import (
remove_quant_scheme,
)
from auto_round.calib_dataset import get_dataloader
-from auto_round.data_type.base import canonical_data_type
+from auto_round.data_type.gguf import (
+ quant_tensor_gguf_asym_dq,
+ quant_tensor_gguf_sym_dq,
+ search_gguf_scale_min_asym,
+ search_gguf_scale_min_sym,
+)
+from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.logger import logger
from auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
@@ -280,6 +286,278 @@ class AutoSchemeWrapperLinear(WrapperLinear):
return qdq_w, 1.0, None
+class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
+ """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""
+
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ enable_torch_compile=True,
+ **kwargs,
+ ):
+ """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ enable_torch_compile=enable_torch_compile,
+ **kwargs,
+ )
+ self.act_score = 0.0
+ self.avg_act_score = 0.0
+ self.act_cnt = 0.0
+ self.weight_score = 0.0
+ self.mix_score = 0.0
+ self.super_qdq_func = super()._qdq_weight
+ self.act_qdq_func = super()._qdq_act
+ self.max_act_value = 0
+ self.need_weight_grad = need_weight_grad
+ self.grad_mode = False
+ if self.need_weight_grad:
+ self.orig_layer.weight.requires_grad = True
+ self.weight_search_quant_func, _ = get_quant_func(
+ orig_layer.data_type,
+ orig_layer.bits,
+ orig_layer.sym,
+ disable_opt_rtn=False,
+ group_size=orig_layer.group_size,
+ iters=0,
+ )
+ self._custom_score_forward = False
+ self.post_init_qdqw(device)
+
+ @torch.no_grad()
+ def post_init_qdqw(self, device):
+ """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,
+ registering a backward hook on it to accumulate ``weight_score``.
+ """
+ qdq_w, _, _ = self.weight_search_quant_func(
+ self.orig_layer.weight.to(device),
+ bits=self.orig_layer.bits,
+ group_size=self.orig_layer.group_size,
+ v=torch.tensor(0, device=device),
+ min_scale=torch.tensor(1.0, device=device),
+ max_scale=torch.tensor(1.0, device=device),
+ scale_dtype=self.orig_layer.scale_dtype,
+ data_type=self.data_type,
+ q_scale_thresh=self.q_scale_thresh,
+ imatrix=self.orig_layer.imatrix.to(device) if hasattr(self.orig_layer, "imatrix") else None,
+ global_scale=getattr(self, "weight_global_scale", None),
+ )
+
+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
+
+ def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
+ self.mix_score = self.weight_score + self.act_score
+ return None
+
+ self.qdq_w.requires_grad_(True)
+ self.orig_layer.weight.requires_grad_(False)
+
+ self.qdq_w.register_hook(save_grad)
+
+ def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):
+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that
+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.
+ """
+ if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:
+ return x, 1.0, None
+
+ qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)
+ if self.grad_mode:
+ with torch.no_grad():
+ max_act_value = torch.abs(x).max()
+ self.max_act_value = max_act_value
+ if max_act_value != 0:
+ self.act_cnt += 1
+ x_diff = (x - qdq_x).to("cpu")
+
+ def save_grad(grad):
+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""
+ if max_act_value == 0:
+ if torch.abs(grad).max() != 0:
+ raise ValueError
+ """
+ this ut will cause NAN issue sometimes, need to investigate
+ @multi_card
+ def test_multi_card(self):
+ model_name = "/models/Qwen3-8B"
+ """
+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():
+ self.act_cnt -= 1
+ return None
+
+ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()
+ self.mix_score = self.weight_score + self.act_score
+ return None
+
+ if qdq_x.requires_grad:
+ qdq_x.register_hook(save_grad)
+ return qdq_x, scale, zp
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
+ return self.qdq_w, 1.0, None
+
+
+class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
+ """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""
+
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ **kwargs,
+ ):
+ """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ need_weight_grad,
+ **kwargs,
+ )
+ self._custom_score_forward = False
+ self.post_init_qdqw(device)
+
+ @torch.no_grad()
+ def post_init_qdqw(self, device):
+ """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,
+ registering a backward hook on it to accumulate ``weight_score``.
+ """
+ qdq_w, scale, zp = self.super_qdq_func(
+ torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)
+ )
+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
+
+ def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
+ # TODO strange, grad could be in CPU
+ self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()
+ self.mix_score = self.weight_score + self.act_score
+ return None
+
+ self.qdq_w.requires_grad_(True)
+ self.orig_layer.weight.requires_grad_(False)
+ self.qdq_w.register_hook(save_grad)
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
+ return self.qdq_w, 1.0, None
+
+
+class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
+ """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant
+ search (``_init_scale``).
+ """
+
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ enable_torch_compile=True,
+ **kwargs,
+ ):
+ """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to
+ build ``qdq_w``.
+ """
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ **kwargs,
+ )
+ self._custom_score_forward = False
+ self.post_init_qdqw(device)
+
+ @torch.no_grad()
+ def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher
+ """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer
+ ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.
+ """
+ qdq_w = self._init_scale(device).detach()
+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
+
+ def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
+ self.mix_score = self.weight_score + self.act_score
+ return None
+
+ self.qdq_w.requires_grad_(True)
+ self.orig_layer.weight.requires_grad_(False)
+
+ self.qdq_w.register_hook(save_grad)
+
+ @torch.no_grad()
+ def _init_scale(self, device):
+ """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in
+ [2,3,4,5,6], returned in the original weight dtype.
+ """
+ tensor = self.orig_layer.weight.data.to(device)
+ bits = self.orig_layer.bits
+ scale_dtype = self.orig_layer.scale_dtype
+ imatrix = self.orig_layer.imatrix.to(tensor.device)
+ orig_dtype = tensor.dtype
+ if self.orig_layer.bits in [2, 4, 5]:
+ group_size = 16 if bits == 2 else 32
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)
+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+
+ qdq_w, _, _ = quant_tensor_gguf_asym_dq(
+ tensor=tensor,
+ bits=bits,
+ scale_dtype=scale_dtype,
+ imatrix=imatrix,
+ scale=scale,
+ wmin=wmin,
+ d_scale=d_scale,
+ d_wmin=d_wmin,
+ )
+ elif bits in [3, 6]:
+ qdq_w, _, _ = quant_tensor_gguf_sym_dq(
+ tensor=tensor,
+ bits=bits,
+ scale_dtype=scale_dtype,
+ imatrix=imatrix,
+ split_num=1,
+ )
+ else:
+ raise ValueError("bits must be in [2,3,4,5,6]")
+ return qdq_w.to(orig_dtype)
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
+ return self.qdq_w, 1.0, None
+
+
def register_imatrix_hook(model):
"""Registers hooks to accumulate activation squared norms into `imatrix`."""
@@ -536,13 +814,7 @@ def _replay_retain_graph(block_module) -> bool:
"""
for _, module in block_module.named_modules():
data_type = getattr(module, "data_type", None)
- if not isinstance(data_type, str):
- continue
- try:
- canonical_id = canonical_data_type(data_type)
- except LookupError:
- continue
- if canonical_id.startswith("mx_"):
+ if isinstance(data_type, str) and data_type.startswith("mx"):
return True
return False
@@ -952,6 +1224,15 @@ def get_score_for_scheme(
if hasattr(m, "bias") and m.bias is not None:
m.bias.requires_grad = False
+ has_imatrix = False
+ for name in quant_layer_names:
+ if name in fixed_layer_scheme.keys():
+ continue
+ m = get_module(model, name)
+ if hasattr(m, "imatrix") and m.imatrix is not None:
+ has_imatrix = True
+ break
+
for name in quant_layer_names:
if offload_context is not None:
offload_context.ensure_loaded(model, name)
@@ -972,6 +1253,14 @@ def get_score_for_scheme(
m.scale_dtype = torch.bfloat16
WrapperLayer = AutoSchemeWrapperLinear
+ # if has_imatrix: # no better result
+ # WrapperLayer = AutoSchemeWrapperLinearIMatrix
+ if hasattr(m, "super_group_size") and m.super_group_size is not None:
+ if has_imatrix:
+ WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix
+ else:
+ WrapperLayer = AutoSchemeWrapperLinearForGGUFK
+
with torch.no_grad():
if low_gpu_mem_usage:
device = m.tuning_device if hasattr(m, "tuning_device") else major_device
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 7beb61ccf..4e834ba94 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1300,34 +1300,8 @@ class BaseOrchestrator(object):
return get_torch_compile_off_reason(model)
def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:
- """Apply torch.compile disabling rules for the current compressor state.
-
- This is intentionally kept beside the compressor state it reads. The
- rules are not reusable policy: every input comes from this instance,
- and preserving that context makes the precedence easy to audit.
- """
- from auto_round.algorithms.quantization.rtn.config import RTNConfig
-
- _, is_valid_act_static = self._get_torch_compile_guard_state()
- reason = None
- if enable_torch_compile:
- if is_valid_act_static:
- reason = "activation is static"
- else:
- reason = self._torch_compile_unsupported_arch_reason()
- user_specified = getattr(self, "_torch_compile_user_specified", False)
- is_auto_scheme = getattr(self, "is_auto_scheme", False) or isinstance(
- getattr(self, "scheme", None), AutoScheme
- )
- if reason is None and not user_specified and not is_auto_scheme:
- if isinstance(self.quantize_config, RTNConfig):
- reason = "RTN/OPT-RTN quantizes each layer in a single pass"
- else:
- iters = getattr(self.quantize_config, "iters", None)
- if iters is not None and iters < MIN_ITERS_FOR_TORCH_COMPILE:
- reason = f"`iters`={iters} is below {MIN_ITERS_FOR_TORCH_COMPILE}"
-
- self.enable_torch_compile = enable_torch_compile and reason is None
+ """Apply torch.compile disabling rules for the current compressor state."""
+ self.enable_torch_compile = enable_torch_compile
# Why compilation ended up off, used by ``_log_torch_compile_state``. When the
# incoming value is already False, keep the reason recorded by the earlier
# precheck pass instead of dropping it.
@@ -1339,9 +1313,33 @@ class BaseOrchestrator(object):
or getattr(self, "_torch_compile_default_off_reason", None)
)
)
- if reason is not None:
- self._torch_compile_off_reason = reason
- logger.warning_once("reset enable_torch_compile to `False` as %s", reason)
+ _, is_valid_act_static = self._get_torch_compile_guard_state()
+
+ # On HPU, we rely on torch.compile to speed up the model execution.
+ if self.enable_torch_compile and is_valid_act_static:
+ self.enable_torch_compile = False
+ self._torch_compile_off_reason = "activation is static"
+ logger.warning_once("reset enable_torch_compile to `False` as activation is static")
+
+ # Architecture-level hard block (DeepSeek / GLM-5.3-Flash DSA families). These
+ # hit dynamo's recompile_limit and cannot be overridden by an explicit
+ # ``enable_torch_compile=True``.
+ if self.enable_torch_compile:
+ arch_reason = self._torch_compile_unsupported_arch_reason()
+ if arch_reason is not None:
+ self.enable_torch_compile = False
+ self._torch_compile_off_reason = arch_reason
+ logger.warning_once("reset enable_torch_compile to `False` as %s", arch_reason)
+
+ if self.enable_torch_compile:
+ disabled_reason = self._torch_compile_disabled_reason()
+ if disabled_reason is not None:
+ self.enable_torch_compile = False
+ self._torch_compile_off_reason = disabled_reason
+ logger.warning_once(
+ "reset enable_torch_compile to `False` as %s, " "so compilation cost would outweigh its benefit",
+ disabled_reason,
+ )
def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:
"""Apply early torch.compile adjustments before scheme resolution.
diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.py
index d790485fb..48df479ed 100644
--- a/auto_round/compressors/layer_config_resolver.py
+++ b/auto_round/compressors/layer_config_resolver.py
@@ -38,31 +38,6 @@ from auto_round.utils import (
from auto_round.utils.model import get_module
-def normalize_layer_config_entry(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
- """Resolve one public layer override through the existing preset machinery."""
- from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
-
- scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
- if isinstance(item, str):
- config = asdict(preset_name_to_scheme(item.upper()))
- elif isinstance(item, QuantizationScheme):
- config = asdict(item)
- elif isinstance(item, dict):
- item = {k: v for k, v in item.items() if k != "in_blocks"}
- scheme_name = item.pop("scheme", None)
- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
- invalid = set(item) - set(scheme_keys + ("fixed_by_user",))
- if invalid:
- raise ValueError(f"Invalid keys {invalid} in layer_config for '{layer_name}'. Allowed keys: {scheme_keys}")
- config.update(item)
- else:
- raise TypeError(
- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
- "Expected str, dict, or QuantizationScheme."
- )
- return {key: value for key, value in config.items() if value is not None}
-
-
def apply_layer_config_special_cases(
layer_config,
model,
@@ -195,6 +170,34 @@ def _resolve_layer_config_presets(
bits-inference, and default-filling."""
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
+ """Convert config entry into dict and validate keys."""
+ if isinstance(item, str):
+ config = asdict(preset_name_to_scheme(item.upper()))
+ elif isinstance(item, QuantizationScheme):
+ config = asdict(item)
+ elif isinstance(item, dict):
+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
+ # silently drop it before validation.
+ item = {k: v for k, v in item.items() if k != "in_blocks"}
+ scheme_name = item.pop("scheme", None)
+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
+ if invalid:
+ raise ValueError(
+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
+ )
+ config.update(item)
+ else:
+ raise TypeError(
+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
+ f"Expected str, dict, or QuantizationScheme."
+ )
+ # Clean up
+ config = {k: v for k, v in config.items() if v is not None}
+ config["fixed_by_user"] = True
+ return config
+
extra_scheme_keys = ("scale_dtype",)
scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
layer_config = copy.deepcopy(layer_config) or {}
@@ -215,7 +218,7 @@ def _resolve_layer_config_presets(
}
# 2. normalize
- layer_config = {k: {**normalize_layer_config_entry(v, k), "fixed_by_user": True} for k, v in layer_config.items()}
+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
# entries whose sym was set by the user (dict key or a QuantizationScheme
# object) rather than inherited from the default scheme below
explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.py
index 58bd9076a..58c746470 100644
--- a/auto_round/data_type/__init__.py
+++ b/auto_round/data_type/__init__.py
@@ -15,7 +15,7 @@
import auto_round.data_type.int
import auto_round.data_type.mxfp
import auto_round.data_type.fp8
-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
import auto_round.data_type.w4fp8
from auto_round.data_type.utils import (
get_quant_func,
@@ -24,8 +24,3 @@ from auto_round.data_type.utils import (
)
import auto_round.data_type.nvfp
import auto_round.data_type.gguf
-from auto_round.data_type.base import (
- DataTypeQuantizer,
- canonical_data_type,
- create_quantizer,
-)
diff --git a/auto_round/data_type/base.py b/auto_round/data_type/base.py
deleted file mode 100644
index c34d336fb..000000000
--- a/auto_round/data_type/base.py
+++ /dev/null
@@ -1,313 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-"""Small public boundary between quantization algorithms and data types.
-
-Algorithms describe *what* a layer needs quantized. This module resolves the
-requested datatype and delegates all datatype-specific work to that datatype's
-class. It deliberately contains no quantization math.
-"""
-
-from collections.abc import Mapping
-from dataclasses import dataclass, replace
-from types import SimpleNamespace
-from typing import Any, Callable
-
-import torch
-
-# Low-level tensor primitives remain functions. Quantizer classes below own
-# their lifecycle and are the only interface used by algorithms and wrappers.
-QUANT_FUNC_WITH_DTYPE: dict[str, Callable] = {}
-_QUANTIZER_CLASSES: dict[str, type] = {}
-_CANONICAL_DATA_TYPES: dict[str, str] = {}
-
-
-def _normalize_data_type(data_type: str) -> str:
- """Normalize spelling differences in datatype names for lookup."""
- return data_type.lower().replace("-", "").replace("_", "")
-
-
-def register_dtype(names):
- """Register a low-level tensor quantization primitive by datatype name."""
-
- def register(function):
- for name in (names,) if isinstance(names, str) else names:
- QUANT_FUNC_WITH_DTYPE[name] = function
- return function
-
- return register
-
-
-def register_quantizer(canonical_id: str, *, aliases: tuple[str, ...] = ()):
- """Register a datatype class under its canonical name and aliases.
-
- Datatype modules use this decorator at import time. The mapping exists
- solely to turn user-facing aliases such as ``"int"`` into the class that
- owns their implementation.
- """
-
- def register(quantizer_class):
- names = (canonical_id, *aliases)
- normalized = tuple(dict.fromkeys(_normalize_data_type(name) for name in names))
- duplicate = next((name for name in normalized if name in _QUANTIZER_CLASSES), None)
- if duplicate is not None:
- raise ValueError(f"Datatype name {duplicate!r} is already registered")
- for name in normalized:
- _QUANTIZER_CLASSES[name] = quantizer_class
- _CANONICAL_DATA_TYPES[name] = canonical_id
- return quantizer_class
-
- return register
-
-
-def canonical_data_type(data_type: str) -> str:
- """Return an alias's canonical name for datatype-dependent policy.
-
- This is intentionally separate from construction: callers such as loss
- policy and block preparation need to classify a datatype without creating
- a quantizer or allocating tensors.
- """
- try:
- return _CANONICAL_DATA_TYPES[_normalize_data_type(data_type)]
- except KeyError as error:
- raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error
-
-
-def _quantizer_class(data_type: str) -> type:
- """Look up the class that owns a datatype implementation."""
- try:
- return _QUANTIZER_CLASSES[_normalize_data_type(data_type)]
- except KeyError as error:
- raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error
-
-
-def _create_weight_quantizer(data_type: str, spec: "WeightQuantizationSpec"):
- """Create the implementation owned by ``data_type`` for one weight request."""
- canonical = canonical_data_type(data_type)
- quantizer_class = _quantizer_class(data_type)
- try:
- return quantizer_class.from_spec(spec, canonical)
- except AttributeError as error:
- raise LookupError(f"Datatype {data_type!r} supports activation quantization only") from error
-
-
-def _create_activation_quantizer(data_type: str, spec: "ActivationQuantizationSpec"):
- """Create the activation implementation owned by ``data_type``."""
- canonical = canonical_data_type(data_type)
- try:
- return _quantizer_class(data_type).create_activation(replace(spec, data_type=canonical))
- except AttributeError as error:
- raise LookupError(f"Datatype {data_type!r} has no activation quantizer") from error
-
-
-def prepare_data_type_block(data_type: str, block, layer_runtimes) -> None:
- """Run the optional block-level setup owned by one datatype class."""
- quantizer_class = _QUANTIZER_CLASSES.get(_normalize_data_type(data_type))
- prepare = getattr(quantizer_class, "prepare_block", None)
- if prepare is not None:
- prepare(block, layer_runtimes)
-
-
-@dataclass(frozen=True)
-class WeightQuantizationSpec:
- """Static properties needed to quantize one layer's weights."""
-
- data_type: str
- bits: int
- group_size: int | tuple[int, int]
- sym: bool
- scale_dtype: torch.dtype
- q_scale_thresh: float = 1e-5
- super_bits: int | None = None
- super_group_size: int | None = None
- global_scale: torch.Tensor | None = None
- clip_min: torch.Tensor | None = None
- clip_max: torch.Tensor | None = None
-
-
-@dataclass(frozen=True)
-class ActivationQuantizationSpec:
- """Static properties needed to quantize one layer's activations."""
-
- data_type: str
- bits: int
- group_size: int | tuple[int, int]
- sym: bool
- scale_dtype: torch.dtype
- dynamic: bool
- q_scale_thresh: float = 1e-5
-
-
-@dataclass(frozen=True)
-class WeightQuantizationResult:
- """Weight QDQ output plus the tensors required by export and inference."""
-
- weight: torch.Tensor
- scale: torch.Tensor | dict[str, torch.Tensor] | None = None
- zero_point: torch.Tensor | dict[str, torch.Tensor] | int | float | None = None
- metadata: object | None = None
- # Conv1D weights are transposed before export. Keep the original output
- # dimension so per-output-channel tensors retain their intended layout.
- logical_rows: int | None = None
-
-
-class DataTypeQuantizer:
- """Run a datatype implementation through initialize, QDQ, and write-back.
-
- This is the only lifecycle wrapper shared by all data types. Datatype
- classes keep their own state and numerical code; this class only manages
- the sequence and the trainable parameter dictionary.
- """
-
- def __init__(self, implementation: Any, mode: str, tune_rounding: bool, tune_minmax: bool, apply_result):
- self._implementation = implementation
- self._mode = mode
- self._tune_rounding = tune_rounding
- self._tune_minmax = tune_minmax
- self._apply_result = apply_result
- self._state = None
- self.parameters: dict[str, torch.Tensor] = {}
-
- def initialize(self, weight: torch.Tensor, *, imatrix=None) -> None:
- """Initialize datatype state and expose its trainable tensors."""
- self._state = self._implementation.create_state(
- weight,
- imatrix=imatrix,
- mode=self._mode,
- tune_rounding=self._tune_rounding,
- tune_minmax=self._tune_minmax,
- )
- tunables = self._state.get("tunables", self._state) if isinstance(self._state, dict) else self._state.tunables
- self.parameters = dict(tunables)
-
- def refresh(self, weight: torch.Tensor, *, imatrix=None) -> None:
- """Rebuild fixed datatype state while preserving existing trainable tensors."""
- previous_parameters = self.parameters
- self.initialize(weight, imatrix=imatrix)
- if isinstance(self._state, dict):
- self._state["tunables"] = previous_parameters
- else:
- self._state = replace(self._state, tunables=previous_parameters)
- self.parameters = previous_parameters
-
- def quantize(self, weight: torch.Tensor, **parameters) -> torch.Tensor:
- """Return QDQ weights for the current trainable parameter values."""
- return self.qdq(weight, **parameters).weight
-
- def qdq(self, weight: torch.Tensor, *, materialize=False, **parameters) -> WeightQuantizationResult:
- """Run datatype QDQ and optionally include tensors needed for export."""
- self._ensure_initialized()
- return self._implementation.qdq(
- weight,
- self._state,
- tunables={**self.parameters, **parameters},
- materialize=materialize,
- )
-
- def apply_result(self, layer, result: WeightQuantizationResult, *, transpose=False) -> None:
- """Store an already computed QDQ result on its source layer."""
- logical_rows = result.weight.shape[0]
- if transpose:
- result = WeightQuantizationResult(
- result.weight.t(), result.scale, result.zero_point, result.metadata, logical_rows
- )
- if result.weight.device != layer.weight.device:
- result = WeightQuantizationResult(
- result.weight.to(layer.weight.device),
- result.scale,
- result.zero_point,
- result.metadata,
- result.logical_rows,
- )
- self._apply_result(layer, result)
-
- def write_back(self, layer, weight: torch.Tensor, *, transpose=False, **parameters) -> None:
- """Materialize QDQ payload and store it on an exportable layer."""
- result = self.qdq(weight, materialize=True, **parameters)
- self.apply_result(layer, result, transpose=transpose)
-
- def _ensure_initialized(self) -> None:
- """Reject use before a datatype has created its per-layer state."""
- if self._state is None:
- raise RuntimeError("Call initialize() before quantize() or write_back().")
-
-
-def _weight_spec(layer) -> WeightQuantizationSpec:
- """Extract the weight request from a resolved quantized layer."""
- scale_dtype = getattr(layer, "scale_dtype", torch.float32)
- return WeightQuantizationSpec(
- data_type=layer.data_type,
- bits=layer.bits,
- group_size=layer.group_size,
- sym=layer.sym,
- scale_dtype=scale_dtype,
- q_scale_thresh=getattr(layer, "q_scale_thresh", 1e-5),
- super_bits=getattr(layer, "super_bits", None),
- super_group_size=getattr(layer, "super_group_size", None),
- global_scale=getattr(layer, "weight_global_scale", None),
- clip_min=getattr(layer, "awq_clip_min", None),
- clip_max=getattr(layer, "awq_clip_max", None),
- )
-
-
-def _activation_spec(layer) -> ActivationQuantizationSpec:
- """Extract the activation request from a resolved quantized layer."""
- return ActivationQuantizationSpec(
- data_type=getattr(layer, "act_data_type", "int_sym"),
- bits=getattr(layer, "act_bits", 16),
- group_size=getattr(layer, "act_group_size", -1),
- sym=getattr(layer, "act_sym", True),
- scale_dtype=getattr(layer, "scale_dtype", torch.float32),
- dynamic=getattr(layer, "act_dynamic", True),
- )
-
-
-def create_quantizer(layer, *, iters=None, disable_opt_rtn=False, tune_rounding=False, tune_minmax=False):
- """Create the datatype-owned weight quantizer for a resolved layer."""
- if isinstance(layer, Mapping):
- layer = SimpleNamespace(**{"q_scale_thresh": 1e-5, **layer})
- spec = _weight_spec(layer)
- iters = getattr(layer, "iters", 0) if iters is None else iters
- mode = "tuned" if iters > 0 else "rtn" if disable_opt_rtn else "optimized_rtn"
- implementation = _create_weight_quantizer(spec.data_type, spec)
- return DataTypeQuantizer(implementation, mode, tune_rounding, tune_minmax, implementation.apply_result)
-
-
-def cache_activation_quantizer(layer):
- """Create and cache a layer's activation quantizer when activation is quantized."""
- if getattr(layer, "act_bits", 16) > 8:
- return None
- if hasattr(layer, "_ar_activation_quantizer"):
- return layer._ar_activation_quantizer
- spec = _activation_spec(layer)
- quantizer = _create_activation_quantizer(spec.data_type, spec)
- layer._ar_activation_quantizer = quantizer
- return quantizer
-
-
-def activation_quantizer_for_layer(layer, *, scale_dtype=None):
- """Create the activation quantizer described by a layer's configuration."""
- spec = _activation_spec(layer)
- if scale_dtype is not None:
- from dataclasses import replace
-
- spec = replace(spec, scale_dtype=scale_dtype)
- return _create_activation_quantizer(spec.data_type, spec)
-
-
-def quantize_activation(tensor, config):
- """Apply dynamic activation quantization using a simple configuration mapping."""
- spec = ActivationQuantizationSpec(
- data_type=config["data_type"],
- bits=config["bits"],
- group_size=config["group_size"],
- sym=config["sym"],
- scale_dtype=tensor.dtype,
- dynamic=True,
- )
- return _create_activation_quantizer(spec.data_type, spec).qdq(tensor)
diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.py
index 708390e63..11f4594c2 100644
--- a/auto_round/data_type/fp8.py
+++ b/auto_round/data_type/fp8.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.base import register_dtype, register_quantizer
+from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import (
float8_e4m3fn_ste,
float8_e5m2_ste,
@@ -366,136 +366,3 @@ if is_gaudi2():
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
qdq_res = qdq_res.to(orig_dtype)
return qdq_res, scale, None
-
-
-class _FP8WeightQuantizer:
- """Select the requested FP8 format and materialize its scale layout."""
-
- def __init__(self, spec, primitive=None, scale_layout=None, data_type=None):
- self.spec = spec
- data_type = (data_type or spec.data_type).lower()
- if primitive is None:
- if data_type in ("fp8_e5m2",):
- primitive, scale_layout = quant_fp8_e5m2, "row"
- elif data_type in ("fp8_unit_scale",):
- primitive, scale_layout = quant_fp8_unit_scale, "scalar"
- elif data_type in ("fp8_e5m2_unit_scale",):
- primitive, scale_layout = quant_fp8_e5m2_unit_scale, "scalar"
- elif data_type == "fp8_gaudi3_sym":
- primitive, scale_layout = quant_fp8_sym_gaudi3, "row"
- elif isinstance(spec.group_size, tuple):
- primitive, scale_layout = quant_block_fp_sym, "block"
- else:
- primitive, scale_layout = quant_fp8_sym, "row"
- self.primitive = primitive
- self.scale_layout = scale_layout
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the FP8 weight quantizer selected by the requested format."""
- return cls(spec, data_type=canonical)
-
- @staticmethod
- def create_activation(spec):
- """Create the FP8 activation quantizer for the same format."""
- return _create_fp8_activation(spec)
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- reduction_dims = 2 if self.scale_layout == "block" else 1
- tunable_shape = grouped.shape[:-reduction_dims]
- tunables = {}
- if tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if tune_minmax:
- tunables["max_scale"] = torch.nn.Parameter(
- torch.ones(tunable_shape, device=weight.device, dtype=torch.float32)
- )
- return tunables
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- quantized, scale, zero_point = self.primitive(
- weight,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- v=tunables.get("value", 0),
- max_scale=tunables.get("max_scale", 1.0),
- )
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(
- quantized,
- scale if materialize else None,
- zero_point if materialize else None,
- self.scale_layout if materialize else None,
- )
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("FP8 weight result was not materialized")
- scale = result.scale
- rows = result.logical_rows or result.weight.shape[0]
- if result.metadata == "row" and scale.numel() % rows == 0:
- scale = scale.reshape(rows, -1)
- module.weight.data.copy_(result.weight)
- module.scale = scale.cpu()
- module.zp = result.zero_point
-
-
-class _FP8ActivationQuantizer:
- """Apply the matching FP8 primitive to dynamic or calibrated activations."""
-
- def __init__(self, spec, data_type, primitive, scale_layout):
- self.spec = spec
- self.data_type = data_type
- self.primitive = primitive
- self.scale_layout = scale_layout
- self.bits = spec.bits
- self.group_size = spec.group_size
- self.requires_calibration = not spec.dynamic
-
- def observe(self, activation, current):
- if self.scale_layout == "tensor":
- maximum = activation.detach().abs().max()
- else:
- grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)
- dims = (-2, -1) if self.scale_layout == "block" else -1
- maximum = grouped.detach().abs().amax(dim=dims)
- return maximum if current is None else torch.maximum(maximum.to(current), current)
-
- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- if self.requires_calibration and observed_max is None:
- raise ValueError(f"{self.data_type} activation requires observed_max")
- return self.primitive(
- activation,
- bits=self.bits,
- group_size=self.group_size,
- tensor_max=observed_max if self.requires_calibration else None,
- max_scale=max_scale,
- )
-
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- quantized, _, _ = self.qdq_with_scale(
- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
- )
- return quantized
-
-
-def _create_fp8_activation(spec):
- """Build an activation quantizer using the same format choice as weights."""
- if spec.data_type == "fp8_gaudi3_sym":
- return _FP8ActivationQuantizer(spec, spec.data_type, quant_fp8_sym_gaudi3, "tensor")
- weight_quantizer = _FP8WeightQuantizer(spec)
- return _FP8ActivationQuantizer(spec, spec.data_type, weight_quantizer.primitive, weight_quantizer.scale_layout)
-
-
-register_quantizer(
- "fp8_sym",
- aliases=("fp", "float", "fp8", "fp8_e4m3", "block_fp8_sym", "block_fp8", "block_fp8_e4m3"),
-)(_FP8WeightQuantizer)
-register_quantizer("fp8_e5m2")(_FP8WeightQuantizer)
-register_quantizer("fp8_unit_scale")(_FP8WeightQuantizer)
-register_quantizer("fp8_e5m2_unit_scale")(_FP8WeightQuantizer)
-if is_gaudi2():
- register_quantizer("fp8_gaudi3_sym")(_FP8WeightQuantizer)
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index dea59a70e..2b4de2b7e 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -11,14 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from collections.abc import Mapping
-from dataclasses import dataclass
from typing import Any, Callable, Union
import torch
-from auto_round.data_type.base import register_dtype, register_quantizer
+from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
from auto_round.export.export_to_gguf.packing import (
_LARGE_TENSOR_BYTES_THRESHOLD,
make_q3_quants,
@@ -26,15 +25,6 @@ from auto_round.export.export_to_gguf.packing import (
make_qx_quants_chunk,
)
from auto_round.logger import logger
-
-
-def get_gguf_quant_constants():
- """Return GGUF block-layout constants without exposing export internals to algorithms."""
- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
-
- return GGML_QUANT_SIZES, QK_K
-
-
from auto_round.utils import get_reciprocal
from auto_round.utils.device import clear_memory
@@ -496,18 +486,9 @@ def _imatrix_handle_zero(
@torch.inference_mode()
-def search_gguf_scale_min_asym(
- tensor,
- bits=4,
- scale_dtype=torch.float16,
- imatrix=None,
- split_num=1,
- v=0,
- super_bits=None,
- super_group_size=None,
-):
- super_bits = super_bits if super_bits is not None else 4 if bits == 2 else 6
- super_group_size = super_group_size if super_group_size is not None else 16 if bits == 2 else 8
+def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1, v=0):
+ super_bits = 4 if bits == 2 else 6
+ super_group_size = 16 if bits == 2 else 8
quant_weights = None
if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
@@ -623,9 +604,6 @@ def search_gguf_scale_min_asym(
def quant_tensor_gguf_asym_dq(
tensor: torch.Tensor,
bits: int = 4,
- group_size=None,
- super_bits=None,
- super_group_size=None,
v=0,
scale_dtype=torch.float16,
imatrix=None,
@@ -656,7 +634,7 @@ def quant_tensor_gguf_asym_dq(
raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")
orig_dtype = tensor.dtype
maxq = int(2.0**bits) - 1
- group_size = group_size if group_size is not None else 16 if bits == 2 else 32
+ group_size = 16 if bits == 2 else 32
# The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),
# unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large
# tensors. For very large single tensors (e.g. huge vocab embed/output projections),
@@ -687,8 +665,6 @@ def quant_tensor_gguf_asym_dq(
imatrix,
split_num=split_num,
v=v,
- super_bits=super_bits,
- super_group_size=super_group_size,
)
scale = scale.clone()
wmin = wmin.clone()
@@ -927,9 +903,6 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0
def quant_tensor_gguf_sym_dq(
tensor,
bits=3,
- group_size=16,
- super_bits=None,
- super_group_size=16,
v=0,
imatrix=None,
scale=None,
@@ -957,15 +930,21 @@ def quant_tensor_gguf_sym_dq(
Quantized and de-quantized tensor, scale, zero-point
"""
+ from auto_round.export.export_to_gguf.config import K_SCALE_SIZE
+
if bits not in [3, 6]:
raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")
maxq = int(2.0 ** (bits - 1))
+ group_size = 16
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
orig_dtype = tensor.dtype
- super_bits = super_bits if super_bits is not None else 6 if bits == 3 else 8
+ super_bits = 6 if bits == 3 else 8
+ super_group_size = 16
+ ggml_type = f"q{bits}_k"
+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]
tensor = tensor.to(torch.float32)
- n_blocks = tensor.nelement() // (super_group_size * group_size)
+ n_blocks = tensor.nelement() // block_size
# Chunk the candidate-scale search for very large single tensors (e.g. huge vocab
# embed/output projections). This search allocates several full-size float32 buffers
# at once; unchunked, that can need tens of GB for a single tensor and OOM even though
@@ -976,7 +955,8 @@ def quant_tensor_gguf_sym_dq(
# ordinary attention/FFN weights never needed it and would just get slower for no benefit.
if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:
split_num = 16
- tensor = tensor.reshape(n_blocks, super_group_size, group_size)
+ # (nb, 16, 16)
+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
if iter is None:
iter = 0
use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1
@@ -1000,161 +980,3 @@ def quant_tensor_gguf_sym_dq(
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
return tensor, {"scale": scale, "d_scale": d_scale}, maxq
-
-
-@dataclass
-class _GGUFState:
- """Persistent GGUF RTN scales and double-quantization payload between QDQ calls."""
-
- tunables: Mapping[str, torch.Tensor]
- tensor_min: torch.Tensor | None
- tensor_max: torch.Tensor | None
- imatrix: torch.Tensor | float | int | None
- scale: torch.Tensor | None = None
- wmin: torch.Tensor | None = None
- d_scale: torch.Tensor | None = None
- d_wmin: torch.Tensor | None = None
- prev_scale: torch.Tensor | None = None
- prev_wmin: torch.Tensor | None = None
- prev_d_scale: torch.Tensor | None = None
- prev_d_wmin: torch.Tensor | None = None
- iteration: int = 0
-
-
-class _GGUFWeightQuantizer:
- """Own GGUF double-quant weight QDQ and its export payload."""
-
- def __init__(self, spec, family="plain", kind=None):
- self.spec = spec
- self.family = family
- self.kind = kind or (
- "sym" if spec.data_type == "int_sym_dq" else "float_zp" if spec.data_type == "int_asym_float_zp" else "asym"
- )
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the GGUF quantizer; its state selects tuned or RTN behavior."""
- kind = {"int_sym_dq": "sym", "int_asym_float_zp": "float_zp"}.get(canonical, "asym")
- return cls(spec, kind=kind)
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- self.family = "rtn" if mode in ("rtn", "optimized_rtn") else "plain"
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)
- tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)
- tunables = {}
- if self.family == "plain" and tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if self.family == "plain" and tune_minmax:
- shape = tensor_min.shape
- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
- return _GGUFState(tunables, tensor_min, tensor_max, imatrix)
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- value = tunables.get("value", 0)
- min_scale = tunables.get("min_scale", 1.0)
- max_scale = tunables.get("max_scale", 1.0)
- if isinstance(min_scale, torch.Tensor):
- min_scale.data.clamp_(0.0, 1.0)
- if isinstance(max_scale, torch.Tensor):
- max_scale.data.clamp_(0.0, 1.0)
-
- if self.kind == "float_zp" and self.family == "plain":
- quantized, scale, zero_point = quant_tensor_asym_float_zp(
- weight,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- v=value,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.spec.scale_dtype,
- tensor_min=state.tensor_min,
- tensor_max=state.tensor_max,
- q_scale_thresh=self.spec.q_scale_thresh,
- )
- elif self.kind == "float_zp":
- quantized, scale, zero_point = quant_tensor_asym_float_zp_rtn(
- weight.clone(),
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- scale_dtype=self.spec.scale_dtype,
- tensor_min=state.tensor_min,
- tensor_max=state.tensor_max.clone(),
- q_scale_thresh=self.spec.q_scale_thresh,
- )
- elif self.family == "rtn":
- primitive = quant_tensor_gguf_sym_dq if self.kind == "sym" else quant_tensor_gguf_asym_dq
- quantized, scale, zero_point = primitive(
- weight.clone(),
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- super_bits=self.spec.super_bits,
- super_group_size=self.spec.super_group_size,
- v=value,
- scale_dtype=self.spec.scale_dtype,
- imatrix=state.imatrix,
- prev_scale=state.prev_scale,
- prev_wmin=state.prev_wmin,
- prev_d_scale=state.prev_d_scale,
- prev_d_wmin=state.prev_d_wmin,
- iter=state.iteration,
- )
- state.scale = scale["scale"]
- state.d_scale = scale["d_scale"]
- state.prev_scale = state.scale
- state.prev_d_scale = state.d_scale
- if isinstance(zero_point, dict):
- state.wmin = zero_point["wmin"]
- state.d_wmin = zero_point["d_wmin"]
- state.prev_wmin = state.wmin
- state.prev_d_wmin = state.d_wmin
- state.iteration += 1
- else:
- primitive = quant_tensor_sym_dq if self.kind == "sym" else quant_tensor_asym_dq
- quantized, scale, zero_point = primitive(
- weight,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- v=value,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.spec.scale_dtype,
- tensor_min=state.tensor_min,
- tensor_max=state.tensor_max,
- q_scale_thresh=self.spec.q_scale_thresh,
- super_bits=self.spec.super_bits,
- super_group_size=self.spec.super_group_size,
- )
-
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("GGUF weight result was not materialized")
- module.weight.data.copy_(result.weight)
- rows = result.logical_rows or result.weight.shape[0]
-
- def set_values(values, primary):
- for key, value in values.items():
- name = primary if key == primary else "w_" + key
- setattr(module, name, (value.reshape(rows, -1) if key == primary else value).cpu())
-
- if isinstance(result.scale, dict):
- set_values(result.scale, "scale")
- else:
- module.scale = result.scale.reshape(rows, -1).cpu()
- if isinstance(result.zero_point, dict):
- set_values(result.zero_point, "zp")
- elif isinstance(result.zero_point, torch.Tensor):
- module.zp = result.zero_point.reshape(rows, -1).cpu()
- else:
- module.zp = result.zero_point
-
-
-register_quantizer("int_sym_dq", aliases=("rtn_int_sym_dq",))(_GGUFWeightQuantizer)
-register_quantizer("int_asym_dq", aliases=("rtn_int_asym_dq",))(_GGUFWeightQuantizer)
-register_quantizer("int_asym_float_zp")(_GGUFWeightQuantizer)
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index ab2b9b4c5..d301f0b2e 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -11,14 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from collections.abc import Mapping
-from dataclasses import dataclass
from typing import Union
import torch
from auto_round import envs
-from auto_round.data_type.base import register_dtype, register_quantizer
+from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.utils import get_reciprocal
@@ -133,7 +131,7 @@ def quant_tensor_rtn_sym(
min_scale=1.0,
max_scale=1.0,
scale_dtype=torch.float16,
- **kwargs,
+ **kwargs
):
"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community
@@ -177,7 +175,7 @@ def quant_tensor_sym(
tensor_max=None,
q_scale_thresh=1e-5,
init_scale=None,
- **kwargs,
+ **kwargs
):
"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community
@@ -252,7 +250,7 @@ def quant_tensor_asym(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs,
+ **kwargs
):
"""Quantize and de-quantize tensor asymmetrically.
@@ -312,7 +310,7 @@ def quant_tensor_sym_gptq(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs,
+ **kwargs
):
"""Quantize and de-quantize tensor asymmetrically.
@@ -378,7 +376,7 @@ def quant_tensor_asym_wo_round(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs,
+ **kwargs
):
"""Quantize and de-quantize tensor asymmetrically without rounding, this is mainly for tuning bias, norm.
@@ -425,193 +423,3 @@ def quant_tensor_asym_wo_round(
qdq_result = (scale * (q - zp)).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
return qdq_result, scale, zp
-
-
-@dataclass(frozen=True)
-class _IntState:
- """Per-layer integer ranges, optional optimized scale, and trainable values."""
-
- tunables: Mapping[str, torch.Tensor]
- tensor_min: torch.Tensor
- tensor_max: torch.Tensor
- optimized_init: torch.Tensor | None
-
-
-class _IntWeightQuantizer:
- """Own integer weight QDQ and choose tuned, RTN, or optimized RTN internally."""
-
- def __init__(self, spec, family="plain"):
- self.spec = spec
- self.family = family
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the integer weight quantizer for a resolved layer."""
- return cls(spec)
-
- @staticmethod
- def create_activation(spec):
- """Create the matching integer activation quantizer."""
- return _IntActivationQuantizer(spec)
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- if mode == "optimized_rtn" and self.spec.sym:
- self.family = "optimized"
- elif mode == "rtn" and self.spec.sym:
- self.family = "rtn"
- else:
- self.family = "plain"
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)
- tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)
- if self.spec.clip_max is not None:
- clip_max = self.spec.clip_max.reshape(-1).to(weight.device, tensor_max.dtype)
- clip_min = (
- -clip_max
- if self.spec.clip_min is None
- else self.spec.clip_min.reshape(-1).to(weight.device, tensor_min.dtype)
- )
- if clip_min.numel() == tensor_min.numel() and clip_max.numel() == tensor_max.numel():
- tensor_min = torch.maximum(tensor_min, clip_min.reshape_as(tensor_min))
- tensor_max = torch.minimum(tensor_max, clip_max.reshape_as(tensor_max))
-
- tunables = {}
- if self.family == "plain" and tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if self.family != "optimized" and tune_minmax:
- shape = tensor_min.shape
- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
-
- optimized_init = None
- if self.family == "optimized" and self.spec.sym:
- search_weight = weight
- if self.spec.clip_min is not None or self.spec.clip_max is not None:
- search_weight = torch.clamp(weight, min=self.spec.clip_min, max=self.spec.clip_max)
- _, optimized_init, _ = quant_tensor_opt_rtn_sym(
- search_weight.clone(),
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- q_scale_thresh=self.spec.q_scale_thresh,
- imatrix=imatrix,
- )
- return _IntState(tunables, tensor_min, tensor_max, optimized_init)
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- value = tunables.get("value", 0)
- min_scale = tunables.get("min_scale", 1.0)
- max_scale = tunables.get("max_scale", 1.0)
- if isinstance(min_scale, torch.Tensor):
- min_scale.data.clamp_(0.0, 1.0)
- if isinstance(max_scale, torch.Tensor):
- max_scale.data.clamp_(0.0, 1.0)
-
- kwargs = {
- "bits": self.spec.bits,
- "group_size": self.spec.group_size,
- "q_scale_thresh": self.spec.q_scale_thresh,
- }
- if self.family == "optimized" and self.spec.sym:
- quantized, scale, zero_point = quant_tensor_sym(
- weight,
- init_scale=state.optimized_init,
- scale_dtype=self.spec.scale_dtype,
- **kwargs,
- )
- elif self.family == "rtn" and self.spec.sym:
- quantized, scale, zero_point = quant_tensor_rtn_sym(
- weight,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.spec.scale_dtype,
- **kwargs,
- )
- else:
- primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym
- quantized, scale, zero_point = primitive(
- weight,
- v=value,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.spec.scale_dtype,
- tensor_min=state.tensor_min,
- tensor_max=state.tensor_max,
- **kwargs,
- )
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("INT weight result was not materialized")
- module.weight.data.copy_(result.weight)
- rows = result.logical_rows or result.weight.shape[0]
- module.scale = result.scale.reshape(rows, -1).cpu()
- module.zp = (
- result.zero_point.reshape(rows, -1).cpu()
- if isinstance(result.zero_point, torch.Tensor)
- else result.zero_point
- )
-
-
-class _IntActivationQuantizer:
- """Quantize integer activations, optionally using a calibrated maximum."""
-
- def __init__(self, spec):
- self.spec = spec
- self.requires_calibration = not spec.dynamic
-
- def observe(self, activation, current):
- grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)
- maximum = grouped.abs().amax(dim=-1)
- return maximum if current is None else torch.maximum(maximum.to(current), current)
-
- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- if self.requires_calibration and observed_max is None:
- raise ValueError(f"{self.spec.data_type} activation requires observed_max")
- primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym
- kwargs = {}
- if observed_max is not None:
- kwargs.update(tensor_min=-observed_max, tensor_max=observed_max)
- return primitive(
- activation,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.spec.scale_dtype,
- q_scale_thresh=self.spec.q_scale_thresh,
- **kwargs,
- )
-
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- quantized, _, _ = self.qdq_with_scale(
- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
- )
- return quantized
-
-
-register_quantizer(
- "int_sym",
- aliases=(
- "int",
- "int4",
- "int4_sym",
- "int8",
- "int8_sym",
- "rtn_int",
- "rtn_int4",
- "rtn_int_sym",
- "rtn_int4_sym",
- "opt_rtn_int",
- "opt_rtn_int4",
- "opt_rtn_int_sym",
- "opt_rtn_int4_sym",
- ),
-)(_IntWeightQuantizer)
-register_quantizer(
- "int_asym",
- aliases=("int4_asym", "rtn_int_asym", "rtn_int4_asym", "opt_rtn_int_asym", "opt_rtn_int4_asym"),
-)(_IntWeightQuantizer)
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 29982f2d4..7f453066a 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE, register_dtype, register_quantizer
+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE, register_dtype
from auto_round.data_type.utils import (
ceil_ste,
floor_ste,
@@ -414,168 +414,6 @@ QUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceil
QUANT_FUNC_WITH_DTYPE["mx_fp4_rceil_v2"] = quant_mx_rceil_v2
QUANT_FUNC_WITH_DTYPE["opt_rtn_mx_fp"] = quant_mx_opt_rtn
-
-class _MXQuantizer:
- """Own MX weight QDQ and derive the concrete MX format from the request."""
-
- def __init__(self, spec, data_type=None, family="plain"):
- self.spec = spec
- self.data_type = data_type or self._data_type(spec)
- self.family = family
-
- @staticmethod
- def _data_type(spec):
- data_type = spec.data_type.lower()
- if data_type in ("mx_fp", "mx_int"):
- data_type = f"{data_type}{spec.bits}"
- if data_type not in MXFP_FORMAT_CACHE and not data_type.endswith("_rceil"):
- raise ValueError(f"Unsupported MX datatype {spec.data_type!r}")
- return data_type
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the MX quantizer selected by the requested MX format."""
- data_type = canonical or spec.data_type
- if data_type in ("mx_fp", "mx_int"):
- data_type = f"{data_type}{spec.bits}"
- return cls(spec, data_type)
-
- @classmethod
- def create_activation(cls, spec):
- """Create the dynamic MX activation quantizer for the same format."""
- return _MXActivationQuantizer(spec, cls._data_type(spec))
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- self.family = "optimized" if mode == "optimized_rtn" else "plain"
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tunables = {}
- if self.family == "plain" and tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if self.family == "plain" and tune_minmax:
- tunables["max_scale"] = torch.nn.Parameter(
- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
- )
- return {"tunables": tunables, "imatrix": imatrix}
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- kwargs = {
- "bits": self.spec.bits,
- "group_size": self.spec.group_size,
- "data_type": self.data_type,
- }
- if self.family == "optimized":
- quantized, exponent, zero_point = quant_mx_opt_rtn(weight, imatrix=state["imatrix"], **kwargs)
- elif self.data_type.endswith("_rceil"):
- quantized, exponent, zero_point = quant_mx_rceil(weight, **kwargs)
- else:
- quantized, exponent, zero_point = quant_mx(
- weight,
- v=tunables.get("value", 0),
- max_scale=tunables.get("max_scale", 1.0),
- **kwargs,
- )
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(
- quantized, exponent if materialize else None, zero_point if materialize else None
- )
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("MX weight result was not materialized")
- module.weight.data.copy_(result.weight)
- rows = result.logical_rows or result.weight.shape[0]
- module.scale = result.scale.reshape(rows, -1).cpu()
- module.zp = None
-
-
-class _MXActivationQuantizer:
- """Apply dynamic MX activation QDQ; MX formats require no calibration pass."""
-
- requires_calibration = False
-
- def __init__(self, spec, data_type):
- if not spec.dynamic:
- raise ValueError(f"MX datatype {data_type!r} supports only dynamic activation quantization")
- self.spec = spec
- self.data_type = data_type
-
- def observe(self, activation, current):
- raise RuntimeError("Dynamic MX activation quantization does not require calibration")
-
- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- primitive = quant_mx_rceil if self.data_type.endswith("_rceil") else quant_mx
- return primitive(
- activation,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- data_type=self.data_type.removesuffix("_rceil"),
- max_scale=max_scale,
- )
-
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- quantized, _, _ = self.qdq_with_scale(
- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
- )
- return quantized
-
-
-_MX_ALIASES = {
- "mx_int8": ("mxint8",),
- "mx_int4": (
- "mxint4",
- "rtn_mx_int4",
- "rtn_mx_int4_sym",
- "opt_rtn_mx_int4",
- "opt_rtn_mx_int4_sym",
- ),
- "mx_int2": ("mxint2",),
- "mx_fp8e5m2": ("mxfp8e5m2",),
- "mx_fp8": ("mxfp8",),
- "mx_fp8e4m3": ("mxfp8e4m3",),
- "mx_fp6e3m2": ("mxfp6e3m2",),
- "mx_fp6": ("mxfp6",),
- "mx_fp6e2m3": ("mxfp6e2m3",),
- "mx_fp4": (
- "mxfp4",
- "mx_fp4_sym",
- "rtn_mx_fp4",
- "rtn_mx_fp4_sym",
- "opt_rtn_mx_fp4",
- "opt_rtn_mx_fp4_sym",
- ),
- "mx_fp4e2m1": ("mxfp4e2m1",),
- "mx_float16": ("mxfloat16",),
- "mx_fp16": ("mxfp16",),
- "mx_bfloat16": ("mxbfloat16",),
- "mx_bf16": ("mxbf16",),
-}
-
-
-for _data_type in MXFP_FORMAT_CACHE:
- register_quantizer(
- _data_type,
- aliases=_MX_ALIASES[_data_type],
- )(_MXQuantizer)
-
-for _family in ("fp", "int"):
- _canonical = f"mx_{_family}"
-
- register_quantizer(
- _canonical,
- aliases=(
- f"mx_{_family}_sym",
- f"rtn_mx_{_family}",
- f"rtn_mx_{_family}_sym",
- f"opt_rtn_mx_{_family}",
- f"opt_rtn_mx_{_family}_sym",
- ),
- )(_MXQuantizer)
-
-register_quantizer("mx_fp_rceil", aliases=("mxfprceil",))(_MXQuantizer)
-register_quantizer("mx_fp4e2m1_rceil")(_MXQuantizer)
-
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])
data1 = quant_element(data, 2, 3, 6.0)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index eed0c05fe..624e91b0c 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -12,14 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from collections.abc import Mapping
-from dataclasses import dataclass
-
import torch
-from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.fp8 import float8_e4m3fn_ste
from auto_round.data_type.gguf import _imatrix_handle_zero
+from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.logger import logger
@@ -380,6 +377,8 @@ def search_nvfp4_scale(tensor, bits=4, qw=None, quant_func=None, group_size=16):
def search_nvfp4_v2_scale(tensor, bits=4, qw=None):
+ """Search per-group scales for the legacy-registry NVFP4 E5M3 quantizer."""
+ qw = 1.0 if qw is None else qw
return search_nvfp4_scale(tensor, bits, qw, quant_func=nvfp4_v2, group_size=tensor.shape[-1]).squeeze(-1)
@@ -433,239 +432,6 @@ def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=Non
return qdq_res.to(orig_dtype), scale, None
-@dataclass(frozen=True)
-class _NVFPState:
- """NVFP tuning values plus optional global and optimized scales."""
-
- tunables: Mapping[str, torch.Tensor]
- global_scale: torch.Tensor | None
- optimized_init: torch.Tensor | None
-
-
-class _NVFPWeightQuantizer:
- """Own NVFP4 weight QDQ, including its global-scale preparation."""
-
- def __init__(self, spec, family="plain"):
- self.spec = spec
- self.family = family
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the NVFP4 weight quantizer for a resolved layer."""
- return cls(spec)
-
- @staticmethod
- def create_activation(spec):
- """Create dynamic NVFP4 activation quantization."""
- return _NVFPActivationQuantizer(spec, "nv_fp4")
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- self.family = "optimized" if mode == "optimized_rtn" else "plain"
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tunables = {}
- if self.family == "plain" and tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if self.family != "optimized" and tune_minmax:
- tunables["max_scale"] = torch.nn.Parameter(
- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
- )
-
- global_scale = self.spec.global_scale
- if global_scale is None:
- global_scale = calculate_gparam(weight, self.spec.group_size, weight.device)
- else:
- global_scale = global_scale.to(weight.device)
- optimized_init = None
- if self.family == "optimized":
- if isinstance(imatrix, torch.Tensor):
- imatrix = imatrix.reshape(1, -1)
- imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)
- imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)
- qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)
- else:
- qw = 1.0
- optimized_init = search_nvfp4_scale(grouped, self.spec.bits, qw)
- return _NVFPState(tunables, global_scale, optimized_init)
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- kwargs = {
- "bits": self.spec.bits,
- "group_size": self.spec.group_size,
- "v": tunables.get("value", 0),
- "max_scale": tunables.get("max_scale", 1.0),
- }
- kwargs["global_scale"] = state.global_scale
- if state.optimized_init is not None:
- kwargs["init_scale"] = state.optimized_init
- quantized, scale, zero_point = nv_fp4(weight, **kwargs)
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(
- quantized,
- scale if materialize else None,
- zero_point if materialize else None,
- state.global_scale if materialize else None,
- )
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("NVFP weight result was not materialized")
- module.weight.data.copy_(result.weight)
- rows = result.logical_rows or result.weight.shape[0]
- module.scale = result.scale.reshape(rows, -1).cpu()
- module.zp = None
- if result.metadata is not None:
- module.weight_global_scale = result.metadata.cpu()
-
-
-class _NVFPV2WeightQuantizer:
- """Own the independent NVFP4-v2 weight QDQ implementation."""
-
- def __init__(self, spec):
- self.spec = spec
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the NVFP4-v2 weight quantizer for a resolved layer."""
- return cls(spec)
-
- @staticmethod
- def create_activation(spec):
- """Create dynamic NVFP4-v2 activation quantization."""
- return _NVFPActivationQuantizer(spec, "nvfp4_v2")
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tunables = {}
- optimized_rtn = mode == "optimized_rtn"
- if tune_rounding and not optimized_rtn:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if tune_minmax and not optimized_rtn:
- tunables["max_scale"] = torch.nn.Parameter(
- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
- )
- optimized_scale = None
- if optimized_rtn:
- if isinstance(imatrix, torch.Tensor):
- imatrix = imatrix.reshape(1, -1)
- imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)
- imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)
- qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)
- else:
- qw = 1.0
- optimized_scale = search_nvfp4_v2_scale(grouped, self.spec.bits, qw)
- return _NVFPState(tunables, None, optimized_scale)
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- quantized, scale, zero_point = nvfp4_v2(
- weight,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- v=tunables.get("value", 0),
- max_scale=tunables.get("max_scale", state.optimized_init if state.optimized_init is not None else 1.0),
- )
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
-
- apply_result = staticmethod(_NVFPWeightQuantizer.apply_result)
-
-
-class _NVFPActivationQuantizer:
- """Quantize dynamic or calibrated NVFP activations for one NVFP format."""
-
- def __init__(self, spec, data_type):
- is_static = data_type in ("nv_fp4_with_static_gs", "nvfp4_v2_with_global_scale")
- if not is_static and not spec.dynamic:
- raise ValueError(f"NVFP datatype {data_type!r} supports only dynamic activation quantization")
- self.spec = spec
- self.data_type = data_type
- self.requires_calibration = is_static
-
- def observe(self, activation, current):
- if not self.requires_calibration:
- raise RuntimeError("Dynamic NVFP activation quantization does not require calibration")
- observed = activation.detach().float().abs().max().unsqueeze(0)
- return observed if current is None else torch.maximum(current.to(observed.device), observed)
-
- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):
- if self.requires_calibration and observed_max is None:
- raise ValueError(f"{self.data_type} activation requires observed_max")
- primitive = {
- "nv_fp4": nv_fp4,
- "nv_fp4_with_static_gs": nv_fp4_with_static_gs,
- "nvfp4_v2": nvfp4_v2,
- "nvfp4_v2_with_global_scale": nvfp4_v2_with_global_scale,
- }[self.data_type]
- kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}
- if self.requires_calibration:
- kwargs["tensor_max"] = observed_max
- if global_scale is not None:
- kwargs["global_scale"] = global_scale
- return primitive(activation, **kwargs)
-
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):
- quantized, _, _ = self.qdq_with_scale(
- activation,
- observed_max=observed_max,
- min_scale=min_scale,
- max_scale=max_scale,
- global_scale=global_scale,
- )
- return quantized
-
-
-def _prepare_nvfp_block(block, layer_runtimes):
- """Materialize shared NVFP scales once after a block's observations."""
- from auto_round.data_type.utils import update_block_global_scale_if_needed
-
- runtime = next(iter(layer_runtimes.values()), None)
- if runtime is None:
- return
- update_block_global_scale_if_needed(block, runtime["data_type"], runtime["group_size"])
-
-
-_NVFPWeightQuantizer.prepare_block = staticmethod(_prepare_nvfp_block)
-
-
-class _NVFPStaticActivation:
- """Datatype entry that provides static NVFP activation quantization only."""
-
- @staticmethod
- def create_activation(spec):
- return _NVFPActivationQuantizer(spec, "nv_fp4_with_static_gs")
-
-
-class _NVFPV2StaticActivation:
- """Datatype entry that provides global-scale NVFP-v2 activation only."""
-
- @staticmethod
- def create_activation(spec):
- return _NVFPActivationQuantizer(spec, "nvfp4_v2_with_global_scale")
-
-
-register_quantizer(
- "nv_fp4",
- aliases=(
- "nv_fp",
- "nv_fp_sym",
- "nv_fp4_sym",
- "rtn_nv_fp",
- "rtn_nv_fp4",
- "rtn_nv_fp_sym",
- "rtn_nv_fp4_sym",
- "opt_rtn_nv_fp",
- "opt_rtn_nv_fp4",
- "opt_rtn_nv_fp_sym",
- "opt_rtn_nv_fp4_sym",
- ),
-)(_NVFPWeightQuantizer)
-register_quantizer("nvfp4_v2")(_NVFPV2WeightQuantizer)
-register_quantizer("nv_fp4_with_static_gs", aliases=("rtn_nv_fp4_with_static_gs",))(_NVFPStaticActivation)
-register_quantizer("nvfp4_v2_with_global_scale")(_NVFPV2StaticActivation)
-
-
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])
data1 = cast_to_fp4(data)
diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.py
new file mode 100644
index 000000000..fca259ed6
--- /dev/null
+++ b/auto_round/data_type/register.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+QUANT_FUNC_WITH_DTYPE = {}
+
+
+def register_dtype(names):
+ """Class decorator to register a EXPORT subclass to the registry.
+
+ Decorator function used before a Pattern subclass.
+
+ Args:
+ names: A string. Define the export type.
+
+ Returns:
+ cls: The class of register.
+ """
+
+ def register(dtype):
+ if isinstance(names, (tuple, list)):
+ for name in names:
+ QUANT_FUNC_WITH_DTYPE[name] = dtype
+ else:
+ QUANT_FUNC_WITH_DTYPE[names] = dtype
+
+ return dtype
+
+ return register
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index d5b3acc52..7bca6fe97 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -22,17 +22,10 @@ from torch.nn import Linear, Module
from auto_round import envs
from auto_round.compressors.utils import is_nv_fp
-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
from auto_round.utils import check_to_quantized, logger
-def quantize_bias_without_round(*args, **kwargs):
- """Quantize bias/norm weights without exposing the INT primitive to algorithms."""
- from auto_round.data_type.int import quant_tensor_asym_wo_round
-
- return quant_tensor_asym_wo_round(*args, **kwargs)
-
-
def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):
"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.
@@ -183,6 +176,141 @@ def get_quant_func(
)
+def _resolve_optimized_dtype_funcs(data_type: str, q_scale_thresh: float = 1e-5):
+ """Resolve the SignRound optimized ``(scale_search_fn, quant_func)`` for a data type.
+
+ Single source of truth for the optimized-path dispatch shared by
+ ``SignRoundOptimizedWrapperLinear`` and AWQ's internal QDQ:
+
+ * ``scale_search_fn(weight_reshape, bits, imatrix) -> init_scale`` searches the
+ data-type-specific per-group init scale (the int variant clamps to
+ ``q_scale_thresh`` to avoid a degenerate zero scale).
+ * ``quant_func`` is the matching *plain* quant function (``init_scale`` already
+ encodes the searched scale, so no opt-rtn / rtn variant is needed).
+
+ Returns ``(None, None)`` for data types without an optimized path
+ (asym int, ``*_dq``, or unrelated types).
+ """
+ dt = str(data_type)
+ if dt.endswith("dq"):
+ return None, None
+ if dt.startswith("int"):
+ # The optimized int init-scale search is symmetric-only; asym int uses
+ # the standard tensor_min/tensor_max range instead.
+ if "asym" in dt:
+ return None, None
+ from auto_round.data_type.int import quant_tensor_sym, search_scales
+
+ def search_int(weight_reshape, bits, imatrix):
+ init_scale = search_scales(weight_reshape, bits, imatrix)
+ return torch.where(
+ init_scale < 0,
+ torch.clamp(init_scale, max=-q_scale_thresh),
+ torch.clamp(init_scale, min=q_scale_thresh),
+ )
+
+ return search_int, quant_tensor_sym
+ if dt.startswith("mx"):
+ from auto_round.data_type.mxfp import quant_mx, search_mx_scale
+
+ return search_mx_scale, quant_mx
+ if dt.startswith("nv"):
+ from auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale
+
+ return search_nvfp4_scale, nv_fp4
+ return None, None
+
+
+def search_optimized_init_scale(
+ weight_reshape: torch.Tensor,
+ data_type: str,
+ bits: int,
+ imatrix=None,
+ q_scale_thresh: float = 1e-5,
+):
+ """Compute the SignRoundV2 optimized per-group ``init_scale`` for a grouped weight.
+
+ Mirrors ``SignRoundOptimizedWrapperLinear``: dispatches on ``data_type`` so that
+ any caller (the optimized wrapper itself or AWQ's internal QDQ used for the
+ smooth/clip grid search) seeds the quantizer with the same initial scale.
+ Returns ``None`` for data types that do not use the optimized init-scale search
+ (asym int, ``*_dq``, or unrelated types).
+
+ Args:
+ weight_reshape: Weight reshaped/padded to ``[..., group_size]``.
+ data_type: Resolved weight data type (e.g. ``"int_sym"``, ``"mx_fp4"``, ``"nv_fp4"``).
+ bits: Weight bit-width.
+ imatrix: Per-element importance matrix matching ``weight_reshape`` (or ``None``/scalar).
+ q_scale_thresh: Minimum scale magnitude used to clamp the int init_scale.
+
+ Returns:
+ The per-group ``init_scale`` tensor, or ``None`` if unsupported.
+ """
+ search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)
+ if search_fn is None:
+ return None
+ if imatrix is None or not isinstance(imatrix, torch.Tensor):
+ imatrix = torch.ones_like(weight_reshape)
+ return search_fn(weight_reshape, bits, imatrix)
+
+
+def get_optimized_quant_func(data_type: str):
+ """Return the plain quant function used by the SignRound optimized path.
+
+ The optimized init-scale search always pairs with the *plain* (non opt-rtn /
+ non rtn) quant function for the data type, since ``init_scale`` already
+ encodes the searched scale. Returns ``None`` for data types without an
+ optimized path (asym int, ``*_dq``, or unrelated types).
+ """
+ _, quant_func = _resolve_optimized_dtype_funcs(data_type)
+ return quant_func
+
+
+def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size):
+ """Reshape/pad an importance matrix to match a group-reshaped weight.
+
+ Encapsulates the imatrix grouping logic shared by the SignRound optimized
+ wrapper and AWQ's internal QDQ so callers never handle the low-level reshape.
+ Returns a tensor of ones when no imatrix is available (uniform importance),
+ keeping every downstream optimized dtype implementation on its tensor path.
+ """
+ if imatrix is None or not isinstance(imatrix, torch.Tensor):
+ return torch.ones_like(weight_reshape)
+ imatrix = imatrix.reshape(1, -1)
+ imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)
+ imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)
+ return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)
+
+
+def compute_optimized_init_scale(
+ weight: torch.Tensor,
+ data_type: str,
+ bits: int,
+ group_size,
+ imatrix=None,
+ q_scale_thresh: float = 1e-5,
+):
+ """Compute the SignRound optimized per-group ``init_scale`` for a full weight.
+
+ Group-reshapes ``weight``, prepares the ``imatrix`` layout, and runs the
+ data-type-specific scale search. Unlike :func:`search_optimized_init_scale`
+ (which expects an already group-reshaped weight), this is the entry point for
+ callers holding a full 2-D weight, e.g. AWQ's internal QDQ. Pair it with
+ :func:`get_optimized_quant_func` (resolved once) to obtain the matching quant
+ function, so the smooth/clip grid-search loss mirrors what
+ ``SignRoundOptimizedWrapperLinear`` applies.
+
+ Returns ``None`` for data types without an optimized init-scale path
+ (asym int, ``*_dq``, or unrelated types).
+ """
+ search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)
+ if search_fn is None:
+ return None
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
+ imatrix = reshape_imatrix_for_weight(imatrix, weight_reshape, group_size)
+ return search_fn(weight_reshape, bits, imatrix)
+
+
def round_ste(x: torch.Tensor):
"""Straight-Through Estimator for rounding.
diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.py
index 6d7e35b41..b8e987d4c 100644
--- a/auto_round/data_type/w4fp8.py
+++ b/auto_round/data_type/w4fp8.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.base import register_dtype, register_quantizer
+from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func
@@ -72,58 +72,3 @@ def progressive_quant_fp8_int4(
bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale
return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4
-
-
-@register_quantizer("fp8_to_int_sym")
-class _ProgressiveFP8WeightQuantizer:
- """Own the progressive BF16-to-FP8-to-INT4 weight quantization path."""
-
- def __init__(self, spec):
- self.spec = spec
-
- @classmethod
- def from_spec(cls, spec, canonical=None):
- """Create the progressive FP8-to-INT4 weight quantizer."""
- return cls(spec)
-
- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
-
- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
- tunables = {}
- if tune_rounding:
- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
- if tune_minmax:
- shape = grouped.shape[:-1]
- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
- return tunables
-
- def qdq(self, weight, state, *, tunables, materialize=False):
- quantized, scales, zero_point = progressive_quant_fp8_int4(
- weight,
- bits=self.spec.bits,
- group_size=self.spec.group_size,
- v=tunables.get("value", 0),
- min_scale=tunables.get("min_scale", 1.0),
- max_scale=tunables.get("max_scale", 1.0),
- q_scale_thresh=self.spec.q_scale_thresh,
- )
- from auto_round.data_type.base import WeightQuantizationResult
-
- return WeightQuantizationResult(
- quantized,
- scales["scale"] if materialize else None,
- zero_point if materialize else None,
- scales["bf16_to_fp8_scale"] if materialize else None,
- )
-
- @staticmethod
- def apply_result(module, result):
- if result.scale is None:
- raise ValueError("Progressive FP8 weight result was not materialized")
- module.weight.data.copy_(result.weight)
- rows = result.logical_rows or result.weight.shape[0]
- module.scale = result.scale.reshape(rows, -1).cpu()
- module.zp = result.zero_point
- module.w_bf16_to_fp8_scale = result.metadata.cpu()
diff --git a/auto_round/modeling/fused_moe/grouped_experts.py b/auto_round/modeling/fused_moe/grouped_experts.py
index 48d8511e5..c9a755316 100644
--- a/auto_round/modeling/fused_moe/grouped_experts.py
+++ b/auto_round/modeling/fused_moe/grouped_experts.py
@@ -490,11 +490,6 @@ def _batched_qdq_weights(
eligible and the caller must go per-expert.
"""
ref = layers[0]
- # Datatype quantizers intentionally keep their per-layer QDQ state private.
- # Without the legacy compiled function, that state cannot be safely merged,
- # so retain correctness by using the normal per-expert path.
- if not hasattr(ref, "weight_quant_func"):
- return None
orig = ref.orig_layer
out_features, in_features = ref.weight.shape
num_experts = len(layers)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index e018ca364..2f874158e 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2231,9 +2231,7 @@ def set_amax_for_uncalibrated_experts(
)
return uncalibrated_experts
# Flatten all tensors to 1D before concatenation
- device = amax_values[0].device
- dtype = amax_values[0].dtype
- flat_values = [t.reshape(-1).to(device=device, dtype=dtype) for t in amax_values]
+ flat_values = [t.reshape(-1) for t in amax_values]
all_values = torch.cat(flat_values)
set_amax_value = torch.max(all_values)
set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_value
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index 239234ac6..4e0379fe2 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -1227,7 +1227,8 @@ def _quantize_weight_nvfp4_e5m3(
disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
"""Fake-quantize a 2D weight tensor to NVFP4 E5M3 and return its high-precision QDQ weight."""
- from auto_round.data_type.base import create_quantizer
+ from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
out_features, in_features = weight.shape
if group_size != 16:
@@ -1239,13 +1240,11 @@ def _quantize_weight_nvfp4_e5m3(
)
weight_dev = weight.to(device)
- quantizer = create_quantizer(
- {"data_type": "nvfp4_v2", "bits": 4, "group_size": group_size, "sym": True, "scale_dtype": torch.float32},
- iters=0,
- disable_opt_rtn=disable_opt_rtn,
- )
- quantizer.initialize(weight_dev)
- qdq_weight = quantizer.quantize(weight_dev)
+ max_scale = 1.0
+ if not disable_opt_rtn:
+ grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
+ max_scale = search_nvfp4_v2_scale(grouped)
+ qdq_weight, _, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)
return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}
@@ -1280,7 +1279,8 @@ def _pack_weight_nvfp4_e5m3(
disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
"""Pack FP4 E2M1 weights with unsigned E5M3 block scales."""
- from auto_round.data_type.base import create_quantizer
+ from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
out_features, in_features = weight.shape
@@ -1289,13 +1289,11 @@ def _pack_weight_nvfp4_e5m3(
f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'."
)
weight_dev = weight.to(device)
- quantizer = create_quantizer(
- {"data_type": "nvfp4_v2", "bits": 4, "group_size": group_size, "sym": True, "scale_dtype": torch.float32},
- iters=0,
- disable_opt_rtn=disable_opt_rtn,
- )
- quantizer.initialize(weight_dev)
- scale = quantizer.qdq(weight_dev, materialize=True).scale
+ max_scale = 1.0
+ if not disable_opt_rtn:
+ grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
+ max_scale = search_nvfp4_v2_scale(grouped)
+ _, scale, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)
# nvfp4_v2 may return a flattened per-group scale layout (e.g. [N, 1]);
# normalize to [out_features, in_features // group_size] before packing
# so serialized .weight_scale keeps the expected 2D shape.
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 6b70865f2..36ffbb173 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -20,8 +20,8 @@ import transformers
from torch.functional import F
import auto_round.envs as envs
-from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer
-from auto_round.data_type.utils import quantize_bias_without_round, reshape_pad_tensor_by_group_size
+from auto_round.compressors.utils import is_nv_fp
+from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -84,7 +84,6 @@ class WrapperLinear(torch.nn.Module):
enable_round_tuning=True,
enable_torch_compile=True,
disable_opt_rtn=True,
- weight_qdq_builder=None,
**kwargs,
):
"""Initializes the WrapperLinear module.
@@ -106,35 +105,18 @@ class WrapperLinear(torch.nn.Module):
self.enable_torch_compile = enable_torch_compile
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
- # Observation may have cached the quantizer on the original layer. Do
- # not consume it until wrapper initialization has completed successfully.
- cached_activation_quantizer = getattr(self.orig_layer, "_ar_activation_quantizer", None)
- self.activation_quantizer = cached_activation_quantizer
- if self.enable_act_quant and self.activation_quantizer is None:
- self.activation_quantizer = activation_quantizer_for_layer(orig_layer)
self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
+ if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)
+ setattr(self, "weight_global_scale", weight_global_scale)
+ self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
self.q_scale_thresh = 1e-8
else:
self.q_scale_thresh = 1e-5
self._init_tuning_params_and_quant_func()
-
- def default_qdq(weight, *, materialize=False):
- """Keep direct wrapper use working when no algorithm supplies a builder."""
- return self.weight_quantizer.qdq(weight, materialize=materialize, **self.params)
-
- self.weight_qdq = default_qdq
- if weight_qdq_builder is not None:
- weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
- if type(orig_layer) == transformers.pytorch_utils.Conv1D:
- weight = weight.t()
- self.weight_qdq = weight_qdq_builder(
- weight=weight.to(self.device), default_qdq=self.weight_quantizer.qdq, parameters=self.params
- )
- for name, parameter in self.params.items():
- setattr(self, name, parameter)
- if cached_activation_quantizer is not None:
- self.orig_layer.__dict__.pop("_ar_activation_quantizer", None)
if deepspeed_exists:
if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):
self.orig_forward = self.linear_forward
@@ -165,25 +147,69 @@ class WrapperLinear(torch.nn.Module):
orig_layer = self.orig_layer
orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
- if type(orig_layer) == transformers.pytorch_utils.Conv1D:
+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
orig_weight = orig_weight.t()
- self.weight_quantizer = create_quantizer(
- orig_layer,
- disable_opt_rtn=self.disable_opt_rtn,
- tune_rounding=self.enable_round_tuning,
- tune_minmax=self.enable_minmax_tuning,
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)
+
+ if self.enable_round_tuning:
+ self.weight_min = (
+ torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)
+ if isinstance(orig_layer.group_size, tuple)
+ else torch.clamp(weight_reshape.min(1)[0], max=0)
+ )
+ self.weight_max = (
+ torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)
+ if isinstance(orig_layer.group_size, tuple)
+ else torch.clamp(weight_reshape.max(1)[0], min=0)
+ )
+ else:
+ self.weight_min = None
+ self.weight_max = None
+ # AWQ clip-as-init: cap the tunable weight range to the per-group clip
+ # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
+ # This initializes the range used by quant_tensor_sym/asym, leaving
+ # min_scale/max_scale to tune a coefficient on top. Only the standard
+ # (non-tuple) group layout maps onto weight_min/weight_max here.
+ awq_clip_min = getattr(orig_layer, "awq_clip_min", None)
+ awq_clip_max = getattr(orig_layer, "awq_clip_max", None)
+ if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):
+ clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
+ if awq_clip_min is None:
+ clip_min_flat = -clip_max_flat
+ else:
+ clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)
+ if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():
+ self.weight_max = torch.minimum(self.weight_max, clip_max_flat)
+ self.weight_min = torch.maximum(self.weight_min, clip_min_flat)
+ self._init_params(
+ "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16
)
- self.weight_quantizer.initialize(orig_weight, imatrix=getattr(orig_layer, "imatrix", None))
- self.params = dict(self.weight_quantizer.parameters)
- for name, value in self.params.items():
- setattr(self, name, value)
- self.value = getattr(self, "value", torch.tensor(0.0, device=self.device, dtype=p_dtype))
- self.min_scale = getattr(self, "min_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))
- self.max_scale = getattr(self, "max_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))
- self.data_type = orig_layer.data_type
+ # Min-max scale initialization
+ shape = get_scale_shape(orig_weight, orig_layer.group_size)
+ self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
+ self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
+
+ self.weight_quant_func, self.data_type = get_quant_func(
+ orig_layer.data_type,
+ orig_layer.bits,
+ orig_layer.sym,
+ self.disable_opt_rtn,
+ orig_layer.group_size,
+ iters=orig_layer.iters,
+ )
+ if self.enable_torch_compile:
+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
if self.enable_act_quant:
- self.act_data_type = orig_layer.act_data_type
+ self.act_quant_func, self.act_data_type = get_quant_func(
+ orig_layer.act_data_type,
+ orig_layer.act_bits,
+ orig_layer.act_sym,
+ disable_opt_rtn=True,
+ iters=orig_layer.iters,
+ )
+ if self.enable_torch_compile:
+ self.act_quant_func = compile_func(self.act_quant_func, self.device)
self._init_params(
"act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic)
)
@@ -192,7 +218,9 @@ class WrapperLinear(torch.nn.Module):
# Bias tuning
if self.enable_norm_bias_tuning:
self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)
- self.bias_quant_func = quantize_bias_without_round
+ from auto_round.data_type.int import quant_tensor_asym_wo_round
+
+ self.bias_quant_func = quant_tensor_asym_wo_round
self.params["bias_v"] = self.bias_v
def _init_params(self, name, dtype, shape, value, tunable):
@@ -226,17 +254,43 @@ class WrapperLinear(torch.nn.Module):
"""
if self.orig_layer.bits >= 16:
return self.orig_layer.weight, None, None
+ min_bound, max_bound = self.minmax_scale_bound
+ min_scale.data.clamp_(min_bound, max_bound)
+ max_scale.data.clamp_(min_bound, max_bound)
weight = self.orig_layer.weight
if weight.device.type == "meta":
weight = self.orig_layer.get_weight().to(self.device)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight = weight.t()
- self.params.update(value=value, min_scale=min_scale, max_scale=max_scale)
- weight_q = self.weight_qdq(weight.to(self.device)).weight.to(weight.dtype)
+ quant_kwargs = {}
+ if hasattr(self.orig_layer, "super_bits"):
+ quant_kwargs["super_bits"] = self.orig_layer.super_bits
+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
+ if hasattr(self, "_extra_quant_kwargs"):
+ quant_kwargs.update(self._extra_quant_kwargs())
+
+ weight_q, scale, zp = self.weight_quant_func(
+ weight.to(self.device),
+ bits=self.orig_layer.bits,
+ group_size=self.orig_layer.group_size,
+ v=value,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.orig_layer.scale_dtype,
+ tensor_min=self.weight_min,
+ tensor_max=self.weight_max,
+ data_type=self.data_type,
+ q_scale_thresh=self.q_scale_thresh,
+ imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,
+ global_scale=getattr(self, "weight_global_scale", None),
+ init_scale=getattr(self, "init_scale", None),
+ **quant_kwargs,
+ )
+ weight_q = weight_q.to(weight.dtype)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight_q = weight_q.t()
- return weight_q, None, None
+ return weight_q, scale, zp
def _qdq_act(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):
"""Quantizes and dequantizes activations.
@@ -252,12 +306,19 @@ class WrapperLinear(torch.nn.Module):
act_max_scale.data.clamp_(0, 1.0)
act_min_scale.data.clamp_(0, 1.0)
env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set
- return self.activation_quantizer.qdq_with_scale(
+ x, scale, zp = self.act_quant_func(
x,
- observed_max=act_max,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.q_scale_thresh,
+ data_type=self.act_data_type,
+ tensor_max=act_max, # for static
max_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
+ global_scale=getattr(self, "input_global_scale", None),
)
+ return x, scale, zp
def _qdq_bias(self, bias, bias_v):
"""Quantizes and dequantizes bias.
@@ -300,28 +361,59 @@ class WrapperLinear(torch.nn.Module):
v = best_params.get("value", torch.tensor(0.0)).to(self.device)
min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)
max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)
- # Preserve the dictionary captured by QDQ closures and restore all
- # algorithm parameters, including names unknown to the wrapper.
- self.params.update({name: value.to(self.device) for name, value in best_params.items()})
- self.params.update(value=v, min_scale=min_scale, max_scale=max_scale)
if self.orig_layer.weight.device.type == "meta":
self.orig_layer.to(self.device)
- if self.orig_layer.bits < 16:
- weight = self.orig_layer.weight
- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
- weight = weight.t()
- result = self.weight_qdq(
- weight.to(self.device),
- materialize=True,
- )
- self.weight_quantizer.apply_result(
- self.orig_layer,
- result,
- transpose=type(self.orig_layer) == transformers.pytorch_utils.Conv1D,
- )
+ # Unwrapper weight
+ qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
+ # if hasattr(self.orig_layer, "imatrix"):
+ # self.orig_layer.imatrix = None
+ self.orig_layer.weight.data.copy_(qdq_weight)
self.orig_layer.weight.grad = None
+ shape = qdq_weight.shape
+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
+ shape = qdq_weight.t().shape
+
+ def _set_dict_attr(attr_dict, attr_name):
+ for key in attr_dict.keys():
+ if key == attr_name:
+ setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))
+ else:
+ name = "w_" + key
+ setattr(self.orig_layer, name, attr_dict[key].to("cpu"))
+
+ if not isinstance(self.orig_layer.group_size, tuple):
+ if isinstance(scale, dict):
+ _set_dict_attr(scale, "scale")
+ elif scale is None:
+ self.orig_layer.scale = None
+ elif scale.numel() > 1:
+ self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
+ else:
+ self.orig_layer.scale = scale.view(-1).to("cpu")
+ else:
+ self.orig_layer.scale = scale.to("cpu")
+
+ if zp is not None:
+ if isinstance(zp, dict):
+ _set_dict_attr(zp, "zp")
+ elif isinstance(zp, torch.Tensor):
+ if zp.numel() > 1:
+ zp = zp.reshape(shape[0], -1)
+ self.orig_layer.zp = zp.to("cpu")
+ else:
+ self.orig_layer.zp = zp.view(-1).to("cpu")
+ else:
+ self.orig_layer.zp = zp
+ else:
+ self.orig_layer.zp = None
+
+ if self.weight_global_scale is not None:
+ global_scale = self.weight_global_scale
+ assert global_scale.numel() == 1
+ self.orig_layer.weight_global_scale = global_scale.to("cpu")
+
# Unwrapper bias
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"].to(self.device)
@@ -365,11 +457,11 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.act_max_scale = self.act_max_scale
self.orig_layer.act_data_type = self.act_data_type
+ self.orig_layer.act_quant_func = self.act_quant_func
wrapper_layer = WrapperWALayer(
self.orig_layer,
enable_torch_compile=self.enable_torch_compile,
device=self.device,
- activation_quantizer=self.activation_quantizer,
)
return _preserve_global_name(wrapper_layer)
@@ -475,16 +567,16 @@ class WrapperLinear(torch.nn.Module):
class WrapperWALayer(torch.nn.Module):
- def __init__(self, orig_layer, enable_torch_compile=True, device="cpu", activation_quantizer=None):
+ def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):
super(WrapperWALayer, self).__init__()
self.orig_layer = orig_layer
self.enable_torch_compile = enable_torch_compile
self.device = device
self.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else None
self.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None
- self.activation_quantizer = activation_quantizer or orig_layer.__dict__.pop("_ar_activation_quantizer", None)
- if self.activation_quantizer is None:
- self.activation_quantizer = activation_quantizer_for_layer(orig_layer)
+ self.act_quant_func = self.orig_layer.act_quant_func
+ if self.enable_torch_compile:
+ self.act_quant_func = compile_func(self.act_quant_func, self.device)
self.extra_repr_org = orig_layer.extra_repr
# Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)
@@ -518,10 +610,32 @@ class WrapperWALayer(torch.nn.Module):
max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
- quant_kwargs = {"observed_max": act_max, "min_scale": min_scale, "max_scale": max_scale}
+ quant_kwargs = {}
if hasattr(self.orig_layer, "input_global_scale"):
quant_kwargs["global_scale"] = self.orig_layer.input_global_scale
- x = self.activation_quantizer.qdq(x, **quant_kwargs)
+ if act_max is None:
+ x, _, _ = self.orig_layer.act_quant_func(
+ x,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.orig_layer.q_scale_thresh,
+ data_type=self.orig_layer.act_data_type,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ **quant_kwargs,
+ )
+ else:
+ x, _, _ = self.orig_layer.act_quant_func(
+ x,
+ bits=self.orig_layer.act_bits,
+ group_size=self.orig_layer.act_group_size,
+ scale_dtype=self.orig_layer.scale_dtype,
+ q_scale_thresh=self.orig_layer.q_scale_thresh,
+ data_type=self.orig_layer.act_data_type,
+ act_max=act_max,
+ **quant_kwargs,
+ )
# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)
return self.orig_layer.forward(x)
@@ -553,7 +667,9 @@ class WrapperLayerNorm(torch.nn.Module):
requires_grad=True,
)
self.params = {"v": self.v}
- self.quant_func = quantize_bias_without_round
+ from auto_round.data_type.int import quant_tensor_asym_wo_round
+
+ self.quant_func = quant_tensor_asym_wo_round
def unwrapper(self, best_params):
if best_params is None:
@@ -602,7 +718,9 @@ class WrapperLlamaNorm(torch.nn.Module):
requires_grad=True,
)
self.params = {"v": self.v}
- self.quant_func = quantize_bias_without_round
+ from auto_round.data_type.int import quant_tensor_asym_wo_round
+
+ self.quant_func = quant_tensor_asym_wo_round
def unwrapper(self, best_params):
if best_params is None:
diff --git a/test/unit/common/algorithms/test_algorithm_datatype_parity.py b/test/unit/common/algorithms/test_algorithm_datatype_parity.py
deleted file mode 100644
index 2d4829c05..000000000
--- a/test/unit/common/algorithms/test_algorithm_datatype_parity.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Check the public datatype API used by wrappers and algorithms."""
-
-from types import SimpleNamespace
-
-import torch
-
-from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer
-from auto_round.data_type.int import quant_tensor_sym
-
-
-def test_weight_quantization_reuses_initialization_and_matches_int_math():
- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
- config = dict(data_type="int", bits=4, group_size=4, sym=True, scale_dtype=torch.float32)
- quantizer = create_quantizer(config, disable_opt_rtn=True)
- quantizer.initialize(weight)
- first = quantizer.quantize(weight)
- second = quantizer.quantize(weight)
- expected = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)[0]
- torch.testing.assert_close(first, expected, rtol=0, atol=0)
- assert torch.equal(first, second)
-
-
-def test_weight_training_and_writeback_share_parameters():
- layer = torch.nn.Linear(4, 2, bias=False)
- layer.data_type = "int"
- layer.bits = 4
- layer.group_size = 4
- layer.sym = True
- layer.scale_dtype = torch.float32
- quantizer = create_quantizer(layer, iters=10, tune_rounding=True, tune_minmax=True)
- quantizer.initialize(layer.weight.detach())
- output = quantizer.quantize(layer.weight.detach())
- output.square().sum().backward()
- assert all(parameter.grad is not None for parameter in quantizer.parameters.values())
- expected = output.detach().clone()
- with torch.no_grad():
- quantizer.write_back(layer, layer.weight.detach())
- torch.testing.assert_close(layer.weight, expected, rtol=0, atol=0)
- assert layer.scale.shape == (2, 1)
- assert layer.zp is not None
-
-
-def test_reinitializing_search_quantizer_does_not_reuse_previous_weight_range():
- config = dict(data_type="int", bits=4, group_size=4, sym=False, scale_dtype=torch.float32)
- quantizer = create_quantizer(config, disable_opt_rtn=True)
- first = torch.tensor([[1.0, -0.75, 0.25, -0.5]])
- second = first * 5
- quantizer.initialize(first)
- quantizer.quantize(first)
- quantizer.initialize(second)
- actual = quantizer.quantize(second)
- fresh = create_quantizer(config, disable_opt_rtn=True)
- fresh.initialize(second)
- assert torch.equal(actual, fresh.quantize(second))
-
-
-def test_activation_adapter_static_and_dynamic_contracts():
- activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)
- for dynamic in (False, True):
- layer = SimpleNamespace(
- act_data_type="int",
- act_bits=8,
- act_group_size=4,
- act_sym=True,
- scale_dtype=torch.float32,
- act_dynamic=dynamic,
- )
- quantizer = activation_quantizer_for_layer(layer)
- observed = quantizer.observe(activation, None)
- qdq = quantizer.qdq(activation, observed_max=observed)
- assert qdq.shape == activation.shape
- assert torch.isfinite(qdq).all()
diff --git a/test/unit/common/algorithms/transforms/hadamard/test_patch.py b/test/unit/common/algorithms/transforms/hadamard/test_patch.py
index 6c8d176a8..ff63079dc 100644
--- a/test/unit/common/algorithms/transforms/hadamard/test_patch.py
+++ b/test/unit/common/algorithms/transforms/hadamard/test_patch.py
@@ -179,14 +179,14 @@ class TestPatchWrapperLinearIdempotency:
# Call the patched _qdq_act on a wrapper
wrapper = _make_wrapper_linear()
- # Activation QDQ is now owned by the datatype quantizer.
- wrapper.activation_quantizer.qdq_with_scale = MagicMock(return_value=(torch.zeros(1, 8), None, None))
+ # Replace the wrapper's act_quant_func with a spy
+ wrapper.act_quant_func = MagicMock(return_value=(torch.zeros(1, 8), None, None))
x = torch.ones(1, 8)
act_min = torch.tensor(1.0)
act_max = torch.tensor(1.0)
- wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max, act_max=act_max)
+ wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max)
# The mock should have been called with x * 2
- called_args = wrapper.activation_quantizer.qdq_with_scale.call_args[0]
+ called_args = wrapper.act_quant_func.call_args[0]
assert torch.equal(called_args[0], x * 2.0)
def test_qdq_weight_falls_through_for_high_bits(self):
@@ -294,7 +294,6 @@ class TestPatchWrapperWALayerIdempotency:
patch_wrapperwalayer_forward_to_apply_transform(inp_transform)
wrapper = _make_wrapper_wa_layer()
- wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))
# Patch the orig_layer.forward to be a spy
orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))
wrapper.orig_layer.forward = orig_forward_spy
@@ -302,8 +301,8 @@ class TestPatchWrapperWALayerIdempotency:
x = torch.ones(1, 8)
wrapper(x)
- # Datatype-owned activation QDQ receives the rotated input.
- called_args = wrapper.activation_quantizer.qdq.call_args[0]
+ # The act_quant_func should have been called with x * 4
+ called_args = wrapper.orig_layer.act_quant_func.call_args[0]
assert torch.equal(called_args[0], x * 4.0)
def test_act_max_passed_when_present(self):
@@ -316,7 +315,6 @@ class TestPatchWrapperWALayerIdempotency:
wrapper = _make_wrapper_wa_layer()
wrapper.orig_layer.act_max = torch.tensor(2.0)
- wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))
orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))
wrapper.orig_layer.forward = orig_forward_spy
@@ -324,7 +322,9 @@ class TestPatchWrapperWALayerIdempotency:
x = torch.ones(1, 8)
wrapper(x)
- assert wrapper.activation_quantizer.qdq.call_args.kwargs["observed_max"] is not None
+ # tensor_max should be the act_max value
+ kwargs = wrapper.orig_layer.act_quant_func.call_args.kwargs
+ assert kwargs.get("tensor_max") is not None
class TestPatchQuantLinearIdempotency:
diff --git a/test/unit/common/core/test_wrapper_utils.py b/test/unit/common/core/test_wrapper_utils.py
index 68de62b29..a603d0da0 100644
--- a/test/unit/common/core/test_wrapper_utils.py
+++ b/test/unit/common/core/test_wrapper_utils.py
@@ -335,95 +335,6 @@ class TestWrapperLinearDeviceTransfer:
assert wrapper.device == "cpu"
assert wrapper.orig_layer is orig_layer
- def test_unwrapper_keeps_weight_for_activation_only_quantization(self):
- """W16A8 writes activation state only; its weight stays full precision."""
- from auto_round.wrapper import WrapperLinear
-
- layer = torch.nn.Linear(4, 2, bias=False)
- layer.bits = 16
- layer.sym = True
- layer.group_size = -1
- layer.scale_dtype = torch.float32
- layer.data_type = "int"
- layer.act_bits = 8
- layer.act_data_type = "int"
- layer.act_sym = True
- layer.act_dynamic = True
- layer.act_group_size = -1
- layer.iters = 0
- layer.tuning_device = "cpu"
- expected = layer.weight.detach().clone()
-
- WrapperLinear(layer, device="cpu", disable_opt_rtn=True).unwrapper({})
-
- assert torch.equal(layer.weight, expected)
- assert not hasattr(layer, "scale")
-
- def test_custom_weight_qdq_is_shared_by_forward_and_write_back(self):
- """A per-layer algorithm hook controls both tuning and final materialization."""
- from auto_round.wrapper import WrapperLinear
-
- orig_layer = torch.nn.Linear(4, 2, bias=False)
- orig_layer.bits = 4
- orig_layer.sym = True
- orig_layer.group_size = 4
- orig_layer.scale_dtype = torch.float32
- orig_layer.data_type = "int"
- orig_layer.act_bits = 16
- orig_layer.act_data_type = "float"
- orig_layer.act_sym = True
- orig_layer.act_dynamic = True
- orig_layer.act_group_size = -1
- orig_layer.tuning_device = "cpu"
- calls = []
-
- def build_weight_qdq(*, weight, default_qdq, parameters):
- assert weight.shape == (2, 4)
- parameters["offset"] = torch.nn.Parameter(weight.new_tensor(0.1))
-
- def custom_qdq(weight, *, materialize=False):
- calls.append(materialize)
- return default_qdq(weight + parameters["offset"], materialize=materialize, **parameters)
-
- return custom_qdq
-
- wrapper = WrapperLinear(
- orig_layer,
- device="cpu",
- disable_opt_rtn=True,
- iters=0,
- weight_qdq_builder=build_weight_qdq,
- )
-
- result, _, _ = wrapper._qdq_weight(wrapper.value, wrapper.min_scale, wrapper.max_scale)
- result.sum().backward()
- assert wrapper.params["offset"].grad is not None
- best = torch.tensor(0.5)
- wrapper.params["offset"] = best
- expected = wrapper.weight_qdq(orig_layer.weight, materialize=True).weight.detach().clone()
- wrapper.params["offset"] = torch.tensor(0.9)
- wrapper.unwrapper({"offset": best})
-
- assert calls == [False, True, True]
- assert torch.equal(orig_layer.weight, expected)
-
- def test_base_algorithm_uses_datatype_qdq_by_default(self):
- """Algorithms only override QDQ when they explicitly need custom behavior."""
- from auto_round.algorithms.quantization.base import BaseQuantizer
- from auto_round.data_type.base import create_quantizer
-
- weight = torch.tensor([[1.0, -0.5, 0.25, -0.75]])
- datatype = create_quantizer(dict(data_type="int", bits=4, group_size=4, sym=True), disable_opt_rtn=True)
- datatype.initialize(weight)
- parameters = {"value": torch.tensor(0.0)}
- qdq = BaseQuantizer.build_weight_qdq(object(), weight=weight, default_qdq=datatype.qdq, parameters=parameters)
- torch.testing.assert_close(qdq(weight).weight, datatype.qdq(weight, value=0.0).weight)
- parameters["value"] = torch.tensor(0.75)
- result = qdq(weight, materialize=True)
- expected = datatype.qdq(weight, value=0.75, materialize=True)
- torch.testing.assert_close(result.weight, expected.weight)
- torch.testing.assert_close(result.scale, expected.scale)
-
def test_wrapper_linear_forward(self):
from auto_round.wrapper import WrapperLinear
diff --git a/test/unit/common/data_type/__init__.py b/test/unit/common/data_type/__init__.py
index 14a492441..e69de29bb 100644
--- a/test/unit/common/data_type/__init__.py
+++ b/test/unit/common/data_type/__init__.py
@@ -1,13 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/test/unit/common/data_type/test_quantizer_parity.py b/test/unit/common/data_type/test_quantizer_parity.py
deleted file mode 100644
index 9b23109cc..000000000
--- a/test/unit/common/data_type/test_quantizer_parity.py
+++ /dev/null
@@ -1,189 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Focused coverage for the datatype public lifecycle."""
-
-import pytest
-import torch
-
-from auto_round.data_type.base import (
- activation_quantizer_for_layer,
- cache_activation_quantizer,
- canonical_data_type,
- create_quantizer,
-)
-from auto_round.data_type.fp8 import quant_fp8_e5m2
-from auto_round.data_type.gguf import _GGUFWeightQuantizer
-from auto_round.data_type.int import quant_tensor_sym
-from auto_round.data_type.mxfp import quant_mx, quant_mx_rceil
-
-
-def _weight_config(data_type, bits, group_size, *, sym=True):
- return dict(data_type=data_type, bits=bits, group_size=group_size, sym=sym, scale_dtype=torch.float32)
-
-
-def test_integer_lifecycle_matches_primitive_and_materializes_payload():
- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
- quantizer.initialize(weight)
- actual = quantizer.quantize(weight)
- expected, _, _ = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)
-
- assert torch.equal(actual, expected)
-
-
-def test_qdq_returns_the_same_result_used_for_write_back():
- """Algorithms can execute QDQ once and hand its materialized result to the datatype."""
- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
- layer = torch.nn.Linear(4, 1, bias=False)
- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
- quantizer.initialize(weight)
-
- result = quantizer.qdq(weight, materialize=True)
- quantizer.apply_result(layer, result)
-
- assert torch.equal(layer.weight, result.weight)
- assert layer.scale.shape == (1, 1)
-
-
-def test_write_back_keeps_conv1d_scale_indexed_by_output_channels():
- """Conv1D stores a transposed weight, but its scale stays per output channel."""
- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5], [-1.0, 0.75, -0.25, 0.5]], dtype=torch.float32)
- layer = torch.nn.Module()
- layer.weight = torch.nn.Parameter(torch.empty(4, 2))
- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
- quantizer.initialize(weight)
-
- quantizer.write_back(layer, weight, transpose=True)
-
- assert layer.weight.shape == (4, 2)
- assert layer.scale.shape == (2, 1)
-
-
-def test_fp8_format_selection_does_not_turn_e5m2_into_block_e4m3():
- weight = torch.randn(16, 32)
- quantizer = create_quantizer(_weight_config("fp8_e5m2", 8, (8, 16)), iters=1)
- quantizer.initialize(weight)
- result = quantizer.quantize(weight)
- expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=(8, 16))
-
- assert torch.equal(result, expected)
-
-
-def test_generic_mx_name_uses_requested_bit_width():
- weight = torch.randn(8, 32)
- quantizer = create_quantizer(_weight_config("mx_fp", 6, 32), iters=1)
- quantizer.initialize(weight)
- result = quantizer.quantize(weight)
- expected, _, _ = quant_mx(weight, bits=6, group_size=32, data_type="mx_fp6")
-
- assert torch.equal(result, expected)
-
-
[email protected]("dynamic", (True, False))
-def test_activation_lifecycle_handles_dynamic_and_calibrated_integer_quantization(dynamic):
- activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)
- layer = type(
- "Layer",
- (),
- dict(act_data_type="int", act_bits=8, act_group_size=4, act_sym=True, act_dynamic=dynamic),
- )()
- quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)
- observed = None if dynamic else quantizer.observe(activation, None)
-
- result = quantizer.qdq(activation, observed_max=observed)
-
- assert result.shape == activation.shape
- assert torch.isfinite(result).all()
-
-
-def test_nvfp_static_global_scale_accepts_dynamic_calibration():
- """NVFP4 calibrates a static global scale while keeping dynamic QDQ enabled."""
- layer = type(
- "Layer",
- (),
- dict(
- act_data_type="nv_fp4_with_static_gs",
- act_bits=4,
- act_group_size=16,
- act_sym=True,
- act_dynamic=True,
- ),
- )()
-
- quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)
-
- assert quantizer.requires_calibration is True
-
-
-def test_aliases_keep_a_single_canonical_datatype_for_policy():
- assert canonical_data_type("int") == "int_sym"
- assert canonical_data_type("INT4_ASYM") == "int_asym"
-
-
-def test_activation_quantizer_uses_the_spec_default_datatype():
- """Partial layer metadata keeps the public ``int_sym`` activation default."""
- layer = type("Layer", (), dict(act_bits=8, act_group_size=4, act_sym=True, act_dynamic=True))()
-
- assert type(cache_activation_quantizer(layer)).__name__ == "_IntActivationQuantizer"
- assert type(activation_quantizer_for_layer(layer)).__name__ == "_IntActivationQuantizer"
-
-
-def test_format_aliases_keep_the_requested_quantization_format():
- weight = torch.randn(4, 32)
- fp8 = create_quantizer(_weight_config("fp8-e5m2", 8, -1), iters=1)
- fp8.initialize(weight)
- expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=-1)
-
- gguf = create_quantizer(_weight_config("INT_SYM_DQ", 4, 32), disable_opt_rtn=True)
- legacy_gguf = create_quantizer(_weight_config("rtn_int_sym_dq", 4, 32), disable_opt_rtn=True)
-
- assert torch.equal(fp8.quantize(weight), expected)
- assert isinstance(gguf._implementation, _GGUFWeightQuantizer)
- assert gguf._implementation.kind == "sym"
- assert isinstance(legacy_gguf._implementation, _GGUFWeightQuantizer)
-
-
-def test_mx_aliases_and_rceil_keep_the_requested_format():
- weight = torch.randn(4, 32)
- layer = type(
- "Layer", (), dict(act_data_type="mxfp4", act_bits=4, act_group_size=32, act_sym=True, act_dynamic=True)
- )()
- activation = activation_quantizer_for_layer(layer)
- quantizer = create_quantizer(_weight_config("mx_fp4e2m1_rceil", 4, 32), iters=1)
- quantizer.initialize(weight)
- expected, _, _ = quant_mx_rceil(weight, bits=4, group_size=32, data_type="mx_fp4e2m1")
-
- assert activation.data_type == "mx_fp4"
- assert torch.equal(quantizer.quantize(weight), expected)
-
-
-def test_awq_clip_ranges_are_flattened_before_integer_qdq():
- """AWQ stores [output, group] clips while INT QDQ uses flattened group rows."""
- weight = torch.randn(4, 8)
- clip = weight.abs().reshape(4, 2, 4).amax(dim=-1)
- quantizer = create_quantizer(
- {
- **_weight_config("int", 4, 4),
- "awq_clip_min": -clip,
- "awq_clip_max": clip,
- },
- iters=1,
- )
- quantizer.initialize(weight)
-
- result = quantizer.quantize(weight)
-
- assert result.shape == weight.shape
- assert torch.isfinite(result).all()
diff --git a/test/unit/common/models/test_moe_experts_interface.py b/test/unit/common/models/test_moe_experts_interface.py
index 2656d1c58..49f0f9125 100644
--- a/test/unit/common/models/test_moe_experts_interface.py
+++ b/test/unit/common/models/test_moe_experts_interface.py
@@ -21,13 +21,10 @@ This verifies that:
3. The forward pass produces correct results
"""
-from types import SimpleNamespace
-
import pytest
import torch
from torch import nn
-from auto_round.modeling.fused_moe.grouped_experts import _batched_qdq_weights
from auto_round.modeling.fused_moe.replace_modules import (
ReplacementModuleBase,
_apply_custom_replacements,
@@ -45,11 +42,6 @@ def _skip_if_no_linear_loop():
pytest.skip("transformers MOE integration not available")
-def test_batched_qdq_falls_back_without_legacy_quant_function():
- """Datatype-owned QDQ safely falls back when its private state cannot be fused."""
- assert _batched_qdq_weights([SimpleNamespace()]) is None
-
-
def test_linear_loop_registration():
"""Test that linear_loop is registered with transformers."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.py
index 0931831cc..837a96605 100644
--- a/test/unit/common/schemes/test_auto_scheme.py
+++ b/test/unit/common/schemes/test_auto_scheme.py
@@ -16,7 +16,6 @@ from auto_round.auto_scheme.delta_loss import (
_vram_inventory_text,
)
from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
-from auto_round.data_type.base import WeightQuantizationResult
from auto_round.export.export_to_gguf.export import _clear_gguf_model_instances
@@ -1201,38 +1200,24 @@ class TestScoreAnchorWeightScoring:
wrapper.max_act_value = 0
wrapper._score_qdq_cpu = None
- class _State:
- tunables = {}
+ def _fake_quant(weight, **kwargs):
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return ((weight / scale).round().clamp(-7, 7) * scale), scale, None
- class _Quantizer:
- def __init__(self):
- self.calls = 0
-
- def quantize(self, weight, **kwargs):
- self.calls += 1
- scale = weight.abs().amax().clamp(min=1e-4) / 7.0
- return (weight / scale).round().clamp(-7, 7) * scale
-
- wrapper.params = {}
- wrapper.weight_state = _State()
- wrapper.weight_quantizer = _Quantizer()
- wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(
- wrapper.weight_quantizer.quantize(weight, **kwargs)
- )
+ wrapper.weight_quant_func = _fake_quant
layer.weight.requires_grad = True
return wrapper, layer
def test_qdq_weight_quantizes_once_and_reuses_cache(self):
wrapper, layer = self._make_wrapper()
calls = {"n": 0}
- quantizer = wrapper.weight_quantizer
- original_quantize = quantizer.quantize
+ real_quant = wrapper.weight_quant_func
- def counting_quantize(*args, **kwargs):
+ def counting_quant(weight, **kwargs):
calls["n"] += 1
- return original_quantize(*args, **kwargs)
+ return real_quant(weight, **kwargs)
- quantizer.quantize = counting_quantize
+ wrapper.weight_quant_func = counting_quant
args = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
first, _, _ = wrapper._qdq_weight(*args)
second, _, _ = wrapper._qdq_weight(*args)
@@ -1411,20 +1396,11 @@ class TestScoreLinearRecompute:
wrapper.min_scale = torch.tensor(1.0)
wrapper.max_scale = torch.tensor(1.0)
- class _State:
- tunables = {}
+ def _fake_quant(weight, **kwargs):
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return (weight / scale).round().clamp(-7, 7) * scale, scale, None
- class _Quantizer:
- def quantize(self, weight, **kwargs):
- scale = weight.abs().amax().clamp(min=1e-4) / 7.0
- return (weight / scale).round().clamp(-7, 7) * scale
-
- wrapper.params = {}
- wrapper.weight_state = _State()
- wrapper.weight_quantizer = _Quantizer()
- wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(
- wrapper.weight_quantizer.quantize(weight, **kwargs)
- )
+ wrapper.weight_quant_func = _fake_quant
layer.weight.requires_grad = True
return wrapper, layer
diff --git a/test/unit/test_cpu/algorithms/test_awq.py b/test/unit/test_cpu/algorithms/test_awq.py
index c53773b5e..380d76661 100644
--- a/test/unit/test_cpu/algorithms/test_awq.py
+++ b/test/unit/test_cpu/algorithms/test_awq.py
@@ -492,7 +492,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -522,14 +522,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quantizer(params):
- return f"{params['data_type']}_{params['bits']}"
+ def fake_resolve_quant_funcs(params):
+ return f"{params['data_type']}_{params['bits']}", None
- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quantizer))
+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quant_func))
return weight
- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -1009,19 +1009,32 @@ class TestAWQUseV2ScaleSearch:
q._qdq_tool.configure(compressor)
assert q._qdq_tool.disable_opt_rtn is True
- params = {
- "bits": 4,
- "group_size": 128,
- "sym": True,
- "data_type": "int",
- "disable_opt_rtn": True,
- }
- assert q._qdq_tool.resolve_quantizer(params)._mode == "rtn"
compressor = self._make_compressor(self._signroundv2_config(data_type="mx_fp"))
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is True
+ def test_init_scale_dispatch_by_data_type(self):
+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
+
+ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
+ weight = torch.randn(32, 128)
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
+ assert init_scale is not None, dt
+ assert init_scale.shape[0] == weight_reshape.shape[0]
+
+ # Scalar imatrix sentinels mean uniform importance and must not enter
+ # the tensor reshape path.
+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
+ assert scalar_init_scale is not None
+ assert scalar_init_scale.shape == init_scale.shape
+
+ # asym int and *_dq are not part of the optimized init-scale path.
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
+
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4
diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.py
index c643d93f1..b1cee31b2 100644
--- a/test/unit/test_cpu/export/test_fake_format.py
+++ b/test/unit/test_cpu/export/test_fake_format.py
@@ -57,16 +57,21 @@ def test_fake_nvfp4_qdq_uses_saved_input_global_scale(monkeypatch):
def test_fake_evaluation_wrapper_uses_saved_input_global_scale():
captured_kwargs = {}
linear = torch.nn.Linear(16, 4)
+ linear.act_bits = 4
+ linear.act_group_size = 16
+ linear.scale_dtype = torch.float32
+ linear.q_scale_thresh = 1e-5
+ linear.act_data_type = "nv_fp4_with_static_gs"
linear.act_max_scale = torch.ones(1)
linear.act_min_scale = torch.ones(1)
linear.input_global_scale = torch.tensor([1.5], dtype=torch.float32)
- class ActivationQuantizer:
- def qdq(self, activation, **kwargs):
- captured_kwargs.update(kwargs)
- return activation
+ def quant_func(activation, **kwargs):
+ captured_kwargs.update(kwargs)
+ return activation, None, None
- wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())
+ linear.act_quant_func = quant_func
+ wrapper = WrapperWALayer(linear, enable_torch_compile=False)
wrapper(torch.randn(2, 16))
@@ -75,14 +80,20 @@ def test_fake_evaluation_wrapper_uses_saved_input_global_scale():
def test_fake_evaluation_wrapper_does_not_pass_global_scale_to_other_quantizers():
linear = torch.nn.Linear(16, 4)
+ linear.act_bits = 8
+ linear.act_group_size = 16
+ linear.scale_dtype = torch.float32
+ linear.q_scale_thresh = 1e-5
+ linear.act_data_type = "int"
linear.act_max_scale = torch.ones(1)
linear.act_min_scale = torch.ones(1)
- class ActivationQuantizer:
- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
- return activation
+ def quant_func(activation, *, global_scale=None, **kwargs):
+ assert global_scale is None
+ return activation, None, None
- wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())
+ linear.act_quant_func = quant_func
+ wrapper = WrapperWALayer(linear, enable_torch_compile=False)
wrapper(torch.randn(2, 16))
diff --git a/test/unit/test_cuda/algorithms/test_alg_ext.py b/test/unit/test_cuda/algorithms/test_alg_ext.py
index a618a6c04..936e1522d 100644
--- a/test/unit/test_cuda/algorithms/test_alg_ext.py
+++ b/test/unit/test_cuda/algorithms/test_alg_ext.py
@@ -35,6 +35,38 @@ class TestAlgExt:
yield
shutil.rmtree("runs", ignore_errors=True)
+ # def test_gguf_q2_k_s_uses_dq_wrapper_block(self, tiny_qwen_model_path):
+ # """Regression test: enable_alg_ext + gguf:q2_k_s must use DQWrapperLinear.
+ #
+ # gguf:q2_k_s overrides data_type to "int_asym_dq" at format-resolution
+ # time. The quantizer must be created *after* that override so that
+ # SignRoundV2 sees the final data_type and sets dq_wrapper_block
+ # (which wraps layers with DQWrapperLinear) instead of falling back to
+ # the plain wrapper_block (which produces WrapperLinear).
+ # """
+ # from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundDQWrapperLinear
+ #
+ # ar = AutoRound(
+ # tiny_qwen_model_path,
+ # bits=4,
+ # format="gguf:q2_k_s",
+ # iters=1,
+ # nsamples=1,
+ # seqlen=32,
+ # enable_alg_ext=True,
+ # )
+ # # post_init() runs the full pipeline (resolve_scheme → resolve_formats →
+ # # create_quantizer → ...). quantizer only exists afterwards.
+ # ar.post_init()
+ # quantizer = ar.alg_composer.block_quantizer
+ #
+ # assert quantizer.wrapper_block.keywords["wrapper_cls"] is SignRoundDQWrapperLinear, (
+ # f"Expected wrapper_block to use '{SignRoundDQWrapperLinear.__name__}', "
+ # f"got '{quantizer.wrapper_block.__name__}'. "
+ # "This likely means the quantizer was created before GGUF format "
+ # "overrides were applied (data_type was not yet 'int_asym_dq')."
+ # )
+
def test_int2_g64_asym_enable_alg_ext_keeps_config(self, tiny_qwen_model_path):
"""Regression test: asym int2/g64 keeps the requested tuning config."""
diff --git a/test/unit/test_cuda/algorithms/test_awq.py b/test/unit/test_cuda/algorithms/test_awq.py
index 106b7e3a7..da1459920 100644
--- a/test/unit/test_cuda/algorithms/test_awq.py
+++ b/test/unit/test_cuda/algorithms/test_awq.py
@@ -498,7 +498,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -528,14 +528,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quantizer(params):
- return f"{params['data_type']}_{params['bits']}"
+ def fake_resolve_quant_funcs(params):
+ return f"{params['data_type']}_{params['bits']}", None
- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quantizer))
+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quant_func))
return weight
- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -986,6 +986,27 @@ class TestAWQUseV2ScaleSearch:
compressor = self._make_compressor(block)
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False
+ def test_init_scale_dispatch_by_data_type(self):
+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
+
+ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
+ weight = torch.randn(32, 128)
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
+ assert init_scale is not None, dt
+ assert init_scale.shape[0] == weight_reshape.shape[0]
+
+ # Scalar imatrix sentinels mean uniform importance and must not enter
+ # the tensor reshape path.
+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
+ assert scalar_init_scale is not None
+ assert scalar_init_scale.shape == init_scale.shape
+
+ # asym int and *_dq are not part of the optimized init-scale path.
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
+
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4
diff --git a/test/unit/test_xpu/algorithms/test_awq.py b/test/unit/test_xpu/algorithms/test_awq.py
index 7558302cc..9bbb8cb1a 100644
--- a/test/unit/test_xpu/algorithms/test_awq.py
+++ b/test/unit/test_xpu/algorithms/test_awq.py
@@ -494,7 +494,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -524,14 +524,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quantizer(params):
- return f"{params['data_type']}_{params['bits']}"
+ def fake_resolve_quant_funcs(params):
+ return f"{params['data_type']}_{params['bits']}", None
- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quantizer))
+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quant_func))
return weight
- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -981,6 +981,27 @@ class TestAWQUseV2ScaleSearch:
compressor = self._make_compressor(block)
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False
+ def test_init_scale_dispatch_by_data_type(self):
+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
+
+ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
+ weight = torch.randn(32, 128)
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
+ assert init_scale is not None, dt
+ assert init_scale.shape[0] == weight_reshape.shape[0]
+
+ # Scalar imatrix sentinels mean uniform importance and must not enter
+ # the tensor reshape path.
+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
+ assert scalar_init_scale is not None
+ assert scalar_init_scale.shape == init_scale.shape
+
+ # asym int and *_dq are not part of the optimized init-scale path.
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
+
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4