PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Revert "refactor: decouple datatype quantization from algorithms (#2406)
Heng Guo86e97adf13cd
diff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.pyindex cfc921152..b3fd72535 100644--- a/auto_round/algorithms/base.py+++ b/auto_round/algorithms/base.py@@ -127,10 +127,6 @@ class BaseAlgorithm:def scheme(self) -> "QuantizationScheme | None":return self.__run_ctx.scheme if self.__run_ctx is not None else None- @property- def run_context(self) -> "QuantizationRunContext | None":- return self.__run_ctx-# ── Derived convenience properties ────────────────────────────────────────@propertydef model(self) -> "torch.nn.Module | None":diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 4ea01e8d8..725ead4fc 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -42,9 +42,8 @@ from auto_round.algorithms.config_resolver import (split_quantization_configs,)from auto_round.algorithms.utils import _has_nvfp4_layer-from auto_round.data_type.base import canonical_data_type, prepare_data_type_blockfrom auto_round.logger import logger-from auto_round.utils import check_to_quantized, clear_memory+from auto_round.utils import clear_memoryfrom auto_round.utils.device_manager import device_managerif TYPE_CHECKING: # avoid circular imports at runtime@@ -211,6 +210,10 @@ class AlgorithmComposer:", ".join(blockers),)+ if _has_nvfp4_layer(orchestrator):+ can_compile_block_forward = False+ logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")+# Bind compressor-level infrastructure (set before _build_quantizer is called).self.block_forward = (BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)@@ -242,15 +245,31 @@ class AlgorithmComposer:Returns a list of hook handles that the caller must remove when done."""- from auto_round.data_type.base import cache_activation_quantizer+ from auto_round.compressors.utils import is_nv_fp+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size++ is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)def collect_act_max(module, input, output):input = input[0] if isinstance(input, (tuple, list)) else inputif input.numel() == 0:return- quantizer = cache_activation_quantizer(module)- if quantizer is not None:- module.act_max = quantizer.observe(input, getattr(module, "act_max", None))+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fp+ input, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ if is_module_act_nv_fp:+ max_val = act_max.max()+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ return+ act_max = act_max.to(module.act_max.device)+ if is_module_act_nv_fp:+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ module.act_max = torch.max(act_max, module.act_max)def should_collect(name, module):from auto_round.compressors.utils import check_need_act_calibration@@ -259,8 +278,8 @@ class AlgorithmComposer:if isinstance(module, tuple(SUPPORTED_LAYER_TYPES)):return (hasattr(module, "act_dynamic")+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)and check_to_quantized(module)- and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False)))if hasattr(module, "bits"):act_dynamic = getattr(module, "act_dynamic", True)@@ -268,8 +287,8 @@ class AlgorithmComposer:act_bits = getattr(module, "act_bits", 16)return (module.bits <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)and check_to_quantized(module)- and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False)))return False@@ -311,19 +330,39 @@ class AlgorithmComposer:q_inputs: Optional list of quantized input tensors; used instead of``fp_inputs`` when provided."""- from auto_round.data_type.base import cache_activation_quantizer+ from auto_round.compressors.utils import is_nv_fp+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizetarget_input = q_inputs or fp_inputs- quantizer = cache_activation_quantizer(layer)- if quantizer is None:- return+ act_group_size = getattr(layer, "act_group_size")+ if act_group_size is None:+ act_group_size = layer.group_size+ act_data_type = getattr(layer, "act_data_type")+ if act_data_type is None:+ act_data_type = layer.data_type+ is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else Falsefor inp in target_input:if isinstance(inp, (tuple, list)):inp = inp[0]if inp.numel() == 0:continue- layer.act_max = quantizer.observe(inp, getattr(layer, "act_max", None))+ inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)+ act_max = torch.max(torch.abs(inp), dim=-1).values++ if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:+ layer.act_max = act_max+ if is_act_nv_fp_flag:+ max_val = act_max.max()+ layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ continue++ act_max = act_max.to(layer.act_max.device)+ if is_act_nv_fp_flag:+ max_val = torch.max(act_max.max(), layer.act_max.max())+ layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ layer.act_max = torch.max(act_max, layer.act_max)def need_quanted_input(self):for preprocessor in self.preprocessors:@@ -432,26 +471,12 @@ class AlgorithmComposer:act_data_type = self.scheme.act_data_type if self.scheme else data_typeif act_data_type is not None or not act_dynamic:from auto_round.compressors.utils import is_nv_fp+ from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.utils import set_amax_for_all_moe_layersif is_nv_fp(act_data_type) or not act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")-- runtimes_by_datatype = {}- for name, module in block.named_modules():- if not check_to_quantized(module):- continue- module_data_type = getattr(module, "data_type", data_type)- try:- canonical_id = canonical_data_type(module_data_type)- except LookupError:- continue- runtimes_by_datatype.setdefault(canonical_id, {})[name] = {- "data_type": module_data_type,- "group_size": getattr(module, "group_size", group_size),- }- for canonical_id, runtimes in runtimes_by_datatype.items():- prepare_data_type_block(canonical_id, block, runtimes)+ update_block_global_scale_if_needed(block, data_type, group_size)if q_inputs is not None and fp_inputs is not q_inputs:clear_memory(fp_inputs)diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex fe463fe20..70b3299e4 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -21,7 +21,7 @@ if TYPE_CHECKING:from auto_round.algorithms.base import BaseAlgorithmfrom auto_round.algorithms.quantization.config import QuantizationConfig-from auto_round.data_type.base import create_quantizer+from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.logger import loggerfrom auto_round.utils import (check_to_quantized,@@ -57,49 +57,6 @@ class BaseQuantizer(BaseAlgorithm):def can_compile_block_forward(self):return True- def build_weight_qdq(self, *, weight: torch.Tensor, default_qdq, parameters: dict[str, torch.Tensor]):- """Return the per-layer weight QDQ callable used for tuning and export.-- Algorithms that need different QDQ behavior may override this method.- The returned callable accepts ``weight`` and keyword ``materialize=False``- and returns ``WeightQuantizationResult``. Bind settings in a closure and- add trainable tensors to ``parameters`` before returning. This per-layer- dictionary is also used by the optimizer and best-parameter restoration.- Read its entries at call time; do not replace the dictionary.-- Args:- weight: Logical [out_features, in_features] weights on the tuning- device, including transposed Conv1D weights. Use only to create- parameters with the right shape/device; do not capture this- tensor in the returned closure.- default_qdq: Datatype function accepting weight, materialize=False,- and explicit datatype parameters. This builder binds the current- parameter dictionary to that function. Existing weight parameters- are datatype-dependent: value is a rounding offset, while- min_scale/max_scale adjust clipping bounds where supported.- parameters: Per-layer tensors collected for optimization. Add a- torch.nn.Parameter to train it; fixed settings belong in the- closure. Use distinct names that do not collide with wrapper- attributes. Read entries on each call to see restored best values.-- Returns:- Callable qdq(weight, *, materialize=False). With materialize=True,- its WeightQuantizationResult must contain scale/zero-point/metadata- required by the chosen datatype's exporter. Preserve autograd during- training.-- To customize, override this method and use ``weight`` to initialize- any new entries in ``parameters``. In the returned function, read those- entries and transform the input or explicitly pass the datatype options- you need. The implementation below is the default binding template.- """-- def qdq(weight, *, materialize=False):- """Use current parameters during training and restored best values at export."""- return default_qdq(weight, materialize=materialize, **parameters)-- return qdq-# ── Calibration hook registration ─────────────────────────────────────────def register_fp_input_forward_hooks(self, block: torch.nn.Module) -> list:"""Register hooks that fire during the reference (FP-input) block forward.@@ -137,22 +94,53 @@ class BaseQuantizer(BaseAlgorithm):if not check_to_quantized(module):continueis_quantized = True+ bits = getattr(module, "bits", None)+ group_size = getattr(module, "group_size", None)+ sym = getattr(module, "sym", None)+ data_type = getattr(module, "data_type", None)super_bits = getattr(module, "super_bits", None)super_group_size = getattr(module, "super_group_size", None)+ scale_dtype = self.scale_dtype+ quant_dtype = data_type+ if quant_dtype not in QUANT_FUNC_WITH_DTYPE:+ quant_dtype = f"{quant_dtype}_{'sym' if sym else 'asym'}"+ if not hasattr(self, "iters") or self.iters <= 0: # pylint: disable=E1101+ tmp_dtype = "rtn_" + quant_dtype+ if tmp_dtype in QUANT_FUNC_WITH_DTYPE:+ quant_dtype = tmp_dtype+ quant_func = QUANT_FUNC_WITH_DTYPE[quant_dtype]# float32 is used in RTN scale search; avoids caching a bf16 copy.weight_dtype = torch.float32 if super_group_size is not None else module.weight.dtype- quantizer = create_quantizer(module, disable_opt_rtn=True)+ quant_kwargs = {+ "bits": bits,+ "group_size": group_size,+ "super_bits": super_bits,+ "super_group_size": super_group_size,+ "scale_dtype": scale_dtype,+ }try:- weight = module.weight.to(dtype=weight_dtype, device=device_manager.device)- quantizer.initialize(weight)- quantizer.write_back(module, weight)+ weight, scale, zp = quant_func(+ module.weight.to(dtype=weight_dtype, device=device_manager.device),+ **quant_kwargs,+ )except torch.OutOfMemoryError:- logger.error(traceback.format_exc())- logger.warning("falling back to CPU")- weight = module.weight.to("cpu")- quantizer.initialize(weight)- quantizer.write_back(module, weight)- del weight, quantizer+ cuda_error_msg = traceback.format_exc()+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU")+ weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)+ except Exception:+ raise+ module.weight.data.copy_(weight.cpu())+ for param_name, val in zip(["scale", "zp"], [scale, zp]):+ if isinstance(val, dict):+ for k, v in val.items():+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())+ elif isinstance(val, torch.Tensor):+ setattr(module, param_name, val.cpu())+ else:+ setattr(module, param_name, val)+ del weight, scale, zpclear_memory()return is_quantized@@ -244,7 +232,6 @@ class BaseQuantizer(BaseAlgorithm):enable_torch_compile=self.compress_context.enable_torch_compile,disable_opt_rtn=disable_opt_rtn,iters=0,- weight_qdq_builder=self.build_weight_qdq,)layer = layer.unwrapper({})except torch.OutOfMemoryError:@@ -261,7 +248,6 @@ class BaseQuantizer(BaseAlgorithm):enable_round_tuning=False,enable_torch_compile=self.compress_context.enable_torch_compile,iters=0,- weight_qdq_builder=self.build_weight_qdq,)layer = layer.unwrapper({})except Exception:diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 278240c87..116e5b2e9 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -374,7 +374,6 @@ class SignRoundQuantizer(BaseQuantizer):self.enable_norm_bias_tuning,enable_torch_compile=self.compress_context.enable_torch_compile,device=device,- weight_qdq_builder=self.build_weight_qdq,)round_params = []@@ -504,6 +503,9 @@ class SignRoundQuantizer(BaseQuantizer):cache_budget,device,)+ if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):+ for n, m in block.named_modules():+ m.cur_iter = itotal_loss = 0global_indices = index_sampler.next_batch()if valid_token_mask:@@ -664,7 +666,6 @@ class SignRoundQuantizer(BaseQuantizer):enable_minmax_tuning=self.enable_minmax_tuning,enable_torch_compile=self.compress_context.enable_torch_compile,device=device,- weight_qdq_builder=self.build_weight_qdq,).to(device)round_params = []minmax_params = []@@ -672,7 +673,7 @@ class SignRoundQuantizer(BaseQuantizer):if "min" in key or "max" in key:minmax_params.append(wrapper_linear.params[key])else:- round_params.append(wrapper_linear.params[key])+ round_params.append(wrapper_linear.value)if len(round_params) + len(minmax_params) <= 0:dump_info = f"quantized {layer_name}"logger.info(dump_info)diff --git a/auto_round/algorithms/quantization/sign_roundv2/__init__.py b/auto_round/algorithms/quantization/sign_roundv2/__init__.pyindex b567f1ca6..4eb376f53 100644--- a/auto_round/algorithms/quantization/sign_roundv2/__init__.py+++ b/auto_round/algorithms/quantization/sign_roundv2/__init__.py@@ -12,8 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer+from auto_round.algorithms.quantization.sign_roundv2.quantizer import (+ SignRoundDQWrapperLinear,+ SignRoundOptimizedWrapperLinear,+ SignRoundV2Quantizer,+)__all__ = [+ "SignRoundDQWrapperLinear",+ "SignRoundOptimizedWrapperLinear","SignRoundV2Quantizer",]diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex 4a7cabc4e..eb9c977c6 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -13,20 +13,309 @@# limitations under the License.from contextlib import nullcontext+from functools import partialfrom typing import TYPE_CHECKING, Callable, Unionimport torch+import transformersfrom torch import autocastfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Configfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer-from auto_round.algorithms.registry import register_pipeline_member-from auto_round.data_type.base import canonical_data_type-from auto_round.logger import logger-from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantizedif TYPE_CHECKING:from auto_round.algorithms.composer import AlgorithmComposer+from auto_round.algorithms.registry import register_pipeline_member+from auto_round.data_type.gguf import (+ double_quant_tensor_sym_rtn,+ quant_tensor_gguf_asym_dq,+ quant_tensor_gguf_sym_dq,+ search_gguf_scale_min_asym,+ search_gguf_scale_min_sym,+)+from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym+from auto_round.data_type.utils import (+ get_optimized_quant_func,+ reshape_imatrix_for_weight,+ reshape_pad_tensor_by_group_size,+ revert_tensor_by_pad,+ round_ste,+ search_optimized_init_scale,+)+from auto_round.logger import logger+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal+from auto_round.wrapper import WrapperLinear, wrapper_block+++def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):+ """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ maxq = int(2.0**bits) - 1+ inverse_scale = get_reciprocal(scale)+ int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)+ qdq = (scale * int_w - wmin).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdq+++def _dq_sym_qdq(tensor, scale, bits, v=0):+ """Pure sym double-quant qdq math given precomputed scale (compilable)."""+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ group_size = 16+ super_group_size = 16+ maxq = int(2.0 ** (bits - 1))+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = tensor.nelement() // block_size+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ else:+ v_r = v+ zp = torch.full_like(scale, maxq)+ inverse_scale = get_reciprocal(scale)+ int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq+ qdq = (scale * (int_w - zp)).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdq+++def _named_wrapper_block(wrapper_cls, name: str):+ wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)+ wrapped.__name__ = name+ return wrapped+++def _dq_asym_group_size(bits: int) -> int:+ """Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""+ return 16 if bits == 2 else 32+++class SignRoundOptimizedWrapperLinear(WrapperLinear):+ minmax_scale_bound = (0.0, 2.0)++ def _init_tuning_params_and_quant_func(self):+ super()._init_tuning_params_and_quant_func()++ layer = self.orig_layer+ data_type = layer.data_type+ weight_reshape = self._prepare_init_scale_weight()+ imatrix = reshape_imatrix_for_weight(getattr(layer, "imatrix", None), weight_reshape, layer.group_size)++ self.init_scale = search_optimized_init_scale(+ weight_reshape, data_type, layer.bits, imatrix, self.q_scale_thresh+ )+ self.weight_quant_func = get_optimized_quant_func(data_type)+ if self.init_scale is None or self.weight_quant_func is None:+ raise ValueError(+ f"SignRound optimized path does not support data_type={data_type!r}; "+ "expected a symmetric int / mx / nv type."+ )++ self.data_type = data_type+ if hasattr(layer, "imatrix"):+ del layer.imatrix+ if self.enable_torch_compile:+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)++ def _prepare_init_scale_weight(self) -> torch.Tensor:+ """Return the group-reshaped weight that seeds the init-scale search.++ Reproduces the layout the quant func sees at tuning time: the (optionally+ transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in+ ``clip_as_init`` mode and stored a per-group clip range on the layer, the+ weight is clamped to that range first so the searched ``init_scale``+ already reflects the AWQ clip (``max_scale`` then tunes a coefficient on+ top of it).+ """+ layer = self.orig_layer+ weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight+ if isinstance(layer, transformers.pytorch_utils.Conv1D):+ weight = weight.t()+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)++ clip_min = getattr(layer, "awq_clip_min", None)+ clip_max = getattr(layer, "awq_clip_max", None)+ if clip_max is not None:+ clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)+ clip_min = (+ -clip_max+ if clip_min is None+ else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)+ )+ if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:+ weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)+ else:+ logger.warning_once(+ "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "+ "grouped weight shape %s."+ % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))+ )+ return weight_reshape+++class SignRoundDQWrapperLinear(WrapperLinear):+ minmax_scale_bound = (0.5, 1.5)++ def __init__(self, *args, **kwargs) -> None:+ if "enable_minmax_tuning" in kwargs:+ logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")+ kwargs["enable_minmax_tuning"] = False+ super().__init__(*args, **kwargs)+ self.prev_scale = None+ self.prev_wmin = None+ self.prev_d_scale = None+ self.prev_d_wmin = None++ def _init_tuning_params_and_quant_func(self):+ super()._init_tuning_params_and_quant_func()+ # The double-quant search path is data-dependent and kept un-compiled,+ # while ``weight_quant_func`` is the compilable pure-math half.+ self._is_dq_path = False+ self._dq_kind = None+ self.search_func = None+ if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:+ self._is_dq_path = True+ if self.orig_layer.data_type == "int_asym_dq":+ self.search_func = search_gguf_scale_min_asym+ self.weight_quant_func = _dq_asym_qdq+ self._dq_kind = "asym"+ else:+ self.search_func = search_gguf_scale_min_sym+ self.weight_quant_func = _dq_sym_qdq+ self._dq_kind = "sym"+ elif self.orig_layer.sym:+ self.weight_quant_func = quant_tensor_sym+ else:+ self.weight_quant_func = quant_tensor_asym+ self.data_type = self.orig_layer.data_type+ if self.enable_act_quant:+ self.act_quant_func = (+ quant_tensor_gguf_asym_dq+ if self.orig_layer.act_data_type == "int_asym_dq"+ else quant_tensor_gguf_sym_dq+ )+ if self.enable_torch_compile:+ self.act_quant_func = compile_func(self.act_quant_func, self.device)+ self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)+ if self.enable_torch_compile:+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)++ @torch.no_grad()+ def _run_search(self, weight, v):+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ bits = self.orig_layer.bits+ scale_dtype = self.orig_layer.scale_dtype+ imatrix = getattr(self.orig_layer, "imatrix", None)++ if self._dq_kind == "asym":+ group_size = _dq_asym_group_size(bits)+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ scale, wmin, d_scale, d_wmin = self.search_func(+ t,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ split_num=1,+ v=v_r,+ )+ return {+ "scale": scale.clone(),+ "wmin": wmin.clone(),+ "d_scale": d_scale.clone(),+ "d_wmin": d_wmin.clone(),+ }++ group_size = 16+ super_group_size = 16+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = t.nelement() // block_size+ t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ super_bits = 6 if bits == 3 else 8+ scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)+ scale = scale.to(scale_dtype)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)+ scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)+ scale = scale.unsqueeze(-1)+ return {"scale": scale.clone(), "d_scale": d_scale.clone()}++ def _qdq_weight(self, value, min_scale, max_scale):+ if not self._is_dq_path:+ return super()._qdq_weight(value, min_scale, max_scale)++ if self.orig_layer.bits >= 16:+ return self.orig_layer.weight, None, None+ min_bound, max_bound = self.minmax_scale_bound+ min_scale.data.clamp_(min_bound, max_bound)+ max_scale.data.clamp_(min_bound, max_bound)+ weight = self.orig_layer.weight+ if weight.device.type == "meta":+ weight = self.orig_layer.get_weight().to(self.device)+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):+ weight = weight.t()++ iter_v = getattr(self, "cur_iter", 0)+ need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)+ if need_search:+ params = self._run_search(weight, value)+ self.prev_scale = params["scale"]+ self.prev_d_scale = params["d_scale"]+ if self._dq_kind == "asym":+ self.prev_wmin = params["wmin"]+ self.prev_d_wmin = params["d_wmin"]+ else:+ params = {+ "scale": self.prev_scale.detach(),+ "d_scale": self.prev_d_scale.detach(),+ }+ if self._dq_kind == "asym":+ params["wmin"] = self.prev_wmin.detach()+ params["d_wmin"] = self.prev_d_wmin.detach()++ bits = self.orig_layer.bits+ if self._dq_kind == "asym":+ group_size = _dq_asym_group_size(bits)+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ params["wmin"],+ bits,+ group_size,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}+ else:+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ bits,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))++ weight_q = weight_q.to(weight.dtype)+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):+ weight_q = weight_q.t()+ return weight_q, scale_out, zp_out@register_pipeline_member(SignRoundV2Config)@@ -42,10 +331,18 @@ class SignRoundV2Quantizer(SignRoundQuantizer):"""Model-level setup: initialise scheme-dependent state once before block iteration."""super().prepare_run(composer=composer)- canonical_dtype = canonical_data_type(self.scheme.data_type)- is_optimized_family = canonical_dtype.startswith(("int_", "mx_", "nv_", "nvfp"))- if self.scheme.sym and self.scheme.super_group_size is None and is_optimized_family:- if self.scheme.bits > 2 and canonical_dtype.startswith("int_"):+ if (+ self.scheme.sym+ and self.scheme.super_group_size is None+ and (+ self.scheme.data_type.startswith("int")+ or self.scheme.data_type.startswith("mx")+ or self.scheme.data_type.startswith("nv")+ )+ ):+ if self.scheme.bits > 2 and not (+ self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")+ ):logger.warning_once("algorithm extension has only undergone limited validation on ""W2A16,INT4, MXFP4 and NVFP4; use with caution."@@ -54,9 +351,10 @@ class SignRoundV2Quantizer(SignRoundQuantizer):self._use_outlier_suppressed_loss = Trueelse:self._use_outlier_suppressed_loss = False- # QDQ ownership stays with the generic wrapper. Datatype adapters- # select optimized and double-quant behavior from the layer request;- # SignRoundV2 only owns its loss policy.+ self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")++ if self.scheme.data_type.endswith("dq"):+ self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")def can_compile_block_forward(self):return Falsediff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.pyindex d3822c2f6..5ef7e1d71 100644--- a/auto_round/algorithms/quantization/utils.py+++ b/auto_round/algorithms/quantization/utils.py@@ -14,7 +14,8 @@import torch-from auto_round.data_type.base import cache_activation_quantizer+from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp+from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized@@ -25,15 +26,28 @@ def register_act_max_hooks(quantizer, model):input = input[0]if input.numel() == 0:return- activation_quantizer = cache_activation_quantizer(module)- if activation_quantizer is not None:- module.act_max = activation_quantizer.observe(input, getattr(module, "act_max", None))+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp+ input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ if is_module_act_nv_fp:+ max_val = act_max.max()+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ act_max = act_max.to(module.act_max.device)+ if is_module_act_nv_fp:+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ module.act_max = torch.max(act_max, module.act_max)hook_handles = []if isinstance(model, SUPPORTED_LAYER_TYPES):if (hasattr(model, "act_dynamic")- and getattr(cache_activation_quantizer(model), "requires_calibration", False)+ and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)and check_to_quantized(model)):hook_handles.append(model.register_forward_hook(get_act_max_hook))@@ -42,7 +56,7 @@ def register_act_max_hooks(quantizer, model):for name, module in model.named_modules():if (hasattr(module, "act_dynamic")- and getattr(cache_activation_quantizer(module), "requires_calibration", False)+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)and check_to_quantized(module)):hook_handles.append(module.register_forward_hook(get_act_max_hook))@@ -50,14 +64,12 @@ def register_act_max_hooks(quantizer, model):if name in quantizer.layer_config:config = quantizer.layer_config[name]- module.act_dynamic = config.get("act_dynamic", True)- module.act_data_type = config.get("act_data_type", "int_sym")- module.act_bits = config.get("act_bits", 16)- module.act_group_size = config.get("act_group_size", quantizer.act_group_size)- module.act_sym = config.get("act_sym", True)+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_bits", 16)if (config["bits"] <= 8- and getattr(cache_activation_quantizer(module), "requires_calibration", False)+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)and check_to_quantized(config)):hook_handles.append(module.register_forward_hook(get_act_max_hook))diff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.pyindex 40c839419..2c8332e5e 100644--- a/auto_round/algorithms/transforms/awq/base.py+++ b/auto_round/algorithms/transforms/awq/base.py@@ -730,7 +730,7 @@ class AWQTransform(BasePreprocessor):# the grid-search loop. Normal AWQ flow requires one mapping to have# compatible quant params, but keeping this per-layer avoids hidden# coupling to the first layer and makes direct calls robust.- bl_quantizers = {bl: self._qdq_tool.resolve_quantizer(bl_params[bl]) for bl in mapping.balance_layers}+ bl_quant_funcs = {bl: self._qdq_tool.resolve_quant_funcs(bl_params[bl]) for bl in mapping.balance_layers}best_error = float("inf")best_scales = None@@ -751,11 +751,12 @@ class AWQTransform(BasePreprocessor):# de-smoothed result back, so the parent forward below sees the# weights the layer would actually compute with.for bl in mapping.balance_layers:- quantizer = bl_quantizers[bl]+ quant_func, opt_quant_func = bl_quant_funcs[bl]w_qdq = self._qdq_tool.qdq(orig_state[bl] * scales_view,bl_params[bl],- quantizer=quantizer,+ quant_func=quant_func,+ opt_quant_func=opt_quant_func,imatrix=getattr(bl, "imatrix", None),)bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)@@ -766,12 +767,13 @@ class AWQTransform(BasePreprocessor):else:total_loss = 0.0for bl in mapping.balance_layers:- quantizer = bl_quantizers[bl]+ quant_func, opt_quant_func = bl_quant_funcs[bl]w_orig = orig_weights[bl].to(device)w_qdq = self._qdq_tool.qdq(w_orig * scales_view,bl_params[bl],- quantizer=quantizer,+ quant_func=quant_func,+ opt_quant_func=opt_quant_func,imatrix=getattr(bl, "imatrix", None),)total_loss += (w_orig - w_qdq / scales_view).pow(2).sum().item()@@ -1030,8 +1032,8 @@ class AWQTransform(BasePreprocessor):# group size and drop super-block (double-quant) params, which the clip# path does not apply.clip_params = {**params, "group_size": gs, "super_bits": None, "super_group_size": None}- quantizer = self._qdq_tool.resolve_quantizer(clip_params)- if quantizer is None:+ quant_func, _ = self._qdq_tool.resolve_quant_funcs(clip_params)+ if quant_func is None:return Nonefeat = input_feat.to(device).reshape(-1, in_features)@@ -1070,7 +1072,7 @@ class AWQTransform(BasePreprocessor):max_val = org_max_val * shrinkcur_w = torch.clamp(w_b, min_val, max_val)cur_w_flat = cur_w.reshape(cur_w.shape[0], n_group * gs)- q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quantizer=quantizer).reshape(cur_w.shape)+ q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quant_func=quant_func).reshape(cur_w.shape)cur_out = (feat * q_w).sum(dim=-1)err = (cur_out - org_out).pow(2).mean(dim=1).view(min_errs.shape)improved = err < min_errsdiff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.pyindex e0bb05ed5..2a51956a2 100644--- a/auto_round/algorithms/transforms/awq/qdq.py+++ b/auto_round/algorithms/transforms/awq/qdq.py@@ -28,15 +28,19 @@ Usage (the same unified QDQ serves both the scale grid-search and clip-search)::tool.configure(compressor) # opt_rtn / v2 flagsparams = tool.resolve_params(layer) # per-layer scheme- quantizer = tool.resolve_quantizer(params) # dispatch once, reuse in loop- w_qdq = tool.qdq(weight, params, quantizer=quantizer)+ qf, opt_qf = tool.resolve_quant_funcs(params) # dispatch once, reuse in loop+ w_qdq = tool.qdq(weight, params, quant_func=qf, opt_quant_func=opt_qf)"""from __future__ import annotationsimport torch-from auto_round.data_type.base import create_quantizer+from auto_round.data_type.utils import (+ compute_optimized_init_scale,+ get_optimized_quant_func,+ get_quant_func,+)class QDQTool:@@ -94,7 +98,7 @@ class QDQTool:"""Resolve the per-layer weight-quant params (``layer_config`` + fallbacks).Single source of every scheme field the QDQ needs, so callers pass the- returned dict straight to :meth:`resolve_quantizer` and :meth:`qdq`.+ returned dict straight to :meth:`resolve_quant_funcs` and :meth:`qdq`.``super_bits`` / ``super_group_size`` are the GGUF double-quantsuper-block params and are ``None`` for non-GGUF schemes."""@@ -109,12 +113,25 @@ class QDQTool:"super_group_size": cfg.get("super_group_size", None),}- def resolve_quantizer(self, params: dict):- """Create one configured quantizer to reuse throughout a weight search."""- return create_quantizer(- {**params, "scale_dtype": torch.float32},+ def resolve_quant_funcs(self, params: dict):+ """Dispatch ``(quant_func, opt_quant_func)`` for ``params``.++ ``opt_quant_func`` is non-``None`` only when the SignRoundV2 optimized+ init-scale path applies (it depends solely on ``data_type`` / ``sym``).+ Hoisting this out of a search loop avoids repeated dispatch.+ """+ quant_func, _ = get_quant_func(+ params["data_type"],+ params["bits"],+ params["sym"],disable_opt_rtn=params["disable_opt_rtn"],+ group_size=params["group_size"],+ iters=0,)+ opt_quant_func = None+ if self.use_v2_scale_search and params["sym"]:+ opt_quant_func = get_optimized_quant_func(params["data_type"])+ return quant_func, opt_quant_func# ── the unified QDQ for AWQ search ──────────────────────@torch.no_grad()@@ -123,11 +140,48 @@ class QDQTool:weight: torch.Tensor,params: dict,*,- quantizer=None,+ quant_func=None,+ opt_quant_func=None,imatrix: torch.Tensor | None = None,) -> torch.Tensor:- """Simulate quantization for a search candidate without changing stored weights."""- if quantizer is None:- quantizer = self.resolve_quantizer(params)- quantizer.initialize(weight, imatrix=imatrix)- return quantizer.quantize(weight)+ """Quantize-dequantize ``weight`` under the resolved ``params``.++ The single QDQ entry point for every scheme: ``bits`` / ``group_size`` /+ ``sym`` / ``data_type`` plus the optional GGUF super-block params drive+ the dispatched ``quant_func``; a non-``None`` ``opt_quant_func`` enables+ the SignRoundV2 optimized init-scale path for this weight. Used for+ both the scale grid-search and the clip-search -- reference/loss only,+ it never mutates stored weights.+ """+ if quant_func is None:+ quant_func, opt_quant_func = self.resolve_quant_funcs(params)+ if quant_func is None:+ raise RuntimeError(+ "QDQTool: no quantization function resolved for "+ f"data_type={params['data_type']}, bits={params['bits']}, "+ f"sym={params['sym']}, group_size={params['group_size']}."+ )++ quant_kwargs = {+ "bits": params["bits"],+ "group_size": params["group_size"],+ "data_type": params["data_type"],+ "sym": params["sym"],+ }+ if params.get("super_bits") is not None:+ quant_kwargs["super_bits"] = params["super_bits"]+ if params.get("super_group_size") is not None:+ quant_kwargs["super_group_size"] = params["super_group_size"]++ active_quant_func = quant_func+ if opt_quant_func is not None:+ init_scale = compute_optimized_init_scale(+ weight, params["data_type"], params["bits"], params["group_size"], imatrix=imatrix+ )+ if init_scale is not None:+ quant_kwargs["init_scale"] = init_scale+ quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)+ active_quant_func = opt_quant_func++ qdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)+ return qdq_weightdiff --git a/auto_round/algorithms/transforms/hadamard/patch.py b/auto_round/algorithms/transforms/hadamard/patch.pyindex 6971cd304..e3f5f9f95 100644--- a/auto_round/algorithms/transforms/hadamard/patch.py+++ b/auto_round/algorithms/transforms/hadamard/patch.py@@ -24,8 +24,6 @@ Each patch is idempotent: calling it twice has no effect.from __future__ import annotations-import math-import torchimport transformers@@ -77,8 +75,6 @@ def patch_wrapperlinear_to_apply_transform(if is_conv1d:new_weight = new_weight.t().contiguous()self.orig_layer.weight.data.copy_(new_weight)- logical_weight = new_weight.t() if is_conv1d else new_weight- self.weight_quantizer.refresh(logical_weight, imatrix=getattr(self.orig_layer, "imatrix", None))self.applied_weight_hadamard = Truereturn _orig_qdq_weight(self, value, min_scale, max_scale)@@ -110,18 +106,16 @@ def patch_wrapperwalayer_forward_to_apply_transform(_orig_forward = WrapperWALayer.forwarddef _forward_patched(self, x):- x = inp_transform(x)- import auto_round.envs as envs-act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None- act_scale = envs.AR_ACT_SCALE- max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale- min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale- x = self.activation_quantizer.qdq(+ x = inp_transform(x)+ x, _, _ = self.orig_layer.act_quant_func(x,- observed_max=act_max,- min_scale=min_scale,- max_scale=max_scale,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ tensor_max=act_max,)return self.orig_layer.forward(x)diff --git a/auto_round/algorithms/transforms/svdquant/residual.py b/auto_round/algorithms/transforms/svdquant/residual.pyindex aa3515b3e..461f6763b 100644--- a/auto_round/algorithms/transforms/svdquant/residual.py+++ b/auto_round/algorithms/transforms/svdquant/residual.py@@ -26,7 +26,7 @@ from dataclasses import dataclassimport torch-from auto_round.data_type.base import create_quantizer, quantize_activation+from auto_round.data_type.utils import get_quant_func_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})@@ -108,12 +108,16 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.requested_dtype = values["data_type"]if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:requested_dtype = f"{requested_dtype}_rceil"- quantizer = create_quantizer(- {**values, "data_type": requested_dtype, "scale_dtype": tensor.dtype},+ quant_func, resolved_dtype = get_quant_func(+ dtype=requested_dtype,+ bits=values["bits"],+ sym=values["sym"],disable_opt_rtn=True,+ group_size=values["group_size"],+ iters=0,)- quantizer.initialize(tensor)- resolved_base_dtype = requested_dtype.removesuffix("_rceil")+ logical_dtype = resolved_dtype.removeprefix("rtn_")+ resolved_base_dtype = logical_dtype.removesuffix("_rceil")if (resolved_base_dtype in _MXFP4_ALIASESand values["bits"] == 4@@ -128,7 +132,12 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.f"got group_size={values['group_size']!r}.")- qdq = quantizer.quantize(tensor)+ qdq, _, _ = quant_func(+ tensor=tensor,+ bits=values["bits"],+ group_size=values["group_size"],+ data_type=logical_dtype,+ )if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:raise ValueError(f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "@@ -153,13 +162,7 @@ def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch@torch.inference_mode()def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:"""Apply deployment-compatible dynamic activation quantize-dequantize."""- values = _validate_scheme_values(scheme)- qdq = quantize_activation(activation, values)- if qdq.shape != activation.shape or qdq.dtype != activation.dtype or qdq.device != activation.device:- raise ValueError("Activation RTN QDQ must preserve shape, dtype, and device.")- if not torch.isfinite(qdq).all():- raise ValueError("Activation RTN QDQ produced non-finite values.")- return qdq+ return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")@torch.inference_mode()diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 8d383e311..8d23ff152 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -44,7 +44,13 @@ from auto_round.auto_scheme.utils import (remove_quant_scheme,)from auto_round.calib_dataset import get_dataloader-from auto_round.data_type.base import canonical_data_type+from auto_round.data_type.gguf import (+ quant_tensor_gguf_asym_dq,+ quant_tensor_gguf_sym_dq,+ search_gguf_scale_min_asym,+ search_gguf_scale_min_sym,+)+from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.logger import loggerfrom auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_from auto_round.schemes import QuantizationScheme, preset_name_to_scheme@@ -280,6 +286,278 @@ class AutoSchemeWrapperLinear(WrapperLinear):return qdq_w, 1.0, None+class AutoSchemeWrapperLinearIMatrix(WrapperLinear):+ """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""++ def __init__(+ self,+ orig_layer,+ enable_minmax_tuning=True,+ enable_norm_bias_tuning=False,+ device="cpu",+ enable_round_tuning=True,+ need_weight_grad=False,+ enable_torch_compile=True,+ **kwargs,+ ):+ """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""+ super().__init__(+ orig_layer,+ enable_minmax_tuning,+ enable_norm_bias_tuning,+ device,+ enable_round_tuning,+ enable_torch_compile=enable_torch_compile,+ **kwargs,+ )+ self.act_score = 0.0+ self.avg_act_score = 0.0+ self.act_cnt = 0.0+ self.weight_score = 0.0+ self.mix_score = 0.0+ self.super_qdq_func = super()._qdq_weight+ self.act_qdq_func = super()._qdq_act+ self.max_act_value = 0+ self.need_weight_grad = need_weight_grad+ self.grad_mode = False+ if self.need_weight_grad:+ self.orig_layer.weight.requires_grad = True+ self.weight_search_quant_func, _ = get_quant_func(+ orig_layer.data_type,+ orig_layer.bits,+ orig_layer.sym,+ disable_opt_rtn=False,+ group_size=orig_layer.group_size,+ iters=0,+ )+ self._custom_score_forward = False+ self.post_init_qdqw(device)++ @torch.no_grad()+ def post_init_qdqw(self, device):+ """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,+ registering a backward hook on it to accumulate ``weight_score``.+ """+ qdq_w, _, _ = self.weight_search_quant_func(+ self.orig_layer.weight.to(device),+ bits=self.orig_layer.bits,+ group_size=self.orig_layer.group_size,+ v=torch.tensor(0, device=device),+ min_scale=torch.tensor(1.0, device=device),+ max_scale=torch.tensor(1.0, device=device),+ scale_dtype=self.orig_layer.scale_dtype,+ data_type=self.data_type,+ q_scale_thresh=self.q_scale_thresh,+ imatrix=self.orig_layer.imatrix.to(device) if hasattr(self.orig_layer, "imatrix") else None,+ global_scale=getattr(self, "weight_global_scale", None),+ )++ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))++ def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ self.mix_score = self.weight_score + self.act_score+ return None++ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)++ self.qdq_w.register_hook(save_grad)++ def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.+ """+ if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:+ return x, 1.0, None++ qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)+ if self.grad_mode:+ with torch.no_grad():+ max_act_value = torch.abs(x).max()+ self.max_act_value = max_act_value+ if max_act_value != 0:+ self.act_cnt += 1+ x_diff = (x - qdq_x).to("cpu")++ def save_grad(grad):+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""+ if max_act_value == 0:+ if torch.abs(grad).max() != 0:+ raise ValueError+ """+ this ut will cause NAN issue sometimes, need to investigate+ @multi_card+ def test_multi_card(self):+ model_name = "/models/Qwen3-8B"+ """+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():+ self.act_cnt -= 1+ return None++ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()+ self.mix_score = self.weight_score + self.act_score+ return None++ if qdq_x.requires_grad:+ qdq_x.register_hook(save_grad)+ return qdq_x, scale, zp++ def _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""+ return self.qdq_w, 1.0, None+++class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):+ """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""++ def __init__(+ self,+ orig_layer,+ enable_minmax_tuning=True,+ enable_norm_bias_tuning=False,+ device="cpu",+ enable_round_tuning=True,+ need_weight_grad=False,+ **kwargs,+ ):+ """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""+ super().__init__(+ orig_layer,+ enable_minmax_tuning,+ enable_norm_bias_tuning,+ device,+ enable_round_tuning,+ need_weight_grad,+ **kwargs,+ )+ self._custom_score_forward = False+ self.post_init_qdqw(device)++ @torch.no_grad()+ def post_init_qdqw(self, device):+ """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,+ registering a backward hook on it to accumulate ``weight_score``.+ """+ qdq_w, scale, zp = self.super_qdq_func(+ torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)+ )+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))++ def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ # TODO strange, grad could be in CPU+ self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()+ self.mix_score = self.weight_score + self.act_score+ return None++ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)+ self.qdq_w.register_hook(save_grad)++ def _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""+ return self.qdq_w, 1.0, None+++class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):+ """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant+ search (``_init_scale``).+ """++ def __init__(+ self,+ orig_layer,+ enable_minmax_tuning=True,+ enable_norm_bias_tuning=False,+ device="cpu",+ enable_round_tuning=True,+ need_weight_grad=False,+ enable_torch_compile=True,+ **kwargs,+ ):+ """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to+ build ``qdq_w``.+ """+ super().__init__(+ orig_layer,+ enable_minmax_tuning,+ enable_norm_bias_tuning,+ device,+ enable_round_tuning,+ need_weight_grad,+ enable_torch_compile=enable_torch_compile,+ **kwargs,+ )+ self._custom_score_forward = False+ self.post_init_qdqw(device)++ @torch.no_grad()+ def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher+ """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer+ ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.+ """+ qdq_w = self._init_scale(device).detach()+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))++ def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ self.mix_score = self.weight_score + self.act_score+ return None++ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)++ self.qdq_w.register_hook(save_grad)++ @torch.no_grad()+ def _init_scale(self, device):+ """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in+ [2,3,4,5,6], returned in the original weight dtype.+ """+ tensor = self.orig_layer.weight.data.to(device)+ bits = self.orig_layer.bits+ scale_dtype = self.orig_layer.scale_dtype+ imatrix = self.orig_layer.imatrix.to(tensor.device)+ orig_dtype = tensor.dtype+ if self.orig_layer.bits in [2, 4, 5]:+ group_size = 16 if bits == 2 else 32+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)++ qdq_w, _, _ = quant_tensor_gguf_asym_dq(+ tensor=tensor,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ scale=scale,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ )+ elif bits in [3, 6]:+ qdq_w, _, _ = quant_tensor_gguf_sym_dq(+ tensor=tensor,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ split_num=1,+ )+ else:+ raise ValueError("bits must be in [2,3,4,5,6]")+ return qdq_w.to(orig_dtype)++ def _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""+ return self.qdq_w, 1.0, None++def register_imatrix_hook(model):"""Registers hooks to accumulate activation squared norms into `imatrix`."""@@ -536,13 +814,7 @@ def _replay_retain_graph(block_module) -> bool:"""for _, module in block_module.named_modules():data_type = getattr(module, "data_type", None)- if not isinstance(data_type, str):- continue- try:- canonical_id = canonical_data_type(data_type)- except LookupError:- continue- if canonical_id.startswith("mx_"):+ if isinstance(data_type, str) and data_type.startswith("mx"):return Truereturn False@@ -952,6 +1224,15 @@ def get_score_for_scheme(if hasattr(m, "bias") and m.bias is not None:m.bias.requires_grad = False+ has_imatrix = False+ for name in quant_layer_names:+ if name in fixed_layer_scheme.keys():+ continue+ m = get_module(model, name)+ if hasattr(m, "imatrix") and m.imatrix is not None:+ has_imatrix = True+ break+for name in quant_layer_names:if offload_context is not None:offload_context.ensure_loaded(model, name)@@ -972,6 +1253,14 @@ def get_score_for_scheme(m.scale_dtype = torch.bfloat16WrapperLayer = AutoSchemeWrapperLinear+ # if has_imatrix: # no better result+ # WrapperLayer = AutoSchemeWrapperLinearIMatrix+ if hasattr(m, "super_group_size") and m.super_group_size is not None:+ if has_imatrix:+ WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix+ else:+ WrapperLayer = AutoSchemeWrapperLinearForGGUFK+with torch.no_grad():if low_gpu_mem_usage:device = m.tuning_device if hasattr(m, "tuning_device") else major_devicediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 7beb61ccf..4e834ba94 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1300,34 +1300,8 @@ class BaseOrchestrator(object):return get_torch_compile_off_reason(model)def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:- """Apply torch.compile disabling rules for the current compressor state.-- This is intentionally kept beside the compressor state it reads. The- rules are not reusable policy: every input comes from this instance,- and preserving that context makes the precedence easy to audit.- """- from auto_round.algorithms.quantization.rtn.config import RTNConfig-- _, is_valid_act_static = self._get_torch_compile_guard_state()- reason = None- if enable_torch_compile:- if is_valid_act_static:- reason = "activation is static"- else:- reason = self._torch_compile_unsupported_arch_reason()- user_specified = getattr(self, "_torch_compile_user_specified", False)- is_auto_scheme = getattr(self, "is_auto_scheme", False) or isinstance(- getattr(self, "scheme", None), AutoScheme- )- if reason is None and not user_specified and not is_auto_scheme:- if isinstance(self.quantize_config, RTNConfig):- reason = "RTN/OPT-RTN quantizes each layer in a single pass"- else:- iters = getattr(self.quantize_config, "iters", None)- if iters is not None and iters < MIN_ITERS_FOR_TORCH_COMPILE:- reason = f"`iters`={iters} is below {MIN_ITERS_FOR_TORCH_COMPILE}"-- self.enable_torch_compile = enable_torch_compile and reason is None+ """Apply torch.compile disabling rules for the current compressor state."""+ self.enable_torch_compile = enable_torch_compile# Why compilation ended up off, used by ``_log_torch_compile_state``. When the# incoming value is already False, keep the reason recorded by the earlier# precheck pass instead of dropping it.@@ -1339,9 +1313,33 @@ class BaseOrchestrator(object):or getattr(self, "_torch_compile_default_off_reason", None)))- if reason is not None:- self._torch_compile_off_reason = reason- logger.warning_once("reset enable_torch_compile to `False` as %s", reason)+ _, is_valid_act_static = self._get_torch_compile_guard_state()++ # On HPU, we rely on torch.compile to speed up the model execution.+ if self.enable_torch_compile and is_valid_act_static:+ self.enable_torch_compile = False+ self._torch_compile_off_reason = "activation is static"+ logger.warning_once("reset enable_torch_compile to `False` as activation is static")++ # Architecture-level hard block (DeepSeek / GLM-5.3-Flash DSA families). These+ # hit dynamo's recompile_limit and cannot be overridden by an explicit+ # ``enable_torch_compile=True``.+ if self.enable_torch_compile:+ arch_reason = self._torch_compile_unsupported_arch_reason()+ if arch_reason is not None:+ self.enable_torch_compile = False+ self._torch_compile_off_reason = arch_reason+ logger.warning_once("reset enable_torch_compile to `False` as %s", arch_reason)++ if self.enable_torch_compile:+ disabled_reason = self._torch_compile_disabled_reason()+ if disabled_reason is not None:+ self.enable_torch_compile = False+ self._torch_compile_off_reason = disabled_reason+ logger.warning_once(+ "reset enable_torch_compile to `False` as %s, " "so compilation cost would outweigh its benefit",+ disabled_reason,+ )def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:"""Apply early torch.compile adjustments before scheme resolution.diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.pyindex d790485fb..48df479ed 100644--- a/auto_round/compressors/layer_config_resolver.py+++ b/auto_round/compressors/layer_config_resolver.py@@ -38,31 +38,6 @@ from auto_round.utils import (from auto_round.utils.model import get_module-def normalize_layer_config_entry(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:- """Resolve one public layer override through the existing preset machinery."""- from auto_round.schemes import QuantizationScheme, preset_name_to_scheme-- scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)- if isinstance(item, str):- config = asdict(preset_name_to_scheme(item.upper()))- elif isinstance(item, QuantizationScheme):- config = asdict(item)- elif isinstance(item, dict):- item = {k: v for k, v in item.items() if k != "in_blocks"}- scheme_name = item.pop("scheme", None)- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}- invalid = set(item) - set(scheme_keys + ("fixed_by_user",))- if invalid:- raise ValueError(f"Invalid keys {invalid} in layer_config for '{layer_name}'. Allowed keys: {scheme_keys}")- config.update(item)- else:- raise TypeError(- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "- "Expected str, dict, or QuantizationScheme."- )- return {key: value for key, value in config.items() if value is not None}--def apply_layer_config_special_cases(layer_config,model,@@ -195,6 +170,34 @@ def _resolve_layer_config_presets(bits-inference, and default-filling."""from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:+ """Convert config entry into dict and validate keys."""+ if isinstance(item, str):+ config = asdict(preset_name_to_scheme(item.upper()))+ elif isinstance(item, QuantizationScheme):+ config = asdict(item)+ elif isinstance(item, dict):+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;+ # silently drop it before validation.+ item = {k: v for k, v in item.items() if k != "in_blocks"}+ scheme_name = item.pop("scheme", None)+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))+ if invalid:+ raise ValueError(+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"+ )+ config.update(item)+ else:+ raise TypeError(+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "+ f"Expected str, dict, or QuantizationScheme."+ )+ # Clean up+ config = {k: v for k, v in config.items() if v is not None}+ config["fixed_by_user"] = True+ return config+extra_scheme_keys = ("scale_dtype",)scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)layer_config = copy.deepcopy(layer_config) or {}@@ -215,7 +218,7 @@ def _resolve_layer_config_presets(}# 2. normalize- layer_config = {k: {**normalize_layer_config_entry(v, k), "fixed_by_user": True} for k, v in layer_config.items()}+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}# entries whose sym was set by the user (dict key or a QuantizationScheme# object) rather than inherited from the default scheme belowexplicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex 58bd9076a..58c746470 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -15,7 +15,7 @@import auto_round.data_type.intimport auto_round.data_type.mxfpimport auto_round.data_type.fp8-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEimport auto_round.data_type.w4fp8from auto_round.data_type.utils import (get_quant_func,@@ -24,8 +24,3 @@ from auto_round.data_type.utils import ()import auto_round.data_type.nvfpimport auto_round.data_type.gguf-from auto_round.data_type.base import (- DataTypeQuantizer,- canonical_data_type,- create_quantizer,-)diff --git a/auto_round/data_type/base.py b/auto_round/data_type/base.pydeleted file mode 100644index c34d336fb..000000000--- a/auto_round/data_type/base.py+++ /dev/null@@ -1,313 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0--"""Small public boundary between quantization algorithms and data types.--Algorithms describe *what* a layer needs quantized. This module resolves the-requested datatype and delegates all datatype-specific work to that datatype's-class. It deliberately contains no quantization math.-"""--from collections.abc import Mapping-from dataclasses import dataclass, replace-from types import SimpleNamespace-from typing import Any, Callable--import torch--# Low-level tensor primitives remain functions. Quantizer classes below own-# their lifecycle and are the only interface used by algorithms and wrappers.-QUANT_FUNC_WITH_DTYPE: dict[str, Callable] = {}-_QUANTIZER_CLASSES: dict[str, type] = {}-_CANONICAL_DATA_TYPES: dict[str, str] = {}---def _normalize_data_type(data_type: str) -> str:- """Normalize spelling differences in datatype names for lookup."""- return data_type.lower().replace("-", "").replace("_", "")---def register_dtype(names):- """Register a low-level tensor quantization primitive by datatype name."""-- def register(function):- for name in (names,) if isinstance(names, str) else names:- QUANT_FUNC_WITH_DTYPE[name] = function- return function-- return register---def register_quantizer(canonical_id: str, *, aliases: tuple[str, ...] = ()):- """Register a datatype class under its canonical name and aliases.-- Datatype modules use this decorator at import time. The mapping exists- solely to turn user-facing aliases such as ``"int"`` into the class that- owns their implementation.- """-- def register(quantizer_class):- names = (canonical_id, *aliases)- normalized = tuple(dict.fromkeys(_normalize_data_type(name) for name in names))- duplicate = next((name for name in normalized if name in _QUANTIZER_CLASSES), None)- if duplicate is not None:- raise ValueError(f"Datatype name {duplicate!r} is already registered")- for name in normalized:- _QUANTIZER_CLASSES[name] = quantizer_class- _CANONICAL_DATA_TYPES[name] = canonical_id- return quantizer_class-- return register---def canonical_data_type(data_type: str) -> str:- """Return an alias's canonical name for datatype-dependent policy.-- This is intentionally separate from construction: callers such as loss- policy and block preparation need to classify a datatype without creating- a quantizer or allocating tensors.- """- try:- return _CANONICAL_DATA_TYPES[_normalize_data_type(data_type)]- except KeyError as error:- raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error---def _quantizer_class(data_type: str) -> type:- """Look up the class that owns a datatype implementation."""- try:- return _QUANTIZER_CLASSES[_normalize_data_type(data_type)]- except KeyError as error:- raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error---def _create_weight_quantizer(data_type: str, spec: "WeightQuantizationSpec"):- """Create the implementation owned by ``data_type`` for one weight request."""- canonical = canonical_data_type(data_type)- quantizer_class = _quantizer_class(data_type)- try:- return quantizer_class.from_spec(spec, canonical)- except AttributeError as error:- raise LookupError(f"Datatype {data_type!r} supports activation quantization only") from error---def _create_activation_quantizer(data_type: str, spec: "ActivationQuantizationSpec"):- """Create the activation implementation owned by ``data_type``."""- canonical = canonical_data_type(data_type)- try:- return _quantizer_class(data_type).create_activation(replace(spec, data_type=canonical))- except AttributeError as error:- raise LookupError(f"Datatype {data_type!r} has no activation quantizer") from error---def prepare_data_type_block(data_type: str, block, layer_runtimes) -> None:- """Run the optional block-level setup owned by one datatype class."""- quantizer_class = _QUANTIZER_CLASSES.get(_normalize_data_type(data_type))- prepare = getattr(quantizer_class, "prepare_block", None)- if prepare is not None:- prepare(block, layer_runtimes)---@dataclass(frozen=True)-class WeightQuantizationSpec:- """Static properties needed to quantize one layer's weights."""-- data_type: str- bits: int- group_size: int | tuple[int, int]- sym: bool- scale_dtype: torch.dtype- q_scale_thresh: float = 1e-5- super_bits: int | None = None- super_group_size: int | None = None- global_scale: torch.Tensor | None = None- clip_min: torch.Tensor | None = None- clip_max: torch.Tensor | None = None---@dataclass(frozen=True)-class ActivationQuantizationSpec:- """Static properties needed to quantize one layer's activations."""-- data_type: str- bits: int- group_size: int | tuple[int, int]- sym: bool- scale_dtype: torch.dtype- dynamic: bool- q_scale_thresh: float = 1e-5---@dataclass(frozen=True)-class WeightQuantizationResult:- """Weight QDQ output plus the tensors required by export and inference."""-- weight: torch.Tensor- scale: torch.Tensor | dict[str, torch.Tensor] | None = None- zero_point: torch.Tensor | dict[str, torch.Tensor] | int | float | None = None- metadata: object | None = None- # Conv1D weights are transposed before export. Keep the original output- # dimension so per-output-channel tensors retain their intended layout.- logical_rows: int | None = None---class DataTypeQuantizer:- """Run a datatype implementation through initialize, QDQ, and write-back.-- This is the only lifecycle wrapper shared by all data types. Datatype- classes keep their own state and numerical code; this class only manages- the sequence and the trainable parameter dictionary.- """-- def __init__(self, implementation: Any, mode: str, tune_rounding: bool, tune_minmax: bool, apply_result):- self._implementation = implementation- self._mode = mode- self._tune_rounding = tune_rounding- self._tune_minmax = tune_minmax- self._apply_result = apply_result- self._state = None- self.parameters: dict[str, torch.Tensor] = {}-- def initialize(self, weight: torch.Tensor, *, imatrix=None) -> None:- """Initialize datatype state and expose its trainable tensors."""- self._state = self._implementation.create_state(- weight,- imatrix=imatrix,- mode=self._mode,- tune_rounding=self._tune_rounding,- tune_minmax=self._tune_minmax,- )- tunables = self._state.get("tunables", self._state) if isinstance(self._state, dict) else self._state.tunables- self.parameters = dict(tunables)-- def refresh(self, weight: torch.Tensor, *, imatrix=None) -> None:- """Rebuild fixed datatype state while preserving existing trainable tensors."""- previous_parameters = self.parameters- self.initialize(weight, imatrix=imatrix)- if isinstance(self._state, dict):- self._state["tunables"] = previous_parameters- else:- self._state = replace(self._state, tunables=previous_parameters)- self.parameters = previous_parameters-- def quantize(self, weight: torch.Tensor, **parameters) -> torch.Tensor:- """Return QDQ weights for the current trainable parameter values."""- return self.qdq(weight, **parameters).weight-- def qdq(self, weight: torch.Tensor, *, materialize=False, **parameters) -> WeightQuantizationResult:- """Run datatype QDQ and optionally include tensors needed for export."""- self._ensure_initialized()- return self._implementation.qdq(- weight,- self._state,- tunables={**self.parameters, **parameters},- materialize=materialize,- )-- def apply_result(self, layer, result: WeightQuantizationResult, *, transpose=False) -> None:- """Store an already computed QDQ result on its source layer."""- logical_rows = result.weight.shape[0]- if transpose:- result = WeightQuantizationResult(- result.weight.t(), result.scale, result.zero_point, result.metadata, logical_rows- )- if result.weight.device != layer.weight.device:- result = WeightQuantizationResult(- result.weight.to(layer.weight.device),- result.scale,- result.zero_point,- result.metadata,- result.logical_rows,- )- self._apply_result(layer, result)-- def write_back(self, layer, weight: torch.Tensor, *, transpose=False, **parameters) -> None:- """Materialize QDQ payload and store it on an exportable layer."""- result = self.qdq(weight, materialize=True, **parameters)- self.apply_result(layer, result, transpose=transpose)-- def _ensure_initialized(self) -> None:- """Reject use before a datatype has created its per-layer state."""- if self._state is None:- raise RuntimeError("Call initialize() before quantize() or write_back().")---def _weight_spec(layer) -> WeightQuantizationSpec:- """Extract the weight request from a resolved quantized layer."""- scale_dtype = getattr(layer, "scale_dtype", torch.float32)- return WeightQuantizationSpec(- data_type=layer.data_type,- bits=layer.bits,- group_size=layer.group_size,- sym=layer.sym,- scale_dtype=scale_dtype,- q_scale_thresh=getattr(layer, "q_scale_thresh", 1e-5),- super_bits=getattr(layer, "super_bits", None),- super_group_size=getattr(layer, "super_group_size", None),- global_scale=getattr(layer, "weight_global_scale", None),- clip_min=getattr(layer, "awq_clip_min", None),- clip_max=getattr(layer, "awq_clip_max", None),- )---def _activation_spec(layer) -> ActivationQuantizationSpec:- """Extract the activation request from a resolved quantized layer."""- return ActivationQuantizationSpec(- data_type=getattr(layer, "act_data_type", "int_sym"),- bits=getattr(layer, "act_bits", 16),- group_size=getattr(layer, "act_group_size", -1),- sym=getattr(layer, "act_sym", True),- scale_dtype=getattr(layer, "scale_dtype", torch.float32),- dynamic=getattr(layer, "act_dynamic", True),- )---def create_quantizer(layer, *, iters=None, disable_opt_rtn=False, tune_rounding=False, tune_minmax=False):- """Create the datatype-owned weight quantizer for a resolved layer."""- if isinstance(layer, Mapping):- layer = SimpleNamespace(**{"q_scale_thresh": 1e-5, **layer})- spec = _weight_spec(layer)- iters = getattr(layer, "iters", 0) if iters is None else iters- mode = "tuned" if iters > 0 else "rtn" if disable_opt_rtn else "optimized_rtn"- implementation = _create_weight_quantizer(spec.data_type, spec)- return DataTypeQuantizer(implementation, mode, tune_rounding, tune_minmax, implementation.apply_result)---def cache_activation_quantizer(layer):- """Create and cache a layer's activation quantizer when activation is quantized."""- if getattr(layer, "act_bits", 16) > 8:- return None- if hasattr(layer, "_ar_activation_quantizer"):- return layer._ar_activation_quantizer- spec = _activation_spec(layer)- quantizer = _create_activation_quantizer(spec.data_type, spec)- layer._ar_activation_quantizer = quantizer- return quantizer---def activation_quantizer_for_layer(layer, *, scale_dtype=None):- """Create the activation quantizer described by a layer's configuration."""- spec = _activation_spec(layer)- if scale_dtype is not None:- from dataclasses import replace-- spec = replace(spec, scale_dtype=scale_dtype)- return _create_activation_quantizer(spec.data_type, spec)---def quantize_activation(tensor, config):- """Apply dynamic activation quantization using a simple configuration mapping."""- spec = ActivationQuantizationSpec(- data_type=config["data_type"],- bits=config["bits"],- group_size=config["group_size"],- sym=config["sym"],- scale_dtype=tensor.dtype,- dynamic=True,- )- return _create_activation_quantizer(spec.data_type, spec).qdq(tensor)diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex 708390e63..11f4594c2 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.base import register_dtype, register_quantizer+from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import (float8_e4m3fn_ste,float8_e5m2_ste,@@ -366,136 +366,3 @@ if is_gaudi2():qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)qdq_res = qdq_res.to(orig_dtype)return qdq_res, scale, None---class _FP8WeightQuantizer:- """Select the requested FP8 format and materialize its scale layout."""-- def __init__(self, spec, primitive=None, scale_layout=None, data_type=None):- self.spec = spec- data_type = (data_type or spec.data_type).lower()- if primitive is None:- if data_type in ("fp8_e5m2",):- primitive, scale_layout = quant_fp8_e5m2, "row"- elif data_type in ("fp8_unit_scale",):- primitive, scale_layout = quant_fp8_unit_scale, "scalar"- elif data_type in ("fp8_e5m2_unit_scale",):- primitive, scale_layout = quant_fp8_e5m2_unit_scale, "scalar"- elif data_type == "fp8_gaudi3_sym":- primitive, scale_layout = quant_fp8_sym_gaudi3, "row"- elif isinstance(spec.group_size, tuple):- primitive, scale_layout = quant_block_fp_sym, "block"- else:- primitive, scale_layout = quant_fp8_sym, "row"- self.primitive = primitive- self.scale_layout = scale_layout-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the FP8 weight quantizer selected by the requested format."""- return cls(spec, data_type=canonical)-- @staticmethod- def create_activation(spec):- """Create the FP8 activation quantizer for the same format."""- return _create_fp8_activation(spec)-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- reduction_dims = 2 if self.scale_layout == "block" else 1- tunable_shape = grouped.shape[:-reduction_dims]- tunables = {}- if tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if tune_minmax:- tunables["max_scale"] = torch.nn.Parameter(- torch.ones(tunable_shape, device=weight.device, dtype=torch.float32)- )- return tunables-- def qdq(self, weight, state, *, tunables, materialize=False):- quantized, scale, zero_point = self.primitive(- weight,- bits=self.spec.bits,- group_size=self.spec.group_size,- v=tunables.get("value", 0),- max_scale=tunables.get("max_scale", 1.0),- )- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(- quantized,- scale if materialize else None,- zero_point if materialize else None,- self.scale_layout if materialize else None,- )-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("FP8 weight result was not materialized")- scale = result.scale- rows = result.logical_rows or result.weight.shape[0]- if result.metadata == "row" and scale.numel() % rows == 0:- scale = scale.reshape(rows, -1)- module.weight.data.copy_(result.weight)- module.scale = scale.cpu()- module.zp = result.zero_point---class _FP8ActivationQuantizer:- """Apply the matching FP8 primitive to dynamic or calibrated activations."""-- def __init__(self, spec, data_type, primitive, scale_layout):- self.spec = spec- self.data_type = data_type- self.primitive = primitive- self.scale_layout = scale_layout- self.bits = spec.bits- self.group_size = spec.group_size- self.requires_calibration = not spec.dynamic-- def observe(self, activation, current):- if self.scale_layout == "tensor":- maximum = activation.detach().abs().max()- else:- grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)- dims = (-2, -1) if self.scale_layout == "block" else -1- maximum = grouped.detach().abs().amax(dim=dims)- return maximum if current is None else torch.maximum(maximum.to(current), current)-- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- if self.requires_calibration and observed_max is None:- raise ValueError(f"{self.data_type} activation requires observed_max")- return self.primitive(- activation,- bits=self.bits,- group_size=self.group_size,- tensor_max=observed_max if self.requires_calibration else None,- max_scale=max_scale,- )-- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- quantized, _, _ = self.qdq_with_scale(- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale- )- return quantized---def _create_fp8_activation(spec):- """Build an activation quantizer using the same format choice as weights."""- if spec.data_type == "fp8_gaudi3_sym":- return _FP8ActivationQuantizer(spec, spec.data_type, quant_fp8_sym_gaudi3, "tensor")- weight_quantizer = _FP8WeightQuantizer(spec)- return _FP8ActivationQuantizer(spec, spec.data_type, weight_quantizer.primitive, weight_quantizer.scale_layout)---register_quantizer(- "fp8_sym",- aliases=("fp", "float", "fp8", "fp8_e4m3", "block_fp8_sym", "block_fp8", "block_fp8_e4m3"),-)(_FP8WeightQuantizer)-register_quantizer("fp8_e5m2")(_FP8WeightQuantizer)-register_quantizer("fp8_unit_scale")(_FP8WeightQuantizer)-register_quantizer("fp8_e5m2_unit_scale")(_FP8WeightQuantizer)-if is_gaudi2():- register_quantizer("fp8_gaudi3_sym")(_FP8WeightQuantizer)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex dea59a70e..2b4de2b7e 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -11,14 +11,13 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from collections.abc import Mapping-from dataclasses import dataclassfrom typing import Any, Callable, Unionimport torch-from auto_round.data_type.base import register_dtype, register_quantizer+from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_Kfrom auto_round.export.export_to_gguf.packing import (_LARGE_TENSOR_BYTES_THRESHOLD,make_q3_quants,@@ -26,15 +25,6 @@ from auto_round.export.export_to_gguf.packing import (make_qx_quants_chunk,)from auto_round.logger import logger---def get_gguf_quant_constants():- """Return GGUF block-layout constants without exposing export internals to algorithms."""- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-- return GGML_QUANT_SIZES, QK_K--from auto_round.utils import get_reciprocalfrom auto_round.utils.device import clear_memory@@ -496,18 +486,9 @@ def _imatrix_handle_zero(@torch.inference_mode()-def search_gguf_scale_min_asym(- tensor,- bits=4,- scale_dtype=torch.float16,- imatrix=None,- split_num=1,- v=0,- super_bits=None,- super_group_size=None,-):- super_bits = super_bits if super_bits is not None else 4 if bits == 2 else 6- super_group_size = super_group_size if super_group_size is not None else 16 if bits == 2 else 8+def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1, v=0):+ super_bits = 4 if bits == 2 else 6+ super_group_size = 16 if bits == 2 else 8quant_weights = Noneif imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):@@ -623,9 +604,6 @@ def search_gguf_scale_min_asym(def quant_tensor_gguf_asym_dq(tensor: torch.Tensor,bits: int = 4,- group_size=None,- super_bits=None,- super_group_size=None,v=0,scale_dtype=torch.float16,imatrix=None,@@ -656,7 +634,7 @@ def quant_tensor_gguf_asym_dq(raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")orig_dtype = tensor.dtypemaxq = int(2.0**bits) - 1- group_size = group_size if group_size is not None else 16 if bits == 2 else 32+ group_size = 16 if bits == 2 else 32# The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),# unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large# tensors. For very large single tensors (e.g. huge vocab embed/output projections),@@ -687,8 +665,6 @@ def quant_tensor_gguf_asym_dq(imatrix,split_num=split_num,v=v,- super_bits=super_bits,- super_group_size=super_group_size,)scale = scale.clone()wmin = wmin.clone()@@ -927,9 +903,6 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0def quant_tensor_gguf_sym_dq(tensor,bits=3,- group_size=16,- super_bits=None,- super_group_size=16,v=0,imatrix=None,scale=None,@@ -957,15 +930,21 @@ def quant_tensor_gguf_sym_dq(Quantized and de-quantized tensor, scale, zero-point"""+ from auto_round.export.export_to_gguf.config import K_SCALE_SIZE+if bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")maxq = int(2.0 ** (bits - 1))+ group_size = 16tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)orig_dtype = tensor.dtype- super_bits = super_bits if super_bits is not None else 6 if bits == 3 else 8+ super_bits = 6 if bits == 3 else 8+ super_group_size = 16+ ggml_type = f"q{bits}_k"+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]tensor = tensor.to(torch.float32)- n_blocks = tensor.nelement() // (super_group_size * group_size)+ n_blocks = tensor.nelement() // block_size# Chunk the candidate-scale search for very large single tensors (e.g. huge vocab# embed/output projections). This search allocates several full-size float32 buffers# at once; unchunked, that can need tens of GB for a single tensor and OOM even though@@ -976,7 +955,8 @@ def quant_tensor_gguf_sym_dq(# ordinary attention/FFN weights never needed it and would just get slower for no benefit.if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:split_num = 16- tensor = tensor.reshape(n_blocks, super_group_size, group_size)+ # (nb, 16, 16)+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)if iter is None:iter = 0use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1@@ -1000,161 +980,3 @@ def quant_tensor_gguf_sym_dq(tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)return tensor, {"scale": scale, "d_scale": d_scale}, maxq---@dataclass-class _GGUFState:- """Persistent GGUF RTN scales and double-quantization payload between QDQ calls."""-- tunables: Mapping[str, torch.Tensor]- tensor_min: torch.Tensor | None- tensor_max: torch.Tensor | None- imatrix: torch.Tensor | float | int | None- scale: torch.Tensor | None = None- wmin: torch.Tensor | None = None- d_scale: torch.Tensor | None = None- d_wmin: torch.Tensor | None = None- prev_scale: torch.Tensor | None = None- prev_wmin: torch.Tensor | None = None- prev_d_scale: torch.Tensor | None = None- prev_d_wmin: torch.Tensor | None = None- iteration: int = 0---class _GGUFWeightQuantizer:- """Own GGUF double-quant weight QDQ and its export payload."""-- def __init__(self, spec, family="plain", kind=None):- self.spec = spec- self.family = family- self.kind = kind or (- "sym" if spec.data_type == "int_sym_dq" else "float_zp" if spec.data_type == "int_asym_float_zp" else "asym"- )-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the GGUF quantizer; its state selects tuned or RTN behavior."""- kind = {"int_sym_dq": "sym", "int_asym_float_zp": "float_zp"}.get(canonical, "asym")- return cls(spec, kind=kind)-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- self.family = "rtn" if mode in ("rtn", "optimized_rtn") else "plain"- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)- tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)- tunables = {}- if self.family == "plain" and tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if self.family == "plain" and tune_minmax:- shape = tensor_min.shape- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))- return _GGUFState(tunables, tensor_min, tensor_max, imatrix)-- def qdq(self, weight, state, *, tunables, materialize=False):- value = tunables.get("value", 0)- min_scale = tunables.get("min_scale", 1.0)- max_scale = tunables.get("max_scale", 1.0)- if isinstance(min_scale, torch.Tensor):- min_scale.data.clamp_(0.0, 1.0)- if isinstance(max_scale, torch.Tensor):- max_scale.data.clamp_(0.0, 1.0)-- if self.kind == "float_zp" and self.family == "plain":- quantized, scale, zero_point = quant_tensor_asym_float_zp(- weight,- bits=self.spec.bits,- group_size=self.spec.group_size,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.spec.scale_dtype,- tensor_min=state.tensor_min,- tensor_max=state.tensor_max,- q_scale_thresh=self.spec.q_scale_thresh,- )- elif self.kind == "float_zp":- quantized, scale, zero_point = quant_tensor_asym_float_zp_rtn(- weight.clone(),- bits=self.spec.bits,- group_size=self.spec.group_size,- scale_dtype=self.spec.scale_dtype,- tensor_min=state.tensor_min,- tensor_max=state.tensor_max.clone(),- q_scale_thresh=self.spec.q_scale_thresh,- )- elif self.family == "rtn":- primitive = quant_tensor_gguf_sym_dq if self.kind == "sym" else quant_tensor_gguf_asym_dq- quantized, scale, zero_point = primitive(- weight.clone(),- bits=self.spec.bits,- group_size=self.spec.group_size,- super_bits=self.spec.super_bits,- super_group_size=self.spec.super_group_size,- v=value,- scale_dtype=self.spec.scale_dtype,- imatrix=state.imatrix,- prev_scale=state.prev_scale,- prev_wmin=state.prev_wmin,- prev_d_scale=state.prev_d_scale,- prev_d_wmin=state.prev_d_wmin,- iter=state.iteration,- )- state.scale = scale["scale"]- state.d_scale = scale["d_scale"]- state.prev_scale = state.scale- state.prev_d_scale = state.d_scale- if isinstance(zero_point, dict):- state.wmin = zero_point["wmin"]- state.d_wmin = zero_point["d_wmin"]- state.prev_wmin = state.wmin- state.prev_d_wmin = state.d_wmin- state.iteration += 1- else:- primitive = quant_tensor_sym_dq if self.kind == "sym" else quant_tensor_asym_dq- quantized, scale, zero_point = primitive(- weight,- bits=self.spec.bits,- group_size=self.spec.group_size,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.spec.scale_dtype,- tensor_min=state.tensor_min,- tensor_max=state.tensor_max,- q_scale_thresh=self.spec.q_scale_thresh,- super_bits=self.spec.super_bits,- super_group_size=self.spec.super_group_size,- )-- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("GGUF weight result was not materialized")- module.weight.data.copy_(result.weight)- rows = result.logical_rows or result.weight.shape[0]-- def set_values(values, primary):- for key, value in values.items():- name = primary if key == primary else "w_" + key- setattr(module, name, (value.reshape(rows, -1) if key == primary else value).cpu())-- if isinstance(result.scale, dict):- set_values(result.scale, "scale")- else:- module.scale = result.scale.reshape(rows, -1).cpu()- if isinstance(result.zero_point, dict):- set_values(result.zero_point, "zp")- elif isinstance(result.zero_point, torch.Tensor):- module.zp = result.zero_point.reshape(rows, -1).cpu()- else:- module.zp = result.zero_point---register_quantizer("int_sym_dq", aliases=("rtn_int_sym_dq",))(_GGUFWeightQuantizer)-register_quantizer("int_asym_dq", aliases=("rtn_int_asym_dq",))(_GGUFWeightQuantizer)-register_quantizer("int_asym_float_zp")(_GGUFWeightQuantizer)diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex ab2b9b4c5..d301f0b2e 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -11,14 +11,12 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from collections.abc import Mapping-from dataclasses import dataclassfrom typing import Unionimport torchfrom auto_round import envs-from auto_round.data_type.base import register_dtype, register_quantizer+from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.utils import get_reciprocal@@ -133,7 +131,7 @@ def quant_tensor_rtn_sym(min_scale=1.0,max_scale=1.0,scale_dtype=torch.float16,- **kwargs,+ **kwargs):"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community@@ -177,7 +175,7 @@ def quant_tensor_sym(tensor_max=None,q_scale_thresh=1e-5,init_scale=None,- **kwargs,+ **kwargs):"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community@@ -252,7 +250,7 @@ def quant_tensor_asym(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs,+ **kwargs):"""Quantize and de-quantize tensor asymmetrically.@@ -312,7 +310,7 @@ def quant_tensor_sym_gptq(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs,+ **kwargs):"""Quantize and de-quantize tensor asymmetrically.@@ -378,7 +376,7 @@ def quant_tensor_asym_wo_round(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs,+ **kwargs):"""Quantize and de-quantize tensor asymmetrically without rounding, this is mainly for tuning bias, norm.@@ -425,193 +423,3 @@ def quant_tensor_asym_wo_round(qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, scale, zp---@dataclass(frozen=True)-class _IntState:- """Per-layer integer ranges, optional optimized scale, and trainable values."""-- tunables: Mapping[str, torch.Tensor]- tensor_min: torch.Tensor- tensor_max: torch.Tensor- optimized_init: torch.Tensor | None---class _IntWeightQuantizer:- """Own integer weight QDQ and choose tuned, RTN, or optimized RTN internally."""-- def __init__(self, spec, family="plain"):- self.spec = spec- self.family = family-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the integer weight quantizer for a resolved layer."""- return cls(spec)-- @staticmethod- def create_activation(spec):- """Create the matching integer activation quantizer."""- return _IntActivationQuantizer(spec)-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- if mode == "optimized_rtn" and self.spec.sym:- self.family = "optimized"- elif mode == "rtn" and self.spec.sym:- self.family = "rtn"- else:- self.family = "plain"- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)- tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)- if self.spec.clip_max is not None:- clip_max = self.spec.clip_max.reshape(-1).to(weight.device, tensor_max.dtype)- clip_min = (- -clip_max- if self.spec.clip_min is None- else self.spec.clip_min.reshape(-1).to(weight.device, tensor_min.dtype)- )- if clip_min.numel() == tensor_min.numel() and clip_max.numel() == tensor_max.numel():- tensor_min = torch.maximum(tensor_min, clip_min.reshape_as(tensor_min))- tensor_max = torch.minimum(tensor_max, clip_max.reshape_as(tensor_max))-- tunables = {}- if self.family == "plain" and tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if self.family != "optimized" and tune_minmax:- shape = tensor_min.shape- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))-- optimized_init = None- if self.family == "optimized" and self.spec.sym:- search_weight = weight- if self.spec.clip_min is not None or self.spec.clip_max is not None:- search_weight = torch.clamp(weight, min=self.spec.clip_min, max=self.spec.clip_max)- _, optimized_init, _ = quant_tensor_opt_rtn_sym(- search_weight.clone(),- bits=self.spec.bits,- group_size=self.spec.group_size,- q_scale_thresh=self.spec.q_scale_thresh,- imatrix=imatrix,- )- return _IntState(tunables, tensor_min, tensor_max, optimized_init)-- def qdq(self, weight, state, *, tunables, materialize=False):- value = tunables.get("value", 0)- min_scale = tunables.get("min_scale", 1.0)- max_scale = tunables.get("max_scale", 1.0)- if isinstance(min_scale, torch.Tensor):- min_scale.data.clamp_(0.0, 1.0)- if isinstance(max_scale, torch.Tensor):- max_scale.data.clamp_(0.0, 1.0)-- kwargs = {- "bits": self.spec.bits,- "group_size": self.spec.group_size,- "q_scale_thresh": self.spec.q_scale_thresh,- }- if self.family == "optimized" and self.spec.sym:- quantized, scale, zero_point = quant_tensor_sym(- weight,- init_scale=state.optimized_init,- scale_dtype=self.spec.scale_dtype,- **kwargs,- )- elif self.family == "rtn" and self.spec.sym:- quantized, scale, zero_point = quant_tensor_rtn_sym(- weight,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.spec.scale_dtype,- **kwargs,- )- else:- primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym- quantized, scale, zero_point = primitive(- weight,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.spec.scale_dtype,- tensor_min=state.tensor_min,- tensor_max=state.tensor_max,- **kwargs,- )- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("INT weight result was not materialized")- module.weight.data.copy_(result.weight)- rows = result.logical_rows or result.weight.shape[0]- module.scale = result.scale.reshape(rows, -1).cpu()- module.zp = (- result.zero_point.reshape(rows, -1).cpu()- if isinstance(result.zero_point, torch.Tensor)- else result.zero_point- )---class _IntActivationQuantizer:- """Quantize integer activations, optionally using a calibrated maximum."""-- def __init__(self, spec):- self.spec = spec- self.requires_calibration = not spec.dynamic-- def observe(self, activation, current):- grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)- maximum = grouped.abs().amax(dim=-1)- return maximum if current is None else torch.maximum(maximum.to(current), current)-- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- if self.requires_calibration and observed_max is None:- raise ValueError(f"{self.spec.data_type} activation requires observed_max")- primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym- kwargs = {}- if observed_max is not None:- kwargs.update(tensor_min=-observed_max, tensor_max=observed_max)- return primitive(- activation,- bits=self.spec.bits,- group_size=self.spec.group_size,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.spec.scale_dtype,- q_scale_thresh=self.spec.q_scale_thresh,- **kwargs,- )-- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- quantized, _, _ = self.qdq_with_scale(- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale- )- return quantized---register_quantizer(- "int_sym",- aliases=(- "int",- "int4",- "int4_sym",- "int8",- "int8_sym",- "rtn_int",- "rtn_int4",- "rtn_int_sym",- "rtn_int4_sym",- "opt_rtn_int",- "opt_rtn_int4",- "opt_rtn_int_sym",- "opt_rtn_int4_sym",- ),-)(_IntWeightQuantizer)-register_quantizer(- "int_asym",- aliases=("int4_asym", "rtn_int_asym", "rtn_int4_asym", "opt_rtn_int_asym", "opt_rtn_int4_asym"),-)(_IntWeightQuantizer)diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 29982f2d4..7f453066a 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE, register_dtype, register_quantizer+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE, register_dtypefrom auto_round.data_type.utils import (ceil_ste,floor_ste,@@ -414,168 +414,6 @@ QUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceilQUANT_FUNC_WITH_DTYPE["mx_fp4_rceil_v2"] = quant_mx_rceil_v2QUANT_FUNC_WITH_DTYPE["opt_rtn_mx_fp"] = quant_mx_opt_rtn--class _MXQuantizer:- """Own MX weight QDQ and derive the concrete MX format from the request."""-- def __init__(self, spec, data_type=None, family="plain"):- self.spec = spec- self.data_type = data_type or self._data_type(spec)- self.family = family-- @staticmethod- def _data_type(spec):- data_type = spec.data_type.lower()- if data_type in ("mx_fp", "mx_int"):- data_type = f"{data_type}{spec.bits}"- if data_type not in MXFP_FORMAT_CACHE and not data_type.endswith("_rceil"):- raise ValueError(f"Unsupported MX datatype {spec.data_type!r}")- return data_type-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the MX quantizer selected by the requested MX format."""- data_type = canonical or spec.data_type- if data_type in ("mx_fp", "mx_int"):- data_type = f"{data_type}{spec.bits}"- return cls(spec, data_type)-- @classmethod- def create_activation(cls, spec):- """Create the dynamic MX activation quantizer for the same format."""- return _MXActivationQuantizer(spec, cls._data_type(spec))-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- self.family = "optimized" if mode == "optimized_rtn" else "plain"- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tunables = {}- if self.family == "plain" and tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if self.family == "plain" and tune_minmax:- tunables["max_scale"] = torch.nn.Parameter(- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)- )- return {"tunables": tunables, "imatrix": imatrix}-- def qdq(self, weight, state, *, tunables, materialize=False):- kwargs = {- "bits": self.spec.bits,- "group_size": self.spec.group_size,- "data_type": self.data_type,- }- if self.family == "optimized":- quantized, exponent, zero_point = quant_mx_opt_rtn(weight, imatrix=state["imatrix"], **kwargs)- elif self.data_type.endswith("_rceil"):- quantized, exponent, zero_point = quant_mx_rceil(weight, **kwargs)- else:- quantized, exponent, zero_point = quant_mx(- weight,- v=tunables.get("value", 0),- max_scale=tunables.get("max_scale", 1.0),- **kwargs,- )- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(- quantized, exponent if materialize else None, zero_point if materialize else None- )-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("MX weight result was not materialized")- module.weight.data.copy_(result.weight)- rows = result.logical_rows or result.weight.shape[0]- module.scale = result.scale.reshape(rows, -1).cpu()- module.zp = None---class _MXActivationQuantizer:- """Apply dynamic MX activation QDQ; MX formats require no calibration pass."""-- requires_calibration = False-- def __init__(self, spec, data_type):- if not spec.dynamic:- raise ValueError(f"MX datatype {data_type!r} supports only dynamic activation quantization")- self.spec = spec- self.data_type = data_type-- def observe(self, activation, current):- raise RuntimeError("Dynamic MX activation quantization does not require calibration")-- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- primitive = quant_mx_rceil if self.data_type.endswith("_rceil") else quant_mx- return primitive(- activation,- bits=self.spec.bits,- group_size=self.spec.group_size,- data_type=self.data_type.removesuffix("_rceil"),- max_scale=max_scale,- )-- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- quantized, _, _ = self.qdq_with_scale(- activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale- )- return quantized---_MX_ALIASES = {- "mx_int8": ("mxint8",),- "mx_int4": (- "mxint4",- "rtn_mx_int4",- "rtn_mx_int4_sym",- "opt_rtn_mx_int4",- "opt_rtn_mx_int4_sym",- ),- "mx_int2": ("mxint2",),- "mx_fp8e5m2": ("mxfp8e5m2",),- "mx_fp8": ("mxfp8",),- "mx_fp8e4m3": ("mxfp8e4m3",),- "mx_fp6e3m2": ("mxfp6e3m2",),- "mx_fp6": ("mxfp6",),- "mx_fp6e2m3": ("mxfp6e2m3",),- "mx_fp4": (- "mxfp4",- "mx_fp4_sym",- "rtn_mx_fp4",- "rtn_mx_fp4_sym",- "opt_rtn_mx_fp4",- "opt_rtn_mx_fp4_sym",- ),- "mx_fp4e2m1": ("mxfp4e2m1",),- "mx_float16": ("mxfloat16",),- "mx_fp16": ("mxfp16",),- "mx_bfloat16": ("mxbfloat16",),- "mx_bf16": ("mxbf16",),-}---for _data_type in MXFP_FORMAT_CACHE:- register_quantizer(- _data_type,- aliases=_MX_ALIASES[_data_type],- )(_MXQuantizer)--for _family in ("fp", "int"):- _canonical = f"mx_{_family}"-- register_quantizer(- _canonical,- aliases=(- f"mx_{_family}_sym",- f"rtn_mx_{_family}",- f"rtn_mx_{_family}_sym",- f"opt_rtn_mx_{_family}",- f"opt_rtn_mx_{_family}_sym",- ),- )(_MXQuantizer)--register_quantizer("mx_fp_rceil", aliases=("mxfprceil",))(_MXQuantizer)-register_quantizer("mx_fp4e2m1_rceil")(_MXQuantizer)-if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])data1 = quant_element(data, 2, 3, 6.0)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex eed0c05fe..624e91b0c 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -12,14 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.-from collections.abc import Mapping-from dataclasses import dataclass-import torch-from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.fp8 import float8_e4m3fn_stefrom auto_round.data_type.gguf import _imatrix_handle_zero+from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.logger import logger@@ -380,6 +377,8 @@ def search_nvfp4_scale(tensor, bits=4, qw=None, quant_func=None, group_size=16):def search_nvfp4_v2_scale(tensor, bits=4, qw=None):+ """Search per-group scales for the legacy-registry NVFP4 E5M3 quantizer."""+ qw = 1.0 if qw is None else qwreturn search_nvfp4_scale(tensor, bits, qw, quant_func=nvfp4_v2, group_size=tensor.shape[-1]).squeeze(-1)@@ -433,239 +432,6 @@ def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=Nonreturn qdq_res.to(orig_dtype), scale, None-@dataclass(frozen=True)-class _NVFPState:- """NVFP tuning values plus optional global and optimized scales."""-- tunables: Mapping[str, torch.Tensor]- global_scale: torch.Tensor | None- optimized_init: torch.Tensor | None---class _NVFPWeightQuantizer:- """Own NVFP4 weight QDQ, including its global-scale preparation."""-- def __init__(self, spec, family="plain"):- self.spec = spec- self.family = family-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the NVFP4 weight quantizer for a resolved layer."""- return cls(spec)-- @staticmethod- def create_activation(spec):- """Create dynamic NVFP4 activation quantization."""- return _NVFPActivationQuantizer(spec, "nv_fp4")-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- self.family = "optimized" if mode == "optimized_rtn" else "plain"- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tunables = {}- if self.family == "plain" and tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if self.family != "optimized" and tune_minmax:- tunables["max_scale"] = torch.nn.Parameter(- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)- )-- global_scale = self.spec.global_scale- if global_scale is None:- global_scale = calculate_gparam(weight, self.spec.group_size, weight.device)- else:- global_scale = global_scale.to(weight.device)- optimized_init = None- if self.family == "optimized":- if isinstance(imatrix, torch.Tensor):- imatrix = imatrix.reshape(1, -1)- imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)- imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)- qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)- else:- qw = 1.0- optimized_init = search_nvfp4_scale(grouped, self.spec.bits, qw)- return _NVFPState(tunables, global_scale, optimized_init)-- def qdq(self, weight, state, *, tunables, materialize=False):- kwargs = {- "bits": self.spec.bits,- "group_size": self.spec.group_size,- "v": tunables.get("value", 0),- "max_scale": tunables.get("max_scale", 1.0),- }- kwargs["global_scale"] = state.global_scale- if state.optimized_init is not None:- kwargs["init_scale"] = state.optimized_init- quantized, scale, zero_point = nv_fp4(weight, **kwargs)- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(- quantized,- scale if materialize else None,- zero_point if materialize else None,- state.global_scale if materialize else None,- )-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("NVFP weight result was not materialized")- module.weight.data.copy_(result.weight)- rows = result.logical_rows or result.weight.shape[0]- module.scale = result.scale.reshape(rows, -1).cpu()- module.zp = None- if result.metadata is not None:- module.weight_global_scale = result.metadata.cpu()---class _NVFPV2WeightQuantizer:- """Own the independent NVFP4-v2 weight QDQ implementation."""-- def __init__(self, spec):- self.spec = spec-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the NVFP4-v2 weight quantizer for a resolved layer."""- return cls(spec)-- @staticmethod- def create_activation(spec):- """Create dynamic NVFP4-v2 activation quantization."""- return _NVFPActivationQuantizer(spec, "nvfp4_v2")-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tunables = {}- optimized_rtn = mode == "optimized_rtn"- if tune_rounding and not optimized_rtn:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if tune_minmax and not optimized_rtn:- tunables["max_scale"] = torch.nn.Parameter(- torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)- )- optimized_scale = None- if optimized_rtn:- if isinstance(imatrix, torch.Tensor):- imatrix = imatrix.reshape(1, -1)- imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)- imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)- qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)- else:- qw = 1.0- optimized_scale = search_nvfp4_v2_scale(grouped, self.spec.bits, qw)- return _NVFPState(tunables, None, optimized_scale)-- def qdq(self, weight, state, *, tunables, materialize=False):- quantized, scale, zero_point = nvfp4_v2(- weight,- bits=self.spec.bits,- group_size=self.spec.group_size,- v=tunables.get("value", 0),- max_scale=tunables.get("max_scale", state.optimized_init if state.optimized_init is not None else 1.0),- )- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)-- apply_result = staticmethod(_NVFPWeightQuantizer.apply_result)---class _NVFPActivationQuantizer:- """Quantize dynamic or calibrated NVFP activations for one NVFP format."""-- def __init__(self, spec, data_type):- is_static = data_type in ("nv_fp4_with_static_gs", "nvfp4_v2_with_global_scale")- if not is_static and not spec.dynamic:- raise ValueError(f"NVFP datatype {data_type!r} supports only dynamic activation quantization")- self.spec = spec- self.data_type = data_type- self.requires_calibration = is_static-- def observe(self, activation, current):- if not self.requires_calibration:- raise RuntimeError("Dynamic NVFP activation quantization does not require calibration")- observed = activation.detach().float().abs().max().unsqueeze(0)- return observed if current is None else torch.maximum(current.to(observed.device), observed)-- def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):- if self.requires_calibration and observed_max is None:- raise ValueError(f"{self.data_type} activation requires observed_max")- primitive = {- "nv_fp4": nv_fp4,- "nv_fp4_with_static_gs": nv_fp4_with_static_gs,- "nvfp4_v2": nvfp4_v2,- "nvfp4_v2_with_global_scale": nvfp4_v2_with_global_scale,- }[self.data_type]- kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}- if self.requires_calibration:- kwargs["tensor_max"] = observed_max- if global_scale is not None:- kwargs["global_scale"] = global_scale- return primitive(activation, **kwargs)-- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0, global_scale=None):- quantized, _, _ = self.qdq_with_scale(- activation,- observed_max=observed_max,- min_scale=min_scale,- max_scale=max_scale,- global_scale=global_scale,- )- return quantized---def _prepare_nvfp_block(block, layer_runtimes):- """Materialize shared NVFP scales once after a block's observations."""- from auto_round.data_type.utils import update_block_global_scale_if_needed-- runtime = next(iter(layer_runtimes.values()), None)- if runtime is None:- return- update_block_global_scale_if_needed(block, runtime["data_type"], runtime["group_size"])---_NVFPWeightQuantizer.prepare_block = staticmethod(_prepare_nvfp_block)---class _NVFPStaticActivation:- """Datatype entry that provides static NVFP activation quantization only."""-- @staticmethod- def create_activation(spec):- return _NVFPActivationQuantizer(spec, "nv_fp4_with_static_gs")---class _NVFPV2StaticActivation:- """Datatype entry that provides global-scale NVFP-v2 activation only."""-- @staticmethod- def create_activation(spec):- return _NVFPActivationQuantizer(spec, "nvfp4_v2_with_global_scale")---register_quantizer(- "nv_fp4",- aliases=(- "nv_fp",- "nv_fp_sym",- "nv_fp4_sym",- "rtn_nv_fp",- "rtn_nv_fp4",- "rtn_nv_fp_sym",- "rtn_nv_fp4_sym",- "opt_rtn_nv_fp",- "opt_rtn_nv_fp4",- "opt_rtn_nv_fp_sym",- "opt_rtn_nv_fp4_sym",- ),-)(_NVFPWeightQuantizer)-register_quantizer("nvfp4_v2")(_NVFPV2WeightQuantizer)-register_quantizer("nv_fp4_with_static_gs", aliases=("rtn_nv_fp4_with_static_gs",))(_NVFPStaticActivation)-register_quantizer("nvfp4_v2_with_global_scale")(_NVFPV2StaticActivation)--if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])data1 = cast_to_fp4(data)diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pynew file mode 100644index 000000000..fca259ed6--- /dev/null+++ b/auto_round/data_type/register.py@@ -0,0 +1,40 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++QUANT_FUNC_WITH_DTYPE = {}+++def register_dtype(names):+ """Class decorator to register a EXPORT subclass to the registry.++ Decorator function used before a Pattern subclass.++ Args:+ names: A string. Define the export type.++ Returns:+ cls: The class of register.+ """++ def register(dtype):+ if isinstance(names, (tuple, list)):+ for name in names:+ QUANT_FUNC_WITH_DTYPE[name] = dtype+ else:+ QUANT_FUNC_WITH_DTYPE[names] = dtype++ return dtype++ return registerdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex d5b3acc52..7bca6fe97 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -22,17 +22,10 @@ from torch.nn import Linear, Modulefrom auto_round import envsfrom auto_round.compressors.utils import is_nv_fp-from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import check_to_quantized, logger-def quantize_bias_without_round(*args, **kwargs):- """Quantize bias/norm weights without exposing the INT primitive to algorithms."""- from auto_round.data_type.int import quant_tensor_asym_wo_round-- return quant_tensor_asym_wo_round(*args, **kwargs)--def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.@@ -183,6 +176,141 @@ def get_quant_func()+def _resolve_optimized_dtype_funcs(data_type: str, q_scale_thresh: float = 1e-5):+ """Resolve the SignRound optimized ``(scale_search_fn, quant_func)`` for a data type.++ Single source of truth for the optimized-path dispatch shared by+ ``SignRoundOptimizedWrapperLinear`` and AWQ's internal QDQ:++ * ``scale_search_fn(weight_reshape, bits, imatrix) -> init_scale`` searches the+ data-type-specific per-group init scale (the int variant clamps to+ ``q_scale_thresh`` to avoid a degenerate zero scale).+ * ``quant_func`` is the matching *plain* quant function (``init_scale`` already+ encodes the searched scale, so no opt-rtn / rtn variant is needed).++ Returns ``(None, None)`` for data types without an optimized path+ (asym int, ``*_dq``, or unrelated types).+ """+ dt = str(data_type)+ if dt.endswith("dq"):+ return None, None+ if dt.startswith("int"):+ # The optimized int init-scale search is symmetric-only; asym int uses+ # the standard tensor_min/tensor_max range instead.+ if "asym" in dt:+ return None, None+ from auto_round.data_type.int import quant_tensor_sym, search_scales++ def search_int(weight_reshape, bits, imatrix):+ init_scale = search_scales(weight_reshape, bits, imatrix)+ return torch.where(+ init_scale < 0,+ torch.clamp(init_scale, max=-q_scale_thresh),+ torch.clamp(init_scale, min=q_scale_thresh),+ )++ return search_int, quant_tensor_sym+ if dt.startswith("mx"):+ from auto_round.data_type.mxfp import quant_mx, search_mx_scale++ return search_mx_scale, quant_mx+ if dt.startswith("nv"):+ from auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale++ return search_nvfp4_scale, nv_fp4+ return None, None+++def search_optimized_init_scale(+ weight_reshape: torch.Tensor,+ data_type: str,+ bits: int,+ imatrix=None,+ q_scale_thresh: float = 1e-5,+):+ """Compute the SignRoundV2 optimized per-group ``init_scale`` for a grouped weight.++ Mirrors ``SignRoundOptimizedWrapperLinear``: dispatches on ``data_type`` so that+ any caller (the optimized wrapper itself or AWQ's internal QDQ used for the+ smooth/clip grid search) seeds the quantizer with the same initial scale.+ Returns ``None`` for data types that do not use the optimized init-scale search+ (asym int, ``*_dq``, or unrelated types).++ Args:+ weight_reshape: Weight reshaped/padded to ``[..., group_size]``.+ data_type: Resolved weight data type (e.g. ``"int_sym"``, ``"mx_fp4"``, ``"nv_fp4"``).+ bits: Weight bit-width.+ imatrix: Per-element importance matrix matching ``weight_reshape`` (or ``None``/scalar).+ q_scale_thresh: Minimum scale magnitude used to clamp the int init_scale.++ Returns:+ The per-group ``init_scale`` tensor, or ``None`` if unsupported.+ """+ search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)+ if search_fn is None:+ return None+ if imatrix is None or not isinstance(imatrix, torch.Tensor):+ imatrix = torch.ones_like(weight_reshape)+ return search_fn(weight_reshape, bits, imatrix)+++def get_optimized_quant_func(data_type: str):+ """Return the plain quant function used by the SignRound optimized path.++ The optimized init-scale search always pairs with the *plain* (non opt-rtn /+ non rtn) quant function for the data type, since ``init_scale`` already+ encodes the searched scale. Returns ``None`` for data types without an+ optimized path (asym int, ``*_dq``, or unrelated types).+ """+ _, quant_func = _resolve_optimized_dtype_funcs(data_type)+ return quant_func+++def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size):+ """Reshape/pad an importance matrix to match a group-reshaped weight.++ Encapsulates the imatrix grouping logic shared by the SignRound optimized+ wrapper and AWQ's internal QDQ so callers never handle the low-level reshape.+ Returns a tensor of ones when no imatrix is available (uniform importance),+ keeping every downstream optimized dtype implementation on its tensor path.+ """+ if imatrix is None or not isinstance(imatrix, torch.Tensor):+ return torch.ones_like(weight_reshape)+ imatrix = imatrix.reshape(1, -1)+ imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)+ imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)+ return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)+++def compute_optimized_init_scale(+ weight: torch.Tensor,+ data_type: str,+ bits: int,+ group_size,+ imatrix=None,+ q_scale_thresh: float = 1e-5,+):+ """Compute the SignRound optimized per-group ``init_scale`` for a full weight.++ Group-reshapes ``weight``, prepares the ``imatrix`` layout, and runs the+ data-type-specific scale search. Unlike :func:`search_optimized_init_scale`+ (which expects an already group-reshaped weight), this is the entry point for+ callers holding a full 2-D weight, e.g. AWQ's internal QDQ. Pair it with+ :func:`get_optimized_quant_func` (resolved once) to obtain the matching quant+ function, so the smooth/clip grid-search loss mirrors what+ ``SignRoundOptimizedWrapperLinear`` applies.++ Returns ``None`` for data types without an optimized init-scale path+ (asym int, ``*_dq``, or unrelated types).+ """+ search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)+ if search_fn is None:+ return None+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)+ imatrix = reshape_imatrix_for_weight(imatrix, weight_reshape, group_size)+ return search_fn(weight_reshape, bits, imatrix)++def round_ste(x: torch.Tensor):"""Straight-Through Estimator for rounding.diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.pyindex 6d7e35b41..b8e987d4c 100644--- a/auto_round/data_type/w4fp8.py+++ b/auto_round/data_type/w4fp8.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.base import register_dtype, register_quantizer+from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func@@ -72,58 +72,3 @@ def progressive_quant_fp8_int4(bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scalereturn qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4---@register_quantizer("fp8_to_int_sym")-class _ProgressiveFP8WeightQuantizer:- """Own the progressive BF16-to-FP8-to-INT4 weight quantization path."""-- def __init__(self, spec):- self.spec = spec-- @classmethod- def from_spec(cls, spec, canonical=None):- """Create the progressive FP8-to-INT4 weight quantizer."""- return cls(spec)-- def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size-- grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)- tunables = {}- if tune_rounding:- tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))- if tune_minmax:- shape = grouped.shape[:-1]- tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))- tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))- return tunables-- def qdq(self, weight, state, *, tunables, materialize=False):- quantized, scales, zero_point = progressive_quant_fp8_int4(- weight,- bits=self.spec.bits,- group_size=self.spec.group_size,- v=tunables.get("value", 0),- min_scale=tunables.get("min_scale", 1.0),- max_scale=tunables.get("max_scale", 1.0),- q_scale_thresh=self.spec.q_scale_thresh,- )- from auto_round.data_type.base import WeightQuantizationResult-- return WeightQuantizationResult(- quantized,- scales["scale"] if materialize else None,- zero_point if materialize else None,- scales["bf16_to_fp8_scale"] if materialize else None,- )-- @staticmethod- def apply_result(module, result):- if result.scale is None:- raise ValueError("Progressive FP8 weight result was not materialized")- module.weight.data.copy_(result.weight)- rows = result.logical_rows or result.weight.shape[0]- module.scale = result.scale.reshape(rows, -1).cpu()- module.zp = result.zero_point- module.w_bf16_to_fp8_scale = result.metadata.cpu()diff --git a/auto_round/modeling/fused_moe/grouped_experts.py b/auto_round/modeling/fused_moe/grouped_experts.pyindex 48d8511e5..c9a755316 100644--- a/auto_round/modeling/fused_moe/grouped_experts.py+++ b/auto_round/modeling/fused_moe/grouped_experts.py@@ -490,11 +490,6 @@ def _batched_qdq_weights(eligible and the caller must go per-expert."""ref = layers[0]- # Datatype quantizers intentionally keep their per-layer QDQ state private.- # Without the legacy compiled function, that state cannot be safely merged,- # so retain correctness by using the normal per-expert path.- if not hasattr(ref, "weight_quant_func"):- return Noneorig = ref.orig_layerout_features, in_features = ref.weight.shapenum_experts = len(layers)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex e018ca364..2f874158e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2231,9 +2231,7 @@ def set_amax_for_uncalibrated_experts()return uncalibrated_experts# Flatten all tensors to 1D before concatenation- device = amax_values[0].device- dtype = amax_values[0].dtype- flat_values = [t.reshape(-1).to(device=device, dtype=dtype) for t in amax_values]+ flat_values = [t.reshape(-1) for t in amax_values]all_values = torch.cat(flat_values)set_amax_value = torch.max(all_values)set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_valuediff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 239234ac6..4e0379fe2 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -1227,7 +1227,8 @@ def _quantize_weight_nvfp4_e5m3(disable_opt_rtn: bool = False,) -> dict[str, torch.Tensor]:"""Fake-quantize a 2D weight tensor to NVFP4 E5M3 and return its high-precision QDQ weight."""- from auto_round.data_type.base import create_quantizer+ from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizeout_features, in_features = weight.shapeif group_size != 16:@@ -1239,13 +1240,11 @@ def _quantize_weight_nvfp4_e5m3()weight_dev = weight.to(device)- quantizer = create_quantizer(- {"data_type": "nvfp4_v2", "bits": 4, "group_size": group_size, "sym": True, "scale_dtype": torch.float32},- iters=0,- disable_opt_rtn=disable_opt_rtn,- )- quantizer.initialize(weight_dev)- qdq_weight = quantizer.quantize(weight_dev)+ max_scale = 1.0+ if not disable_opt_rtn:+ grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]+ max_scale = search_nvfp4_v2_scale(grouped)+ qdq_weight, _, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}@@ -1280,7 +1279,8 @@ def _pack_weight_nvfp4_e5m3(disable_opt_rtn: bool = False,) -> dict[str, torch.Tensor]:"""Pack FP4 E2M1 weights with unsigned E5M3 block scales."""- from auto_round.data_type.base import create_quantizer+ from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinearout_features, in_features = weight.shape@@ -1289,13 +1289,11 @@ def _pack_weight_nvfp4_e5m3(f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'.")weight_dev = weight.to(device)- quantizer = create_quantizer(- {"data_type": "nvfp4_v2", "bits": 4, "group_size": group_size, "sym": True, "scale_dtype": torch.float32},- iters=0,- disable_opt_rtn=disable_opt_rtn,- )- quantizer.initialize(weight_dev)- scale = quantizer.qdq(weight_dev, materialize=True).scale+ max_scale = 1.0+ if not disable_opt_rtn:+ grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]+ max_scale = search_nvfp4_v2_scale(grouped)+ _, scale, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)# nvfp4_v2 may return a flattened per-group scale layout (e.g. [N, 1]);# normalize to [out_features, in_features // group_size] before packing# so serialized .weight_scale keeps the expected 2D shape.diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 6b70865f2..36ffbb173 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -20,8 +20,8 @@ import transformersfrom torch.functional import Fimport auto_round.envs as envs-from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer-from auto_round.data_type.utils import quantize_bias_without_round, reshape_pad_tensor_by_group_size+from auto_round.compressors.utils import is_nv_fp+from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_sizefrom auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -84,7 +84,6 @@ class WrapperLinear(torch.nn.Module):enable_round_tuning=True,enable_torch_compile=True,disable_opt_rtn=True,- weight_qdq_builder=None,**kwargs,):"""Initializes the WrapperLinear module.@@ -106,35 +105,18 @@ class WrapperLinear(torch.nn.Module):self.enable_torch_compile = enable_torch_compileself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8- # Observation may have cached the quantizer on the original layer. Do- # not consume it until wrapper initialization has completed successfully.- cached_activation_quantizer = getattr(self.orig_layer, "_ar_activation_quantizer", None)- self.activation_quantizer = cached_activation_quantizer- if self.enable_act_quant and self.activation_quantizer is None:- self.activation_quantizer = activation_quantizer_for_layer(orig_layer)self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)+ if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:+ from auto_round.data_type.nvfp import calculate_gparam++ weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)+ setattr(self, "weight_global_scale", weight_global_scale)+ self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:self.q_scale_thresh = 1e-8else:self.q_scale_thresh = 1e-5self._init_tuning_params_and_quant_func()-- def default_qdq(weight, *, materialize=False):- """Keep direct wrapper use working when no algorithm supplies a builder."""- return self.weight_quantizer.qdq(weight, materialize=materialize, **self.params)-- self.weight_qdq = default_qdq- if weight_qdq_builder is not None:- weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()- if type(orig_layer) == transformers.pytorch_utils.Conv1D:- weight = weight.t()- self.weight_qdq = weight_qdq_builder(- weight=weight.to(self.device), default_qdq=self.weight_quantizer.qdq, parameters=self.params- )- for name, parameter in self.params.items():- setattr(self, name, parameter)- if cached_activation_quantizer is not None:- self.orig_layer.__dict__.pop("_ar_activation_quantizer", None)if deepspeed_exists:if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):self.orig_forward = self.linear_forward@@ -165,25 +147,69 @@ class WrapperLinear(torch.nn.Module):orig_layer = self.orig_layerorig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()- if type(orig_layer) == transformers.pytorch_utils.Conv1D:+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:orig_weight = orig_weight.t()- self.weight_quantizer = create_quantizer(- orig_layer,- disable_opt_rtn=self.disable_opt_rtn,- tune_rounding=self.enable_round_tuning,- tune_minmax=self.enable_minmax_tuning,+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)++ if self.enable_round_tuning:+ self.weight_min = (+ torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)+ if isinstance(orig_layer.group_size, tuple)+ else torch.clamp(weight_reshape.min(1)[0], max=0)+ )+ self.weight_max = (+ torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)+ if isinstance(orig_layer.group_size, tuple)+ else torch.clamp(weight_reshape.max(1)[0], min=0)+ )+ else:+ self.weight_min = None+ self.weight_max = None+ # AWQ clip-as-init: cap the tunable weight range to the per-group clip+ # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).+ # This initializes the range used by quant_tensor_sym/asym, leaving+ # min_scale/max_scale to tune a coefficient on top. Only the standard+ # (non-tuple) group layout maps onto weight_min/weight_max here.+ awq_clip_min = getattr(orig_layer, "awq_clip_min", None)+ awq_clip_max = getattr(orig_layer, "awq_clip_max", None)+ if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):+ clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)+ if awq_clip_min is None:+ clip_min_flat = -clip_max_flat+ else:+ clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)+ if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():+ self.weight_max = torch.minimum(self.weight_max, clip_max_flat)+ self.weight_min = torch.maximum(self.weight_min, clip_min_flat)+ self._init_params(+ "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16)- self.weight_quantizer.initialize(orig_weight, imatrix=getattr(orig_layer, "imatrix", None))- self.params = dict(self.weight_quantizer.parameters)- for name, value in self.params.items():- setattr(self, name, value)- self.value = getattr(self, "value", torch.tensor(0.0, device=self.device, dtype=p_dtype))- self.min_scale = getattr(self, "min_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))- self.max_scale = getattr(self, "max_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))- self.data_type = orig_layer.data_type+ # Min-max scale initialization+ shape = get_scale_shape(orig_weight, orig_layer.group_size)+ self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))+ self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))++ self.weight_quant_func, self.data_type = get_quant_func(+ orig_layer.data_type,+ orig_layer.bits,+ orig_layer.sym,+ self.disable_opt_rtn,+ orig_layer.group_size,+ iters=orig_layer.iters,+ )+ if self.enable_torch_compile:+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)if self.enable_act_quant:- self.act_data_type = orig_layer.act_data_type+ self.act_quant_func, self.act_data_type = get_quant_func(+ orig_layer.act_data_type,+ orig_layer.act_bits,+ orig_layer.act_sym,+ disable_opt_rtn=True,+ iters=orig_layer.iters,+ )+ if self.enable_torch_compile:+ self.act_quant_func = compile_func(self.act_quant_func, self.device)self._init_params("act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic))@@ -192,7 +218,9 @@ class WrapperLinear(torch.nn.Module):# Bias tuningif self.enable_norm_bias_tuning:self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)- self.bias_quant_func = quantize_bias_without_round+ from auto_round.data_type.int import quant_tensor_asym_wo_round++ self.bias_quant_func = quant_tensor_asym_wo_roundself.params["bias_v"] = self.bias_vdef _init_params(self, name, dtype, shape, value, tunable):@@ -226,17 +254,43 @@ class WrapperLinear(torch.nn.Module):"""if self.orig_layer.bits >= 16:return self.orig_layer.weight, None, None+ min_bound, max_bound = self.minmax_scale_bound+ min_scale.data.clamp_(min_bound, max_bound)+ max_scale.data.clamp_(min_bound, max_bound)weight = self.orig_layer.weightif weight.device.type == "meta":weight = self.orig_layer.get_weight().to(self.device)if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:weight = weight.t()- self.params.update(value=value, min_scale=min_scale, max_scale=max_scale)- weight_q = self.weight_qdq(weight.to(self.device)).weight.to(weight.dtype)+ quant_kwargs = {}+ if hasattr(self.orig_layer, "super_bits"):+ quant_kwargs["super_bits"] = self.orig_layer.super_bits+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_size+ if hasattr(self, "_extra_quant_kwargs"):+ quant_kwargs.update(self._extra_quant_kwargs())++ weight_q, scale, zp = self.weight_quant_func(+ weight.to(self.device),+ bits=self.orig_layer.bits,+ group_size=self.orig_layer.group_size,+ v=value,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.orig_layer.scale_dtype,+ tensor_min=self.weight_min,+ tensor_max=self.weight_max,+ data_type=self.data_type,+ q_scale_thresh=self.q_scale_thresh,+ imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,+ global_scale=getattr(self, "weight_global_scale", None),+ init_scale=getattr(self, "init_scale", None),+ **quant_kwargs,+ )+ weight_q = weight_q.to(weight.dtype)if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:weight_q = weight_q.t()- return weight_q, None, None+ return weight_q, scale, zpdef _qdq_act(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):"""Quantizes and dequantizes activations.@@ -252,12 +306,19 @@ class WrapperLinear(torch.nn.Module):act_max_scale.data.clamp_(0, 1.0)act_min_scale.data.clamp_(0, 1.0)env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set- return self.activation_quantizer.qdq_with_scale(+ x, scale, zp = self.act_quant_func(x,- observed_max=act_max,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.q_scale_thresh,+ data_type=self.act_data_type,+ tensor_max=act_max, # for staticmax_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,+ global_scale=getattr(self, "input_global_scale", None),)+ return x, scale, zpdef _qdq_bias(self, bias, bias_v):"""Quantizes and dequantizes bias.@@ -300,28 +361,59 @@ class WrapperLinear(torch.nn.Module):v = best_params.get("value", torch.tensor(0.0)).to(self.device)min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)- # Preserve the dictionary captured by QDQ closures and restore all- # algorithm parameters, including names unknown to the wrapper.- self.params.update({name: value.to(self.device) for name, value in best_params.items()})- self.params.update(value=v, min_scale=min_scale, max_scale=max_scale)if self.orig_layer.weight.device.type == "meta":self.orig_layer.to(self.device)- if self.orig_layer.bits < 16:- weight = self.orig_layer.weight- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:- weight = weight.t()- result = self.weight_qdq(- weight.to(self.device),- materialize=True,- )- self.weight_quantizer.apply_result(- self.orig_layer,- result,- transpose=type(self.orig_layer) == transformers.pytorch_utils.Conv1D,- )+ # Unwrapper weight+ qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)+ # if hasattr(self.orig_layer, "imatrix"):+ # self.orig_layer.imatrix = None+ self.orig_layer.weight.data.copy_(qdq_weight)self.orig_layer.weight.grad = None+ shape = qdq_weight.shape+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:+ shape = qdq_weight.t().shape++ def _set_dict_attr(attr_dict, attr_name):+ for key in attr_dict.keys():+ if key == attr_name:+ setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))+ else:+ name = "w_" + key+ setattr(self.orig_layer, name, attr_dict[key].to("cpu"))++ if not isinstance(self.orig_layer.group_size, tuple):+ if isinstance(scale, dict):+ _set_dict_attr(scale, "scale")+ elif scale is None:+ self.orig_layer.scale = None+ elif scale.numel() > 1:+ self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")+ else:+ self.orig_layer.scale = scale.view(-1).to("cpu")+ else:+ self.orig_layer.scale = scale.to("cpu")++ if zp is not None:+ if isinstance(zp, dict):+ _set_dict_attr(zp, "zp")+ elif isinstance(zp, torch.Tensor):+ if zp.numel() > 1:+ zp = zp.reshape(shape[0], -1)+ self.orig_layer.zp = zp.to("cpu")+ else:+ self.orig_layer.zp = zp.view(-1).to("cpu")+ else:+ self.orig_layer.zp = zp+ else:+ self.orig_layer.zp = None++ if self.weight_global_scale is not None:+ global_scale = self.weight_global_scale+ assert global_scale.numel() == 1+ self.orig_layer.weight_global_scale = global_scale.to("cpu")+# Unwrapper biasif self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"].to(self.device)@@ -365,11 +457,11 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.act_max_scale = self.act_max_scaleself.orig_layer.act_data_type = self.act_data_type+ self.orig_layer.act_quant_func = self.act_quant_funcwrapper_layer = WrapperWALayer(self.orig_layer,enable_torch_compile=self.enable_torch_compile,device=self.device,- activation_quantizer=self.activation_quantizer,)return _preserve_global_name(wrapper_layer)@@ -475,16 +567,16 @@ class WrapperLinear(torch.nn.Module):class WrapperWALayer(torch.nn.Module):- def __init__(self, orig_layer, enable_torch_compile=True, device="cpu", activation_quantizer=None):+ def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):super(WrapperWALayer, self).__init__()self.orig_layer = orig_layerself.enable_torch_compile = enable_torch_compileself.device = deviceself.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else Noneself.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None- self.activation_quantizer = activation_quantizer or orig_layer.__dict__.pop("_ar_activation_quantizer", None)- if self.activation_quantizer is None:- self.activation_quantizer = activation_quantizer_for_layer(orig_layer)+ self.act_quant_func = self.orig_layer.act_quant_func+ if self.enable_torch_compile:+ self.act_quant_func = compile_func(self.act_quant_func, self.device)self.extra_repr_org = orig_layer.extra_repr# Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)@@ -518,10 +610,32 @@ class WrapperWALayer(torch.nn.Module):max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scalemin_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale- quant_kwargs = {"observed_max": act_max, "min_scale": min_scale, "max_scale": max_scale}+ quant_kwargs = {}if hasattr(self.orig_layer, "input_global_scale"):quant_kwargs["global_scale"] = self.orig_layer.input_global_scale- x = self.activation_quantizer.qdq(x, **quant_kwargs)+ if act_max is None:+ x, _, _ = self.orig_layer.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ min_scale=min_scale,+ max_scale=max_scale,+ **quant_kwargs,+ )+ else:+ x, _, _ = self.orig_layer.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ act_max=act_max,+ **quant_kwargs,+ )# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)return self.orig_layer.forward(x)@@ -553,7 +667,9 @@ class WrapperLayerNorm(torch.nn.Module):requires_grad=True,)self.params = {"v": self.v}- self.quant_func = quantize_bias_without_round+ from auto_round.data_type.int import quant_tensor_asym_wo_round++ self.quant_func = quant_tensor_asym_wo_rounddef unwrapper(self, best_params):if best_params is None:@@ -602,7 +718,9 @@ class WrapperLlamaNorm(torch.nn.Module):requires_grad=True,)self.params = {"v": self.v}- self.quant_func = quantize_bias_without_round+ from auto_round.data_type.int import quant_tensor_asym_wo_round++ self.quant_func = quant_tensor_asym_wo_rounddef unwrapper(self, best_params):if best_params is None:diff --git a/test/unit/common/algorithms/test_algorithm_datatype_parity.py b/test/unit/common/algorithms/test_algorithm_datatype_parity.pydeleted file mode 100644index 2d4829c05..000000000--- a/test/unit/common/algorithms/test_algorithm_datatype_parity.py+++ /dev/null@@ -1,86 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--"""Check the public datatype API used by wrappers and algorithms."""--from types import SimpleNamespace--import torch--from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer-from auto_round.data_type.int import quant_tensor_sym---def test_weight_quantization_reuses_initialization_and_matches_int_math():- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)- config = dict(data_type="int", bits=4, group_size=4, sym=True, scale_dtype=torch.float32)- quantizer = create_quantizer(config, disable_opt_rtn=True)- quantizer.initialize(weight)- first = quantizer.quantize(weight)- second = quantizer.quantize(weight)- expected = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)[0]- torch.testing.assert_close(first, expected, rtol=0, atol=0)- assert torch.equal(first, second)---def test_weight_training_and_writeback_share_parameters():- layer = torch.nn.Linear(4, 2, bias=False)- layer.data_type = "int"- layer.bits = 4- layer.group_size = 4- layer.sym = True- layer.scale_dtype = torch.float32- quantizer = create_quantizer(layer, iters=10, tune_rounding=True, tune_minmax=True)- quantizer.initialize(layer.weight.detach())- output = quantizer.quantize(layer.weight.detach())- output.square().sum().backward()- assert all(parameter.grad is not None for parameter in quantizer.parameters.values())- expected = output.detach().clone()- with torch.no_grad():- quantizer.write_back(layer, layer.weight.detach())- torch.testing.assert_close(layer.weight, expected, rtol=0, atol=0)- assert layer.scale.shape == (2, 1)- assert layer.zp is not None---def test_reinitializing_search_quantizer_does_not_reuse_previous_weight_range():- config = dict(data_type="int", bits=4, group_size=4, sym=False, scale_dtype=torch.float32)- quantizer = create_quantizer(config, disable_opt_rtn=True)- first = torch.tensor([[1.0, -0.75, 0.25, -0.5]])- second = first * 5- quantizer.initialize(first)- quantizer.quantize(first)- quantizer.initialize(second)- actual = quantizer.quantize(second)- fresh = create_quantizer(config, disable_opt_rtn=True)- fresh.initialize(second)- assert torch.equal(actual, fresh.quantize(second))---def test_activation_adapter_static_and_dynamic_contracts():- activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)- for dynamic in (False, True):- layer = SimpleNamespace(- act_data_type="int",- act_bits=8,- act_group_size=4,- act_sym=True,- scale_dtype=torch.float32,- act_dynamic=dynamic,- )- quantizer = activation_quantizer_for_layer(layer)- observed = quantizer.observe(activation, None)- qdq = quantizer.qdq(activation, observed_max=observed)- assert qdq.shape == activation.shape- assert torch.isfinite(qdq).all()diff --git a/test/unit/common/algorithms/transforms/hadamard/test_patch.py b/test/unit/common/algorithms/transforms/hadamard/test_patch.pyindex 6c8d176a8..ff63079dc 100644--- a/test/unit/common/algorithms/transforms/hadamard/test_patch.py+++ b/test/unit/common/algorithms/transforms/hadamard/test_patch.py@@ -179,14 +179,14 @@ class TestPatchWrapperLinearIdempotency:# Call the patched _qdq_act on a wrapperwrapper = _make_wrapper_linear()- # Activation QDQ is now owned by the datatype quantizer.- wrapper.activation_quantizer.qdq_with_scale = MagicMock(return_value=(torch.zeros(1, 8), None, None))+ # Replace the wrapper's act_quant_func with a spy+ wrapper.act_quant_func = MagicMock(return_value=(torch.zeros(1, 8), None, None))x = torch.ones(1, 8)act_min = torch.tensor(1.0)act_max = torch.tensor(1.0)- wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max, act_max=act_max)+ wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max)# The mock should have been called with x * 2- called_args = wrapper.activation_quantizer.qdq_with_scale.call_args[0]+ called_args = wrapper.act_quant_func.call_args[0]assert torch.equal(called_args[0], x * 2.0)def test_qdq_weight_falls_through_for_high_bits(self):@@ -294,7 +294,6 @@ class TestPatchWrapperWALayerIdempotency:patch_wrapperwalayer_forward_to_apply_transform(inp_transform)wrapper = _make_wrapper_wa_layer()- wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))# Patch the orig_layer.forward to be a spyorig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))wrapper.orig_layer.forward = orig_forward_spy@@ -302,8 +301,8 @@ class TestPatchWrapperWALayerIdempotency:x = torch.ones(1, 8)wrapper(x)- # Datatype-owned activation QDQ receives the rotated input.- called_args = wrapper.activation_quantizer.qdq.call_args[0]+ # The act_quant_func should have been called with x * 4+ called_args = wrapper.orig_layer.act_quant_func.call_args[0]assert torch.equal(called_args[0], x * 4.0)def test_act_max_passed_when_present(self):@@ -316,7 +315,6 @@ class TestPatchWrapperWALayerIdempotency:wrapper = _make_wrapper_wa_layer()wrapper.orig_layer.act_max = torch.tensor(2.0)- wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))wrapper.orig_layer.forward = orig_forward_spy@@ -324,7 +322,9 @@ class TestPatchWrapperWALayerIdempotency:x = torch.ones(1, 8)wrapper(x)- assert wrapper.activation_quantizer.qdq.call_args.kwargs["observed_max"] is not None+ # tensor_max should be the act_max value+ kwargs = wrapper.orig_layer.act_quant_func.call_args.kwargs+ assert kwargs.get("tensor_max") is not Noneclass TestPatchQuantLinearIdempotency:diff --git a/test/unit/common/core/test_wrapper_utils.py b/test/unit/common/core/test_wrapper_utils.pyindex 68de62b29..a603d0da0 100644--- a/test/unit/common/core/test_wrapper_utils.py+++ b/test/unit/common/core/test_wrapper_utils.py@@ -335,95 +335,6 @@ class TestWrapperLinearDeviceTransfer:assert wrapper.device == "cpu"assert wrapper.orig_layer is orig_layer- def test_unwrapper_keeps_weight_for_activation_only_quantization(self):- """W16A8 writes activation state only; its weight stays full precision."""- from auto_round.wrapper import WrapperLinear-- layer = torch.nn.Linear(4, 2, bias=False)- layer.bits = 16- layer.sym = True- layer.group_size = -1- layer.scale_dtype = torch.float32- layer.data_type = "int"- layer.act_bits = 8- layer.act_data_type = "int"- layer.act_sym = True- layer.act_dynamic = True- layer.act_group_size = -1- layer.iters = 0- layer.tuning_device = "cpu"- expected = layer.weight.detach().clone()-- WrapperLinear(layer, device="cpu", disable_opt_rtn=True).unwrapper({})-- assert torch.equal(layer.weight, expected)- assert not hasattr(layer, "scale")-- def test_custom_weight_qdq_is_shared_by_forward_and_write_back(self):- """A per-layer algorithm hook controls both tuning and final materialization."""- from auto_round.wrapper import WrapperLinear-- orig_layer = torch.nn.Linear(4, 2, bias=False)- orig_layer.bits = 4- orig_layer.sym = True- orig_layer.group_size = 4- orig_layer.scale_dtype = torch.float32- orig_layer.data_type = "int"- orig_layer.act_bits = 16- orig_layer.act_data_type = "float"- orig_layer.act_sym = True- orig_layer.act_dynamic = True- orig_layer.act_group_size = -1- orig_layer.tuning_device = "cpu"- calls = []-- def build_weight_qdq(*, weight, default_qdq, parameters):- assert weight.shape == (2, 4)- parameters["offset"] = torch.nn.Parameter(weight.new_tensor(0.1))-- def custom_qdq(weight, *, materialize=False):- calls.append(materialize)- return default_qdq(weight + parameters["offset"], materialize=materialize, **parameters)-- return custom_qdq-- wrapper = WrapperLinear(- orig_layer,- device="cpu",- disable_opt_rtn=True,- iters=0,- weight_qdq_builder=build_weight_qdq,- )-- result, _, _ = wrapper._qdq_weight(wrapper.value, wrapper.min_scale, wrapper.max_scale)- result.sum().backward()- assert wrapper.params["offset"].grad is not None- best = torch.tensor(0.5)- wrapper.params["offset"] = best- expected = wrapper.weight_qdq(orig_layer.weight, materialize=True).weight.detach().clone()- wrapper.params["offset"] = torch.tensor(0.9)- wrapper.unwrapper({"offset": best})-- assert calls == [False, True, True]- assert torch.equal(orig_layer.weight, expected)-- def test_base_algorithm_uses_datatype_qdq_by_default(self):- """Algorithms only override QDQ when they explicitly need custom behavior."""- from auto_round.algorithms.quantization.base import BaseQuantizer- from auto_round.data_type.base import create_quantizer-- weight = torch.tensor([[1.0, -0.5, 0.25, -0.75]])- datatype = create_quantizer(dict(data_type="int", bits=4, group_size=4, sym=True), disable_opt_rtn=True)- datatype.initialize(weight)- parameters = {"value": torch.tensor(0.0)}- qdq = BaseQuantizer.build_weight_qdq(object(), weight=weight, default_qdq=datatype.qdq, parameters=parameters)- torch.testing.assert_close(qdq(weight).weight, datatype.qdq(weight, value=0.0).weight)- parameters["value"] = torch.tensor(0.75)- result = qdq(weight, materialize=True)- expected = datatype.qdq(weight, value=0.75, materialize=True)- torch.testing.assert_close(result.weight, expected.weight)- torch.testing.assert_close(result.scale, expected.scale)-def test_wrapper_linear_forward(self):from auto_round.wrapper import WrapperLineardiff --git a/test/unit/common/data_type/__init__.py b/test/unit/common/data_type/__init__.pyindex 14a492441..e69de29bb 100644--- a/test/unit/common/data_type/__init__.py+++ b/test/unit/common/data_type/__init__.py@@ -1,13 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.diff --git a/test/unit/common/data_type/test_quantizer_parity.py b/test/unit/common/data_type/test_quantizer_parity.pydeleted file mode 100644index 9b23109cc..000000000--- a/test/unit/common/data_type/test_quantizer_parity.py+++ /dev/null@@ -1,189 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--"""Focused coverage for the datatype public lifecycle."""--import pytest-import torch--from auto_round.data_type.base import (- activation_quantizer_for_layer,- cache_activation_quantizer,- canonical_data_type,- create_quantizer,-)-from auto_round.data_type.fp8 import quant_fp8_e5m2-from auto_round.data_type.gguf import _GGUFWeightQuantizer-from auto_round.data_type.int import quant_tensor_sym-from auto_round.data_type.mxfp import quant_mx, quant_mx_rceil---def _weight_config(data_type, bits, group_size, *, sym=True):- return dict(data_type=data_type, bits=bits, group_size=group_size, sym=sym, scale_dtype=torch.float32)---def test_integer_lifecycle_matches_primitive_and_materializes_payload():- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)- quantizer.initialize(weight)- actual = quantizer.quantize(weight)- expected, _, _ = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)-- assert torch.equal(actual, expected)---def test_qdq_returns_the_same_result_used_for_write_back():- """Algorithms can execute QDQ once and hand its materialized result to the datatype."""- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)- layer = torch.nn.Linear(4, 1, bias=False)- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)- quantizer.initialize(weight)-- result = quantizer.qdq(weight, materialize=True)- quantizer.apply_result(layer, result)-- assert torch.equal(layer.weight, result.weight)- assert layer.scale.shape == (1, 1)---def test_write_back_keeps_conv1d_scale_indexed_by_output_channels():- """Conv1D stores a transposed weight, but its scale stays per output channel."""- weight = torch.tensor([[1.0, -0.75, 0.25, -0.5], [-1.0, 0.75, -0.25, 0.5]], dtype=torch.float32)- layer = torch.nn.Module()- layer.weight = torch.nn.Parameter(torch.empty(4, 2))- quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)- quantizer.initialize(weight)-- quantizer.write_back(layer, weight, transpose=True)-- assert layer.weight.shape == (4, 2)- assert layer.scale.shape == (2, 1)---def test_fp8_format_selection_does_not_turn_e5m2_into_block_e4m3():- weight = torch.randn(16, 32)- quantizer = create_quantizer(_weight_config("fp8_e5m2", 8, (8, 16)), iters=1)- quantizer.initialize(weight)- result = quantizer.quantize(weight)- expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=(8, 16))-- assert torch.equal(result, expected)---def test_generic_mx_name_uses_requested_bit_width():- weight = torch.randn(8, 32)- quantizer = create_quantizer(_weight_config("mx_fp", 6, 32), iters=1)- quantizer.initialize(weight)- result = quantizer.quantize(weight)- expected, _, _ = quant_mx(weight, bits=6, group_size=32, data_type="mx_fp6")-- assert torch.equal(result, expected)--[email protected]("dynamic", (True, False))-def test_activation_lifecycle_handles_dynamic_and_calibrated_integer_quantization(dynamic):- activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)- layer = type(- "Layer",- (),- dict(act_data_type="int", act_bits=8, act_group_size=4, act_sym=True, act_dynamic=dynamic),- )()- quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)- observed = None if dynamic else quantizer.observe(activation, None)-- result = quantizer.qdq(activation, observed_max=observed)-- assert result.shape == activation.shape- assert torch.isfinite(result).all()---def test_nvfp_static_global_scale_accepts_dynamic_calibration():- """NVFP4 calibrates a static global scale while keeping dynamic QDQ enabled."""- layer = type(- "Layer",- (),- dict(- act_data_type="nv_fp4_with_static_gs",- act_bits=4,- act_group_size=16,- act_sym=True,- act_dynamic=True,- ),- )()-- quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)-- assert quantizer.requires_calibration is True---def test_aliases_keep_a_single_canonical_datatype_for_policy():- assert canonical_data_type("int") == "int_sym"- assert canonical_data_type("INT4_ASYM") == "int_asym"---def test_activation_quantizer_uses_the_spec_default_datatype():- """Partial layer metadata keeps the public ``int_sym`` activation default."""- layer = type("Layer", (), dict(act_bits=8, act_group_size=4, act_sym=True, act_dynamic=True))()-- assert type(cache_activation_quantizer(layer)).__name__ == "_IntActivationQuantizer"- assert type(activation_quantizer_for_layer(layer)).__name__ == "_IntActivationQuantizer"---def test_format_aliases_keep_the_requested_quantization_format():- weight = torch.randn(4, 32)- fp8 = create_quantizer(_weight_config("fp8-e5m2", 8, -1), iters=1)- fp8.initialize(weight)- expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=-1)-- gguf = create_quantizer(_weight_config("INT_SYM_DQ", 4, 32), disable_opt_rtn=True)- legacy_gguf = create_quantizer(_weight_config("rtn_int_sym_dq", 4, 32), disable_opt_rtn=True)-- assert torch.equal(fp8.quantize(weight), expected)- assert isinstance(gguf._implementation, _GGUFWeightQuantizer)- assert gguf._implementation.kind == "sym"- assert isinstance(legacy_gguf._implementation, _GGUFWeightQuantizer)---def test_mx_aliases_and_rceil_keep_the_requested_format():- weight = torch.randn(4, 32)- layer = type(- "Layer", (), dict(act_data_type="mxfp4", act_bits=4, act_group_size=32, act_sym=True, act_dynamic=True)- )()- activation = activation_quantizer_for_layer(layer)- quantizer = create_quantizer(_weight_config("mx_fp4e2m1_rceil", 4, 32), iters=1)- quantizer.initialize(weight)- expected, _, _ = quant_mx_rceil(weight, bits=4, group_size=32, data_type="mx_fp4e2m1")-- assert activation.data_type == "mx_fp4"- assert torch.equal(quantizer.quantize(weight), expected)---def test_awq_clip_ranges_are_flattened_before_integer_qdq():- """AWQ stores [output, group] clips while INT QDQ uses flattened group rows."""- weight = torch.randn(4, 8)- clip = weight.abs().reshape(4, 2, 4).amax(dim=-1)- quantizer = create_quantizer(- {- **_weight_config("int", 4, 4),- "awq_clip_min": -clip,- "awq_clip_max": clip,- },- iters=1,- )- quantizer.initialize(weight)-- result = quantizer.quantize(weight)-- assert result.shape == weight.shape- assert torch.isfinite(result).all()diff --git a/test/unit/common/models/test_moe_experts_interface.py b/test/unit/common/models/test_moe_experts_interface.pyindex 2656d1c58..49f0f9125 100644--- a/test/unit/common/models/test_moe_experts_interface.py+++ b/test/unit/common/models/test_moe_experts_interface.py@@ -21,13 +21,10 @@ This verifies that:3. The forward pass produces correct results"""-from types import SimpleNamespace-import pytestimport torchfrom torch import nn-from auto_round.modeling.fused_moe.grouped_experts import _batched_qdq_weightsfrom auto_round.modeling.fused_moe.replace_modules import (ReplacementModuleBase,_apply_custom_replacements,@@ -45,11 +42,6 @@ def _skip_if_no_linear_loop():pytest.skip("transformers MOE integration not available")-def test_batched_qdq_falls_back_without_legacy_quant_function():- """Datatype-owned QDQ safely falls back when its private state cannot be fused."""- assert _batched_qdq_weights([SimpleNamespace()]) is None--def test_linear_loop_registration():"""Test that linear_loop is registered with transformers."""from auto_round.modeling.fused_moe.moe_experts_interface import (diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.pyindex 0931831cc..837a96605 100644--- a/test/unit/common/schemes/test_auto_scheme.py+++ b/test/unit/common/schemes/test_auto_scheme.py@@ -16,7 +16,6 @@ from auto_round.auto_scheme.delta_loss import (_vram_inventory_text,)from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name-from auto_round.data_type.base import WeightQuantizationResultfrom auto_round.export.export_to_gguf.export import _clear_gguf_model_instances@@ -1201,38 +1200,24 @@ class TestScoreAnchorWeightScoring:wrapper.max_act_value = 0wrapper._score_qdq_cpu = None- class _State:- tunables = {}+ def _fake_quant(weight, **kwargs):+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return ((weight / scale).round().clamp(-7, 7) * scale), scale, None- class _Quantizer:- def __init__(self):- self.calls = 0-- def quantize(self, weight, **kwargs):- self.calls += 1- scale = weight.abs().amax().clamp(min=1e-4) / 7.0- return (weight / scale).round().clamp(-7, 7) * scale-- wrapper.params = {}- wrapper.weight_state = _State()- wrapper.weight_quantizer = _Quantizer()- wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(- wrapper.weight_quantizer.quantize(weight, **kwargs)- )+ wrapper.weight_quant_func = _fake_quantlayer.weight.requires_grad = Truereturn wrapper, layerdef test_qdq_weight_quantizes_once_and_reuses_cache(self):wrapper, layer = self._make_wrapper()calls = {"n": 0}- quantizer = wrapper.weight_quantizer- original_quantize = quantizer.quantize+ real_quant = wrapper.weight_quant_func- def counting_quantize(*args, **kwargs):+ def counting_quant(weight, **kwargs):calls["n"] += 1- return original_quantize(*args, **kwargs)+ return real_quant(weight, **kwargs)- quantizer.quantize = counting_quantize+ wrapper.weight_quant_func = counting_quantargs = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))first, _, _ = wrapper._qdq_weight(*args)second, _, _ = wrapper._qdq_weight(*args)@@ -1411,20 +1396,11 @@ class TestScoreLinearRecompute:wrapper.min_scale = torch.tensor(1.0)wrapper.max_scale = torch.tensor(1.0)- class _State:- tunables = {}+ def _fake_quant(weight, **kwargs):+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return (weight / scale).round().clamp(-7, 7) * scale, scale, None- class _Quantizer:- def quantize(self, weight, **kwargs):- scale = weight.abs().amax().clamp(min=1e-4) / 7.0- return (weight / scale).round().clamp(-7, 7) * scale-- wrapper.params = {}- wrapper.weight_state = _State()- wrapper.weight_quantizer = _Quantizer()- wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(- wrapper.weight_quantizer.quantize(weight, **kwargs)- )+ wrapper.weight_quant_func = _fake_quantlayer.weight.requires_grad = Truereturn wrapper, layerdiff --git a/test/unit/test_cpu/algorithms/test_awq.py b/test/unit/test_cpu/algorithms/test_awq.pyindex c53773b5e..380d76661 100644--- a/test/unit/test_cpu/algorithms/test_awq.py+++ b/test/unit/test_cpu/algorithms/test_awq.py@@ -492,7 +492,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -522,14 +522,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quantizer(params):- return f"{params['data_type']}_{params['bits']}"+ def fake_resolve_quant_funcs(params):+ return f"{params['data_type']}_{params['bits']}", None- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):- records.append((params["data_type"], params["bits"], quantizer))+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quant_func))return weight- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcstransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -1009,19 +1009,32 @@ class TestAWQUseV2ScaleSearch:q._qdq_tool.configure(compressor)assert q._qdq_tool.disable_opt_rtn is True- params = {- "bits": 4,- "group_size": 128,- "sym": True,- "data_type": "int",- "disable_opt_rtn": True,- }- assert q._qdq_tool.resolve_quantizer(params)._mode == "rtn"compressor = self._make_compressor(self._signroundv2_config(data_type="mx_fp"))assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is True+ def test_init_scale_dispatch_by_data_type(self):+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale++ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):+ weight = torch.randn(32, 128)+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)+ assert init_scale is not None, dt+ assert init_scale.shape[0] == weight_reshape.shape[0]++ # Scalar imatrix sentinels mean uniform importance and must not enter+ # the tensor reshape path.+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)+ assert scalar_init_scale is not None+ assert scalar_init_scale.shape == init_scale.shape++ # asym int and *_dq are not part of the optimized init-scale path.+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None+def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.pyindex c643d93f1..b1cee31b2 100644--- a/test/unit/test_cpu/export/test_fake_format.py+++ b/test/unit/test_cpu/export/test_fake_format.py@@ -57,16 +57,21 @@ def test_fake_nvfp4_qdq_uses_saved_input_global_scale(monkeypatch):def test_fake_evaluation_wrapper_uses_saved_input_global_scale():captured_kwargs = {}linear = torch.nn.Linear(16, 4)+ linear.act_bits = 4+ linear.act_group_size = 16+ linear.scale_dtype = torch.float32+ linear.q_scale_thresh = 1e-5+ linear.act_data_type = "nv_fp4_with_static_gs"linear.act_max_scale = torch.ones(1)linear.act_min_scale = torch.ones(1)linear.input_global_scale = torch.tensor([1.5], dtype=torch.float32)- class ActivationQuantizer:- def qdq(self, activation, **kwargs):- captured_kwargs.update(kwargs)- return activation+ def quant_func(activation, **kwargs):+ captured_kwargs.update(kwargs)+ return activation, None, None- wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())+ linear.act_quant_func = quant_func+ wrapper = WrapperWALayer(linear, enable_torch_compile=False)wrapper(torch.randn(2, 16))@@ -75,14 +80,20 @@ def test_fake_evaluation_wrapper_uses_saved_input_global_scale():def test_fake_evaluation_wrapper_does_not_pass_global_scale_to_other_quantizers():linear = torch.nn.Linear(16, 4)+ linear.act_bits = 8+ linear.act_group_size = 16+ linear.scale_dtype = torch.float32+ linear.q_scale_thresh = 1e-5+ linear.act_data_type = "int"linear.act_max_scale = torch.ones(1)linear.act_min_scale = torch.ones(1)- class ActivationQuantizer:- def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):- return activation+ def quant_func(activation, *, global_scale=None, **kwargs):+ assert global_scale is None+ return activation, None, None- wrapper = WrapperWALayer(linear, enable_torch_compile=False, activation_quantizer=ActivationQuantizer())+ linear.act_quant_func = quant_func+ wrapper = WrapperWALayer(linear, enable_torch_compile=False)wrapper(torch.randn(2, 16))diff --git a/test/unit/test_cuda/algorithms/test_alg_ext.py b/test/unit/test_cuda/algorithms/test_alg_ext.pyindex a618a6c04..936e1522d 100644--- a/test/unit/test_cuda/algorithms/test_alg_ext.py+++ b/test/unit/test_cuda/algorithms/test_alg_ext.py@@ -35,6 +35,38 @@ class TestAlgExt:yieldshutil.rmtree("runs", ignore_errors=True)+ # def test_gguf_q2_k_s_uses_dq_wrapper_block(self, tiny_qwen_model_path):+ # """Regression test: enable_alg_ext + gguf:q2_k_s must use DQWrapperLinear.+ #+ # gguf:q2_k_s overrides data_type to "int_asym_dq" at format-resolution+ # time. The quantizer must be created *after* that override so that+ # SignRoundV2 sees the final data_type and sets dq_wrapper_block+ # (which wraps layers with DQWrapperLinear) instead of falling back to+ # the plain wrapper_block (which produces WrapperLinear).+ # """+ # from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundDQWrapperLinear+ #+ # ar = AutoRound(+ # tiny_qwen_model_path,+ # bits=4,+ # format="gguf:q2_k_s",+ # iters=1,+ # nsamples=1,+ # seqlen=32,+ # enable_alg_ext=True,+ # )+ # # post_init() runs the full pipeline (resolve_scheme → resolve_formats →+ # # create_quantizer → ...). quantizer only exists afterwards.+ # ar.post_init()+ # quantizer = ar.alg_composer.block_quantizer+ #+ # assert quantizer.wrapper_block.keywords["wrapper_cls"] is SignRoundDQWrapperLinear, (+ # f"Expected wrapper_block to use '{SignRoundDQWrapperLinear.__name__}', "+ # f"got '{quantizer.wrapper_block.__name__}'. "+ # "This likely means the quantizer was created before GGUF format "+ # "overrides were applied (data_type was not yet 'int_asym_dq')."+ # )+def test_int2_g64_asym_enable_alg_ext_keeps_config(self, tiny_qwen_model_path):"""Regression test: asym int2/g64 keeps the requested tuning config."""diff --git a/test/unit/test_cuda/algorithms/test_awq.py b/test/unit/test_cuda/algorithms/test_awq.pyindex 106b7e3a7..da1459920 100644--- a/test/unit/test_cuda/algorithms/test_awq.py+++ b/test/unit/test_cuda/algorithms/test_awq.py@@ -498,7 +498,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -528,14 +528,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quantizer(params):- return f"{params['data_type']}_{params['bits']}"+ def fake_resolve_quant_funcs(params):+ return f"{params['data_type']}_{params['bits']}", None- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):- records.append((params["data_type"], params["bits"], quantizer))+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quant_func))return weight- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcstransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -986,6 +986,27 @@ class TestAWQUseV2ScaleSearch:compressor = self._make_compressor(block)assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False+ def test_init_scale_dispatch_by_data_type(self):+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale++ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):+ weight = torch.randn(32, 128)+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)+ assert init_scale is not None, dt+ assert init_scale.shape[0] == weight_reshape.shape[0]++ # Scalar imatrix sentinels mean uniform importance and must not enter+ # the tensor reshape path.+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)+ assert scalar_init_scale is not None+ assert scalar_init_scale.shape == init_scale.shape++ # asym int and *_dq are not part of the optimized init-scale path.+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None+def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4diff --git a/test/unit/test_xpu/algorithms/test_awq.py b/test/unit/test_xpu/algorithms/test_awq.pyindex 7558302cc..9bbb8cb1a 100644--- a/test/unit/test_xpu/algorithms/test_awq.py+++ b/test/unit/test_xpu/algorithms/test_awq.py@@ -494,7 +494,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quantizer(self):+ def test_awq_grid_search_uses_per_balance_layer_quant_func(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -524,14 +524,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quantizer(params):- return f"{params['data_type']}_{params['bits']}"+ def fake_resolve_quant_funcs(params):+ return f"{params['data_type']}_{params['bits']}", None- def fake_qdq(weight, params, *, quantizer=None, imatrix=None):- records.append((params["data_type"], params["bits"], quantizer))+ def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quant_func))return weight- transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer+ transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcstransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -981,6 +981,27 @@ class TestAWQUseV2ScaleSearch:compressor = self._make_compressor(block)assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False+ def test_init_scale_dispatch_by_data_type(self):+ """``search_optimized_init_scale`` injects only for sym int/mx/nv."""+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale++ for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):+ weight = torch.randn(32, 128)+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)+ init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)+ assert init_scale is not None, dt+ assert init_scale.shape[0] == weight_reshape.shape[0]++ # Scalar imatrix sentinels mean uniform importance and must not enter+ # the tensor reshape path.+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)+ assert scalar_init_scale is not None+ assert scalar_init_scale.shape == init_scale.shape++ # asym int and *_dq are not part of the optimized init-scale path.+ assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None+ assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None+def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4