PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor: decouple datatype quantization from algorithms (#2360)
Heng Guo2e82448a6d79
diff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.pyindex b3fd72535..cfc921152 100644--- a/auto_round/algorithms/base.py+++ b/auto_round/algorithms/base.py@@ -127,6 +127,10 @@ class BaseAlgorithm:def scheme(self) -> "QuantizationScheme | None":return self.__run_ctx.scheme if self.__run_ctx is not None else None+ @property+ def run_context(self) -> "QuantizationRunContext | None":+ return self.__run_ctx+# ── Derived convenience properties ────────────────────────────────────────@propertydef model(self) -> "torch.nn.Module | None":diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 7f455239b..26bdffd6d 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -42,8 +42,9 @@ from auto_round.algorithms.config_resolver import (split_quantization_configs,)from auto_round.algorithms.utils import _has_nvfp4_layer+from auto_round.data_type.base import canonical_data_type, prepare_data_type_blockfrom auto_round.logger import logger-from auto_round.utils import clear_memory+from auto_round.utils import check_to_quantized, clear_memoryfrom auto_round.utils.device_manager import device_managerif TYPE_CHECKING: # avoid circular imports at runtime@@ -200,10 +201,6 @@ class AlgorithmComposer:", ".join(blockers),)- if _has_nvfp4_layer(orchestrator):- can_compile_block_forward = False- logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")-# Bind compressor-level infrastructure (set before _build_quantizer is called).self.block_forward = (BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)@@ -222,31 +219,15 @@ class AlgorithmComposer:Returns a list of hook handles that the caller must remove when done."""- from auto_round.compressors.utils import is_nv_fp- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size-- is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)+ from auto_round.data_type.base import cache_activation_quantizerdef collect_act_max(module, input, output):input = input[0] if isinstance(input, (tuple, list)) else inputif input.numel() == 0:return- module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fp- input, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- if is_module_act_nv_fp:- max_val = act_max.max()- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- return- act_max = act_max.to(module.act_max.device)- if is_module_act_nv_fp:- max_val = torch.max(act_max.max(), module.act_max.max())- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- module.act_max = torch.max(act_max, module.act_max)+ quantizer = cache_activation_quantizer(module)+ if quantizer is not None:+ module.act_max = quantizer.observe(input, getattr(module, "act_max", None))def should_collect(name, module):from auto_round.compressors.utils import check_need_act_calibration@@ -255,8 +236,8 @@ class AlgorithmComposer:if isinstance(module, tuple(SUPPORTED_LAYER_TYPES)):return (hasattr(module, "act_dynamic")- and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)and check_to_quantized(module)+ and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False)))if hasattr(module, "bits"):act_dynamic = getattr(module, "act_dynamic", True)@@ -264,8 +245,8 @@ class AlgorithmComposer:act_bits = getattr(module, "act_bits", 16)return (module.bits <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)and check_to_quantized(module)+ and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False)))return False@@ -307,39 +288,19 @@ class AlgorithmComposer:q_inputs: Optional list of quantized input tensors; used instead of``fp_inputs`` when provided."""- from auto_round.compressors.utils import is_nv_fp- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+ from auto_round.data_type.base import cache_activation_quantizertarget_input = q_inputs or fp_inputs- act_group_size = getattr(layer, "act_group_size")- if act_group_size is None:- act_group_size = layer.group_size- act_data_type = getattr(layer, "act_data_type")- if act_data_type is None:- act_data_type = layer.data_type- is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False+ quantizer = cache_activation_quantizer(layer)+ if quantizer is None:+ returnfor inp in target_input:if isinstance(inp, (tuple, list)):inp = inp[0]if inp.numel() == 0:continue- inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)- act_max = torch.max(torch.abs(inp), dim=-1).values-- if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:- layer.act_max = act_max- if is_act_nv_fp_flag:- max_val = act_max.max()- layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- continue-- act_max = act_max.to(layer.act_max.device)- if is_act_nv_fp_flag:- max_val = torch.max(act_max.max(), layer.act_max.max())- layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- layer.act_max = torch.max(act_max, layer.act_max)+ layer.act_max = quantizer.observe(inp, getattr(layer, "act_max", None))def need_quanted_input(self):for preprocessor in self.preprocessors:@@ -441,12 +402,26 @@ class AlgorithmComposer:act_data_type = self.scheme.act_data_type if self.scheme else data_typeif act_data_type is not None or not act_dynamic:from auto_round.compressors.utils import is_nv_fp- from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.utils import set_amax_for_all_moe_layersif is_nv_fp(act_data_type) or not act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")- update_block_global_scale_if_needed(block, data_type, group_size)++ runtimes_by_datatype = {}+ for name, module in block.named_modules():+ if not check_to_quantized(module):+ continue+ module_data_type = getattr(module, "data_type", data_type)+ try:+ canonical_id = canonical_data_type(module_data_type)+ except LookupError:+ continue+ runtimes_by_datatype.setdefault(canonical_id, {})[name] = {+ "data_type": module_data_type,+ "group_size": getattr(module, "group_size", group_size),+ }+ for canonical_id, runtimes in runtimes_by_datatype.items():+ prepare_data_type_block(canonical_id, block, runtimes)if q_inputs is not None and fp_inputs is not q_inputs:clear_memory(fp_inputs)diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex 70b3299e4..fe463fe20 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -21,7 +21,7 @@ if TYPE_CHECKING:from auto_round.algorithms.base import BaseAlgorithmfrom auto_round.algorithms.quantization.config import QuantizationConfig-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.base import create_quantizerfrom auto_round.logger import loggerfrom auto_round.utils import (check_to_quantized,@@ -57,6 +57,49 @@ class BaseQuantizer(BaseAlgorithm):def can_compile_block_forward(self):return True+ def build_weight_qdq(self, *, weight: torch.Tensor, default_qdq, parameters: dict[str, torch.Tensor]):+ """Return the per-layer weight QDQ callable used for tuning and export.++ Algorithms that need different QDQ behavior may override this method.+ The returned callable accepts ``weight`` and keyword ``materialize=False``+ and returns ``WeightQuantizationResult``. Bind settings in a closure and+ add trainable tensors to ``parameters`` before returning. This per-layer+ dictionary is also used by the optimizer and best-parameter restoration.+ Read its entries at call time; do not replace the dictionary.++ Args:+ weight: Logical [out_features, in_features] weights on the tuning+ device, including transposed Conv1D weights. Use only to create+ parameters with the right shape/device; do not capture this+ tensor in the returned closure.+ default_qdq: Datatype function accepting weight, materialize=False,+ and explicit datatype parameters. This builder binds the current+ parameter dictionary to that function. Existing weight parameters+ are datatype-dependent: value is a rounding offset, while+ min_scale/max_scale adjust clipping bounds where supported.+ parameters: Per-layer tensors collected for optimization. Add a+ torch.nn.Parameter to train it; fixed settings belong in the+ closure. Use distinct names that do not collide with wrapper+ attributes. Read entries on each call to see restored best values.++ Returns:+ Callable qdq(weight, *, materialize=False). With materialize=True,+ its WeightQuantizationResult must contain scale/zero-point/metadata+ required by the chosen datatype's exporter. Preserve autograd during+ training.++ To customize, override this method and use ``weight`` to initialize+ any new entries in ``parameters``. In the returned function, read those+ entries and transform the input or explicitly pass the datatype options+ you need. The implementation below is the default binding template.+ """++ def qdq(weight, *, materialize=False):+ """Use current parameters during training and restored best values at export."""+ return default_qdq(weight, materialize=materialize, **parameters)++ return qdq+# ── Calibration hook registration ─────────────────────────────────────────def register_fp_input_forward_hooks(self, block: torch.nn.Module) -> list:"""Register hooks that fire during the reference (FP-input) block forward.@@ -94,53 +137,22 @@ class BaseQuantizer(BaseAlgorithm):if not check_to_quantized(module):continueis_quantized = True- bits = getattr(module, "bits", None)- group_size = getattr(module, "group_size", None)- sym = getattr(module, "sym", None)- data_type = getattr(module, "data_type", None)super_bits = getattr(module, "super_bits", None)super_group_size = getattr(module, "super_group_size", None)- scale_dtype = self.scale_dtype- quant_dtype = data_type- if quant_dtype not in QUANT_FUNC_WITH_DTYPE:- quant_dtype = f"{quant_dtype}_{'sym' if sym else 'asym'}"- if not hasattr(self, "iters") or self.iters <= 0: # pylint: disable=E1101- tmp_dtype = "rtn_" + quant_dtype- if tmp_dtype in QUANT_FUNC_WITH_DTYPE:- quant_dtype = tmp_dtype- quant_func = QUANT_FUNC_WITH_DTYPE[quant_dtype]# float32 is used in RTN scale search; avoids caching a bf16 copy.weight_dtype = torch.float32 if super_group_size is not None else module.weight.dtype- quant_kwargs = {- "bits": bits,- "group_size": group_size,- "super_bits": super_bits,- "super_group_size": super_group_size,- "scale_dtype": scale_dtype,- }+ quantizer = create_quantizer(module, disable_opt_rtn=True)try:- weight, scale, zp = quant_func(- module.weight.to(dtype=weight_dtype, device=device_manager.device),- **quant_kwargs,- )+ weight = module.weight.to(dtype=weight_dtype, device=device_manager.device)+ quantizer.initialize(weight)+ quantizer.write_back(module, weight)except torch.OutOfMemoryError:- cuda_error_msg = traceback.format_exc()- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU")- weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)- except Exception:- raise- module.weight.data.copy_(weight.cpu())- for param_name, val in zip(["scale", "zp"], [scale, zp]):- if isinstance(val, dict):- for k, v in val.items():- setattr(module, k if k == "scale" else f"w_{k}", v.cpu())- elif isinstance(val, torch.Tensor):- setattr(module, param_name, val.cpu())- else:- setattr(module, param_name, val)- del weight, scale, zp+ logger.error(traceback.format_exc())+ logger.warning("falling back to CPU")+ weight = module.weight.to("cpu")+ quantizer.initialize(weight)+ quantizer.write_back(module, weight)+ del weight, quantizerclear_memory()return is_quantized@@ -232,6 +244,7 @@ class BaseQuantizer(BaseAlgorithm):enable_torch_compile=self.compress_context.enable_torch_compile,disable_opt_rtn=disable_opt_rtn,iters=0,+ weight_qdq_builder=self.build_weight_qdq,)layer = layer.unwrapper({})except torch.OutOfMemoryError:@@ -248,6 +261,7 @@ class BaseQuantizer(BaseAlgorithm):enable_round_tuning=False,enable_torch_compile=self.compress_context.enable_torch_compile,iters=0,+ weight_qdq_builder=self.build_weight_qdq,)layer = layer.unwrapper({})except Exception:diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex df70764db..25e7d5457 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -374,6 +374,7 @@ class SignRoundQuantizer(BaseQuantizer):self.enable_norm_bias_tuning,enable_torch_compile=self.compress_context.enable_torch_compile,device=device,+ weight_qdq_builder=self.build_weight_qdq,)round_params = []@@ -503,9 +504,6 @@ class SignRoundQuantizer(BaseQuantizer):cache_budget,device,)- if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):- for n, m in block.named_modules():- m.cur_iter = itotal_loss = 0global_indices = index_sampler.next_batch()if valid_token_mask:@@ -666,6 +664,7 @@ class SignRoundQuantizer(BaseQuantizer):enable_minmax_tuning=self.enable_minmax_tuning,enable_torch_compile=self.compress_context.enable_torch_compile,device=device,+ weight_qdq_builder=self.build_weight_qdq,).to(device)round_params = []minmax_params = []@@ -673,7 +672,7 @@ class SignRoundQuantizer(BaseQuantizer):if "min" in key or "max" in key:minmax_params.append(wrapper_linear.params[key])else:- round_params.append(wrapper_linear.value)+ round_params.append(wrapper_linear.params[key])if len(round_params) + len(minmax_params) <= 0:dump_info = f"quantized {layer_name}"logger.info(dump_info)diff --git a/auto_round/algorithms/quantization/sign_roundv2/__init__.py b/auto_round/algorithms/quantization/sign_roundv2/__init__.pyindex 4eb376f53..b567f1ca6 100644--- a/auto_round/algorithms/quantization/sign_roundv2/__init__.py+++ b/auto_round/algorithms/quantization/sign_roundv2/__init__.py@@ -12,14 +12,8 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.algorithms.quantization.sign_roundv2.quantizer import (- SignRoundDQWrapperLinear,- SignRoundOptimizedWrapperLinear,- SignRoundV2Quantizer,-)+from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer__all__ = [- "SignRoundDQWrapperLinear",- "SignRoundOptimizedWrapperLinear","SignRoundV2Quantizer",]diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex 338599e3a..d01778f58 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -13,309 +13,20 @@# limitations under the License.from contextlib import nullcontext-from functools import partialfrom typing import TYPE_CHECKING, Callable, Unionimport torch-import transformersfrom torch import autocastfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Configfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer--if TYPE_CHECKING:- from auto_round.algorithms.composer import AlgorithmComposerfrom auto_round.algorithms.registry import register_pipeline_member-from auto_round.data_type.gguf import (- double_quant_tensor_sym_rtn,- quant_tensor_gguf_asym_dq,- quant_tensor_gguf_sym_dq,- search_gguf_scale_min_asym,- search_gguf_scale_min_sym,-)-from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym-from auto_round.data_type.utils import (- get_optimized_quant_func,- reshape_imatrix_for_weight,- reshape_pad_tensor_by_group_size,- revert_tensor_by_pad,- round_ste,- search_optimized_init_scale,-)+from auto_round.data_type.base import canonical_data_typefrom auto_round.logger import logger-from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal-from auto_round.wrapper import WrapperLinear, wrapper_block---def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):- """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- orig_dtype = tensor.dtype- tensor = tensor.to(torch.float32)- maxq = int(2.0**bits) - 1- inverse_scale = get_reciprocal(scale)- int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)- qdq = (scale * int_w - wmin).to(orig_dtype)- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)- return qdq---def _dq_sym_qdq(tensor, scale, bits, v=0):- """Pure sym double-quant qdq math given precomputed scale (compilable)."""- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-- group_size = 16- super_group_size = 16- maxq = int(2.0 ** (bits - 1))- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- orig_dtype = tensor.dtype- tensor = tensor.to(torch.float32)- ggml_type = f"q{bits}_k"- block_size, _ = GGML_QUANT_SIZES[ggml_type]- n_blocks = tensor.nelement() // block_size- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)- else:- v_r = v- zp = torch.full_like(scale, maxq)- inverse_scale = get_reciprocal(scale)- int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq- qdq = (scale * (int_w - zp)).to(orig_dtype)- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)- return qdq---def _named_wrapper_block(wrapper_cls, name: str):- wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)- wrapped.__name__ = name- return wrapped---def _dq_asym_group_size(bits: int) -> int:- """Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""- return 16 if bits == 2 else 32+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized--class SignRoundOptimizedWrapperLinear(WrapperLinear):- minmax_scale_bound = (0.0, 2.0)-- def _init_tuning_params_and_quant_func(self):- super()._init_tuning_params_and_quant_func()-- layer = self.orig_layer- data_type = layer.data_type- weight_reshape = self._prepare_init_scale_weight()- imatrix = reshape_imatrix_for_weight(getattr(layer, "imatrix", None), weight_reshape, layer.group_size)-- self.init_scale = search_optimized_init_scale(- weight_reshape, data_type, layer.bits, imatrix, self.q_scale_thresh- )- self.weight_quant_func = get_optimized_quant_func(data_type)- if self.init_scale is None or self.weight_quant_func is None:- raise ValueError(- f"SignRound optimized path does not support data_type={data_type!r}; "- "expected a symmetric int / mx / nv type."- )-- self.data_type = data_type- if hasattr(layer, "imatrix"):- del layer.imatrix- if self.enable_torch_compile:- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)-- def _prepare_init_scale_weight(self) -> torch.Tensor:- """Return the group-reshaped weight that seeds the init-scale search.-- Reproduces the layout the quant func sees at tuning time: the (optionally- transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in- ``clip_as_init`` mode and stored a per-group clip range on the layer, the- weight is clamped to that range first so the searched ``init_scale``- already reflects the AWQ clip (``max_scale`` then tunes a coefficient on- top of it).- """- layer = self.orig_layer- weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight- if isinstance(layer, transformers.pytorch_utils.Conv1D):- weight = weight.t()- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)-- clip_min = getattr(layer, "awq_clip_min", None)- clip_max = getattr(layer, "awq_clip_max", None)- if clip_max is not None:- clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)- clip_min = (- -clip_max- if clip_min is None- else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)- )- if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:- weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)- else:- logger.warning_once(- "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "- "grouped weight shape %s."- % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))- )- return weight_reshape---class SignRoundDQWrapperLinear(WrapperLinear):- minmax_scale_bound = (0.5, 1.5)-- def __init__(self, *args, **kwargs) -> None:- if "enable_minmax_tuning" in kwargs:- logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")- kwargs["enable_minmax_tuning"] = False- super().__init__(*args, **kwargs)- self.prev_scale = None- self.prev_wmin = None- self.prev_d_scale = None- self.prev_d_wmin = None-- def _init_tuning_params_and_quant_func(self):- super()._init_tuning_params_and_quant_func()- # The double-quant search path is data-dependent and kept un-compiled,- # while ``weight_quant_func`` is the compilable pure-math half.- self._is_dq_path = False- self._dq_kind = None- self.search_func = None- if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:- self._is_dq_path = True- if self.orig_layer.data_type == "int_asym_dq":- self.search_func = search_gguf_scale_min_asym- self.weight_quant_func = _dq_asym_qdq- self._dq_kind = "asym"- else:- self.search_func = search_gguf_scale_min_sym- self.weight_quant_func = _dq_sym_qdq- self._dq_kind = "sym"- elif self.orig_layer.sym:- self.weight_quant_func = quant_tensor_sym- else:- self.weight_quant_func = quant_tensor_asym- self.data_type = self.orig_layer.data_type- if self.enable_act_quant:- self.act_quant_func = (- quant_tensor_gguf_asym_dq- if self.orig_layer.act_data_type == "int_asym_dq"- else quant_tensor_gguf_sym_dq- )- if self.enable_torch_compile:- self.act_quant_func = compile_func(self.act_quant_func, self.device)- self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)- if self.enable_torch_compile:- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)-- @torch.no_grad()- def _run_search(self, weight, v):- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-- bits = self.orig_layer.bits- scale_dtype = self.orig_layer.scale_dtype- imatrix = getattr(self.orig_layer, "imatrix", None)-- if self._dq_kind == "asym":- group_size = _dq_asym_group_size(bits)- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)- v_r = v- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- scale, wmin, d_scale, d_wmin = self.search_func(- t,- bits=bits,- scale_dtype=scale_dtype,- imatrix=imatrix,- split_num=1,- v=v_r,- )- return {- "scale": scale.clone(),- "wmin": wmin.clone(),- "d_scale": d_scale.clone(),- "d_wmin": d_wmin.clone(),- }-- group_size = 16- super_group_size = 16- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)- ggml_type = f"q{bits}_k"- block_size, _ = GGML_QUANT_SIZES[ggml_type]- n_blocks = t.nelement() // block_size- t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)- v_r = v- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)- super_bits = 6 if bits == 3 else 8- scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)- scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)- scale = scale.unsqueeze(-1)- return {"scale": scale.clone(), "d_scale": d_scale.clone()}-- def _qdq_weight(self, value, min_scale, max_scale):- if not self._is_dq_path:- return super()._qdq_weight(value, min_scale, max_scale)-- if self.orig_layer.bits >= 16:- return self.orig_layer.weight, None, None- min_bound, max_bound = self.minmax_scale_bound- min_scale.data.clamp_(min_bound, max_bound)- max_scale.data.clamp_(min_bound, max_bound)- weight = self.orig_layer.weight- if weight.device.type == "meta":- weight = self.orig_layer.get_weight().to(self.device)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight = weight.t()-- iter_v = getattr(self, "cur_iter", 0)- need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)- if need_search:- params = self._run_search(weight, value)- self.prev_scale = params["scale"]- self.prev_d_scale = params["d_scale"]- if self._dq_kind == "asym":- self.prev_wmin = params["wmin"]- self.prev_d_wmin = params["d_wmin"]- else:- params = {- "scale": self.prev_scale.detach(),- "d_scale": self.prev_d_scale.detach(),- }- if self._dq_kind == "asym":- params["wmin"] = self.prev_wmin.detach()- params["d_wmin"] = self.prev_d_wmin.detach()-- bits = self.orig_layer.bits- if self._dq_kind == "asym":- group_size = _dq_asym_group_size(bits)- weight_q = self.weight_quant_func(- weight,- params["scale"],- params["wmin"],- bits,- group_size,- v=value,- )- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}- zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}- else:- weight_q = self.weight_quant_func(- weight,- params["scale"],- bits,- v=value,- )- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}- zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))-- weight_q = weight_q.to(weight.dtype)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight_q = weight_q.t()- return weight_q, scale_out, zp_out+if TYPE_CHECKING:+ from auto_round.algorithms.composer import AlgorithmComposer@register_pipeline_member(SignRoundV2Config)@@ -331,18 +42,10 @@ class SignRoundV2Quantizer(SignRoundQuantizer):"""Model-level setup: initialise scheme-dependent state once before block iteration."""super().prepare_run(composer=composer)- if (- self.scheme.sym- and self.scheme.super_group_size is None- and (- self.scheme.data_type.startswith("int")- or self.scheme.data_type.startswith("mx")- or self.scheme.data_type.startswith("nv")- )- ):- if self.scheme.bits > 2 and not (- self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")- ):+ canonical_dtype = canonical_data_type(self.scheme.data_type)+ is_optimized_family = canonical_dtype.startswith(("int_", "mx_", "nv_", "nvfp"))+ if self.scheme.sym and self.scheme.super_group_size is None and is_optimized_family:+ if self.scheme.bits > 2 and canonical_dtype.startswith("int_"):logger.warning_once("algorithm extension has only undergone limited validation on ""W2A16,INT4, MXFP4 and NVFP4; use with caution."@@ -351,10 +54,9 @@ class SignRoundV2Quantizer(SignRoundQuantizer):self._use_outlier_suppressed_loss = Trueelse:self._use_outlier_suppressed_loss = False- self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")-- if self.scheme.data_type.endswith("dq"):- self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")+ # QDQ ownership stays with the generic wrapper. Datatype adapters+ # select optimized and double-quant behavior from the layer request;+ # SignRoundV2 only owns its loss policy.def can_compile_block_forward(self):return Falsediff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.pyindex 5ef7e1d71..d3822c2f6 100644--- a/auto_round/algorithms/quantization/utils.py+++ b/auto_round/algorithms/quantization/utils.py@@ -14,8 +14,7 @@import torch-from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.data_type.base import cache_activation_quantizerfrom auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized@@ -26,28 +25,15 @@ def register_act_max_hooks(quantizer, model):input = input[0]if input.numel() == 0:return- module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp- input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- if is_module_act_nv_fp:- max_val = act_max.max()- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- act_max = act_max.to(module.act_max.device)- if is_module_act_nv_fp:- max_val = torch.max(act_max.max(), module.act_max.max())- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- module.act_max = torch.max(act_max, module.act_max)+ activation_quantizer = cache_activation_quantizer(module)+ if activation_quantizer is not None:+ module.act_max = activation_quantizer.observe(input, getattr(module, "act_max", None))hook_handles = []if isinstance(model, SUPPORTED_LAYER_TYPES):if (hasattr(model, "act_dynamic")- and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)+ and getattr(cache_activation_quantizer(model), "requires_calibration", False)and check_to_quantized(model)):hook_handles.append(model.register_forward_hook(get_act_max_hook))@@ -56,7 +42,7 @@ def register_act_max_hooks(quantizer, model):for name, module in model.named_modules():if (hasattr(module, "act_dynamic")- and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)+ and getattr(cache_activation_quantizer(module), "requires_calibration", False)and check_to_quantized(module)):hook_handles.append(module.register_forward_hook(get_act_max_hook))@@ -64,12 +50,14 @@ def register_act_max_hooks(quantizer, model):if name in quantizer.layer_config:config = quantizer.layer_config[name]- act_dynamic = config.get("act_dynamic", True)- act_data_type = config.get("act_data_type", None)- act_bits = config.get("act_bits", 16)+ module.act_dynamic = config.get("act_dynamic", True)+ module.act_data_type = config.get("act_data_type", "int_sym")+ module.act_bits = config.get("act_bits", 16)+ module.act_group_size = config.get("act_group_size", quantizer.act_group_size)+ module.act_sym = config.get("act_sym", True)if (config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and getattr(cache_activation_quantizer(module), "requires_calibration", False)and check_to_quantized(config)):hook_handles.append(module.register_forward_hook(get_act_max_hook))diff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.pyindex 2c8332e5e..40c839419 100644--- a/auto_round/algorithms/transforms/awq/base.py+++ b/auto_round/algorithms/transforms/awq/base.py@@ -730,7 +730,7 @@ class AWQTransform(BasePreprocessor):# the grid-search loop. Normal AWQ flow requires one mapping to have# compatible quant params, but keeping this per-layer avoids hidden# coupling to the first layer and makes direct calls robust.- bl_quant_funcs = {bl: self._qdq_tool.resolve_quant_funcs(bl_params[bl]) for bl in mapping.balance_layers}+ bl_quantizers = {bl: self._qdq_tool.resolve_quantizer(bl_params[bl]) for bl in mapping.balance_layers}best_error = float("inf")best_scales = None@@ -751,12 +751,11 @@ class AWQTransform(BasePreprocessor):# de-smoothed result back, so the parent forward below sees the# weights the layer would actually compute with.for bl in mapping.balance_layers:- quant_func, opt_quant_func = bl_quant_funcs[bl]+ quantizer = bl_quantizers[bl]w_qdq = self._qdq_tool.qdq(orig_state[bl] * scales_view,bl_params[bl],- quant_func=quant_func,- opt_quant_func=opt_quant_func,+ quantizer=quantizer,imatrix=getattr(bl, "imatrix", None),)bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)@@ -767,13 +766,12 @@ class AWQTransform(BasePreprocessor):else:total_loss = 0.0for bl in mapping.balance_layers:- quant_func, opt_quant_func = bl_quant_funcs[bl]+ quantizer = bl_quantizers[bl]w_orig = orig_weights[bl].to(device)w_qdq = self._qdq_tool.qdq(w_orig * scales_view,bl_params[bl],- quant_func=quant_func,- opt_quant_func=opt_quant_func,+ quantizer=quantizer,imatrix=getattr(bl, "imatrix", None),)total_loss += (w_orig - w_qdq / scales_view).pow(2).sum().item()@@ -1032,8 +1030,8 @@ class AWQTransform(BasePreprocessor):# group size and drop super-block (double-quant) params, which the clip# path does not apply.clip_params = {**params, "group_size": gs, "super_bits": None, "super_group_size": None}- quant_func, _ = self._qdq_tool.resolve_quant_funcs(clip_params)- if quant_func is None:+ quantizer = self._qdq_tool.resolve_quantizer(clip_params)+ if quantizer is None:return Nonefeat = input_feat.to(device).reshape(-1, in_features)@@ -1072,7 +1070,7 @@ class AWQTransform(BasePreprocessor):max_val = org_max_val * shrinkcur_w = torch.clamp(w_b, min_val, max_val)cur_w_flat = cur_w.reshape(cur_w.shape[0], n_group * gs)- q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quant_func=quant_func).reshape(cur_w.shape)+ q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quantizer=quantizer).reshape(cur_w.shape)cur_out = (feat * q_w).sum(dim=-1)err = (cur_out - org_out).pow(2).mean(dim=1).view(min_errs.shape)improved = err < min_errsdiff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.pyindex 2a51956a2..e0bb05ed5 100644--- a/auto_round/algorithms/transforms/awq/qdq.py+++ b/auto_round/algorithms/transforms/awq/qdq.py@@ -28,19 +28,15 @@ Usage (the same unified QDQ serves both the scale grid-search and clip-search)::tool.configure(compressor) # opt_rtn / v2 flagsparams = tool.resolve_params(layer) # per-layer scheme- qf, opt_qf = tool.resolve_quant_funcs(params) # dispatch once, reuse in loop- w_qdq = tool.qdq(weight, params, quant_func=qf, opt_quant_func=opt_qf)+ quantizer = tool.resolve_quantizer(params) # dispatch once, reuse in loop+ w_qdq = tool.qdq(weight, params, quantizer=quantizer)"""from __future__ import annotationsimport torch-from auto_round.data_type.utils import (- compute_optimized_init_scale,- get_optimized_quant_func,- get_quant_func,-)+from auto_round.data_type.base import create_quantizerclass QDQTool:@@ -98,7 +94,7 @@ class QDQTool:"""Resolve the per-layer weight-quant params (``layer_config`` + fallbacks).Single source of every scheme field the QDQ needs, so callers pass the- returned dict straight to :meth:`resolve_quant_funcs` and :meth:`qdq`.+ returned dict straight to :meth:`resolve_quantizer` and :meth:`qdq`.``super_bits`` / ``super_group_size`` are the GGUF double-quantsuper-block params and are ``None`` for non-GGUF schemes."""@@ -113,25 +109,12 @@ class QDQTool:"super_group_size": cfg.get("super_group_size", None),}- def resolve_quant_funcs(self, params: dict):- """Dispatch ``(quant_func, opt_quant_func)`` for ``params``.-- ``opt_quant_func`` is non-``None`` only when the SignRoundV2 optimized- init-scale path applies (it depends solely on ``data_type`` / ``sym``).- Hoisting this out of a search loop avoids repeated dispatch.- """- quant_func, _ = get_quant_func(- params["data_type"],- params["bits"],- params["sym"],+ def resolve_quantizer(self, params: dict):+ """Create one configured quantizer to reuse throughout a weight search."""+ return create_quantizer(+ {**params, "scale_dtype": torch.float32},disable_opt_rtn=params["disable_opt_rtn"],- group_size=params["group_size"],- iters=0,)- opt_quant_func = None- if self.use_v2_scale_search and params["sym"]:- opt_quant_func = get_optimized_quant_func(params["data_type"])- return quant_func, opt_quant_func# ── the unified QDQ for AWQ search ──────────────────────@torch.no_grad()@@ -140,48 +123,11 @@ class QDQTool:weight: torch.Tensor,params: dict,*,- quant_func=None,- opt_quant_func=None,+ quantizer=None,imatrix: torch.Tensor | None = None,) -> torch.Tensor:- """Quantize-dequantize ``weight`` under the resolved ``params``.-- The single QDQ entry point for every scheme: ``bits`` / ``group_size`` /- ``sym`` / ``data_type`` plus the optional GGUF super-block params drive- the dispatched ``quant_func``; a non-``None`` ``opt_quant_func`` enables- the SignRoundV2 optimized init-scale path for this weight. Used for- both the scale grid-search and the clip-search -- reference/loss only,- it never mutates stored weights.- """- if quant_func is None:- quant_func, opt_quant_func = self.resolve_quant_funcs(params)- if quant_func is None:- raise RuntimeError(- "QDQTool: no quantization function resolved for "- f"data_type={params['data_type']}, bits={params['bits']}, "- f"sym={params['sym']}, group_size={params['group_size']}."- )-- quant_kwargs = {- "bits": params["bits"],- "group_size": params["group_size"],- "data_type": params["data_type"],- "sym": params["sym"],- }- if params.get("super_bits") is not None:- quant_kwargs["super_bits"] = params["super_bits"]- if params.get("super_group_size") is not None:- quant_kwargs["super_group_size"] = params["super_group_size"]-- active_quant_func = quant_func- if opt_quant_func is not None:- init_scale = compute_optimized_init_scale(- weight, params["data_type"], params["bits"], params["group_size"], imatrix=imatrix- )- if init_scale is not None:- quant_kwargs["init_scale"] = init_scale- quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)- active_quant_func = opt_quant_func-- qdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)- return qdq_weight+ """Simulate quantization for a search candidate without changing stored weights."""+ if quantizer is None:+ quantizer = self.resolve_quantizer(params)+ quantizer.initialize(weight, imatrix=imatrix)+ return quantizer.quantize(weight)diff --git a/auto_round/algorithms/transforms/hadamard/patch.py b/auto_round/algorithms/transforms/hadamard/patch.pyindex e3f5f9f95..6971cd304 100644--- a/auto_round/algorithms/transforms/hadamard/patch.py+++ b/auto_round/algorithms/transforms/hadamard/patch.py@@ -24,6 +24,8 @@ Each patch is idempotent: calling it twice has no effect.from __future__ import annotations+import math+import torchimport transformers@@ -75,6 +77,8 @@ def patch_wrapperlinear_to_apply_transform(if is_conv1d:new_weight = new_weight.t().contiguous()self.orig_layer.weight.data.copy_(new_weight)+ logical_weight = new_weight.t() if is_conv1d else new_weight+ self.weight_quantizer.refresh(logical_weight, imatrix=getattr(self.orig_layer, "imatrix", None))self.applied_weight_hadamard = Truereturn _orig_qdq_weight(self, value, min_scale, max_scale)@@ -106,16 +110,18 @@ def patch_wrapperwalayer_forward_to_apply_transform(_orig_forward = WrapperWALayer.forwarddef _forward_patched(self, x):- act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else Nonex = inp_transform(x)- x, _, _ = self.orig_layer.act_quant_func(+ import auto_round.envs as envs++ act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None+ act_scale = envs.AR_ACT_SCALE+ max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale+ min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale+ x = self.activation_quantizer.qdq(x,- bits=self.orig_layer.act_bits,- group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.orig_layer.q_scale_thresh,- data_type=self.orig_layer.act_data_type,- tensor_max=act_max,+ observed_max=act_max,+ min_scale=min_scale,+ max_scale=max_scale,)return self.orig_layer.forward(x)diff --git a/auto_round/algorithms/transforms/svdquant/residual.py b/auto_round/algorithms/transforms/svdquant/residual.pyindex 461f6763b..aa3515b3e 100644--- a/auto_round/algorithms/transforms/svdquant/residual.py+++ b/auto_round/algorithms/transforms/svdquant/residual.py@@ -26,7 +26,7 @@ from dataclasses import dataclassimport torch-from auto_round.data_type.utils import get_quant_func+from auto_round.data_type.base import create_quantizer, quantize_activation_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})@@ -108,16 +108,12 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.requested_dtype = values["data_type"]if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:requested_dtype = f"{requested_dtype}_rceil"- quant_func, resolved_dtype = get_quant_func(- dtype=requested_dtype,- bits=values["bits"],- sym=values["sym"],+ quantizer = create_quantizer(+ {**values, "data_type": requested_dtype, "scale_dtype": tensor.dtype},disable_opt_rtn=True,- group_size=values["group_size"],- iters=0,)- logical_dtype = resolved_dtype.removeprefix("rtn_")- resolved_base_dtype = logical_dtype.removesuffix("_rceil")+ quantizer.initialize(tensor)+ resolved_base_dtype = requested_dtype.removesuffix("_rceil")if (resolved_base_dtype in _MXFP4_ALIASESand values["bits"] == 4@@ -132,12 +128,7 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.f"got group_size={values['group_size']!r}.")- qdq, _, _ = quant_func(- tensor=tensor,- bits=values["bits"],- group_size=values["group_size"],- data_type=logical_dtype,- )+ qdq = quantizer.quantize(tensor)if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:raise ValueError(f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "@@ -162,7 +153,13 @@ def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch@torch.inference_mode()def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:"""Apply deployment-compatible dynamic activation quantize-dequantize."""- return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")+ values = _validate_scheme_values(scheme)+ qdq = quantize_activation(activation, values)+ if qdq.shape != activation.shape or qdq.dtype != activation.dtype or qdq.device != activation.device:+ raise ValueError("Activation RTN QDQ must preserve shape, dtype, and device.")+ if not torch.isfinite(qdq).all():+ raise ValueError("Activation RTN QDQ produced non-finite values.")+ return qdq@torch.inference_mode()diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 8d23ff152..8d383e311 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -44,13 +44,7 @@ from auto_round.auto_scheme.utils import (remove_quant_scheme,)from auto_round.calib_dataset import get_dataloader-from auto_round.data_type.gguf import (- quant_tensor_gguf_asym_dq,- quant_tensor_gguf_sym_dq,- search_gguf_scale_min_asym,- search_gguf_scale_min_sym,-)-from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.base import canonical_data_typefrom auto_round.logger import loggerfrom auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_from auto_round.schemes import QuantizationScheme, preset_name_to_scheme@@ -286,278 +280,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):return qdq_w, 1.0, None-class AutoSchemeWrapperLinearIMatrix(WrapperLinear):- """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""-- def __init__(- self,- orig_layer,- enable_minmax_tuning=True,- enable_norm_bias_tuning=False,- device="cpu",- enable_round_tuning=True,- need_weight_grad=False,- enable_torch_compile=True,- **kwargs,- ):- """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""- super().__init__(- orig_layer,- enable_minmax_tuning,- enable_norm_bias_tuning,- device,- enable_round_tuning,- enable_torch_compile=enable_torch_compile,- **kwargs,- )- self.act_score = 0.0- self.avg_act_score = 0.0- self.act_cnt = 0.0- self.weight_score = 0.0- self.mix_score = 0.0- self.super_qdq_func = super()._qdq_weight- self.act_qdq_func = super()._qdq_act- self.max_act_value = 0- self.need_weight_grad = need_weight_grad- self.grad_mode = False- if self.need_weight_grad:- self.orig_layer.weight.requires_grad = True- self.weight_search_quant_func, _ = get_quant_func(- orig_layer.data_type,- orig_layer.bits,- orig_layer.sym,- disable_opt_rtn=False,- group_size=orig_layer.group_size,- iters=0,- )- self._custom_score_forward = False- self.post_init_qdqw(device)-- @torch.no_grad()- def post_init_qdqw(self, device):- """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,- registering a backward hook on it to accumulate ``weight_score``.- """- qdq_w, _, _ = self.weight_search_quant_func(- self.orig_layer.weight.to(device),- bits=self.orig_layer.bits,- group_size=self.orig_layer.group_size,- v=torch.tensor(0, device=device),- min_scale=torch.tensor(1.0, device=device),- max_scale=torch.tensor(1.0, device=device),- scale_dtype=self.orig_layer.scale_dtype,- data_type=self.data_type,- q_scale_thresh=self.q_scale_thresh,- imatrix=self.orig_layer.imatrix.to(device) if hasattr(self.orig_layer, "imatrix") else None,- global_scale=getattr(self, "weight_global_scale", None),- )-- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))-- def save_grad(grad):- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()- self.mix_score = self.weight_score + self.act_score- return None-- self.qdq_w.requires_grad_(True)- self.orig_layer.weight.requires_grad_(False)-- self.qdq_w.register_hook(save_grad)-- def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):- """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that- accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.- """- if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:- return x, 1.0, None-- qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)- if self.grad_mode:- with torch.no_grad():- max_act_value = torch.abs(x).max()- self.max_act_value = max_act_value- if max_act_value != 0:- self.act_cnt += 1- x_diff = (x - qdq_x).to("cpu")-- def save_grad(grad):- """Backward hook: accumulate activation score from grad * (x - qdq_x)."""- if max_act_value == 0:- if torch.abs(grad).max() != 0:- raise ValueError- """- this ut will cause NAN issue sometimes, need to investigate- @multi_card- def test_multi_card(self):- model_name = "/models/Qwen3-8B"- """- if torch.isnan(grad).any() or torch.isnan(x_diff).any():- self.act_cnt -= 1- return None-- self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()- self.mix_score = self.weight_score + self.act_score- return None-- if qdq_x.requires_grad:- qdq_x.register_hook(save_grad)- return qdq_x, scale, zp-- def _qdq_weight(self, value, min_scale, max_scale):- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""- return self.qdq_w, 1.0, None---class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):- """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""-- def __init__(- self,- orig_layer,- enable_minmax_tuning=True,- enable_norm_bias_tuning=False,- device="cpu",- enable_round_tuning=True,- need_weight_grad=False,- **kwargs,- ):- """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""- super().__init__(- orig_layer,- enable_minmax_tuning,- enable_norm_bias_tuning,- device,- enable_round_tuning,- need_weight_grad,- **kwargs,- )- self._custom_score_forward = False- self.post_init_qdqw(device)-- @torch.no_grad()- def post_init_qdqw(self, device):- """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,- registering a backward hook on it to accumulate ``weight_score``.- """- qdq_w, scale, zp = self.super_qdq_func(- torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)- )- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))-- def save_grad(grad):- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- # TODO strange, grad could be in CPU- self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()- self.mix_score = self.weight_score + self.act_score- return None-- self.qdq_w.requires_grad_(True)- self.orig_layer.weight.requires_grad_(False)- self.qdq_w.register_hook(save_grad)-- def _qdq_weight(self, value, min_scale, max_scale):- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""- return self.qdq_w, 1.0, None---class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):- """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant- search (``_init_scale``).- """-- def __init__(- self,- orig_layer,- enable_minmax_tuning=True,- enable_norm_bias_tuning=False,- device="cpu",- enable_round_tuning=True,- need_weight_grad=False,- enable_torch_compile=True,- **kwargs,- ):- """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to- build ``qdq_w``.- """- super().__init__(- orig_layer,- enable_minmax_tuning,- enable_norm_bias_tuning,- device,- enable_round_tuning,- need_weight_grad,- enable_torch_compile=enable_torch_compile,- **kwargs,- )- self._custom_score_forward = False- self.post_init_qdqw(device)-- @torch.no_grad()- def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher- """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer- ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.- """- qdq_w = self._init_scale(device).detach()- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))-- def save_grad(grad):- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()- self.mix_score = self.weight_score + self.act_score- return None-- self.qdq_w.requires_grad_(True)- self.orig_layer.weight.requires_grad_(False)-- self.qdq_w.register_hook(save_grad)-- @torch.no_grad()- def _init_scale(self, device):- """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in- [2,3,4,5,6], returned in the original weight dtype.- """- tensor = self.orig_layer.weight.data.to(device)- bits = self.orig_layer.bits- scale_dtype = self.orig_layer.scale_dtype- imatrix = self.orig_layer.imatrix.to(tensor.device)- orig_dtype = tensor.dtype- if self.orig_layer.bits in [2, 4, 5]:- group_size = 16 if bits == 2 else 32- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)-- qdq_w, _, _ = quant_tensor_gguf_asym_dq(- tensor=tensor,- bits=bits,- scale_dtype=scale_dtype,- imatrix=imatrix,- scale=scale,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- )- elif bits in [3, 6]:- qdq_w, _, _ = quant_tensor_gguf_sym_dq(- tensor=tensor,- bits=bits,- scale_dtype=scale_dtype,- imatrix=imatrix,- split_num=1,- )- else:- raise ValueError("bits must be in [2,3,4,5,6]")- return qdq_w.to(orig_dtype)-- def _qdq_weight(self, value, min_scale, max_scale):- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""- return self.qdq_w, 1.0, None--def register_imatrix_hook(model):"""Registers hooks to accumulate activation squared norms into `imatrix`."""@@ -814,7 +536,13 @@ def _replay_retain_graph(block_module) -> bool:"""for _, module in block_module.named_modules():data_type = getattr(module, "data_type", None)- if isinstance(data_type, str) and data_type.startswith("mx"):+ if not isinstance(data_type, str):+ continue+ try:+ canonical_id = canonical_data_type(data_type)+ except LookupError:+ continue+ if canonical_id.startswith("mx_"):return Truereturn False@@ -1224,15 +952,6 @@ def get_score_for_scheme(if hasattr(m, "bias") and m.bias is not None:m.bias.requires_grad = False- has_imatrix = False- for name in quant_layer_names:- if name in fixed_layer_scheme.keys():- continue- m = get_module(model, name)- if hasattr(m, "imatrix") and m.imatrix is not None:- has_imatrix = True- break-for name in quant_layer_names:if offload_context is not None:offload_context.ensure_loaded(model, name)@@ -1253,14 +972,6 @@ def get_score_for_scheme(m.scale_dtype = torch.bfloat16WrapperLayer = AutoSchemeWrapperLinear- # if has_imatrix: # no better result- # WrapperLayer = AutoSchemeWrapperLinearIMatrix- if hasattr(m, "super_group_size") and m.super_group_size is not None:- if has_imatrix:- WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix- else:- WrapperLayer = AutoSchemeWrapperLinearForGGUFK-with torch.no_grad():if low_gpu_mem_usage:device = m.tuning_device if hasattr(m, "tuning_device") else major_devicediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 4e1968684..d5617e7aa 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1295,8 +1295,34 @@ class BaseOrchestrator(object):return get_torch_compile_off_reason(model)def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:- """Apply torch.compile disabling rules for the current compressor state."""- self.enable_torch_compile = enable_torch_compile+ """Apply torch.compile disabling rules for the current compressor state.++ This is intentionally kept beside the compressor state it reads. The+ rules are not reusable policy: every input comes from this instance,+ and preserving that context makes the precedence easy to audit.+ """+ from auto_round.algorithms.quantization.rtn.config import RTNConfig++ _, is_valid_act_static = self._get_torch_compile_guard_state()+ reason = None+ if enable_torch_compile:+ if is_valid_act_static:+ reason = "activation is static"+ else:+ reason = self._torch_compile_unsupported_arch_reason()+ user_specified = getattr(self, "_torch_compile_user_specified", False)+ is_auto_scheme = getattr(self, "is_auto_scheme", False) or isinstance(+ getattr(self, "scheme", None), AutoScheme+ )+ if reason is None and not user_specified and not is_auto_scheme:+ if isinstance(self.quantize_config, RTNConfig):+ reason = "RTN/OPT-RTN quantizes each layer in a single pass"+ else:+ iters = getattr(self.quantize_config, "iters", None)+ if iters is not None and iters < MIN_ITERS_FOR_TORCH_COMPILE:+ reason = f"`iters`={iters} is below {MIN_ITERS_FOR_TORCH_COMPILE}"++ self.enable_torch_compile = enable_torch_compile and reason is None# Why compilation ended up off, used by ``_log_torch_compile_state``. When the# incoming value is already False, keep the reason recorded by the earlier# precheck pass instead of dropping it.@@ -1308,33 +1334,9 @@ class BaseOrchestrator(object):or getattr(self, "_torch_compile_default_off_reason", None)))- _, is_valid_act_static = self._get_torch_compile_guard_state()-- # On HPU, we rely on torch.compile to speed up the model execution.- if self.enable_torch_compile and is_valid_act_static:- self.enable_torch_compile = False- self._torch_compile_off_reason = "activation is static"- logger.warning_once("reset enable_torch_compile to `False` as activation is static")-- # Architecture-level hard block (DeepSeek / GLM-5.3-Flash DSA families). These- # hit dynamo's recompile_limit and cannot be overridden by an explicit- # ``enable_torch_compile=True``.- if self.enable_torch_compile:- arch_reason = self._torch_compile_unsupported_arch_reason()- if arch_reason is not None:- self.enable_torch_compile = False- self._torch_compile_off_reason = arch_reason- logger.warning_once("reset enable_torch_compile to `False` as %s", arch_reason)-- if self.enable_torch_compile:- disabled_reason = self._torch_compile_disabled_reason()- if disabled_reason is not None:- self.enable_torch_compile = False- self._torch_compile_off_reason = disabled_reason- logger.warning_once(- "reset enable_torch_compile to `False` as %s, " "so compilation cost would outweigh its benefit",- disabled_reason,- )+ if reason is not None:+ self._torch_compile_off_reason = reason+ logger.warning_once("reset enable_torch_compile to `False` as %s", reason)def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:"""Apply early torch.compile adjustments before scheme resolution.diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.pyindex 48df479ed..d790485fb 100644--- a/auto_round/compressors/layer_config_resolver.py+++ b/auto_round/compressors/layer_config_resolver.py@@ -38,6 +38,31 @@ from auto_round.utils import (from auto_round.utils.model import get_module+def normalize_layer_config_entry(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:+ """Resolve one public layer override through the existing preset machinery."""+ from auto_round.schemes import QuantizationScheme, preset_name_to_scheme++ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)+ if isinstance(item, str):+ config = asdict(preset_name_to_scheme(item.upper()))+ elif isinstance(item, QuantizationScheme):+ config = asdict(item)+ elif isinstance(item, dict):+ item = {k: v for k, v in item.items() if k != "in_blocks"}+ scheme_name = item.pop("scheme", None)+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}+ invalid = set(item) - set(scheme_keys + ("fixed_by_user",))+ if invalid:+ raise ValueError(f"Invalid keys {invalid} in layer_config for '{layer_name}'. Allowed keys: {scheme_keys}")+ config.update(item)+ else:+ raise TypeError(+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "+ "Expected str, dict, or QuantizationScheme."+ )+ return {key: value for key, value in config.items() if value is not None}++def apply_layer_config_special_cases(layer_config,model,@@ -170,34 +195,6 @@ def _resolve_layer_config_presets(bits-inference, and default-filling."""from auto_round.schemes import QuantizationScheme, preset_name_to_scheme- def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:- """Convert config entry into dict and validate keys."""- if isinstance(item, str):- config = asdict(preset_name_to_scheme(item.upper()))- elif isinstance(item, QuantizationScheme):- config = asdict(item)- elif isinstance(item, dict):- # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;- # silently drop it before validation.- item = {k: v for k, v in item.items() if k != "in_blocks"}- scheme_name = item.pop("scheme", None)- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}- invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))- if invalid:- raise ValueError(- f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"- )- config.update(item)- else:- raise TypeError(- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "- f"Expected str, dict, or QuantizationScheme."- )- # Clean up- config = {k: v for k, v in config.items() if v is not None}- config["fixed_by_user"] = True- return config-extra_scheme_keys = ("scale_dtype",)scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)layer_config = copy.deepcopy(layer_config) or {}@@ -218,7 +215,7 @@ def _resolve_layer_config_presets(}# 2. normalize- layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}+ layer_config = {k: {**normalize_layer_config_entry(v, k), "fixed_by_user": True} for k, v in layer_config.items()}# entries whose sym was set by the user (dict key or a QuantizationScheme# object) rather than inherited from the default scheme belowexplicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex 58c746470..58bd9076a 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -15,7 +15,7 @@import auto_round.data_type.intimport auto_round.data_type.mxfpimport auto_round.data_type.fp8-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPEimport auto_round.data_type.w4fp8from auto_round.data_type.utils import (get_quant_func,@@ -24,3 +24,8 @@ from auto_round.data_type.utils import ()import auto_round.data_type.nvfpimport auto_round.data_type.gguf+from auto_round.data_type.base import (+ DataTypeQuantizer,+ canonical_data_type,+ create_quantizer,+)diff --git a/auto_round/data_type/base.py b/auto_round/data_type/base.pynew file mode 100644index 000000000..c34d336fb--- /dev/null+++ b/auto_round/data_type/base.py@@ -0,0 +1,313 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0++"""Small public boundary between quantization algorithms and data types.++Algorithms describe *what* a layer needs quantized. This module resolves the+requested datatype and delegates all datatype-specific work to that datatype's+class. It deliberately contains no quantization math.+"""++from collections.abc import Mapping+from dataclasses import dataclass, replace+from types import SimpleNamespace+from typing import Any, Callable++import torch++# Low-level tensor primitives remain functions. Quantizer classes below own+# their lifecycle and are the only interface used by algorithms and wrappers.+QUANT_FUNC_WITH_DTYPE: dict[str, Callable] = {}+_QUANTIZER_CLASSES: dict[str, type] = {}+_CANONICAL_DATA_TYPES: dict[str, str] = {}+++def _normalize_data_type(data_type: str) -> str:+ """Normalize spelling differences in datatype names for lookup."""+ return data_type.lower().replace("-", "").replace("_", "")+++def register_dtype(names):+ """Register a low-level tensor quantization primitive by datatype name."""++ def register(function):+ for name in (names,) if isinstance(names, str) else names:+ QUANT_FUNC_WITH_DTYPE[name] = function+ return function++ return register+++def register_quantizer(canonical_id: str, *, aliases: tuple[str, ...] = ()):+ """Register a datatype class under its canonical name and aliases.++ Datatype modules use this decorator at import time. The mapping exists+ solely to turn user-facing aliases such as ``"int"`` into the class that+ owns their implementation.+ """++ def register(quantizer_class):+ names = (canonical_id, *aliases)+ normalized = tuple(dict.fromkeys(_normalize_data_type(name) for name in names))+ duplicate = next((name for name in normalized if name in _QUANTIZER_CLASSES), None)+ if duplicate is not None:+ raise ValueError(f"Datatype name {duplicate!r} is already registered")+ for name in normalized:+ _QUANTIZER_CLASSES[name] = quantizer_class+ _CANONICAL_DATA_TYPES[name] = canonical_id+ return quantizer_class++ return register+++def canonical_data_type(data_type: str) -> str:+ """Return an alias's canonical name for datatype-dependent policy.++ This is intentionally separate from construction: callers such as loss+ policy and block preparation need to classify a datatype without creating+ a quantizer or allocating tensors.+ """+ try:+ return _CANONICAL_DATA_TYPES[_normalize_data_type(data_type)]+ except KeyError as error:+ raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error+++def _quantizer_class(data_type: str) -> type:+ """Look up the class that owns a datatype implementation."""+ try:+ return _QUANTIZER_CLASSES[_normalize_data_type(data_type)]+ except KeyError as error:+ raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error+++def _create_weight_quantizer(data_type: str, spec: "WeightQuantizationSpec"):+ """Create the implementation owned by ``data_type`` for one weight request."""+ canonical = canonical_data_type(data_type)+ quantizer_class = _quantizer_class(data_type)+ try:+ return quantizer_class.from_spec(spec, canonical)+ except AttributeError as error:+ raise LookupError(f"Datatype {data_type!r} supports activation quantization only") from error+++def _create_activation_quantizer(data_type: str, spec: "ActivationQuantizationSpec"):+ """Create the activation implementation owned by ``data_type``."""+ canonical = canonical_data_type(data_type)+ try:+ return _quantizer_class(data_type).create_activation(replace(spec, data_type=canonical))+ except AttributeError as error:+ raise LookupError(f"Datatype {data_type!r} has no activation quantizer") from error+++def prepare_data_type_block(data_type: str, block, layer_runtimes) -> None:+ """Run the optional block-level setup owned by one datatype class."""+ quantizer_class = _QUANTIZER_CLASSES.get(_normalize_data_type(data_type))+ prepare = getattr(quantizer_class, "prepare_block", None)+ if prepare is not None:+ prepare(block, layer_runtimes)+++@dataclass(frozen=True)+class WeightQuantizationSpec:+ """Static properties needed to quantize one layer's weights."""++ data_type: str+ bits: int+ group_size: int | tuple[int, int]+ sym: bool+ scale_dtype: torch.dtype+ q_scale_thresh: float = 1e-5+ super_bits: int | None = None+ super_group_size: int | None = None+ global_scale: torch.Tensor | None = None+ clip_min: torch.Tensor | None = None+ clip_max: torch.Tensor | None = None+++@dataclass(frozen=True)+class ActivationQuantizationSpec:+ """Static properties needed to quantize one layer's activations."""++ data_type: str+ bits: int+ group_size: int | tuple[int, int]+ sym: bool+ scale_dtype: torch.dtype+ dynamic: bool+ q_scale_thresh: float = 1e-5+++@dataclass(frozen=True)+class WeightQuantizationResult:+ """Weight QDQ output plus the tensors required by export and inference."""++ weight: torch.Tensor+ scale: torch.Tensor | dict[str, torch.Tensor] | None = None+ zero_point: torch.Tensor | dict[str, torch.Tensor] | int | float | None = None+ metadata: object | None = None+ # Conv1D weights are transposed before export. Keep the original output+ # dimension so per-output-channel tensors retain their intended layout.+ logical_rows: int | None = None+++class DataTypeQuantizer:+ """Run a datatype implementation through initialize, QDQ, and write-back.++ This is the only lifecycle wrapper shared by all data types. Datatype+ classes keep their own state and numerical code; this class only manages+ the sequence and the trainable parameter dictionary.+ """++ def __init__(self, implementation: Any, mode: str, tune_rounding: bool, tune_minmax: bool, apply_result):+ self._implementation = implementation+ self._mode = mode+ self._tune_rounding = tune_rounding+ self._tune_minmax = tune_minmax+ self._apply_result = apply_result+ self._state = None+ self.parameters: dict[str, torch.Tensor] = {}++ def initialize(self, weight: torch.Tensor, *, imatrix=None) -> None:+ """Initialize datatype state and expose its trainable tensors."""+ self._state = self._implementation.create_state(+ weight,+ imatrix=imatrix,+ mode=self._mode,+ tune_rounding=self._tune_rounding,+ tune_minmax=self._tune_minmax,+ )+ tunables = self._state.get("tunables", self._state) if isinstance(self._state, dict) else self._state.tunables+ self.parameters = dict(tunables)++ def refresh(self, weight: torch.Tensor, *, imatrix=None) -> None:+ """Rebuild fixed datatype state while preserving existing trainable tensors."""+ previous_parameters = self.parameters+ self.initialize(weight, imatrix=imatrix)+ if isinstance(self._state, dict):+ self._state["tunables"] = previous_parameters+ else:+ self._state = replace(self._state, tunables=previous_parameters)+ self.parameters = previous_parameters++ def quantize(self, weight: torch.Tensor, **parameters) -> torch.Tensor:+ """Return QDQ weights for the current trainable parameter values."""+ return self.qdq(weight, **parameters).weight++ def qdq(self, weight: torch.Tensor, *, materialize=False, **parameters) -> WeightQuantizationResult:+ """Run datatype QDQ and optionally include tensors needed for export."""+ self._ensure_initialized()+ return self._implementation.qdq(+ weight,+ self._state,+ tunables={**self.parameters, **parameters},+ materialize=materialize,+ )++ def apply_result(self, layer, result: WeightQuantizationResult, *, transpose=False) -> None:+ """Store an already computed QDQ result on its source layer."""+ logical_rows = result.weight.shape[0]+ if transpose:+ result = WeightQuantizationResult(+ result.weight.t(), result.scale, result.zero_point, result.metadata, logical_rows+ )+ if result.weight.device != layer.weight.device:+ result = WeightQuantizationResult(+ result.weight.to(layer.weight.device),+ result.scale,+ result.zero_point,+ result.metadata,+ result.logical_rows,+ )+ self._apply_result(layer, result)++ def write_back(self, layer, weight: torch.Tensor, *, transpose=False, **parameters) -> None:+ """Materialize QDQ payload and store it on an exportable layer."""+ result = self.qdq(weight, materialize=True, **parameters)+ self.apply_result(layer, result, transpose=transpose)++ def _ensure_initialized(self) -> None:+ """Reject use before a datatype has created its per-layer state."""+ if self._state is None:+ raise RuntimeError("Call initialize() before quantize() or write_back().")+++def _weight_spec(layer) -> WeightQuantizationSpec:+ """Extract the weight request from a resolved quantized layer."""+ scale_dtype = getattr(layer, "scale_dtype", torch.float32)+ return WeightQuantizationSpec(+ data_type=layer.data_type,+ bits=layer.bits,+ group_size=layer.group_size,+ sym=layer.sym,+ scale_dtype=scale_dtype,+ q_scale_thresh=getattr(layer, "q_scale_thresh", 1e-5),+ super_bits=getattr(layer, "super_bits", None),+ super_group_size=getattr(layer, "super_group_size", None),+ global_scale=getattr(layer, "weight_global_scale", None),+ clip_min=getattr(layer, "awq_clip_min", None),+ clip_max=getattr(layer, "awq_clip_max", None),+ )+++def _activation_spec(layer) -> ActivationQuantizationSpec:+ """Extract the activation request from a resolved quantized layer."""+ return ActivationQuantizationSpec(+ data_type=getattr(layer, "act_data_type", "int_sym"),+ bits=getattr(layer, "act_bits", 16),+ group_size=getattr(layer, "act_group_size", -1),+ sym=getattr(layer, "act_sym", True),+ scale_dtype=getattr(layer, "scale_dtype", torch.float32),+ dynamic=getattr(layer, "act_dynamic", True),+ )+++def create_quantizer(layer, *, iters=None, disable_opt_rtn=False, tune_rounding=False, tune_minmax=False):+ """Create the datatype-owned weight quantizer for a resolved layer."""+ if isinstance(layer, Mapping):+ layer = SimpleNamespace(**{"q_scale_thresh": 1e-5, **layer})+ spec = _weight_spec(layer)+ iters = getattr(layer, "iters", 0) if iters is None else iters+ mode = "tuned" if iters > 0 else "rtn" if disable_opt_rtn else "optimized_rtn"+ implementation = _create_weight_quantizer(spec.data_type, spec)+ return DataTypeQuantizer(implementation, mode, tune_rounding, tune_minmax, implementation.apply_result)+++def cache_activation_quantizer(layer):+ """Create and cache a layer's activation quantizer when activation is quantized."""+ if getattr(layer, "act_bits", 16) > 8:+ return None+ if hasattr(layer, "_ar_activation_quantizer"):+ return layer._ar_activation_quantizer+ spec = _activation_spec(layer)+ quantizer = _create_activation_quantizer(spec.data_type, spec)+ layer._ar_activation_quantizer = quantizer+ return quantizer+++def activation_quantizer_for_layer(layer, *, scale_dtype=None):+ """Create the activation quantizer described by a layer's configuration."""+ spec = _activation_spec(layer)+ if scale_dtype is not None:+ from dataclasses import replace++ spec = replace(spec, scale_dtype=scale_dtype)+ return _create_activation_quantizer(spec.data_type, spec)+++def quantize_activation(tensor, config):+ """Apply dynamic activation quantization using a simple configuration mapping."""+ spec = ActivationQuantizationSpec(+ data_type=config["data_type"],+ bits=config["bits"],+ group_size=config["group_size"],+ sym=config["sym"],+ scale_dtype=tensor.dtype,+ dynamic=True,+ )+ return _create_activation_quantizer(spec.data_type, spec).qdq(tensor)diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex 11f4594c2..708390e63 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.register import register_dtype+from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.utils import (float8_e4m3fn_ste,float8_e5m2_ste,@@ -366,3 +366,136 @@ if is_gaudi2():qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)qdq_res = qdq_res.to(orig_dtype)return qdq_res, scale, None+++class _FP8WeightQuantizer:+ """Select the requested FP8 format and materialize its scale layout."""++ def __init__(self, spec, primitive=None, scale_layout=None, data_type=None):+ self.spec = spec+ data_type = (data_type or spec.data_type).lower()+ if primitive is None:+ if data_type in ("fp8_e5m2",):+ primitive, scale_layout = quant_fp8_e5m2, "row"+ elif data_type in ("fp8_unit_scale",):+ primitive, scale_layout = quant_fp8_unit_scale, "scalar"+ elif data_type in ("fp8_e5m2_unit_scale",):+ primitive, scale_layout = quant_fp8_e5m2_unit_scale, "scalar"+ elif data_type == "fp8_gaudi3_sym":+ primitive, scale_layout = quant_fp8_sym_gaudi3, "row"+ elif isinstance(spec.group_size, tuple):+ primitive, scale_layout = quant_block_fp_sym, "block"+ else:+ primitive, scale_layout = quant_fp8_sym, "row"+ self.primitive = primitive+ self.scale_layout = scale_layout++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the FP8 weight quantizer selected by the requested format."""+ return cls(spec, data_type=canonical)++ @staticmethod+ def create_activation(spec):+ """Create the FP8 activation quantizer for the same format."""+ return _create_fp8_activation(spec)++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ reduction_dims = 2 if self.scale_layout == "block" else 1+ tunable_shape = grouped.shape[:-reduction_dims]+ tunables = {}+ if tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if tune_minmax:+ tunables["max_scale"] = torch.nn.Parameter(+ torch.ones(tunable_shape, device=weight.device, dtype=torch.float32)+ )+ return tunables++ def qdq(self, weight, state, *, tunables, materialize=False):+ quantized, scale, zero_point = self.primitive(+ weight,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ v=tunables.get("value", 0),+ max_scale=tunables.get("max_scale", 1.0),+ )+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(+ quantized,+ scale if materialize else None,+ zero_point if materialize else None,+ self.scale_layout if materialize else None,+ )++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("FP8 weight result was not materialized")+ scale = result.scale+ rows = result.logical_rows or result.weight.shape[0]+ if result.metadata == "row" and scale.numel() % rows == 0:+ scale = scale.reshape(rows, -1)+ module.weight.data.copy_(result.weight)+ module.scale = scale.cpu()+ module.zp = result.zero_point+++class _FP8ActivationQuantizer:+ """Apply the matching FP8 primitive to dynamic or calibrated activations."""++ def __init__(self, spec, data_type, primitive, scale_layout):+ self.spec = spec+ self.data_type = data_type+ self.primitive = primitive+ self.scale_layout = scale_layout+ self.bits = spec.bits+ self.group_size = spec.group_size+ self.requires_calibration = not spec.dynamic++ def observe(self, activation, current):+ if self.scale_layout == "tensor":+ maximum = activation.detach().abs().max()+ else:+ grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)+ dims = (-2, -1) if self.scale_layout == "block" else -1+ maximum = grouped.detach().abs().amax(dim=dims)+ return maximum if current is None else torch.maximum(maximum.to(current), current)++ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ if self.requires_calibration and observed_max is None:+ raise ValueError(f"{self.data_type} activation requires observed_max")+ return self.primitive(+ activation,+ bits=self.bits,+ group_size=self.group_size,+ tensor_max=observed_max if self.requires_calibration else None,+ max_scale=max_scale,+ )++ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ quantized, _, _ = self.qdq_with_scale(+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale+ )+ return quantized+++def _create_fp8_activation(spec):+ """Build an activation quantizer using the same format choice as weights."""+ if spec.data_type == "fp8_gaudi3_sym":+ return _FP8ActivationQuantizer(spec, spec.data_type, quant_fp8_sym_gaudi3, "tensor")+ weight_quantizer = _FP8WeightQuantizer(spec)+ return _FP8ActivationQuantizer(spec, spec.data_type, weight_quantizer.primitive, weight_quantizer.scale_layout)+++register_quantizer(+ "fp8_sym",+ aliases=("fp", "float", "fp8", "fp8_e4m3", "block_fp8_sym", "block_fp8", "block_fp8_e4m3"),+)(_FP8WeightQuantizer)+register_quantizer("fp8_e5m2")(_FP8WeightQuantizer)+register_quantizer("fp8_unit_scale")(_FP8WeightQuantizer)+register_quantizer("fp8_e5m2_unit_scale")(_FP8WeightQuantizer)+if is_gaudi2():+ register_quantizer("fp8_gaudi3_sym")(_FP8WeightQuantizer)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 2b4de2b7e..dea59a70e 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -11,13 +11,14 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from collections.abc import Mapping+from dataclasses import dataclassfrom typing import Any, Callable, Unionimport torch-from auto_round.data_type.register import register_dtype+from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_Kfrom auto_round.export.export_to_gguf.packing import (_LARGE_TENSOR_BYTES_THRESHOLD,make_q3_quants,@@ -25,6 +26,15 @@ from auto_round.export.export_to_gguf.packing import (make_qx_quants_chunk,)from auto_round.logger import logger+++def get_gguf_quant_constants():+ """Return GGUF block-layout constants without exposing export internals to algorithms."""+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ return GGML_QUANT_SIZES, QK_K++from auto_round.utils import get_reciprocalfrom auto_round.utils.device import clear_memory@@ -486,9 +496,18 @@ def _imatrix_handle_zero(@torch.inference_mode()-def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1, v=0):- super_bits = 4 if bits == 2 else 6- super_group_size = 16 if bits == 2 else 8+def search_gguf_scale_min_asym(+ tensor,+ bits=4,+ scale_dtype=torch.float16,+ imatrix=None,+ split_num=1,+ v=0,+ super_bits=None,+ super_group_size=None,+):+ super_bits = super_bits if super_bits is not None else 4 if bits == 2 else 6+ super_group_size = super_group_size if super_group_size is not None else 16 if bits == 2 else 8quant_weights = Noneif imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):@@ -604,6 +623,9 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatridef quant_tensor_gguf_asym_dq(tensor: torch.Tensor,bits: int = 4,+ group_size=None,+ super_bits=None,+ super_group_size=None,v=0,scale_dtype=torch.float16,imatrix=None,@@ -634,7 +656,7 @@ def quant_tensor_gguf_asym_dq(raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")orig_dtype = tensor.dtypemaxq = int(2.0**bits) - 1- group_size = 16 if bits == 2 else 32+ group_size = group_size if group_size is not None else 16 if bits == 2 else 32# The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),# unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large# tensors. For very large single tensors (e.g. huge vocab embed/output projections),@@ -665,6 +687,8 @@ def quant_tensor_gguf_asym_dq(imatrix,split_num=split_num,v=v,+ super_bits=super_bits,+ super_group_size=super_group_size,)scale = scale.clone()wmin = wmin.clone()@@ -903,6 +927,9 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0def quant_tensor_gguf_sym_dq(tensor,bits=3,+ group_size=16,+ super_bits=None,+ super_group_size=16,v=0,imatrix=None,scale=None,@@ -930,21 +957,15 @@ def quant_tensor_gguf_sym_dq(Quantized and de-quantized tensor, scale, zero-point"""- from auto_round.export.export_to_gguf.config import K_SCALE_SIZE-if bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")maxq = int(2.0 ** (bits - 1))- group_size = 16tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)orig_dtype = tensor.dtype- super_bits = 6 if bits == 3 else 8- super_group_size = 16- ggml_type = f"q{bits}_k"- block_size, type_size = GGML_QUANT_SIZES[ggml_type]+ super_bits = super_bits if super_bits is not None else 6 if bits == 3 else 8tensor = tensor.to(torch.float32)- n_blocks = tensor.nelement() // block_size+ n_blocks = tensor.nelement() // (super_group_size * group_size)# Chunk the candidate-scale search for very large single tensors (e.g. huge vocab# embed/output projections). This search allocates several full-size float32 buffers# at once; unchunked, that can need tens of GB for a single tensor and OOM even though@@ -955,8 +976,7 @@ def quant_tensor_gguf_sym_dq(# ordinary attention/FFN weights never needed it and would just get slower for no benefit.if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:split_num = 16- # (nb, 16, 16)- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ tensor = tensor.reshape(n_blocks, super_group_size, group_size)if iter is None:iter = 0use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1@@ -980,3 +1000,161 @@ def quant_tensor_gguf_sym_dq(tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)return tensor, {"scale": scale, "d_scale": d_scale}, maxq+++@dataclass+class _GGUFState:+ """Persistent GGUF RTN scales and double-quantization payload between QDQ calls."""++ tunables: Mapping[str, torch.Tensor]+ tensor_min: torch.Tensor | None+ tensor_max: torch.Tensor | None+ imatrix: torch.Tensor | float | int | None+ scale: torch.Tensor | None = None+ wmin: torch.Tensor | None = None+ d_scale: torch.Tensor | None = None+ d_wmin: torch.Tensor | None = None+ prev_scale: torch.Tensor | None = None+ prev_wmin: torch.Tensor | None = None+ prev_d_scale: torch.Tensor | None = None+ prev_d_wmin: torch.Tensor | None = None+ iteration: int = 0+++class _GGUFWeightQuantizer:+ """Own GGUF double-quant weight QDQ and its export payload."""++ def __init__(self, spec, family="plain", kind=None):+ self.spec = spec+ self.family = family+ self.kind = kind or (+ "sym" if spec.data_type == "int_sym_dq" else "float_zp" if spec.data_type == "int_asym_float_zp" else "asym"+ )++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the GGUF quantizer; its state selects tuned or RTN behavior."""+ kind = {"int_sym_dq": "sym", "int_asym_float_zp": "float_zp"}.get(canonical, "asym")+ return cls(spec, kind=kind)++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ self.family = "rtn" if mode in ("rtn", "optimized_rtn") else "plain"+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)+ tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)+ tunables = {}+ if self.family == "plain" and tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if self.family == "plain" and tune_minmax:+ shape = tensor_min.shape+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))+ return _GGUFState(tunables, tensor_min, tensor_max, imatrix)++ def qdq(self, weight, state, *, tunables, materialize=False):+ value = tunables.get("value", 0)+ min_scale = tunables.get("min_scale", 1.0)+ max_scale = tunables.get("max_scale", 1.0)+ if isinstance(min_scale, torch.Tensor):+ min_scale.data.clamp_(0.0, 1.0)+ if isinstance(max_scale, torch.Tensor):+ max_scale.data.clamp_(0.0, 1.0)++ if self.kind == "float_zp" and self.family == "plain":+ quantized, scale, zero_point = quant_tensor_asym_float_zp(+ weight,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ v=value,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.spec.scale_dtype,+ tensor_min=state.tensor_min,+ tensor_max=state.tensor_max,+ q_scale_thresh=self.spec.q_scale_thresh,+ )+ elif self.kind == "float_zp":+ quantized, scale, zero_point = quant_tensor_asym_float_zp_rtn(+ weight.clone(),+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ scale_dtype=self.spec.scale_dtype,+ tensor_min=state.tensor_min,+ tensor_max=state.tensor_max.clone(),+ q_scale_thresh=self.spec.q_scale_thresh,+ )+ elif self.family == "rtn":+ primitive = quant_tensor_gguf_sym_dq if self.kind == "sym" else quant_tensor_gguf_asym_dq+ quantized, scale, zero_point = primitive(+ weight.clone(),+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ super_bits=self.spec.super_bits,+ super_group_size=self.spec.super_group_size,+ v=value,+ scale_dtype=self.spec.scale_dtype,+ imatrix=state.imatrix,+ prev_scale=state.prev_scale,+ prev_wmin=state.prev_wmin,+ prev_d_scale=state.prev_d_scale,+ prev_d_wmin=state.prev_d_wmin,+ iter=state.iteration,+ )+ state.scale = scale["scale"]+ state.d_scale = scale["d_scale"]+ state.prev_scale = state.scale+ state.prev_d_scale = state.d_scale+ if isinstance(zero_point, dict):+ state.wmin = zero_point["wmin"]+ state.d_wmin = zero_point["d_wmin"]+ state.prev_wmin = state.wmin+ state.prev_d_wmin = state.d_wmin+ state.iteration += 1+ else:+ primitive = quant_tensor_sym_dq if self.kind == "sym" else quant_tensor_asym_dq+ quantized, scale, zero_point = primitive(+ weight,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ v=value,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.spec.scale_dtype,+ tensor_min=state.tensor_min,+ tensor_max=state.tensor_max,+ q_scale_thresh=self.spec.q_scale_thresh,+ super_bits=self.spec.super_bits,+ super_group_size=self.spec.super_group_size,+ )++ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("GGUF weight result was not materialized")+ module.weight.data.copy_(result.weight)+ rows = result.logical_rows or result.weight.shape[0]++ def set_values(values, primary):+ for key, value in values.items():+ name = primary if key == primary else "w_" + key+ setattr(module, name, (value.reshape(rows, -1) if key == primary else value).cpu())++ if isinstance(result.scale, dict):+ set_values(result.scale, "scale")+ else:+ module.scale = result.scale.reshape(rows, -1).cpu()+ if isinstance(result.zero_point, dict):+ set_values(result.zero_point, "zp")+ elif isinstance(result.zero_point, torch.Tensor):+ module.zp = result.zero_point.reshape(rows, -1).cpu()+ else:+ module.zp = result.zero_point+++register_quantizer("int_sym_dq", aliases=("rtn_int_sym_dq",))(_GGUFWeightQuantizer)+register_quantizer("int_asym_dq", aliases=("rtn_int_asym_dq",))(_GGUFWeightQuantizer)+register_quantizer("int_asym_float_zp")(_GGUFWeightQuantizer)diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex d301f0b2e..ab2b9b4c5 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -11,12 +11,14 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from collections.abc import Mapping+from dataclasses import dataclassfrom typing import Unionimport torchfrom auto_round import envs-from auto_round.data_type.register import register_dtype+from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.utils import get_reciprocal@@ -131,7 +133,7 @@ def quant_tensor_rtn_sym(min_scale=1.0,max_scale=1.0,scale_dtype=torch.float16,- **kwargs+ **kwargs,):"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community@@ -175,7 +177,7 @@ def quant_tensor_sym(tensor_max=None,q_scale_thresh=1e-5,init_scale=None,- **kwargs+ **kwargs,):"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community@@ -250,7 +252,7 @@ def quant_tensor_asym(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs+ **kwargs,):"""Quantize and de-quantize tensor asymmetrically.@@ -310,7 +312,7 @@ def quant_tensor_sym_gptq(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs+ **kwargs,):"""Quantize and de-quantize tensor asymmetrically.@@ -376,7 +378,7 @@ def quant_tensor_asym_wo_round(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- **kwargs+ **kwargs,):"""Quantize and de-quantize tensor asymmetrically without rounding, this is mainly for tuning bias, norm.@@ -423,3 +425,193 @@ def quant_tensor_asym_wo_round(qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, scale, zp+++@dataclass(frozen=True)+class _IntState:+ """Per-layer integer ranges, optional optimized scale, and trainable values."""++ tunables: Mapping[str, torch.Tensor]+ tensor_min: torch.Tensor+ tensor_max: torch.Tensor+ optimized_init: torch.Tensor | None+++class _IntWeightQuantizer:+ """Own integer weight QDQ and choose tuned, RTN, or optimized RTN internally."""++ def __init__(self, spec, family="plain"):+ self.spec = spec+ self.family = family++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the integer weight quantizer for a resolved layer."""+ return cls(spec)++ @staticmethod+ def create_activation(spec):+ """Create the matching integer activation quantizer."""+ return _IntActivationQuantizer(spec)++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ if mode == "optimized_rtn" and self.spec.sym:+ self.family = "optimized"+ elif mode == "rtn" and self.spec.sym:+ self.family = "rtn"+ else:+ self.family = "plain"+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)+ tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)+ if self.spec.clip_max is not None:+ clip_max = self.spec.clip_max.reshape(-1).to(weight.device, tensor_max.dtype)+ clip_min = (+ -clip_max+ if self.spec.clip_min is None+ else self.spec.clip_min.reshape(-1).to(weight.device, tensor_min.dtype)+ )+ if clip_min.numel() == tensor_min.numel() and clip_max.numel() == tensor_max.numel():+ tensor_min = torch.maximum(tensor_min, clip_min.reshape_as(tensor_min))+ tensor_max = torch.minimum(tensor_max, clip_max.reshape_as(tensor_max))++ tunables = {}+ if self.family == "plain" and tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if self.family != "optimized" and tune_minmax:+ shape = tensor_min.shape+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))++ optimized_init = None+ if self.family == "optimized" and self.spec.sym:+ search_weight = weight+ if self.spec.clip_min is not None or self.spec.clip_max is not None:+ search_weight = torch.clamp(weight, min=self.spec.clip_min, max=self.spec.clip_max)+ _, optimized_init, _ = quant_tensor_opt_rtn_sym(+ search_weight.clone(),+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ q_scale_thresh=self.spec.q_scale_thresh,+ imatrix=imatrix,+ )+ return _IntState(tunables, tensor_min, tensor_max, optimized_init)++ def qdq(self, weight, state, *, tunables, materialize=False):+ value = tunables.get("value", 0)+ min_scale = tunables.get("min_scale", 1.0)+ max_scale = tunables.get("max_scale", 1.0)+ if isinstance(min_scale, torch.Tensor):+ min_scale.data.clamp_(0.0, 1.0)+ if isinstance(max_scale, torch.Tensor):+ max_scale.data.clamp_(0.0, 1.0)++ kwargs = {+ "bits": self.spec.bits,+ "group_size": self.spec.group_size,+ "q_scale_thresh": self.spec.q_scale_thresh,+ }+ if self.family == "optimized" and self.spec.sym:+ quantized, scale, zero_point = quant_tensor_sym(+ weight,+ init_scale=state.optimized_init,+ scale_dtype=self.spec.scale_dtype,+ **kwargs,+ )+ elif self.family == "rtn" and self.spec.sym:+ quantized, scale, zero_point = quant_tensor_rtn_sym(+ weight,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.spec.scale_dtype,+ **kwargs,+ )+ else:+ primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym+ quantized, scale, zero_point = primitive(+ weight,+ v=value,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.spec.scale_dtype,+ tensor_min=state.tensor_min,+ tensor_max=state.tensor_max,+ **kwargs,+ )+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("INT weight result was not materialized")+ module.weight.data.copy_(result.weight)+ rows = result.logical_rows or result.weight.shape[0]+ module.scale = result.scale.reshape(rows, -1).cpu()+ module.zp = (+ result.zero_point.reshape(rows, -1).cpu()+ if isinstance(result.zero_point, torch.Tensor)+ else result.zero_point+ )+++class _IntActivationQuantizer:+ """Quantize integer activations, optionally using a calibrated maximum."""++ def __init__(self, spec):+ self.spec = spec+ self.requires_calibration = not spec.dynamic++ def observe(self, activation, current):+ grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)+ maximum = grouped.abs().amax(dim=-1)+ return maximum if current is None else torch.maximum(maximum.to(current), current)++ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ if self.requires_calibration and observed_max is None:+ raise ValueError(f"{self.spec.data_type} activation requires observed_max")+ primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym+ kwargs = {}+ if observed_max is not None:+ kwargs.update(tensor_min=-observed_max, tensor_max=observed_max)+ return primitive(+ activation,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=self.spec.scale_dtype,+ q_scale_thresh=self.spec.q_scale_thresh,+ **kwargs,+ )++ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ quantized, _, _ = self.qdq_with_scale(+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale+ )+ return quantized+++register_quantizer(+ "int_sym",+ aliases=(+ "int",+ "int4",+ "int4_sym",+ "int8",+ "int8_sym",+ "rtn_int",+ "rtn_int4",+ "rtn_int_sym",+ "rtn_int4_sym",+ "opt_rtn_int",+ "opt_rtn_int4",+ "opt_rtn_int_sym",+ "opt_rtn_int4_sym",+ ),+)(_IntWeightQuantizer)+register_quantizer(+ "int_asym",+ aliases=("int4_asym", "rtn_int_asym", "rtn_int4_asym", "opt_rtn_int_asym", "opt_rtn_int4_asym"),+)(_IntWeightQuantizer)diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 7f453066a..29982f2d4 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE, register_dtype+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE, register_dtype, register_quantizerfrom auto_round.data_type.utils import (ceil_ste,floor_ste,@@ -414,6 +414,168 @@ QUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceilQUANT_FUNC_WITH_DTYPE["mx_fp4_rceil_v2"] = quant_mx_rceil_v2QUANT_FUNC_WITH_DTYPE["opt_rtn_mx_fp"] = quant_mx_opt_rtn++class _MXQuantizer:+ """Own MX weight QDQ and derive the concrete MX format from the request."""++ def __init__(self, spec, data_type=None, family="plain"):+ self.spec = spec+ self.data_type = data_type or self._data_type(spec)+ self.family = family++ @staticmethod+ def _data_type(spec):+ data_type = spec.data_type.lower()+ if data_type in ("mx_fp", "mx_int"):+ data_type = f"{data_type}{spec.bits}"+ if data_type not in MXFP_FORMAT_CACHE and not data_type.endswith("_rceil"):+ raise ValueError(f"Unsupported MX datatype {spec.data_type!r}")+ return data_type++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the MX quantizer selected by the requested MX format."""+ data_type = canonical or spec.data_type+ if data_type in ("mx_fp", "mx_int"):+ data_type = f"{data_type}{spec.bits}"+ return cls(spec, data_type)++ @classmethod+ def create_activation(cls, spec):+ """Create the dynamic MX activation quantizer for the same format."""+ return _MXActivationQuantizer(spec, cls._data_type(spec))++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ self.family = "optimized" if mode == "optimized_rtn" else "plain"+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tunables = {}+ if self.family == "plain" and tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if self.family == "plain" and tune_minmax:+ tunables["max_scale"] = torch.nn.Parameter(+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)+ )+ return {"tunables": tunables, "imatrix": imatrix}++ def qdq(self, weight, state, *, tunables, materialize=False):+ kwargs = {+ "bits": self.spec.bits,+ "group_size": self.spec.group_size,+ "data_type": self.data_type,+ }+ if self.family == "optimized":+ quantized, exponent, zero_point = quant_mx_opt_rtn(weight, imatrix=state["imatrix"], **kwargs)+ elif self.data_type.endswith("_rceil"):+ quantized, exponent, zero_point = quant_mx_rceil(weight, **kwargs)+ else:+ quantized, exponent, zero_point = quant_mx(+ weight,+ v=tunables.get("value", 0),+ max_scale=tunables.get("max_scale", 1.0),+ **kwargs,+ )+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(+ quantized, exponent if materialize else None, zero_point if materialize else None+ )++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("MX weight result was not materialized")+ module.weight.data.copy_(result.weight)+ rows = result.logical_rows or result.weight.shape[0]+ module.scale = result.scale.reshape(rows, -1).cpu()+ module.zp = None+++class _MXActivationQuantizer:+ """Apply dynamic MX activation QDQ; MX formats require no calibration pass."""++ requires_calibration = False++ def __init__(self, spec, data_type):+ if not spec.dynamic:+ raise ValueError(f"MX datatype {data_type!r} supports only dynamic activation quantization")+ self.spec = spec+ self.data_type = data_type++ def observe(self, activation, current):+ raise RuntimeError("Dynamic MX activation quantization does not require calibration")++ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ primitive = quant_mx_rceil if self.data_type.endswith("_rceil") else quant_mx+ return primitive(+ activation,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ data_type=self.data_type.removesuffix("_rceil"),+ max_scale=max_scale,+ )++ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ quantized, _, _ = self.qdq_with_scale(+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale+ )+ return quantized+++_MX_ALIASES = {+ "mx_int8": ("mxint8",),+ "mx_int4": (+ "mxint4",+ "rtn_mx_int4",+ "rtn_mx_int4_sym",+ "opt_rtn_mx_int4",+ "opt_rtn_mx_int4_sym",+ ),+ "mx_int2": ("mxint2",),+ "mx_fp8e5m2": ("mxfp8e5m2",),+ "mx_fp8": ("mxfp8",),+ "mx_fp8e4m3": ("mxfp8e4m3",),+ "mx_fp6e3m2": ("mxfp6e3m2",),+ "mx_fp6": ("mxfp6",),+ "mx_fp6e2m3": ("mxfp6e2m3",),+ "mx_fp4": (+ "mxfp4",+ "mx_fp4_sym",+ "rtn_mx_fp4",+ "rtn_mx_fp4_sym",+ "opt_rtn_mx_fp4",+ "opt_rtn_mx_fp4_sym",+ ),+ "mx_fp4e2m1": ("mxfp4e2m1",),+ "mx_float16": ("mxfloat16",),+ "mx_fp16": ("mxfp16",),+ "mx_bfloat16": ("mxbfloat16",),+ "mx_bf16": ("mxbf16",),+}+++for _data_type in MXFP_FORMAT_CACHE:+ register_quantizer(+ _data_type,+ aliases=_MX_ALIASES[_data_type],+ )(_MXQuantizer)++for _family in ("fp", "int"):+ _canonical = f"mx_{_family}"++ register_quantizer(+ _canonical,+ aliases=(+ f"mx_{_family}_sym",+ f"rtn_mx_{_family}",+ f"rtn_mx_{_family}_sym",+ f"opt_rtn_mx_{_family}",+ f"opt_rtn_mx_{_family}_sym",+ ),+ )(_MXQuantizer)++register_quantizer("mx_fp_rceil", aliases=("mxfprceil",))(_MXQuantizer)+register_quantizer("mx_fp4e2m1_rceil")(_MXQuantizer)+if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])data1 = quant_element(data, 2, 3, 6.0)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex fbe025843..cb4e2efca 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -12,11 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.+from collections.abc import Mapping+from dataclasses import dataclass+import torch+from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.fp8 import float8_e4m3fn_stefrom auto_round.data_type.gguf import _imatrix_handle_zero-from auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.logger import logger@@ -422,6 +425,222 @@ def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=Nonreturn qdq_res.to(orig_dtype), scale, None+@dataclass(frozen=True)+class _NVFPState:+ """NVFP tuning values plus optional global and optimized scales."""++ tunables: Mapping[str, torch.Tensor]+ global_scale: torch.Tensor | None+ optimized_init: torch.Tensor | None+++class _NVFPWeightQuantizer:+ """Own NVFP4 weight QDQ, including its global-scale preparation."""++ def __init__(self, spec, family="plain"):+ self.spec = spec+ self.family = family++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the NVFP4 weight quantizer for a resolved layer."""+ return cls(spec)++ @staticmethod+ def create_activation(spec):+ """Create dynamic NVFP4 activation quantization."""+ return _NVFPActivationQuantizer(spec, "nv_fp4")++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ self.family = "optimized" if mode == "optimized_rtn" else "plain"+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tunables = {}+ if self.family == "plain" and tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if self.family != "optimized" and tune_minmax:+ tunables["max_scale"] = torch.nn.Parameter(+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)+ )++ global_scale = self.spec.global_scale+ if global_scale is None:+ global_scale = calculate_gparam(weight, self.spec.group_size, weight.device)+ else:+ global_scale = global_scale.to(weight.device)+ optimized_init = None+ if self.family == "optimized":+ if isinstance(imatrix, torch.Tensor):+ imatrix = imatrix.reshape(1, -1)+ imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)+ imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)+ qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)+ else:+ qw = 1.0+ optimized_init = search_nvfp4_scale(grouped, self.spec.bits, qw)+ return _NVFPState(tunables, global_scale, optimized_init)++ def qdq(self, weight, state, *, tunables, materialize=False):+ kwargs = {+ "bits": self.spec.bits,+ "group_size": self.spec.group_size,+ "v": tunables.get("value", 0),+ "max_scale": tunables.get("max_scale", 1.0),+ }+ kwargs["global_scale"] = state.global_scale+ if state.optimized_init is not None:+ kwargs["init_scale"] = state.optimized_init+ quantized, scale, zero_point = nv_fp4(weight, **kwargs)+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(+ quantized,+ scale if materialize else None,+ zero_point if materialize else None,+ state.global_scale if materialize else None,+ )++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("NVFP weight result was not materialized")+ module.weight.data.copy_(result.weight)+ rows = result.logical_rows or result.weight.shape[0]+ module.scale = result.scale.reshape(rows, -1).cpu()+ module.zp = None+ if result.metadata is not None:+ module.weight_global_scale = result.metadata.cpu()+++class _NVFPV2WeightQuantizer:+ """Own the independent NVFP4-v2 weight QDQ implementation."""++ def __init__(self, spec):+ self.spec = spec++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the NVFP4-v2 weight quantizer for a resolved layer."""+ return cls(spec)++ @staticmethod+ def create_activation(spec):+ """Create dynamic NVFP4-v2 activation quantization."""+ return _NVFPActivationQuantizer(spec, "nvfp4_v2")++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tunables = {}+ if tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if tune_minmax:+ tunables["max_scale"] = torch.nn.Parameter(+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)+ )+ return _NVFPState(tunables, None, None)++ def qdq(self, weight, state, *, tunables, materialize=False):+ quantized, scale, zero_point = nvfp4_v2(+ weight,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ v=tunables.get("value", 0),+ max_scale=tunables.get("max_scale", 1.0),+ )+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)++ apply_result = staticmethod(_NVFPWeightQuantizer.apply_result)+++class _NVFPActivationQuantizer:+ """Quantize dynamic or calibrated NVFP activations for one NVFP format."""++ def __init__(self, spec, data_type):+ is_static = data_type in ("nv_fp4_with_static_gs", "nvfp4_v2_with_global_scale")+ if not is_static and not spec.dynamic:+ raise ValueError(f"NVFP datatype {data_type!r} supports only dynamic activation quantization")+ self.spec = spec+ self.data_type = data_type+ self.requires_calibration = is_static++ def observe(self, activation, current):+ if not self.requires_calibration:+ raise RuntimeError("Dynamic NVFP activation quantization does not require calibration")+ observed = activation.detach().float().abs().max().unsqueeze(0)+ return observed if current is None else torch.maximum(current.to(observed.device), observed)++ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ if self.requires_calibration and observed_max is None:+ raise ValueError(f"{self.data_type} activation requires observed_max")+ primitive = {+ "nv_fp4": nv_fp4,+ "nv_fp4_with_static_gs": nv_fp4_with_static_gs,+ "nvfp4_v2": nvfp4_v2,+ "nvfp4_v2_with_global_scale": nvfp4_v2_with_global_scale,+ }[self.data_type]+ kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}+ if self.requires_calibration:+ kwargs["tensor_max"] = observed_max+ return primitive(activation, **kwargs)++ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):+ quantized, _, _ = self.qdq_with_scale(+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale+ )+ return quantized+++def _prepare_nvfp_block(block, layer_runtimes):+ """Materialize shared NVFP scales once after a block's observations."""+ from auto_round.data_type.utils import update_block_global_scale_if_needed++ runtime = next(iter(layer_runtimes.values()), None)+ if runtime is None:+ return+ update_block_global_scale_if_needed(block, runtime["data_type"], runtime["group_size"])+++_NVFPWeightQuantizer.prepare_block = staticmethod(_prepare_nvfp_block)+++class _NVFPStaticActivation:+ """Datatype entry that provides static NVFP activation quantization only."""++ @staticmethod+ def create_activation(spec):+ return _NVFPActivationQuantizer(spec, "nv_fp4_with_static_gs")+++class _NVFPV2StaticActivation:+ """Datatype entry that provides global-scale NVFP-v2 activation only."""++ @staticmethod+ def create_activation(spec):+ return _NVFPActivationQuantizer(spec, "nvfp4_v2_with_global_scale")+++register_quantizer(+ "nv_fp4",+ aliases=(+ "nv_fp",+ "nv_fp_sym",+ "nv_fp4_sym",+ "rtn_nv_fp",+ "rtn_nv_fp4",+ "rtn_nv_fp_sym",+ "rtn_nv_fp4_sym",+ "opt_rtn_nv_fp",+ "opt_rtn_nv_fp4",+ "opt_rtn_nv_fp_sym",+ "opt_rtn_nv_fp4_sym",+ ),+)(_NVFPWeightQuantizer)+register_quantizer("nvfp4_v2")(_NVFPV2WeightQuantizer)+register_quantizer("nv_fp4_with_static_gs", aliases=("rtn_nv_fp4_with_static_gs",))(_NVFPStaticActivation)+register_quantizer("nvfp4_v2_with_global_scale")(_NVFPV2StaticActivation)++if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])data1 = cast_to_fp4(data)diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pydeleted file mode 100644index fca259ed6..000000000--- a/auto_round/data_type/register.py+++ /dev/null@@ -1,40 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.---QUANT_FUNC_WITH_DTYPE = {}---def register_dtype(names):- """Class decorator to register a EXPORT subclass to the registry.-- Decorator function used before a Pattern subclass.-- Args:- names: A string. Define the export type.-- Returns:- cls: The class of register.- """-- def register(dtype):- if isinstance(names, (tuple, list)):- for name in names:- QUANT_FUNC_WITH_DTYPE[name] = dtype- else:- QUANT_FUNC_WITH_DTYPE[names] = dtype-- return dtype-- return registerdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex c412b2445..50b383ca2 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -22,10 +22,17 @@ from torch.nn import Linear, Modulefrom auto_round import envsfrom auto_round.compressors.utils import is_nv_fp-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import check_to_quantized, logger+def quantize_bias_without_round(*args, **kwargs):+ """Quantize bias/norm weights without exposing the INT primitive to algorithms."""+ from auto_round.data_type.int import quant_tensor_asym_wo_round++ return quant_tensor_asym_wo_round(*args, **kwargs)++def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.@@ -176,141 +183,6 @@ def get_quant_func()-def _resolve_optimized_dtype_funcs(data_type: str, q_scale_thresh: float = 1e-5):- """Resolve the SignRound optimized ``(scale_search_fn, quant_func)`` for a data type.-- Single source of truth for the optimized-path dispatch shared by- ``SignRoundOptimizedWrapperLinear`` and AWQ's internal QDQ:-- * ``scale_search_fn(weight_reshape, bits, imatrix) -> init_scale`` searches the- data-type-specific per-group init scale (the int variant clamps to- ``q_scale_thresh`` to avoid a degenerate zero scale).- * ``quant_func`` is the matching *plain* quant function (``init_scale`` already- encodes the searched scale, so no opt-rtn / rtn variant is needed).-- Returns ``(None, None)`` for data types without an optimized path- (asym int, ``*_dq``, or unrelated types).- """- dt = str(data_type)- if dt.endswith("dq"):- return None, None- if dt.startswith("int"):- # The optimized int init-scale search is symmetric-only; asym int uses- # the standard tensor_min/tensor_max range instead.- if "asym" in dt:- return None, None- from auto_round.data_type.int import quant_tensor_sym, search_scales-- def search_int(weight_reshape, bits, imatrix):- init_scale = search_scales(weight_reshape, bits, imatrix)- return torch.where(- init_scale < 0,- torch.clamp(init_scale, max=-q_scale_thresh),- torch.clamp(init_scale, min=q_scale_thresh),- )-- return search_int, quant_tensor_sym- if dt.startswith("mx"):- from auto_round.data_type.mxfp import quant_mx, search_mx_scale-- return search_mx_scale, quant_mx- if dt.startswith("nv"):- from auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale-- return search_nvfp4_scale, nv_fp4- return None, None---def search_optimized_init_scale(- weight_reshape: torch.Tensor,- data_type: str,- bits: int,- imatrix=None,- q_scale_thresh: float = 1e-5,-):- """Compute the SignRoundV2 optimized per-group ``init_scale`` for a grouped weight.-- Mirrors ``SignRoundOptimizedWrapperLinear``: dispatches on ``data_type`` so that- any caller (the optimized wrapper itself or AWQ's internal QDQ used for the- smooth/clip grid search) seeds the quantizer with the same initial scale.- Returns ``None`` for data types that do not use the optimized init-scale search- (asym int, ``*_dq``, or unrelated types).-- Args:- weight_reshape: Weight reshaped/padded to ``[..., group_size]``.- data_type: Resolved weight data type (e.g. ``"int_sym"``, ``"mx_fp4"``, ``"nv_fp4"``).- bits: Weight bit-width.- imatrix: Per-element importance matrix matching ``weight_reshape`` (or ``None``/scalar).- q_scale_thresh: Minimum scale magnitude used to clamp the int init_scale.-- Returns:- The per-group ``init_scale`` tensor, or ``None`` if unsupported.- """- search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)- if search_fn is None:- return None- if imatrix is None or not isinstance(imatrix, torch.Tensor):- imatrix = torch.ones_like(weight_reshape)- return search_fn(weight_reshape, bits, imatrix)---def get_optimized_quant_func(data_type: str):- """Return the plain quant function used by the SignRound optimized path.-- The optimized init-scale search always pairs with the *plain* (non opt-rtn /- non rtn) quant function for the data type, since ``init_scale`` already- encodes the searched scale. Returns ``None`` for data types without an- optimized path (asym int, ``*_dq``, or unrelated types).- """- _, quant_func = _resolve_optimized_dtype_funcs(data_type)- return quant_func---def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size):- """Reshape/pad an importance matrix to match a group-reshaped weight.-- Encapsulates the imatrix grouping logic shared by the SignRound optimized- wrapper and AWQ's internal QDQ so callers never handle the low-level reshape.- Returns a tensor of ones when no imatrix is available (uniform importance),- keeping every downstream optimized dtype implementation on its tensor path.- """- if imatrix is None or not isinstance(imatrix, torch.Tensor):- return torch.ones_like(weight_reshape)- imatrix = imatrix.reshape(1, -1)- imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)- imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)- return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)---def compute_optimized_init_scale(- weight: torch.Tensor,- data_type: str,- bits: int,- group_size,- imatrix=None,- q_scale_thresh: float = 1e-5,-):- """Compute the SignRound optimized per-group ``init_scale`` for a full weight.-- Group-reshapes ``weight``, prepares the ``imatrix`` layout, and runs the- data-type-specific scale search. Unlike :func:`search_optimized_init_scale`- (which expects an already group-reshaped weight), this is the entry point for- callers holding a full 2-D weight, e.g. AWQ's internal QDQ. Pair it with- :func:`get_optimized_quant_func` (resolved once) to obtain the matching quant- function, so the smooth/clip grid-search loss mirrors what- ``SignRoundOptimizedWrapperLinear`` applies.-- Returns ``None`` for data types without an optimized init-scale path- (asym int, ``*_dq``, or unrelated types).- """- search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)- if search_fn is None:- return None- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)- imatrix = reshape_imatrix_for_weight(imatrix, weight_reshape, group_size)- return search_fn(weight_reshape, bits, imatrix)--def round_ste(x: torch.Tensor):"""Straight-Through Estimator for rounding.diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.pyindex b8e987d4c..6d7e35b41 100644--- a/auto_round/data_type/w4fp8.py+++ b/auto_round/data_type/w4fp8.py@@ -14,7 +14,7 @@import torch-from auto_round.data_type.register import register_dtype+from auto_round.data_type.base import register_dtype, register_quantizerfrom auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func@@ -72,3 +72,58 @@ def progressive_quant_fp8_int4(bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scalereturn qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4+++@register_quantizer("fp8_to_int_sym")+class _ProgressiveFP8WeightQuantizer:+ """Own the progressive BF16-to-FP8-to-INT4 weight quantization path."""++ def __init__(self, spec):+ self.spec = spec++ @classmethod+ def from_spec(cls, spec, canonical=None):+ """Create the progressive FP8-to-INT4 weight quantizer."""+ return cls(spec)++ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size++ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)+ tunables = {}+ if tune_rounding:+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))+ if tune_minmax:+ shape = grouped.shape[:-1]+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))+ return tunables++ def qdq(self, weight, state, *, tunables, materialize=False):+ quantized, scales, zero_point = progressive_quant_fp8_int4(+ weight,+ bits=self.spec.bits,+ group_size=self.spec.group_size,+ v=tunables.get("value", 0),+ min_scale=tunables.get("min_scale", 1.0),+ max_scale=tunables.get("max_scale", 1.0),+ q_scale_thresh=self.spec.q_scale_thresh,+ )+ from auto_round.data_type.base import WeightQuantizationResult++ return WeightQuantizationResult(+ quantized,+ scales["scale"] if materialize else None,+ zero_point if materialize else None,+ scales["bf16_to_fp8_scale"] if materialize else None,+ )++ @staticmethod+ def apply_result(module, result):+ if result.scale is None:+ raise ValueError("Progressive FP8 weight result was not materialized")+ module.weight.data.copy_(result.weight)+ rows = result.logical_rows or result.weight.shape[0]+ module.scale = result.scale.reshape(rows, -1).cpu()+ module.zp = result.zero_point+ module.w_bf16_to_fp8_scale = result.metadata.cpu()diff --git a/auto_round/modeling/fused_moe/grouped_experts.py b/auto_round/modeling/fused_moe/grouped_experts.pyindex c9a755316..48d8511e5 100644--- a/auto_round/modeling/fused_moe/grouped_experts.py+++ b/auto_round/modeling/fused_moe/grouped_experts.py@@ -490,6 +490,11 @@ def _batched_qdq_weights(eligible and the caller must go per-expert."""ref = layers[0]+ # Datatype quantizers intentionally keep their per-layer QDQ state private.+ # Without the legacy compiled function, that state cannot be safely merged,+ # so retain correctness by using the normal per-expert path.+ if not hasattr(ref, "weight_quant_func"):+ return Noneorig = ref.orig_layerout_features, in_features = ref.weight.shapenum_experts = len(layers)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex d7de6366f..12c21c58c 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2180,7 +2180,9 @@ def set_amax_for_uncalibrated_experts()return uncalibrated_experts# Flatten all tensors to 1D before concatenation- flat_values = [t.reshape(-1) for t in amax_values]+ device = amax_values[0].device+ dtype = amax_values[0].dtype+ flat_values = [t.reshape(-1).to(device=device, dtype=dtype) for t in amax_values]all_values = torch.cat(flat_values)set_amax_value = torch.max(all_values)set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_valuediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex f36b67e00..2c9193902 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -20,8 +20,8 @@ import transformersfrom torch.functional import Fimport auto_round.envs as envs-from auto_round.compressors.utils import is_nv_fp-from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size+from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer+from auto_round.data_type.utils import quantize_bias_without_round, reshape_pad_tensor_by_group_sizefrom auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -84,6 +84,7 @@ class WrapperLinear(torch.nn.Module):enable_round_tuning=True,enable_torch_compile=True,disable_opt_rtn=True,+ weight_qdq_builder=None,**kwargs,):"""Initializes the WrapperLinear module.@@ -105,18 +106,35 @@ class WrapperLinear(torch.nn.Module):self.enable_torch_compile = enable_torch_compileself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8+ # Observation may have cached the quantizer on the original layer. Do+ # not consume it until wrapper initialization has completed successfully.+ cached_activation_quantizer = getattr(self.orig_layer, "_ar_activation_quantizer", None)+ self.activation_quantizer = cached_activation_quantizer+ if self.enable_act_quant and self.activation_quantizer is None:+ self.activation_quantizer = activation_quantizer_for_layer(orig_layer)self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)- if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:- from auto_round.data_type.nvfp import calculate_gparam-- weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)- setattr(self, "weight_global_scale", weight_global_scale)- self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:self.q_scale_thresh = 1e-8else:self.q_scale_thresh = 1e-5self._init_tuning_params_and_quant_func()++ def default_qdq(weight, *, materialize=False):+ """Keep direct wrapper use working when no algorithm supplies a builder."""+ return self.weight_quantizer.qdq(weight, materialize=materialize, **self.params)++ self.weight_qdq = default_qdq+ if weight_qdq_builder is not None:+ weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()+ if type(orig_layer) == transformers.pytorch_utils.Conv1D:+ weight = weight.t()+ self.weight_qdq = weight_qdq_builder(+ weight=weight.to(self.device), default_qdq=self.weight_quantizer.qdq, parameters=self.params+ )+ for name, parameter in self.params.items():+ setattr(self, name, parameter)+ if cached_activation_quantizer is not None:+ self.orig_layer.__dict__.pop("_ar_activation_quantizer", None)if deepspeed_exists:if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):self.orig_forward = self.linear_forward@@ -147,69 +165,25 @@ class WrapperLinear(torch.nn.Module):orig_layer = self.orig_layerorig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:+ if type(orig_layer) == transformers.pytorch_utils.Conv1D:orig_weight = orig_weight.t()- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)-- if self.enable_round_tuning:- self.weight_min = (- torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)- if isinstance(orig_layer.group_size, tuple)- else torch.clamp(weight_reshape.min(1)[0], max=0)- )- self.weight_max = (- torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)- if isinstance(orig_layer.group_size, tuple)- else torch.clamp(weight_reshape.max(1)[0], min=0)- )- else:- self.weight_min = None- self.weight_max = None- # AWQ clip-as-init: cap the tunable weight range to the per-group clip- # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).- # This initializes the range used by quant_tensor_sym/asym, leaving- # min_scale/max_scale to tune a coefficient on top. Only the standard- # (non-tuple) group layout maps onto weight_min/weight_max here.- awq_clip_min = getattr(orig_layer, "awq_clip_min", None)- awq_clip_max = getattr(orig_layer, "awq_clip_max", None)- if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):- clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)- if awq_clip_min is None:- clip_min_flat = -clip_max_flat- else:- clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)- if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():- self.weight_max = torch.minimum(self.weight_max, clip_max_flat)- self.weight_min = torch.maximum(self.weight_min, clip_min_flat)- self._init_params(- "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16- )- # Min-max scale initialization- shape = get_scale_shape(orig_weight, orig_layer.group_size)- self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))- self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))-- self.weight_quant_func, self.data_type = get_quant_func(- orig_layer.data_type,- orig_layer.bits,- orig_layer.sym,- self.disable_opt_rtn,- orig_layer.group_size,- iters=orig_layer.iters,+ self.weight_quantizer = create_quantizer(+ orig_layer,+ disable_opt_rtn=self.disable_opt_rtn,+ tune_rounding=self.enable_round_tuning,+ tune_minmax=self.enable_minmax_tuning,)- if self.enable_torch_compile:- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)+ self.weight_quantizer.initialize(orig_weight, imatrix=getattr(orig_layer, "imatrix", None))+ self.params = dict(self.weight_quantizer.parameters)+ for name, value in self.params.items():+ setattr(self, name, value)+ self.value = getattr(self, "value", torch.tensor(0.0, device=self.device, dtype=p_dtype))+ self.min_scale = getattr(self, "min_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))+ self.max_scale = getattr(self, "max_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))+ self.data_type = orig_layer.data_typeif self.enable_act_quant:- self.act_quant_func, self.act_data_type = get_quant_func(- orig_layer.act_data_type,- orig_layer.act_bits,- orig_layer.act_sym,- disable_opt_rtn=True,- iters=orig_layer.iters,- )- if self.enable_torch_compile:- self.act_quant_func = compile_func(self.act_quant_func, self.device)+ self.act_data_type = orig_layer.act_data_typeself._init_params("act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic))@@ -218,9 +192,7 @@ class WrapperLinear(torch.nn.Module):# Bias tuningif self.enable_norm_bias_tuning:self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)- from auto_round.data_type.int import quant_tensor_asym_wo_round-- self.bias_quant_func = quant_tensor_asym_wo_round+ self.bias_quant_func = quantize_bias_without_roundself.params["bias_v"] = self.bias_vdef _init_params(self, name, dtype, shape, value, tunable):@@ -254,43 +226,17 @@ class WrapperLinear(torch.nn.Module):"""if self.orig_layer.bits >= 16:return self.orig_layer.weight, None, None- min_bound, max_bound = self.minmax_scale_bound- min_scale.data.clamp_(min_bound, max_bound)- max_scale.data.clamp_(min_bound, max_bound)weight = self.orig_layer.weightif weight.device.type == "meta":weight = self.orig_layer.get_weight().to(self.device)if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:weight = weight.t()- quant_kwargs = {}- if hasattr(self.orig_layer, "super_bits"):- quant_kwargs["super_bits"] = self.orig_layer.super_bits- quant_kwargs["super_group_size"] = self.orig_layer.super_group_size- if hasattr(self, "_extra_quant_kwargs"):- quant_kwargs.update(self._extra_quant_kwargs())-- weight_q, scale, zp = self.weight_quant_func(- weight.to(self.device),- bits=self.orig_layer.bits,- group_size=self.orig_layer.group_size,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.orig_layer.scale_dtype,- tensor_min=self.weight_min,- tensor_max=self.weight_max,- data_type=self.data_type,- q_scale_thresh=self.q_scale_thresh,- imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,- global_scale=getattr(self, "weight_global_scale", None),- init_scale=getattr(self, "init_scale", None),- **quant_kwargs,- )- weight_q = weight_q.to(weight.dtype)+ self.params.update(value=value, min_scale=min_scale, max_scale=max_scale)+ weight_q = self.weight_qdq(weight.to(self.device)).weight.to(weight.dtype)if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:weight_q = weight_q.t()- return weight_q, scale, zp+ return weight_q, None, Nonedef _qdq_act(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):"""Quantizes and dequantizes activations.@@ -306,19 +252,12 @@ class WrapperLinear(torch.nn.Module):act_max_scale.data.clamp_(0, 1.0)act_min_scale.data.clamp_(0, 1.0)env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set- x, scale, zp = self.act_quant_func(+ return self.activation_quantizer.qdq_with_scale(x,- bits=self.orig_layer.act_bits,- group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.q_scale_thresh,- data_type=self.act_data_type,- tensor_max=act_max, # for static+ observed_max=act_max,max_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,- global_scale=getattr(self, "input_global_scale", None),)- return x, scale, zpdef _qdq_bias(self, bias, bias_v):"""Quantizes and dequantizes bias.@@ -361,59 +300,28 @@ class WrapperLinear(torch.nn.Module):v = best_params.get("value", torch.tensor(0.0)).to(self.device)min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)+ # Preserve the dictionary captured by QDQ closures and restore all+ # algorithm parameters, including names unknown to the wrapper.+ self.params.update({name: value.to(self.device) for name, value in best_params.items()})+ self.params.update(value=v, min_scale=min_scale, max_scale=max_scale)if self.orig_layer.weight.device.type == "meta":self.orig_layer.to(self.device)- # Unwrapper weight- qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)- # if hasattr(self.orig_layer, "imatrix"):- # self.orig_layer.imatrix = None- self.orig_layer.weight.data.copy_(qdq_weight)+ if self.orig_layer.bits < 16:+ weight = self.orig_layer.weight+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:+ weight = weight.t()+ result = self.weight_qdq(+ weight.to(self.device),+ materialize=True,+ )+ self.weight_quantizer.apply_result(+ self.orig_layer,+ result,+ transpose=type(self.orig_layer) == transformers.pytorch_utils.Conv1D,+ )self.orig_layer.weight.grad = None- shape = qdq_weight.shape- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:- shape = qdq_weight.t().shape-- def _set_dict_attr(attr_dict, attr_name):- for key in attr_dict.keys():- if key == attr_name:- setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))- else:- name = "w_" + key- setattr(self.orig_layer, name, attr_dict[key].to("cpu"))-- if not isinstance(self.orig_layer.group_size, tuple):- if isinstance(scale, dict):- _set_dict_attr(scale, "scale")- elif scale is None:- self.orig_layer.scale = None- elif scale.numel() > 1:- self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")- else:- self.orig_layer.scale = scale.view(-1).to("cpu")- else:- self.orig_layer.scale = scale.to("cpu")-- if zp is not None:- if isinstance(zp, dict):- _set_dict_attr(zp, "zp")- elif isinstance(zp, torch.Tensor):- if zp.numel() > 1:- zp = zp.reshape(shape[0], -1)- self.orig_layer.zp = zp.to("cpu")- else:- self.orig_layer.zp = zp.view(-1).to("cpu")- else:- self.orig_layer.zp = zp- else:- self.orig_layer.zp = None-- if self.weight_global_scale is not None:- global_scale = self.weight_global_scale- assert global_scale.numel() == 1- self.orig_layer.weight_global_scale = global_scale.to("cpu")-# Unwrapper biasif self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"].to(self.device)@@ -457,11 +365,11 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.act_max_scale = self.act_max_scaleself.orig_layer.act_data_type = self.act_data_type- self.orig_layer.act_quant_func = self.act_quant_funcwrapper_layer = WrapperWALayer(self.orig_layer,enable_torch_compile=self.enable_torch_compile,device=self.device,+ activation_quantizer=self.activation_quantizer,)return _preserve_global_name(wrapper_layer)@@ -567,16 +475,16 @@ class WrapperLinear(torch.nn.Module):class WrapperWALayer(torch.nn.Module):- def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):+ def __init__(self, orig_layer, enable_torch_compile=True, device="cpu", activation_quantizer=None):super(WrapperWALayer, self).__init__()self.orig_layer = orig_layerself.enable_torch_compile = enable_torch_compileself.device = deviceself.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else Noneself.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None- self.act_quant_func = self.orig_layer.act_quant_func- if self.enable_torch_compile:- self.act_quant_func = compile_func(self.act_quant_func, self.device)+ self.activation_quantizer = activation_quantizer or orig_layer.__dict__.pop("_ar_activation_quantizer", None)+ if self.activation_quantizer is None:+ self.activation_quantizer = activation_quantizer_for_layer(orig_layer)self.extra_repr_org = orig_layer.extra_repr# Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)@@ -610,27 +518,12 @@ class WrapperWALayer(torch.nn.Module):max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scalemin_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale- if act_max is None:- x, _, _ = self.orig_layer.act_quant_func(- x,- bits=self.orig_layer.act_bits,- group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.orig_layer.q_scale_thresh,- data_type=self.orig_layer.act_data_type,- min_scale=min_scale,- max_scale=max_scale,- )- else:- x, _, _ = self.orig_layer.act_quant_func(- x,- bits=self.orig_layer.act_bits,- group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.orig_layer.q_scale_thresh,- data_type=self.orig_layer.act_data_type,- act_max=act_max,- )+ x = self.activation_quantizer.qdq(+ x,+ observed_max=act_max,+ min_scale=min_scale,+ max_scale=max_scale,+ )# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)return self.orig_layer.forward(x)@@ -662,9 +555,7 @@ class WrapperLayerNorm(torch.nn.Module):requires_grad=True,)self.params = {"v": self.v}- from auto_round.data_type.int import quant_tensor_asym_wo_round-- self.quant_func = quant_tensor_asym_wo_round+ self.quant_func = quantize_bias_without_rounddef unwrapper(self, best_params):if best_params is None:@@ -713,9 +604,7 @@ class WrapperLlamaNorm(torch.nn.Module):requires_grad=True,)self.params = {"v": self.v}- from auto_round.data_type.int import quant_tensor_asym_wo_round-- self.quant_func = quant_tensor_asym_wo_round+ self.quant_func = quantize_bias_without_rounddef unwrapper(self, best_params):if best_params is None:diff --git a/benchmark_both.py b/benchmark_both.pydeleted file mode 100644index 320f18b11..000000000--- a/benchmark_both.py+++ /dev/null@@ -1,83 +0,0 @@-"""Quick A/B benchmark: old (compressors) vs new (compressors_new) architecture.--Uses AR_DISABLE_NEW_ARCH env-var to toggle. Runs each configuration in a-subprocess to avoid cross-contamination, with a warmup run to fill OS page cache.-"""--import json-import os-import subprocess-import sys-import time--MODEL = "Qwen/Qwen3-0.6B"-ITERS = "200"-SCHEME = "W4A16"-DEVICE = "cuda:0"--CMD_TEMPLATE = [- sys.executable,- "-m",- "auto_round",- "--model_name",- MODEL,- "--scheme",- SCHEME,- "--iters",- ITERS,- "--device",- DEVICE,-]---def run_once(label: str, env_override: dict) -> float:- env = {**os.environ, **env_override}- print(f"\n{'='*60}")- print(f" Running: {label}")- print(f" AR_DISABLE_NEW_ARCH={env.get('AR_DISABLE_NEW_ARCH', 'unset')}")- print(f"{'='*60}", flush=True)- t0 = time.perf_counter()- proc = subprocess.run(CMD_TEMPLATE, env=env, capture_output=True, text=True)- elapsed = time.perf_counter() - t0- if proc.returncode != 0:- print(f"STDERR:\n{proc.stderr[-2000:]}")- raise RuntimeError(f"{label} failed with rc={proc.returncode}")- print(f" {label}: {elapsed:.1f}s")- return elapsed---def main():- # Warmup: fill OS page cache & JIT caches- print("Warmup run (old arch)...")- run_once("warmup", {"AR_DISABLE_NEW_ARCH": "1"})-- # Interleaved runs to reduce bias- results = {"old": [], "new": []}- for trial in range(2):- if trial % 2 == 0:- first, second = ("old", "1"), ("new", "0")- else:- first, second = ("new", "0"), ("old", "1")- for label, flag in [first, second]:- t = run_once(f"{label} (trial {trial+1})", {"AR_DISABLE_NEW_ARCH": flag})- results[label].append(t)-- # Summary- print("\n" + "=" * 60)- print("SUMMARY")- print("=" * 60)- for arch in ["old", "new"]:- times = results[arch]- avg = sum(times) / len(times)- print(f" {arch}: {[f'{t:.1f}' for t in times]} avg={avg:.1f}s")- old_avg = sum(results["old"]) / len(results["old"])- new_avg = sum(results["new"]) / len(results["new"])- diff_pct = (new_avg - old_avg) / old_avg * 100- print(f"\n Diff: {diff_pct:+.1f}% (new vs old)")- print(f" {'PASS' if abs(diff_pct) < 5 else 'FAIL'} (threshold: ±5%)")-- json.dump(results, open("benchmark_results/latest.json", "w"), indent=2)---if __name__ == "__main__":- main()diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex f2c62a93e..2ff9edf2b 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -174,7 +174,7 @@ adopted within the community, **only 4-bits quantization is supported**. Please**MLX Format**[Experimental Feature]: This format targets Apple Silicon (M1/M2/M3/...) and is loaded directly by [`mlx-lm`](https://github.com/ml-explore/mlx-lm) (text-only LLM) or [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm) (vision/audio + language).- Supports **2, 3, 4, 5, 6, 8 bits** (5/6 bits are MLX-exclusive — GPTQ/AWQ have no standard packing for them).-- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--schemes "..." --bits 3.5`);+- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--schemes "..." --bits 3.5`);- Use `--format mlx` for a native MLX checkpoint; use `--format auto_round:mlx` if you want HuggingFace `transformers` + AutoRound to load it (post-init repacks each layer into MLX `QuantLinear` on Darwin).- Limitation: embedding layer quantization has not supported#### Format and scheme support matrixdiff --git a/performance_ut.sh b/performance_ut.shdeleted file mode 100644index 12e78906b..000000000--- a/performance_ut.sh+++ /dev/null@@ -1,115 +0,0 @@-#!/bin/bash-set -euo pipefail--PATTERN='[-a-zA-Z0-9_]*='--for i in "$@"; do- case $i in- --model_name=*)- model_name=$(echo $i | sed "s/${PATTERN}//")- ;;- --scheme=*)- scheme=$(echo $i | sed "s/${PATTERN}//")- ;;- *)- echo "Parameter $i not recognized."- exit 1- ;;- esac-done--readonly WORKSPACE_DIR="/auto-round"-readonly LOG_DIR="${WORKSPACE_DIR}/log_dir"-readonly PERF_SCRIPT_DIR="${WORKSPACE_DIR}/.azure-pipelines/scripts/performance"-readonly BASELINE_GIT_URL="git+https://github.com/intel/auto-round.git"-readonly ITERS=200--log_group_start() { echo "##[group]$1"; }-log_group_end() { echo "##[endgroup]"; }-log_info() { echo -e "[\033[32mINFO\033[0m] $1"; }-log_err() { echo -e "[\033[31mERROR\033[0m] $1" >&2; }--function setup_environment() {- log_group_start "Set up environment..."-- export TZ='Asia/Shanghai'- export TQDM_MININTERVAL=60- export HF_HUB_DISABLE_PROGRESS_BARS=1- export UV_NO_PROGRESS=1- export UV_SYSTEM_PYTHON=1-- log_info "Creating log directory: ${LOG_DIR}"- mkdir -p "${LOG_DIR}"-- log_info "Downloading model: ${model_name}"- hf download "${model_name}"-- log_group_end-}--function install_auto_round() {- local install_source=$1- local mode_name=$2-- log_group_start "Install requirements for [${mode_name}]..."-- (- cd "${WORKSPACE_DIR}"- log_info "Uninstalling existing auto-round..."- uv pip uninstall auto-round || true-- log_info "Installing auto-round from: ${install_source}"- BUILD_HPU_ONLY=1 uv pip install "${install_source}"- )-- log_group_end-}--function run_performance_test() {- local test_mode=$1- local log_file="${LOG_DIR}/perf_test_${test_mode}.log"-- log_group_start "Run ${test_mode} performance test (${scheme})..."-- (- cd "${PERF_SCRIPT_DIR}"- log_info "Executing auto-round for ${scheme}. Logging to ${log_file}"- auto-round \- --model_name "${model_name}" \- --scheme "${scheme}" \- --iters "${ITERS}" \- --enable_torch_compile \- --device hpu \- --output_dir "./${test_mode}" 2>&1 | tee -a "${log_file}"- )-- log_group_end-}--function run_performance_check() {- log_group_start "Check performance results..."-- (- cd "${PERF_SCRIPT_DIR}"- log_info "Executing check_performance.py"- python check_performance.py- )-- log_group_end-}--function main() {- setup_environment-- install_auto_round "." "current"- run_performance_test "current"-- install_auto_round "${BASELINE_GIT_URL}" "baseline"- run_performance_test "baseline"-- run_performance_check-- log_info "All tasks completed successfully."-}--main "$@"\ No newline at end of filediff --git a/profile_rss_per_block.py b/profile_rss_per_block.pydeleted file mode 100644index 66b13ce59..000000000--- a/profile_rss_per_block.py+++ /dev/null@@ -1,203 +0,0 @@-"""Granular per-block RSS profiling for peak RAM regression diagnosis.--Instruments both old and new architecture to measure RSS at key points-within the per-block quantization loop.--Usage:- # New arch:- python profile_rss_per_block.py- # Old arch:- AR_DISABLE_NEW_ARCH=1 python profile_rss_per_block.py-"""--import gc-import os-import resource-import sys-import time---def rss_mb():- """Get current RSS in MB (no gc.collect - raw measurement)."""- return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB -> MB on Linux---def rss_mb_clean():- """Get current RSS in MB after gc.collect."""- gc.collect()- return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024---# Use psutil for live RSS (ru_maxrss is peak, not current)-import psutil--_proc = psutil.Process()---def live_rss_mb():- """Current RSS in MB (not peak)."""- return _proc.memory_info().rss / (1024 * 1024)---def live_rss_mb_clean():- gc.collect()- try:- import ctypes-- libc = ctypes.CDLL("libc.so.6")- libc.malloc_trim(0)- except Exception:- pass- return _proc.memory_info().rss / (1024 * 1024)---arch = os.environ.get("AR_DISABLE_NEW_ARCH", "0")-arch_label = "OLD" if arch == "1" else "NEW"-print(f"\n{'='*70}")-print(f" {arch_label} Architecture - Granular Per-Block RSS Profiling")-print(f"{'='*70}")-print(f"Before import RSS: {live_rss_mb():.1f} MB")--# Monkey-patch to add instrumentation-if arch != "1":- # NEW ARCH: patch CalibCompressor._quantize_single_block- from auto_round.compressors_new import calib as calib_mod-- _orig_quantize_single_block = calib_mod.CalibCompressor._quantize_single_block- _orig_quantize_blocks = calib_mod.CalibCompressor._quantize_blocks-- _block_rss_log = []-- def _patched_quantize_single_block(self, model, m, input_ids, input_others, q_input):- block_idx = len(_block_rss_log)- rss_before = live_rss_mb()-- result = _orig_quantize_single_block(self, model, m, input_ids, input_others, q_input)-- rss_after_return = live_rss_mb()- gc.collect()- rss_after_gc = live_rss_mb()- try:- import ctypes-- libc = ctypes.CDLL("libc.so.6")- libc.malloc_trim(0)- except Exception:- pass- rss_after_trim = live_rss_mb()-- entry = {- "block": block_idx,- "before": rss_before,- "after_return": rss_after_return,- "after_gc": rss_after_gc,- "after_trim": rss_after_trim,- "delta_return": rss_after_return - rss_before,- "delta_gc": rss_after_gc - rss_before,- "delta_trim": rss_after_trim - rss_before,- }- _block_rss_log.append(entry)- print(- f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "- f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "- f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",- flush=True,- )- return result-- calib_mod.CalibCompressor._quantize_single_block = _patched_quantize_single_block--else:- # OLD ARCH: patch LLMCompressor._quantize_block- from auto_round.compressors import base as base_mod-- _orig_quantize_block = base_mod.LLMCompressor._quantize_block-- _block_rss_log = []-- def _patched_quantize_block(self, block, input_ids, input_others, q_input=None, device="cpu", auto_offload=True):- block_idx = len(_block_rss_log)- rss_before = live_rss_mb()-- result = _orig_quantize_block(self, block, input_ids, input_others, q_input, device, auto_offload)-- rss_after_return = live_rss_mb()- gc.collect()- rss_after_gc = live_rss_mb()- try:- import ctypes-- libc = ctypes.CDLL("libc.so.6")- libc.malloc_trim(0)- except Exception:- pass- rss_after_trim = live_rss_mb()-- entry = {- "block": block_idx,- "before": rss_before,- "after_return": rss_after_return,- "after_gc": rss_after_gc,- "after_trim": rss_after_trim,- "delta_return": rss_after_return - rss_before,- "delta_gc": rss_after_gc - rss_before,- "delta_trim": rss_after_trim - rss_before,- }- _block_rss_log.append(entry)- print(- f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "- f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "- f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",- flush=True,- )- return result-- base_mod.LLMCompressor._quantize_block = _patched_quantize_block--print(f"After import RSS: {live_rss_mb():.1f} MB")--from auto_round import AutoRound--print(f"After AutoRound import RSS: {live_rss_mb():.1f} MB")--import shutil--save_dir = "/tmp/profile_rss_output"-shutil.rmtree(save_dir, ignore_errors=True)--print("\nCreating AutoRound instance...")-ar = AutoRound(- model="Qwen/Qwen3-0.6B",- scheme="FP8_STATIC",- iters=200,- nsamples=128,- enable_torch_compile=True,-)-print(f"After init RSS: {live_rss_mb():.1f} MB")-print(f"After init RSS (clean): {live_rss_mb_clean():.1f} MB")--print("\nStarting quantize_and_save...\n")-model, folder = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")--print(f"\n{'='*70}")-print(f" SUMMARY ({arch_label} Architecture)")-print(f"{'='*70}")-print(f"Final RSS: {live_rss_mb():.1f} MB")-print(f"Final RSS (clean): {live_rss_mb_clean():.1f} MB")-print("\nPer-block deltas (after return, after gc+trim):")-for e in _block_rss_log:- print(- f" Block {e['block']:2d}: delta_ret={e['delta_return']:+.1f} delta_trim={e['delta_trim']:+.1f} MB "- f"(abs: {e['after_trim']:.1f} MB)"- )--# Compute growth rate-if len(_block_rss_log) >= 2:- first = _block_rss_log[0]["after_trim"]- last = _block_rss_log[-1]["after_trim"]- n = len(_block_rss_log) - 1- print(f"\nGrowth: {first:.1f} -> {last:.1f} MB over {n} blocks = {(last-first)/n:.1f} MB/block avg")--print(f"\nPeak RSS (ru_maxrss): {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.1f} MB")--shutil.rmtree(save_dir, ignore_errors=True)diff --git a/test/unit/common/algorithms/test_algorithm_datatype_parity.py b/test/unit/common/algorithms/test_algorithm_datatype_parity.pynew file mode 100644index 000000000..2d4829c05--- /dev/null+++ b/test/unit/common/algorithms/test_algorithm_datatype_parity.py@@ -0,0 +1,86 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Check the public datatype API used by wrappers and algorithms."""++from types import SimpleNamespace++import torch++from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer+from auto_round.data_type.int import quant_tensor_sym+++def test_weight_quantization_reuses_initialization_and_matches_int_math():+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)+ config = dict(data_type="int", bits=4, group_size=4, sym=True, scale_dtype=torch.float32)+ quantizer = create_quantizer(config, disable_opt_rtn=True)+ quantizer.initialize(weight)+ first = quantizer.quantize(weight)+ second = quantizer.quantize(weight)+ expected = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)[0]+ torch.testing.assert_close(first, expected, rtol=0, atol=0)+ assert torch.equal(first, second)+++def test_weight_training_and_writeback_share_parameters():+ layer = torch.nn.Linear(4, 2, bias=False)+ layer.data_type = "int"+ layer.bits = 4+ layer.group_size = 4+ layer.sym = True+ layer.scale_dtype = torch.float32+ quantizer = create_quantizer(layer, iters=10, tune_rounding=True, tune_minmax=True)+ quantizer.initialize(layer.weight.detach())+ output = quantizer.quantize(layer.weight.detach())+ output.square().sum().backward()+ assert all(parameter.grad is not None for parameter in quantizer.parameters.values())+ expected = output.detach().clone()+ with torch.no_grad():+ quantizer.write_back(layer, layer.weight.detach())+ torch.testing.assert_close(layer.weight, expected, rtol=0, atol=0)+ assert layer.scale.shape == (2, 1)+ assert layer.zp is not None+++def test_reinitializing_search_quantizer_does_not_reuse_previous_weight_range():+ config = dict(data_type="int", bits=4, group_size=4, sym=False, scale_dtype=torch.float32)+ quantizer = create_quantizer(config, disable_opt_rtn=True)+ first = torch.tensor([[1.0, -0.75, 0.25, -0.5]])+ second = first * 5+ quantizer.initialize(first)+ quantizer.quantize(first)+ quantizer.initialize(second)+ actual = quantizer.quantize(second)+ fresh = create_quantizer(config, disable_opt_rtn=True)+ fresh.initialize(second)+ assert torch.equal(actual, fresh.quantize(second))+++def test_activation_adapter_static_and_dynamic_contracts():+ activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)+ for dynamic in (False, True):+ layer = SimpleNamespace(+ act_data_type="int",+ act_bits=8,+ act_group_size=4,+ act_sym=True,+ scale_dtype=torch.float32,+ act_dynamic=dynamic,+ )+ quantizer = activation_quantizer_for_layer(layer)+ observed = quantizer.observe(activation, None)+ qdq = quantizer.qdq(activation, observed_max=observed)+ assert qdq.shape == activation.shape+ assert torch.isfinite(qdq).all()diff --git a/test/unit/common/algorithms/transforms/hadamard/test_patch.py b/test/unit/common/algorithms/transforms/hadamard/test_patch.pyindex ff63079dc..6c8d176a8 100644--- a/test/unit/common/algorithms/transforms/hadamard/test_patch.py+++ b/test/unit/common/algorithms/transforms/hadamard/test_patch.py@@ -179,14 +179,14 @@ class TestPatchWrapperLinearIdempotency:# Call the patched _qdq_act on a wrapperwrapper = _make_wrapper_linear()- # Replace the wrapper's act_quant_func with a spy- wrapper.act_quant_func = MagicMock(return_value=(torch.zeros(1, 8), None, None))+ # Activation QDQ is now owned by the datatype quantizer.+ wrapper.activation_quantizer.qdq_with_scale = MagicMock(return_value=(torch.zeros(1, 8), None, None))x = torch.ones(1, 8)act_min = torch.tensor(1.0)act_max = torch.tensor(1.0)- wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max)+ wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max, act_max=act_max)# The mock should have been called with x * 2- called_args = wrapper.act_quant_func.call_args[0]+ called_args = wrapper.activation_quantizer.qdq_with_scale.call_args[0]assert torch.equal(called_args[0], x * 2.0)def test_qdq_weight_falls_through_for_high_bits(self):@@ -294,6 +294,7 @@ class TestPatchWrapperWALayerIdempotency:patch_wrapperwalayer_forward_to_apply_transform(inp_transform)wrapper = _make_wrapper_wa_layer()+ wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))# Patch the orig_layer.forward to be a spyorig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))wrapper.orig_layer.forward = orig_forward_spy@@ -301,8 +302,8 @@ class TestPatchWrapperWALayerIdempotency:x = torch.ones(1, 8)wrapper(x)- # The act_quant_func should have been called with x * 4- called_args = wrapper.orig_layer.act_quant_func.call_args[0]+ # Datatype-owned activation QDQ receives the rotated input.+ called_args = wrapper.activation_quantizer.qdq.call_args[0]assert torch.equal(called_args[0], x * 4.0)def test_act_max_passed_when_present(self):@@ -315,6 +316,7 @@ class TestPatchWrapperWALayerIdempotency:wrapper = _make_wrapper_wa_layer()wrapper.orig_layer.act_max = torch.tensor(2.0)+ wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))wrapper.orig_layer.forward = orig_forward_spy@@ -322,9 +324,7 @@ class TestPatchWrapperWALayerIdempotency:x = torch.ones(1, 8)wrapper(x)- # tensor_max should be the act_max value- kwargs = wrapper.orig_layer.act_quant_func.call_args.kwargs- assert kwargs.get("tensor_max") is not None+ assert wrapper.activation_quantizer.qdq.call_args.kwargs["observed_max"] is not Noneclass TestPatchQuantLinearIdempotency:diff --git a/test/unit/common/core/test_wrapper_utils.py b/test/unit/common/core/test_wrapper_utils.pyindex a603d0da0..68de62b29 100644--- a/test/unit/common/core/test_wrapper_utils.py+++ b/test/unit/common/core/test_wrapper_utils.py@@ -335,6 +335,95 @@ class TestWrapperLinearDeviceTransfer:assert wrapper.device == "cpu"assert wrapper.orig_layer is orig_layer+ def test_unwrapper_keeps_weight_for_activation_only_quantization(self):+ """W16A8 writes activation state only; its weight stays full precision."""+ from auto_round.wrapper import WrapperLinear++ layer = torch.nn.Linear(4, 2, bias=False)+ layer.bits = 16+ layer.sym = True+ layer.group_size = -1+ layer.scale_dtype = torch.float32+ layer.data_type = "int"+ layer.act_bits = 8+ layer.act_data_type = "int"+ layer.act_sym = True+ layer.act_dynamic = True+ layer.act_group_size = -1+ layer.iters = 0+ layer.tuning_device = "cpu"+ expected = layer.weight.detach().clone()++ WrapperLinear(layer, device="cpu", disable_opt_rtn=True).unwrapper({})++ assert torch.equal(layer.weight, expected)+ assert not hasattr(layer, "scale")++ def test_custom_weight_qdq_is_shared_by_forward_and_write_back(self):+ """A per-layer algorithm hook controls both tuning and final materialization."""+ from auto_round.wrapper import WrapperLinear++ orig_layer = torch.nn.Linear(4, 2, bias=False)+ orig_layer.bits = 4+ orig_layer.sym = True+ orig_layer.group_size = 4+ orig_layer.scale_dtype = torch.float32+ orig_layer.data_type = "int"+ orig_layer.act_bits = 16+ orig_layer.act_data_type = "float"+ orig_layer.act_sym = True+ orig_layer.act_dynamic = True+ orig_layer.act_group_size = -1+ orig_layer.tuning_device = "cpu"+ calls = []++ def build_weight_qdq(*, weight, default_qdq, parameters):+ assert weight.shape == (2, 4)+ parameters["offset"] = torch.nn.Parameter(weight.new_tensor(0.1))++ def custom_qdq(weight, *, materialize=False):+ calls.append(materialize)+ return default_qdq(weight + parameters["offset"], materialize=materialize, **parameters)++ return custom_qdq++ wrapper = WrapperLinear(+ orig_layer,+ device="cpu",+ disable_opt_rtn=True,+ iters=0,+ weight_qdq_builder=build_weight_qdq,+ )++ result, _, _ = wrapper._qdq_weight(wrapper.value, wrapper.min_scale, wrapper.max_scale)+ result.sum().backward()+ assert wrapper.params["offset"].grad is not None+ best = torch.tensor(0.5)+ wrapper.params["offset"] = best+ expected = wrapper.weight_qdq(orig_layer.weight, materialize=True).weight.detach().clone()+ wrapper.params["offset"] = torch.tensor(0.9)+ wrapper.unwrapper({"offset": best})++ assert calls == [False, True, True]+ assert torch.equal(orig_layer.weight, expected)++ def test_base_algorithm_uses_datatype_qdq_by_default(self):+ """Algorithms only override QDQ when they explicitly need custom behavior."""+ from auto_round.algorithms.quantization.base import BaseQuantizer+ from auto_round.data_type.base import create_quantizer++ weight = torch.tensor([[1.0, -0.5, 0.25, -0.75]])+ datatype = create_quantizer(dict(data_type="int", bits=4, group_size=4, sym=True), disable_opt_rtn=True)+ datatype.initialize(weight)+ parameters = {"value": torch.tensor(0.0)}+ qdq = BaseQuantizer.build_weight_qdq(object(), weight=weight, default_qdq=datatype.qdq, parameters=parameters)+ torch.testing.assert_close(qdq(weight).weight, datatype.qdq(weight, value=0.0).weight)+ parameters["value"] = torch.tensor(0.75)+ result = qdq(weight, materialize=True)+ expected = datatype.qdq(weight, value=0.75, materialize=True)+ torch.testing.assert_close(result.weight, expected.weight)+ torch.testing.assert_close(result.scale, expected.scale)+def test_wrapper_linear_forward(self):from auto_round.wrapper import WrapperLineardiff --git a/test/unit/common/data_type/__init__.py b/test/unit/common/data_type/__init__.pyindex e69de29bb..14a492441 100644--- a/test/unit/common/data_type/__init__.py+++ b/test/unit/common/data_type/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/test/unit/common/data_type/test_quantizer_parity.py b/test/unit/common/data_type/test_quantizer_parity.pynew file mode 100644index 000000000..9b23109cc--- /dev/null+++ b/test/unit/common/data_type/test_quantizer_parity.py@@ -0,0 +1,189 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Focused coverage for the datatype public lifecycle."""++import pytest+import torch++from auto_round.data_type.base import (+ activation_quantizer_for_layer,+ cache_activation_quantizer,+ canonical_data_type,+ create_quantizer,+)+from auto_round.data_type.fp8 import quant_fp8_e5m2+from auto_round.data_type.gguf import _GGUFWeightQuantizer+from auto_round.data_type.int import quant_tensor_sym+from auto_round.data_type.mxfp import quant_mx, quant_mx_rceil+++def _weight_config(data_type, bits, group_size, *, sym=True):+ return dict(data_type=data_type, bits=bits, group_size=group_size, sym=sym, scale_dtype=torch.float32)+++def test_integer_lifecycle_matches_primitive_and_materializes_payload():+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)+ quantizer.initialize(weight)+ actual = quantizer.quantize(weight)+ expected, _, _ = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)++ assert torch.equal(actual, expected)+++def test_qdq_returns_the_same_result_used_for_write_back():+ """Algorithms can execute QDQ once and hand its materialized result to the datatype."""+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)+ layer = torch.nn.Linear(4, 1, bias=False)+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)+ quantizer.initialize(weight)++ result = quantizer.qdq(weight, materialize=True)+ quantizer.apply_result(layer, result)++ assert torch.equal(layer.weight, result.weight)+ assert layer.scale.shape == (1, 1)+++def test_write_back_keeps_conv1d_scale_indexed_by_output_channels():+ """Conv1D stores a transposed weight, but its scale stays per output channel."""+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5], [-1.0, 0.75, -0.25, 0.5]], dtype=torch.float32)+ layer = torch.nn.Module()+ layer.weight = torch.nn.Parameter(torch.empty(4, 2))+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)+ quantizer.initialize(weight)++ quantizer.write_back(layer, weight, transpose=True)++ assert layer.weight.shape == (4, 2)+ assert layer.scale.shape == (2, 1)+++def test_fp8_format_selection_does_not_turn_e5m2_into_block_e4m3():+ weight = torch.randn(16, 32)+ quantizer = create_quantizer(_weight_config("fp8_e5m2", 8, (8, 16)), iters=1)+ quantizer.initialize(weight)+ result = quantizer.quantize(weight)+ expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=(8, 16))++ assert torch.equal(result, expected)+++def test_generic_mx_name_uses_requested_bit_width():+ weight = torch.randn(8, 32)+ quantizer = create_quantizer(_weight_config("mx_fp", 6, 32), iters=1)+ quantizer.initialize(weight)+ result = quantizer.quantize(weight)+ expected, _, _ = quant_mx(weight, bits=6, group_size=32, data_type="mx_fp6")++ assert torch.equal(result, expected)++[email protected]("dynamic", (True, False))+def test_activation_lifecycle_handles_dynamic_and_calibrated_integer_quantization(dynamic):+ activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)+ layer = type(+ "Layer",+ (),+ dict(act_data_type="int", act_bits=8, act_group_size=4, act_sym=True, act_dynamic=dynamic),+ )()+ quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)+ observed = None if dynamic else quantizer.observe(activation, None)++ result = quantizer.qdq(activation, observed_max=observed)++ assert result.shape == activation.shape+ assert torch.isfinite(result).all()+++def test_nvfp_static_global_scale_accepts_dynamic_calibration():+ """NVFP4 calibrates a static global scale while keeping dynamic QDQ enabled."""+ layer = type(+ "Layer",+ (),+ dict(+ act_data_type="nv_fp4_with_static_gs",+ act_bits=4,+ act_group_size=16,+ act_sym=True,+ act_dynamic=True,+ ),+ )()++ quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)++ assert quantizer.requires_calibration is True+++def test_aliases_keep_a_single_canonical_datatype_for_policy():+ assert canonical_data_type("int") == "int_sym"+ assert canonical_data_type("INT4_ASYM") == "int_asym"+++def test_activation_quantizer_uses_the_spec_default_datatype():+ """Partial layer metadata keeps the public ``int_sym`` activation default."""+ layer = type("Layer", (), dict(act_bits=8, act_group_size=4, act_sym=True, act_dynamic=True))()++ assert type(cache_activation_quantizer(layer)).__name__ == "_IntActivationQuantizer"+ assert type(activation_quantizer_for_layer(layer)).__name__ == "_IntActivationQuantizer"+++def test_format_aliases_keep_the_requested_quantization_format():+ weight = torch.randn(4, 32)+ fp8 = create_quantizer(_weight_config("fp8-e5m2", 8, -1), iters=1)+ fp8.initialize(weight)+ expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=-1)++ gguf = create_quantizer(_weight_config("INT_SYM_DQ", 4, 32), disable_opt_rtn=True)+ legacy_gguf = create_quantizer(_weight_config("rtn_int_sym_dq", 4, 32), disable_opt_rtn=True)++ assert torch.equal(fp8.quantize(weight), expected)+ assert isinstance(gguf._implementation, _GGUFWeightQuantizer)+ assert gguf._implementation.kind == "sym"+ assert isinstance(legacy_gguf._implementation, _GGUFWeightQuantizer)+++def test_mx_aliases_and_rceil_keep_the_requested_format():+ weight = torch.randn(4, 32)+ layer = type(+ "Layer", (), dict(act_data_type="mxfp4", act_bits=4, act_group_size=32, act_sym=True, act_dynamic=True)+ )()+ activation = activation_quantizer_for_layer(layer)+ quantizer = create_quantizer(_weight_config("mx_fp4e2m1_rceil", 4, 32), iters=1)+ quantizer.initialize(weight)+ expected, _, _ = quant_mx_rceil(weight, bits=4, group_size=32, data_type="mx_fp4e2m1")++ assert activation.data_type == "mx_fp4"+ assert torch.equal(quantizer.quantize(weight), expected)+++def test_awq_clip_ranges_are_flattened_before_integer_qdq():+ """AWQ stores [output, group] clips while INT QDQ uses flattened group rows."""+ weight = torch.randn(4, 8)+ clip = weight.abs().reshape(4, 2, 4).amax(dim=-1)+ quantizer = create_quantizer(+ {+ **_weight_config("int", 4, 4),+ "awq_clip_min": -clip,+ "awq_clip_max": clip,+ },+ iters=1,+ )+ quantizer.initialize(weight)++ result = quantizer.quantize(weight)++ assert result.shape == weight.shape+ assert torch.isfinite(result).all()diff --git a/test/unit/common/models/test_moe_experts_interface.py b/test/unit/common/models/test_moe_experts_interface.pyindex 49f0f9125..2656d1c58 100644--- a/test/unit/common/models/test_moe_experts_interface.py+++ b/test/unit/common/models/test_moe_experts_interface.py@@ -21,10 +21,13 @@ This verifies that:3. The forward pass produces correct results"""+from types import SimpleNamespace+import pytestimport torchfrom torch import nn+from auto_round.modeling.fused_moe.grouped_experts import _batched_qdq_weightsfrom auto_round.modeling.fused_moe.replace_modules import (ReplacementModuleBase,_apply_custom_replacements,@@ -42,6 +45,11 @@ def _skip_if_no_linear_loop():pytest.skip("transformers MOE integration not available")+def test_batched_qdq_falls_back_without_legacy_quant_function():+ """Datatype-owned QDQ safely falls back when its private state cannot be fused."""+ assert _batched_qdq_weights([SimpleNamespace()]) is None++def test_linear_loop_registration():"""Test that linear_loop is registered with transformers."""from auto_round.modeling.fused_moe.moe_experts_interface import (diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.pyindex 837a96605..0931831cc 100644--- a/test/unit/common/schemes/test_auto_scheme.py+++ b/test/unit/common/schemes/test_auto_scheme.py@@ -16,6 +16,7 @@ from auto_round.auto_scheme.delta_loss import (_vram_inventory_text,)from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name+from auto_round.data_type.base import WeightQuantizationResultfrom auto_round.export.export_to_gguf.export import _clear_gguf_model_instances@@ -1200,24 +1201,38 @@ class TestScoreAnchorWeightScoring:wrapper.max_act_value = 0wrapper._score_qdq_cpu = None- def _fake_quant(weight, **kwargs):- scale = weight.abs().amax().clamp(min=1e-4) / 7.0- return ((weight / scale).round().clamp(-7, 7) * scale), scale, None+ class _State:+ tunables = {}- wrapper.weight_quant_func = _fake_quant+ class _Quantizer:+ def __init__(self):+ self.calls = 0++ def quantize(self, weight, **kwargs):+ self.calls += 1+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return (weight / scale).round().clamp(-7, 7) * scale++ wrapper.params = {}+ wrapper.weight_state = _State()+ wrapper.weight_quantizer = _Quantizer()+ wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(+ wrapper.weight_quantizer.quantize(weight, **kwargs)+ )layer.weight.requires_grad = Truereturn wrapper, layerdef test_qdq_weight_quantizes_once_and_reuses_cache(self):wrapper, layer = self._make_wrapper()calls = {"n": 0}- real_quant = wrapper.weight_quant_func+ quantizer = wrapper.weight_quantizer+ original_quantize = quantizer.quantize- def counting_quant(weight, **kwargs):+ def counting_quantize(*args, **kwargs):calls["n"] += 1- return real_quant(weight, **kwargs)+ return original_quantize(*args, **kwargs)- wrapper.weight_quant_func = counting_quant+ quantizer.quantize = counting_quantizeargs = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))first, _, _ = wrapper._qdq_weight(*args)second, _, _ = wrapper._qdq_weight(*args)@@ -1396,11 +1411,20 @@ class TestScoreLinearRecompute:wrapper.min_scale = torch.tensor(1.0)wrapper.max_scale = torch.tensor(1.0)- def _fake_quant(weight, **kwargs):- scale = weight.abs().amax().clamp(min=1e-4) / 7.0- return (weight / scale).round().clamp(-7, 7) * scale, scale, None+ class _State:+ tunables = {}- wrapper.weight_quant_func = _fake_quant+ class _Quantizer:+ def quantize(self, weight, **kwargs):+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return (weight / scale).round().clamp(-7, 7) * scale++ wrapper.params = {}+ wrapper.weight_state = _State()+ wrapper.weight_quantizer = _Quantizer()+ wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(+ wrapper.weight_quantizer.quantize(weight, **kwargs)+ )layer.weight.requires_grad = Truereturn wrapper, layerdiff --git a/test/unit/test_cpu/algorithms/test_awq.py b/test/unit/test_cpu/algorithms/test_awq.pyindex 380d76661..c53773b5e 100644--- a/test/unit/test_cpu/algorithms/test_awq.py+++ b/test/unit/test_cpu/algorithms/test_awq.py@@ -492,7 +492,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -522,14 +522,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quant_funcs(params):- return f"{params['data_type']}_{params['bits']}", None+ def fake_resolve_quantizer(params):+ return f"{params['data_type']}_{params['bits']}"- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):- records.append((params["data_type"], params["bits"], quant_func))+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quantizer))return weight- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizertransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -1009,32 +1009,19 @@ class TestAWQUseV2ScaleSearch:q._qdq_tool.configure(compressor)assert q._qdq_tool.disable_opt_rtn is True+ params = {+ "bits": 4,+ "group_size": 128,+ "sym": True,+ "data_type": "int",+ "disable_opt_rtn": True,+ }+ assert q._qdq_tool.resolve_quantizer(params)._mode == "rtn"compressor = self._make_compressor(self._signroundv2_config(data_type="mx_fp"))assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is True- def test_init_scale_dispatch_by_data_type(self):- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale-- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):- weight = torch.randn(32, 128)- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)- assert init_scale is not None, dt- assert init_scale.shape[0] == weight_reshape.shape[0]-- # Scalar imatrix sentinels mean uniform importance and must not enter- # the tensor reshape path.- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)- assert scalar_init_scale is not None- assert scalar_init_scale.shape == init_scale.shape-- # asym int and *_dq are not part of the optimized init-scale path.- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None-def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4diff --git a/test/unit/test_cuda/algorithms/test_alg_ext.py b/test/unit/test_cuda/algorithms/test_alg_ext.pyindex 936e1522d..a618a6c04 100644--- a/test/unit/test_cuda/algorithms/test_alg_ext.py+++ b/test/unit/test_cuda/algorithms/test_alg_ext.py@@ -35,38 +35,6 @@ class TestAlgExt:yieldshutil.rmtree("runs", ignore_errors=True)- # def test_gguf_q2_k_s_uses_dq_wrapper_block(self, tiny_qwen_model_path):- # """Regression test: enable_alg_ext + gguf:q2_k_s must use DQWrapperLinear.- #- # gguf:q2_k_s overrides data_type to "int_asym_dq" at format-resolution- # time. The quantizer must be created *after* that override so that- # SignRoundV2 sees the final data_type and sets dq_wrapper_block- # (which wraps layers with DQWrapperLinear) instead of falling back to- # the plain wrapper_block (which produces WrapperLinear).- # """- # from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundDQWrapperLinear- #- # ar = AutoRound(- # tiny_qwen_model_path,- # bits=4,- # format="gguf:q2_k_s",- # iters=1,- # nsamples=1,- # seqlen=32,- # enable_alg_ext=True,- # )- # # post_init() runs the full pipeline (resolve_scheme → resolve_formats →- # # create_quantizer → ...). quantizer only exists afterwards.- # ar.post_init()- # quantizer = ar.alg_composer.block_quantizer- #- # assert quantizer.wrapper_block.keywords["wrapper_cls"] is SignRoundDQWrapperLinear, (- # f"Expected wrapper_block to use '{SignRoundDQWrapperLinear.__name__}', "- # f"got '{quantizer.wrapper_block.__name__}'. "- # "This likely means the quantizer was created before GGUF format "- # "overrides were applied (data_type was not yet 'int_asym_dq')."- # )-def test_int2_g64_asym_enable_alg_ext_keeps_config(self, tiny_qwen_model_path):"""Regression test: asym int2/g64 keeps the requested tuning config."""diff --git a/test/unit/test_cuda/algorithms/test_awq.py b/test/unit/test_cuda/algorithms/test_awq.pyindex da1459920..106b7e3a7 100644--- a/test/unit/test_cuda/algorithms/test_awq.py+++ b/test/unit/test_cuda/algorithms/test_awq.py@@ -498,7 +498,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -528,14 +528,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quant_funcs(params):- return f"{params['data_type']}_{params['bits']}", None+ def fake_resolve_quantizer(params):+ return f"{params['data_type']}_{params['bits']}"- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):- records.append((params["data_type"], params["bits"], quant_func))+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quantizer))return weight- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizertransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -986,27 +986,6 @@ class TestAWQUseV2ScaleSearch:compressor = self._make_compressor(block)assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False- def test_init_scale_dispatch_by_data_type(self):- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale-- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):- weight = torch.randn(32, 128)- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)- assert init_scale is not None, dt- assert init_scale.shape[0] == weight_reshape.shape[0]-- # Scalar imatrix sentinels mean uniform importance and must not enter- # the tensor reshape path.- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)- assert scalar_init_scale is not None- assert scalar_init_scale.shape == init_scale.shape-- # asym int and *_dq are not part of the optimized init-scale path.- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None-def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4diff --git a/test/unit/test_xpu/algorithms/test_awq.py b/test/unit/test_xpu/algorithms/test_awq.pyindex 9bbb8cb1a..7558302cc 100644--- a/test/unit/test_xpu/algorithms/test_awq.py+++ b/test/unit/test_xpu/algorithms/test_awq.py@@ -494,7 +494,7 @@ class TestAWQMoE:assert transform._mapping_is_smoothable(mapping) is Falseassert any("different quantization parameters" in warning for warning in warnings)- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):"""Direct grid search should pass each balance layer's own resolved quant function."""import torch.nn as nn@@ -524,14 +524,14 @@ class TestAWQMoE:}records = []- def fake_resolve_quant_funcs(params):- return f"{params['data_type']}_{params['bits']}", None+ def fake_resolve_quantizer(params):+ return f"{params['data_type']}_{params['bits']}"- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):- records.append((params["data_type"], params["bits"], quant_func))+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):+ records.append((params["data_type"], params["bits"], quantizer))return weight- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizertransform._qdq_tool.qdq = fake_qdqtransform._grid_search_scales(mapping, torch.ones(8))@@ -981,27 +981,6 @@ class TestAWQUseV2ScaleSearch:compressor = self._make_compressor(block)assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False- def test_init_scale_dispatch_by_data_type(self):- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale-- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):- weight = torch.randn(32, 128)- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)- assert init_scale is not None, dt- assert init_scale.shape[0] == weight_reshape.shape[0]-- # Scalar imatrix sentinels mean uniform importance and must not enter- # the tensor reshape path.- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)- assert scalar_init_scale is not None- assert scalar_init_scale.shape == init_scale.shape-- # asym int and *_dq are not part of the optimized init-scale path.- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None-def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4