Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refactor: decouple datatype quantization from algorithms (#2360)

Heng Guo2e82448a6d79
diff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.py
index b3fd72535..cfc921152 100644
--- a/auto_round/algorithms/base.py
+++ b/auto_round/algorithms/base.py
@@ -127,6 +127,10 @@ class BaseAlgorithm:
def scheme(self) -> "QuantizationScheme | None":
return self.__run_ctx.scheme if self.__run_ctx is not None else None
+ @property
+ def run_context(self) -> "QuantizationRunContext | None":
+ return self.__run_ctx
+
# ── Derived convenience properties ────────────────────────────────────────
@property
def model(self) -> "torch.nn.Module | None":
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 7f455239b..26bdffd6d 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -42,8 +42,9 @@ from auto_round.algorithms.config_resolver import (
split_quantization_configs,
)
from auto_round.algorithms.utils import _has_nvfp4_layer
+from auto_round.data_type.base import canonical_data_type, prepare_data_type_block
from auto_round.logger import logger
-from auto_round.utils import clear_memory
+from auto_round.utils import check_to_quantized, clear_memory
from auto_round.utils.device_manager import device_manager
if TYPE_CHECKING: # avoid circular imports at runtime
@@ -200,10 +201,6 @@ class AlgorithmComposer:
", ".join(blockers),
)
- if _has_nvfp4_layer(orchestrator):
- can_compile_block_forward = False
- logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")
-
# Bind compressor-level infrastructure (set before _build_quantizer is called).
self.block_forward = (
BlockForwardRunner.from_orchestrator(orchestrator, enable_torch_compile=can_compile_block_forward)
@@ -222,31 +219,15 @@ class AlgorithmComposer:
Returns a list of hook handles that the caller must remove when done.
"""
- from auto_round.compressors.utils import is_nv_fp
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
-
- is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)
+ from auto_round.data_type.base import cache_activation_quantizer
def collect_act_max(module, input, output):
input = input[0] if isinstance(input, (tuple, list)) else input
if input.numel() == 0:
return
- module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fp
- input, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)
- act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max") or module.act_max.numel() == 0:
- module.act_max = act_max
- if is_module_act_nv_fp:
- max_val = act_max.max()
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- return
- act_max = act_max.to(module.act_max.device)
- if is_module_act_nv_fp:
- max_val = torch.max(act_max.max(), module.act_max.max())
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- module.act_max = torch.max(act_max, module.act_max)
+ quantizer = cache_activation_quantizer(module)
+ if quantizer is not None:
+ module.act_max = quantizer.observe(input, getattr(module, "act_max", None))
def should_collect(name, module):
from auto_round.compressors.utils import check_need_act_calibration
@@ -255,8 +236,8 @@ class AlgorithmComposer:
if isinstance(module, tuple(SUPPORTED_LAYER_TYPES)):
return (
hasattr(module, "act_dynamic")
- and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)
and check_to_quantized(module)
+ and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False))
)
if hasattr(module, "bits"):
act_dynamic = getattr(module, "act_dynamic", True)
@@ -264,8 +245,8 @@ class AlgorithmComposer:
act_bits = getattr(module, "act_bits", 16)
return (
module.bits <= 8
- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
and check_to_quantized(module)
+ and bool(getattr(cache_activation_quantizer(module), "requires_calibration", False))
)
return False
@@ -307,39 +288,19 @@ class AlgorithmComposer:
q_inputs: Optional list of quantized input tensors; used instead of
``fp_inputs`` when provided.
"""
- from auto_round.compressors.utils import is_nv_fp
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+ from auto_round.data_type.base import cache_activation_quantizer
target_input = q_inputs or fp_inputs
- act_group_size = getattr(layer, "act_group_size")
- if act_group_size is None:
- act_group_size = layer.group_size
- act_data_type = getattr(layer, "act_data_type")
- if act_data_type is None:
- act_data_type = layer.data_type
- is_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False
+ quantizer = cache_activation_quantizer(layer)
+ if quantizer is None:
+ return
for inp in target_input:
if isinstance(inp, (tuple, list)):
inp = inp[0]
if inp.numel() == 0:
continue
- inp, _, _ = reshape_pad_tensor_by_group_size(inp, act_group_size)
- act_max = torch.max(torch.abs(inp), dim=-1).values
-
- if not hasattr(layer, "act_max") or layer.act_max.numel() == 0:
- layer.act_max = act_max
- if is_act_nv_fp_flag:
- max_val = act_max.max()
- layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- continue
-
- act_max = act_max.to(layer.act_max.device)
- if is_act_nv_fp_flag:
- max_val = torch.max(act_max.max(), layer.act_max.max())
- layer.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- layer.act_max = torch.max(act_max, layer.act_max)
+ layer.act_max = quantizer.observe(inp, getattr(layer, "act_max", None))
def need_quanted_input(self):
for preprocessor in self.preprocessors:
@@ -441,12 +402,26 @@ class AlgorithmComposer:
act_data_type = self.scheme.act_data_type if self.scheme else data_type
if act_data_type is not None or not act_dynamic:
from auto_round.compressors.utils import is_nv_fp
- from auto_round.data_type.utils import update_block_global_scale_if_needed
from auto_round.utils import set_amax_for_all_moe_layers
if is_nv_fp(act_data_type) or not act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
- update_block_global_scale_if_needed(block, data_type, group_size)
+
+ runtimes_by_datatype = {}
+ for name, module in block.named_modules():
+ if not check_to_quantized(module):
+ continue
+ module_data_type = getattr(module, "data_type", data_type)
+ try:
+ canonical_id = canonical_data_type(module_data_type)
+ except LookupError:
+ continue
+ runtimes_by_datatype.setdefault(canonical_id, {})[name] = {
+ "data_type": module_data_type,
+ "group_size": getattr(module, "group_size", group_size),
+ }
+ for canonical_id, runtimes in runtimes_by_datatype.items():
+ prepare_data_type_block(canonical_id, block, runtimes)
if q_inputs is not None and fp_inputs is not q_inputs:
clear_memory(fp_inputs)
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index 70b3299e4..fe463fe20 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
from auto_round.algorithms.base import BaseAlgorithm
from auto_round.algorithms.quantization.config import QuantizationConfig
-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.base import create_quantizer
from auto_round.logger import logger
from auto_round.utils import (
check_to_quantized,
@@ -57,6 +57,49 @@ class BaseQuantizer(BaseAlgorithm):
def can_compile_block_forward(self):
return True
+ def build_weight_qdq(self, *, weight: torch.Tensor, default_qdq, parameters: dict[str, torch.Tensor]):
+ """Return the per-layer weight QDQ callable used for tuning and export.
+
+ Algorithms that need different QDQ behavior may override this method.
+ The returned callable accepts ``weight`` and keyword ``materialize=False``
+ and returns ``WeightQuantizationResult``. Bind settings in a closure and
+ add trainable tensors to ``parameters`` before returning. This per-layer
+ dictionary is also used by the optimizer and best-parameter restoration.
+ Read its entries at call time; do not replace the dictionary.
+
+ Args:
+ weight: Logical [out_features, in_features] weights on the tuning
+ device, including transposed Conv1D weights. Use only to create
+ parameters with the right shape/device; do not capture this
+ tensor in the returned closure.
+ default_qdq: Datatype function accepting weight, materialize=False,
+ and explicit datatype parameters. This builder binds the current
+ parameter dictionary to that function. Existing weight parameters
+ are datatype-dependent: value is a rounding offset, while
+ min_scale/max_scale adjust clipping bounds where supported.
+ parameters: Per-layer tensors collected for optimization. Add a
+ torch.nn.Parameter to train it; fixed settings belong in the
+ closure. Use distinct names that do not collide with wrapper
+ attributes. Read entries on each call to see restored best values.
+
+ Returns:
+ Callable qdq(weight, *, materialize=False). With materialize=True,
+ its WeightQuantizationResult must contain scale/zero-point/metadata
+ required by the chosen datatype's exporter. Preserve autograd during
+ training.
+
+ To customize, override this method and use ``weight`` to initialize
+ any new entries in ``parameters``. In the returned function, read those
+ entries and transform the input or explicitly pass the datatype options
+ you need. The implementation below is the default binding template.
+ """
+
+ def qdq(weight, *, materialize=False):
+ """Use current parameters during training and restored best values at export."""
+ return default_qdq(weight, materialize=materialize, **parameters)
+
+ return qdq
+
# ── Calibration hook registration ─────────────────────────────────────────
def register_fp_input_forward_hooks(self, block: torch.nn.Module) -> list:
"""Register hooks that fire during the reference (FP-input) block forward.
@@ -94,53 +137,22 @@ class BaseQuantizer(BaseAlgorithm):
if not check_to_quantized(module):
continue
is_quantized = True
- bits = getattr(module, "bits", None)
- group_size = getattr(module, "group_size", None)
- sym = getattr(module, "sym", None)
- data_type = getattr(module, "data_type", None)
super_bits = getattr(module, "super_bits", None)
super_group_size = getattr(module, "super_group_size", None)
- scale_dtype = self.scale_dtype
- quant_dtype = data_type
- if quant_dtype not in QUANT_FUNC_WITH_DTYPE:
- quant_dtype = f"{quant_dtype}_{'sym' if sym else 'asym'}"
- if not hasattr(self, "iters") or self.iters <= 0: # pylint: disable=E1101
- tmp_dtype = "rtn_" + quant_dtype
- if tmp_dtype in QUANT_FUNC_WITH_DTYPE:
- quant_dtype = tmp_dtype
- quant_func = QUANT_FUNC_WITH_DTYPE[quant_dtype]
# float32 is used in RTN scale search; avoids caching a bf16 copy.
weight_dtype = torch.float32 if super_group_size is not None else module.weight.dtype
- quant_kwargs = {
- "bits": bits,
- "group_size": group_size,
- "super_bits": super_bits,
- "super_group_size": super_group_size,
- "scale_dtype": scale_dtype,
- }
+ quantizer = create_quantizer(module, disable_opt_rtn=True)
try:
- weight, scale, zp = quant_func(
- module.weight.to(dtype=weight_dtype, device=device_manager.device),
- **quant_kwargs,
- )
+ weight = module.weight.to(dtype=weight_dtype, device=device_manager.device)
+ quantizer.initialize(weight)
+ quantizer.write_back(module, weight)
except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU")
- weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)
- except Exception:
- raise
- module.weight.data.copy_(weight.cpu())
- for param_name, val in zip(["scale", "zp"], [scale, zp]):
- if isinstance(val, dict):
- for k, v in val.items():
- setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
- elif isinstance(val, torch.Tensor):
- setattr(module, param_name, val.cpu())
- else:
- setattr(module, param_name, val)
- del weight, scale, zp
+ logger.error(traceback.format_exc())
+ logger.warning("falling back to CPU")
+ weight = module.weight.to("cpu")
+ quantizer.initialize(weight)
+ quantizer.write_back(module, weight)
+ del weight, quantizer
clear_memory()
return is_quantized
@@ -232,6 +244,7 @@ class BaseQuantizer(BaseAlgorithm):
enable_torch_compile=self.compress_context.enable_torch_compile,
disable_opt_rtn=disable_opt_rtn,
iters=0,
+ weight_qdq_builder=self.build_weight_qdq,
)
layer = layer.unwrapper({})
except torch.OutOfMemoryError:
@@ -248,6 +261,7 @@ class BaseQuantizer(BaseAlgorithm):
enable_round_tuning=False,
enable_torch_compile=self.compress_context.enable_torch_compile,
iters=0,
+ weight_qdq_builder=self.build_weight_qdq,
)
layer = layer.unwrapper({})
except Exception:
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index df70764db..25e7d5457 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -374,6 +374,7 @@ class SignRoundQuantizer(BaseQuantizer):
self.enable_norm_bias_tuning,
enable_torch_compile=self.compress_context.enable_torch_compile,
device=device,
+ weight_qdq_builder=self.build_weight_qdq,
)
round_params = []
@@ -503,9 +504,6 @@ class SignRoundQuantizer(BaseQuantizer):
cache_budget,
device,
)
- if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):
- for n, m in block.named_modules():
- m.cur_iter = i
total_loss = 0
global_indices = index_sampler.next_batch()
if valid_token_mask:
@@ -666,6 +664,7 @@ class SignRoundQuantizer(BaseQuantizer):
enable_minmax_tuning=self.enable_minmax_tuning,
enable_torch_compile=self.compress_context.enable_torch_compile,
device=device,
+ weight_qdq_builder=self.build_weight_qdq,
).to(device)
round_params = []
minmax_params = []
@@ -673,7 +672,7 @@ class SignRoundQuantizer(BaseQuantizer):
if "min" in key or "max" in key:
minmax_params.append(wrapper_linear.params[key])
else:
- round_params.append(wrapper_linear.value)
+ round_params.append(wrapper_linear.params[key])
if len(round_params) + len(minmax_params) <= 0:
dump_info = f"quantized {layer_name}"
logger.info(dump_info)
diff --git a/auto_round/algorithms/quantization/sign_roundv2/__init__.py b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
index 4eb376f53..b567f1ca6 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/__init__.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
@@ -12,14 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.algorithms.quantization.sign_roundv2.quantizer import (
- SignRoundDQWrapperLinear,
- SignRoundOptimizedWrapperLinear,
- SignRoundV2Quantizer,
-)
+from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer
__all__ = [
- "SignRoundDQWrapperLinear",
- "SignRoundOptimizedWrapperLinear",
"SignRoundV2Quantizer",
]
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
index 338599e3a..d01778f58 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -13,309 +13,20 @@
# limitations under the License.
from contextlib import nullcontext
-from functools import partial
from typing import TYPE_CHECKING, Callable, Union
import torch
-import transformers
from torch import autocast
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
-
-if TYPE_CHECKING:
- from auto_round.algorithms.composer import AlgorithmComposer
from auto_round.algorithms.registry import register_pipeline_member
-from auto_round.data_type.gguf import (
- double_quant_tensor_sym_rtn,
- quant_tensor_gguf_asym_dq,
- quant_tensor_gguf_sym_dq,
- search_gguf_scale_min_asym,
- search_gguf_scale_min_sym,
-)
-from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym
-from auto_round.data_type.utils import (
- get_optimized_quant_func,
- reshape_imatrix_for_weight,
- reshape_pad_tensor_by_group_size,
- revert_tensor_by_pad,
- round_ste,
- search_optimized_init_scale,
-)
+from auto_round.data_type.base import canonical_data_type
from auto_round.logger import logger
-from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal
-from auto_round.wrapper import WrapperLinear, wrapper_block
-
-
-def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):
- """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""
- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- orig_dtype = tensor.dtype
- tensor = tensor.to(torch.float32)
- maxq = int(2.0**bits) - 1
- inverse_scale = get_reciprocal(scale)
- int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
- qdq = (scale * int_w - wmin).to(orig_dtype)
- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
- return qdq
-
-
-def _dq_sym_qdq(tensor, scale, bits, v=0):
- """Pure sym double-quant qdq math given precomputed scale (compilable)."""
- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
-
- group_size = 16
- super_group_size = 16
- maxq = int(2.0 ** (bits - 1))
- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- orig_dtype = tensor.dtype
- tensor = tensor.to(torch.float32)
- ggml_type = f"q{bits}_k"
- block_size, _ = GGML_QUANT_SIZES[ggml_type]
- n_blocks = tensor.nelement() // block_size
- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- if isinstance(v, torch.Tensor):
- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- else:
- v_r = v
- zp = torch.full_like(scale, maxq)
- inverse_scale = get_reciprocal(scale)
- int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq
- qdq = (scale * (int_w - zp)).to(orig_dtype)
- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
- return qdq
-
-
-def _named_wrapper_block(wrapper_cls, name: str):
- wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)
- wrapped.__name__ = name
- return wrapped
-
-
-def _dq_asym_group_size(bits: int) -> int:
- """Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""
- return 16 if bits == 2 else 32
+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
-
-class SignRoundOptimizedWrapperLinear(WrapperLinear):
- minmax_scale_bound = (0.0, 2.0)
-
- def _init_tuning_params_and_quant_func(self):
- super()._init_tuning_params_and_quant_func()
-
- layer = self.orig_layer
- data_type = layer.data_type
- weight_reshape = self._prepare_init_scale_weight()
- imatrix = reshape_imatrix_for_weight(getattr(layer, "imatrix", None), weight_reshape, layer.group_size)
-
- self.init_scale = search_optimized_init_scale(
- weight_reshape, data_type, layer.bits, imatrix, self.q_scale_thresh
- )
- self.weight_quant_func = get_optimized_quant_func(data_type)
- if self.init_scale is None or self.weight_quant_func is None:
- raise ValueError(
- f"SignRound optimized path does not support data_type={data_type!r}; "
- "expected a symmetric int / mx / nv type."
- )
-
- self.data_type = data_type
- if hasattr(layer, "imatrix"):
- del layer.imatrix
- if self.enable_torch_compile:
- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
-
- def _prepare_init_scale_weight(self) -> torch.Tensor:
- """Return the group-reshaped weight that seeds the init-scale search.
-
- Reproduces the layout the quant func sees at tuning time: the (optionally
- transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in
- ``clip_as_init`` mode and stored a per-group clip range on the layer, the
- weight is clamped to that range first so the searched ``init_scale``
- already reflects the AWQ clip (``max_scale`` then tunes a coefficient on
- top of it).
- """
- layer = self.orig_layer
- weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight
- if isinstance(layer, transformers.pytorch_utils.Conv1D):
- weight = weight.t()
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)
-
- clip_min = getattr(layer, "awq_clip_min", None)
- clip_max = getattr(layer, "awq_clip_max", None)
- if clip_max is not None:
- clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
- clip_min = (
- -clip_max
- if clip_min is None
- else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
- )
- if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:
- weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)
- else:
- logger.warning_once(
- "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "
- "grouped weight shape %s."
- % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))
- )
- return weight_reshape
-
-
-class SignRoundDQWrapperLinear(WrapperLinear):
- minmax_scale_bound = (0.5, 1.5)
-
- def __init__(self, *args, **kwargs) -> None:
- if "enable_minmax_tuning" in kwargs:
- logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")
- kwargs["enable_minmax_tuning"] = False
- super().__init__(*args, **kwargs)
- self.prev_scale = None
- self.prev_wmin = None
- self.prev_d_scale = None
- self.prev_d_wmin = None
-
- def _init_tuning_params_and_quant_func(self):
- super()._init_tuning_params_and_quant_func()
- # The double-quant search path is data-dependent and kept un-compiled,
- # while ``weight_quant_func`` is the compilable pure-math half.
- self._is_dq_path = False
- self._dq_kind = None
- self.search_func = None
- if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:
- self._is_dq_path = True
- if self.orig_layer.data_type == "int_asym_dq":
- self.search_func = search_gguf_scale_min_asym
- self.weight_quant_func = _dq_asym_qdq
- self._dq_kind = "asym"
- else:
- self.search_func = search_gguf_scale_min_sym
- self.weight_quant_func = _dq_sym_qdq
- self._dq_kind = "sym"
- elif self.orig_layer.sym:
- self.weight_quant_func = quant_tensor_sym
- else:
- self.weight_quant_func = quant_tensor_asym
- self.data_type = self.orig_layer.data_type
- if self.enable_act_quant:
- self.act_quant_func = (
- quant_tensor_gguf_asym_dq
- if self.orig_layer.act_data_type == "int_asym_dq"
- else quant_tensor_gguf_sym_dq
- )
- if self.enable_torch_compile:
- self.act_quant_func = compile_func(self.act_quant_func, self.device)
- self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)
- if self.enable_torch_compile:
- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
-
- @torch.no_grad()
- def _run_search(self, weight, v):
- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
-
- bits = self.orig_layer.bits
- scale_dtype = self.orig_layer.scale_dtype
- imatrix = getattr(self.orig_layer, "imatrix", None)
-
- if self._dq_kind == "asym":
- group_size = _dq_asym_group_size(bits)
- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
- v_r = v
- if isinstance(v, torch.Tensor):
- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
- scale, wmin, d_scale, d_wmin = self.search_func(
- t,
- bits=bits,
- scale_dtype=scale_dtype,
- imatrix=imatrix,
- split_num=1,
- v=v_r,
- )
- return {
- "scale": scale.clone(),
- "wmin": wmin.clone(),
- "d_scale": d_scale.clone(),
- "d_wmin": d_wmin.clone(),
- }
-
- group_size = 16
- super_group_size = 16
- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
- ggml_type = f"q{bits}_k"
- block_size, _ = GGML_QUANT_SIZES[ggml_type]
- n_blocks = t.nelement() // block_size
- t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- v_r = v
- if isinstance(v, torch.Tensor):
- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- super_bits = 6 if bits == 3 else 8
- scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)
- scale = scale.to(scale_dtype)
- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
- scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
- scale = scale.unsqueeze(-1)
- return {"scale": scale.clone(), "d_scale": d_scale.clone()}
-
- def _qdq_weight(self, value, min_scale, max_scale):
- if not self._is_dq_path:
- return super()._qdq_weight(value, min_scale, max_scale)
-
- if self.orig_layer.bits >= 16:
- return self.orig_layer.weight, None, None
- min_bound, max_bound = self.minmax_scale_bound
- min_scale.data.clamp_(min_bound, max_bound)
- max_scale.data.clamp_(min_bound, max_bound)
- weight = self.orig_layer.weight
- if weight.device.type == "meta":
- weight = self.orig_layer.get_weight().to(self.device)
- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
- weight = weight.t()
-
- iter_v = getattr(self, "cur_iter", 0)
- need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)
- if need_search:
- params = self._run_search(weight, value)
- self.prev_scale = params["scale"]
- self.prev_d_scale = params["d_scale"]
- if self._dq_kind == "asym":
- self.prev_wmin = params["wmin"]
- self.prev_d_wmin = params["d_wmin"]
- else:
- params = {
- "scale": self.prev_scale.detach(),
- "d_scale": self.prev_d_scale.detach(),
- }
- if self._dq_kind == "asym":
- params["wmin"] = self.prev_wmin.detach()
- params["d_wmin"] = self.prev_d_wmin.detach()
-
- bits = self.orig_layer.bits
- if self._dq_kind == "asym":
- group_size = _dq_asym_group_size(bits)
- weight_q = self.weight_quant_func(
- weight,
- params["scale"],
- params["wmin"],
- bits,
- group_size,
- v=value,
- )
- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
- zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}
- else:
- weight_q = self.weight_quant_func(
- weight,
- params["scale"],
- bits,
- v=value,
- )
- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
- zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))
-
- weight_q = weight_q.to(weight.dtype)
- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
- weight_q = weight_q.t()
- return weight_q, scale_out, zp_out
+if TYPE_CHECKING:
+ from auto_round.algorithms.composer import AlgorithmComposer
@register_pipeline_member(SignRoundV2Config)
@@ -331,18 +42,10 @@ class SignRoundV2Quantizer(SignRoundQuantizer):
"""Model-level setup: initialise scheme-dependent state once before block iteration."""
super().prepare_run(composer=composer)
- if (
- self.scheme.sym
- and self.scheme.super_group_size is None
- and (
- self.scheme.data_type.startswith("int")
- or self.scheme.data_type.startswith("mx")
- or self.scheme.data_type.startswith("nv")
- )
- ):
- if self.scheme.bits > 2 and not (
- self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")
- ):
+ canonical_dtype = canonical_data_type(self.scheme.data_type)
+ is_optimized_family = canonical_dtype.startswith(("int_", "mx_", "nv_", "nvfp"))
+ if self.scheme.sym and self.scheme.super_group_size is None and is_optimized_family:
+ if self.scheme.bits > 2 and canonical_dtype.startswith("int_"):
logger.warning_once(
"algorithm extension has only undergone limited validation on "
"W2A16,INT4, MXFP4 and NVFP4; use with caution."
@@ -351,10 +54,9 @@ class SignRoundV2Quantizer(SignRoundQuantizer):
self._use_outlier_suppressed_loss = True
else:
self._use_outlier_suppressed_loss = False
- self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")
-
- if self.scheme.data_type.endswith("dq"):
- self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")
+ # QDQ ownership stays with the generic wrapper. Datatype adapters
+ # select optimized and double-quant behavior from the layer request;
+ # SignRoundV2 only owns its loss policy.
def can_compile_block_forward(self):
return False
diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.py
index 5ef7e1d71..d3822c2f6 100644
--- a/auto_round/algorithms/quantization/utils.py
+++ b/auto_round/algorithms/quantization/utils.py
@@ -14,8 +14,7 @@
import torch
-from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.data_type.base import cache_activation_quantizer
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
@@ -26,28 +25,15 @@ def register_act_max_hooks(quantizer, model):
input = input[0]
if input.numel() == 0:
return
- module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp
- input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)
- act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max") or module.act_max.numel() == 0:
- module.act_max = act_max
- if is_module_act_nv_fp:
- max_val = act_max.max()
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- act_max = act_max.to(module.act_max.device)
- if is_module_act_nv_fp:
- max_val = torch.max(act_max.max(), module.act_max.max())
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- module.act_max = torch.max(act_max, module.act_max)
+ activation_quantizer = cache_activation_quantizer(module)
+ if activation_quantizer is not None:
+ module.act_max = activation_quantizer.observe(input, getattr(module, "act_max", None))
hook_handles = []
if isinstance(model, SUPPORTED_LAYER_TYPES):
if (
hasattr(model, "act_dynamic")
- and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)
+ and getattr(cache_activation_quantizer(model), "requires_calibration", False)
and check_to_quantized(model)
):
hook_handles.append(model.register_forward_hook(get_act_max_hook))
@@ -56,7 +42,7 @@ def register_act_max_hooks(quantizer, model):
for name, module in model.named_modules():
if (
hasattr(module, "act_dynamic")
- and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)
+ and getattr(cache_activation_quantizer(module), "requires_calibration", False)
and check_to_quantized(module)
):
hook_handles.append(module.register_forward_hook(get_act_max_hook))
@@ -64,12 +50,14 @@ def register_act_max_hooks(quantizer, model):
if name in quantizer.layer_config:
config = quantizer.layer_config[name]
- act_dynamic = config.get("act_dynamic", True)
- act_data_type = config.get("act_data_type", None)
- act_bits = config.get("act_bits", 16)
+ module.act_dynamic = config.get("act_dynamic", True)
+ module.act_data_type = config.get("act_data_type", "int_sym")
+ module.act_bits = config.get("act_bits", 16)
+ module.act_group_size = config.get("act_group_size", quantizer.act_group_size)
+ module.act_sym = config.get("act_sym", True)
if (
config["bits"] <= 8
- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
+ and getattr(cache_activation_quantizer(module), "requires_calibration", False)
and check_to_quantized(config)
):
hook_handles.append(module.register_forward_hook(get_act_max_hook))
diff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.py
index 2c8332e5e..40c839419 100644
--- a/auto_round/algorithms/transforms/awq/base.py
+++ b/auto_round/algorithms/transforms/awq/base.py
@@ -730,7 +730,7 @@ class AWQTransform(BasePreprocessor):
# the grid-search loop. Normal AWQ flow requires one mapping to have
# compatible quant params, but keeping this per-layer avoids hidden
# coupling to the first layer and makes direct calls robust.
- bl_quant_funcs = {bl: self._qdq_tool.resolve_quant_funcs(bl_params[bl]) for bl in mapping.balance_layers}
+ bl_quantizers = {bl: self._qdq_tool.resolve_quantizer(bl_params[bl]) for bl in mapping.balance_layers}
best_error = float("inf")
best_scales = None
@@ -751,12 +751,11 @@ class AWQTransform(BasePreprocessor):
# de-smoothed result back, so the parent forward below sees the
# weights the layer would actually compute with.
for bl in mapping.balance_layers:
- quant_func, opt_quant_func = bl_quant_funcs[bl]
+ quantizer = bl_quantizers[bl]
w_qdq = self._qdq_tool.qdq(
orig_state[bl] * scales_view,
bl_params[bl],
- quant_func=quant_func,
- opt_quant_func=opt_quant_func,
+ quantizer=quantizer,
imatrix=getattr(bl, "imatrix", None),
)
bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)
@@ -767,13 +766,12 @@ class AWQTransform(BasePreprocessor):
else:
total_loss = 0.0
for bl in mapping.balance_layers:
- quant_func, opt_quant_func = bl_quant_funcs[bl]
+ quantizer = bl_quantizers[bl]
w_orig = orig_weights[bl].to(device)
w_qdq = self._qdq_tool.qdq(
w_orig * scales_view,
bl_params[bl],
- quant_func=quant_func,
- opt_quant_func=opt_quant_func,
+ quantizer=quantizer,
imatrix=getattr(bl, "imatrix", None),
)
total_loss += (w_orig - w_qdq / scales_view).pow(2).sum().item()
@@ -1032,8 +1030,8 @@ class AWQTransform(BasePreprocessor):
# group size and drop super-block (double-quant) params, which the clip
# path does not apply.
clip_params = {**params, "group_size": gs, "super_bits": None, "super_group_size": None}
- quant_func, _ = self._qdq_tool.resolve_quant_funcs(clip_params)
- if quant_func is None:
+ quantizer = self._qdq_tool.resolve_quantizer(clip_params)
+ if quantizer is None:
return None
feat = input_feat.to(device).reshape(-1, in_features)
@@ -1072,7 +1070,7 @@ class AWQTransform(BasePreprocessor):
max_val = org_max_val * shrink
cur_w = torch.clamp(w_b, min_val, max_val)
cur_w_flat = cur_w.reshape(cur_w.shape[0], n_group * gs)
- q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quant_func=quant_func).reshape(cur_w.shape)
+ q_w = self._qdq_tool.qdq(cur_w_flat, clip_params, quantizer=quantizer).reshape(cur_w.shape)
cur_out = (feat * q_w).sum(dim=-1)
err = (cur_out - org_out).pow(2).mean(dim=1).view(min_errs.shape)
improved = err < min_errs
diff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.py
index 2a51956a2..e0bb05ed5 100644
--- a/auto_round/algorithms/transforms/awq/qdq.py
+++ b/auto_round/algorithms/transforms/awq/qdq.py
@@ -28,19 +28,15 @@ Usage (the same unified QDQ serves both the scale grid-search and clip-search)::
tool.configure(compressor) # opt_rtn / v2 flags
params = tool.resolve_params(layer) # per-layer scheme
- qf, opt_qf = tool.resolve_quant_funcs(params) # dispatch once, reuse in loop
- w_qdq = tool.qdq(weight, params, quant_func=qf, opt_quant_func=opt_qf)
+ quantizer = tool.resolve_quantizer(params) # dispatch once, reuse in loop
+ w_qdq = tool.qdq(weight, params, quantizer=quantizer)
"""
from __future__ import annotations
import torch
-from auto_round.data_type.utils import (
- compute_optimized_init_scale,
- get_optimized_quant_func,
- get_quant_func,
-)
+from auto_round.data_type.base import create_quantizer
class QDQTool:
@@ -98,7 +94,7 @@ class QDQTool:
"""Resolve the per-layer weight-quant params (``layer_config`` + fallbacks).
Single source of every scheme field the QDQ needs, so callers pass the
- returned dict straight to :meth:`resolve_quant_funcs` and :meth:`qdq`.
+ returned dict straight to :meth:`resolve_quantizer` and :meth:`qdq`.
``super_bits`` / ``super_group_size`` are the GGUF double-quant
super-block params and are ``None`` for non-GGUF schemes.
"""
@@ -113,25 +109,12 @@ class QDQTool:
"super_group_size": cfg.get("super_group_size", None),
}
- def resolve_quant_funcs(self, params: dict):
- """Dispatch ``(quant_func, opt_quant_func)`` for ``params``.
-
- ``opt_quant_func`` is non-``None`` only when the SignRoundV2 optimized
- init-scale path applies (it depends solely on ``data_type`` / ``sym``).
- Hoisting this out of a search loop avoids repeated dispatch.
- """
- quant_func, _ = get_quant_func(
- params["data_type"],
- params["bits"],
- params["sym"],
+ def resolve_quantizer(self, params: dict):
+ """Create one configured quantizer to reuse throughout a weight search."""
+ return create_quantizer(
+ {**params, "scale_dtype": torch.float32},
disable_opt_rtn=params["disable_opt_rtn"],
- group_size=params["group_size"],
- iters=0,
)
- opt_quant_func = None
- if self.use_v2_scale_search and params["sym"]:
- opt_quant_func = get_optimized_quant_func(params["data_type"])
- return quant_func, opt_quant_func
# ── the unified QDQ for AWQ search ──────────────────────
@torch.no_grad()
@@ -140,48 +123,11 @@ class QDQTool:
weight: torch.Tensor,
params: dict,
*,
- quant_func=None,
- opt_quant_func=None,
+ quantizer=None,
imatrix: torch.Tensor | None = None,
) -> torch.Tensor:
- """Quantize-dequantize ``weight`` under the resolved ``params``.
-
- The single QDQ entry point for every scheme: ``bits`` / ``group_size`` /
- ``sym`` / ``data_type`` plus the optional GGUF super-block params drive
- the dispatched ``quant_func``; a non-``None`` ``opt_quant_func`` enables
- the SignRoundV2 optimized init-scale path for this weight. Used for
- both the scale grid-search and the clip-search -- reference/loss only,
- it never mutates stored weights.
- """
- if quant_func is None:
- quant_func, opt_quant_func = self.resolve_quant_funcs(params)
- if quant_func is None:
- raise RuntimeError(
- "QDQTool: no quantization function resolved for "
- f"data_type={params['data_type']}, bits={params['bits']}, "
- f"sym={params['sym']}, group_size={params['group_size']}."
- )
-
- quant_kwargs = {
- "bits": params["bits"],
- "group_size": params["group_size"],
- "data_type": params["data_type"],
- "sym": params["sym"],
- }
- if params.get("super_bits") is not None:
- quant_kwargs["super_bits"] = params["super_bits"]
- if params.get("super_group_size") is not None:
- quant_kwargs["super_group_size"] = params["super_group_size"]
-
- active_quant_func = quant_func
- if opt_quant_func is not None:
- init_scale = compute_optimized_init_scale(
- weight, params["data_type"], params["bits"], params["group_size"], imatrix=imatrix
- )
- if init_scale is not None:
- quant_kwargs["init_scale"] = init_scale
- quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)
- active_quant_func = opt_quant_func
-
- qdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)
- return qdq_weight
+ """Simulate quantization for a search candidate without changing stored weights."""
+ if quantizer is None:
+ quantizer = self.resolve_quantizer(params)
+ quantizer.initialize(weight, imatrix=imatrix)
+ return quantizer.quantize(weight)
diff --git a/auto_round/algorithms/transforms/hadamard/patch.py b/auto_round/algorithms/transforms/hadamard/patch.py
index e3f5f9f95..6971cd304 100644
--- a/auto_round/algorithms/transforms/hadamard/patch.py
+++ b/auto_round/algorithms/transforms/hadamard/patch.py
@@ -24,6 +24,8 @@ Each patch is idempotent: calling it twice has no effect.
from __future__ import annotations
+import math
+
import torch
import transformers
@@ -75,6 +77,8 @@ def patch_wrapperlinear_to_apply_transform(
if is_conv1d:
new_weight = new_weight.t().contiguous()
self.orig_layer.weight.data.copy_(new_weight)
+ logical_weight = new_weight.t() if is_conv1d else new_weight
+ self.weight_quantizer.refresh(logical_weight, imatrix=getattr(self.orig_layer, "imatrix", None))
self.applied_weight_hadamard = True
return _orig_qdq_weight(self, value, min_scale, max_scale)
@@ -106,16 +110,18 @@ def patch_wrapperwalayer_forward_to_apply_transform(
_orig_forward = WrapperWALayer.forward
def _forward_patched(self, x):
- act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
x = inp_transform(x)
- x, _, _ = self.orig_layer.act_quant_func(
+ import auto_round.envs as envs
+
+ act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
+ act_scale = envs.AR_ACT_SCALE
+ max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
+ min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
+ x = self.activation_quantizer.qdq(
x,
- bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.act_group_size,
- scale_dtype=self.orig_layer.scale_dtype,
- q_scale_thresh=self.orig_layer.q_scale_thresh,
- data_type=self.orig_layer.act_data_type,
- tensor_max=act_max,
+ observed_max=act_max,
+ min_scale=min_scale,
+ max_scale=max_scale,
)
return self.orig_layer.forward(x)
diff --git a/auto_round/algorithms/transforms/svdquant/residual.py b/auto_round/algorithms/transforms/svdquant/residual.py
index 461f6763b..aa3515b3e 100644
--- a/auto_round/algorithms/transforms/svdquant/residual.py
+++ b/auto_round/algorithms/transforms/svdquant/residual.py
@@ -26,7 +26,7 @@ from dataclasses import dataclass
import torch
-from auto_round.data_type.utils import get_quant_func
+from auto_round.data_type.base import create_quantizer, quantize_activation
_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})
_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})
@@ -108,16 +108,12 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.
requested_dtype = values["data_type"]
if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:
requested_dtype = f"{requested_dtype}_rceil"
- quant_func, resolved_dtype = get_quant_func(
- dtype=requested_dtype,
- bits=values["bits"],
- sym=values["sym"],
+ quantizer = create_quantizer(
+ {**values, "data_type": requested_dtype, "scale_dtype": tensor.dtype},
disable_opt_rtn=True,
- group_size=values["group_size"],
- iters=0,
)
- logical_dtype = resolved_dtype.removeprefix("rtn_")
- resolved_base_dtype = logical_dtype.removesuffix("_rceil")
+ quantizer.initialize(tensor)
+ resolved_base_dtype = requested_dtype.removesuffix("_rceil")
if (
resolved_base_dtype in _MXFP4_ALIASES
and values["bits"] == 4
@@ -132,12 +128,7 @@ def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.
f"got group_size={values['group_size']!r}."
)
- qdq, _, _ = quant_func(
- tensor=tensor,
- bits=values["bits"],
- group_size=values["group_size"],
- data_type=logical_dtype,
- )
+ qdq = quantizer.quantize(tensor)
if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:
raise ValueError(
f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "
@@ -162,7 +153,13 @@ def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch
@torch.inference_mode()
def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:
"""Apply deployment-compatible dynamic activation quantize-dequantize."""
- return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")
+ values = _validate_scheme_values(scheme)
+ qdq = quantize_activation(activation, values)
+ if qdq.shape != activation.shape or qdq.dtype != activation.dtype or qdq.device != activation.device:
+ raise ValueError("Activation RTN QDQ must preserve shape, dtype, and device.")
+ if not torch.isfinite(qdq).all():
+ raise ValueError("Activation RTN QDQ produced non-finite values.")
+ return qdq
@torch.inference_mode()
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 8d23ff152..8d383e311 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -44,13 +44,7 @@ from auto_round.auto_scheme.utils import (
remove_quant_scheme,
)
from auto_round.calib_dataset import get_dataloader
-from auto_round.data_type.gguf import (
- quant_tensor_gguf_asym_dq,
- quant_tensor_gguf_sym_dq,
- search_gguf_scale_min_asym,
- search_gguf_scale_min_sym,
-)
-from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.data_type.base import canonical_data_type
from auto_round.logger import logger
from auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
@@ -286,278 +280,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):
return qdq_w, 1.0, None
-class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
- """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""
-
- def __init__(
- self,
- orig_layer,
- enable_minmax_tuning=True,
- enable_norm_bias_tuning=False,
- device="cpu",
- enable_round_tuning=True,
- need_weight_grad=False,
- enable_torch_compile=True,
- **kwargs,
- ):
- """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""
- super().__init__(
- orig_layer,
- enable_minmax_tuning,
- enable_norm_bias_tuning,
- device,
- enable_round_tuning,
- enable_torch_compile=enable_torch_compile,
- **kwargs,
- )
- self.act_score = 0.0
- self.avg_act_score = 0.0
- self.act_cnt = 0.0
- self.weight_score = 0.0
- self.mix_score = 0.0
- self.super_qdq_func = super()._qdq_weight
- self.act_qdq_func = super()._qdq_act
- self.max_act_value = 0
- self.need_weight_grad = need_weight_grad
- self.grad_mode = False
- if self.need_weight_grad:
- self.orig_layer.weight.requires_grad = True
- self.weight_search_quant_func, _ = get_quant_func(
- orig_layer.data_type,
- orig_layer.bits,
- orig_layer.sym,
- disable_opt_rtn=False,
- group_size=orig_layer.group_size,
- iters=0,
- )
- self._custom_score_forward = False
- self.post_init_qdqw(device)
-
- @torch.no_grad()
- def post_init_qdqw(self, device):
- """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,
- registering a backward hook on it to accumulate ``weight_score``.
- """
- qdq_w, _, _ = self.weight_search_quant_func(
- self.orig_layer.weight.to(device),
- bits=self.orig_layer.bits,
- group_size=self.orig_layer.group_size,
- v=torch.tensor(0, device=device),
- min_scale=torch.tensor(1.0, device=device),
- max_scale=torch.tensor(1.0, device=device),
- scale_dtype=self.orig_layer.scale_dtype,
- data_type=self.data_type,
- q_scale_thresh=self.q_scale_thresh,
- imatrix=self.orig_layer.imatrix.to(device) if hasattr(self.orig_layer, "imatrix") else None,
- global_scale=getattr(self, "weight_global_scale", None),
- )
-
- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
-
- def save_grad(grad):
- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
- self.mix_score = self.weight_score + self.act_score
- return None
-
- self.qdq_w.requires_grad_(True)
- self.orig_layer.weight.requires_grad_(False)
-
- self.qdq_w.register_hook(save_grad)
-
- def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):
- """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that
- accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.
- """
- if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:
- return x, 1.0, None
-
- qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)
- if self.grad_mode:
- with torch.no_grad():
- max_act_value = torch.abs(x).max()
- self.max_act_value = max_act_value
- if max_act_value != 0:
- self.act_cnt += 1
- x_diff = (x - qdq_x).to("cpu")
-
- def save_grad(grad):
- """Backward hook: accumulate activation score from grad * (x - qdq_x)."""
- if max_act_value == 0:
- if torch.abs(grad).max() != 0:
- raise ValueError
- """
- this ut will cause NAN issue sometimes, need to investigate
- @multi_card
- def test_multi_card(self):
- model_name = "/models/Qwen3-8B"
- """
- if torch.isnan(grad).any() or torch.isnan(x_diff).any():
- self.act_cnt -= 1
- return None
-
- self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()
- self.mix_score = self.weight_score + self.act_score
- return None
-
- if qdq_x.requires_grad:
- qdq_x.register_hook(save_grad)
- return qdq_x, scale, zp
-
- def _qdq_weight(self, value, min_scale, max_scale):
- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
- return self.qdq_w, 1.0, None
-
-
-class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
- """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""
-
- def __init__(
- self,
- orig_layer,
- enable_minmax_tuning=True,
- enable_norm_bias_tuning=False,
- device="cpu",
- enable_round_tuning=True,
- need_weight_grad=False,
- **kwargs,
- ):
- """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""
- super().__init__(
- orig_layer,
- enable_minmax_tuning,
- enable_norm_bias_tuning,
- device,
- enable_round_tuning,
- need_weight_grad,
- **kwargs,
- )
- self._custom_score_forward = False
- self.post_init_qdqw(device)
-
- @torch.no_grad()
- def post_init_qdqw(self, device):
- """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,
- registering a backward hook on it to accumulate ``weight_score``.
- """
- qdq_w, scale, zp = self.super_qdq_func(
- torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)
- )
- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
-
- def save_grad(grad):
- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
- # TODO strange, grad could be in CPU
- self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()
- self.mix_score = self.weight_score + self.act_score
- return None
-
- self.qdq_w.requires_grad_(True)
- self.orig_layer.weight.requires_grad_(False)
- self.qdq_w.register_hook(save_grad)
-
- def _qdq_weight(self, value, min_scale, max_scale):
- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
- return self.qdq_w, 1.0, None
-
-
-class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
- """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant
- search (``_init_scale``).
- """
-
- def __init__(
- self,
- orig_layer,
- enable_minmax_tuning=True,
- enable_norm_bias_tuning=False,
- device="cpu",
- enable_round_tuning=True,
- need_weight_grad=False,
- enable_torch_compile=True,
- **kwargs,
- ):
- """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to
- build ``qdq_w``.
- """
- super().__init__(
- orig_layer,
- enable_minmax_tuning,
- enable_norm_bias_tuning,
- device,
- enable_round_tuning,
- need_weight_grad,
- enable_torch_compile=enable_torch_compile,
- **kwargs,
- )
- self._custom_score_forward = False
- self.post_init_qdqw(device)
-
- @torch.no_grad()
- def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher
- """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer
- ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.
- """
- qdq_w = self._init_scale(device).detach()
- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
-
- def save_grad(grad):
- """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
- self.mix_score = self.weight_score + self.act_score
- return None
-
- self.qdq_w.requires_grad_(True)
- self.orig_layer.weight.requires_grad_(False)
-
- self.qdq_w.register_hook(save_grad)
-
- @torch.no_grad()
- def _init_scale(self, device):
- """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in
- [2,3,4,5,6], returned in the original weight dtype.
- """
- tensor = self.orig_layer.weight.data.to(device)
- bits = self.orig_layer.bits
- scale_dtype = self.orig_layer.scale_dtype
- imatrix = self.orig_layer.imatrix.to(tensor.device)
- orig_dtype = tensor.dtype
- if self.orig_layer.bits in [2, 4, 5]:
- group_size = 16 if bits == 2 else 32
- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)
- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
-
- qdq_w, _, _ = quant_tensor_gguf_asym_dq(
- tensor=tensor,
- bits=bits,
- scale_dtype=scale_dtype,
- imatrix=imatrix,
- scale=scale,
- wmin=wmin,
- d_scale=d_scale,
- d_wmin=d_wmin,
- )
- elif bits in [3, 6]:
- qdq_w, _, _ = quant_tensor_gguf_sym_dq(
- tensor=tensor,
- bits=bits,
- scale_dtype=scale_dtype,
- imatrix=imatrix,
- split_num=1,
- )
- else:
- raise ValueError("bits must be in [2,3,4,5,6]")
- return qdq_w.to(orig_dtype)
-
- def _qdq_weight(self, value, min_scale, max_scale):
- """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
- return self.qdq_w, 1.0, None
-
-
def register_imatrix_hook(model):
"""Registers hooks to accumulate activation squared norms into `imatrix`."""
@@ -814,7 +536,13 @@ def _replay_retain_graph(block_module) -> bool:
"""
for _, module in block_module.named_modules():
data_type = getattr(module, "data_type", None)
- if isinstance(data_type, str) and data_type.startswith("mx"):
+ if not isinstance(data_type, str):
+ continue
+ try:
+ canonical_id = canonical_data_type(data_type)
+ except LookupError:
+ continue
+ if canonical_id.startswith("mx_"):
return True
return False
@@ -1224,15 +952,6 @@ def get_score_for_scheme(
if hasattr(m, "bias") and m.bias is not None:
m.bias.requires_grad = False
- has_imatrix = False
- for name in quant_layer_names:
- if name in fixed_layer_scheme.keys():
- continue
- m = get_module(model, name)
- if hasattr(m, "imatrix") and m.imatrix is not None:
- has_imatrix = True
- break
-
for name in quant_layer_names:
if offload_context is not None:
offload_context.ensure_loaded(model, name)
@@ -1253,14 +972,6 @@ def get_score_for_scheme(
m.scale_dtype = torch.bfloat16
WrapperLayer = AutoSchemeWrapperLinear
- # if has_imatrix: # no better result
- # WrapperLayer = AutoSchemeWrapperLinearIMatrix
- if hasattr(m, "super_group_size") and m.super_group_size is not None:
- if has_imatrix:
- WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix
- else:
- WrapperLayer = AutoSchemeWrapperLinearForGGUFK
-
with torch.no_grad():
if low_gpu_mem_usage:
device = m.tuning_device if hasattr(m, "tuning_device") else major_device
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 4e1968684..d5617e7aa 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1295,8 +1295,34 @@ class BaseOrchestrator(object):
return get_torch_compile_off_reason(model)
def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:
- """Apply torch.compile disabling rules for the current compressor state."""
- self.enable_torch_compile = enable_torch_compile
+ """Apply torch.compile disabling rules for the current compressor state.
+
+ This is intentionally kept beside the compressor state it reads. The
+ rules are not reusable policy: every input comes from this instance,
+ and preserving that context makes the precedence easy to audit.
+ """
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+
+ _, is_valid_act_static = self._get_torch_compile_guard_state()
+ reason = None
+ if enable_torch_compile:
+ if is_valid_act_static:
+ reason = "activation is static"
+ else:
+ reason = self._torch_compile_unsupported_arch_reason()
+ user_specified = getattr(self, "_torch_compile_user_specified", False)
+ is_auto_scheme = getattr(self, "is_auto_scheme", False) or isinstance(
+ getattr(self, "scheme", None), AutoScheme
+ )
+ if reason is None and not user_specified and not is_auto_scheme:
+ if isinstance(self.quantize_config, RTNConfig):
+ reason = "RTN/OPT-RTN quantizes each layer in a single pass"
+ else:
+ iters = getattr(self.quantize_config, "iters", None)
+ if iters is not None and iters < MIN_ITERS_FOR_TORCH_COMPILE:
+ reason = f"`iters`={iters} is below {MIN_ITERS_FOR_TORCH_COMPILE}"
+
+ self.enable_torch_compile = enable_torch_compile and reason is None
# Why compilation ended up off, used by ``_log_torch_compile_state``. When the
# incoming value is already False, keep the reason recorded by the earlier
# precheck pass instead of dropping it.
@@ -1308,33 +1334,9 @@ class BaseOrchestrator(object):
or getattr(self, "_torch_compile_default_off_reason", None)
)
)
- _, is_valid_act_static = self._get_torch_compile_guard_state()
-
- # On HPU, we rely on torch.compile to speed up the model execution.
- if self.enable_torch_compile and is_valid_act_static:
- self.enable_torch_compile = False
- self._torch_compile_off_reason = "activation is static"
- logger.warning_once("reset enable_torch_compile to `False` as activation is static")
-
- # Architecture-level hard block (DeepSeek / GLM-5.3-Flash DSA families). These
- # hit dynamo's recompile_limit and cannot be overridden by an explicit
- # ``enable_torch_compile=True``.
- if self.enable_torch_compile:
- arch_reason = self._torch_compile_unsupported_arch_reason()
- if arch_reason is not None:
- self.enable_torch_compile = False
- self._torch_compile_off_reason = arch_reason
- logger.warning_once("reset enable_torch_compile to `False` as %s", arch_reason)
-
- if self.enable_torch_compile:
- disabled_reason = self._torch_compile_disabled_reason()
- if disabled_reason is not None:
- self.enable_torch_compile = False
- self._torch_compile_off_reason = disabled_reason
- logger.warning_once(
- "reset enable_torch_compile to `False` as %s, " "so compilation cost would outweigh its benefit",
- disabled_reason,
- )
+ if reason is not None:
+ self._torch_compile_off_reason = reason
+ logger.warning_once("reset enable_torch_compile to `False` as %s", reason)
def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:
"""Apply early torch.compile adjustments before scheme resolution.
diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.py
index 48df479ed..d790485fb 100644
--- a/auto_round/compressors/layer_config_resolver.py
+++ b/auto_round/compressors/layer_config_resolver.py
@@ -38,6 +38,31 @@ from auto_round.utils import (
from auto_round.utils.model import get_module
+def normalize_layer_config_entry(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
+ """Resolve one public layer override through the existing preset machinery."""
+ from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+
+ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
+ if isinstance(item, str):
+ config = asdict(preset_name_to_scheme(item.upper()))
+ elif isinstance(item, QuantizationScheme):
+ config = asdict(item)
+ elif isinstance(item, dict):
+ item = {k: v for k, v in item.items() if k != "in_blocks"}
+ scheme_name = item.pop("scheme", None)
+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
+ invalid = set(item) - set(scheme_keys + ("fixed_by_user",))
+ if invalid:
+ raise ValueError(f"Invalid keys {invalid} in layer_config for '{layer_name}'. Allowed keys: {scheme_keys}")
+ config.update(item)
+ else:
+ raise TypeError(
+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
+ "Expected str, dict, or QuantizationScheme."
+ )
+ return {key: value for key, value in config.items() if value is not None}
+
+
def apply_layer_config_special_cases(
layer_config,
model,
@@ -170,34 +195,6 @@ def _resolve_layer_config_presets(
bits-inference, and default-filling."""
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
- def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
- """Convert config entry into dict and validate keys."""
- if isinstance(item, str):
- config = asdict(preset_name_to_scheme(item.upper()))
- elif isinstance(item, QuantizationScheme):
- config = asdict(item)
- elif isinstance(item, dict):
- # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
- # silently drop it before validation.
- item = {k: v for k, v in item.items() if k != "in_blocks"}
- scheme_name = item.pop("scheme", None)
- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
- invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
- if invalid:
- raise ValueError(
- f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
- )
- config.update(item)
- else:
- raise TypeError(
- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
- f"Expected str, dict, or QuantizationScheme."
- )
- # Clean up
- config = {k: v for k, v in config.items() if v is not None}
- config["fixed_by_user"] = True
- return config
-
extra_scheme_keys = ("scale_dtype",)
scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
layer_config = copy.deepcopy(layer_config) or {}
@@ -218,7 +215,7 @@ def _resolve_layer_config_presets(
}
# 2. normalize
- layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
+ layer_config = {k: {**normalize_layer_config_entry(v, k), "fixed_by_user": True} for k, v in layer_config.items()}
# entries whose sym was set by the user (dict key or a QuantizationScheme
# object) rather than inherited from the default scheme below
explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.py
index 58c746470..58bd9076a 100644
--- a/auto_round/data_type/__init__.py
+++ b/auto_round/data_type/__init__.py
@@ -15,7 +15,7 @@
import auto_round.data_type.int
import auto_round.data_type.mxfp
import auto_round.data_type.fp8
-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE
import auto_round.data_type.w4fp8
from auto_round.data_type.utils import (
get_quant_func,
@@ -24,3 +24,8 @@ from auto_round.data_type.utils import (
)
import auto_round.data_type.nvfp
import auto_round.data_type.gguf
+from auto_round.data_type.base import (
+ DataTypeQuantizer,
+ canonical_data_type,
+ create_quantizer,
+)
diff --git a/auto_round/data_type/base.py b/auto_round/data_type/base.py
new file mode 100644
index 000000000..c34d336fb
--- /dev/null
+++ b/auto_round/data_type/base.py
@@ -0,0 +1,313 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+
+"""Small public boundary between quantization algorithms and data types.
+
+Algorithms describe *what* a layer needs quantized. This module resolves the
+requested datatype and delegates all datatype-specific work to that datatype's
+class. It deliberately contains no quantization math.
+"""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from types import SimpleNamespace
+from typing import Any, Callable
+
+import torch
+
+# Low-level tensor primitives remain functions. Quantizer classes below own
+# their lifecycle and are the only interface used by algorithms and wrappers.
+QUANT_FUNC_WITH_DTYPE: dict[str, Callable] = {}
+_QUANTIZER_CLASSES: dict[str, type] = {}
+_CANONICAL_DATA_TYPES: dict[str, str] = {}
+
+
+def _normalize_data_type(data_type: str) -> str:
+ """Normalize spelling differences in datatype names for lookup."""
+ return data_type.lower().replace("-", "").replace("_", "")
+
+
+def register_dtype(names):
+ """Register a low-level tensor quantization primitive by datatype name."""
+
+ def register(function):
+ for name in (names,) if isinstance(names, str) else names:
+ QUANT_FUNC_WITH_DTYPE[name] = function
+ return function
+
+ return register
+
+
+def register_quantizer(canonical_id: str, *, aliases: tuple[str, ...] = ()):
+ """Register a datatype class under its canonical name and aliases.
+
+ Datatype modules use this decorator at import time. The mapping exists
+ solely to turn user-facing aliases such as ``"int"`` into the class that
+ owns their implementation.
+ """
+
+ def register(quantizer_class):
+ names = (canonical_id, *aliases)
+ normalized = tuple(dict.fromkeys(_normalize_data_type(name) for name in names))
+ duplicate = next((name for name in normalized if name in _QUANTIZER_CLASSES), None)
+ if duplicate is not None:
+ raise ValueError(f"Datatype name {duplicate!r} is already registered")
+ for name in normalized:
+ _QUANTIZER_CLASSES[name] = quantizer_class
+ _CANONICAL_DATA_TYPES[name] = canonical_id
+ return quantizer_class
+
+ return register
+
+
+def canonical_data_type(data_type: str) -> str:
+ """Return an alias's canonical name for datatype-dependent policy.
+
+ This is intentionally separate from construction: callers such as loss
+ policy and block preparation need to classify a datatype without creating
+ a quantizer or allocating tensors.
+ """
+ try:
+ return _CANONICAL_DATA_TYPES[_normalize_data_type(data_type)]
+ except KeyError as error:
+ raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error
+
+
+def _quantizer_class(data_type: str) -> type:
+ """Look up the class that owns a datatype implementation."""
+ try:
+ return _QUANTIZER_CLASSES[_normalize_data_type(data_type)]
+ except KeyError as error:
+ raise LookupError(f"No datatype quantizer registered for {data_type!r}") from error
+
+
+def _create_weight_quantizer(data_type: str, spec: "WeightQuantizationSpec"):
+ """Create the implementation owned by ``data_type`` for one weight request."""
+ canonical = canonical_data_type(data_type)
+ quantizer_class = _quantizer_class(data_type)
+ try:
+ return quantizer_class.from_spec(spec, canonical)
+ except AttributeError as error:
+ raise LookupError(f"Datatype {data_type!r} supports activation quantization only") from error
+
+
+def _create_activation_quantizer(data_type: str, spec: "ActivationQuantizationSpec"):
+ """Create the activation implementation owned by ``data_type``."""
+ canonical = canonical_data_type(data_type)
+ try:
+ return _quantizer_class(data_type).create_activation(replace(spec, data_type=canonical))
+ except AttributeError as error:
+ raise LookupError(f"Datatype {data_type!r} has no activation quantizer") from error
+
+
+def prepare_data_type_block(data_type: str, block, layer_runtimes) -> None:
+ """Run the optional block-level setup owned by one datatype class."""
+ quantizer_class = _QUANTIZER_CLASSES.get(_normalize_data_type(data_type))
+ prepare = getattr(quantizer_class, "prepare_block", None)
+ if prepare is not None:
+ prepare(block, layer_runtimes)
+
+
+@dataclass(frozen=True)
+class WeightQuantizationSpec:
+ """Static properties needed to quantize one layer's weights."""
+
+ data_type: str
+ bits: int
+ group_size: int | tuple[int, int]
+ sym: bool
+ scale_dtype: torch.dtype
+ q_scale_thresh: float = 1e-5
+ super_bits: int | None = None
+ super_group_size: int | None = None
+ global_scale: torch.Tensor | None = None
+ clip_min: torch.Tensor | None = None
+ clip_max: torch.Tensor | None = None
+
+
+@dataclass(frozen=True)
+class ActivationQuantizationSpec:
+ """Static properties needed to quantize one layer's activations."""
+
+ data_type: str
+ bits: int
+ group_size: int | tuple[int, int]
+ sym: bool
+ scale_dtype: torch.dtype
+ dynamic: bool
+ q_scale_thresh: float = 1e-5
+
+
+@dataclass(frozen=True)
+class WeightQuantizationResult:
+ """Weight QDQ output plus the tensors required by export and inference."""
+
+ weight: torch.Tensor
+ scale: torch.Tensor | dict[str, torch.Tensor] | None = None
+ zero_point: torch.Tensor | dict[str, torch.Tensor] | int | float | None = None
+ metadata: object | None = None
+ # Conv1D weights are transposed before export. Keep the original output
+ # dimension so per-output-channel tensors retain their intended layout.
+ logical_rows: int | None = None
+
+
+class DataTypeQuantizer:
+ """Run a datatype implementation through initialize, QDQ, and write-back.
+
+ This is the only lifecycle wrapper shared by all data types. Datatype
+ classes keep their own state and numerical code; this class only manages
+ the sequence and the trainable parameter dictionary.
+ """
+
+ def __init__(self, implementation: Any, mode: str, tune_rounding: bool, tune_minmax: bool, apply_result):
+ self._implementation = implementation
+ self._mode = mode
+ self._tune_rounding = tune_rounding
+ self._tune_minmax = tune_minmax
+ self._apply_result = apply_result
+ self._state = None
+ self.parameters: dict[str, torch.Tensor] = {}
+
+ def initialize(self, weight: torch.Tensor, *, imatrix=None) -> None:
+ """Initialize datatype state and expose its trainable tensors."""
+ self._state = self._implementation.create_state(
+ weight,
+ imatrix=imatrix,
+ mode=self._mode,
+ tune_rounding=self._tune_rounding,
+ tune_minmax=self._tune_minmax,
+ )
+ tunables = self._state.get("tunables", self._state) if isinstance(self._state, dict) else self._state.tunables
+ self.parameters = dict(tunables)
+
+ def refresh(self, weight: torch.Tensor, *, imatrix=None) -> None:
+ """Rebuild fixed datatype state while preserving existing trainable tensors."""
+ previous_parameters = self.parameters
+ self.initialize(weight, imatrix=imatrix)
+ if isinstance(self._state, dict):
+ self._state["tunables"] = previous_parameters
+ else:
+ self._state = replace(self._state, tunables=previous_parameters)
+ self.parameters = previous_parameters
+
+ def quantize(self, weight: torch.Tensor, **parameters) -> torch.Tensor:
+ """Return QDQ weights for the current trainable parameter values."""
+ return self.qdq(weight, **parameters).weight
+
+ def qdq(self, weight: torch.Tensor, *, materialize=False, **parameters) -> WeightQuantizationResult:
+ """Run datatype QDQ and optionally include tensors needed for export."""
+ self._ensure_initialized()
+ return self._implementation.qdq(
+ weight,
+ self._state,
+ tunables={**self.parameters, **parameters},
+ materialize=materialize,
+ )
+
+ def apply_result(self, layer, result: WeightQuantizationResult, *, transpose=False) -> None:
+ """Store an already computed QDQ result on its source layer."""
+ logical_rows = result.weight.shape[0]
+ if transpose:
+ result = WeightQuantizationResult(
+ result.weight.t(), result.scale, result.zero_point, result.metadata, logical_rows
+ )
+ if result.weight.device != layer.weight.device:
+ result = WeightQuantizationResult(
+ result.weight.to(layer.weight.device),
+ result.scale,
+ result.zero_point,
+ result.metadata,
+ result.logical_rows,
+ )
+ self._apply_result(layer, result)
+
+ def write_back(self, layer, weight: torch.Tensor, *, transpose=False, **parameters) -> None:
+ """Materialize QDQ payload and store it on an exportable layer."""
+ result = self.qdq(weight, materialize=True, **parameters)
+ self.apply_result(layer, result, transpose=transpose)
+
+ def _ensure_initialized(self) -> None:
+ """Reject use before a datatype has created its per-layer state."""
+ if self._state is None:
+ raise RuntimeError("Call initialize() before quantize() or write_back().")
+
+
+def _weight_spec(layer) -> WeightQuantizationSpec:
+ """Extract the weight request from a resolved quantized layer."""
+ scale_dtype = getattr(layer, "scale_dtype", torch.float32)
+ return WeightQuantizationSpec(
+ data_type=layer.data_type,
+ bits=layer.bits,
+ group_size=layer.group_size,
+ sym=layer.sym,
+ scale_dtype=scale_dtype,
+ q_scale_thresh=getattr(layer, "q_scale_thresh", 1e-5),
+ super_bits=getattr(layer, "super_bits", None),
+ super_group_size=getattr(layer, "super_group_size", None),
+ global_scale=getattr(layer, "weight_global_scale", None),
+ clip_min=getattr(layer, "awq_clip_min", None),
+ clip_max=getattr(layer, "awq_clip_max", None),
+ )
+
+
+def _activation_spec(layer) -> ActivationQuantizationSpec:
+ """Extract the activation request from a resolved quantized layer."""
+ return ActivationQuantizationSpec(
+ data_type=getattr(layer, "act_data_type", "int_sym"),
+ bits=getattr(layer, "act_bits", 16),
+ group_size=getattr(layer, "act_group_size", -1),
+ sym=getattr(layer, "act_sym", True),
+ scale_dtype=getattr(layer, "scale_dtype", torch.float32),
+ dynamic=getattr(layer, "act_dynamic", True),
+ )
+
+
+def create_quantizer(layer, *, iters=None, disable_opt_rtn=False, tune_rounding=False, tune_minmax=False):
+ """Create the datatype-owned weight quantizer for a resolved layer."""
+ if isinstance(layer, Mapping):
+ layer = SimpleNamespace(**{"q_scale_thresh": 1e-5, **layer})
+ spec = _weight_spec(layer)
+ iters = getattr(layer, "iters", 0) if iters is None else iters
+ mode = "tuned" if iters > 0 else "rtn" if disable_opt_rtn else "optimized_rtn"
+ implementation = _create_weight_quantizer(spec.data_type, spec)
+ return DataTypeQuantizer(implementation, mode, tune_rounding, tune_minmax, implementation.apply_result)
+
+
+def cache_activation_quantizer(layer):
+ """Create and cache a layer's activation quantizer when activation is quantized."""
+ if getattr(layer, "act_bits", 16) > 8:
+ return None
+ if hasattr(layer, "_ar_activation_quantizer"):
+ return layer._ar_activation_quantizer
+ spec = _activation_spec(layer)
+ quantizer = _create_activation_quantizer(spec.data_type, spec)
+ layer._ar_activation_quantizer = quantizer
+ return quantizer
+
+
+def activation_quantizer_for_layer(layer, *, scale_dtype=None):
+ """Create the activation quantizer described by a layer's configuration."""
+ spec = _activation_spec(layer)
+ if scale_dtype is not None:
+ from dataclasses import replace
+
+ spec = replace(spec, scale_dtype=scale_dtype)
+ return _create_activation_quantizer(spec.data_type, spec)
+
+
+def quantize_activation(tensor, config):
+ """Apply dynamic activation quantization using a simple configuration mapping."""
+ spec = ActivationQuantizationSpec(
+ data_type=config["data_type"],
+ bits=config["bits"],
+ group_size=config["group_size"],
+ sym=config["sym"],
+ scale_dtype=tensor.dtype,
+ dynamic=True,
+ )
+ return _create_activation_quantizer(spec.data_type, spec).qdq(tensor)
diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.py
index 11f4594c2..708390e63 100644
--- a/auto_round/data_type/fp8.py
+++ b/auto_round/data_type/fp8.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.register import register_dtype
+from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.utils import (
float8_e4m3fn_ste,
float8_e5m2_ste,
@@ -366,3 +366,136 @@ if is_gaudi2():
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
qdq_res = qdq_res.to(orig_dtype)
return qdq_res, scale, None
+
+
+class _FP8WeightQuantizer:
+ """Select the requested FP8 format and materialize its scale layout."""
+
+ def __init__(self, spec, primitive=None, scale_layout=None, data_type=None):
+ self.spec = spec
+ data_type = (data_type or spec.data_type).lower()
+ if primitive is None:
+ if data_type in ("fp8_e5m2",):
+ primitive, scale_layout = quant_fp8_e5m2, "row"
+ elif data_type in ("fp8_unit_scale",):
+ primitive, scale_layout = quant_fp8_unit_scale, "scalar"
+ elif data_type in ("fp8_e5m2_unit_scale",):
+ primitive, scale_layout = quant_fp8_e5m2_unit_scale, "scalar"
+ elif data_type == "fp8_gaudi3_sym":
+ primitive, scale_layout = quant_fp8_sym_gaudi3, "row"
+ elif isinstance(spec.group_size, tuple):
+ primitive, scale_layout = quant_block_fp_sym, "block"
+ else:
+ primitive, scale_layout = quant_fp8_sym, "row"
+ self.primitive = primitive
+ self.scale_layout = scale_layout
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the FP8 weight quantizer selected by the requested format."""
+ return cls(spec, data_type=canonical)
+
+ @staticmethod
+ def create_activation(spec):
+ """Create the FP8 activation quantizer for the same format."""
+ return _create_fp8_activation(spec)
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ reduction_dims = 2 if self.scale_layout == "block" else 1
+ tunable_shape = grouped.shape[:-reduction_dims]
+ tunables = {}
+ if tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if tune_minmax:
+ tunables["max_scale"] = torch.nn.Parameter(
+ torch.ones(tunable_shape, device=weight.device, dtype=torch.float32)
+ )
+ return tunables
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ quantized, scale, zero_point = self.primitive(
+ weight,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ v=tunables.get("value", 0),
+ max_scale=tunables.get("max_scale", 1.0),
+ )
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(
+ quantized,
+ scale if materialize else None,
+ zero_point if materialize else None,
+ self.scale_layout if materialize else None,
+ )
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("FP8 weight result was not materialized")
+ scale = result.scale
+ rows = result.logical_rows or result.weight.shape[0]
+ if result.metadata == "row" and scale.numel() % rows == 0:
+ scale = scale.reshape(rows, -1)
+ module.weight.data.copy_(result.weight)
+ module.scale = scale.cpu()
+ module.zp = result.zero_point
+
+
+class _FP8ActivationQuantizer:
+ """Apply the matching FP8 primitive to dynamic or calibrated activations."""
+
+ def __init__(self, spec, data_type, primitive, scale_layout):
+ self.spec = spec
+ self.data_type = data_type
+ self.primitive = primitive
+ self.scale_layout = scale_layout
+ self.bits = spec.bits
+ self.group_size = spec.group_size
+ self.requires_calibration = not spec.dynamic
+
+ def observe(self, activation, current):
+ if self.scale_layout == "tensor":
+ maximum = activation.detach().abs().max()
+ else:
+ grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)
+ dims = (-2, -1) if self.scale_layout == "block" else -1
+ maximum = grouped.detach().abs().amax(dim=dims)
+ return maximum if current is None else torch.maximum(maximum.to(current), current)
+
+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ if self.requires_calibration and observed_max is None:
+ raise ValueError(f"{self.data_type} activation requires observed_max")
+ return self.primitive(
+ activation,
+ bits=self.bits,
+ group_size=self.group_size,
+ tensor_max=observed_max if self.requires_calibration else None,
+ max_scale=max_scale,
+ )
+
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ quantized, _, _ = self.qdq_with_scale(
+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
+ )
+ return quantized
+
+
+def _create_fp8_activation(spec):
+ """Build an activation quantizer using the same format choice as weights."""
+ if spec.data_type == "fp8_gaudi3_sym":
+ return _FP8ActivationQuantizer(spec, spec.data_type, quant_fp8_sym_gaudi3, "tensor")
+ weight_quantizer = _FP8WeightQuantizer(spec)
+ return _FP8ActivationQuantizer(spec, spec.data_type, weight_quantizer.primitive, weight_quantizer.scale_layout)
+
+
+register_quantizer(
+ "fp8_sym",
+ aliases=("fp", "float", "fp8", "fp8_e4m3", "block_fp8_sym", "block_fp8", "block_fp8_e4m3"),
+)(_FP8WeightQuantizer)
+register_quantizer("fp8_e5m2")(_FP8WeightQuantizer)
+register_quantizer("fp8_unit_scale")(_FP8WeightQuantizer)
+register_quantizer("fp8_e5m2_unit_scale")(_FP8WeightQuantizer)
+if is_gaudi2():
+ register_quantizer("fp8_gaudi3_sym")(_FP8WeightQuantizer)
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 2b4de2b7e..dea59a70e 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -11,13 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from collections.abc import Mapping
+from dataclasses import dataclass
from typing import Any, Callable, Union
import torch
-from auto_round.data_type.register import register_dtype
+from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
from auto_round.export.export_to_gguf.packing import (
_LARGE_TENSOR_BYTES_THRESHOLD,
make_q3_quants,
@@ -25,6 +26,15 @@ from auto_round.export.export_to_gguf.packing import (
make_qx_quants_chunk,
)
from auto_round.logger import logger
+
+
+def get_gguf_quant_constants():
+ """Return GGUF block-layout constants without exposing export internals to algorithms."""
+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
+
+ return GGML_QUANT_SIZES, QK_K
+
+
from auto_round.utils import get_reciprocal
from auto_round.utils.device import clear_memory
@@ -486,9 +496,18 @@ def _imatrix_handle_zero(
@torch.inference_mode()
-def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1, v=0):
- super_bits = 4 if bits == 2 else 6
- super_group_size = 16 if bits == 2 else 8
+def search_gguf_scale_min_asym(
+ tensor,
+ bits=4,
+ scale_dtype=torch.float16,
+ imatrix=None,
+ split_num=1,
+ v=0,
+ super_bits=None,
+ super_group_size=None,
+):
+ super_bits = super_bits if super_bits is not None else 4 if bits == 2 else 6
+ super_group_size = super_group_size if super_group_size is not None else 16 if bits == 2 else 8
quant_weights = None
if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
@@ -604,6 +623,9 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
def quant_tensor_gguf_asym_dq(
tensor: torch.Tensor,
bits: int = 4,
+ group_size=None,
+ super_bits=None,
+ super_group_size=None,
v=0,
scale_dtype=torch.float16,
imatrix=None,
@@ -634,7 +656,7 @@ def quant_tensor_gguf_asym_dq(
raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")
orig_dtype = tensor.dtype
maxq = int(2.0**bits) - 1
- group_size = 16 if bits == 2 else 32
+ group_size = group_size if group_size is not None else 16 if bits == 2 else 32
# The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),
# unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large
# tensors. For very large single tensors (e.g. huge vocab embed/output projections),
@@ -665,6 +687,8 @@ def quant_tensor_gguf_asym_dq(
imatrix,
split_num=split_num,
v=v,
+ super_bits=super_bits,
+ super_group_size=super_group_size,
)
scale = scale.clone()
wmin = wmin.clone()
@@ -903,6 +927,9 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0
def quant_tensor_gguf_sym_dq(
tensor,
bits=3,
+ group_size=16,
+ super_bits=None,
+ super_group_size=16,
v=0,
imatrix=None,
scale=None,
@@ -930,21 +957,15 @@ def quant_tensor_gguf_sym_dq(
Quantized and de-quantized tensor, scale, zero-point
"""
- from auto_round.export.export_to_gguf.config import K_SCALE_SIZE
-
if bits not in [3, 6]:
raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")
maxq = int(2.0 ** (bits - 1))
- group_size = 16
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
orig_dtype = tensor.dtype
- super_bits = 6 if bits == 3 else 8
- super_group_size = 16
- ggml_type = f"q{bits}_k"
- block_size, type_size = GGML_QUANT_SIZES[ggml_type]
+ super_bits = super_bits if super_bits is not None else 6 if bits == 3 else 8
tensor = tensor.to(torch.float32)
- n_blocks = tensor.nelement() // block_size
+ n_blocks = tensor.nelement() // (super_group_size * group_size)
# Chunk the candidate-scale search for very large single tensors (e.g. huge vocab
# embed/output projections). This search allocates several full-size float32 buffers
# at once; unchunked, that can need tens of GB for a single tensor and OOM even though
@@ -955,8 +976,7 @@ def quant_tensor_gguf_sym_dq(
# ordinary attention/FFN weights never needed it and would just get slower for no benefit.
if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:
split_num = 16
- # (nb, 16, 16)
- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+ tensor = tensor.reshape(n_blocks, super_group_size, group_size)
if iter is None:
iter = 0
use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1
@@ -980,3 +1000,161 @@ def quant_tensor_gguf_sym_dq(
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
return tensor, {"scale": scale, "d_scale": d_scale}, maxq
+
+
+@dataclass
+class _GGUFState:
+ """Persistent GGUF RTN scales and double-quantization payload between QDQ calls."""
+
+ tunables: Mapping[str, torch.Tensor]
+ tensor_min: torch.Tensor | None
+ tensor_max: torch.Tensor | None
+ imatrix: torch.Tensor | float | int | None
+ scale: torch.Tensor | None = None
+ wmin: torch.Tensor | None = None
+ d_scale: torch.Tensor | None = None
+ d_wmin: torch.Tensor | None = None
+ prev_scale: torch.Tensor | None = None
+ prev_wmin: torch.Tensor | None = None
+ prev_d_scale: torch.Tensor | None = None
+ prev_d_wmin: torch.Tensor | None = None
+ iteration: int = 0
+
+
+class _GGUFWeightQuantizer:
+ """Own GGUF double-quant weight QDQ and its export payload."""
+
+ def __init__(self, spec, family="plain", kind=None):
+ self.spec = spec
+ self.family = family
+ self.kind = kind or (
+ "sym" if spec.data_type == "int_sym_dq" else "float_zp" if spec.data_type == "int_asym_float_zp" else "asym"
+ )
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the GGUF quantizer; its state selects tuned or RTN behavior."""
+ kind = {"int_sym_dq": "sym", "int_asym_float_zp": "float_zp"}.get(canonical, "asym")
+ return cls(spec, kind=kind)
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ self.family = "rtn" if mode in ("rtn", "optimized_rtn") else "plain"
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)
+ tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)
+ tunables = {}
+ if self.family == "plain" and tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if self.family == "plain" and tune_minmax:
+ shape = tensor_min.shape
+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+ return _GGUFState(tunables, tensor_min, tensor_max, imatrix)
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ value = tunables.get("value", 0)
+ min_scale = tunables.get("min_scale", 1.0)
+ max_scale = tunables.get("max_scale", 1.0)
+ if isinstance(min_scale, torch.Tensor):
+ min_scale.data.clamp_(0.0, 1.0)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale.data.clamp_(0.0, 1.0)
+
+ if self.kind == "float_zp" and self.family == "plain":
+ quantized, scale, zero_point = quant_tensor_asym_float_zp(
+ weight,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ v=value,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.spec.scale_dtype,
+ tensor_min=state.tensor_min,
+ tensor_max=state.tensor_max,
+ q_scale_thresh=self.spec.q_scale_thresh,
+ )
+ elif self.kind == "float_zp":
+ quantized, scale, zero_point = quant_tensor_asym_float_zp_rtn(
+ weight.clone(),
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ scale_dtype=self.spec.scale_dtype,
+ tensor_min=state.tensor_min,
+ tensor_max=state.tensor_max.clone(),
+ q_scale_thresh=self.spec.q_scale_thresh,
+ )
+ elif self.family == "rtn":
+ primitive = quant_tensor_gguf_sym_dq if self.kind == "sym" else quant_tensor_gguf_asym_dq
+ quantized, scale, zero_point = primitive(
+ weight.clone(),
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ super_bits=self.spec.super_bits,
+ super_group_size=self.spec.super_group_size,
+ v=value,
+ scale_dtype=self.spec.scale_dtype,
+ imatrix=state.imatrix,
+ prev_scale=state.prev_scale,
+ prev_wmin=state.prev_wmin,
+ prev_d_scale=state.prev_d_scale,
+ prev_d_wmin=state.prev_d_wmin,
+ iter=state.iteration,
+ )
+ state.scale = scale["scale"]
+ state.d_scale = scale["d_scale"]
+ state.prev_scale = state.scale
+ state.prev_d_scale = state.d_scale
+ if isinstance(zero_point, dict):
+ state.wmin = zero_point["wmin"]
+ state.d_wmin = zero_point["d_wmin"]
+ state.prev_wmin = state.wmin
+ state.prev_d_wmin = state.d_wmin
+ state.iteration += 1
+ else:
+ primitive = quant_tensor_sym_dq if self.kind == "sym" else quant_tensor_asym_dq
+ quantized, scale, zero_point = primitive(
+ weight,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ v=value,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.spec.scale_dtype,
+ tensor_min=state.tensor_min,
+ tensor_max=state.tensor_max,
+ q_scale_thresh=self.spec.q_scale_thresh,
+ super_bits=self.spec.super_bits,
+ super_group_size=self.spec.super_group_size,
+ )
+
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("GGUF weight result was not materialized")
+ module.weight.data.copy_(result.weight)
+ rows = result.logical_rows or result.weight.shape[0]
+
+ def set_values(values, primary):
+ for key, value in values.items():
+ name = primary if key == primary else "w_" + key
+ setattr(module, name, (value.reshape(rows, -1) if key == primary else value).cpu())
+
+ if isinstance(result.scale, dict):
+ set_values(result.scale, "scale")
+ else:
+ module.scale = result.scale.reshape(rows, -1).cpu()
+ if isinstance(result.zero_point, dict):
+ set_values(result.zero_point, "zp")
+ elif isinstance(result.zero_point, torch.Tensor):
+ module.zp = result.zero_point.reshape(rows, -1).cpu()
+ else:
+ module.zp = result.zero_point
+
+
+register_quantizer("int_sym_dq", aliases=("rtn_int_sym_dq",))(_GGUFWeightQuantizer)
+register_quantizer("int_asym_dq", aliases=("rtn_int_asym_dq",))(_GGUFWeightQuantizer)
+register_quantizer("int_asym_float_zp")(_GGUFWeightQuantizer)
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index d301f0b2e..ab2b9b4c5 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -11,12 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from collections.abc import Mapping
+from dataclasses import dataclass
from typing import Union
import torch
from auto_round import envs
-from auto_round.data_type.register import register_dtype
+from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.utils import get_reciprocal
@@ -131,7 +133,7 @@ def quant_tensor_rtn_sym(
min_scale=1.0,
max_scale=1.0,
scale_dtype=torch.float16,
- **kwargs
+ **kwargs,
):
"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community
@@ -175,7 +177,7 @@ def quant_tensor_sym(
tensor_max=None,
q_scale_thresh=1e-5,
init_scale=None,
- **kwargs
+ **kwargs,
):
"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community
@@ -250,7 +252,7 @@ def quant_tensor_asym(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs
+ **kwargs,
):
"""Quantize and de-quantize tensor asymmetrically.
@@ -310,7 +312,7 @@ def quant_tensor_sym_gptq(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs
+ **kwargs,
):
"""Quantize and de-quantize tensor asymmetrically.
@@ -376,7 +378,7 @@ def quant_tensor_asym_wo_round(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
- **kwargs
+ **kwargs,
):
"""Quantize and de-quantize tensor asymmetrically without rounding, this is mainly for tuning bias, norm.
@@ -423,3 +425,193 @@ def quant_tensor_asym_wo_round(
qdq_result = (scale * (q - zp)).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
return qdq_result, scale, zp
+
+
+@dataclass(frozen=True)
+class _IntState:
+ """Per-layer integer ranges, optional optimized scale, and trainable values."""
+
+ tunables: Mapping[str, torch.Tensor]
+ tensor_min: torch.Tensor
+ tensor_max: torch.Tensor
+ optimized_init: torch.Tensor | None
+
+
+class _IntWeightQuantizer:
+ """Own integer weight QDQ and choose tuned, RTN, or optimized RTN internally."""
+
+ def __init__(self, spec, family="plain"):
+ self.spec = spec
+ self.family = family
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the integer weight quantizer for a resolved layer."""
+ return cls(spec)
+
+ @staticmethod
+ def create_activation(spec):
+ """Create the matching integer activation quantizer."""
+ return _IntActivationQuantizer(spec)
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ if mode == "optimized_rtn" and self.spec.sym:
+ self.family = "optimized"
+ elif mode == "rtn" and self.spec.sym:
+ self.family = "rtn"
+ else:
+ self.family = "plain"
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tensor_min = torch.clamp(grouped.amin(dim=-1), max=0)
+ tensor_max = torch.clamp(grouped.amax(dim=-1), min=0)
+ if self.spec.clip_max is not None:
+ clip_max = self.spec.clip_max.reshape(-1).to(weight.device, tensor_max.dtype)
+ clip_min = (
+ -clip_max
+ if self.spec.clip_min is None
+ else self.spec.clip_min.reshape(-1).to(weight.device, tensor_min.dtype)
+ )
+ if clip_min.numel() == tensor_min.numel() and clip_max.numel() == tensor_max.numel():
+ tensor_min = torch.maximum(tensor_min, clip_min.reshape_as(tensor_min))
+ tensor_max = torch.minimum(tensor_max, clip_max.reshape_as(tensor_max))
+
+ tunables = {}
+ if self.family == "plain" and tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if self.family != "optimized" and tune_minmax:
+ shape = tensor_min.shape
+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+
+ optimized_init = None
+ if self.family == "optimized" and self.spec.sym:
+ search_weight = weight
+ if self.spec.clip_min is not None or self.spec.clip_max is not None:
+ search_weight = torch.clamp(weight, min=self.spec.clip_min, max=self.spec.clip_max)
+ _, optimized_init, _ = quant_tensor_opt_rtn_sym(
+ search_weight.clone(),
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ q_scale_thresh=self.spec.q_scale_thresh,
+ imatrix=imatrix,
+ )
+ return _IntState(tunables, tensor_min, tensor_max, optimized_init)
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ value = tunables.get("value", 0)
+ min_scale = tunables.get("min_scale", 1.0)
+ max_scale = tunables.get("max_scale", 1.0)
+ if isinstance(min_scale, torch.Tensor):
+ min_scale.data.clamp_(0.0, 1.0)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale.data.clamp_(0.0, 1.0)
+
+ kwargs = {
+ "bits": self.spec.bits,
+ "group_size": self.spec.group_size,
+ "q_scale_thresh": self.spec.q_scale_thresh,
+ }
+ if self.family == "optimized" and self.spec.sym:
+ quantized, scale, zero_point = quant_tensor_sym(
+ weight,
+ init_scale=state.optimized_init,
+ scale_dtype=self.spec.scale_dtype,
+ **kwargs,
+ )
+ elif self.family == "rtn" and self.spec.sym:
+ quantized, scale, zero_point = quant_tensor_rtn_sym(
+ weight,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.spec.scale_dtype,
+ **kwargs,
+ )
+ else:
+ primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym
+ quantized, scale, zero_point = primitive(
+ weight,
+ v=value,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.spec.scale_dtype,
+ tensor_min=state.tensor_min,
+ tensor_max=state.tensor_max,
+ **kwargs,
+ )
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("INT weight result was not materialized")
+ module.weight.data.copy_(result.weight)
+ rows = result.logical_rows or result.weight.shape[0]
+ module.scale = result.scale.reshape(rows, -1).cpu()
+ module.zp = (
+ result.zero_point.reshape(rows, -1).cpu()
+ if isinstance(result.zero_point, torch.Tensor)
+ else result.zero_point
+ )
+
+
+class _IntActivationQuantizer:
+ """Quantize integer activations, optionally using a calibrated maximum."""
+
+ def __init__(self, spec):
+ self.spec = spec
+ self.requires_calibration = not spec.dynamic
+
+ def observe(self, activation, current):
+ grouped, _, _ = reshape_pad_tensor_by_group_size(activation, self.spec.group_size)
+ maximum = grouped.abs().amax(dim=-1)
+ return maximum if current is None else torch.maximum(maximum.to(current), current)
+
+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ if self.requires_calibration and observed_max is None:
+ raise ValueError(f"{self.spec.data_type} activation requires observed_max")
+ primitive = quant_tensor_sym if self.spec.sym else quant_tensor_asym
+ kwargs = {}
+ if observed_max is not None:
+ kwargs.update(tensor_min=-observed_max, tensor_max=observed_max)
+ return primitive(
+ activation,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ scale_dtype=self.spec.scale_dtype,
+ q_scale_thresh=self.spec.q_scale_thresh,
+ **kwargs,
+ )
+
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ quantized, _, _ = self.qdq_with_scale(
+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
+ )
+ return quantized
+
+
+register_quantizer(
+ "int_sym",
+ aliases=(
+ "int",
+ "int4",
+ "int4_sym",
+ "int8",
+ "int8_sym",
+ "rtn_int",
+ "rtn_int4",
+ "rtn_int_sym",
+ "rtn_int4_sym",
+ "opt_rtn_int",
+ "opt_rtn_int4",
+ "opt_rtn_int_sym",
+ "opt_rtn_int4_sym",
+ ),
+)(_IntWeightQuantizer)
+register_quantizer(
+ "int_asym",
+ aliases=("int4_asym", "rtn_int_asym", "rtn_int4_asym", "opt_rtn_int_asym", "opt_rtn_int4_asym"),
+)(_IntWeightQuantizer)
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 7f453066a..29982f2d4 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE, register_dtype
+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE, register_dtype, register_quantizer
from auto_round.data_type.utils import (
ceil_ste,
floor_ste,
@@ -414,6 +414,168 @@ QUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceil
QUANT_FUNC_WITH_DTYPE["mx_fp4_rceil_v2"] = quant_mx_rceil_v2
QUANT_FUNC_WITH_DTYPE["opt_rtn_mx_fp"] = quant_mx_opt_rtn
+
+class _MXQuantizer:
+ """Own MX weight QDQ and derive the concrete MX format from the request."""
+
+ def __init__(self, spec, data_type=None, family="plain"):
+ self.spec = spec
+ self.data_type = data_type or self._data_type(spec)
+ self.family = family
+
+ @staticmethod
+ def _data_type(spec):
+ data_type = spec.data_type.lower()
+ if data_type in ("mx_fp", "mx_int"):
+ data_type = f"{data_type}{spec.bits}"
+ if data_type not in MXFP_FORMAT_CACHE and not data_type.endswith("_rceil"):
+ raise ValueError(f"Unsupported MX datatype {spec.data_type!r}")
+ return data_type
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the MX quantizer selected by the requested MX format."""
+ data_type = canonical or spec.data_type
+ if data_type in ("mx_fp", "mx_int"):
+ data_type = f"{data_type}{spec.bits}"
+ return cls(spec, data_type)
+
+ @classmethod
+ def create_activation(cls, spec):
+ """Create the dynamic MX activation quantizer for the same format."""
+ return _MXActivationQuantizer(spec, cls._data_type(spec))
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ self.family = "optimized" if mode == "optimized_rtn" else "plain"
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tunables = {}
+ if self.family == "plain" and tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if self.family == "plain" and tune_minmax:
+ tunables["max_scale"] = torch.nn.Parameter(
+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
+ )
+ return {"tunables": tunables, "imatrix": imatrix}
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ kwargs = {
+ "bits": self.spec.bits,
+ "group_size": self.spec.group_size,
+ "data_type": self.data_type,
+ }
+ if self.family == "optimized":
+ quantized, exponent, zero_point = quant_mx_opt_rtn(weight, imatrix=state["imatrix"], **kwargs)
+ elif self.data_type.endswith("_rceil"):
+ quantized, exponent, zero_point = quant_mx_rceil(weight, **kwargs)
+ else:
+ quantized, exponent, zero_point = quant_mx(
+ weight,
+ v=tunables.get("value", 0),
+ max_scale=tunables.get("max_scale", 1.0),
+ **kwargs,
+ )
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(
+ quantized, exponent if materialize else None, zero_point if materialize else None
+ )
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("MX weight result was not materialized")
+ module.weight.data.copy_(result.weight)
+ rows = result.logical_rows or result.weight.shape[0]
+ module.scale = result.scale.reshape(rows, -1).cpu()
+ module.zp = None
+
+
+class _MXActivationQuantizer:
+ """Apply dynamic MX activation QDQ; MX formats require no calibration pass."""
+
+ requires_calibration = False
+
+ def __init__(self, spec, data_type):
+ if not spec.dynamic:
+ raise ValueError(f"MX datatype {data_type!r} supports only dynamic activation quantization")
+ self.spec = spec
+ self.data_type = data_type
+
+ def observe(self, activation, current):
+ raise RuntimeError("Dynamic MX activation quantization does not require calibration")
+
+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ primitive = quant_mx_rceil if self.data_type.endswith("_rceil") else quant_mx
+ return primitive(
+ activation,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ data_type=self.data_type.removesuffix("_rceil"),
+ max_scale=max_scale,
+ )
+
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ quantized, _, _ = self.qdq_with_scale(
+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
+ )
+ return quantized
+
+
+_MX_ALIASES = {
+ "mx_int8": ("mxint8",),
+ "mx_int4": (
+ "mxint4",
+ "rtn_mx_int4",
+ "rtn_mx_int4_sym",
+ "opt_rtn_mx_int4",
+ "opt_rtn_mx_int4_sym",
+ ),
+ "mx_int2": ("mxint2",),
+ "mx_fp8e5m2": ("mxfp8e5m2",),
+ "mx_fp8": ("mxfp8",),
+ "mx_fp8e4m3": ("mxfp8e4m3",),
+ "mx_fp6e3m2": ("mxfp6e3m2",),
+ "mx_fp6": ("mxfp6",),
+ "mx_fp6e2m3": ("mxfp6e2m3",),
+ "mx_fp4": (
+ "mxfp4",
+ "mx_fp4_sym",
+ "rtn_mx_fp4",
+ "rtn_mx_fp4_sym",
+ "opt_rtn_mx_fp4",
+ "opt_rtn_mx_fp4_sym",
+ ),
+ "mx_fp4e2m1": ("mxfp4e2m1",),
+ "mx_float16": ("mxfloat16",),
+ "mx_fp16": ("mxfp16",),
+ "mx_bfloat16": ("mxbfloat16",),
+ "mx_bf16": ("mxbf16",),
+}
+
+
+for _data_type in MXFP_FORMAT_CACHE:
+ register_quantizer(
+ _data_type,
+ aliases=_MX_ALIASES[_data_type],
+ )(_MXQuantizer)
+
+for _family in ("fp", "int"):
+ _canonical = f"mx_{_family}"
+
+ register_quantizer(
+ _canonical,
+ aliases=(
+ f"mx_{_family}_sym",
+ f"rtn_mx_{_family}",
+ f"rtn_mx_{_family}_sym",
+ f"opt_rtn_mx_{_family}",
+ f"opt_rtn_mx_{_family}_sym",
+ ),
+ )(_MXQuantizer)
+
+register_quantizer("mx_fp_rceil", aliases=("mxfprceil",))(_MXQuantizer)
+register_quantizer("mx_fp4e2m1_rceil")(_MXQuantizer)
+
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])
data1 = quant_element(data, 2, 3, 6.0)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index fbe025843..cb4e2efca 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -12,11 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from collections.abc import Mapping
+from dataclasses import dataclass
+
import torch
+from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.fp8 import float8_e4m3fn_ste
from auto_round.data_type.gguf import _imatrix_handle_zero
-from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.logger import logger
@@ -422,6 +425,222 @@ def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=Non
return qdq_res.to(orig_dtype), scale, None
+@dataclass(frozen=True)
+class _NVFPState:
+ """NVFP tuning values plus optional global and optimized scales."""
+
+ tunables: Mapping[str, torch.Tensor]
+ global_scale: torch.Tensor | None
+ optimized_init: torch.Tensor | None
+
+
+class _NVFPWeightQuantizer:
+ """Own NVFP4 weight QDQ, including its global-scale preparation."""
+
+ def __init__(self, spec, family="plain"):
+ self.spec = spec
+ self.family = family
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the NVFP4 weight quantizer for a resolved layer."""
+ return cls(spec)
+
+ @staticmethod
+ def create_activation(spec):
+ """Create dynamic NVFP4 activation quantization."""
+ return _NVFPActivationQuantizer(spec, "nv_fp4")
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ self.family = "optimized" if mode == "optimized_rtn" else "plain"
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tunables = {}
+ if self.family == "plain" and tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if self.family != "optimized" and tune_minmax:
+ tunables["max_scale"] = torch.nn.Parameter(
+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
+ )
+
+ global_scale = self.spec.global_scale
+ if global_scale is None:
+ global_scale = calculate_gparam(weight, self.spec.group_size, weight.device)
+ else:
+ global_scale = global_scale.to(weight.device)
+ optimized_init = None
+ if self.family == "optimized":
+ if isinstance(imatrix, torch.Tensor):
+ imatrix = imatrix.reshape(1, -1)
+ imatrix = reshape_pad_tensor_by_group_size(imatrix, self.spec.group_size, val=1e-5)[0].view(1, -1)
+ imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1).reshape(grouped.shape)
+ qw = _imatrix_handle_zero(imatrix, grouped, self.spec.bits, self.spec.group_size)
+ else:
+ qw = 1.0
+ optimized_init = search_nvfp4_scale(grouped, self.spec.bits, qw)
+ return _NVFPState(tunables, global_scale, optimized_init)
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ kwargs = {
+ "bits": self.spec.bits,
+ "group_size": self.spec.group_size,
+ "v": tunables.get("value", 0),
+ "max_scale": tunables.get("max_scale", 1.0),
+ }
+ kwargs["global_scale"] = state.global_scale
+ if state.optimized_init is not None:
+ kwargs["init_scale"] = state.optimized_init
+ quantized, scale, zero_point = nv_fp4(weight, **kwargs)
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(
+ quantized,
+ scale if materialize else None,
+ zero_point if materialize else None,
+ state.global_scale if materialize else None,
+ )
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("NVFP weight result was not materialized")
+ module.weight.data.copy_(result.weight)
+ rows = result.logical_rows or result.weight.shape[0]
+ module.scale = result.scale.reshape(rows, -1).cpu()
+ module.zp = None
+ if result.metadata is not None:
+ module.weight_global_scale = result.metadata.cpu()
+
+
+class _NVFPV2WeightQuantizer:
+ """Own the independent NVFP4-v2 weight QDQ implementation."""
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the NVFP4-v2 weight quantizer for a resolved layer."""
+ return cls(spec)
+
+ @staticmethod
+ def create_activation(spec):
+ """Create dynamic NVFP4-v2 activation quantization."""
+ return _NVFPActivationQuantizer(spec, "nvfp4_v2")
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tunables = {}
+ if tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if tune_minmax:
+ tunables["max_scale"] = torch.nn.Parameter(
+ torch.ones(grouped.shape[:-1], device=weight.device, dtype=torch.float32)
+ )
+ return _NVFPState(tunables, None, None)
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ quantized, scale, zero_point = nvfp4_v2(
+ weight,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ v=tunables.get("value", 0),
+ max_scale=tunables.get("max_scale", 1.0),
+ )
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(quantized, scale if materialize else None, zero_point if materialize else None)
+
+ apply_result = staticmethod(_NVFPWeightQuantizer.apply_result)
+
+
+class _NVFPActivationQuantizer:
+ """Quantize dynamic or calibrated NVFP activations for one NVFP format."""
+
+ def __init__(self, spec, data_type):
+ is_static = data_type in ("nv_fp4_with_static_gs", "nvfp4_v2_with_global_scale")
+ if not is_static and not spec.dynamic:
+ raise ValueError(f"NVFP datatype {data_type!r} supports only dynamic activation quantization")
+ self.spec = spec
+ self.data_type = data_type
+ self.requires_calibration = is_static
+
+ def observe(self, activation, current):
+ if not self.requires_calibration:
+ raise RuntimeError("Dynamic NVFP activation quantization does not require calibration")
+ observed = activation.detach().float().abs().max().unsqueeze(0)
+ return observed if current is None else torch.maximum(current.to(observed.device), observed)
+
+ def qdq_with_scale(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ if self.requires_calibration and observed_max is None:
+ raise ValueError(f"{self.data_type} activation requires observed_max")
+ primitive = {
+ "nv_fp4": nv_fp4,
+ "nv_fp4_with_static_gs": nv_fp4_with_static_gs,
+ "nvfp4_v2": nvfp4_v2,
+ "nvfp4_v2_with_global_scale": nvfp4_v2_with_global_scale,
+ }[self.data_type]
+ kwargs = {"bits": self.spec.bits, "group_size": self.spec.group_size, "max_scale": max_scale}
+ if self.requires_calibration:
+ kwargs["tensor_max"] = observed_max
+ return primitive(activation, **kwargs)
+
+ def qdq(self, activation, *, observed_max=None, min_scale=1.0, max_scale=1.0):
+ quantized, _, _ = self.qdq_with_scale(
+ activation, observed_max=observed_max, min_scale=min_scale, max_scale=max_scale
+ )
+ return quantized
+
+
+def _prepare_nvfp_block(block, layer_runtimes):
+ """Materialize shared NVFP scales once after a block's observations."""
+ from auto_round.data_type.utils import update_block_global_scale_if_needed
+
+ runtime = next(iter(layer_runtimes.values()), None)
+ if runtime is None:
+ return
+ update_block_global_scale_if_needed(block, runtime["data_type"], runtime["group_size"])
+
+
+_NVFPWeightQuantizer.prepare_block = staticmethod(_prepare_nvfp_block)
+
+
+class _NVFPStaticActivation:
+ """Datatype entry that provides static NVFP activation quantization only."""
+
+ @staticmethod
+ def create_activation(spec):
+ return _NVFPActivationQuantizer(spec, "nv_fp4_with_static_gs")
+
+
+class _NVFPV2StaticActivation:
+ """Datatype entry that provides global-scale NVFP-v2 activation only."""
+
+ @staticmethod
+ def create_activation(spec):
+ return _NVFPActivationQuantizer(spec, "nvfp4_v2_with_global_scale")
+
+
+register_quantizer(
+ "nv_fp4",
+ aliases=(
+ "nv_fp",
+ "nv_fp_sym",
+ "nv_fp4_sym",
+ "rtn_nv_fp",
+ "rtn_nv_fp4",
+ "rtn_nv_fp_sym",
+ "rtn_nv_fp4_sym",
+ "opt_rtn_nv_fp",
+ "opt_rtn_nv_fp4",
+ "opt_rtn_nv_fp_sym",
+ "opt_rtn_nv_fp4_sym",
+ ),
+)(_NVFPWeightQuantizer)
+register_quantizer("nvfp4_v2")(_NVFPV2WeightQuantizer)
+register_quantizer("nv_fp4_with_static_gs", aliases=("rtn_nv_fp4_with_static_gs",))(_NVFPStaticActivation)
+register_quantizer("nvfp4_v2_with_global_scale")(_NVFPV2StaticActivation)
+
+
if __name__ == "__main__":
data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1, 6.0, 6.2, 8.9])
data1 = cast_to_fp4(data)
diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.py
deleted file mode 100644
index fca259ed6..000000000
--- a/auto_round/data_type/register.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-QUANT_FUNC_WITH_DTYPE = {}
-
-
-def register_dtype(names):
- """Class decorator to register a EXPORT subclass to the registry.
-
- Decorator function used before a Pattern subclass.
-
- Args:
- names: A string. Define the export type.
-
- Returns:
- cls: The class of register.
- """
-
- def register(dtype):
- if isinstance(names, (tuple, list)):
- for name in names:
- QUANT_FUNC_WITH_DTYPE[name] = dtype
- else:
- QUANT_FUNC_WITH_DTYPE[names] = dtype
-
- return dtype
-
- return register
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index c412b2445..50b383ca2 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -22,10 +22,17 @@ from torch.nn import Linear, Module
from auto_round import envs
from auto_round.compressors.utils import is_nv_fp
-from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.base import QUANT_FUNC_WITH_DTYPE
from auto_round.utils import check_to_quantized, logger
+def quantize_bias_without_round(*args, **kwargs):
+ """Quantize bias/norm weights without exposing the INT primitive to algorithms."""
+ from auto_round.data_type.int import quant_tensor_asym_wo_round
+
+ return quant_tensor_asym_wo_round(*args, **kwargs)
+
+
def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):
"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.
@@ -176,141 +183,6 @@ def get_quant_func(
)
-def _resolve_optimized_dtype_funcs(data_type: str, q_scale_thresh: float = 1e-5):
- """Resolve the SignRound optimized ``(scale_search_fn, quant_func)`` for a data type.
-
- Single source of truth for the optimized-path dispatch shared by
- ``SignRoundOptimizedWrapperLinear`` and AWQ's internal QDQ:
-
- * ``scale_search_fn(weight_reshape, bits, imatrix) -> init_scale`` searches the
- data-type-specific per-group init scale (the int variant clamps to
- ``q_scale_thresh`` to avoid a degenerate zero scale).
- * ``quant_func`` is the matching *plain* quant function (``init_scale`` already
- encodes the searched scale, so no opt-rtn / rtn variant is needed).
-
- Returns ``(None, None)`` for data types without an optimized path
- (asym int, ``*_dq``, or unrelated types).
- """
- dt = str(data_type)
- if dt.endswith("dq"):
- return None, None
- if dt.startswith("int"):
- # The optimized int init-scale search is symmetric-only; asym int uses
- # the standard tensor_min/tensor_max range instead.
- if "asym" in dt:
- return None, None
- from auto_round.data_type.int import quant_tensor_sym, search_scales
-
- def search_int(weight_reshape, bits, imatrix):
- init_scale = search_scales(weight_reshape, bits, imatrix)
- return torch.where(
- init_scale < 0,
- torch.clamp(init_scale, max=-q_scale_thresh),
- torch.clamp(init_scale, min=q_scale_thresh),
- )
-
- return search_int, quant_tensor_sym
- if dt.startswith("mx"):
- from auto_round.data_type.mxfp import quant_mx, search_mx_scale
-
- return search_mx_scale, quant_mx
- if dt.startswith("nv"):
- from auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale
-
- return search_nvfp4_scale, nv_fp4
- return None, None
-
-
-def search_optimized_init_scale(
- weight_reshape: torch.Tensor,
- data_type: str,
- bits: int,
- imatrix=None,
- q_scale_thresh: float = 1e-5,
-):
- """Compute the SignRoundV2 optimized per-group ``init_scale`` for a grouped weight.
-
- Mirrors ``SignRoundOptimizedWrapperLinear``: dispatches on ``data_type`` so that
- any caller (the optimized wrapper itself or AWQ's internal QDQ used for the
- smooth/clip grid search) seeds the quantizer with the same initial scale.
- Returns ``None`` for data types that do not use the optimized init-scale search
- (asym int, ``*_dq``, or unrelated types).
-
- Args:
- weight_reshape: Weight reshaped/padded to ``[..., group_size]``.
- data_type: Resolved weight data type (e.g. ``"int_sym"``, ``"mx_fp4"``, ``"nv_fp4"``).
- bits: Weight bit-width.
- imatrix: Per-element importance matrix matching ``weight_reshape`` (or ``None``/scalar).
- q_scale_thresh: Minimum scale magnitude used to clamp the int init_scale.
-
- Returns:
- The per-group ``init_scale`` tensor, or ``None`` if unsupported.
- """
- search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)
- if search_fn is None:
- return None
- if imatrix is None or not isinstance(imatrix, torch.Tensor):
- imatrix = torch.ones_like(weight_reshape)
- return search_fn(weight_reshape, bits, imatrix)
-
-
-def get_optimized_quant_func(data_type: str):
- """Return the plain quant function used by the SignRound optimized path.
-
- The optimized init-scale search always pairs with the *plain* (non opt-rtn /
- non rtn) quant function for the data type, since ``init_scale`` already
- encodes the searched scale. Returns ``None`` for data types without an
- optimized path (asym int, ``*_dq``, or unrelated types).
- """
- _, quant_func = _resolve_optimized_dtype_funcs(data_type)
- return quant_func
-
-
-def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size):
- """Reshape/pad an importance matrix to match a group-reshaped weight.
-
- Encapsulates the imatrix grouping logic shared by the SignRound optimized
- wrapper and AWQ's internal QDQ so callers never handle the low-level reshape.
- Returns a tensor of ones when no imatrix is available (uniform importance),
- keeping every downstream optimized dtype implementation on its tensor path.
- """
- if imatrix is None or not isinstance(imatrix, torch.Tensor):
- return torch.ones_like(weight_reshape)
- imatrix = imatrix.reshape(1, -1)
- imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)
- imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)
- return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)
-
-
-def compute_optimized_init_scale(
- weight: torch.Tensor,
- data_type: str,
- bits: int,
- group_size,
- imatrix=None,
- q_scale_thresh: float = 1e-5,
-):
- """Compute the SignRound optimized per-group ``init_scale`` for a full weight.
-
- Group-reshapes ``weight``, prepares the ``imatrix`` layout, and runs the
- data-type-specific scale search. Unlike :func:`search_optimized_init_scale`
- (which expects an already group-reshaped weight), this is the entry point for
- callers holding a full 2-D weight, e.g. AWQ's internal QDQ. Pair it with
- :func:`get_optimized_quant_func` (resolved once) to obtain the matching quant
- function, so the smooth/clip grid-search loss mirrors what
- ``SignRoundOptimizedWrapperLinear`` applies.
-
- Returns ``None`` for data types without an optimized init-scale path
- (asym int, ``*_dq``, or unrelated types).
- """
- search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)
- if search_fn is None:
- return None
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
- imatrix = reshape_imatrix_for_weight(imatrix, weight_reshape, group_size)
- return search_fn(weight_reshape, bits, imatrix)
-
-
def round_ste(x: torch.Tensor):
"""Straight-Through Estimator for rounding.
diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.py
index b8e987d4c..6d7e35b41 100644
--- a/auto_round/data_type/w4fp8.py
+++ b/auto_round/data_type/w4fp8.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.data_type.register import register_dtype
+from auto_round.data_type.base import register_dtype, register_quantizer
from auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func
@@ -72,3 +72,58 @@ def progressive_quant_fp8_int4(
bf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale
return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4
+
+
+@register_quantizer("fp8_to_int_sym")
+class _ProgressiveFP8WeightQuantizer:
+ """Own the progressive BF16-to-FP8-to-INT4 weight quantization path."""
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ @classmethod
+ def from_spec(cls, spec, canonical=None):
+ """Create the progressive FP8-to-INT4 weight quantizer."""
+ return cls(spec)
+
+ def create_state(self, weight, *, imatrix=None, mode, tune_rounding, tune_minmax):
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+
+ grouped, _, _ = reshape_pad_tensor_by_group_size(weight, self.spec.group_size)
+ tunables = {}
+ if tune_rounding:
+ tunables["value"] = torch.nn.Parameter(torch.zeros_like(grouped, dtype=torch.float32))
+ if tune_minmax:
+ shape = grouped.shape[:-1]
+ tunables["min_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+ tunables["max_scale"] = torch.nn.Parameter(torch.ones(shape, device=weight.device, dtype=torch.float32))
+ return tunables
+
+ def qdq(self, weight, state, *, tunables, materialize=False):
+ quantized, scales, zero_point = progressive_quant_fp8_int4(
+ weight,
+ bits=self.spec.bits,
+ group_size=self.spec.group_size,
+ v=tunables.get("value", 0),
+ min_scale=tunables.get("min_scale", 1.0),
+ max_scale=tunables.get("max_scale", 1.0),
+ q_scale_thresh=self.spec.q_scale_thresh,
+ )
+ from auto_round.data_type.base import WeightQuantizationResult
+
+ return WeightQuantizationResult(
+ quantized,
+ scales["scale"] if materialize else None,
+ zero_point if materialize else None,
+ scales["bf16_to_fp8_scale"] if materialize else None,
+ )
+
+ @staticmethod
+ def apply_result(module, result):
+ if result.scale is None:
+ raise ValueError("Progressive FP8 weight result was not materialized")
+ module.weight.data.copy_(result.weight)
+ rows = result.logical_rows or result.weight.shape[0]
+ module.scale = result.scale.reshape(rows, -1).cpu()
+ module.zp = result.zero_point
+ module.w_bf16_to_fp8_scale = result.metadata.cpu()
diff --git a/auto_round/modeling/fused_moe/grouped_experts.py b/auto_round/modeling/fused_moe/grouped_experts.py
index c9a755316..48d8511e5 100644
--- a/auto_round/modeling/fused_moe/grouped_experts.py
+++ b/auto_round/modeling/fused_moe/grouped_experts.py
@@ -490,6 +490,11 @@ def _batched_qdq_weights(
eligible and the caller must go per-expert.
"""
ref = layers[0]
+ # Datatype quantizers intentionally keep their per-layer QDQ state private.
+ # Without the legacy compiled function, that state cannot be safely merged,
+ # so retain correctness by using the normal per-expert path.
+ if not hasattr(ref, "weight_quant_func"):
+ return None
orig = ref.orig_layer
out_features, in_features = ref.weight.shape
num_experts = len(layers)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index d7de6366f..12c21c58c 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2180,7 +2180,9 @@ def set_amax_for_uncalibrated_experts(
)
return uncalibrated_experts
# Flatten all tensors to 1D before concatenation
- flat_values = [t.reshape(-1) for t in amax_values]
+ device = amax_values[0].device
+ dtype = amax_values[0].dtype
+ flat_values = [t.reshape(-1).to(device=device, dtype=dtype) for t in amax_values]
all_values = torch.cat(flat_values)
set_amax_value = torch.max(all_values)
set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_value
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index f36b67e00..2c9193902 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -20,8 +20,8 @@ import transformers
from torch.functional import F
import auto_round.envs as envs
-from auto_round.compressors.utils import is_nv_fp
-from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
+from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer
+from auto_round.data_type.utils import quantize_bias_without_round, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -84,6 +84,7 @@ class WrapperLinear(torch.nn.Module):
enable_round_tuning=True,
enable_torch_compile=True,
disable_opt_rtn=True,
+ weight_qdq_builder=None,
**kwargs,
):
"""Initializes the WrapperLinear module.
@@ -105,18 +106,35 @@ class WrapperLinear(torch.nn.Module):
self.enable_torch_compile = enable_torch_compile
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
+ # Observation may have cached the quantizer on the original layer. Do
+ # not consume it until wrapper initialization has completed successfully.
+ cached_activation_quantizer = getattr(self.orig_layer, "_ar_activation_quantizer", None)
+ self.activation_quantizer = cached_activation_quantizer
+ if self.enable_act_quant and self.activation_quantizer is None:
+ self.activation_quantizer = activation_quantizer_for_layer(orig_layer)
self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
- if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:
- from auto_round.data_type.nvfp import calculate_gparam
-
- weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)
- setattr(self, "weight_global_scale", weight_global_scale)
- self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
self.q_scale_thresh = 1e-8
else:
self.q_scale_thresh = 1e-5
self._init_tuning_params_and_quant_func()
+
+ def default_qdq(weight, *, materialize=False):
+ """Keep direct wrapper use working when no algorithm supplies a builder."""
+ return self.weight_quantizer.qdq(weight, materialize=materialize, **self.params)
+
+ self.weight_qdq = default_qdq
+ if weight_qdq_builder is not None:
+ weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
+ if type(orig_layer) == transformers.pytorch_utils.Conv1D:
+ weight = weight.t()
+ self.weight_qdq = weight_qdq_builder(
+ weight=weight.to(self.device), default_qdq=self.weight_quantizer.qdq, parameters=self.params
+ )
+ for name, parameter in self.params.items():
+ setattr(self, name, parameter)
+ if cached_activation_quantizer is not None:
+ self.orig_layer.__dict__.pop("_ar_activation_quantizer", None)
if deepspeed_exists:
if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):
self.orig_forward = self.linear_forward
@@ -147,69 +165,25 @@ class WrapperLinear(torch.nn.Module):
orig_layer = self.orig_layer
orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
+ if type(orig_layer) == transformers.pytorch_utils.Conv1D:
orig_weight = orig_weight.t()
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)
-
- if self.enable_round_tuning:
- self.weight_min = (
- torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)
- if isinstance(orig_layer.group_size, tuple)
- else torch.clamp(weight_reshape.min(1)[0], max=0)
- )
- self.weight_max = (
- torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)
- if isinstance(orig_layer.group_size, tuple)
- else torch.clamp(weight_reshape.max(1)[0], min=0)
- )
- else:
- self.weight_min = None
- self.weight_max = None
- # AWQ clip-as-init: cap the tunable weight range to the per-group clip
- # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
- # This initializes the range used by quant_tensor_sym/asym, leaving
- # min_scale/max_scale to tune a coefficient on top. Only the standard
- # (non-tuple) group layout maps onto weight_min/weight_max here.
- awq_clip_min = getattr(orig_layer, "awq_clip_min", None)
- awq_clip_max = getattr(orig_layer, "awq_clip_max", None)
- if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):
- clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
- if awq_clip_min is None:
- clip_min_flat = -clip_max_flat
- else:
- clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)
- if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():
- self.weight_max = torch.minimum(self.weight_max, clip_max_flat)
- self.weight_min = torch.maximum(self.weight_min, clip_min_flat)
- self._init_params(
- "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16
- )
- # Min-max scale initialization
- shape = get_scale_shape(orig_weight, orig_layer.group_size)
- self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
- self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
-
- self.weight_quant_func, self.data_type = get_quant_func(
- orig_layer.data_type,
- orig_layer.bits,
- orig_layer.sym,
- self.disable_opt_rtn,
- orig_layer.group_size,
- iters=orig_layer.iters,
+ self.weight_quantizer = create_quantizer(
+ orig_layer,
+ disable_opt_rtn=self.disable_opt_rtn,
+ tune_rounding=self.enable_round_tuning,
+ tune_minmax=self.enable_minmax_tuning,
)
- if self.enable_torch_compile:
- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
+ self.weight_quantizer.initialize(orig_weight, imatrix=getattr(orig_layer, "imatrix", None))
+ self.params = dict(self.weight_quantizer.parameters)
+ for name, value in self.params.items():
+ setattr(self, name, value)
+ self.value = getattr(self, "value", torch.tensor(0.0, device=self.device, dtype=p_dtype))
+ self.min_scale = getattr(self, "min_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))
+ self.max_scale = getattr(self, "max_scale", torch.tensor(1.0, device=self.device, dtype=p_dtype))
+ self.data_type = orig_layer.data_type
if self.enable_act_quant:
- self.act_quant_func, self.act_data_type = get_quant_func(
- orig_layer.act_data_type,
- orig_layer.act_bits,
- orig_layer.act_sym,
- disable_opt_rtn=True,
- iters=orig_layer.iters,
- )
- if self.enable_torch_compile:
- self.act_quant_func = compile_func(self.act_quant_func, self.device)
+ self.act_data_type = orig_layer.act_data_type
self._init_params(
"act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic)
)
@@ -218,9 +192,7 @@ class WrapperLinear(torch.nn.Module):
# Bias tuning
if self.enable_norm_bias_tuning:
self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)
- from auto_round.data_type.int import quant_tensor_asym_wo_round
-
- self.bias_quant_func = quant_tensor_asym_wo_round
+ self.bias_quant_func = quantize_bias_without_round
self.params["bias_v"] = self.bias_v
def _init_params(self, name, dtype, shape, value, tunable):
@@ -254,43 +226,17 @@ class WrapperLinear(torch.nn.Module):
"""
if self.orig_layer.bits >= 16:
return self.orig_layer.weight, None, None
- min_bound, max_bound = self.minmax_scale_bound
- min_scale.data.clamp_(min_bound, max_bound)
- max_scale.data.clamp_(min_bound, max_bound)
weight = self.orig_layer.weight
if weight.device.type == "meta":
weight = self.orig_layer.get_weight().to(self.device)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight = weight.t()
- quant_kwargs = {}
- if hasattr(self.orig_layer, "super_bits"):
- quant_kwargs["super_bits"] = self.orig_layer.super_bits
- quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
- if hasattr(self, "_extra_quant_kwargs"):
- quant_kwargs.update(self._extra_quant_kwargs())
-
- weight_q, scale, zp = self.weight_quant_func(
- weight.to(self.device),
- bits=self.orig_layer.bits,
- group_size=self.orig_layer.group_size,
- v=value,
- min_scale=min_scale,
- max_scale=max_scale,
- scale_dtype=self.orig_layer.scale_dtype,
- tensor_min=self.weight_min,
- tensor_max=self.weight_max,
- data_type=self.data_type,
- q_scale_thresh=self.q_scale_thresh,
- imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,
- global_scale=getattr(self, "weight_global_scale", None),
- init_scale=getattr(self, "init_scale", None),
- **quant_kwargs,
- )
- weight_q = weight_q.to(weight.dtype)
+ self.params.update(value=value, min_scale=min_scale, max_scale=max_scale)
+ weight_q = self.weight_qdq(weight.to(self.device)).weight.to(weight.dtype)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight_q = weight_q.t()
- return weight_q, scale, zp
+ return weight_q, None, None
def _qdq_act(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):
"""Quantizes and dequantizes activations.
@@ -306,19 +252,12 @@ class WrapperLinear(torch.nn.Module):
act_max_scale.data.clamp_(0, 1.0)
act_min_scale.data.clamp_(0, 1.0)
env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set
- x, scale, zp = self.act_quant_func(
+ return self.activation_quantizer.qdq_with_scale(
x,
- bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.act_group_size,
- scale_dtype=self.orig_layer.scale_dtype,
- q_scale_thresh=self.q_scale_thresh,
- data_type=self.act_data_type,
- tensor_max=act_max, # for static
+ observed_max=act_max,
max_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
- global_scale=getattr(self, "input_global_scale", None),
)
- return x, scale, zp
def _qdq_bias(self, bias, bias_v):
"""Quantizes and dequantizes bias.
@@ -361,59 +300,28 @@ class WrapperLinear(torch.nn.Module):
v = best_params.get("value", torch.tensor(0.0)).to(self.device)
min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)
max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)
+ # Preserve the dictionary captured by QDQ closures and restore all
+ # algorithm parameters, including names unknown to the wrapper.
+ self.params.update({name: value.to(self.device) for name, value in best_params.items()})
+ self.params.update(value=v, min_scale=min_scale, max_scale=max_scale)
if self.orig_layer.weight.device.type == "meta":
self.orig_layer.to(self.device)
- # Unwrapper weight
- qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
- # if hasattr(self.orig_layer, "imatrix"):
- # self.orig_layer.imatrix = None
- self.orig_layer.weight.data.copy_(qdq_weight)
+ if self.orig_layer.bits < 16:
+ weight = self.orig_layer.weight
+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
+ weight = weight.t()
+ result = self.weight_qdq(
+ weight.to(self.device),
+ materialize=True,
+ )
+ self.weight_quantizer.apply_result(
+ self.orig_layer,
+ result,
+ transpose=type(self.orig_layer) == transformers.pytorch_utils.Conv1D,
+ )
self.orig_layer.weight.grad = None
- shape = qdq_weight.shape
- if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
- shape = qdq_weight.t().shape
-
- def _set_dict_attr(attr_dict, attr_name):
- for key in attr_dict.keys():
- if key == attr_name:
- setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))
- else:
- name = "w_" + key
- setattr(self.orig_layer, name, attr_dict[key].to("cpu"))
-
- if not isinstance(self.orig_layer.group_size, tuple):
- if isinstance(scale, dict):
- _set_dict_attr(scale, "scale")
- elif scale is None:
- self.orig_layer.scale = None
- elif scale.numel() > 1:
- self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
- else:
- self.orig_layer.scale = scale.view(-1).to("cpu")
- else:
- self.orig_layer.scale = scale.to("cpu")
-
- if zp is not None:
- if isinstance(zp, dict):
- _set_dict_attr(zp, "zp")
- elif isinstance(zp, torch.Tensor):
- if zp.numel() > 1:
- zp = zp.reshape(shape[0], -1)
- self.orig_layer.zp = zp.to("cpu")
- else:
- self.orig_layer.zp = zp.view(-1).to("cpu")
- else:
- self.orig_layer.zp = zp
- else:
- self.orig_layer.zp = None
-
- if self.weight_global_scale is not None:
- global_scale = self.weight_global_scale
- assert global_scale.numel() == 1
- self.orig_layer.weight_global_scale = global_scale.to("cpu")
-
# Unwrapper bias
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"].to(self.device)
@@ -457,11 +365,11 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.act_max_scale = self.act_max_scale
self.orig_layer.act_data_type = self.act_data_type
- self.orig_layer.act_quant_func = self.act_quant_func
wrapper_layer = WrapperWALayer(
self.orig_layer,
enable_torch_compile=self.enable_torch_compile,
device=self.device,
+ activation_quantizer=self.activation_quantizer,
)
return _preserve_global_name(wrapper_layer)
@@ -567,16 +475,16 @@ class WrapperLinear(torch.nn.Module):
class WrapperWALayer(torch.nn.Module):
- def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):
+ def __init__(self, orig_layer, enable_torch_compile=True, device="cpu", activation_quantizer=None):
super(WrapperWALayer, self).__init__()
self.orig_layer = orig_layer
self.enable_torch_compile = enable_torch_compile
self.device = device
self.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else None
self.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None
- self.act_quant_func = self.orig_layer.act_quant_func
- if self.enable_torch_compile:
- self.act_quant_func = compile_func(self.act_quant_func, self.device)
+ self.activation_quantizer = activation_quantizer or orig_layer.__dict__.pop("_ar_activation_quantizer", None)
+ if self.activation_quantizer is None:
+ self.activation_quantizer = activation_quantizer_for_layer(orig_layer)
self.extra_repr_org = orig_layer.extra_repr
# Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)
@@ -610,27 +518,12 @@ class WrapperWALayer(torch.nn.Module):
max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
- if act_max is None:
- x, _, _ = self.orig_layer.act_quant_func(
- x,
- bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.act_group_size,
- scale_dtype=self.orig_layer.scale_dtype,
- q_scale_thresh=self.orig_layer.q_scale_thresh,
- data_type=self.orig_layer.act_data_type,
- min_scale=min_scale,
- max_scale=max_scale,
- )
- else:
- x, _, _ = self.orig_layer.act_quant_func(
- x,
- bits=self.orig_layer.act_bits,
- group_size=self.orig_layer.act_group_size,
- scale_dtype=self.orig_layer.scale_dtype,
- q_scale_thresh=self.orig_layer.q_scale_thresh,
- data_type=self.orig_layer.act_data_type,
- act_max=act_max,
- )
+ x = self.activation_quantizer.qdq(
+ x,
+ observed_max=act_max,
+ min_scale=min_scale,
+ max_scale=max_scale,
+ )
# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)
return self.orig_layer.forward(x)
@@ -662,9 +555,7 @@ class WrapperLayerNorm(torch.nn.Module):
requires_grad=True,
)
self.params = {"v": self.v}
- from auto_round.data_type.int import quant_tensor_asym_wo_round
-
- self.quant_func = quant_tensor_asym_wo_round
+ self.quant_func = quantize_bias_without_round
def unwrapper(self, best_params):
if best_params is None:
@@ -713,9 +604,7 @@ class WrapperLlamaNorm(torch.nn.Module):
requires_grad=True,
)
self.params = {"v": self.v}
- from auto_round.data_type.int import quant_tensor_asym_wo_round
-
- self.quant_func = quant_tensor_asym_wo_round
+ self.quant_func = quantize_bias_without_round
def unwrapper(self, best_params):
if best_params is None:
diff --git a/benchmark_both.py b/benchmark_both.py
deleted file mode 100644
index 320f18b11..000000000
--- a/benchmark_both.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""Quick A/B benchmark: old (compressors) vs new (compressors_new) architecture.
-
-Uses AR_DISABLE_NEW_ARCH env-var to toggle. Runs each configuration in a
-subprocess to avoid cross-contamination, with a warmup run to fill OS page cache.
-"""
-
-import json
-import os
-import subprocess
-import sys
-import time
-
-MODEL = "Qwen/Qwen3-0.6B"
-ITERS = "200"
-SCHEME = "W4A16"
-DEVICE = "cuda:0"
-
-CMD_TEMPLATE = [
- sys.executable,
- "-m",
- "auto_round",
- "--model_name",
- MODEL,
- "--scheme",
- SCHEME,
- "--iters",
- ITERS,
- "--device",
- DEVICE,
-]
-
-
-def run_once(label: str, env_override: dict) -> float:
- env = {**os.environ, **env_override}
- print(f"\n{'='*60}")
- print(f" Running: {label}")
- print(f" AR_DISABLE_NEW_ARCH={env.get('AR_DISABLE_NEW_ARCH', 'unset')}")
- print(f"{'='*60}", flush=True)
- t0 = time.perf_counter()
- proc = subprocess.run(CMD_TEMPLATE, env=env, capture_output=True, text=True)
- elapsed = time.perf_counter() - t0
- if proc.returncode != 0:
- print(f"STDERR:\n{proc.stderr[-2000:]}")
- raise RuntimeError(f"{label} failed with rc={proc.returncode}")
- print(f" {label}: {elapsed:.1f}s")
- return elapsed
-
-
-def main():
- # Warmup: fill OS page cache & JIT caches
- print("Warmup run (old arch)...")
- run_once("warmup", {"AR_DISABLE_NEW_ARCH": "1"})
-
- # Interleaved runs to reduce bias
- results = {"old": [], "new": []}
- for trial in range(2):
- if trial % 2 == 0:
- first, second = ("old", "1"), ("new", "0")
- else:
- first, second = ("new", "0"), ("old", "1")
- for label, flag in [first, second]:
- t = run_once(f"{label} (trial {trial+1})", {"AR_DISABLE_NEW_ARCH": flag})
- results[label].append(t)
-
- # Summary
- print("\n" + "=" * 60)
- print("SUMMARY")
- print("=" * 60)
- for arch in ["old", "new"]:
- times = results[arch]
- avg = sum(times) / len(times)
- print(f" {arch}: {[f'{t:.1f}' for t in times]} avg={avg:.1f}s")
- old_avg = sum(results["old"]) / len(results["old"])
- new_avg = sum(results["new"]) / len(results["new"])
- diff_pct = (new_avg - old_avg) / old_avg * 100
- print(f"\n Diff: {diff_pct:+.1f}% (new vs old)")
- print(f" {'PASS' if abs(diff_pct) < 5 else 'FAIL'} (threshold: ±5%)")
-
- json.dump(results, open("benchmark_results/latest.json", "w"), indent=2)
-
-
-if __name__ == "__main__":
- main()
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index f2c62a93e..2ff9edf2b 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -174,7 +174,7 @@ adopted within the community, **only 4-bits quantization is supported**. Please
**MLX Format**[Experimental Feature]: This format targets Apple Silicon (M1/M2/M3/...) and is loaded directly by [`mlx-lm`](https://github.com/ml-explore/mlx-lm) (text-only LLM) or [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm) (vision/audio + language).
- Supports **2, 3, 4, 5, 6, 8 bits** (5/6 bits are MLX-exclusive — GPTQ/AWQ have no standard packing for them).
-- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--schemes "..." --bits 3.5`);
+- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--schemes "..." --bits 3.5`);
- Use `--format mlx` for a native MLX checkpoint; use `--format auto_round:mlx` if you want HuggingFace `transformers` + AutoRound to load it (post-init repacks each layer into MLX `QuantLinear` on Darwin).
- Limitation: embedding layer quantization has not supported
#### Format and scheme support matrix
diff --git a/performance_ut.sh b/performance_ut.sh
deleted file mode 100644
index 12e78906b..000000000
--- a/performance_ut.sh
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-PATTERN='[-a-zA-Z0-9_]*='
-
-for i in "$@"; do
- case $i in
- --model_name=*)
- model_name=$(echo $i | sed "s/${PATTERN}//")
- ;;
- --scheme=*)
- scheme=$(echo $i | sed "s/${PATTERN}//")
- ;;
- *)
- echo "Parameter $i not recognized."
- exit 1
- ;;
- esac
-done
-
-readonly WORKSPACE_DIR="/auto-round"
-readonly LOG_DIR="${WORKSPACE_DIR}/log_dir"
-readonly PERF_SCRIPT_DIR="${WORKSPACE_DIR}/.azure-pipelines/scripts/performance"
-readonly BASELINE_GIT_URL="git+https://github.com/intel/auto-round.git"
-readonly ITERS=200
-
-log_group_start() { echo "##[group]$1"; }
-log_group_end() { echo "##[endgroup]"; }
-log_info() { echo -e "[\033[32mINFO\033[0m] $1"; }
-log_err() { echo -e "[\033[31mERROR\033[0m] $1" >&2; }
-
-function setup_environment() {
- log_group_start "Set up environment..."
-
- export TZ='Asia/Shanghai'
- export TQDM_MININTERVAL=60
- export HF_HUB_DISABLE_PROGRESS_BARS=1
- export UV_NO_PROGRESS=1
- export UV_SYSTEM_PYTHON=1
-
- log_info "Creating log directory: ${LOG_DIR}"
- mkdir -p "${LOG_DIR}"
-
- log_info "Downloading model: ${model_name}"
- hf download "${model_name}"
-
- log_group_end
-}
-
-function install_auto_round() {
- local install_source=$1
- local mode_name=$2
-
- log_group_start "Install requirements for [${mode_name}]..."
-
- (
- cd "${WORKSPACE_DIR}"
- log_info "Uninstalling existing auto-round..."
- uv pip uninstall auto-round || true
-
- log_info "Installing auto-round from: ${install_source}"
- BUILD_HPU_ONLY=1 uv pip install "${install_source}"
- )
-
- log_group_end
-}
-
-function run_performance_test() {
- local test_mode=$1
- local log_file="${LOG_DIR}/perf_test_${test_mode}.log"
-
- log_group_start "Run ${test_mode} performance test (${scheme})..."
-
- (
- cd "${PERF_SCRIPT_DIR}"
- log_info "Executing auto-round for ${scheme}. Logging to ${log_file}"
- auto-round \
- --model_name "${model_name}" \
- --scheme "${scheme}" \
- --iters "${ITERS}" \
- --enable_torch_compile \
- --device hpu \
- --output_dir "./${test_mode}" 2>&1 | tee -a "${log_file}"
- )
-
- log_group_end
-}
-
-function run_performance_check() {
- log_group_start "Check performance results..."
-
- (
- cd "${PERF_SCRIPT_DIR}"
- log_info "Executing check_performance.py"
- python check_performance.py
- )
-
- log_group_end
-}
-
-function main() {
- setup_environment
-
- install_auto_round "." "current"
- run_performance_test "current"
-
- install_auto_round "${BASELINE_GIT_URL}" "baseline"
- run_performance_test "baseline"
-
- run_performance_check
-
- log_info "All tasks completed successfully."
-}
-
-main "$@"
\ No newline at end of file
diff --git a/profile_rss_per_block.py b/profile_rss_per_block.py
deleted file mode 100644
index 66b13ce59..000000000
--- a/profile_rss_per_block.py
+++ /dev/null
@@ -1,203 +0,0 @@
-"""Granular per-block RSS profiling for peak RAM regression diagnosis.
-
-Instruments both old and new architecture to measure RSS at key points
-within the per-block quantization loop.
-
-Usage:
- # New arch:
- python profile_rss_per_block.py
- # Old arch:
- AR_DISABLE_NEW_ARCH=1 python profile_rss_per_block.py
-"""
-
-import gc
-import os
-import resource
-import sys
-import time
-
-
-def rss_mb():
- """Get current RSS in MB (no gc.collect - raw measurement)."""
- return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB -> MB on Linux
-
-
-def rss_mb_clean():
- """Get current RSS in MB after gc.collect."""
- gc.collect()
- return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
-
-
-# Use psutil for live RSS (ru_maxrss is peak, not current)
-import psutil
-
-_proc = psutil.Process()
-
-
-def live_rss_mb():
- """Current RSS in MB (not peak)."""
- return _proc.memory_info().rss / (1024 * 1024)
-
-
-def live_rss_mb_clean():
- gc.collect()
- try:
- import ctypes
-
- libc = ctypes.CDLL("libc.so.6")
- libc.malloc_trim(0)
- except Exception:
- pass
- return _proc.memory_info().rss / (1024 * 1024)
-
-
-arch = os.environ.get("AR_DISABLE_NEW_ARCH", "0")
-arch_label = "OLD" if arch == "1" else "NEW"
-print(f"\n{'='*70}")
-print(f" {arch_label} Architecture - Granular Per-Block RSS Profiling")
-print(f"{'='*70}")
-print(f"Before import RSS: {live_rss_mb():.1f} MB")
-
-# Monkey-patch to add instrumentation
-if arch != "1":
- # NEW ARCH: patch CalibCompressor._quantize_single_block
- from auto_round.compressors_new import calib as calib_mod
-
- _orig_quantize_single_block = calib_mod.CalibCompressor._quantize_single_block
- _orig_quantize_blocks = calib_mod.CalibCompressor._quantize_blocks
-
- _block_rss_log = []
-
- def _patched_quantize_single_block(self, model, m, input_ids, input_others, q_input):
- block_idx = len(_block_rss_log)
- rss_before = live_rss_mb()
-
- result = _orig_quantize_single_block(self, model, m, input_ids, input_others, q_input)
-
- rss_after_return = live_rss_mb()
- gc.collect()
- rss_after_gc = live_rss_mb()
- try:
- import ctypes
-
- libc = ctypes.CDLL("libc.so.6")
- libc.malloc_trim(0)
- except Exception:
- pass
- rss_after_trim = live_rss_mb()
-
- entry = {
- "block": block_idx,
- "before": rss_before,
- "after_return": rss_after_return,
- "after_gc": rss_after_gc,
- "after_trim": rss_after_trim,
- "delta_return": rss_after_return - rss_before,
- "delta_gc": rss_after_gc - rss_before,
- "delta_trim": rss_after_trim - rss_before,
- }
- _block_rss_log.append(entry)
- print(
- f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "
- f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "
- f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",
- flush=True,
- )
- return result
-
- calib_mod.CalibCompressor._quantize_single_block = _patched_quantize_single_block
-
-else:
- # OLD ARCH: patch LLMCompressor._quantize_block
- from auto_round.compressors import base as base_mod
-
- _orig_quantize_block = base_mod.LLMCompressor._quantize_block
-
- _block_rss_log = []
-
- def _patched_quantize_block(self, block, input_ids, input_others, q_input=None, device="cpu", auto_offload=True):
- block_idx = len(_block_rss_log)
- rss_before = live_rss_mb()
-
- result = _orig_quantize_block(self, block, input_ids, input_others, q_input, device, auto_offload)
-
- rss_after_return = live_rss_mb()
- gc.collect()
- rss_after_gc = live_rss_mb()
- try:
- import ctypes
-
- libc = ctypes.CDLL("libc.so.6")
- libc.malloc_trim(0)
- except Exception:
- pass
- rss_after_trim = live_rss_mb()
-
- entry = {
- "block": block_idx,
- "before": rss_before,
- "after_return": rss_after_return,
- "after_gc": rss_after_gc,
- "after_trim": rss_after_trim,
- "delta_return": rss_after_return - rss_before,
- "delta_gc": rss_after_gc - rss_before,
- "delta_trim": rss_after_trim - rss_before,
- }
- _block_rss_log.append(entry)
- print(
- f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "
- f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "
- f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",
- flush=True,
- )
- return result
-
- base_mod.LLMCompressor._quantize_block = _patched_quantize_block
-
-print(f"After import RSS: {live_rss_mb():.1f} MB")
-
-from auto_round import AutoRound
-
-print(f"After AutoRound import RSS: {live_rss_mb():.1f} MB")
-
-import shutil
-
-save_dir = "/tmp/profile_rss_output"
-shutil.rmtree(save_dir, ignore_errors=True)
-
-print("\nCreating AutoRound instance...")
-ar = AutoRound(
- model="Qwen/Qwen3-0.6B",
- scheme="FP8_STATIC",
- iters=200,
- nsamples=128,
- enable_torch_compile=True,
-)
-print(f"After init RSS: {live_rss_mb():.1f} MB")
-print(f"After init RSS (clean): {live_rss_mb_clean():.1f} MB")
-
-print("\nStarting quantize_and_save...\n")
-model, folder = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")
-
-print(f"\n{'='*70}")
-print(f" SUMMARY ({arch_label} Architecture)")
-print(f"{'='*70}")
-print(f"Final RSS: {live_rss_mb():.1f} MB")
-print(f"Final RSS (clean): {live_rss_mb_clean():.1f} MB")
-print("\nPer-block deltas (after return, after gc+trim):")
-for e in _block_rss_log:
- print(
- f" Block {e['block']:2d}: delta_ret={e['delta_return']:+.1f} delta_trim={e['delta_trim']:+.1f} MB "
- f"(abs: {e['after_trim']:.1f} MB)"
- )
-
-# Compute growth rate
-if len(_block_rss_log) >= 2:
- first = _block_rss_log[0]["after_trim"]
- last = _block_rss_log[-1]["after_trim"]
- n = len(_block_rss_log) - 1
- print(f"\nGrowth: {first:.1f} -> {last:.1f} MB over {n} blocks = {(last-first)/n:.1f} MB/block avg")
-
-print(f"\nPeak RSS (ru_maxrss): {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.1f} MB")
-
-shutil.rmtree(save_dir, ignore_errors=True)
diff --git a/test/unit/common/algorithms/test_algorithm_datatype_parity.py b/test/unit/common/algorithms/test_algorithm_datatype_parity.py
new file mode 100644
index 000000000..2d4829c05
--- /dev/null
+++ b/test/unit/common/algorithms/test_algorithm_datatype_parity.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Check the public datatype API used by wrappers and algorithms."""
+
+from types import SimpleNamespace
+
+import torch
+
+from auto_round.data_type.base import activation_quantizer_for_layer, create_quantizer
+from auto_round.data_type.int import quant_tensor_sym
+
+
+def test_weight_quantization_reuses_initialization_and_matches_int_math():
+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
+ config = dict(data_type="int", bits=4, group_size=4, sym=True, scale_dtype=torch.float32)
+ quantizer = create_quantizer(config, disable_opt_rtn=True)
+ quantizer.initialize(weight)
+ first = quantizer.quantize(weight)
+ second = quantizer.quantize(weight)
+ expected = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)[0]
+ torch.testing.assert_close(first, expected, rtol=0, atol=0)
+ assert torch.equal(first, second)
+
+
+def test_weight_training_and_writeback_share_parameters():
+ layer = torch.nn.Linear(4, 2, bias=False)
+ layer.data_type = "int"
+ layer.bits = 4
+ layer.group_size = 4
+ layer.sym = True
+ layer.scale_dtype = torch.float32
+ quantizer = create_quantizer(layer, iters=10, tune_rounding=True, tune_minmax=True)
+ quantizer.initialize(layer.weight.detach())
+ output = quantizer.quantize(layer.weight.detach())
+ output.square().sum().backward()
+ assert all(parameter.grad is not None for parameter in quantizer.parameters.values())
+ expected = output.detach().clone()
+ with torch.no_grad():
+ quantizer.write_back(layer, layer.weight.detach())
+ torch.testing.assert_close(layer.weight, expected, rtol=0, atol=0)
+ assert layer.scale.shape == (2, 1)
+ assert layer.zp is not None
+
+
+def test_reinitializing_search_quantizer_does_not_reuse_previous_weight_range():
+ config = dict(data_type="int", bits=4, group_size=4, sym=False, scale_dtype=torch.float32)
+ quantizer = create_quantizer(config, disable_opt_rtn=True)
+ first = torch.tensor([[1.0, -0.75, 0.25, -0.5]])
+ second = first * 5
+ quantizer.initialize(first)
+ quantizer.quantize(first)
+ quantizer.initialize(second)
+ actual = quantizer.quantize(second)
+ fresh = create_quantizer(config, disable_opt_rtn=True)
+ fresh.initialize(second)
+ assert torch.equal(actual, fresh.quantize(second))
+
+
+def test_activation_adapter_static_and_dynamic_contracts():
+ activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)
+ for dynamic in (False, True):
+ layer = SimpleNamespace(
+ act_data_type="int",
+ act_bits=8,
+ act_group_size=4,
+ act_sym=True,
+ scale_dtype=torch.float32,
+ act_dynamic=dynamic,
+ )
+ quantizer = activation_quantizer_for_layer(layer)
+ observed = quantizer.observe(activation, None)
+ qdq = quantizer.qdq(activation, observed_max=observed)
+ assert qdq.shape == activation.shape
+ assert torch.isfinite(qdq).all()
diff --git a/test/unit/common/algorithms/transforms/hadamard/test_patch.py b/test/unit/common/algorithms/transforms/hadamard/test_patch.py
index ff63079dc..6c8d176a8 100644
--- a/test/unit/common/algorithms/transforms/hadamard/test_patch.py
+++ b/test/unit/common/algorithms/transforms/hadamard/test_patch.py
@@ -179,14 +179,14 @@ class TestPatchWrapperLinearIdempotency:
# Call the patched _qdq_act on a wrapper
wrapper = _make_wrapper_linear()
- # Replace the wrapper's act_quant_func with a spy
- wrapper.act_quant_func = MagicMock(return_value=(torch.zeros(1, 8), None, None))
+ # Activation QDQ is now owned by the datatype quantizer.
+ wrapper.activation_quantizer.qdq_with_scale = MagicMock(return_value=(torch.zeros(1, 8), None, None))
x = torch.ones(1, 8)
act_min = torch.tensor(1.0)
act_max = torch.tensor(1.0)
- wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max)
+ wrapper._qdq_act(x, act_min_scale=act_min, act_max_scale=act_max, act_max=act_max)
# The mock should have been called with x * 2
- called_args = wrapper.act_quant_func.call_args[0]
+ called_args = wrapper.activation_quantizer.qdq_with_scale.call_args[0]
assert torch.equal(called_args[0], x * 2.0)
def test_qdq_weight_falls_through_for_high_bits(self):
@@ -294,6 +294,7 @@ class TestPatchWrapperWALayerIdempotency:
patch_wrapperwalayer_forward_to_apply_transform(inp_transform)
wrapper = _make_wrapper_wa_layer()
+ wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))
# Patch the orig_layer.forward to be a spy
orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))
wrapper.orig_layer.forward = orig_forward_spy
@@ -301,8 +302,8 @@ class TestPatchWrapperWALayerIdempotency:
x = torch.ones(1, 8)
wrapper(x)
- # The act_quant_func should have been called with x * 4
- called_args = wrapper.orig_layer.act_quant_func.call_args[0]
+ # Datatype-owned activation QDQ receives the rotated input.
+ called_args = wrapper.activation_quantizer.qdq.call_args[0]
assert torch.equal(called_args[0], x * 4.0)
def test_act_max_passed_when_present(self):
@@ -315,6 +316,7 @@ class TestPatchWrapperWALayerIdempotency:
wrapper = _make_wrapper_wa_layer()
wrapper.orig_layer.act_max = torch.tensor(2.0)
+ wrapper.activation_quantizer.qdq = MagicMock(return_value=torch.zeros(1, 8))
orig_forward_spy = MagicMock(return_value=torch.zeros(1, 8))
wrapper.orig_layer.forward = orig_forward_spy
@@ -322,9 +324,7 @@ class TestPatchWrapperWALayerIdempotency:
x = torch.ones(1, 8)
wrapper(x)
- # tensor_max should be the act_max value
- kwargs = wrapper.orig_layer.act_quant_func.call_args.kwargs
- assert kwargs.get("tensor_max") is not None
+ assert wrapper.activation_quantizer.qdq.call_args.kwargs["observed_max"] is not None
class TestPatchQuantLinearIdempotency:
diff --git a/test/unit/common/core/test_wrapper_utils.py b/test/unit/common/core/test_wrapper_utils.py
index a603d0da0..68de62b29 100644
--- a/test/unit/common/core/test_wrapper_utils.py
+++ b/test/unit/common/core/test_wrapper_utils.py
@@ -335,6 +335,95 @@ class TestWrapperLinearDeviceTransfer:
assert wrapper.device == "cpu"
assert wrapper.orig_layer is orig_layer
+ def test_unwrapper_keeps_weight_for_activation_only_quantization(self):
+ """W16A8 writes activation state only; its weight stays full precision."""
+ from auto_round.wrapper import WrapperLinear
+
+ layer = torch.nn.Linear(4, 2, bias=False)
+ layer.bits = 16
+ layer.sym = True
+ layer.group_size = -1
+ layer.scale_dtype = torch.float32
+ layer.data_type = "int"
+ layer.act_bits = 8
+ layer.act_data_type = "int"
+ layer.act_sym = True
+ layer.act_dynamic = True
+ layer.act_group_size = -1
+ layer.iters = 0
+ layer.tuning_device = "cpu"
+ expected = layer.weight.detach().clone()
+
+ WrapperLinear(layer, device="cpu", disable_opt_rtn=True).unwrapper({})
+
+ assert torch.equal(layer.weight, expected)
+ assert not hasattr(layer, "scale")
+
+ def test_custom_weight_qdq_is_shared_by_forward_and_write_back(self):
+ """A per-layer algorithm hook controls both tuning and final materialization."""
+ from auto_round.wrapper import WrapperLinear
+
+ orig_layer = torch.nn.Linear(4, 2, bias=False)
+ orig_layer.bits = 4
+ orig_layer.sym = True
+ orig_layer.group_size = 4
+ orig_layer.scale_dtype = torch.float32
+ orig_layer.data_type = "int"
+ orig_layer.act_bits = 16
+ orig_layer.act_data_type = "float"
+ orig_layer.act_sym = True
+ orig_layer.act_dynamic = True
+ orig_layer.act_group_size = -1
+ orig_layer.tuning_device = "cpu"
+ calls = []
+
+ def build_weight_qdq(*, weight, default_qdq, parameters):
+ assert weight.shape == (2, 4)
+ parameters["offset"] = torch.nn.Parameter(weight.new_tensor(0.1))
+
+ def custom_qdq(weight, *, materialize=False):
+ calls.append(materialize)
+ return default_qdq(weight + parameters["offset"], materialize=materialize, **parameters)
+
+ return custom_qdq
+
+ wrapper = WrapperLinear(
+ orig_layer,
+ device="cpu",
+ disable_opt_rtn=True,
+ iters=0,
+ weight_qdq_builder=build_weight_qdq,
+ )
+
+ result, _, _ = wrapper._qdq_weight(wrapper.value, wrapper.min_scale, wrapper.max_scale)
+ result.sum().backward()
+ assert wrapper.params["offset"].grad is not None
+ best = torch.tensor(0.5)
+ wrapper.params["offset"] = best
+ expected = wrapper.weight_qdq(orig_layer.weight, materialize=True).weight.detach().clone()
+ wrapper.params["offset"] = torch.tensor(0.9)
+ wrapper.unwrapper({"offset": best})
+
+ assert calls == [False, True, True]
+ assert torch.equal(orig_layer.weight, expected)
+
+ def test_base_algorithm_uses_datatype_qdq_by_default(self):
+ """Algorithms only override QDQ when they explicitly need custom behavior."""
+ from auto_round.algorithms.quantization.base import BaseQuantizer
+ from auto_round.data_type.base import create_quantizer
+
+ weight = torch.tensor([[1.0, -0.5, 0.25, -0.75]])
+ datatype = create_quantizer(dict(data_type="int", bits=4, group_size=4, sym=True), disable_opt_rtn=True)
+ datatype.initialize(weight)
+ parameters = {"value": torch.tensor(0.0)}
+ qdq = BaseQuantizer.build_weight_qdq(object(), weight=weight, default_qdq=datatype.qdq, parameters=parameters)
+ torch.testing.assert_close(qdq(weight).weight, datatype.qdq(weight, value=0.0).weight)
+ parameters["value"] = torch.tensor(0.75)
+ result = qdq(weight, materialize=True)
+ expected = datatype.qdq(weight, value=0.75, materialize=True)
+ torch.testing.assert_close(result.weight, expected.weight)
+ torch.testing.assert_close(result.scale, expected.scale)
+
def test_wrapper_linear_forward(self):
from auto_round.wrapper import WrapperLinear
diff --git a/test/unit/common/data_type/__init__.py b/test/unit/common/data_type/__init__.py
index e69de29bb..14a492441 100644
--- a/test/unit/common/data_type/__init__.py
+++ b/test/unit/common/data_type/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/test/unit/common/data_type/test_quantizer_parity.py b/test/unit/common/data_type/test_quantizer_parity.py
new file mode 100644
index 000000000..9b23109cc
--- /dev/null
+++ b/test/unit/common/data_type/test_quantizer_parity.py
@@ -0,0 +1,189 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Focused coverage for the datatype public lifecycle."""
+
+import pytest
+import torch
+
+from auto_round.data_type.base import (
+ activation_quantizer_for_layer,
+ cache_activation_quantizer,
+ canonical_data_type,
+ create_quantizer,
+)
+from auto_round.data_type.fp8 import quant_fp8_e5m2
+from auto_round.data_type.gguf import _GGUFWeightQuantizer
+from auto_round.data_type.int import quant_tensor_sym
+from auto_round.data_type.mxfp import quant_mx, quant_mx_rceil
+
+
+def _weight_config(data_type, bits, group_size, *, sym=True):
+ return dict(data_type=data_type, bits=bits, group_size=group_size, sym=sym, scale_dtype=torch.float32)
+
+
+def test_integer_lifecycle_matches_primitive_and_materializes_payload():
+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
+ quantizer.initialize(weight)
+ actual = quantizer.quantize(weight)
+ expected, _, _ = quant_tensor_sym(weight, bits=4, group_size=4, scale_dtype=torch.float32)
+
+ assert torch.equal(actual, expected)
+
+
+def test_qdq_returns_the_same_result_used_for_write_back():
+ """Algorithms can execute QDQ once and hand its materialized result to the datatype."""
+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5]], dtype=torch.float32)
+ layer = torch.nn.Linear(4, 1, bias=False)
+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
+ quantizer.initialize(weight)
+
+ result = quantizer.qdq(weight, materialize=True)
+ quantizer.apply_result(layer, result)
+
+ assert torch.equal(layer.weight, result.weight)
+ assert layer.scale.shape == (1, 1)
+
+
+def test_write_back_keeps_conv1d_scale_indexed_by_output_channels():
+ """Conv1D stores a transposed weight, but its scale stays per output channel."""
+ weight = torch.tensor([[1.0, -0.75, 0.25, -0.5], [-1.0, 0.75, -0.25, 0.5]], dtype=torch.float32)
+ layer = torch.nn.Module()
+ layer.weight = torch.nn.Parameter(torch.empty(4, 2))
+ quantizer = create_quantizer(_weight_config("int", 4, 4), disable_opt_rtn=True)
+ quantizer.initialize(weight)
+
+ quantizer.write_back(layer, weight, transpose=True)
+
+ assert layer.weight.shape == (4, 2)
+ assert layer.scale.shape == (2, 1)
+
+
+def test_fp8_format_selection_does_not_turn_e5m2_into_block_e4m3():
+ weight = torch.randn(16, 32)
+ quantizer = create_quantizer(_weight_config("fp8_e5m2", 8, (8, 16)), iters=1)
+ quantizer.initialize(weight)
+ result = quantizer.quantize(weight)
+ expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=(8, 16))
+
+ assert torch.equal(result, expected)
+
+
+def test_generic_mx_name_uses_requested_bit_width():
+ weight = torch.randn(8, 32)
+ quantizer = create_quantizer(_weight_config("mx_fp", 6, 32), iters=1)
+ quantizer.initialize(weight)
+ result = quantizer.quantize(weight)
+ expected, _, _ = quant_mx(weight, bits=6, group_size=32, data_type="mx_fp6")
+
+ assert torch.equal(result, expected)
+
+
[email protected]("dynamic", (True, False))
+def test_activation_lifecycle_handles_dynamic_and_calibrated_integer_quantization(dynamic):
+ activation = torch.tensor([[-1.0, 0.5, 0.25, 1.0]], dtype=torch.float32)
+ layer = type(
+ "Layer",
+ (),
+ dict(act_data_type="int", act_bits=8, act_group_size=4, act_sym=True, act_dynamic=dynamic),
+ )()
+ quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)
+ observed = None if dynamic else quantizer.observe(activation, None)
+
+ result = quantizer.qdq(activation, observed_max=observed)
+
+ assert result.shape == activation.shape
+ assert torch.isfinite(result).all()
+
+
+def test_nvfp_static_global_scale_accepts_dynamic_calibration():
+ """NVFP4 calibrates a static global scale while keeping dynamic QDQ enabled."""
+ layer = type(
+ "Layer",
+ (),
+ dict(
+ act_data_type="nv_fp4_with_static_gs",
+ act_bits=4,
+ act_group_size=16,
+ act_sym=True,
+ act_dynamic=True,
+ ),
+ )()
+
+ quantizer = activation_quantizer_for_layer(layer, scale_dtype=torch.float32)
+
+ assert quantizer.requires_calibration is True
+
+
+def test_aliases_keep_a_single_canonical_datatype_for_policy():
+ assert canonical_data_type("int") == "int_sym"
+ assert canonical_data_type("INT4_ASYM") == "int_asym"
+
+
+def test_activation_quantizer_uses_the_spec_default_datatype():
+ """Partial layer metadata keeps the public ``int_sym`` activation default."""
+ layer = type("Layer", (), dict(act_bits=8, act_group_size=4, act_sym=True, act_dynamic=True))()
+
+ assert type(cache_activation_quantizer(layer)).__name__ == "_IntActivationQuantizer"
+ assert type(activation_quantizer_for_layer(layer)).__name__ == "_IntActivationQuantizer"
+
+
+def test_format_aliases_keep_the_requested_quantization_format():
+ weight = torch.randn(4, 32)
+ fp8 = create_quantizer(_weight_config("fp8-e5m2", 8, -1), iters=1)
+ fp8.initialize(weight)
+ expected, _, _ = quant_fp8_e5m2(weight, bits=8, group_size=-1)
+
+ gguf = create_quantizer(_weight_config("INT_SYM_DQ", 4, 32), disable_opt_rtn=True)
+ legacy_gguf = create_quantizer(_weight_config("rtn_int_sym_dq", 4, 32), disable_opt_rtn=True)
+
+ assert torch.equal(fp8.quantize(weight), expected)
+ assert isinstance(gguf._implementation, _GGUFWeightQuantizer)
+ assert gguf._implementation.kind == "sym"
+ assert isinstance(legacy_gguf._implementation, _GGUFWeightQuantizer)
+
+
+def test_mx_aliases_and_rceil_keep_the_requested_format():
+ weight = torch.randn(4, 32)
+ layer = type(
+ "Layer", (), dict(act_data_type="mxfp4", act_bits=4, act_group_size=32, act_sym=True, act_dynamic=True)
+ )()
+ activation = activation_quantizer_for_layer(layer)
+ quantizer = create_quantizer(_weight_config("mx_fp4e2m1_rceil", 4, 32), iters=1)
+ quantizer.initialize(weight)
+ expected, _, _ = quant_mx_rceil(weight, bits=4, group_size=32, data_type="mx_fp4e2m1")
+
+ assert activation.data_type == "mx_fp4"
+ assert torch.equal(quantizer.quantize(weight), expected)
+
+
+def test_awq_clip_ranges_are_flattened_before_integer_qdq():
+ """AWQ stores [output, group] clips while INT QDQ uses flattened group rows."""
+ weight = torch.randn(4, 8)
+ clip = weight.abs().reshape(4, 2, 4).amax(dim=-1)
+ quantizer = create_quantizer(
+ {
+ **_weight_config("int", 4, 4),
+ "awq_clip_min": -clip,
+ "awq_clip_max": clip,
+ },
+ iters=1,
+ )
+ quantizer.initialize(weight)
+
+ result = quantizer.quantize(weight)
+
+ assert result.shape == weight.shape
+ assert torch.isfinite(result).all()
diff --git a/test/unit/common/models/test_moe_experts_interface.py b/test/unit/common/models/test_moe_experts_interface.py
index 49f0f9125..2656d1c58 100644
--- a/test/unit/common/models/test_moe_experts_interface.py
+++ b/test/unit/common/models/test_moe_experts_interface.py
@@ -21,10 +21,13 @@ This verifies that:
3. The forward pass produces correct results
"""
+from types import SimpleNamespace
+
import pytest
import torch
from torch import nn
+from auto_round.modeling.fused_moe.grouped_experts import _batched_qdq_weights
from auto_round.modeling.fused_moe.replace_modules import (
ReplacementModuleBase,
_apply_custom_replacements,
@@ -42,6 +45,11 @@ def _skip_if_no_linear_loop():
pytest.skip("transformers MOE integration not available")
+def test_batched_qdq_falls_back_without_legacy_quant_function():
+ """Datatype-owned QDQ safely falls back when its private state cannot be fused."""
+ assert _batched_qdq_weights([SimpleNamespace()]) is None
+
+
def test_linear_loop_registration():
"""Test that linear_loop is registered with transformers."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.py
index 837a96605..0931831cc 100644
--- a/test/unit/common/schemes/test_auto_scheme.py
+++ b/test/unit/common/schemes/test_auto_scheme.py
@@ -16,6 +16,7 @@ from auto_round.auto_scheme.delta_loss import (
_vram_inventory_text,
)
from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
+from auto_round.data_type.base import WeightQuantizationResult
from auto_round.export.export_to_gguf.export import _clear_gguf_model_instances
@@ -1200,24 +1201,38 @@ class TestScoreAnchorWeightScoring:
wrapper.max_act_value = 0
wrapper._score_qdq_cpu = None
- def _fake_quant(weight, **kwargs):
- scale = weight.abs().amax().clamp(min=1e-4) / 7.0
- return ((weight / scale).round().clamp(-7, 7) * scale), scale, None
+ class _State:
+ tunables = {}
- wrapper.weight_quant_func = _fake_quant
+ class _Quantizer:
+ def __init__(self):
+ self.calls = 0
+
+ def quantize(self, weight, **kwargs):
+ self.calls += 1
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return (weight / scale).round().clamp(-7, 7) * scale
+
+ wrapper.params = {}
+ wrapper.weight_state = _State()
+ wrapper.weight_quantizer = _Quantizer()
+ wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(
+ wrapper.weight_quantizer.quantize(weight, **kwargs)
+ )
layer.weight.requires_grad = True
return wrapper, layer
def test_qdq_weight_quantizes_once_and_reuses_cache(self):
wrapper, layer = self._make_wrapper()
calls = {"n": 0}
- real_quant = wrapper.weight_quant_func
+ quantizer = wrapper.weight_quantizer
+ original_quantize = quantizer.quantize
- def counting_quant(weight, **kwargs):
+ def counting_quantize(*args, **kwargs):
calls["n"] += 1
- return real_quant(weight, **kwargs)
+ return original_quantize(*args, **kwargs)
- wrapper.weight_quant_func = counting_quant
+ quantizer.quantize = counting_quantize
args = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
first, _, _ = wrapper._qdq_weight(*args)
second, _, _ = wrapper._qdq_weight(*args)
@@ -1396,11 +1411,20 @@ class TestScoreLinearRecompute:
wrapper.min_scale = torch.tensor(1.0)
wrapper.max_scale = torch.tensor(1.0)
- def _fake_quant(weight, **kwargs):
- scale = weight.abs().amax().clamp(min=1e-4) / 7.0
- return (weight / scale).round().clamp(-7, 7) * scale, scale, None
+ class _State:
+ tunables = {}
- wrapper.weight_quant_func = _fake_quant
+ class _Quantizer:
+ def quantize(self, weight, **kwargs):
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return (weight / scale).round().clamp(-7, 7) * scale
+
+ wrapper.params = {}
+ wrapper.weight_state = _State()
+ wrapper.weight_quantizer = _Quantizer()
+ wrapper.weight_qdq = lambda weight, **kwargs: WeightQuantizationResult(
+ wrapper.weight_quantizer.quantize(weight, **kwargs)
+ )
layer.weight.requires_grad = True
return wrapper, layer
diff --git a/test/unit/test_cpu/algorithms/test_awq.py b/test/unit/test_cpu/algorithms/test_awq.py
index 380d76661..c53773b5e 100644
--- a/test/unit/test_cpu/algorithms/test_awq.py
+++ b/test/unit/test_cpu/algorithms/test_awq.py
@@ -492,7 +492,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -522,14 +522,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quant_funcs(params):
- return f"{params['data_type']}_{params['bits']}", None
+ def fake_resolve_quantizer(params):
+ return f"{params['data_type']}_{params['bits']}"
- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quant_func))
+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quantizer))
return weight
- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -1009,32 +1009,19 @@ class TestAWQUseV2ScaleSearch:
q._qdq_tool.configure(compressor)
assert q._qdq_tool.disable_opt_rtn is True
+ params = {
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "disable_opt_rtn": True,
+ }
+ assert q._qdq_tool.resolve_quantizer(params)._mode == "rtn"
compressor = self._make_compressor(self._signroundv2_config(data_type="mx_fp"))
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is True
- def test_init_scale_dispatch_by_data_type(self):
- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
-
- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
- weight = torch.randn(32, 128)
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
- assert init_scale is not None, dt
- assert init_scale.shape[0] == weight_reshape.shape[0]
-
- # Scalar imatrix sentinels mean uniform importance and must not enter
- # the tensor reshape path.
- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
- assert scalar_init_scale is not None
- assert scalar_init_scale.shape == init_scale.shape
-
- # asym int and *_dq are not part of the optimized init-scale path.
- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
-
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4
diff --git a/test/unit/test_cuda/algorithms/test_alg_ext.py b/test/unit/test_cuda/algorithms/test_alg_ext.py
index 936e1522d..a618a6c04 100644
--- a/test/unit/test_cuda/algorithms/test_alg_ext.py
+++ b/test/unit/test_cuda/algorithms/test_alg_ext.py
@@ -35,38 +35,6 @@ class TestAlgExt:
yield
shutil.rmtree("runs", ignore_errors=True)
- # def test_gguf_q2_k_s_uses_dq_wrapper_block(self, tiny_qwen_model_path):
- # """Regression test: enable_alg_ext + gguf:q2_k_s must use DQWrapperLinear.
- #
- # gguf:q2_k_s overrides data_type to "int_asym_dq" at format-resolution
- # time. The quantizer must be created *after* that override so that
- # SignRoundV2 sees the final data_type and sets dq_wrapper_block
- # (which wraps layers with DQWrapperLinear) instead of falling back to
- # the plain wrapper_block (which produces WrapperLinear).
- # """
- # from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundDQWrapperLinear
- #
- # ar = AutoRound(
- # tiny_qwen_model_path,
- # bits=4,
- # format="gguf:q2_k_s",
- # iters=1,
- # nsamples=1,
- # seqlen=32,
- # enable_alg_ext=True,
- # )
- # # post_init() runs the full pipeline (resolve_scheme → resolve_formats →
- # # create_quantizer → ...). quantizer only exists afterwards.
- # ar.post_init()
- # quantizer = ar.alg_composer.block_quantizer
- #
- # assert quantizer.wrapper_block.keywords["wrapper_cls"] is SignRoundDQWrapperLinear, (
- # f"Expected wrapper_block to use '{SignRoundDQWrapperLinear.__name__}', "
- # f"got '{quantizer.wrapper_block.__name__}'. "
- # "This likely means the quantizer was created before GGUF format "
- # "overrides were applied (data_type was not yet 'int_asym_dq')."
- # )
-
def test_int2_g64_asym_enable_alg_ext_keeps_config(self, tiny_qwen_model_path):
"""Regression test: asym int2/g64 keeps the requested tuning config."""
diff --git a/test/unit/test_cuda/algorithms/test_awq.py b/test/unit/test_cuda/algorithms/test_awq.py
index da1459920..106b7e3a7 100644
--- a/test/unit/test_cuda/algorithms/test_awq.py
+++ b/test/unit/test_cuda/algorithms/test_awq.py
@@ -498,7 +498,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -528,14 +528,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quant_funcs(params):
- return f"{params['data_type']}_{params['bits']}", None
+ def fake_resolve_quantizer(params):
+ return f"{params['data_type']}_{params['bits']}"
- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quant_func))
+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quantizer))
return weight
- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -986,27 +986,6 @@ class TestAWQUseV2ScaleSearch:
compressor = self._make_compressor(block)
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False
- def test_init_scale_dispatch_by_data_type(self):
- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
-
- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
- weight = torch.randn(32, 128)
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
- assert init_scale is not None, dt
- assert init_scale.shape[0] == weight_reshape.shape[0]
-
- # Scalar imatrix sentinels mean uniform importance and must not enter
- # the tensor reshape path.
- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
- assert scalar_init_scale is not None
- assert scalar_init_scale.shape == init_scale.shape
-
- # asym int and *_dq are not part of the optimized init-scale path.
- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
-
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4
diff --git a/test/unit/test_xpu/algorithms/test_awq.py b/test/unit/test_xpu/algorithms/test_awq.py
index 9bbb8cb1a..7558302cc 100644
--- a/test/unit/test_xpu/algorithms/test_awq.py
+++ b/test/unit/test_xpu/algorithms/test_awq.py
@@ -494,7 +494,7 @@ class TestAWQMoE:
assert transform._mapping_is_smoothable(mapping) is False
assert any("different quantization parameters" in warning for warning in warnings)
- def test_awq_grid_search_uses_per_balance_layer_quant_func(self):
+ def test_awq_grid_search_uses_per_balance_layer_quantizer(self):
"""Direct grid search should pass each balance layer's own resolved quant function."""
import torch.nn as nn
@@ -524,14 +524,14 @@ class TestAWQMoE:
}
records = []
- def fake_resolve_quant_funcs(params):
- return f"{params['data_type']}_{params['bits']}", None
+ def fake_resolve_quantizer(params):
+ return f"{params['data_type']}_{params['bits']}"
- def fake_qdq(weight, params, *, quant_func=None, opt_quant_func=None, imatrix=None):
- records.append((params["data_type"], params["bits"], quant_func))
+ def fake_qdq(weight, params, *, quantizer=None, imatrix=None):
+ records.append((params["data_type"], params["bits"], quantizer))
return weight
- transform._qdq_tool.resolve_quant_funcs = fake_resolve_quant_funcs
+ transform._qdq_tool.resolve_quantizer = fake_resolve_quantizer
transform._qdq_tool.qdq = fake_qdq
transform._grid_search_scales(mapping, torch.ones(8))
@@ -981,27 +981,6 @@ class TestAWQUseV2ScaleSearch:
compressor = self._make_compressor(block)
assert q._qdq_tool._block_quantizer_is_signroundv2(compressor) is False
- def test_init_scale_dispatch_by_data_type(self):
- """``search_optimized_init_scale`` injects only for sym int/mx/nv."""
- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, search_optimized_init_scale
-
- for dt, gs in (("int_sym", 128), ("mx_fp4", 32), ("nv_fp4", 16)):
- weight = torch.randn(32, 128)
- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight, gs)
- init_scale = search_optimized_init_scale(weight_reshape, dt, 4, None)
- assert init_scale is not None, dt
- assert init_scale.shape[0] == weight_reshape.shape[0]
-
- # Scalar imatrix sentinels mean uniform importance and must not enter
- # the tensor reshape path.
- scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
- assert scalar_init_scale is not None
- assert scalar_init_scale.shape == init_scale.shape
-
- # asym int and *_dq are not part of the optimized init-scale path.
- assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
- assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
-
def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
"""AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4