PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from collections.abc import Callable
from contextlib import nullcontext
from functools import partial
from typing import TYPE_CHECKING
import torch
import transformers
from torch import autocast
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
if TYPE_CHECKING:
from auto_round.algorithms.composer import AlgorithmComposer
from auto_round import envs
from auto_round.algorithms.registry import register_pipeline_member
from auto_round.data_type.gguf import (
double_quant_tensor_sym_rtn,
quant_tensor_gguf_asym_dq,
quant_tensor_gguf_sym_dq,
search_gguf_scale_min_asym,
search_gguf_scale_min_sym,
)
from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym
from auto_round.data_type.utils import (
get_optimized_quant_func,
reshape_imatrix_for_weight,
reshape_pad_tensor_by_group_size,
resolve_optimized_init_scale_fn,
revert_tensor_by_pad,
round_ste,
search_optimized_init_scale,
)
from auto_round.logger import logger
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal
from auto_round.wrapper import WrapperLinear, wrapper_block
def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):
"""Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
orig_dtype = tensor.dtype
tensor = tensor.to(torch.float32)
maxq = int(2.0**bits) - 1
inverse_scale = get_reciprocal(scale)
int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
qdq = (scale * int_w - wmin).to(orig_dtype)
qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
return qdq
def _dq_sym_qdq(tensor, scale, bits, v=0):
"""Pure sym double-quant qdq math given precomputed scale (compilable)."""
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
group_size = 16
super_group_size = 16
maxq = int(2.0 ** (bits - 1))
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
orig_dtype = tensor.dtype
tensor = tensor.to(torch.float32)
ggml_type = f"q{bits}_k"
block_size, _ = GGML_QUANT_SIZES[ggml_type]
n_blocks = tensor.nelement() // block_size
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
if isinstance(v, torch.Tensor):
v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
else:
v_r = v
zp = torch.full_like(scale, maxq)
inverse_scale = get_reciprocal(scale)
int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq
qdq = (scale * (int_w - zp)).to(orig_dtype)
qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
return qdq
def _named_wrapper_block(wrapper_cls, name: str):
wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)
wrapped.__name__ = name
return wrapped
def _dq_asym_group_size(bits: int) -> int:
"""Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""
return 16 if bits == 2 else 32
class SignRoundOptimizedWrapperLinear(WrapperLinear):
minmax_scale_bound = (0.1, 10.0) # TODO widened from 2.0 for low-bit QAD; verify correctness
# Opt-in protocol for wrapper_block: the init-scale search is weight-local
# and row-independent, so same-shape modules can be searched as one stacked
# batch (bit-identical per module, far fewer python/launch round-trips).
supports_batched_search = True
def _init_tuning_params_and_quant_func(self, defer_search: bool = False) -> None:
super()._init_tuning_params_and_quant_func(defer_search=defer_search)
# Log/geometric min-max tuning (opt-in): re-anchor the tunable min/max
# scale coefficients to log space so each SignSGD step is a symmetric
# multiplicative move. Only engages when the coefficients are actually
# trainable (``nn.Parameter``); constant/frozen coefficients keep the
# plain linear path (a constant 1.0 must not be passed through exp()).
self._use_log_minmax = False
_min_tunable = isinstance(getattr(self, "min_scale", None), torch.nn.Parameter)
_max_tunable = isinstance(getattr(self, "max_scale", None), torch.nn.Parameter)
if envs.AR_ENABLE_LOG_MINMAX and _min_tunable and _max_tunable:
self._use_log_minmax = True
warm_start = getattr(self.orig_layer, "_ar_warm_start", None) or {}
saved_log = warm_start.get("minmax_parameterization") == "log"
with torch.no_grad():
for key in ("min_scale", "max_scale"):
param = getattr(self, key)
if not isinstance(param, torch.nn.Parameter):
continue
saved = warm_start.get(key)
compatible = saved is not None and saved.shape == param.shape
if self._use_log_minmax:
if not compatible:
# theta = 0 <-> coeff = exp(0) = 1.0
param.zero_()
elif not saved_log:
param.clamp_min_(torch.finfo(param.dtype).tiny).log_()
elif compatible and saved_log:
param.exp_()
layer = self.orig_layer
data_type = layer.data_type
# Warm-start reuse: a prior pass (SignRound -> QAD chaining) may have
# stashed the searched ``init_scale`` on this layer. Reuse it verbatim so
# the second pass tunes on the exact same scale grid (re-searching here
# would differ because the imatrix was consumed/deleted in the first pass).
warm_start = getattr(layer, "_ar_warm_start", None)
warm_init_scale = warm_start.get("init_scale") if warm_start else None
weight_reshape = self._prepare_init_scale_weight()
# staged deferral keeps only the RAW column imatrix (or None): the
# expanded full-size copy would be pinned per module for the whole
# wrap phase -- N x weight bytes -- while the batched call needs it
# only transiently per chunk (bounded by the batch cap)
imatrix_raw = getattr(layer, "imatrix", None)
imatrix = None if defer_search else reshape_imatrix_for_weight(imatrix_raw, weight_reshape, layer.group_size)
self.weight_quant_func = get_optimized_quant_func(data_type)
search_fn = resolve_optimized_init_scale_fn(data_type, self.q_scale_thresh)
if search_fn is None or self.weight_quant_func is None:
raise ValueError(
f"SignRound optimized path does not support data_type={data_type!r}; "
"expected a symmetric int / mx / nv type."
)
self.data_type = data_type
self.init_scale = None
self._deferred_search_inputs = None
if warm_init_scale is not None:
# Warm-started from a prior pass: reuse the exact scale, skip any search.
self.init_scale = warm_init_scale.to(self.device)
elif defer_search:
# staged for run_batched_wrap_search: the exact callable the
# per-module path would invoke travels with the module
self._deferred_search_inputs = (
weight_reshape,
data_type,
layer.bits,
imatrix_raw,
self.q_scale_thresh,
search_fn,
)
else:
self.init_scale = search_fn(weight_reshape, layer.bits, imatrix)
if hasattr(layer, "imatrix"):
del layer.imatrix
if self.enable_torch_compile:
self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
def _prepare_minmax_scale(self, min_scale, max_scale):
"""Return the multiplicative coefficients the quant func consumes.
In log mode the stored parameters are ``theta`` (SignSGD tunes them with
symmetric fixed steps); clamp them to the symmetric log bound and return
``exp(theta)`` so the quant function still receives plain multiplicative
coefficients, while ``+lr``/``-lr`` steps become ``xe^lr``/``/e^lr``.
Gradients flow back through ``exp`` to the ``theta`` parameters.
"""
if not getattr(self, "_use_log_minmax", False):
return super()._prepare_minmax_scale(min_scale, max_scale)
# The shared ``minmax_scale_bound`` is defined in Euclidean (coefficient)
# space; convert it to the log/theta clamp range here so both the linear
# and log paths read from one bound. ``log`` is undefined at <= 0, so when
# the linear bound starts at 0 fall back to a multiplicatively symmetric
# lower end (the reciprocal of the upper coefficient), keeping shrink/expand
# equidistant from the identity.
coeff_low, coeff_high = self.minmax_scale_bound
if coeff_low <= 0.0:
coeff_low = 1.0 / coeff_high
low, high = math.log(coeff_low), math.log(coeff_high)
min_scale.data.clamp_(low, high)
max_scale.data.clamp_(low, high)
return torch.exp(min_scale), torch.exp(max_scale)
def _run_deferred_search_now(self) -> None:
"""Run the staged search per-module (fallback when batching is off)."""
# Warm-started wrappers (QAD chaining) resolve ``init_scale`` directly and
# stage no search inputs; there is nothing to run for them.
if self._deferred_search_inputs is None:
return
weight, _data_type, bits, imatrix_raw, _thresh, search_fn = self._deferred_search_inputs
imatrix = reshape_imatrix_for_weight(imatrix_raw, weight, self.orig_layer.group_size)
self.init_scale = search_fn(weight, bits, imatrix)
self._deferred_search_inputs = None
def finalize_batched_search(self, init_scale: "torch.Tensor | list[torch.Tensor]") -> None:
"""Adopt a search result computed on a stacked same-shape batch."""
self.init_scale = init_scale
self._deferred_search_inputs = None
def _prepare_init_scale_weight(self) -> torch.Tensor:
"""Return the group-reshaped weight that seeds the init-scale search.
Reproduces the layout the quant func sees at tuning time: the (optionally
transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in
``clip_as_init`` mode and stored a per-group clip range on the layer, the
weight is clamped to that range first so the searched ``init_scale``
already reflects the AWQ clip (``max_scale`` then tunes a coefficient on
top of it).
"""
layer = self.orig_layer
weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight
if isinstance(layer, transformers.pytorch_utils.Conv1D):
weight = weight.t()
weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)
clip_min = getattr(layer, "awq_clip_min", None)
clip_max = getattr(layer, "awq_clip_max", None)
if clip_max is not None:
clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
clip_min = (
-clip_max
if clip_min is None
else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
)
if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:
weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)
else:
logger.warning_once(
f"SignRoundV2: ignoring AWQ clip range with shapes {tuple(clip_min.shape)}/{tuple(clip_max.shape)} "
f"incompatible with grouped weight shape {tuple(weight_reshape.shape)}."
)
return weight_reshape
class SignRoundOptimizedAsymWrapperLinear(SignRoundOptimizedWrapperLinear):
"""Algorithm-extension wrapper for asymmetric INT weights.
The plain asym path seeds the grid from the raw per-group min/max
(``scale=(max-min)/maxq``, ``zp=round(-min/scale)``), which at low bit-widths
is a poor init: the step is too coarse for bell-shaped groups and the integer
zero-point rounding clips one side. This wrapper instead anchors the grid to an
imatrix-weighted joint ``(scale, zp)`` search (NeUQI), expressed as per-group
``(tensor_min, tensor_max)`` margins that ``quant_tensor_asym`` reproduces
exactly, and -- unlike ``--enable_neuqi`` on the plain path -- keeps
``min_scale``/``max_scale`` tunable around that anchor (both shrink and expand).
"""
minmax_scale_bound = (0.5, 1.5)
# The asymmetric anchor search runs inline and does not stage init-scale inputs.
supports_batched_search = False
def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
layer = self.orig_layer
# Let the anchor below own the NeUQI search: disable the base wrapper's
# frozen-margin NeUQI branch so margins stay tunable and the search runs once.
enable_neuqi = self.enable_neuqi
self.enable_neuqi = False
try:
WrapperLinear._init_tuning_params_and_quant_func(self, defer_search=defer_search)
finally:
self.enable_neuqi = enable_neuqi
self.init_scale = None
self._asym_anchor = False
if self.weight_min is None or layer.bits >= 16 or isinstance(layer.group_size, tuple):
return
warm_start = getattr(layer, "_ar_warm_start", None)
if warm_start and warm_start.get("anchor_min") is not None and warm_start.get("anchor_max") is not None:
wmin = warm_start["anchor_min"].to(self.device, torch.float32)
wmax = warm_start["anchor_max"].to(self.device, torch.float32)
else:
from auto_round.data_type.neuqi import neuqi_anchor_margins
weight_reshape = self._prepare_init_scale_weight()
wmin, wmax = neuqi_anchor_margins(
weight_reshape,
layer.bits,
layer.group_size,
getattr(layer, "imatrix", None),
self.device,
sym=False,
)
if wmin.numel() != self.weight_min.numel() or wmax.numel() != self.weight_max.numel():
logger.warning_once("SignRoundV2 asym: anchor shape mismatch, falling back to min/max init.")
return
# Keep the anchor in fp32 so quant_tensor_asym re-derives the searched zp exactly.
self.weight_min = wmin.reshape(self.weight_min.shape).to(self.device, torch.float32)
self.weight_max = wmax.reshape(self.weight_max.shape).to(self.device, torch.float32)
self._asym_anchor = True
if hasattr(layer, "imatrix"):
del layer.imatrix
class SignRoundDQWrapperLinear(WrapperLinear):
minmax_scale_bound = (0.5, 1.5)
def __init__(self, *args, **kwargs) -> None:
if "enable_minmax_tuning" in kwargs:
logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")
kwargs["enable_minmax_tuning"] = False
super().__init__(*args, **kwargs)
self.prev_scale = None
self.prev_wmin = None
self.prev_d_scale = None
self.prev_d_wmin = None
def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
# the DQ scale search stays inline; the parameter keeps the base-class
# call signature uniform (staged batching applies to the optimized path)
super()._init_tuning_params_and_quant_func()
# The double-quant search path is data-dependent and kept un-compiled,
# while ``weight_quant_func`` is the compilable pure-math half.
self._is_dq_path = False
self._dq_kind = None
self.search_func = None
if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:
self._is_dq_path = True
if self.orig_layer.data_type == "int_asym_dq":
self.search_func = search_gguf_scale_min_asym
self.weight_quant_func = _dq_asym_qdq
self._dq_kind = "asym"
else:
self.search_func = search_gguf_scale_min_sym
self.weight_quant_func = _dq_sym_qdq
self._dq_kind = "sym"
elif self.orig_layer.sym:
self.weight_quant_func = quant_tensor_sym
else:
self.weight_quant_func = quant_tensor_asym
self.data_type = self.orig_layer.data_type
if self.enable_act_quant:
self.act_quant_func = (
quant_tensor_gguf_asym_dq
if self.orig_layer.act_data_type == "int_asym_dq"
else quant_tensor_gguf_sym_dq
)
if self.enable_torch_compile:
self.act_quant_func = compile_func(self.act_quant_func, self.device)
self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)
if self.enable_torch_compile:
self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
@torch.no_grad()
def _run_search(self, weight, v):
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
bits = self.orig_layer.bits
scale_dtype = self.orig_layer.scale_dtype
imatrix = getattr(self.orig_layer, "imatrix", None)
if self._dq_kind == "asym":
group_size = _dq_asym_group_size(bits)
t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
v_r = v
if isinstance(v, torch.Tensor):
v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
scale, wmin, d_scale, d_wmin = self.search_func(
t,
bits=bits,
scale_dtype=scale_dtype,
imatrix=imatrix,
split_num=1,
v=v_r,
)
return {
"scale": scale.clone(),
"wmin": wmin.clone(),
"d_scale": d_scale.clone(),
"d_wmin": d_wmin.clone(),
}
group_size = 16
super_group_size = 16
t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
ggml_type = f"q{bits}_k"
block_size, _ = GGML_QUANT_SIZES[ggml_type]
n_blocks = t.nelement() // block_size
t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)
v_r = v
if isinstance(v, torch.Tensor):
v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
super_bits = 6 if bits == 3 else 8
scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)
scale = scale.to(scale_dtype)
scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
scale = scale.unsqueeze(-1)
return {"scale": scale.clone(), "d_scale": d_scale.clone()}
def _qdq_weight(self, value, min_scale, max_scale):
if not self._is_dq_path:
return super()._qdq_weight(value, min_scale, max_scale)
if self.orig_layer.bits >= 16:
return self.orig_layer.weight, None, None
min_bound, max_bound = self.minmax_scale_bound
min_scale.data.clamp_(min_bound, max_bound)
max_scale.data.clamp_(min_bound, max_bound)
weight = self.orig_layer.weight
if weight.device.type == "meta":
weight = self.orig_layer.get_weight().to(self.device)
if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
weight = weight.t()
iter_v = getattr(self, "cur_iter", 0)
need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)
if need_search:
params = self._run_search(weight, value)
self.prev_scale = params["scale"]
self.prev_d_scale = params["d_scale"]
if self._dq_kind == "asym":
self.prev_wmin = params["wmin"]
self.prev_d_wmin = params["d_wmin"]
else:
params = {
"scale": self.prev_scale.detach(),
"d_scale": self.prev_d_scale.detach(),
}
if self._dq_kind == "asym":
params["wmin"] = self.prev_wmin.detach()
params["d_wmin"] = self.prev_d_wmin.detach()
bits = self.orig_layer.bits
if self._dq_kind == "asym":
group_size = _dq_asym_group_size(bits)
weight_q = self.weight_quant_func(
weight,
params["scale"],
params["wmin"],
bits,
group_size,
v=value,
)
scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}
else:
weight_q = self.weight_quant_func(
weight,
params["scale"],
bits,
v=value,
)
scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))
weight_q = weight_q.to(weight.dtype)
if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
weight_q = weight_q.t()
return weight_q, scale_out, zp_out
@register_pipeline_member(SignRoundV2Config)
class SignRoundV2Quantizer(SignRoundQuantizer):
"""SignRound variant using the open algorithm-extension path in the new architecture."""
calibration_stats = {"imatrix": "sum"}
def __init__(self, config: SignRoundConfig) -> None:
super().__init__(config)
self._use_outlier_suppressed_loss = False
logger.info("using algorithm extension for quantization.")
def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:
"""Model-level setup: initialise scheme-dependent state once before block iteration."""
super().prepare_run(composer=composer)
if (
self.scheme.sym
and self.scheme.super_group_size is None
and (
self.scheme.data_type.startswith("int")
or self.scheme.data_type.startswith("mx")
or self.scheme.data_type.startswith("nv")
)
):
if self.scheme.bits > 2 and not (
self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")
):
logger.warning_once(
"algorithm extension has only undergone limited validation on "
"W2A16,INT4, MXFP4 and NVFP4; use with caution."
)
if self.scheme.act_bits <= 4 or self.scheme.bits < 4:
self._use_outlier_suppressed_loss = True
else:
self._use_outlier_suppressed_loss = False
self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")
elif (
not self.scheme.sym
and self.scheme.super_group_size is None
and self.scheme.data_type == "int"
and self.scheme.bits < 16
):
# Asym INT: searched (scale, zp) anchor + tunable margins, and the same
# outlier-suppressed block loss the sym path uses at low bits.
self._use_outlier_suppressed_loss = self.scheme.act_bits <= 4 or self.scheme.bits < 4
self.wrapper_block = _named_wrapper_block(SignRoundOptimizedAsymWrapperLinear, "wrapper_block")
if self.scheme.data_type.endswith("dq"):
self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")
def can_compile_block_forward(self):
# torch.compile on the whole block forward is only reliable on torch >= 2.13
from packaging.version import Version
return Version(torch.__version__.split("+")[0]) >= Version("2.13")
def _get_loss(
self,
pred_output: torch.Tensor,
ref_output: torch.Tensor,
indices: torch.Tensor,
mse_loss: Callable,
device: str | torch.device = "cpu",
valid_token_mask: list[torch.Tensor] | None = None,
):
if self._use_outlier_suppressed_loss:
loss_diff = torch.abs(pred_output - ref_output)
flat_diff = loss_diff.view(-1)
topk = max(1, int(flat_diff.numel() / 1000))
_, top_indices = torch.topk(torch.abs(flat_diff), topk)
mask = torch.zeros_like(flat_diff, dtype=torch.bool)
mask[top_indices] = True
mask = (~mask).view_as(loss_diff)
autocast_ctx = (
autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype) if self.amp else nullcontext()
)
if valid_token_mask:
tmp_attention_mask = [valid_token_mask[i] for i in indices]
tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)
tmp_attention_mask.unsqueeze_(-1)
with autocast_ctx:
return torch.mean(
(
torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32))
* tmp_attention_mask
* mask
)
** 2
)
with autocast_ctx:
return torch.mean((torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32)) * mask) ** 2)
return super()._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
def register_fp_input_forward_hooks(self, block):
"""Register FP-input hooks: imatrix."""
handles = super().register_fp_input_forward_hooks(block)
if not self._is_wint4aint4():
handles.extend(self._register_imatrix_hooks(block))
return handles
def _is_wint4aint4(self):
return (
"int4" in self.scheme.act_data_type or ("int" in self.scheme.act_data_type and self.scheme.act_bits == 4)
) and ("int4" in self.scheme.data_type or ("int" in self.scheme.data_type and self.scheme.bits == 4))
def _register_imatrix_hooks(self, model):
def collect_imatrix(module, input, output):
input = input[0] if isinstance(input, (tuple, list)) else input
flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
if not hasattr(module, "imatrix"):
module.imatrix = squared
return
module.imatrix += squared.to(module.imatrix.device)
handles = []
for _, module in model.named_modules():
if isinstance(module, SUPPORTED_LAYER_TYPES) and check_to_quantized(module):
handles.append(module.register_forward_hook(collect_imatrix))
return handles