Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

27.6 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math
from collections.abc import Callable
from contextlib import nullcontext
from functools import partial
from typing import TYPE_CHECKING

import torch
import transformers
from torch import autocast

from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer

if TYPE_CHECKING:
    from auto_round.algorithms.composer import AlgorithmComposer

from auto_round import envs
from auto_round.algorithms.registry import register_pipeline_member
from auto_round.data_type.gguf import (
    double_quant_tensor_sym_rtn,
    quant_tensor_gguf_asym_dq,
    quant_tensor_gguf_sym_dq,
    search_gguf_scale_min_asym,
    search_gguf_scale_min_sym,
)
from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym
from auto_round.data_type.utils import (
    get_optimized_quant_func,
    reshape_imatrix_for_weight,
    reshape_pad_tensor_by_group_size,
    resolve_optimized_init_scale_fn,
    revert_tensor_by_pad,
    round_ste,
    search_optimized_init_scale,
)
from auto_round.logger import logger
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal
from auto_round.wrapper import WrapperLinear, wrapper_block


def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):
    """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""
    tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
    orig_dtype = tensor.dtype
    tensor = tensor.to(torch.float32)
    maxq = int(2.0**bits) - 1
    inverse_scale = get_reciprocal(scale)
    int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
    qdq = (scale * int_w - wmin).to(orig_dtype)
    qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
    return qdq


def _dq_sym_qdq(tensor, scale, bits, v=0):
    """Pure sym double-quant qdq math given precomputed scale (compilable)."""
    from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K

    group_size = 16
    super_group_size = 16
    maxq = int(2.0 ** (bits - 1))
    tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
    orig_dtype = tensor.dtype
    tensor = tensor.to(torch.float32)
    ggml_type = f"q{bits}_k"
    block_size, _ = GGML_QUANT_SIZES[ggml_type]
    n_blocks = tensor.nelement() // block_size
    tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
    if isinstance(v, torch.Tensor):
        v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
        v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
    else:
        v_r = v
    zp = torch.full_like(scale, maxq)
    inverse_scale = get_reciprocal(scale)
    int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq
    qdq = (scale * (int_w - zp)).to(orig_dtype)
    qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)
    return qdq


def _named_wrapper_block(wrapper_cls, name: str):
    wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)
    wrapped.__name__ = name
    return wrapped


def _dq_asym_group_size(bits: int) -> int:
    """Group size used by the GGUF asym double-quant path (16 for 2-bit, else 32)."""
    return 16 if bits == 2 else 32


class SignRoundOptimizedWrapperLinear(WrapperLinear):
    minmax_scale_bound = (0.1, 10.0)  # TODO widened from 2.0 for low-bit QAD; verify correctness
    # Opt-in protocol for wrapper_block: the init-scale search is weight-local
    # and row-independent, so same-shape modules can be searched as one stacked
    # batch (bit-identical per module, far fewer python/launch round-trips).
    supports_batched_search = True

    def _init_tuning_params_and_quant_func(self, defer_search: bool = False) -> None:
        super()._init_tuning_params_and_quant_func(defer_search=defer_search)

        # Log/geometric min-max tuning (opt-in): re-anchor the tunable min/max
        # scale coefficients to log space so each SignSGD step is a symmetric
        # multiplicative move. Only engages when the coefficients are actually
        # trainable (``nn.Parameter``); constant/frozen coefficients keep the
        # plain linear path (a constant 1.0 must not be passed through exp()).
        self._use_log_minmax = False
        _min_tunable = isinstance(getattr(self, "min_scale", None), torch.nn.Parameter)
        _max_tunable = isinstance(getattr(self, "max_scale", None), torch.nn.Parameter)
        if envs.AR_ENABLE_LOG_MINMAX and _min_tunable and _max_tunable:
            self._use_log_minmax = True
        warm_start = getattr(self.orig_layer, "_ar_warm_start", None) or {}
        saved_log = warm_start.get("minmax_parameterization") == "log"
        with torch.no_grad():
            for key in ("min_scale", "max_scale"):
                param = getattr(self, key)
                if not isinstance(param, torch.nn.Parameter):
                    continue
                saved = warm_start.get(key)
                compatible = saved is not None and saved.shape == param.shape
                if self._use_log_minmax:
                    if not compatible:
                        # theta = 0 <-> coeff = exp(0) = 1.0
                        param.zero_()
                    elif not saved_log:
                        param.clamp_min_(torch.finfo(param.dtype).tiny).log_()
                elif compatible and saved_log:
                    param.exp_()

        layer = self.orig_layer
        data_type = layer.data_type

        # Warm-start reuse: a prior pass (SignRound -> QAD chaining) may have
        # stashed the searched ``init_scale`` on this layer. Reuse it verbatim so
        # the second pass tunes on the exact same scale grid (re-searching here
        # would differ because the imatrix was consumed/deleted in the first pass).
        warm_start = getattr(layer, "_ar_warm_start", None)
        warm_init_scale = warm_start.get("init_scale") if warm_start else None

        weight_reshape = self._prepare_init_scale_weight()
        # staged deferral keeps only the RAW column imatrix (or None): the
        # expanded full-size copy would be pinned per module for the whole
        # wrap phase -- N x weight bytes -- while the batched call needs it
        # only transiently per chunk (bounded by the batch cap)
        imatrix_raw = getattr(layer, "imatrix", None)
        imatrix = None if defer_search else reshape_imatrix_for_weight(imatrix_raw, weight_reshape, layer.group_size)

        self.weight_quant_func = get_optimized_quant_func(data_type)
        search_fn = resolve_optimized_init_scale_fn(data_type, self.q_scale_thresh)
        if search_fn is None or self.weight_quant_func is None:
            raise ValueError(
                f"SignRound optimized path does not support data_type={data_type!r}; "
                "expected a symmetric int / mx / nv type."
            )
        self.data_type = data_type
        self.init_scale = None
        self._deferred_search_inputs = None
        if warm_init_scale is not None:
            # Warm-started from a prior pass: reuse the exact scale, skip any search.
            self.init_scale = warm_init_scale.to(self.device)
        elif defer_search:
            # staged for run_batched_wrap_search: the exact callable the
            # per-module path would invoke travels with the module
            self._deferred_search_inputs = (
                weight_reshape,
                data_type,
                layer.bits,
                imatrix_raw,
                self.q_scale_thresh,
                search_fn,
            )
        else:
            self.init_scale = search_fn(weight_reshape, layer.bits, imatrix)

        if hasattr(layer, "imatrix"):
            del layer.imatrix
        if self.enable_torch_compile:
            self.weight_quant_func = compile_func(self.weight_quant_func, self.device)

    def _prepare_minmax_scale(self, min_scale, max_scale):
        """Return the multiplicative coefficients the quant func consumes.

        In log mode the stored parameters are ``theta`` (SignSGD tunes them with
        symmetric fixed steps); clamp them to the symmetric log bound and return
        ``exp(theta)`` so the quant function still receives plain multiplicative
        coefficients, while ``+lr``/``-lr`` steps become ``xe^lr``/``/e^lr``.
        Gradients flow back through ``exp`` to the ``theta`` parameters.
        """
        if not getattr(self, "_use_log_minmax", False):
            return super()._prepare_minmax_scale(min_scale, max_scale)
        # The shared ``minmax_scale_bound`` is defined in Euclidean (coefficient)
        # space; convert it to the log/theta clamp range here so both the linear
        # and log paths read from one bound. ``log`` is undefined at <= 0, so when
        # the linear bound starts at 0 fall back to a multiplicatively symmetric
        # lower end (the reciprocal of the upper coefficient), keeping shrink/expand
        # equidistant from the identity.
        coeff_low, coeff_high = self.minmax_scale_bound
        if coeff_low <= 0.0:
            coeff_low = 1.0 / coeff_high
        low, high = math.log(coeff_low), math.log(coeff_high)
        min_scale.data.clamp_(low, high)
        max_scale.data.clamp_(low, high)
        return torch.exp(min_scale), torch.exp(max_scale)

    def _run_deferred_search_now(self) -> None:
        """Run the staged search per-module (fallback when batching is off)."""
        # Warm-started wrappers (QAD chaining) resolve ``init_scale`` directly and
        # stage no search inputs; there is nothing to run for them.
        if self._deferred_search_inputs is None:
            return
        weight, _data_type, bits, imatrix_raw, _thresh, search_fn = self._deferred_search_inputs
        imatrix = reshape_imatrix_for_weight(imatrix_raw, weight, self.orig_layer.group_size)
        self.init_scale = search_fn(weight, bits, imatrix)
        self._deferred_search_inputs = None

    def finalize_batched_search(self, init_scale: "torch.Tensor | list[torch.Tensor]") -> None:
        """Adopt a search result computed on a stacked same-shape batch."""
        self.init_scale = init_scale
        self._deferred_search_inputs = None

    def _prepare_init_scale_weight(self) -> torch.Tensor:
        """Return the group-reshaped weight that seeds the init-scale search.

        Reproduces the layout the quant func sees at tuning time: the (optionally
        transposed) FP weight grouped to ``[-1, group_size]``. When AWQ ran in
        ``clip_as_init`` mode and stored a per-group clip range on the layer, the
        weight is clamped to that range first so the searched ``init_scale``
        already reflects the AWQ clip (``max_scale`` then tunes a coefficient on
        top of it).
        """
        layer = self.orig_layer
        weight = layer.get_weight() if hasattr(layer, "get_weight") else layer.weight
        if isinstance(layer, transformers.pytorch_utils.Conv1D):
            weight = weight.t()
        weight_reshape, _, _ = reshape_pad_tensor_by_group_size(weight.data, layer.group_size)

        clip_min = getattr(layer, "awq_clip_min", None)
        clip_max = getattr(layer, "awq_clip_max", None)
        if clip_max is not None:
            clip_max = clip_max.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
            clip_min = (
                -clip_max
                if clip_min is None
                else clip_min.reshape(-1, 1).to(weight_reshape.device, weight_reshape.dtype)
            )
            if clip_max.shape[0] == weight_reshape.shape[0] and clip_min.shape[0] == weight_reshape.shape[0]:
                weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)
            else:
                logger.warning_once(
                    f"SignRoundV2: ignoring AWQ clip range with shapes {tuple(clip_min.shape)}/{tuple(clip_max.shape)} "
                    f"incompatible with grouped weight shape {tuple(weight_reshape.shape)}."
                )
        return weight_reshape


class SignRoundOptimizedAsymWrapperLinear(SignRoundOptimizedWrapperLinear):
    """Algorithm-extension wrapper for asymmetric INT weights.

    The plain asym path seeds the grid from the raw per-group min/max
    (``scale=(max-min)/maxq``, ``zp=round(-min/scale)``), which at low bit-widths
    is a poor init: the step is too coarse for bell-shaped groups and the integer
    zero-point rounding clips one side. This wrapper instead anchors the grid to an
    imatrix-weighted joint ``(scale, zp)`` search (NeUQI), expressed as per-group
    ``(tensor_min, tensor_max)`` margins that ``quant_tensor_asym`` reproduces
    exactly, and -- unlike ``--enable_neuqi`` on the plain path -- keeps
    ``min_scale``/``max_scale`` tunable around that anchor (both shrink and expand).
    """

    minmax_scale_bound = (0.5, 1.5)
    # The asymmetric anchor search runs inline and does not stage init-scale inputs.
    supports_batched_search = False

    def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
        layer = self.orig_layer
        # Let the anchor below own the NeUQI search: disable the base wrapper's
        # frozen-margin NeUQI branch so margins stay tunable and the search runs once.
        enable_neuqi = self.enable_neuqi
        self.enable_neuqi = False
        try:
            WrapperLinear._init_tuning_params_and_quant_func(self, defer_search=defer_search)
        finally:
            self.enable_neuqi = enable_neuqi
        self.init_scale = None
        self._asym_anchor = False

        if self.weight_min is None or layer.bits >= 16 or isinstance(layer.group_size, tuple):
            return

        warm_start = getattr(layer, "_ar_warm_start", None)
        if warm_start and warm_start.get("anchor_min") is not None and warm_start.get("anchor_max") is not None:
            wmin = warm_start["anchor_min"].to(self.device, torch.float32)
            wmax = warm_start["anchor_max"].to(self.device, torch.float32)
        else:
            from auto_round.data_type.neuqi import neuqi_anchor_margins

            weight_reshape = self._prepare_init_scale_weight()
            wmin, wmax = neuqi_anchor_margins(
                weight_reshape,
                layer.bits,
                layer.group_size,
                getattr(layer, "imatrix", None),
                self.device,
                sym=False,
            )
        if wmin.numel() != self.weight_min.numel() or wmax.numel() != self.weight_max.numel():
            logger.warning_once("SignRoundV2 asym: anchor shape mismatch, falling back to min/max init.")
            return
        # Keep the anchor in fp32 so quant_tensor_asym re-derives the searched zp exactly.
        self.weight_min = wmin.reshape(self.weight_min.shape).to(self.device, torch.float32)
        self.weight_max = wmax.reshape(self.weight_max.shape).to(self.device, torch.float32)
        self._asym_anchor = True
        if hasattr(layer, "imatrix"):
            del layer.imatrix


class SignRoundDQWrapperLinear(WrapperLinear):
    minmax_scale_bound = (0.5, 1.5)

    def __init__(self, *args, **kwargs) -> None:
        if "enable_minmax_tuning" in kwargs:
            logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")
            kwargs["enable_minmax_tuning"] = False
        super().__init__(*args, **kwargs)
        self.prev_scale = None
        self.prev_wmin = None
        self.prev_d_scale = None
        self.prev_d_wmin = None

    def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
        # the DQ scale search stays inline; the parameter keeps the base-class
        # call signature uniform (staged batching applies to the optimized path)
        super()._init_tuning_params_and_quant_func()
        # The double-quant search path is data-dependent and kept un-compiled,
        # while ``weight_quant_func`` is the compilable pure-math half.
        self._is_dq_path = False
        self._dq_kind = None
        self.search_func = None
        if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:
            self._is_dq_path = True
            if self.orig_layer.data_type == "int_asym_dq":
                self.search_func = search_gguf_scale_min_asym
                self.weight_quant_func = _dq_asym_qdq
                self._dq_kind = "asym"
            else:
                self.search_func = search_gguf_scale_min_sym
                self.weight_quant_func = _dq_sym_qdq
                self._dq_kind = "sym"
        elif self.orig_layer.sym:
            self.weight_quant_func = quant_tensor_sym
        else:
            self.weight_quant_func = quant_tensor_asym
        self.data_type = self.orig_layer.data_type
        if self.enable_act_quant:
            self.act_quant_func = (
                quant_tensor_gguf_asym_dq
                if self.orig_layer.act_data_type == "int_asym_dq"
                else quant_tensor_gguf_sym_dq
            )
            if self.enable_torch_compile:
                self.act_quant_func = compile_func(self.act_quant_func, self.device)
            self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)
        if self.enable_torch_compile:
            self.weight_quant_func = compile_func(self.weight_quant_func, self.device)

    @torch.no_grad()
    def _run_search(self, weight, v):
        from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K

        bits = self.orig_layer.bits
        scale_dtype = self.orig_layer.scale_dtype
        imatrix = getattr(self.orig_layer, "imatrix", None)

        if self._dq_kind == "asym":
            group_size = _dq_asym_group_size(bits)
            t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
            v_r = v
            if isinstance(v, torch.Tensor):
                v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
            scale, wmin, d_scale, d_wmin = self.search_func(
                t,
                bits=bits,
                scale_dtype=scale_dtype,
                imatrix=imatrix,
                split_num=1,
                v=v_r,
            )
            return {
                "scale": scale.clone(),
                "wmin": wmin.clone(),
                "d_scale": d_scale.clone(),
                "d_wmin": d_wmin.clone(),
            }

        group_size = 16
        super_group_size = 16
        t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)
        ggml_type = f"q{bits}_k"
        block_size, _ = GGML_QUANT_SIZES[ggml_type]
        n_blocks = t.nelement() // block_size
        t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)
        v_r = v
        if isinstance(v, torch.Tensor):
            v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)
            v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)
        super_bits = 6 if bits == 3 else 8
        scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)
        scale = scale.to(scale_dtype)
        scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
        scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
        scale = scale.unsqueeze(-1)
        return {"scale": scale.clone(), "d_scale": d_scale.clone()}

    def _qdq_weight(self, value, min_scale, max_scale):
        if not self._is_dq_path:
            return super()._qdq_weight(value, min_scale, max_scale)

        if self.orig_layer.bits >= 16:
            return self.orig_layer.weight, None, None
        min_bound, max_bound = self.minmax_scale_bound
        min_scale.data.clamp_(min_bound, max_bound)
        max_scale.data.clamp_(min_bound, max_bound)
        weight = self.orig_layer.weight
        if weight.device.type == "meta":
            weight = self.orig_layer.get_weight().to(self.device)
        if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
            weight = weight.t()

        iter_v = getattr(self, "cur_iter", 0)
        need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)
        if need_search:
            params = self._run_search(weight, value)
            self.prev_scale = params["scale"]
            self.prev_d_scale = params["d_scale"]
            if self._dq_kind == "asym":
                self.prev_wmin = params["wmin"]
                self.prev_d_wmin = params["d_wmin"]
        else:
            params = {
                "scale": self.prev_scale.detach(),
                "d_scale": self.prev_d_scale.detach(),
            }
            if self._dq_kind == "asym":
                params["wmin"] = self.prev_wmin.detach()
                params["d_wmin"] = self.prev_d_wmin.detach()

        bits = self.orig_layer.bits
        if self._dq_kind == "asym":
            group_size = _dq_asym_group_size(bits)
            weight_q = self.weight_quant_func(
                weight,
                params["scale"],
                params["wmin"],
                bits,
                group_size,
                v=value,
            )
            scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
            zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}
        else:
            weight_q = self.weight_quant_func(
                weight,
                params["scale"],
                bits,
                v=value,
            )
            scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}
            zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))

        weight_q = weight_q.to(weight.dtype)
        if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
            weight_q = weight_q.t()
        return weight_q, scale_out, zp_out


@register_pipeline_member(SignRoundV2Config)
class SignRoundV2Quantizer(SignRoundQuantizer):
    """SignRound variant using the open algorithm-extension path in the new architecture."""

    calibration_stats = {"imatrix": "sum"}

    def __init__(self, config: SignRoundConfig) -> None:
        super().__init__(config)
        self._use_outlier_suppressed_loss = False
        logger.info("using algorithm extension for quantization.")

    def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:
        """Model-level setup: initialise scheme-dependent state once before block iteration."""
        super().prepare_run(composer=composer)

        if (
            self.scheme.sym
            and self.scheme.super_group_size is None
            and (
                self.scheme.data_type.startswith("int")
                or self.scheme.data_type.startswith("mx")
                or self.scheme.data_type.startswith("nv")
            )
        ):
            if self.scheme.bits > 2 and not (
                self.scheme.data_type.startswith("mx") or self.scheme.data_type.startswith("nv")
            ):
                logger.warning_once(
                    "algorithm extension has only undergone limited validation on "
                    "W2A16,INT4, MXFP4 and NVFP4; use with caution."
                )
            if self.scheme.act_bits <= 4 or self.scheme.bits < 4:
                self._use_outlier_suppressed_loss = True
            else:
                self._use_outlier_suppressed_loss = False
            self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")
        elif (
            not self.scheme.sym
            and self.scheme.super_group_size is None
            and self.scheme.data_type == "int"
            and self.scheme.bits < 16
        ):
            # Asym INT: searched (scale, zp) anchor + tunable margins, and the same
            # outlier-suppressed block loss the sym path uses at low bits.
            self._use_outlier_suppressed_loss = self.scheme.act_bits <= 4 or self.scheme.bits < 4
            self.wrapper_block = _named_wrapper_block(SignRoundOptimizedAsymWrapperLinear, "wrapper_block")

        if self.scheme.data_type.endswith("dq"):
            self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")

    def can_compile_block_forward(self):
        # torch.compile on the whole block forward is only reliable on torch >= 2.13
        from packaging.version import Version

        return Version(torch.__version__.split("+")[0]) >= Version("2.13")

    def _get_loss(
        self,
        pred_output: torch.Tensor,
        ref_output: torch.Tensor,
        indices: torch.Tensor,
        mse_loss: Callable,
        device: str | torch.device = "cpu",
        valid_token_mask: list[torch.Tensor] | None = None,
    ):
        if self._use_outlier_suppressed_loss:
            loss_diff = torch.abs(pred_output - ref_output)
            flat_diff = loss_diff.view(-1)
            topk = max(1, int(flat_diff.numel() / 1000))
            _, top_indices = torch.topk(torch.abs(flat_diff), topk)
            mask = torch.zeros_like(flat_diff, dtype=torch.bool)
            mask[top_indices] = True
            mask = (~mask).view_as(loss_diff)

            autocast_ctx = (
                autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype) if self.amp else nullcontext()
            )
            if valid_token_mask:
                tmp_attention_mask = [valid_token_mask[i] for i in indices]
                tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)
                tmp_attention_mask.unsqueeze_(-1)
                with autocast_ctx:
                    return torch.mean(
                        (
                            torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32))
                            * tmp_attention_mask
                            * mask
                        )
                        ** 2
                    )

            with autocast_ctx:
                return torch.mean((torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32)) * mask) ** 2)
        return super()._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)

    def register_fp_input_forward_hooks(self, block):
        """Register FP-input hooks: imatrix."""
        handles = super().register_fp_input_forward_hooks(block)
        if not self._is_wint4aint4():
            handles.extend(self._register_imatrix_hooks(block))
        return handles

    def _is_wint4aint4(self):
        return (
            "int4" in self.scheme.act_data_type or ("int" in self.scheme.act_data_type and self.scheme.act_bits == 4)
        ) and ("int4" in self.scheme.data_type or ("int" in self.scheme.data_type and self.scheme.bits == 4))

    def _register_imatrix_hooks(self, model):
        def collect_imatrix(module, input, output):
            input = input[0] if isinstance(input, (tuple, list)) else input
            flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
            squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)

            if not hasattr(module, "imatrix"):
                module.imatrix = squared
                return
            module.imatrix += squared.to(module.imatrix.device)

        handles = []
        for _, module in model.named_modules():
            if isinstance(module, SUPPORTED_LAYER_TYPES) and check_to_quantized(module):
                handles.append(module.register_forward_hook(collect_imatrix))
        return handles