Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

54.5 KB
Raw Blame
# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math
from contextlib import contextmanager
from math import ceil

import torch
import transformers
from torch.functional import F

import auto_round.envs as envs
from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import (
    SUPPORTED_LAYER_TYPES,
    check_to_quantized,
    compile_func,
    deepspeed_exists,
    set_module,
)

if deepspeed_exists:
    from deepspeed import comm as dist
    from deepspeed.module_inject import LinearAllreduce, LinearLayer


def get_scale_shape(weight, group_size):
    """Computes the shape of the scale tensor for quantization based on the weight tensor and group size.

    Args:
      weight (torch.Tensor): The weight tensor of the layer.
      group_size (int or tuple): The size of the groups for quantization.

    Returns:
      The shape of the scale tensor to be used for quantization.
    """
    if isinstance(group_size, tuple):
        assert len(weight.shape) == len(group_size), f"Expected group_size is {len(weight.shape)}D but get {group_size}"
        return (weight.shape[0] // group_size[0], weight.shape[1] // group_size[1])
    if group_size == 0:
        return 1
    elif group_size == -1 or weight.shape[1] < group_size:
        shape = weight.shape[0]
    else:
        shape = weight.shape[0] * ceil(weight.shape[1] / group_size)

    return shape


def _restore_warm_start_param(orig_layer, name, param):
    """Restore a compatible parameter without replacing its optimizer identity."""
    saved = getattr(orig_layer, "_ar_warm_start", {}).get(name)
    if saved is not None and saved.shape == param.shape:
        with torch.no_grad():
            param.copy_(saved.to(device=param.device, dtype=param.dtype))


class WrapperLinear(torch.nn.Module):
    """A wrapper for linear/conv1d layers to enable quantization and tuning.

    This module wraps an existing linear or conv1d layer and provides additional functionality
    for quantization, parameter tuning, and activation/bias normalization.

    Args:
        orig_layer (torch.nn.Module): The original layer to be wrapped (linear or conv1d).
        enable_minmax_tuning (bool): Whether to enable min-max scale tuning.
        enable_norm_bias_tuning (bool): Whether to enable normalization and tuning of the bias term.
        enable_torch_compile (bool): Whether to enable torch compilation.
        device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').
    """

    minmax_scale_bound = (0.0, 1.0)

    def __init__(
        self,
        orig_layer,
        enable_minmax_tuning=True,
        enable_norm_bias_tuning=False,
        device="cpu",
        enable_round_tuning=True,
        enable_torch_compile=True,
        disable_opt_rtn=True,
        enable_neuqi=False,
        **kwargs,
    ):
        """Initializes the WrapperLinear module.

        Args:
            orig_layer (torch.nn.Module): The original layer to wrap.
            enable_minmax_tuning (bool): Whether to enable min-max scale tuning.
            enable_norm_bias_tuning (bool): Whether to enable normalization and tuning for the bias term.
            device (str): The computation device, such as 'cpu' or 'cuda'.
        """
        super().__init__()
        self.orig_layer = orig_layer
        self.orig_layer.iters = kwargs.pop("iters", 200)
        self.disable_opt_rtn = disable_opt_rtn
        self.enable_neuqi = enable_neuqi
        self.output_device = device
        self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
        self.enable_minmax_tuning = enable_minmax_tuning
        self.enable_round_tuning = enable_round_tuning
        self.enable_torch_compile = enable_torch_compile
        self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
        self.enable_act_quant = self.orig_layer.act_bits <= 8
        self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
        self.input_global_scale = getattr(self.orig_layer, "input_global_scale", None)
        if needs_nvfp_global_scale(self.orig_layer.data_type) and self.weight_global_scale is None:
            from auto_round.data_type.nvfp import calculate_gparam_by_dtype

            weight_global_scale = calculate_gparam_by_dtype(
                self.orig_layer.weight, self.orig_layer.data_type, self.orig_layer.group_size
            )
            self.weight_global_scale = weight_global_scale
            self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
        if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
            self.q_scale_thresh = 1e-8
        else:
            self.q_scale_thresh = 1e-5
        # consumed here so subclasses never see it in **kwargs; forwarded to the
        # search init (staging for batched same-shape wrap searches)
        self._init_tuning_params_and_quant_func(defer_search=bool(kwargs.pop("defer_search", False)))
        if deepspeed_exists:
            if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):
                self.orig_forward = self.linear_forward
            elif type(self.orig_layer) == LinearAllreduce:
                self.orig_forward = self.all_reduce_linear_forward
                self.mp_group = self.orig_layer.mp_group
            else:
                self.orig_forward = self.conv1d_forward
        else:
            self.orig_forward = self.linear_forward if type(self.orig_layer) == torch.nn.Linear else self.conv1d_forward

    @property
    def weight(self):
        return self.orig_layer.weight

    @property
    def bias(self):
        return self.orig_layer.bias

    def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
        """Initializes tuning parameters and quantization functions.

        This method sets up required parameters and functions for weight quantization,
        activation quantization, and bias/normalization.
        """
        self.params = {}
        p_dtype = torch.float32  ##parameter dtype

        orig_layer = self.orig_layer
        orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
        if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
            orig_weight = orig_weight.t()
        weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)

        if self.enable_round_tuning:
            self.weight_min = (
                torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)
                if isinstance(orig_layer.group_size, tuple)
                else torch.clamp(weight_reshape.min(1)[0], max=0)
            )
            self.weight_max = (
                torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)
                if isinstance(orig_layer.group_size, tuple)
                else torch.clamp(weight_reshape.max(1)[0], min=0)
            )
        else:
            self.weight_min = None
            self.weight_max = None
        # AWQ clip-as-init: cap the tunable weight range to the per-group clip
        # range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
        # This initializes the range used by quant_tensor_sym/asym, leaving
        # min_scale/max_scale to tune a coefficient on top. Only the standard
        # (non-tuple) group layout maps onto weight_min/weight_max here.
        awq_clip_min = getattr(orig_layer, "awq_clip_min", None)
        awq_clip_max = getattr(orig_layer, "awq_clip_max", None)
        if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):
            clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
            if awq_clip_min is None:
                clip_min_flat = -clip_max_flat
            else:
                clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)
            if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():
                self.weight_max = torch.minimum(self.weight_max, clip_max_flat)
                self.weight_min = torch.maximum(self.weight_min, clip_min_flat)
        # NeUQI frozen init (enable_neuqi on the tuning path): anchor
        # the grid to the searched (scale, zp) instead of the raw per-group
        # min/max. Placement mirrors the AWQ clip-as-init above; unsupported
        # layouts (tuple group sizes, >=16 bits, non-int data types) keep the
        # status-quo min/max grid. The zero-shot path needs no anchor -- its
        # search runs inside the quant function dispatch itself.
        self._neuqi_frozen_margins = False
        if (
            self.enable_neuqi
            and self.enable_round_tuning
            and weight_reshape is not None
            and self.weight_min is not None
            and self.orig_layer.bits < 16
            and not isinstance(orig_layer.group_size, tuple)
            and self.orig_layer.data_type == "int"
        ):
            if getattr(orig_layer, "awq_clip_max", None) is not None:
                logger.warning_once(
                    "enable_neuqi anchors the tuning grid to its own search result, which "
                    "overrides the AWQ clip-as-init range on this layer; the clip is ignored."
                )
            if defer_search:
                # staged for the batched anchor driver; the margins keep the
                # plain min/max until the batch pass applies the searched ones
                self._deferred_neuqi_anchor = (
                    weight_reshape,
                    self.orig_layer.bits,
                    orig_layer.group_size,
                    getattr(orig_layer, "imatrix", None),
                    bool(getattr(orig_layer, "sym", True)),
                )
                self._neuqi_frozen_margins = True
            else:
                from auto_round.data_type.neuqi import neuqi_anchor_margins

                _wmin, _wmax = neuqi_anchor_margins(
                    weight_reshape,
                    self.orig_layer.bits,
                    orig_layer.group_size,
                    getattr(orig_layer, "imatrix", None),
                    self.device,
                    sym=bool(getattr(orig_layer, "sym", True)),
                )
                self._neuqi_frozen_margins = True
                self.weight_min = _wmin.to(self.weight_min.dtype)
                self.weight_max = _wmax.to(self.weight_max.dtype)
        self._init_params(
            "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16
        )
        # Min-max scale initialization
        shape = get_scale_shape(orig_weight, orig_layer.group_size)
        _margins_tunable = (self.enable_minmax_tuning and self.orig_layer.bits < 16) and (
            not self._neuqi_frozen_margins
        )
        self._init_params("min_scale", p_dtype, shape, 1.0, _margins_tunable)
        self._init_params("max_scale", p_dtype, shape, 1.0, _margins_tunable)

        # Midrise int (v1 path): the absmax-derived scale (absmax / (maxq-0.5)) is far from the
        # MSE optimum (~0.5x at W2) and the default [0, 1] max_scale bound with SignSGD can barely
        # shrink that far. Seed ``init_scale`` with the same imatrix-weighted search used by the v2
        # path and let ``max_scale`` tune a coefficient around it in both directions.
        # (SignRoundOptimizedWrapperLinear re-runs the search and keeps its own bound.)
        if (
            str(self.orig_layer.data_type).startswith("int_midrise")
            and type(self).minmax_scale_bound == WrapperLinear.minmax_scale_bound  # plain v1 wrapper only
            and self.enable_round_tuning
            and self.orig_layer.bits < 16
            and not isinstance(orig_layer.group_size, tuple)
        ):
            from auto_round.data_type.utils import reshape_imatrix_for_weight, search_optimized_init_scale

            warm_start = getattr(orig_layer, "_ar_warm_start", None)
            if warm_start and warm_start.get("init_scale") is not None:
                self.init_scale = warm_start["init_scale"].to(self.device)
            else:
                w_r = weight_reshape.to(self.device, torch.float32)
                imatrix = reshape_imatrix_for_weight(getattr(orig_layer, "imatrix", None), w_r, orig_layer.group_size)
                self.init_scale = search_optimized_init_scale(
                    w_r, orig_layer.data_type, orig_layer.bits, imatrix.to(w_r.device), self.q_scale_thresh
                )
            self.minmax_scale_bound = (0.5, 1.5)

        # Warm-start: when a prior block-wise pass (e.g. the SignRound -> QAD
        # chaining) stashed its tuned ``value``/``min_scale``/``max_scale`` on
        # this layer via ``store_warm_start_params_and_unwrap``, seed the freshly
        # created tuning parameters with them so this pass continues from that
        # state instead of starting from RTN (value=0, scale coeff=1).
        warm_start = getattr(orig_layer, "_ar_warm_start", None)
        if warm_start:
            for _key in ("value", "min_scale", "max_scale"):
                saved = warm_start.get(_key)
                param = self.params.get(_key)
                if saved is None or param is None:
                    continue
                saved = saved.to(device=param.device, dtype=param.dtype)
                if saved.shape == param.shape:
                    param.data.copy_(saved)
                else:
                    logger.warning_once(
                        f"warm-start param '{_key}' shape {tuple(saved.shape)} does not match "
                        f"{tuple(param.shape)}; keeping the default initialization."
                    )

        self.weight_quant_func, self.data_type = get_quant_func(
            orig_layer.data_type,
            orig_layer.bits,
            orig_layer.sym,
            self.disable_opt_rtn,
            orig_layer.group_size,
            iters=orig_layer.iters,
            enable_neuqi=self.enable_neuqi,
        )
        if self.enable_torch_compile:
            self.weight_quant_func = compile_func(self.weight_quant_func, self.device)

        if self.enable_act_quant:
            self.act_quant_func, self.act_data_type = get_quant_func(
                orig_layer.act_data_type,
                orig_layer.act_bits,
                orig_layer.act_sym,
                disable_opt_rtn=True,
                iters=orig_layer.iters,
            )
            if self.enable_torch_compile:
                self.act_quant_func = compile_func(self.act_quant_func, self.device)
            self._init_params(
                "act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic)
            )
            self._init_params("act_min_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING)
            if warm_start:
                for key in ("act_min_scale", "act_max_scale"):
                    saved = warm_start.get(key)
                    param = self.params.get(key)
                    if saved is not None and param is not None and saved.shape == param.shape:
                        param.data.copy_(saved.to(device=param.device, dtype=param.dtype))

        # Bias tuning
        if self.enable_norm_bias_tuning:
            self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)
            _restore_warm_start_param(self.orig_layer, "bias_v", self.bias_v)
            from auto_round.data_type.int import quant_tensor_asym_wo_round

            self.bias_quant_func = quant_tensor_asym_wo_round
            self.params["bias_v"] = self.bias_v

    def _init_params(self, name, dtype, shape, value, tunable):
        """Initializes a parameter for tuning or uses a constant if tuning is disabled.

        Args:
            name (str): Name of the parameter.
            dtype (torch.dtype): Data type of the parameter.
            shape (tuple): Shape of the parameter.
            value (float): Initial value for the parameter.
            tunable (bool): Whether the parameter should be tunable.
        """
        if tunable:
            p = torch.nn.Parameter(torch.ones(shape, device=self.device, dtype=dtype) * value, requires_grad=True)
            self.params.update({name: p})
        else:
            p = torch.tensor(1.0 * value, device=self.device, dtype=dtype)

        setattr(self, name, p)

    def _apply_qdq(self, qdq_weight: torch.Tensor, scale: "torch.Tensor | dict", zp: "torch.Tensor | dict") -> None:
        """Write a quantize-dequantized result back onto the original layer.

        The single source of the unwrapper's write-back conventions (weight
        copy, scale/zp attribute shapes and devices, grad reset, global
        scale), shared with the batched zero-shot search driver.
        """
        self.orig_layer.weight.data.copy_(qdq_weight)
        self.orig_layer.weight.grad = None

        shape = qdq_weight.shape
        if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
            shape = qdq_weight.t().shape

        def _set_dict_attr(attr_dict: dict, attr_name: str) -> None:
            for key, value in attr_dict.items():
                if key == attr_name:
                    setattr(self.orig_layer, attr_name, value.reshape(shape[0], -1).to("cpu"))
                else:
                    name = "w_" + key
                    setattr(self.orig_layer, name, value.to("cpu"))

        if not isinstance(self.orig_layer.group_size, tuple):
            if isinstance(scale, dict):
                _set_dict_attr(scale, "scale")
            elif scale is None:
                self.orig_layer.scale = None
            elif scale.numel() > 1:
                self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
            else:
                self.orig_layer.scale = scale.view(-1).to("cpu")
        else:
            self.orig_layer.scale = scale.to("cpu")

        if zp is not None:
            if isinstance(zp, dict):
                _set_dict_attr(zp, "zp")
            elif isinstance(zp, torch.Tensor):
                if zp.numel() > 1:
                    zp = zp.reshape(shape[0], -1)
                    self.orig_layer.zp = zp.to("cpu")
                else:
                    self.orig_layer.zp = zp.view(-1).to("cpu")
            else:
                self.orig_layer.zp = zp
        else:
            self.orig_layer.zp = None

        if self.weight_global_scale is not None:
            global_scale = self.weight_global_scale
            assert global_scale.numel() == 1
            self.orig_layer.weight_global_scale = global_scale.to("cpu")

    def _run_deferred_neuqi_anchor_now(self) -> None:
        """Run the staged NeUQI grid anchor per module (fallback when batching is off)."""
        staged = getattr(self, "_deferred_neuqi_anchor", None)
        if staged is None:
            return
        weight_reshape, bits, group_size, imatrix_raw, sym = staged
        from auto_round.data_type.neuqi import neuqi_anchor_margins

        # run on the weight's own device when it is accelerator-resident: the
        # tuning device also hosts the pools and calibration state, so a
        # per-module fallback that pulls every search there can exhaust it
        device = weight_reshape.device if weight_reshape.device.type == "cuda" else self.device
        wmin, wmax = neuqi_anchor_margins(weight_reshape, bits, group_size, imatrix_raw, device, sym=sym)
        self._apply_neuqi_anchor(wmin, wmax)

    def _apply_neuqi_anchor(self, wmin: "torch.Tensor", wmax: "torch.Tensor") -> None:
        """Adopt a searched anchor, computed alone or on a stacked same-shape batch."""
        self.weight_min = wmin.reshape(self.weight_min.shape).to(
            device=self.weight_min.device, dtype=self.weight_min.dtype
        )
        self.weight_max = wmax.reshape(self.weight_max.shape).to(
            device=self.weight_max.device, dtype=self.weight_max.dtype
        )
        self._deferred_neuqi_anchor = None

    def _quant_call_kwargs(
        self,
        value: torch.Tensor,
        min_scale: torch.Tensor,
        max_scale: torch.Tensor,
        imatrix_override: torch.Tensor | None = None,
    ) -> dict:
        """Assemble the weight_quant_func call kwargs from wrapper + layer state.

        Single source of the quant-call contract, shared by the serial
        ``_qdq_weight`` path and the batched zero-shot search driver (which
        passes an explicit stacked ``imatrix_override``). ``weight_quant_func``
        itself takes the weight as its first positional argument.
        """
        quant_kwargs = {}
        if hasattr(self.orig_layer, "super_bits"):
            quant_kwargs["super_bits"] = self.orig_layer.super_bits
            quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
        if hasattr(self, "_extra_quant_kwargs"):
            quant_kwargs.update(self._extra_quant_kwargs())
        imatrix = imatrix_override
        if imatrix is None:
            if hasattr(self.orig_layer, "imatrix"):
                # a meta-resident stored weight (low-memory/offload lanes) must
                # not become the relocation target -- the imatrix would go meta
                # too while the weight itself is read through get_weight() onto
                # the compute device; relocate to the effective compute device
                _target = self.device if self.orig_layer.weight.device.type == "meta" else self.orig_layer.weight.device
                imatrix = self.orig_layer.imatrix.to(_target)
            else:
                imatrix = None
        return {
            "bits": self.orig_layer.bits,
            "group_size": self.orig_layer.group_size,
            "v": value,
            "min_scale": min_scale,
            "max_scale": max_scale,
            "scale_dtype": self.orig_layer.scale_dtype,
            "tensor_min": self.weight_min,
            "tensor_max": self.weight_max,
            "data_type": self.data_type,
            "q_scale_thresh": self.q_scale_thresh,
            "imatrix": imatrix,
            "global_scale": getattr(self, "weight_global_scale", None),
            "init_scale": getattr(self, "init_scale", None),
            **quant_kwargs,
        }

    def _prepare_minmax_scale(self, min_scale, max_scale):
        """Clamp the min/max scale coefficients and return what the quant func sees.

        Default (linear) behavior: the stored parameters *are* the multiplicative
        coefficients, clamped in place to ``minmax_scale_bound`` and passed
        through unchanged. Subclasses may override this to tune the coefficients
        in a different geometry (e.g. log/geometric space) while keeping the
        quant functions -- which always receive plain multiplicative
        coefficients -- untouched.
        """
        min_bound, max_bound = self.minmax_scale_bound
        min_scale.data.clamp_(min_bound, max_bound)
        max_scale.data.clamp_(min_bound, max_bound)
        return min_scale, max_scale

    def _qdq_weight(self, value, min_scale, max_scale):
        presolved = getattr(self, "_presolved_qdq", None)
        if presolved is not None:
            # Batched-search write-back: the search already ran on a stacked
            # tensor; skip the recompute but keep everything downstream of this
            # method (write-back conventions + the full unwrapper tail --
            # bias/meta/update, static-act rescale, WrapperWALayer attachment)
            # identical to the serial path.
            self._presolved_qdq = None
            return presolved
        if self.orig_layer.bits >= 16:
            return self.orig_layer.weight, None, None
        min_scale, max_scale = self._prepare_minmax_scale(min_scale, max_scale)
        weight = self.orig_layer.weight
        if weight.device.type == "meta":
            weight = self.orig_layer.get_weight().to(self.device)
        if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
            weight = weight.t()

        quant_kwargs = {}
        if hasattr(self.orig_layer, "super_bits"):
            quant_kwargs["super_bits"] = self.orig_layer.super_bits
            quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
        if hasattr(self, "_extra_quant_kwargs"):
            quant_kwargs.update(self._extra_quant_kwargs())

        weight_q, scale, zp = self.weight_quant_func(
            weight.to(self.device),
            **self._quant_call_kwargs(
                value,
                min_scale,
                max_scale,
                imatrix_override=None,
            ),
        )
        weight_q = weight_q.to(weight.dtype)
        if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
            weight_q = weight_q.t()
        return weight_q, scale, zp

    def _qdq_act(self, x, act_min_scale=None, act_max_scale=None, act_max=None):
        """Quantizes and dequantizes activations.

        Args:
            x (torch.Tensor): Input activations.
            act_max_scale (torch.Tensor): Maximum scale for the act_max
            act_max (torch.Tensor, optional): Maximum value for activation quantization. Defaults to None.

        Returns:
            tuple: Quantized activation, scale, and zero point.
        """
        if act_min_scale is None:
            act_min_scale = torch.tensor(1.0)
        if act_max_scale is None:
            act_max_scale = torch.tensor(1.0)
        act_max_scale.data.clamp_(0, 1.0)
        act_min_scale.data.clamp_(0, 1.0)
        env_act_scale = envs.AR_ACT_SCALE  # fixed activation ratio,prioritize to use this one if set
        x, scale, zp = self.act_quant_func(
            x,
            bits=self.orig_layer.act_bits,
            group_size=self.orig_layer.act_group_size,
            scale_dtype=self.orig_layer.scale_dtype,
            q_scale_thresh=self.q_scale_thresh,
            data_type=self.act_data_type,
            tensor_max=act_max,  # for static
            max_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
            min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
            global_scale=getattr(self, "input_global_scale", None),
        )
        return x, scale, zp

    def _qdq_bias(self, bias, bias_v):
        """Quantizes and dequantizes bias.

        Args:
            bias (torch.Tensor): Bias tensor to be quantized.
            bias_v (torch.Tensor): Value added for rounding for tuning.

        Returns:
            tuple: Quantized bias, scale, and zero point.
        """
        bias_bits = 4  ## hard code
        bias_group_size = -1
        bias, scale, zp = self.bias_quant_func(
            bias,
            bits=bias_bits,
            group_size=bias_group_size,
            v=bias_v,
            q_scale_thresh=self.q_scale_thresh,
            global_scale=getattr(self, "weight_global_scale", None),
        )
        return bias, scale, zp

    def unwrapper(self, best_params):
        """Restores the original layer by applying the best tuning parameters.

        Args:
            best_params (dict): Dictionary containing the best tuning parameters.

        Returns:
            torch.nn.Module: The unwrapped and restored original layer.
        """

        def _preserve_global_name(layer):
            if hasattr(self.orig_layer, "global_name") and not hasattr(layer, "global_name"):
                layer.global_name = self.orig_layer.global_name
            return layer

        best_params = best_params or {}
        v = best_params.get("value", torch.tensor(0.0)).to(self.device)
        min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)
        max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)

        if self.orig_layer.weight.device.type == "meta":
            self.orig_layer.to(self.device)
        # Unwrapper weight
        qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
        self._apply_qdq(qdq_weight, scale, zp)

        # Unwrapper bias
        if self.enable_norm_bias_tuning and "bias_v" in best_params.keys():  ##fake quant
            bias_v = best_params["bias_v"].to(self.device)
            bias = self.orig_layer.bias
            if bias is not None and bias.device.type == "meta":
                bias = self.orig_layer.get_bias().to(self.device)
            bias, _, _ = self._qdq_bias(bias, bias_v)
            self.orig_layer.bias.grad = None
            self.orig_layer.bias.data.copy_(bias)

        if hasattr(self.orig_layer, "update"):
            self.orig_layer.update()
            self.orig_layer.to("meta")

        # Unwrapper act
        if self.enable_act_quant:
            act_min_scale = best_params.get("act_min_scale", torch.tensor(1.0)).detach().to(self.device).clone()
            act_max_scale = best_params.get("act_max_scale", torch.tensor(1.0)).detach().to(self.device).clone()
            act_min_scale.clamp_(0, 1.0)
            act_max_scale.clamp_(0, 1.0)
            if not self.orig_layer.act_dynamic:
                act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
                if act_max is not None:
                    tmp_shape = 1
                    if self.orig_layer.act_group_size > 1:
                        tmp_shape = (act_max.shape[0], self.orig_layer.act_group_size)
                    elif self.orig_layer.act_group_size == -1:
                        tmp_shape = (act_max.shape[0], 1)
                    _, act_scale, _ = self._qdq_act(
                        torch.zeros(tmp_shape).to(self.device),
                        act_min_scale=act_min_scale,
                        act_max_scale=act_max_scale,
                        act_max=act_max,
                    )
                    # Keep calibration statistics separate from the tuned coefficients.
                    self.orig_layer.act_max = self.orig_layer.act_max.to("cpu")
                else:
                    act_scale = torch.ones(1, dtype=self.orig_layer.scale_dtype)
                self.orig_layer.act_scale = act_scale.to("cpu")

            self.orig_layer.q_scale_thresh = self.q_scale_thresh
            self.orig_layer.data_type = self.data_type
            self.orig_layer.act_min_scale = act_min_scale
            self.orig_layer.act_max_scale = act_max_scale

            self.orig_layer.act_data_type = self.act_data_type
            self.orig_layer.act_quant_func = self.act_quant_func
            wrapper_layer = WrapperWALayer(
                self.orig_layer,
                enable_torch_compile=self.enable_torch_compile,
                device=self.device,
            )
            return _preserve_global_name(wrapper_layer)

        return _preserve_global_name(self.orig_layer)

    def linear_forward(self, x, weight, bias):
        """Performs the forward pass for a linear layer.

        Args:
            x (torch.Tensor): Input tensor.
            weight (torch.Tensor): Weight tensor for the linear layer.
            bias (torch.Tensor): Bias tensor for the linear layer.

        Returns:
            torch.Tensor: Output tensor after applying the linear layer.
        """
        return F.linear(x, weight, bias)  # pylint: disable=E1102

    def all_reduce_linear_forward(self, x, weight, bias):
        """Performs the forward pass for a linear layer.

        Args:
            x (torch.Tensor): Input tensor.
            weight (torch.Tensor): Weight tensor for the linear layer.
            bias (torch.Tensor): Bias tensor for the linear layer.

        Returns:
            torch.Tensor: Output tensor after applying the linear layer.
        """
        output = torch.matmul(x, weight.transpose(-1, -2))
        if self.mp_group is not None:
            dist.inference_all_reduce(output, group=self.mp_group)
        if bias is not None:
            output += bias
        return output

    def conv1d_forward(self, x, weight, bias):
        """Performs the forward pass for a Conv1D layer.

        Args:
            x (torch.Tensor): Input tensor.
            weight (torch.Tensor): Weight tensor for the Conv1D layer.
            bias (torch.Tensor): Bias tensor for the Conv1D layer.

        Returns:
            torch.Tensor: Output tensor after applying the Conv1D layer.
        """
        size_out = x.size()[:-1] + (self.orig_layer.nf,)
        x = torch.addmm(bias, x.view(-1, x.size(-1)), weight)
        x = x.view(*size_out)
        return x

    def forward(self, x):
        """Executes the forward pass with quantized weights and optional bias/activation quantization.

        Args:
            x (torch.Tensor): Input tensor.

        Returns:
            torch.Tensor: Output tensor after applying the wrapped layer.
        """
        # logger.info(self.orig_layer.global_name)
        x = x.to(self.device)
        weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)

        if self.enable_act_quant:
            # Run orig_layer's forward_pre_hooks (e.g., online Hadamard transform)
            # BEFORE activation quantization, to match inference behavior in WrapperWALayer.
            for hook in self.orig_layer._forward_pre_hooks.values():
                result = hook(self.orig_layer, (x,))
                if result is not None:
                    x = result[0] if isinstance(result, tuple) else result
            act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
            x, _, _ = self._qdq_act(
                x, act_max_scale=self.act_max_scale, act_min_scale=self.act_min_scale, act_max=act_max
            )
        elif len(self.orig_layer._forward_pre_hooks) > 0:
            # Even without act_quant, run pre-hooks for online Hadamard correctness
            # (when fuse_online_to_weight=False, pre-hooks transform activations).
            for hook in self.orig_layer._forward_pre_hooks.values():
                result = hook(self.orig_layer, (x,))
                if result is not None:
                    x = result[0] if isinstance(result, tuple) else result

        # pylint: disable=not-callable
        bias = self.orig_layer.bias
        if bias is not None and bias.device.type == "meta":
            bias = self.orig_layer.get_bias().to(self.device)
        if self.enable_norm_bias_tuning:
            bias, _, _ = self._qdq_bias(bias, self.bias_v)

        output = self.orig_forward(x, weight_q, bias).to(self.output_device)

        # Execute post-hooks from orig_layer (e.g., v_proj per-head Hadamard
        # when online rotation is not fused into weights).
        for hook in self.orig_layer._forward_hooks.values():
            hook_result = hook(self.orig_layer, (x,), output)
            if hook_result is not None:
                output = hook_result

        return output


class WrapperWALayer(torch.nn.Module):

    def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):
        super().__init__()
        self.orig_layer = orig_layer
        self.enable_torch_compile = enable_torch_compile
        self.device = device
        self.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else None
        self.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None
        self.act_quant_func = self.orig_layer.act_quant_func
        if self.enable_torch_compile:
            self.act_quant_func = compile_func(self.act_quant_func, self.device)
        self.extra_repr_org = orig_layer.extra_repr

        # Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)
        # and remove them from orig_layer so they won't fire again inside orig_layer.forward().
        # We will run them explicitly in our forward() BEFORE activation quantization.
        self._stolen_pre_hooks = list(orig_layer._forward_pre_hooks.values())
        orig_layer._forward_pre_hooks.clear()

    @property
    def weight(self):
        """Exposes the weight of the wrapped layer for external access."""
        return self.orig_layer.weight

    @property
    def bias(self):
        """Exposes the bias of the wrapped layer for external access."""
        return self.orig_layer.bias

    def forward(self, x):
        # 1) Run stolen pre_hooks first (e.g., online Hadamard) → smooths activation
        for hook in self._stolen_pre_hooks:
            result = hook(self.orig_layer, (x,))
            if result is not None:
                x = result[0] if isinstance(result, tuple) else result

        # 2) Activation quantization on the smoothed activation
        import auto_round.envs as envs

        act_scale = envs.AR_ACT_SCALE
        act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None

        max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
        min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
        quant_kwargs = {}
        if hasattr(self.orig_layer, "input_global_scale"):
            quant_kwargs["global_scale"] = self.orig_layer.input_global_scale
        x, _, _ = self.orig_layer.act_quant_func(
            x,
            bits=self.orig_layer.act_bits,
            group_size=self.orig_layer.act_group_size,
            scale_dtype=self.orig_layer.scale_dtype,
            q_scale_thresh=self.orig_layer.q_scale_thresh,
            data_type=self.orig_layer.act_data_type,
            tensor_max=act_max,
            min_scale=min_scale,
            max_scale=max_scale,
            **quant_kwargs,
        )
        # 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)
        return self.orig_layer.forward(x)

    def extra_repr(self):
        return f"{self.extra_repr_org()}, weight_type={self.data_type}, act_data_type={self.act_data_type}"


class WrapperLayerNorm(torch.nn.Module):
    """A wrapper for layer normalization with quantized weights.

    This class wraps a given layer normalization module and applies quantization without round
    to its weights. The quantization is parameterized by the number of bits and
    an optional group size.
    """

    def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):
        super().__init__()
        self.orig_layer = orig_layer
        self.bits = bit
        self.group_size = group_size
        self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
        self.output_device = device
        weight_dtype = torch.float32
        self.q_scale_thresh = 1e-5
        self.v = torch.nn.Parameter(
            reshape_pad_tensor_by_group_size(
                torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
            )[0],
            requires_grad=True,
        )
        self.params = {"v": self.v}
        _restore_warm_start_param(self.orig_layer, "v", self.v)
        from auto_round.data_type.int import quant_tensor_asym_wo_round

        self.quant_func = quant_tensor_asym_wo_round

    def unwrapper(self, best_params):
        if best_params is None:
            return self.orig_layer
        v = best_params["v"].to(self.device)
        weight_q, _, _ = self.quant_func(
            self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh
        )
        self.orig_layer.q_scale_thresh = self.q_scale_thresh
        self.orig_layer.weight.data.copy_(weight_q)
        return self.orig_layer

    def forward(self, input):
        input = input.to(self.device)
        weight_q, _, _ = self.quant_func(
            self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh
        )
        import torch.nn.functional as F

        return F.layer_norm(
            input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps
        ).to(self.output_device)


class WrapperLlamaNorm(torch.nn.Module):
    """A wrapper for Llama normalization in HF with fake quantized weights without rounding.

    This class wraps a given layer normalization module and applies quantization without rounding
    to its weights. The quantization is parameterized by the number of bits and
    an optional group size.
    """

    def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):
        super().__init__()
        self.orig_layer = orig_layer
        self.bits = bit
        self.group_size = group_size
        self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
        self.output_device = device
        weight_dtype = torch.float32
        self.q_scale_thresh = 1e-5
        self.v = torch.nn.Parameter(
            reshape_pad_tensor_by_group_size(
                torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
            )[0],
            requires_grad=True,
        )
        self.params = {"v": self.v}
        _restore_warm_start_param(self.orig_layer, "v", self.v)
        from auto_round.data_type.int import quant_tensor_asym_wo_round

        self.quant_func = quant_tensor_asym_wo_round

    def unwrapper(self, best_params):
        if best_params is None:
            return self.orig_layer
        v = best_params["v"].to(self.device)
        weight_q, _, _ = self.quant_func(
            self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh
        )
        self.orig_layer.q_scale_thresh = self.q_scale_thresh
        self.orig_layer.weight.data.copy_(weight_q)
        return self.orig_layer

    def forward(self, hidden_states):
        hidden_states = hidden_states.to(self.device)
        weight_q, _, _ = self.quant_func(
            self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh
        )
        input_dtype = hidden_states.dtype
        hidden_states = hidden_states.to(torch.float32)
        variance = hidden_states.pow(2).mean(-1, keepdim=True)
        hidden_states = hidden_states * torch.rsqrt(variance + self.orig_layer.variance_epsilon)
        return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)


NORM_MAPPING = {}
NORM_MAPPING["LayerNorm"] = WrapperLayerNorm
NORM_MAPPING["LlamaRMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen2RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Phi3RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["MistralRMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen3RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen3_5MoeRMSNorm"] = WrapperLlamaNorm


class WrapperMultiblock(torch.nn.Module):
    """A wrapper for a list of modules to be act as a single block.

    Args:
    module_list: The list of modules to wrap.
    """

    def __init__(self, module_list):
        super().__init__()
        self.layers = torch.nn.ModuleList(module_list)

    def forward(self, x, *args, **kwargs):
        hidden_states = x
        for idx, decoder_layer in enumerate(self.layers):
            layer_outputs = decoder_layer(hidden_states, *args, **kwargs)
            hidden_states = layer_outputs
            if isinstance(hidden_states, (tuple, list)):
                hidden_states = layer_outputs[0]
        return hidden_states


def wrapper_block(
    block,
    enable_minmax_tuning,
    enable_norm_bias_tuning,
    enable_torch_compile=True,
    device="cpu",
    wrapper_cls=WrapperLinear,
    **kwargs,
):
    """Wraps the layers in the given block with a custom Wrapper module.

    Args:
        block: The input block containing linear and conv1d layers to be wrapped.
        enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.
        enable_norm_bias_tuning: A boolean indicating whether normalization and bias tuning is enabled.
        enable_torch_compile: A boolean indicating whether to enable torch compilation.
        device: The device to which the wrapped layers should be moved.

    Returns:
        list: A list of names of the wrapped layers and unwrapped layers.
    """
    quantized_layers = []
    unquantized_layers = []

    # Wrap-time searches (SignRoundV2 init-scale, GGUF DQ scale) read only the
    # module's own weight plus per-module statistics, never calibration
    # activations; eligible searches run later in one stacked batched call per
    # (device, shape, config, search fn).
    work = []
    for n, m in block.named_modules():
        if type(m) in SUPPORTED_LAYER_TYPES:
            work.append((n, m, "skip" if not check_to_quantized(m) else "linear"))
        elif enable_norm_bias_tuning and "norm" in m.__class__.__name__.lower():
            work.append((n, m, "norm"))

    from auto_round.algorithms.quantization import search_dispatch

    _defer_batch = (
        bool(getattr(wrapper_cls, "supports_batched_search", False)) and not search_dispatch.batched_search_disabled()
    )
    deferred_wrappers = []

    def _wrap_one(n: str, m: torch.nn.Module, kind: str) -> torch.nn.Module:
        if kind == "norm":
            if m.__class__.__name__ in NORM_MAPPING.keys():
                wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]
                new_m = wrapper_layer_class(m, device=device)
                set_module(block, n, new_m)
            elif "RMSNorm" in m.__class__.__name__:
                logger.warning_once(
                    f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself"
                )
                wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]
                new_m = wrapper_layer_class(m, device=device)
                set_module(block, n, new_m)
            else:
                logger.warning_once(f"{m.__class__.__name__} is not supported")
            return None
        if kind == "skip":
            return "u"
        layer_wrapper_cls = getattr(m, "_ar_warm_start_wrapper_cls", wrapper_cls)
        defer_search = _defer_batch and getattr(layer_wrapper_cls, "supports_batched_search", False)
        extra = {"defer_search": True} if defer_search else {}
        new_m = layer_wrapper_cls(
            m,
            enable_minmax_tuning=enable_minmax_tuning,
            enable_norm_bias_tuning=enable_norm_bias_tuning,
            enable_torch_compile=enable_torch_compile,
            device=device,
            **kwargs,
            **extra,
        )
        set_module(block, n, new_m)
        if defer_search:
            deferred_wrappers.append(new_m)
        return "q"

    results = [_wrap_one(n, m, kind) for n, m, kind in work]

    if _defer_batch and deferred_wrappers:
        anchor_pending = [w for w in deferred_wrappers if getattr(w, "_deferred_neuqi_anchor", None) is not None]
        if anchor_pending:
            if not search_dispatch.run_batched_neuqi_anchors(anchor_pending):
                for w in anchor_pending:
                    w._run_deferred_neuqi_anchor_now()
        # stacked same-shape batches per (device, shape, config, search fn);
        # singletons and the kill-switch fall back to the identical per-module call.
        # Data-parallel ranks shard the searches across the ranks instead.
        from auto_round.algorithms.parallel.spmd import current_rank, run_rank_sharded_wrap_searches

        rank = current_rank()
        if not (rank is not None and run_rank_sharded_wrap_searches(rank, deferred_wrappers)):
            if not search_dispatch.run_batched_wrap_search(deferred_wrappers):
                for w in deferred_wrappers:
                    w._run_deferred_search_now()

    quantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "q"]
    unquantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "u"]
    return quantized_layers, unquantized_layers


@torch.no_grad()
def unwrapper_layer(model, layer, layer_name, best_params):
    """Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.

    Args:
    block: The input block containing wrapped modules to be unwrapped.
    vs: A dictionary of scaling parameters for the wrapped modules.
    min_scales: A dictionary of minimum scaling values for the wrapped modules.
    max_scales: A dictionary of maximum scaling values for the wrapped modules.
    """

    # WrapperWALayer retains activation QDQ and is not a tuning wrapper.
    if hasattr(layer, "orig_layer") and callable(getattr(layer, "unwrapper", None)):
        orig_layer = layer.unwrapper(best_params)
        act_max = getattr(orig_layer, "act_max", None)
        act_scale = getattr(orig_layer, "act_scale", None)
        if (
            "lm_head" in layer_name
            and getattr(layer, "enable_act_quant", False)
            and not getattr(orig_layer, "act_dynamic", True)
            and act_scale is not None
            and act_max is None
        ):
            logger.warning_once(
                "Static activation quantization for lm_head is not fully supported yet. "
                "lm_head activation statistics are missing, so activation scale falls back to unit scale."
            )
        orig_layer = orig_layer.to("cpu")
        set_module(model, layer_name, orig_layer)


@torch.no_grad()
def unwrapper_block(block, best_params):
    """Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.

    Args:
    block: The input block containing wrapped modules to be unwrapped.
    vs: A dictionary of scaling parameters for the wrapped modules.
    min_scales: A dictionary of minimum scaling values for the wrapped modules.
    max_scales: A dictionary of maximum scaling values for the wrapped modules.
    """
    for n, m in block.named_modules():
        # Keep persistent activation wrappers, which do not implement unwrapper.
        if hasattr(m, "orig_layer") and callable(getattr(m, "unwrapper", None)):
            if n in best_params.keys():
                best_param = best_params[n]
            else:
                best_param = None
            orig_layer = m.unwrapper(best_param)
            set_module(block, n, orig_layer)


@contextmanager
def preserve_original_weights(block):
    """Keep a host snapshot for a composer-owned stage that must not bake weights.

    This also covers algorithms that modify parameters in place instead of using
    common wrappers. Only the current block is copied on block-tuning paths;
    ordinary/final stages do not pay this cost. Restore the original module graph
    and parameter dtype/values on success or failure, leaving warm-start metadata
    on the original layers available to subsequent stages.
    """
    modules = list(block.named_modules())
    parameters = [(p, p.detach().to("cpu", copy=True), p.requires_grad) for p in block.parameters()]
    quantization_types = [
        (module, key, getattr(module, key))
        for _, module in modules
        for key in ("data_type", "act_data_type")
        if hasattr(module, key)
    ]
    try:
        yield
    finally:
        with torch.no_grad():
            for name, module in modules:
                if name:
                    set_module(block, name, module)
            for parameter, saved, requires_grad in parameters:
                parameter.data = saved.to(parameter.device)
                parameter.requires_grad_(requires_grad)
            # RTN writes resolved one-shot dtype names during finalization.
            # Subsequent training stages need the original differentiable types.
            for module, key, value in quantization_types:
                setattr(module, key, value)


@torch.no_grad()
def store_warm_start_params_and_unwrap(block, best_params, *, before_unwrap=None):
    """Warm-start variant of :func:`unwrapper_block`: keep the original FP weight.

    Instead of baking the tuned ``value``/``min_scale``/``max_scale`` into the
    (quantized) weight like :func:`unwrapper_block` does, this stashes them on
    each ``orig_layer`` under ``_ar_warm_start`` and restores the untouched
    full-precision layer. A later pass (e.g. QAD whole-model tuning) can then
    re-wrap the same layers and seed its tuning parameters from the stash via the
    warm-start branch in :meth:`WrapperLinear._init_tuning_params_and_quant_func`.

    The optimized SignRound wrapper also exposes a searched ``init_scale`` (not a
    trainable parameter); it is stashed too so the second pass reuses the exact
    same scale grid instead of re-searching it.

    Args:
        block: The block (or single wrapped layer) whose wrappers to collect.
        best_params: Result of ``collect_best_params`` -- either a flat
            ``{param_name: tensor}`` dict (single-wrapper block) or
            ``{submodule_name: {param_name: tensor}}``.
        before_unwrap: Optional callback receiving the still-wrapped block with
            its best parameters loaded. Used to collect quantized outputs before
            restoring FP layers; restoration also happens if the callback raises.
    """
    best_params = best_params or {}
    wrapped_layers = [
        (n, m)
        for n, m in block.named_modules()
        if hasattr(m, "orig_layer") and hasattr(m, "params") and callable(getattr(m, "unwrapper", None))
    ]
    try:
        for n, m in wrapped_layers:
            if n in best_params:
                layer_params = best_params[n]
            elif any(key in best_params for key in m.params):
                # Flat dict: the block itself is the wrapped layer.
                layer_params = best_params
            else:
                layer_params = {}
            store = {}
            for key in m.params:
                if key in layer_params and layer_params[key] is not None:
                    store[key] = layer_params[key].detach().to("cpu", copy=True)
            store["minmax_parameterization"] = "log" if getattr(m, "_use_log_minmax", False) else "linear"
            init_scale = getattr(m, "init_scale", None)
            if init_scale is not None:
                store["init_scale"] = init_scale.detach().to("cpu")
            # Asym optimized wrapper: preserve the searched grid anchor too.
            if getattr(m, "_asym_anchor", False):
                store["anchor_min"] = m.weight_min.detach().to("cpu")
                store["anchor_max"] = m.weight_max.detach().to("cpu")
            m.orig_layer._ar_warm_start = store
            if isinstance(m, WrapperLinear):
                m.orig_layer._ar_warm_start_wrapper_cls = type(m)
            if before_unwrap is not None:
                for key, saved in layer_params.items():
                    param = m.params.get(key)
                    if param is not None and saved is not None:
                        param.copy_(saved.to(device=param.device, dtype=param.dtype))
        if before_unwrap is not None:
            before_unwrap(block)
    finally:
        for n, m in wrapped_layers:
            set_module(block, n, m.orig_layer)