PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from contextlib import contextmanager
from math import ceil
import torch
import transformers
from torch.functional import F
import auto_round.envs as envs
from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_to_quantized,
compile_func,
deepspeed_exists,
set_module,
)
if deepspeed_exists:
from deepspeed import comm as dist
from deepspeed.module_inject import LinearAllreduce, LinearLayer
def get_scale_shape(weight, group_size):
"""Computes the shape of the scale tensor for quantization based on the weight tensor and group size.
Args:
weight (torch.Tensor): The weight tensor of the layer.
group_size (int or tuple): The size of the groups for quantization.
Returns:
The shape of the scale tensor to be used for quantization.
"""
if isinstance(group_size, tuple):
assert len(weight.shape) == len(group_size), f"Expected group_size is {len(weight.shape)}D but get {group_size}"
return (weight.shape[0] // group_size[0], weight.shape[1] // group_size[1])
if group_size == 0:
return 1
elif group_size == -1 or weight.shape[1] < group_size:
shape = weight.shape[0]
else:
shape = weight.shape[0] * ceil(weight.shape[1] / group_size)
return shape
def _restore_warm_start_param(orig_layer, name, param):
"""Restore a compatible parameter without replacing its optimizer identity."""
saved = getattr(orig_layer, "_ar_warm_start", {}).get(name)
if saved is not None and saved.shape == param.shape:
with torch.no_grad():
param.copy_(saved.to(device=param.device, dtype=param.dtype))
class WrapperLinear(torch.nn.Module):
"""A wrapper for linear/conv1d layers to enable quantization and tuning.
This module wraps an existing linear or conv1d layer and provides additional functionality
for quantization, parameter tuning, and activation/bias normalization.
Args:
orig_layer (torch.nn.Module): The original layer to be wrapped (linear or conv1d).
enable_minmax_tuning (bool): Whether to enable min-max scale tuning.
enable_norm_bias_tuning (bool): Whether to enable normalization and tuning of the bias term.
enable_torch_compile (bool): Whether to enable torch compilation.
device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').
"""
minmax_scale_bound = (0.0, 1.0)
def __init__(
self,
orig_layer,
enable_minmax_tuning=True,
enable_norm_bias_tuning=False,
device="cpu",
enable_round_tuning=True,
enable_torch_compile=True,
disable_opt_rtn=True,
enable_neuqi=False,
**kwargs,
):
"""Initializes the WrapperLinear module.
Args:
orig_layer (torch.nn.Module): The original layer to wrap.
enable_minmax_tuning (bool): Whether to enable min-max scale tuning.
enable_norm_bias_tuning (bool): Whether to enable normalization and tuning for the bias term.
device (str): The computation device, such as 'cpu' or 'cuda'.
"""
super().__init__()
self.orig_layer = orig_layer
self.orig_layer.iters = kwargs.pop("iters", 200)
self.disable_opt_rtn = disable_opt_rtn
self.enable_neuqi = enable_neuqi
self.output_device = device
self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.enable_minmax_tuning = enable_minmax_tuning
self.enable_round_tuning = enable_round_tuning
self.enable_torch_compile = enable_torch_compile
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
self.input_global_scale = getattr(self.orig_layer, "input_global_scale", None)
if needs_nvfp_global_scale(self.orig_layer.data_type) and self.weight_global_scale is None:
from auto_round.data_type.nvfp import calculate_gparam_by_dtype
weight_global_scale = calculate_gparam_by_dtype(
self.orig_layer.weight, self.orig_layer.data_type, self.orig_layer.group_size
)
self.weight_global_scale = weight_global_scale
self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
self.q_scale_thresh = 1e-8
else:
self.q_scale_thresh = 1e-5
# consumed here so subclasses never see it in **kwargs; forwarded to the
# search init (staging for batched same-shape wrap searches)
self._init_tuning_params_and_quant_func(defer_search=bool(kwargs.pop("defer_search", False)))
if deepspeed_exists:
if type(self.orig_layer) in (torch.nn.Linear, LinearLayer):
self.orig_forward = self.linear_forward
elif type(self.orig_layer) == LinearAllreduce:
self.orig_forward = self.all_reduce_linear_forward
self.mp_group = self.orig_layer.mp_group
else:
self.orig_forward = self.conv1d_forward
else:
self.orig_forward = self.linear_forward if type(self.orig_layer) == torch.nn.Linear else self.conv1d_forward
@property
def weight(self):
return self.orig_layer.weight
@property
def bias(self):
return self.orig_layer.bias
def _init_tuning_params_and_quant_func(self, defer_search: bool = False):
"""Initializes tuning parameters and quantization functions.
This method sets up required parameters and functions for weight quantization,
activation quantization, and bias/normalization.
"""
self.params = {}
p_dtype = torch.float32 ##parameter dtype
orig_layer = self.orig_layer
orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
orig_weight = orig_weight.t()
weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, orig_layer.group_size)
if self.enable_round_tuning:
self.weight_min = (
torch.clamp(weight_reshape.amin(dim=(-2, -1)), max=0)
if isinstance(orig_layer.group_size, tuple)
else torch.clamp(weight_reshape.min(1)[0], max=0)
)
self.weight_max = (
torch.clamp(weight_reshape.amax(dim=(-2, -1)), min=0)
if isinstance(orig_layer.group_size, tuple)
else torch.clamp(weight_reshape.max(1)[0], min=0)
)
else:
self.weight_min = None
self.weight_max = None
# AWQ clip-as-init: cap the tunable weight range to the per-group clip
# range searched by AWQ (``apply_clip`` with ``clip_as_init=True``).
# This initializes the range used by quant_tensor_sym/asym, leaving
# min_scale/max_scale to tune a coefficient on top. Only the standard
# (non-tuple) group layout maps onto weight_min/weight_max here.
awq_clip_min = getattr(orig_layer, "awq_clip_min", None)
awq_clip_max = getattr(orig_layer, "awq_clip_max", None)
if awq_clip_max is not None and self.weight_min is not None and not isinstance(orig_layer.group_size, tuple):
clip_max_flat = awq_clip_max.reshape(-1).to(self.weight_max.device, self.weight_max.dtype)
if awq_clip_min is None:
clip_min_flat = -clip_max_flat
else:
clip_min_flat = awq_clip_min.reshape(-1).to(self.weight_min.device, self.weight_min.dtype)
if clip_max_flat.numel() == self.weight_max.numel() and clip_min_flat.numel() == self.weight_min.numel():
self.weight_max = torch.minimum(self.weight_max, clip_max_flat)
self.weight_min = torch.maximum(self.weight_min, clip_min_flat)
# NeUQI frozen init (enable_neuqi on the tuning path): anchor
# the grid to the searched (scale, zp) instead of the raw per-group
# min/max. Placement mirrors the AWQ clip-as-init above; unsupported
# layouts (tuple group sizes, >=16 bits, non-int data types) keep the
# status-quo min/max grid. The zero-shot path needs no anchor -- its
# search runs inside the quant function dispatch itself.
self._neuqi_frozen_margins = False
if (
self.enable_neuqi
and self.enable_round_tuning
and weight_reshape is not None
and self.weight_min is not None
and self.orig_layer.bits < 16
and not isinstance(orig_layer.group_size, tuple)
and self.orig_layer.data_type == "int"
):
if getattr(orig_layer, "awq_clip_max", None) is not None:
logger.warning_once(
"enable_neuqi anchors the tuning grid to its own search result, which "
"overrides the AWQ clip-as-init range on this layer; the clip is ignored."
)
if defer_search:
# staged for the batched anchor driver; the margins keep the
# plain min/max until the batch pass applies the searched ones
self._deferred_neuqi_anchor = (
weight_reshape,
self.orig_layer.bits,
orig_layer.group_size,
getattr(orig_layer, "imatrix", None),
bool(getattr(orig_layer, "sym", True)),
)
self._neuqi_frozen_margins = True
else:
from auto_round.data_type.neuqi import neuqi_anchor_margins
_wmin, _wmax = neuqi_anchor_margins(
weight_reshape,
self.orig_layer.bits,
orig_layer.group_size,
getattr(orig_layer, "imatrix", None),
self.device,
sym=bool(getattr(orig_layer, "sym", True)),
)
self._neuqi_frozen_margins = True
self.weight_min = _wmin.to(self.weight_min.dtype)
self.weight_max = _wmax.to(self.weight_max.dtype)
self._init_params(
"value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16
)
# Min-max scale initialization
shape = get_scale_shape(orig_weight, orig_layer.group_size)
_margins_tunable = (self.enable_minmax_tuning and self.orig_layer.bits < 16) and (
not self._neuqi_frozen_margins
)
self._init_params("min_scale", p_dtype, shape, 1.0, _margins_tunable)
self._init_params("max_scale", p_dtype, shape, 1.0, _margins_tunable)
# Midrise int (v1 path): the absmax-derived scale (absmax / (maxq-0.5)) is far from the
# MSE optimum (~0.5x at W2) and the default [0, 1] max_scale bound with SignSGD can barely
# shrink that far. Seed ``init_scale`` with the same imatrix-weighted search used by the v2
# path and let ``max_scale`` tune a coefficient around it in both directions.
# (SignRoundOptimizedWrapperLinear re-runs the search and keeps its own bound.)
if (
str(self.orig_layer.data_type).startswith("int_midrise")
and type(self).minmax_scale_bound == WrapperLinear.minmax_scale_bound # plain v1 wrapper only
and self.enable_round_tuning
and self.orig_layer.bits < 16
and not isinstance(orig_layer.group_size, tuple)
):
from auto_round.data_type.utils import reshape_imatrix_for_weight, search_optimized_init_scale
warm_start = getattr(orig_layer, "_ar_warm_start", None)
if warm_start and warm_start.get("init_scale") is not None:
self.init_scale = warm_start["init_scale"].to(self.device)
else:
w_r = weight_reshape.to(self.device, torch.float32)
imatrix = reshape_imatrix_for_weight(getattr(orig_layer, "imatrix", None), w_r, orig_layer.group_size)
self.init_scale = search_optimized_init_scale(
w_r, orig_layer.data_type, orig_layer.bits, imatrix.to(w_r.device), self.q_scale_thresh
)
self.minmax_scale_bound = (0.5, 1.5)
# Warm-start: when a prior block-wise pass (e.g. the SignRound -> QAD
# chaining) stashed its tuned ``value``/``min_scale``/``max_scale`` on
# this layer via ``store_warm_start_params_and_unwrap``, seed the freshly
# created tuning parameters with them so this pass continues from that
# state instead of starting from RTN (value=0, scale coeff=1).
warm_start = getattr(orig_layer, "_ar_warm_start", None)
if warm_start:
for _key in ("value", "min_scale", "max_scale"):
saved = warm_start.get(_key)
param = self.params.get(_key)
if saved is None or param is None:
continue
saved = saved.to(device=param.device, dtype=param.dtype)
if saved.shape == param.shape:
param.data.copy_(saved)
else:
logger.warning_once(
f"warm-start param '{_key}' shape {tuple(saved.shape)} does not match "
f"{tuple(param.shape)}; keeping the default initialization."
)
self.weight_quant_func, self.data_type = get_quant_func(
orig_layer.data_type,
orig_layer.bits,
orig_layer.sym,
self.disable_opt_rtn,
orig_layer.group_size,
iters=orig_layer.iters,
enable_neuqi=self.enable_neuqi,
)
if self.enable_torch_compile:
self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
if self.enable_act_quant:
self.act_quant_func, self.act_data_type = get_quant_func(
orig_layer.act_data_type,
orig_layer.act_bits,
orig_layer.act_sym,
disable_opt_rtn=True,
iters=orig_layer.iters,
)
if self.enable_torch_compile:
self.act_quant_func = compile_func(self.act_quant_func, self.device)
self._init_params(
"act_max_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING or (not orig_layer.act_dynamic)
)
self._init_params("act_min_scale", p_dtype, (1), 1.0, envs.AR_ENABLE_ACT_MINMAX_TUNING)
if warm_start:
for key in ("act_min_scale", "act_max_scale"):
saved = warm_start.get(key)
param = self.params.get(key)
if saved is not None and param is not None and saved.shape == param.shape:
param.data.copy_(saved.to(device=param.device, dtype=param.dtype))
# Bias tuning
if self.enable_norm_bias_tuning:
self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)
_restore_warm_start_param(self.orig_layer, "bias_v", self.bias_v)
from auto_round.data_type.int import quant_tensor_asym_wo_round
self.bias_quant_func = quant_tensor_asym_wo_round
self.params["bias_v"] = self.bias_v
def _init_params(self, name, dtype, shape, value, tunable):
"""Initializes a parameter for tuning or uses a constant if tuning is disabled.
Args:
name (str): Name of the parameter.
dtype (torch.dtype): Data type of the parameter.
shape (tuple): Shape of the parameter.
value (float): Initial value for the parameter.
tunable (bool): Whether the parameter should be tunable.
"""
if tunable:
p = torch.nn.Parameter(torch.ones(shape, device=self.device, dtype=dtype) * value, requires_grad=True)
self.params.update({name: p})
else:
p = torch.tensor(1.0 * value, device=self.device, dtype=dtype)
setattr(self, name, p)
def _apply_qdq(self, qdq_weight: torch.Tensor, scale: "torch.Tensor | dict", zp: "torch.Tensor | dict") -> None:
"""Write a quantize-dequantized result back onto the original layer.
The single source of the unwrapper's write-back conventions (weight
copy, scale/zp attribute shapes and devices, grad reset, global
scale), shared with the batched zero-shot search driver.
"""
self.orig_layer.weight.data.copy_(qdq_weight)
self.orig_layer.weight.grad = None
shape = qdq_weight.shape
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
shape = qdq_weight.t().shape
def _set_dict_attr(attr_dict: dict, attr_name: str) -> None:
for key, value in attr_dict.items():
if key == attr_name:
setattr(self.orig_layer, attr_name, value.reshape(shape[0], -1).to("cpu"))
else:
name = "w_" + key
setattr(self.orig_layer, name, value.to("cpu"))
if not isinstance(self.orig_layer.group_size, tuple):
if isinstance(scale, dict):
_set_dict_attr(scale, "scale")
elif scale is None:
self.orig_layer.scale = None
elif scale.numel() > 1:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
else:
self.orig_layer.scale = scale.view(-1).to("cpu")
else:
self.orig_layer.scale = scale.to("cpu")
if zp is not None:
if isinstance(zp, dict):
_set_dict_attr(zp, "zp")
elif isinstance(zp, torch.Tensor):
if zp.numel() > 1:
zp = zp.reshape(shape[0], -1)
self.orig_layer.zp = zp.to("cpu")
else:
self.orig_layer.zp = zp.view(-1).to("cpu")
else:
self.orig_layer.zp = zp
else:
self.orig_layer.zp = None
if self.weight_global_scale is not None:
global_scale = self.weight_global_scale
assert global_scale.numel() == 1
self.orig_layer.weight_global_scale = global_scale.to("cpu")
def _run_deferred_neuqi_anchor_now(self) -> None:
"""Run the staged NeUQI grid anchor per module (fallback when batching is off)."""
staged = getattr(self, "_deferred_neuqi_anchor", None)
if staged is None:
return
weight_reshape, bits, group_size, imatrix_raw, sym = staged
from auto_round.data_type.neuqi import neuqi_anchor_margins
# run on the weight's own device when it is accelerator-resident: the
# tuning device also hosts the pools and calibration state, so a
# per-module fallback that pulls every search there can exhaust it
device = weight_reshape.device if weight_reshape.device.type == "cuda" else self.device
wmin, wmax = neuqi_anchor_margins(weight_reshape, bits, group_size, imatrix_raw, device, sym=sym)
self._apply_neuqi_anchor(wmin, wmax)
def _apply_neuqi_anchor(self, wmin: "torch.Tensor", wmax: "torch.Tensor") -> None:
"""Adopt a searched anchor, computed alone or on a stacked same-shape batch."""
self.weight_min = wmin.reshape(self.weight_min.shape).to(
device=self.weight_min.device, dtype=self.weight_min.dtype
)
self.weight_max = wmax.reshape(self.weight_max.shape).to(
device=self.weight_max.device, dtype=self.weight_max.dtype
)
self._deferred_neuqi_anchor = None
def _quant_call_kwargs(
self,
value: torch.Tensor,
min_scale: torch.Tensor,
max_scale: torch.Tensor,
imatrix_override: torch.Tensor | None = None,
) -> dict:
"""Assemble the weight_quant_func call kwargs from wrapper + layer state.
Single source of the quant-call contract, shared by the serial
``_qdq_weight`` path and the batched zero-shot search driver (which
passes an explicit stacked ``imatrix_override``). ``weight_quant_func``
itself takes the weight as its first positional argument.
"""
quant_kwargs = {}
if hasattr(self.orig_layer, "super_bits"):
quant_kwargs["super_bits"] = self.orig_layer.super_bits
quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
if hasattr(self, "_extra_quant_kwargs"):
quant_kwargs.update(self._extra_quant_kwargs())
imatrix = imatrix_override
if imatrix is None:
if hasattr(self.orig_layer, "imatrix"):
# a meta-resident stored weight (low-memory/offload lanes) must
# not become the relocation target -- the imatrix would go meta
# too while the weight itself is read through get_weight() onto
# the compute device; relocate to the effective compute device
_target = self.device if self.orig_layer.weight.device.type == "meta" else self.orig_layer.weight.device
imatrix = self.orig_layer.imatrix.to(_target)
else:
imatrix = None
return {
"bits": self.orig_layer.bits,
"group_size": self.orig_layer.group_size,
"v": value,
"min_scale": min_scale,
"max_scale": max_scale,
"scale_dtype": self.orig_layer.scale_dtype,
"tensor_min": self.weight_min,
"tensor_max": self.weight_max,
"data_type": self.data_type,
"q_scale_thresh": self.q_scale_thresh,
"imatrix": imatrix,
"global_scale": getattr(self, "weight_global_scale", None),
"init_scale": getattr(self, "init_scale", None),
**quant_kwargs,
}
def _prepare_minmax_scale(self, min_scale, max_scale):
"""Clamp the min/max scale coefficients and return what the quant func sees.
Default (linear) behavior: the stored parameters *are* the multiplicative
coefficients, clamped in place to ``minmax_scale_bound`` and passed
through unchanged. Subclasses may override this to tune the coefficients
in a different geometry (e.g. log/geometric space) while keeping the
quant functions -- which always receive plain multiplicative
coefficients -- untouched.
"""
min_bound, max_bound = self.minmax_scale_bound
min_scale.data.clamp_(min_bound, max_bound)
max_scale.data.clamp_(min_bound, max_bound)
return min_scale, max_scale
def _qdq_weight(self, value, min_scale, max_scale):
presolved = getattr(self, "_presolved_qdq", None)
if presolved is not None:
# Batched-search write-back: the search already ran on a stacked
# tensor; skip the recompute but keep everything downstream of this
# method (write-back conventions + the full unwrapper tail --
# bias/meta/update, static-act rescale, WrapperWALayer attachment)
# identical to the serial path.
self._presolved_qdq = None
return presolved
if self.orig_layer.bits >= 16:
return self.orig_layer.weight, None, None
min_scale, max_scale = self._prepare_minmax_scale(min_scale, max_scale)
weight = self.orig_layer.weight
if weight.device.type == "meta":
weight = self.orig_layer.get_weight().to(self.device)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight = weight.t()
quant_kwargs = {}
if hasattr(self.orig_layer, "super_bits"):
quant_kwargs["super_bits"] = self.orig_layer.super_bits
quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
if hasattr(self, "_extra_quant_kwargs"):
quant_kwargs.update(self._extra_quant_kwargs())
weight_q, scale, zp = self.weight_quant_func(
weight.to(self.device),
**self._quant_call_kwargs(
value,
min_scale,
max_scale,
imatrix_override=None,
),
)
weight_q = weight_q.to(weight.dtype)
if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
weight_q = weight_q.t()
return weight_q, scale, zp
def _qdq_act(self, x, act_min_scale=None, act_max_scale=None, act_max=None):
"""Quantizes and dequantizes activations.
Args:
x (torch.Tensor): Input activations.
act_max_scale (torch.Tensor): Maximum scale for the act_max
act_max (torch.Tensor, optional): Maximum value for activation quantization. Defaults to None.
Returns:
tuple: Quantized activation, scale, and zero point.
"""
if act_min_scale is None:
act_min_scale = torch.tensor(1.0)
if act_max_scale is None:
act_max_scale = torch.tensor(1.0)
act_max_scale.data.clamp_(0, 1.0)
act_min_scale.data.clamp_(0, 1.0)
env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set
x, scale, zp = self.act_quant_func(
x,
bits=self.orig_layer.act_bits,
group_size=self.orig_layer.act_group_size,
scale_dtype=self.orig_layer.scale_dtype,
q_scale_thresh=self.q_scale_thresh,
data_type=self.act_data_type,
tensor_max=act_max, # for static
max_scale=act_max_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
min_scale=act_min_scale if math.isclose(env_act_scale, 1.0, rel_tol=1e-6) else env_act_scale,
global_scale=getattr(self, "input_global_scale", None),
)
return x, scale, zp
def _qdq_bias(self, bias, bias_v):
"""Quantizes and dequantizes bias.
Args:
bias (torch.Tensor): Bias tensor to be quantized.
bias_v (torch.Tensor): Value added for rounding for tuning.
Returns:
tuple: Quantized bias, scale, and zero point.
"""
bias_bits = 4 ## hard code
bias_group_size = -1
bias, scale, zp = self.bias_quant_func(
bias,
bits=bias_bits,
group_size=bias_group_size,
v=bias_v,
q_scale_thresh=self.q_scale_thresh,
global_scale=getattr(self, "weight_global_scale", None),
)
return bias, scale, zp
def unwrapper(self, best_params):
"""Restores the original layer by applying the best tuning parameters.
Args:
best_params (dict): Dictionary containing the best tuning parameters.
Returns:
torch.nn.Module: The unwrapped and restored original layer.
"""
def _preserve_global_name(layer):
if hasattr(self.orig_layer, "global_name") and not hasattr(layer, "global_name"):
layer.global_name = self.orig_layer.global_name
return layer
best_params = best_params or {}
v = best_params.get("value", torch.tensor(0.0)).to(self.device)
min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)
max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)
if self.orig_layer.weight.device.type == "meta":
self.orig_layer.to(self.device)
# Unwrapper weight
qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
self._apply_qdq(qdq_weight, scale, zp)
# Unwrapper bias
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"].to(self.device)
bias = self.orig_layer.bias
if bias is not None and bias.device.type == "meta":
bias = self.orig_layer.get_bias().to(self.device)
bias, _, _ = self._qdq_bias(bias, bias_v)
self.orig_layer.bias.grad = None
self.orig_layer.bias.data.copy_(bias)
if hasattr(self.orig_layer, "update"):
self.orig_layer.update()
self.orig_layer.to("meta")
# Unwrapper act
if self.enable_act_quant:
act_min_scale = best_params.get("act_min_scale", torch.tensor(1.0)).detach().to(self.device).clone()
act_max_scale = best_params.get("act_max_scale", torch.tensor(1.0)).detach().to(self.device).clone()
act_min_scale.clamp_(0, 1.0)
act_max_scale.clamp_(0, 1.0)
if not self.orig_layer.act_dynamic:
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
if act_max is not None:
tmp_shape = 1
if self.orig_layer.act_group_size > 1:
tmp_shape = (act_max.shape[0], self.orig_layer.act_group_size)
elif self.orig_layer.act_group_size == -1:
tmp_shape = (act_max.shape[0], 1)
_, act_scale, _ = self._qdq_act(
torch.zeros(tmp_shape).to(self.device),
act_min_scale=act_min_scale,
act_max_scale=act_max_scale,
act_max=act_max,
)
# Keep calibration statistics separate from the tuned coefficients.
self.orig_layer.act_max = self.orig_layer.act_max.to("cpu")
else:
act_scale = torch.ones(1, dtype=self.orig_layer.scale_dtype)
self.orig_layer.act_scale = act_scale.to("cpu")
self.orig_layer.q_scale_thresh = self.q_scale_thresh
self.orig_layer.data_type = self.data_type
self.orig_layer.act_min_scale = act_min_scale
self.orig_layer.act_max_scale = act_max_scale
self.orig_layer.act_data_type = self.act_data_type
self.orig_layer.act_quant_func = self.act_quant_func
wrapper_layer = WrapperWALayer(
self.orig_layer,
enable_torch_compile=self.enable_torch_compile,
device=self.device,
)
return _preserve_global_name(wrapper_layer)
return _preserve_global_name(self.orig_layer)
def linear_forward(self, x, weight, bias):
"""Performs the forward pass for a linear layer.
Args:
x (torch.Tensor): Input tensor.
weight (torch.Tensor): Weight tensor for the linear layer.
bias (torch.Tensor): Bias tensor for the linear layer.
Returns:
torch.Tensor: Output tensor after applying the linear layer.
"""
return F.linear(x, weight, bias) # pylint: disable=E1102
def all_reduce_linear_forward(self, x, weight, bias):
"""Performs the forward pass for a linear layer.
Args:
x (torch.Tensor): Input tensor.
weight (torch.Tensor): Weight tensor for the linear layer.
bias (torch.Tensor): Bias tensor for the linear layer.
Returns:
torch.Tensor: Output tensor after applying the linear layer.
"""
output = torch.matmul(x, weight.transpose(-1, -2))
if self.mp_group is not None:
dist.inference_all_reduce(output, group=self.mp_group)
if bias is not None:
output += bias
return output
def conv1d_forward(self, x, weight, bias):
"""Performs the forward pass for a Conv1D layer.
Args:
x (torch.Tensor): Input tensor.
weight (torch.Tensor): Weight tensor for the Conv1D layer.
bias (torch.Tensor): Bias tensor for the Conv1D layer.
Returns:
torch.Tensor: Output tensor after applying the Conv1D layer.
"""
size_out = x.size()[:-1] + (self.orig_layer.nf,)
x = torch.addmm(bias, x.view(-1, x.size(-1)), weight)
x = x.view(*size_out)
return x
def forward(self, x):
"""Executes the forward pass with quantized weights and optional bias/activation quantization.
Args:
x (torch.Tensor): Input tensor.
Returns:
torch.Tensor: Output tensor after applying the wrapped layer.
"""
# logger.info(self.orig_layer.global_name)
x = x.to(self.device)
weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)
if self.enable_act_quant:
# Run orig_layer's forward_pre_hooks (e.g., online Hadamard transform)
# BEFORE activation quantization, to match inference behavior in WrapperWALayer.
for hook in self.orig_layer._forward_pre_hooks.values():
result = hook(self.orig_layer, (x,))
if result is not None:
x = result[0] if isinstance(result, tuple) else result
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
x, _, _ = self._qdq_act(
x, act_max_scale=self.act_max_scale, act_min_scale=self.act_min_scale, act_max=act_max
)
elif len(self.orig_layer._forward_pre_hooks) > 0:
# Even without act_quant, run pre-hooks for online Hadamard correctness
# (when fuse_online_to_weight=False, pre-hooks transform activations).
for hook in self.orig_layer._forward_pre_hooks.values():
result = hook(self.orig_layer, (x,))
if result is not None:
x = result[0] if isinstance(result, tuple) else result
# pylint: disable=not-callable
bias = self.orig_layer.bias
if bias is not None and bias.device.type == "meta":
bias = self.orig_layer.get_bias().to(self.device)
if self.enable_norm_bias_tuning:
bias, _, _ = self._qdq_bias(bias, self.bias_v)
output = self.orig_forward(x, weight_q, bias).to(self.output_device)
# Execute post-hooks from orig_layer (e.g., v_proj per-head Hadamard
# when online rotation is not fused into weights).
for hook in self.orig_layer._forward_hooks.values():
hook_result = hook(self.orig_layer, (x,), output)
if hook_result is not None:
output = hook_result
return output
class WrapperWALayer(torch.nn.Module):
def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):
super().__init__()
self.orig_layer = orig_layer
self.enable_torch_compile = enable_torch_compile
self.device = device
self.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else None
self.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else None
self.act_quant_func = self.orig_layer.act_quant_func
if self.enable_torch_compile:
self.act_quant_func = compile_func(self.act_quant_func, self.device)
self.extra_repr_org = orig_layer.extra_repr
# Steal forward_pre_hooks from orig_layer (e.g., Hadamard transform hooks)
# and remove them from orig_layer so they won't fire again inside orig_layer.forward().
# We will run them explicitly in our forward() BEFORE activation quantization.
self._stolen_pre_hooks = list(orig_layer._forward_pre_hooks.values())
orig_layer._forward_pre_hooks.clear()
@property
def weight(self):
"""Exposes the weight of the wrapped layer for external access."""
return self.orig_layer.weight
@property
def bias(self):
"""Exposes the bias of the wrapped layer for external access."""
return self.orig_layer.bias
def forward(self, x):
# 1) Run stolen pre_hooks first (e.g., online Hadamard) → smooths activation
for hook in self._stolen_pre_hooks:
result = hook(self.orig_layer, (x,))
if result is not None:
x = result[0] if isinstance(result, tuple) else result
# 2) Activation quantization on the smoothed activation
import auto_round.envs as envs
act_scale = envs.AR_ACT_SCALE
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
max_scale = self.orig_layer.act_max_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
min_scale = self.orig_layer.act_min_scale if math.isclose(act_scale, 1.0, rel_tol=1e-6) else act_scale
quant_kwargs = {}
if hasattr(self.orig_layer, "input_global_scale"):
quant_kwargs["global_scale"] = self.orig_layer.input_global_scale
x, _, _ = self.orig_layer.act_quant_func(
x,
bits=self.orig_layer.act_bits,
group_size=self.orig_layer.act_group_size,
scale_dtype=self.orig_layer.scale_dtype,
q_scale_thresh=self.orig_layer.q_scale_thresh,
data_type=self.orig_layer.act_data_type,
tensor_max=act_max,
min_scale=min_scale,
max_scale=max_scale,
**quant_kwargs,
)
# 3) Linear computation via orig_layer (pre_hooks already removed, no double execution)
return self.orig_layer.forward(x)
def extra_repr(self):
return f"{self.extra_repr_org()}, weight_type={self.data_type}, act_data_type={self.act_data_type}"
class WrapperLayerNorm(torch.nn.Module):
"""A wrapper for layer normalization with quantized weights.
This class wraps a given layer normalization module and applies quantization without round
to its weights. The quantization is parameterized by the number of bits and
an optional group size.
"""
def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):
super().__init__()
self.orig_layer = orig_layer
self.bits = bit
self.group_size = group_size
self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.output_device = device
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
reshape_pad_tensor_by_group_size(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
)[0],
requires_grad=True,
)
self.params = {"v": self.v}
_restore_warm_start_param(self.orig_layer, "v", self.v)
from auto_round.data_type.int import quant_tensor_asym_wo_round
self.quant_func = quant_tensor_asym_wo_round
def unwrapper(self, best_params):
if best_params is None:
return self.orig_layer
v = best_params["v"].to(self.device)
weight_q, _, _ = self.quant_func(
self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh
)
self.orig_layer.q_scale_thresh = self.q_scale_thresh
self.orig_layer.weight.data.copy_(weight_q)
return self.orig_layer
def forward(self, input):
input = input.to(self.device)
weight_q, _, _ = self.quant_func(
self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh
)
import torch.nn.functional as F
return F.layer_norm(
input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps
).to(self.output_device)
class WrapperLlamaNorm(torch.nn.Module):
"""A wrapper for Llama normalization in HF with fake quantized weights without rounding.
This class wraps a given layer normalization module and applies quantization without rounding
to its weights. The quantization is parameterized by the number of bits and
an optional group size.
"""
def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):
super().__init__()
self.orig_layer = orig_layer
self.bits = bit
self.group_size = group_size
self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.output_device = device
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
reshape_pad_tensor_by_group_size(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size
)[0],
requires_grad=True,
)
self.params = {"v": self.v}
_restore_warm_start_param(self.orig_layer, "v", self.v)
from auto_round.data_type.int import quant_tensor_asym_wo_round
self.quant_func = quant_tensor_asym_wo_round
def unwrapper(self, best_params):
if best_params is None:
return self.orig_layer
v = best_params["v"].to(self.device)
weight_q, _, _ = self.quant_func(
self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh
)
self.orig_layer.q_scale_thresh = self.q_scale_thresh
self.orig_layer.weight.data.copy_(weight_q)
return self.orig_layer
def forward(self, hidden_states):
hidden_states = hidden_states.to(self.device)
weight_q, _, _ = self.quant_func(
self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh
)
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.orig_layer.variance_epsilon)
return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)
NORM_MAPPING = {}
NORM_MAPPING["LayerNorm"] = WrapperLayerNorm
NORM_MAPPING["LlamaRMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen2RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Phi3RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["MistralRMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen3RMSNorm"] = WrapperLlamaNorm
NORM_MAPPING["Qwen3_5MoeRMSNorm"] = WrapperLlamaNorm
class WrapperMultiblock(torch.nn.Module):
"""A wrapper for a list of modules to be act as a single block.
Args:
module_list: The list of modules to wrap.
"""
def __init__(self, module_list):
super().__init__()
self.layers = torch.nn.ModuleList(module_list)
def forward(self, x, *args, **kwargs):
hidden_states = x
for idx, decoder_layer in enumerate(self.layers):
layer_outputs = decoder_layer(hidden_states, *args, **kwargs)
hidden_states = layer_outputs
if isinstance(hidden_states, (tuple, list)):
hidden_states = layer_outputs[0]
return hidden_states
def wrapper_block(
block,
enable_minmax_tuning,
enable_norm_bias_tuning,
enable_torch_compile=True,
device="cpu",
wrapper_cls=WrapperLinear,
**kwargs,
):
"""Wraps the layers in the given block with a custom Wrapper module.
Args:
block: The input block containing linear and conv1d layers to be wrapped.
enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.
enable_norm_bias_tuning: A boolean indicating whether normalization and bias tuning is enabled.
enable_torch_compile: A boolean indicating whether to enable torch compilation.
device: The device to which the wrapped layers should be moved.
Returns:
list: A list of names of the wrapped layers and unwrapped layers.
"""
quantized_layers = []
unquantized_layers = []
# Wrap-time searches (SignRoundV2 init-scale, GGUF DQ scale) read only the
# module's own weight plus per-module statistics, never calibration
# activations; eligible searches run later in one stacked batched call per
# (device, shape, config, search fn).
work = []
for n, m in block.named_modules():
if type(m) in SUPPORTED_LAYER_TYPES:
work.append((n, m, "skip" if not check_to_quantized(m) else "linear"))
elif enable_norm_bias_tuning and "norm" in m.__class__.__name__.lower():
work.append((n, m, "norm"))
from auto_round.algorithms.quantization import search_dispatch
_defer_batch = (
bool(getattr(wrapper_cls, "supports_batched_search", False)) and not search_dispatch.batched_search_disabled()
)
deferred_wrappers = []
def _wrap_one(n: str, m: torch.nn.Module, kind: str) -> torch.nn.Module:
if kind == "norm":
if m.__class__.__name__ in NORM_MAPPING.keys():
wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]
new_m = wrapper_layer_class(m, device=device)
set_module(block, n, new_m)
elif "RMSNorm" in m.__class__.__name__:
logger.warning_once(
f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself"
)
wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]
new_m = wrapper_layer_class(m, device=device)
set_module(block, n, new_m)
else:
logger.warning_once(f"{m.__class__.__name__} is not supported")
return None
if kind == "skip":
return "u"
layer_wrapper_cls = getattr(m, "_ar_warm_start_wrapper_cls", wrapper_cls)
defer_search = _defer_batch and getattr(layer_wrapper_cls, "supports_batched_search", False)
extra = {"defer_search": True} if defer_search else {}
new_m = layer_wrapper_cls(
m,
enable_minmax_tuning=enable_minmax_tuning,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
device=device,
**kwargs,
**extra,
)
set_module(block, n, new_m)
if defer_search:
deferred_wrappers.append(new_m)
return "q"
results = [_wrap_one(n, m, kind) for n, m, kind in work]
if _defer_batch and deferred_wrappers:
anchor_pending = [w for w in deferred_wrappers if getattr(w, "_deferred_neuqi_anchor", None) is not None]
if anchor_pending:
if not search_dispatch.run_batched_neuqi_anchors(anchor_pending):
for w in anchor_pending:
w._run_deferred_neuqi_anchor_now()
# stacked same-shape batches per (device, shape, config, search fn);
# singletons and the kill-switch fall back to the identical per-module call.
# Data-parallel ranks shard the searches across the ranks instead.
from auto_round.algorithms.parallel.spmd import current_rank, run_rank_sharded_wrap_searches
rank = current_rank()
if not (rank is not None and run_rank_sharded_wrap_searches(rank, deferred_wrappers)):
if not search_dispatch.run_batched_wrap_search(deferred_wrappers):
for w in deferred_wrappers:
w._run_deferred_search_now()
quantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "q"]
unquantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "u"]
return quantized_layers, unquantized_layers
@torch.no_grad()
def unwrapper_layer(model, layer, layer_name, best_params):
"""Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.
Args:
block: The input block containing wrapped modules to be unwrapped.
vs: A dictionary of scaling parameters for the wrapped modules.
min_scales: A dictionary of minimum scaling values for the wrapped modules.
max_scales: A dictionary of maximum scaling values for the wrapped modules.
"""
# WrapperWALayer retains activation QDQ and is not a tuning wrapper.
if hasattr(layer, "orig_layer") and callable(getattr(layer, "unwrapper", None)):
orig_layer = layer.unwrapper(best_params)
act_max = getattr(orig_layer, "act_max", None)
act_scale = getattr(orig_layer, "act_scale", None)
if (
"lm_head" in layer_name
and getattr(layer, "enable_act_quant", False)
and not getattr(orig_layer, "act_dynamic", True)
and act_scale is not None
and act_max is None
):
logger.warning_once(
"Static activation quantization for lm_head is not fully supported yet. "
"lm_head activation statistics are missing, so activation scale falls back to unit scale."
)
orig_layer = orig_layer.to("cpu")
set_module(model, layer_name, orig_layer)
@torch.no_grad()
def unwrapper_block(block, best_params):
"""Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.
Args:
block: The input block containing wrapped modules to be unwrapped.
vs: A dictionary of scaling parameters for the wrapped modules.
min_scales: A dictionary of minimum scaling values for the wrapped modules.
max_scales: A dictionary of maximum scaling values for the wrapped modules.
"""
for n, m in block.named_modules():
# Keep persistent activation wrappers, which do not implement unwrapper.
if hasattr(m, "orig_layer") and callable(getattr(m, "unwrapper", None)):
if n in best_params.keys():
best_param = best_params[n]
else:
best_param = None
orig_layer = m.unwrapper(best_param)
set_module(block, n, orig_layer)
@contextmanager
def preserve_original_weights(block):
"""Keep a host snapshot for a composer-owned stage that must not bake weights.
This also covers algorithms that modify parameters in place instead of using
common wrappers. Only the current block is copied on block-tuning paths;
ordinary/final stages do not pay this cost. Restore the original module graph
and parameter dtype/values on success or failure, leaving warm-start metadata
on the original layers available to subsequent stages.
"""
modules = list(block.named_modules())
parameters = [(p, p.detach().to("cpu", copy=True), p.requires_grad) for p in block.parameters()]
quantization_types = [
(module, key, getattr(module, key))
for _, module in modules
for key in ("data_type", "act_data_type")
if hasattr(module, key)
]
try:
yield
finally:
with torch.no_grad():
for name, module in modules:
if name:
set_module(block, name, module)
for parameter, saved, requires_grad in parameters:
parameter.data = saved.to(parameter.device)
parameter.requires_grad_(requires_grad)
# RTN writes resolved one-shot dtype names during finalization.
# Subsequent training stages need the original differentiable types.
for module, key, value in quantization_types:
setattr(module, key, value)
@torch.no_grad()
def store_warm_start_params_and_unwrap(block, best_params, *, before_unwrap=None):
"""Warm-start variant of :func:`unwrapper_block`: keep the original FP weight.
Instead of baking the tuned ``value``/``min_scale``/``max_scale`` into the
(quantized) weight like :func:`unwrapper_block` does, this stashes them on
each ``orig_layer`` under ``_ar_warm_start`` and restores the untouched
full-precision layer. A later pass (e.g. QAD whole-model tuning) can then
re-wrap the same layers and seed its tuning parameters from the stash via the
warm-start branch in :meth:`WrapperLinear._init_tuning_params_and_quant_func`.
The optimized SignRound wrapper also exposes a searched ``init_scale`` (not a
trainable parameter); it is stashed too so the second pass reuses the exact
same scale grid instead of re-searching it.
Args:
block: The block (or single wrapped layer) whose wrappers to collect.
best_params: Result of ``collect_best_params`` -- either a flat
``{param_name: tensor}`` dict (single-wrapper block) or
``{submodule_name: {param_name: tensor}}``.
before_unwrap: Optional callback receiving the still-wrapped block with
its best parameters loaded. Used to collect quantized outputs before
restoring FP layers; restoration also happens if the callback raises.
"""
best_params = best_params or {}
wrapped_layers = [
(n, m)
for n, m in block.named_modules()
if hasattr(m, "orig_layer") and hasattr(m, "params") and callable(getattr(m, "unwrapper", None))
]
try:
for n, m in wrapped_layers:
if n in best_params:
layer_params = best_params[n]
elif any(key in best_params for key in m.params):
# Flat dict: the block itself is the wrapped layer.
layer_params = best_params
else:
layer_params = {}
store = {}
for key in m.params:
if key in layer_params and layer_params[key] is not None:
store[key] = layer_params[key].detach().to("cpu", copy=True)
store["minmax_parameterization"] = "log" if getattr(m, "_use_log_minmax", False) else "linear"
init_scale = getattr(m, "init_scale", None)
if init_scale is not None:
store["init_scale"] = init_scale.detach().to("cpu")
# Asym optimized wrapper: preserve the searched grid anchor too.
if getattr(m, "_asym_anchor", False):
store["anchor_min"] = m.weight_min.detach().to("cpu")
store["anchor_max"] = m.weight_max.detach().to("cpu")
m.orig_layer._ar_warm_start = store
if isinstance(m, WrapperLinear):
m.orig_layer._ar_warm_start_wrapper_cls = type(m)
if before_unwrap is not None:
for key, saved in layer_params.items():
param = m.params.get(key)
if param is not None and saved is not None:
param.copy_(saved.to(device=param.device, dtype=param.dtype))
if before_unwrap is not None:
before_unwrap(block)
finally:
for n, m in wrapped_layers:
set_module(block, n, m.orig_layer)