PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2023 MIT HAN Lab
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import gc
import warnings
import torch
import torch.nn as nn
from torch.autograd import Function
from auto_round.utils import get_packing_device
from auto_round.utils.bit_packing import (
pack_awq_bitstream,
pack_scalar_zero,
requires_generic_bit_packing,
unpack_awq_bitstream,
)
# Bit-widths the AWQ packing layout can represent. AutoAWQ's classic layout
# interleaves 8 values per int32 word, which only lines up at 4 bits; 5/6/7-bit
# keep the same 8-value interleave but write it into the generic bit-stream
# layout (exactly what humming's AWQWeightSchema expects).
SUPPORTED_AWQ_BITS = (4, 5, 6, 7)
def awq_gemm_kernel_supported(in_features: int, out_features: int, bits: int, group_size: int) -> bool:
"""Whether the AWQ GEMM kernel can serve a layer of the given shape.
The kernel requires ``in_features`` and ``out_features`` to be multiples of
``group_size`` (``IC/OC is not multiple of Group size``) and ``out_features``
to be a multiple of the 64-wide CTA tile. Bit widths outside
``SUPPORTED_AWQ_BITS`` have no servable packing layout. A non-positive
``group_size`` (per-channel quantization) is treated as servable here;
per-channel compatibility is validated elsewhere.
"""
if bits not in SUPPORTED_AWQ_BITS:
return False
if not isinstance(group_size, int) or group_size <= 0:
return True
return in_features % group_size == 0 and out_features % group_size == 0 and out_features % 64 == 0
def awq_user_forced_quantization(cfg) -> bool:
"""Whether a ``layer_config`` entry explicitly keeps this layer quantized.
User-supplied entries (exact names or expanded regex matches) carry
``fixed_by_user=True`` once normalized; entries AutoRound filled in from
the default scheme carry ``fixed_by_user=False``. At format-resolution
time user entries can still be raw (a dict missing the flag, a preset
string, or a QuantizationScheme), so any present entry counts as user
intent unless it explicitly says ``fixed_by_user=False``.
After ``apply_plan_to_model`` the same flag is also available on the
module itself as ``layer.fixed_by_user``; see ``awq_layer_user_forced``.
"""
if not cfg:
return False
if not hasattr(cfg, "get"):
return True
return cfg.get("fixed_by_user", True) is not False
def awq_layer_user_forced(layer, cfg=None) -> bool:
"""Whether a quantized module explicitly keeps an AWQ-unservable shape quantized.
Reads ``layer.fixed_by_user`` written by ``apply_plan_to_model``. A module
without the flag falls back to its ``layer_config`` entry when given, and
otherwise follows the same rule as ``awq_user_forced_quantization`` for a
plan entry missing the key (e.g. lm_head added via ``quant_lm_head``), so
the marking step and the packers agree.
"""
flag = getattr(layer, "fixed_by_user", None)
if flag is not None:
return bool(flag)
return cfg is None or awq_user_forced_quantization(cfg)
def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
shifts = torch.arange(0, 32, bits, device=qzeros.device)
# unpacking columnwise
iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
torch.int8 # smallest dtype available
)
iweights = iweights.view(iweights.shape[0], -1)
# unpacking columnwise
if qzeros is not None:
izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
torch.int8 # smallest dtype available
)
izeros = izeros.view(izeros.shape[0], -1)
else:
izeros = qzeros
return iweights, izeros
AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):
reverse_order_tensor = torch.arange(
iweights.shape[-1],
dtype=torch.int32,
device=izeros.device,
)
reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
reverse_order_tensor = reverse_order_tensor.view(-1)
if izeros is not None:
izeros = izeros[:, reverse_order_tensor]
iweights = iweights[:, reverse_order_tensor]
return iweights, izeros
def dequantize_gemm(qweight, qzeros, scales, bits, group_size):
if requires_generic_bit_packing(bits):
# 5/6/7-bit: values straddle int32 words, so unpack via the generic
# bit-stream helper (which also undoes the AWQ interleave).
iweight = unpack_awq_bitstream(qweight, bits, dim=1)
izeros = unpack_awq_bitstream(qzeros, bits, dim=1)
else:
# Unpack the qweight and qzeros tensors
iweight, izeros = unpack_awq(qweight, qzeros, bits)
# Reverse the order of the iweight and izeros tensors
iweight, izeros = reverse_awq_order(iweight, izeros, bits)
# overflow checks
iweight = torch.bitwise_and(iweight, (2**bits) - 1)
izeros = torch.bitwise_and(izeros, (2**bits) - 1)
# fp16 weights
scales = scales.repeat_interleave(group_size, dim=0)
izeros = izeros.repeat_interleave(group_size, dim=0)
iweight = (iweight - izeros) * scales
return iweight
class WQLinearMMFunction(Function):
@staticmethod
# ctx is the first argument to forward
def forward(
ctx,
x,
qweight,
qzeros,
scales,
w_bit=4,
group_size=128,
bias=None,
out_features=0,
):
# The forward pass can use ctx.
ctx.save_for_backward(x, qweight, qzeros, scales, bias)
ctx.out_features = out_features
out_shape = x.shape[:-1] + (out_features,)
x = x.to(torch.float16)
out = dequantize_gemm(qweight, qzeros, scales, w_bit, group_size)
out = torch.matmul(x, out)
out = out + bias if bias is not None else out
out = out.reshape(out_shape)
# always want 3D tensor if tensor is 2D
if len(out.shape) == 2:
out = out.unsqueeze(0)
return out
class WQLinear_GEMM(nn.Module):
def __init__(self, w_bit, group_size, in_features, out_features, bias, dev, training=False):
super().__init__()
if w_bit not in SUPPORTED_AWQ_BITS:
raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_AWQ_BITS))}-bit are supported for now.")
self.in_features = in_features
self.out_features = out_features
self.w_bit = w_bit
self.group_size = group_size if group_size != -1 else in_features
self.training = training
self.use_generic_bit_packing = requires_generic_bit_packing(w_bit)
# quick sanity check (make sure alignment)
if self.in_features % self.group_size != 0:
raise ValueError(f"in_features ({self.in_features}) shape mismatch")
# 2/4/8-bit keep AutoAWQ's original ``32 // w_bit`` alignment rule; 5/6/7-bit
# pack a contiguous bit-stream over blocks of 32 values, so the output
# dimension must be a multiple of 32.
pack_align = 32 if self.use_generic_bit_packing else 32 // self.w_bit
if out_features % pack_align != 0:
raise ValueError(f"out_features ({out_features}) must be a multiple of {pack_align} for {w_bit}-bit AWQ")
packed_out_features = out_features * self.w_bit // 32
self.register_buffer(
"qweight",
torch.zeros(
(in_features, packed_out_features),
dtype=torch.int32,
device=dev,
),
)
self.register_buffer(
"qzeros",
torch.zeros(
(in_features // self.group_size, packed_out_features),
dtype=torch.int32,
device=dev,
),
)
self.register_buffer(
"scales",
torch.zeros(
(in_features // self.group_size, out_features),
dtype=torch.float16,
device=dev,
),
)
if bias:
self.register_buffer(
"bias",
torch.zeros(
(out_features),
dtype=torch.float16,
device=dev,
),
)
else:
self.bias = None
@classmethod
def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, device=None):
device = get_packing_device(device)
awq_linear = cls(
w_bit,
group_size,
linear.in_features,
linear.out_features,
linear.bias is not None,
linear.weight.device,
)
if init_only: # just prepare for loading sd
return awq_linear
# need scales and zeros info for real quantization
if scales is None or zeros is None:
raise ValueError("Both 'scales' and 'zeros' must be provided (not None)")
awq_linear.scales = scales.clone().half()
if linear.bias is not None:
awq_linear.bias = linear.bias.clone().half()
pack_num = 32 // awq_linear.w_bit
maxq = 2**awq_linear.w_bit - 1
repeat_size = group_size if group_size != -1 else linear.in_features
repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)
# clamp into [0, maxq]: the searched (scale, zp) grid may leave
# round(W/s + zp) outside the level range (the search evaluates
# the clamped loss); an unclamped value shifts bits into the
# neighboring packed nibble and silently corrupts it.
intweight = (
torch.round(
linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]]
+ repeat_zeros[:, : linear.weight.shape[1]]
)
.clamp_(0, maxq)
.to(torch.int)
.t()
.contiguous()
)
else:
repeat_zeros = zeros
intweight = (
torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]] + repeat_zeros)
.clamp_(0, maxq)
.to(torch.int)
.t()
.contiguous()
)
intweight = intweight.to(dtype=torch.int32)
del repeat_scales
if awq_linear.use_generic_bit_packing:
# 5/6/7-bit: AWQ interleave (groups of 8) + generic bit-stream packing.
awq_linear.qweight = pack_awq_bitstream(intweight, awq_linear.w_bit, dim=1).to("cpu")
packed_cols = scales.shape[1] * awq_linear.w_bit // 32
if isinstance(zeros, torch.Tensor):
zeros = zeros.to(dtype=torch.int32, device=device)
qzeros = pack_awq_bitstream(zeros, awq_linear.w_bit, dim=1)
else:
# A single shared zero point is interleave-invariant, so it can be
# tiled directly without reordering.
qzeros = pack_scalar_zero(
int(zeros), awq_linear.w_bit, packed_cols, (scales.shape[0], packed_cols), device=device
)
awq_linear.qzeros = qzeros.to("cpu")
return awq_linear
intweight = intweight.reshape(-1, intweight.shape[1] // pack_num, pack_num)
new_order_map = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7], device=device) * awq_linear.w_bit
intweight = intweight << new_order_map
intweight = torch.sum(intweight, dim=-1).to(torch.int32)
awq_linear.qweight = intweight.to("cpu")
if isinstance(zeros, torch.Tensor):
zeros = zeros.to(dtype=torch.int32, device=device)
zeros = zeros.reshape(-1, zeros.shape[1] // pack_num, pack_num)
zeros = zeros << new_order_map
qzeros = torch.sum(zeros, dim=-1).to(torch.int32)
else:
value = 0
for i in range(pack_num):
value |= zeros << (i * awq_linear.w_bit)
qzeros = (
torch.ones(
(scales.shape[0], scales.shape[1] // pack_num),
dtype=torch.int32,
device=device,
)
* value
)
awq_linear.qzeros = qzeros.to("cpu")
return awq_linear
def forward(self, x):
out_shape = x.shape[:-1] + (self.out_features,)
input_dtype = x.dtype
if input_dtype != torch.float16:
x = x.half()
if self.training:
out = WQLinearMMFunction.apply(
x,
self.qweight,
self.qzeros,
self.scales,
self.w_bit,
self.group_size,
self.bias,
self.out_features,
)
else:
with torch.no_grad():
out = WQLinearMMFunction.apply(
x,
self.qweight,
self.qzeros,
self.scales,
self.w_bit,
self.group_size,
self.bias,
self.out_features,
)
if input_dtype != torch.float16:
out = out.to(dtype=input_dtype)
return out.reshape(out_shape)
def extra_repr(self) -> str:
return f"in_features={self.in_features}, out_features={self.out_features}, bias={self.bias is not None}, w_bit={self.w_bit}, group_size={self.group_size}"