Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

15.1 KB
Raw Blame
# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright (c) 2023 MIT HAN Lab

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import gc
import warnings

import torch
import torch.nn as nn
from torch.autograd import Function

from auto_round.utils import get_packing_device
from auto_round.utils.bit_packing import (
    pack_awq_bitstream,
    pack_scalar_zero,
    requires_generic_bit_packing,
    unpack_awq_bitstream,
)

# Bit-widths the AWQ packing layout can represent. AutoAWQ's classic layout
# interleaves 8 values per int32 word, which only lines up at 4 bits; 5/6/7-bit
# keep the same 8-value interleave but write it into the generic bit-stream
# layout (exactly what humming's AWQWeightSchema expects).
SUPPORTED_AWQ_BITS = (4, 5, 6, 7)


def awq_gemm_kernel_supported(in_features: int, out_features: int, bits: int, group_size: int) -> bool:
    """Whether the AWQ GEMM kernel can serve a layer of the given shape.

    The kernel requires ``in_features`` and ``out_features`` to be multiples of
    ``group_size`` (``IC/OC is not multiple of Group size``) and ``out_features``
    to be a multiple of the 64-wide CTA tile. Bit widths outside
    ``SUPPORTED_AWQ_BITS`` have no servable packing layout. A non-positive
    ``group_size`` (per-channel quantization) is treated as servable here;
    per-channel compatibility is validated elsewhere.
    """
    if bits not in SUPPORTED_AWQ_BITS:
        return False
    if not isinstance(group_size, int) or group_size <= 0:
        return True
    return in_features % group_size == 0 and out_features % group_size == 0 and out_features % 64 == 0


def awq_user_forced_quantization(cfg) -> bool:
    """Whether a ``layer_config`` entry explicitly keeps this layer quantized.

    User-supplied entries (exact names or expanded regex matches) carry
    ``fixed_by_user=True`` once normalized; entries AutoRound filled in from
    the default scheme carry ``fixed_by_user=False``. At format-resolution
    time user entries can still be raw (a dict missing the flag, a preset
    string, or a QuantizationScheme), so any present entry counts as user
    intent unless it explicitly says ``fixed_by_user=False``.

    After ``apply_plan_to_model`` the same flag is also available on the
    module itself as ``layer.fixed_by_user``; see ``awq_layer_user_forced``.
    """
    if not cfg:
        return False
    if not hasattr(cfg, "get"):
        return True
    return cfg.get("fixed_by_user", True) is not False


def awq_layer_user_forced(layer, cfg=None) -> bool:
    """Whether a quantized module explicitly keeps an AWQ-unservable shape quantized.

    Reads ``layer.fixed_by_user`` written by ``apply_plan_to_model``. A module
    without the flag falls back to its ``layer_config`` entry when given, and
    otherwise follows the same rule as ``awq_user_forced_quantization`` for a
    plan entry missing the key (e.g. lm_head added via ``quant_lm_head``), so
    the marking step and the packers agree.
    """
    flag = getattr(layer, "fixed_by_user", None)
    if flag is not None:
        return bool(flag)
    return cfg is None or awq_user_forced_quantization(cfg)


def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
    shifts = torch.arange(0, 32, bits, device=qzeros.device)

    # unpacking columnwise
    iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
        torch.int8  # smallest dtype available
    )
    iweights = iweights.view(iweights.shape[0], -1)

    # unpacking columnwise
    if qzeros is not None:
        izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
            torch.int8  # smallest dtype available
        )
        izeros = izeros.view(izeros.shape[0], -1)
    else:
        izeros = qzeros

    return iweights, izeros


AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]


def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):
    reverse_order_tensor = torch.arange(
        iweights.shape[-1],
        dtype=torch.int32,
        device=izeros.device,
    )
    reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
    reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
    reverse_order_tensor = reverse_order_tensor.view(-1)

    if izeros is not None:
        izeros = izeros[:, reverse_order_tensor]
    iweights = iweights[:, reverse_order_tensor]

    return iweights, izeros


def dequantize_gemm(qweight, qzeros, scales, bits, group_size):
    if requires_generic_bit_packing(bits):
        # 5/6/7-bit: values straddle int32 words, so unpack via the generic
        # bit-stream helper (which also undoes the AWQ interleave).
        iweight = unpack_awq_bitstream(qweight, bits, dim=1)
        izeros = unpack_awq_bitstream(qzeros, bits, dim=1)
    else:
        # Unpack the qweight and qzeros tensors
        iweight, izeros = unpack_awq(qweight, qzeros, bits)
        # Reverse the order of the iweight and izeros tensors
        iweight, izeros = reverse_awq_order(iweight, izeros, bits)

    # overflow checks
    iweight = torch.bitwise_and(iweight, (2**bits) - 1)
    izeros = torch.bitwise_and(izeros, (2**bits) - 1)

    # fp16 weights
    scales = scales.repeat_interleave(group_size, dim=0)
    izeros = izeros.repeat_interleave(group_size, dim=0)
    iweight = (iweight - izeros) * scales

    return iweight


class WQLinearMMFunction(Function):

    @staticmethod
    # ctx is the first argument to forward
    def forward(
        ctx,
        x,
        qweight,
        qzeros,
        scales,
        w_bit=4,
        group_size=128,
        bias=None,
        out_features=0,
    ):
        # The forward pass can use ctx.
        ctx.save_for_backward(x, qweight, qzeros, scales, bias)
        ctx.out_features = out_features

        out_shape = x.shape[:-1] + (out_features,)
        x = x.to(torch.float16)

        out = dequantize_gemm(qweight, qzeros, scales, w_bit, group_size)
        out = torch.matmul(x, out)

        out = out + bias if bias is not None else out
        out = out.reshape(out_shape)

        # always want 3D tensor if tensor is 2D
        if len(out.shape) == 2:
            out = out.unsqueeze(0)

        return out


class WQLinear_GEMM(nn.Module):

    def __init__(self, w_bit, group_size, in_features, out_features, bias, dev, training=False):
        super().__init__()

        if w_bit not in SUPPORTED_AWQ_BITS:
            raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_AWQ_BITS))}-bit are supported for now.")

        self.in_features = in_features
        self.out_features = out_features
        self.w_bit = w_bit
        self.group_size = group_size if group_size != -1 else in_features
        self.training = training
        self.use_generic_bit_packing = requires_generic_bit_packing(w_bit)

        # quick sanity check (make sure alignment)
        if self.in_features % self.group_size != 0:
            raise ValueError(f"in_features ({self.in_features}) shape mismatch")

        # 2/4/8-bit keep AutoAWQ's original ``32 // w_bit`` alignment rule; 5/6/7-bit
        # pack a contiguous bit-stream over blocks of 32 values, so the output
        # dimension must be a multiple of 32.
        pack_align = 32 if self.use_generic_bit_packing else 32 // self.w_bit
        if out_features % pack_align != 0:
            raise ValueError(f"out_features ({out_features}) must be a multiple of {pack_align} for {w_bit}-bit AWQ")
        packed_out_features = out_features * self.w_bit // 32

        self.register_buffer(
            "qweight",
            torch.zeros(
                (in_features, packed_out_features),
                dtype=torch.int32,
                device=dev,
            ),
        )
        self.register_buffer(
            "qzeros",
            torch.zeros(
                (in_features // self.group_size, packed_out_features),
                dtype=torch.int32,
                device=dev,
            ),
        )
        self.register_buffer(
            "scales",
            torch.zeros(
                (in_features // self.group_size, out_features),
                dtype=torch.float16,
                device=dev,
            ),
        )
        if bias:
            self.register_buffer(
                "bias",
                torch.zeros(
                    (out_features),
                    dtype=torch.float16,
                    device=dev,
                ),
            )
        else:
            self.bias = None

    @classmethod
    def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, device=None):
        device = get_packing_device(device)
        awq_linear = cls(
            w_bit,
            group_size,
            linear.in_features,
            linear.out_features,
            linear.bias is not None,
            linear.weight.device,
        )
        if init_only:  # just prepare for loading sd
            return awq_linear

        # need scales and zeros info for real quantization
        if scales is None or zeros is None:
            raise ValueError("Both 'scales' and 'zeros' must be provided (not None)")

        awq_linear.scales = scales.clone().half()
        if linear.bias is not None:
            awq_linear.bias = linear.bias.clone().half()

        pack_num = 32 // awq_linear.w_bit
        maxq = 2**awq_linear.w_bit - 1
        repeat_size = group_size if group_size != -1 else linear.in_features
        repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)
        if isinstance(zeros, torch.Tensor):
            repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)
            # clamp into [0, maxq]: the searched (scale, zp) grid may leave
            # round(W/s + zp) outside the level range (the search evaluates
            # the clamped loss); an unclamped value shifts bits into the
            # neighboring packed nibble and silently corrupts it.
            intweight = (
                torch.round(
                    linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]]
                    + repeat_zeros[:, : linear.weight.shape[1]]
                )
                .clamp_(0, maxq)
                .to(torch.int)
                .t()
                .contiguous()
            )

        else:
            repeat_zeros = zeros
            intweight = (
                torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]] + repeat_zeros)
                .clamp_(0, maxq)
                .to(torch.int)
                .t()
                .contiguous()
            )

        intweight = intweight.to(dtype=torch.int32)
        del repeat_scales

        if awq_linear.use_generic_bit_packing:
            # 5/6/7-bit: AWQ interleave (groups of 8) + generic bit-stream packing.
            awq_linear.qweight = pack_awq_bitstream(intweight, awq_linear.w_bit, dim=1).to("cpu")
            packed_cols = scales.shape[1] * awq_linear.w_bit // 32
            if isinstance(zeros, torch.Tensor):
                zeros = zeros.to(dtype=torch.int32, device=device)
                qzeros = pack_awq_bitstream(zeros, awq_linear.w_bit, dim=1)
            else:
                # A single shared zero point is interleave-invariant, so it can be
                # tiled directly without reordering.
                qzeros = pack_scalar_zero(
                    int(zeros), awq_linear.w_bit, packed_cols, (scales.shape[0], packed_cols), device=device
                )
            awq_linear.qzeros = qzeros.to("cpu")
            return awq_linear

        intweight = intweight.reshape(-1, intweight.shape[1] // pack_num, pack_num)

        new_order_map = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7], device=device) * awq_linear.w_bit
        intweight = intweight << new_order_map
        intweight = torch.sum(intweight, dim=-1).to(torch.int32)
        awq_linear.qweight = intweight.to("cpu")

        if isinstance(zeros, torch.Tensor):
            zeros = zeros.to(dtype=torch.int32, device=device)
            zeros = zeros.reshape(-1, zeros.shape[1] // pack_num, pack_num)
            zeros = zeros << new_order_map
            qzeros = torch.sum(zeros, dim=-1).to(torch.int32)

        else:
            value = 0
            for i in range(pack_num):
                value |= zeros << (i * awq_linear.w_bit)
            qzeros = (
                torch.ones(
                    (scales.shape[0], scales.shape[1] // pack_num),
                    dtype=torch.int32,
                    device=device,
                )
                * value
            )

        awq_linear.qzeros = qzeros.to("cpu")

        return awq_linear

    def forward(self, x):
        out_shape = x.shape[:-1] + (self.out_features,)

        input_dtype = x.dtype
        if input_dtype != torch.float16:
            x = x.half()

        if self.training:
            out = WQLinearMMFunction.apply(
                x,
                self.qweight,
                self.qzeros,
                self.scales,
                self.w_bit,
                self.group_size,
                self.bias,
                self.out_features,
            )
        else:
            with torch.no_grad():
                out = WQLinearMMFunction.apply(
                    x,
                    self.qweight,
                    self.qzeros,
                    self.scales,
                    self.w_bit,
                    self.group_size,
                    self.bias,
                    self.out_features,
                )

        if input_dtype != torch.float16:
            out = out.to(dtype=input_dtype)

        return out.reshape(out_shape)

    def extra_repr(self) -> str:
        return f"in_features={self.in_features}, out_features={self.out_features}, bias={self.bias is not None}, w_bit={self.w_bit}, group_size={self.group_size}"