PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
autoround_support_qbits_backend (#145)
Wang, Zhedbdc4a39e721
diff --git a/.gitignore b/.gitignorenew file mode 100644index 000000000..ca92a0e08--- /dev/null+++ b/.gitignore@@ -0,0 +1,11 @@+.vs+.vscode+__pycache__+auto_round.egg-info/+build/*+.eggs/+dist/+.cache/+.clangd+CMakeUserPresets.json+tmp_autoround/diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex a49fb2f33..1494e41c1 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -42,6 +42,7 @@ from transformers.quantizers.auto import AUTO_QUANTIZER_MAPPINGfrom transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethodfrom auto_round.utils import get_module, set_module+import auto_round_extension.qbits.qlinear_qbits as qlinear_qbitslogger = getLogger(__name__)import sys@@ -362,9 +363,26 @@ class AutoRoundQuantizer(HfQuantizer):weight_dtype=layer.weight.dtype,)+ if new_layer.qweight.device.type == "cpu": # fallback to qbits linear when qweight on cpu device+ QuantLinear = qlinear_qbits.QuantLinear+ new_layer = QuantLinear( # pylint: disable=E1123+ bits,+ group_size,+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ )+new_layer.device = deviceset_module(module, layer_name, new_layer)+ def qbits_post_init(self, model):+ for layer in model.modules():+ if isinstance(layer,qlinear_qbits.QuantLinear):+ layer.post_init()+ return model+def post_init_model(self, model):"""Post-initialization that require device information, for example buffers initialization on device.@@ -384,6 +402,9 @@ class AutoRoundQuantizer(HfQuantizer):model.quantize_config = StoreAttr()model = autoround_post_init(model)+ # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.+ model = self.qbits_post_init(model)+return modeldef _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):diff --git a/auto_round_extension/qbits/__init__.py b/auto_round_extension/qbits/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.pynew file mode 100644index 000000000..b83619d97--- /dev/null+++ b/auto_round_extension/qbits/qlinear_qbits.py@@ -0,0 +1,209 @@++# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math++import numpy as np+import torch+import torch.nn as nn+import transformers+from intel_extension_for_transformers import qbits # with QBits kernels ()+from auto_round.utils import convert_dtype_torch2str, logging++logger = logging.getLogger(__name__)++BITS_DTYPE_MAPPING = {+ 2: "int2_clip",+ 4: "int4_clip",+ 8: "int8",+}+++class QuantLinear(nn.Module):+ QUANT_TYPE = "qbits"++ def __init__(+ self,+ bits,+ group_size,+ infeatures,+ outfeatures,+ bias,+ kernel_switch_threshold=128,+ trainable=False,+ weight_dtype=torch.bfloat16,+ **kwargs,+ ):+ super().__init__()++ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2, 4,8 bits are supported for QBits.")++ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1+ self.weight_dtype = weight_dtype+ self.asym = True++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits,+ outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=weight_dtype,+ ),+ )+ if bias:+ self.register_buffer("bias", torch.zeros(+ (outfeatures), dtype=torch.float))+ else:+ self.bias = None++ self.kernel_switch_threshold = kernel_switch_threshold++ self.trainable = trainable++ def post_init(self):+ assert self.qweight.device.type == "cpu"+ if self.bias is not None:+ self.bias = self.bias.to(dtype=torch.float32)++ # intweight: k x n, zeros: k / group_size x n+ intweight, zeros = unpack_to_8bit_signed(+ self.qweight, self.qzeros, self.bits)+ if zeros is None:+ zeros = torch.empty(0, dtype=torch.int8)+ self.asym = False+ else:+ # change it to int8 with offset 128+ if self.bits == 8:+ zeros = (zeros.to(torch.int32) -+ (2 ** (self.bits - 1))).to(torch.int8)+ else:+ zeros -= (2**(self.bits - 1))++ if not self.asym:+ intweight -= (2**(self.bits - 1))+ intweight = intweight.to(torch.uint8 if self.asym else torch.int8)+ # due to asym return torch.uint8 but backend request int8,+ # change it to int8 with offset 128+ if self.asym:+ intweight = (intweight.to(torch.int32) -+ (2 ** (self.bits - 1))).to(torch.int8)++ scales = self.scales++ logger.info(+ f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}")+ self.qweight = qbits.repack_quantized_weight(intweight.contiguous(), scales.float().contiguous(), zeros.contiguous(), torch.empty(0),+ # weight_dtype+ BITS_DTYPE_MAPPING[self.bits],+ # scale_dtype+ "fp32",+ # TODO(zhe): consider dynamic-set cmpt for better perf?+ "fp32",+ self.asym,+ self.group_size)+ # free mem+ self.qzeros = torch.empty(0)+ self.scales = torch.empty(0)++ def forward(self, x: torch.Tensor):+ raw_input_dtype = x.dtype+ if raw_input_dtype != torch.float32:+ x = x.to(torch.float32)+ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.view(-1, x.shape[-1]) # convert xd to 2d+ out_2d_shape = x.shape[:-1] + (self.outfeatures,)++ outputs = torch.zeros(out_2d_shape, device=x.device, dtype=torch.float)+ bias = self.bias if self.bias is not None else torch.empty(+ 0, dtype=torch.float)++ qbits.woq_linear(x, self.qweight, bias, outputs,+ convert_dtype_torch2str(torch.float), # compute_dtype+ BITS_DTYPE_MAPPING[self.bits], # weight_dtype+ "fp32", # scale_dtype+ self.asym)+ return outputs.to(raw_input_dtype).view(out_shape)++[email protected]_grad()+def unpack_to_8bit_signed(qweight, qzeros, bits):+ wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)+ zeros = None+ if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):+ zp_shape = list(qzeros.shape)+ zp_shape[1] = zp_shape[1] * (32 // bits)++ zeros = torch.bitwise_right_shift(+ torch.unsqueeze(qzeros, 2).expand(-1, -1,+ 32 // bits), wf.unsqueeze(0)+ ).to(torch.int16 if bits == 8 else torch.int8)+ torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)+ if bits == 8:+ zeros = zeros.to(torch.uint8)+ try:+ zeros = zeros.reshape(zp_shape)+ except:+ # zeros and scales have different iteam numbers.+ # remove 1 (due to 0 + 1 in line 252)+ zeros = zeros[zeros != 1]+ zeros = zeros.reshape(zp_shape)++ weight = torch.bitwise_right_shift(+ torch.unsqueeze(qweight, 1).expand(-1, 32 //+ bits, -1), wf.unsqueeze(-1)+ ).to(torch.int16 if bits == 8 else torch.int8)+ weight.bitwise_and_((2**bits) - 1)+ weight = weight.view(-1, weight.shape[-1])++ return weight, zeros+++# Copied from qlinear_marlin.py[email protected]_grad()+def dequantize_weight(qweight, qzeros, scales, bits):+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(+ qweight, qzeros, bits)+ group_size = unpacked_qweight.shape[0] // scales.shape[0]+ scales = scales.repeat_interleave(group_size, dim=0)+ if unpacked_qzeros is not None:+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)+ else:+ unpacked_qzeros = torch.full_like(+ scales, 8 if bits == 4 else 128, dtype=torch.int32)+ unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales++ return unpacked_qweight, unpacked_qzeros+++__all__ = ["QuantLinear"]diff --git a/setup.py b/setup.pyindex c6b5c1e4d..fd244df2b 100644--- a/setup.py+++ b/setup.py@@ -16,15 +16,18 @@ except Exception as error:version = __version__-def fetch_requirements(path):- with open(path, "r") as fd:- return [r.strip() for r in fd.readlines()]--BUILD_CUDA_EXT = int(os.environ.get('BUILD_CUDA_EXT', '1')) == 1PYPI_RELEASE = os.environ.get('PYPI_RELEASE', None)+def fetch_requirements(path):+ requirements = []+ with open(path, "r") as fd:+ requirements = [r.strip() for r in fd.readlines()]+ if not BUILD_CUDA_EXT:+ requirements.append("intel-extension-for-transformers") # for leverage QBits woq_linear capability.+ return requirements+def detect_local_sm_architectures():"""Detect compute capabilities of one machine's GPUs as PyTorch does.@@ -53,17 +56,6 @@ def detect_local_sm_architectures():UNSUPPORTED_COMPUTE_CAPABILITIES = ['3.5', '3.7', '5.0', '5.2', '5.3']-requirements = [- "torch",- "accelerate",- "datasets",- "sentencepiece",- "safetensors",- "transformers",- "tqdm",- 'py-cpuinfo'- 'sentencepiece'-]if BUILD_CUDA_EXT:try: