PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Porting Numba-based packing from INC (#301)
Yi Liufe19011edc8a
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex 3641b7d3d..5c4289f56 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -9,6 +9,7 @@ pip listcd /auto-round/test || exit 1find . -type f -exec sed -i '/sys\.path\.insert(0, "\.\.")/d' {} ++export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATHexport FORCE_BF16=1export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverageauto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')diff --git a/auto_round/export/export_to_itrex/bit_packer.py b/auto_round/export/export_to_itrex/bit_packer.pynew file mode 100644index 000000000..44cd708ff--- /dev/null+++ b/auto_round/export/export_to_itrex/bit_packer.py@@ -0,0 +1,279 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Utility functions for bit packing."""+++from typing import Callable, Dict, Tuple++import numba+import numpy as np++# key: (bits, compress_bits), value: pack function+bit_packers: Dict[Tuple[int, int], Callable] = {}+++def register_pack_func(orig_bits: int, compress_bits: int):+ """Register the pack function."""++ def decorator(func):+ bit_packers[(orig_bits, compress_bits)] = func+ return func++ return decorator+++@register_pack_func(4, 32)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b4_c32(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=4 and compress_bits=32."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 7] & 0b1111) << 28)+ | ((raw_array[:, i * n_pack + 6] & 0b1111) << 24)+ | ((raw_array[:, i * n_pack + 5] & 0b1111) << 20)+ | ((raw_array[:, i * n_pack + 4] & 0b1111) << 16)+ | ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)+ | (raw_array[:, i * n_pack] & 0b1111)+ )+ return packed_array+++@register_pack_func(4, 16)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b4_c16(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=4 and compress_bits=16."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)+ | (raw_array[:, i * n_pack] & 0b1111)+ )+ return packed_array+++@register_pack_func(4, 8)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b4_c8(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=4 and compress_bits=8."""+ for i in range(new_in_features):+ packed_array[:, i] = ((raw_array[:, i * n_pack + 1] & 0b1111) << 4) | (raw_array[:, i * n_pack] & 0b1111)+ return packed_array+++@register_pack_func(4, 64)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b4_c64(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=4 and compress_bits=64."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 15] & 0b1111) << 60)+ | ((raw_array[:, i * n_pack + 14] & 0b1111) << 56)+ | ((raw_array[:, i * n_pack + 13] & 0b1111) << 52)+ | ((raw_array[:, i * n_pack + 12] & 0b1111) << 48)+ | ((raw_array[:, i * n_pack + 11] & 0b1111) << 44)+ | ((raw_array[:, i * n_pack + 10] & 0b1111) << 40)+ | ((raw_array[:, i * n_pack + 9] & 0b1111) << 36)+ | ((raw_array[:, i * n_pack + 8] & 0b1111) << 32)+ | ((raw_array[:, i * n_pack + 7] & 0b1111) << 28)+ | ((raw_array[:, i * n_pack + 6] & 0b1111) << 24)+ | ((raw_array[:, i * n_pack + 5] & 0b1111) << 20)+ | ((raw_array[:, i * n_pack + 4] & 0b1111) << 16)+ | ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)+ | (raw_array[:, i * n_pack] & 0b1111)+ )+ return packed_array+++@register_pack_func(8, 32)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b8_c32(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=8 and compress_bits=32."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)+ | (raw_array[:, i * n_pack] & 0b11111111)+ )+ return packed_array+++@register_pack_func(8, 16)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b8_c16(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=8 and compress_bits=16."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)+ | (raw_array[:, i * n_pack] & 0b11111111)+ )+ return packed_array+++@register_pack_func(8, 8)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b8_c8(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=8 and compress_bits=8."""+ for i in range(new_in_features):+ packed_array[:, i] = raw_array[:, i * n_pack] & 0b11111111+ return packed_array+++@register_pack_func(8, 64)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b8_c64(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=8 and compress_bits=64."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 7] & 0b11111111) << 56)+ | ((raw_array[:, i * n_pack + 6] & 0b11111111) << 48)+ | ((raw_array[:, i * n_pack + 5] & 0b11111111) << 40)+ | ((raw_array[:, i * n_pack + 4] & 0b11111111) << 32)+ | ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)+ | (raw_array[:, i * n_pack] & 0b11111111)+ )+ return packed_array+++@register_pack_func(2, 32)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b2_c32(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=2 and compress_bits=32."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 15] & 0b11) << 30)+ | ((raw_array[:, i * n_pack + 14] & 0b11) << 28)+ | ((raw_array[:, i * n_pack + 13] & 0b11) << 26)+ | ((raw_array[:, i * n_pack + 12] & 0b11) << 24)+ | ((raw_array[:, i * n_pack + 11] & 0b11) << 22)+ | ((raw_array[:, i * n_pack + 10] & 0b11) << 20)+ | ((raw_array[:, i * n_pack + 9] & 0b11) << 18)+ | ((raw_array[:, i * n_pack + 8] & 0b11) << 16)+ | ((raw_array[:, i * n_pack + 7] & 0b11) << 14)+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)+ | (raw_array[:, i * n_pack] & 0b11)+ )+ return packed_array+++@register_pack_func(2, 16)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b2_c16(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=2 and compress_bits=16."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 7] & 0b11) << 14)+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)+ | (raw_array[:, i * n_pack] & 0b11)+ )+ return packed_array+++@register_pack_func(2, 8)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b2_c8(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=2 and compress_bits=8."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 3] & 0b11) << 6)+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)+ | (raw_array[:, i * n_pack] & 0b11)+ )+ return packed_array+++@register_pack_func(2, 64)[email protected](nopython=True, parallel=True)+def pack_array_with_numba_b2_c64(+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int+) -> np.ndarray:+ """Pack the array with numba when bits=2 and compress_bits=64."""+ for i in range(new_in_features):+ packed_array[:, i] = (+ ((raw_array[:, i * n_pack + 31] & 0b11) << 62)+ | ((raw_array[:, i * n_pack + 30] & 0b11) << 60)+ | ((raw_array[:, i * n_pack + 29] & 0b11) << 58)+ | ((raw_array[:, i * n_pack + 28] & 0b11) << 56)+ | ((raw_array[:, i * n_pack + 27] & 0b11) << 54)+ | ((raw_array[:, i * n_pack + 26] & 0b11) << 52)+ | ((raw_array[:, i * n_pack + 25] & 0b11) << 50)+ | ((raw_array[:, i * n_pack + 24] & 0b11) << 48)+ | ((raw_array[:, i * n_pack + 23] & 0b11) << 46)+ | ((raw_array[:, i * n_pack + 22] & 0b11) << 44)+ | ((raw_array[:, i * n_pack + 21] & 0b11) << 42)+ | ((raw_array[:, i * n_pack + 20] & 0b11) << 40)+ | ((raw_array[:, i * n_pack + 19] & 0b11) << 38)+ | ((raw_array[:, i * n_pack + 18] & 0b11) << 36)+ | ((raw_array[:, i * n_pack + 17] & 0b11) << 34)+ | ((raw_array[:, i * n_pack + 16] & 0b11) << 32)+ | ((raw_array[:, i * n_pack + 15] & 0b11) << 30)+ | ((raw_array[:, i * n_pack + 14] & 0b11) << 28)+ | ((raw_array[:, i * n_pack + 13] & 0b11) << 26)+ | ((raw_array[:, i * n_pack + 12] & 0b11) << 24)+ | ((raw_array[:, i * n_pack + 11] & 0b11) << 22)+ | ((raw_array[:, i * n_pack + 10] & 0b11) << 20)+ | ((raw_array[:, i * n_pack + 9] & 0b11) << 18)+ | ((raw_array[:, i * n_pack + 8] & 0b11) << 16)+ | ((raw_array[:, i * n_pack + 7] & 0b11) << 14)+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)+ | (raw_array[:, i * n_pack] & 0b11)+ )+ return packed_arraydiff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.pyindex 8e9f0aa85..686105fdd 100644--- a/auto_round/export/export_to_itrex/model_wrapper.py+++ b/auto_round/export/export_to_itrex/model_wrapper.py@@ -23,7 +23,7 @@ import torchfrom packaging.version import Versionfrom torch.autograd import Functionfrom torch.nn import functional as F-+import numpy as npfrom auto_round.utils import loggerNF4 = [@@ -54,8 +54,18 @@ NF4_BIT = [7, 1, 2, 3, 4, 5, 6, 0, -8, -7, -6, -5, -4, -3, -2, -1]FP4_BNB_BIT = [-5, -6, -3, -4, -1, -2, -7, 0, 1, 6, 7, 4, 5, 2, 3]FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]-FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}-INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}+FLOAT_MAPPING = {+ "nf4": NF4,+ "fp4": FP4_BNB,+ "fp4_e2m1_bnb": FP4_BNB,+ "fp4_e2m1": FP4_E2M1,+}+INT_MAPPING = {+ "nf4": NF4_BIT,+ "fp4": FP4_BNB_BIT,+ "fp4_e2m1_bnb": FP4_BNB_BIT,+ "fp4_e2m1": FP4_E2M1_BIT,+}def get_torch_version():@@ -71,6 +81,7 @@ PT_VERSION = get_torch_version().releaseclass WeightOnlyLinear(torch.nn.Module):+def __init__(self,in_features,@@ -85,6 +96,7 @@ class WeightOnlyLinear(torch.nn.Module):compression_dim=1,device="cpu",use_optimum_format=True,+ use_legacy_pack = False,):super().__init__()self.use_optimum_format = use_optimum_format@@ -186,6 +198,25 @@ class WeightOnlyLinear(torch.nn.Module):self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))else:self.bias = None+ self.pack_func = (+ self.pack_tensor_native_impl if use_legacy_pack else self.pack_tensor+ )++ def pack_tensor_native_impl(self, raw_tensor):+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)+ target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)+ packed_tensor = torch.zeros(+ raw_tensor.shape[0], target_len, dtype=self.compression_dtype+ ).to(raw_tensor.device)+ for j in range(packed_tensor.shape[1]):+ start = self.n_pack * j+ end = self.n_pack * (j + 1)+ tmp = raw_tensor[:, start:end].type(self.compression_dtype)+ for e in range(tmp.shape[1]):+ tmp[:, e] &= mask+ tmp[:, e] = tmp[:, e] << (self.bits * e)+ packed_tensor[:, j] |= tmp[:, e]+ return packed_tensordef pack(self, int_weight, scale, zp, bias):if self.use_optimum_format:@@ -203,7 +234,9 @@ class WeightOnlyLinear(torch.nn.Module):if bias is not None:assert hasattr(self, "bias"), "bias is not set when initializing."self.bias = bias.type(self.float_type).to(self.device)- assert scale.shape == self.scales.shape, "Scale shape is mismatched."+ assert (+ scale.shape == self.scales.shape+ ), f"Scale shape is mismatched, expect {self.scales.shape}, got {scale.shape}"self.scales = scale.type(self.float_type).to(self.device)if not self.use_optimum_format and self.compression_dim == 0:int_weight = int_weight.t_().contiguous()@@ -211,17 +244,9 @@ class WeightOnlyLinear(torch.nn.Module):origin_shape = int_weight.shapetarget_shape = self.qweight.shapeassert origin_shape[0] == target_shape[0], "output channels mismatch, please check."- mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)# pack weight- for j in range(target_shape[1]):- start = self.n_pack * j- end = self.n_pack * (j + 1)- tmp = int_weight[:, start:end].type(self.compression_dtype)- for e in range(tmp.shape[1]):- tmp[:, e] &= mask- tmp[:, e] = tmp[:, e] << (self.bits * e)- self.qweight[:, j] |= tmp[:, e]+ self.qweight.copy_(self.pack_func(int_weight))if not self.use_optimum_format and self.compression_dim == 0:self.qweight = self.qweight.t_().contiguous()@@ -233,15 +258,7 @@ class WeightOnlyLinear(torch.nn.Module):zp = zp.t_().contiguous()self.qzeros = self.qzeros.t_().contiguous()assert hasattr(self, "qzeros"), "zp is not set when initializing."- target_shape = self.qzeros.shape- for j in range(target_shape[1]):- start = self.n_pack * j- end = self.n_pack * (j + 1)- tmp = zp[:, start:end].type(self.compression_dtype)- for e in range(tmp.shape[1]):- tmp[:, e] &= mask- tmp[:, e] = tmp[:, e] << (self.bits * e)- self.qzeros[:, j] |= tmp[:, e]+ self.qzeros.copy_(self.pack_func(zp))if self.use_optimum_format or self.compression_dim == 0:self.qzeros = self.qzeros.t_().contiguous()if self.use_optimum_format:@@ -351,3 +368,198 @@ class WeightOnlyLinear(torch.nn.Module):if self.use_optimum_format:tmp_str += ", use_optimum_format=True"return tmp_str++ def pack_tensor_with_torch(self, raw_tensor):+ """Pack the tensor with torch.++ Args:+ raw_tensor (tensor): raw tensor.++ Returns:+ tensor: packed tensor.+ """+ target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)+ packed_tensor = torch.zeros(+ raw_tensor.shape[0], target_len, dtype=self.compression_dtype+ ).to(raw_tensor.device)+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(+ raw_tensor.device+ )+ for j in range(packed_tensor.shape[1]):+ start = self.n_pack * j+ end = self.n_pack * (j + 1)+ tmp = raw_tensor[:, start:end].type(self.compression_dtype)+ tmp &= mask+ for e in range(tmp.shape[1]):+ tmp[:, e] = tmp[:, e] << (self.bits * e)+ packed_tensor[:, j] |= tmp[:, e]+ return packed_tensor++ def unpack_tensor_with_torch(self, packed_tensor):+ """Unpack the tensor with torch.++ Args:+ packed_tensor (tensor): packed tensor.++ Returns:+ tensor: unpacked tensor.+ """+ target_dtype = torch.int16+ target_len = packed_tensor.shape[1] * self.n_pack+ unpacked_tensor = torch.zeros(+ packed_tensor.shape[0], target_len, dtype=target_dtype+ ).to(packed_tensor.device)+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(+ packed_tensor.device+ )+ for j in range(packed_tensor.shape[1]):+ for e in range(self.n_pack):+ index = j * self.n_pack + e+ tmp = packed_tensor[:, j]+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))+ tmp = tmp >> self.compress_bits - self.bits+ if hasattr(self, "qzeros"):+ tmp &= mask # remove sign bit+ unpacked_tensor[:, index].copy_(tmp.type(target_dtype))+ return unpacked_tensor++ def pack_array_with_numba(+ self,+ raw_array: np.ndarray,+ n_pack: int,+ bits: int,+ compress_bits: int,+ compression_dtype=np.int32,+ ) -> np.ndarray:+ """Packs the input array by combining elements into a specified bit-width format using NumPy.++ Args:+ raw_array (np.ndarray): The array to be packed. Shape: [out_features, in_features] or [1, in_features].+ n_pack (int): The number of elements to be packed together.+ bits (int): The number of bits for each element.+ compress_bits (int): The number of bits for each element of the compressed array, supported 2, 4, 8.+ compression_dtype (np.dtype, optional): The data type of the compressed array. Defaults to np.int32.++ Returns:+ np.ndarray: The packed array.+ """+ # Try to pack with numba to accelerate the packing process.+ # If numba is not availabll or the packing method is not supported,+ # fallback to the torch implementation.+ try:+ import numba++ numba.config.THREADING_LAYER = "safe"+ except ImportError:+ logger.warning(+ "To accelerate packing, please install numba with `pip install numba tbb`."+ )+ return (+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()+ )+ except Exception as e:+ logger.warning(+ f"Import numba failed with error: {e}, fallback to torch implementation."+ )+ return (+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()+ )+ from auto_round.export.export_to_itrex.bit_packer import bit_packers++ pack_func_name = (bits, compress_bits)+ if pack_func_name not in bit_packers:+ logger.warning(+ (+ f"Unsupported packing with bits: {bits}, compress_bits: {compress_bits} using numba, "+ "fallback to torch implementation."+ )+ )+ return (+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()+ )+ out_features, in_features = raw_array.shape+ new_in_features = (in_features + n_pack - 1) // n_pack+ packed_array = np.zeros(+ (out_features, new_in_features), dtype=compression_dtype+ )+ raw_array = raw_array.astype(compression_dtype)+ pack_method = bit_packers[pack_func_name]+ return pack_method(raw_array, packed_array, n_pack, new_in_features)++ def pack_tensor_with_numpy_impl(self, raw_tensor):+ """The implement of packing tensor with numpy."""+ raw_array = raw_tensor.cpu().numpy()+ target_len = np.ceil(raw_array.shape[1] / self.n_pack).astype(int)+ target_dtype = torch.tensor(0, dtype=self.compression_dtype).numpy().dtype+ packed_array = np.zeros((raw_array.shape[0], target_len), dtype=target_dtype)+ mask = np.uint8(2**self.bits - 1)+ for j in range(packed_array.shape[1]):+ start = self.n_pack * j+ end = self.n_pack * (j + 1)+ tmp = raw_array[:, start:end].astype(target_dtype)+ tmp &= mask+ for e in range(tmp.shape[1]):+ tmp[:, e] = np.left_shift(tmp[:, e], self.bits * e)+ packed_array[:, j] |= tmp[:, e]+ packed_tensor = torch.from_numpy(packed_array).to(device=raw_tensor.device)+ return packed_tensor++ def pack_tensor_with_numpy(self, raw_tensor):+ """Pack the tensor with numpy."""+ if self.bits == 8 and self.compression_dtype == torch.int8:+ return raw_tensor+ if self.bits not in [2, 4, 8]:+ return self.pack_tensor_with_numpy_impl(raw_tensor)+ compression_dtype = torch.tensor(0, dtype=self.compression_dtype).numpy().dtype+ packed_array = self.pack_array_with_numba(+ raw_tensor.cpu().numpy(),+ self.n_pack,+ self.bits,+ self.compress_bits,+ compression_dtype,+ )+ return torch.from_numpy(packed_array).to(device=raw_tensor.device)++ def unpack_tensor_with_numpy(self, packed_tensor):+ """Unpack the packed tensor with numpy."""+ packed_array = packed_tensor.cpu().numpy()+ target_dtype = np.int16+ if (+ self.bits == 8+ and self.compression_dtype == torch.int8+ and hasattr(self, "qzeros")+ ):+ # special case for unpacking uint8 date from int8 compression_dtype+ target_dtype = np.uint8+ target_len = packed_array.shape[1] * self.n_pack+ unpacked_array = np.zeros(+ (packed_array.shape[0], target_len), dtype=target_dtype+ )+ mask = np.uint8(2**self.bits - 1)+ for j in range(packed_array.shape[1]):+ for e in range(self.n_pack):+ index = j * self.n_pack + e+ tmp = packed_array[:, j]+ tmp = np.left_shift(tmp, self.compress_bits - self.bits * (e + 1))+ tmp = np.right_shift(tmp, self.compress_bits - self.bits)+ if hasattr(self, "qzeros"):+ tmp &= mask+ unpacked_array[:, index] = tmp.astype(target_dtype)+ unpacked_tensor = torch.from_numpy(unpacked_array).to(+ device=packed_tensor.device+ )+ return unpacked_tensor++ def pack_tensor(self, raw_tensor):+ """Pack tensor."""+ if "cuda" in raw_tensor.device.type:+ return self.pack_tensor_with_torch(raw_tensor)+ else:+ return self.pack_tensor_with_numpy(raw_tensor)++ def unpack_tensor(self, packed_tensor):+ """Unpack tensor."""+ if "cuda" in packed_tensor.device.type:+ return self.unpack_tensor_with_torch(packed_tensor)+ else:+ return self.unpack_tensor_with_numpy(packed_tensor)diff --git a/requirements.txt b/requirements.txtindex 455c60523..d010ea536 100644--- a/requirements.txt+++ b/requirements.txt@@ -12,3 +12,5 @@ tqdmpackagingauto-gptq>=0.7.1pillow+numba+tbbdiff --git a/test/test_woq_linear.py b/test/test_woq_linear.pynew file mode 100644index 000000000..527ddd8de--- /dev/null+++ b/test/test_woq_linear.py@@ -0,0 +1,84 @@+import pytest+import torch++from auto_round.export.export_to_itrex.model_wrapper import WeightOnlyLinear+++class TestWeightOnlyLinear:+ @pytest.mark.parametrize(+ "bits, compression_dtype",+ [+ (8, torch.int16),+ (8, torch.int32),+ (8, torch.int64),+ (4, torch.int8),+ (4, torch.int16),+ (4, torch.int32),+ (4, torch.int64),+ (2, torch.int8),+ (2, torch.int16),+ (2, torch.int32),+ (2, torch.int64),+ ],+ )+ def test_pack_with_numba(self, bits, compression_dtype):+ m = torch.nn.Linear(1024, 512)+ dtype = "int"+ weight = m.weight.detach()+ group_size = 32+ origin_shape = weight.shape+ from auto_round.data_type.int import quant_tensor_sym+ from auto_round.quantizer import quant_tensor, reshape_tensor++ data = reshape_tensor(weight, group_size=group_size)+ qdq, scale, zp = quant_tensor(+ quant_tensor_sym, data=data, group_size=group_size+ )+ int_weight = (+ qdq.div(scale)+ .add(zp)+ .clamp(0, 2 ** (bits) - 1)+ .to(torch.int32)+ .reshape(origin_shape)+ )+ scale = scale.reshape(weight.shape[0], -1)+ zp = zp.reshape(weight.shape[0], -1).to(torch.int32).clamp(0, 2 ** (bits) - 1)+ module_with_legacy_pack = WeightOnlyLinear(+ in_features=m.in_features,+ out_features=m.out_features,+ dtype=dtype,+ bits=bits,+ groupsize=32,+ zp=zp is not None,+ bias=m.bias is not None,+ use_optimum_format=False,+ compression_dtype=compression_dtype,+ use_legacy_pack=True,+ )+ module_with_legacy_pack.pack(+ int_weight.clone(), scale.clone(), zp.clone(), m.bias+ )+ module_with_new_pack = WeightOnlyLinear(+ in_features=m.in_features,+ out_features=m.out_features,+ dtype=dtype,+ bits=bits,+ groupsize=32,+ zp=zp is not None,+ bias=m.bias is not None,+ use_optimum_format=False,+ compression_dtype=compression_dtype,+ use_legacy_pack=False,+ )+ module_with_new_pack.pack(int_weight.clone(), scale.clone(), zp.clone(), m.bias)++ assert torch.equal(+ module_with_new_pack.qweight, module_with_legacy_pack.qweight+ )++ assert torch.equal(module_with_new_pack.qzeros, module_with_legacy_pack.qzeros)+ assert torch.equal(module_with_new_pack.scales, module_with_legacy_pack.scales)+ unpacked_int_weight = module_with_new_pack.unpack_tensor(+ module_with_legacy_pack.qweight+ )+ assert torch.equal(unpacked_int_weight, int_weight)