Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Porting Numba-based packing from INC (#301)

Yi Liufe19011edc8a
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 3641b7d3d..5c4289f56 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -9,6 +9,7 @@ pip list
cd /auto-round/test || exit 1
find . -type f -exec sed -i '/sys\.path\.insert(0, "\.\.")/d' {} +
+export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
export FORCE_BF16=1
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
diff --git a/auto_round/export/export_to_itrex/bit_packer.py b/auto_round/export/export_to_itrex/bit_packer.py
new file mode 100644
index 000000000..44cd708ff
--- /dev/null
+++ b/auto_round/export/export_to_itrex/bit_packer.py
@@ -0,0 +1,279 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for bit packing."""
+
+
+from typing import Callable, Dict, Tuple
+
+import numba
+import numpy as np
+
+# key: (bits, compress_bits), value: pack function
+bit_packers: Dict[Tuple[int, int], Callable] = {}
+
+
+def register_pack_func(orig_bits: int, compress_bits: int):
+ """Register the pack function."""
+
+ def decorator(func):
+ bit_packers[(orig_bits, compress_bits)] = func
+ return func
+
+ return decorator
+
+
+@register_pack_func(4, 32)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b4_c32(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=4 and compress_bits=32."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 7] & 0b1111) << 28)
+ | ((raw_array[:, i * n_pack + 6] & 0b1111) << 24)
+ | ((raw_array[:, i * n_pack + 5] & 0b1111) << 20)
+ | ((raw_array[:, i * n_pack + 4] & 0b1111) << 16)
+ | ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)
+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)
+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)
+ | (raw_array[:, i * n_pack] & 0b1111)
+ )
+ return packed_array
+
+
+@register_pack_func(4, 16)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b4_c16(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=4 and compress_bits=16."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)
+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)
+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)
+ | (raw_array[:, i * n_pack] & 0b1111)
+ )
+ return packed_array
+
+
+@register_pack_func(4, 8)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b4_c8(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=4 and compress_bits=8."""
+ for i in range(new_in_features):
+ packed_array[:, i] = ((raw_array[:, i * n_pack + 1] & 0b1111) << 4) | (raw_array[:, i * n_pack] & 0b1111)
+ return packed_array
+
+
+@register_pack_func(4, 64)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b4_c64(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=4 and compress_bits=64."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 15] & 0b1111) << 60)
+ | ((raw_array[:, i * n_pack + 14] & 0b1111) << 56)
+ | ((raw_array[:, i * n_pack + 13] & 0b1111) << 52)
+ | ((raw_array[:, i * n_pack + 12] & 0b1111) << 48)
+ | ((raw_array[:, i * n_pack + 11] & 0b1111) << 44)
+ | ((raw_array[:, i * n_pack + 10] & 0b1111) << 40)
+ | ((raw_array[:, i * n_pack + 9] & 0b1111) << 36)
+ | ((raw_array[:, i * n_pack + 8] & 0b1111) << 32)
+ | ((raw_array[:, i * n_pack + 7] & 0b1111) << 28)
+ | ((raw_array[:, i * n_pack + 6] & 0b1111) << 24)
+ | ((raw_array[:, i * n_pack + 5] & 0b1111) << 20)
+ | ((raw_array[:, i * n_pack + 4] & 0b1111) << 16)
+ | ((raw_array[:, i * n_pack + 3] & 0b1111) << 12)
+ | ((raw_array[:, i * n_pack + 2] & 0b1111) << 8)
+ | ((raw_array[:, i * n_pack + 1] & 0b1111) << 4)
+ | (raw_array[:, i * n_pack] & 0b1111)
+ )
+ return packed_array
+
+
+@register_pack_func(8, 32)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b8_c32(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=8 and compress_bits=32."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)
+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)
+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)
+ | (raw_array[:, i * n_pack] & 0b11111111)
+ )
+ return packed_array
+
+
+@register_pack_func(8, 16)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b8_c16(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=8 and compress_bits=16."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)
+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)
+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)
+ | (raw_array[:, i * n_pack] & 0b11111111)
+ )
+ return packed_array
+
+
+@register_pack_func(8, 8)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b8_c8(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=8 and compress_bits=8."""
+ for i in range(new_in_features):
+ packed_array[:, i] = raw_array[:, i * n_pack] & 0b11111111
+ return packed_array
+
+
+@register_pack_func(8, 64)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b8_c64(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=8 and compress_bits=64."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 7] & 0b11111111) << 56)
+ | ((raw_array[:, i * n_pack + 6] & 0b11111111) << 48)
+ | ((raw_array[:, i * n_pack + 5] & 0b11111111) << 40)
+ | ((raw_array[:, i * n_pack + 4] & 0b11111111) << 32)
+ | ((raw_array[:, i * n_pack + 3] & 0b11111111) << 24)
+ | ((raw_array[:, i * n_pack + 2] & 0b11111111) << 16)
+ | ((raw_array[:, i * n_pack + 1] & 0b11111111) << 8)
+ | (raw_array[:, i * n_pack] & 0b11111111)
+ )
+ return packed_array
+
+
+@register_pack_func(2, 32)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b2_c32(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=2 and compress_bits=32."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 15] & 0b11) << 30)
+ | ((raw_array[:, i * n_pack + 14] & 0b11) << 28)
+ | ((raw_array[:, i * n_pack + 13] & 0b11) << 26)
+ | ((raw_array[:, i * n_pack + 12] & 0b11) << 24)
+ | ((raw_array[:, i * n_pack + 11] & 0b11) << 22)
+ | ((raw_array[:, i * n_pack + 10] & 0b11) << 20)
+ | ((raw_array[:, i * n_pack + 9] & 0b11) << 18)
+ | ((raw_array[:, i * n_pack + 8] & 0b11) << 16)
+ | ((raw_array[:, i * n_pack + 7] & 0b11) << 14)
+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)
+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)
+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)
+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)
+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)
+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)
+ | (raw_array[:, i * n_pack] & 0b11)
+ )
+ return packed_array
+
+
+@register_pack_func(2, 16)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b2_c16(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=2 and compress_bits=16."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 7] & 0b11) << 14)
+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)
+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)
+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)
+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)
+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)
+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)
+ | (raw_array[:, i * n_pack] & 0b11)
+ )
+ return packed_array
+
+
+@register_pack_func(2, 8)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b2_c8(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=2 and compress_bits=8."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 3] & 0b11) << 6)
+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)
+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)
+ | (raw_array[:, i * n_pack] & 0b11)
+ )
+ return packed_array
+
+
+@register_pack_func(2, 64)
[email protected](nopython=True, parallel=True)
+def pack_array_with_numba_b2_c64(
+ raw_array: np.ndarray, packed_array: np.ndarray, n_pack: int, new_in_features: int
+) -> np.ndarray:
+ """Pack the array with numba when bits=2 and compress_bits=64."""
+ for i in range(new_in_features):
+ packed_array[:, i] = (
+ ((raw_array[:, i * n_pack + 31] & 0b11) << 62)
+ | ((raw_array[:, i * n_pack + 30] & 0b11) << 60)
+ | ((raw_array[:, i * n_pack + 29] & 0b11) << 58)
+ | ((raw_array[:, i * n_pack + 28] & 0b11) << 56)
+ | ((raw_array[:, i * n_pack + 27] & 0b11) << 54)
+ | ((raw_array[:, i * n_pack + 26] & 0b11) << 52)
+ | ((raw_array[:, i * n_pack + 25] & 0b11) << 50)
+ | ((raw_array[:, i * n_pack + 24] & 0b11) << 48)
+ | ((raw_array[:, i * n_pack + 23] & 0b11) << 46)
+ | ((raw_array[:, i * n_pack + 22] & 0b11) << 44)
+ | ((raw_array[:, i * n_pack + 21] & 0b11) << 42)
+ | ((raw_array[:, i * n_pack + 20] & 0b11) << 40)
+ | ((raw_array[:, i * n_pack + 19] & 0b11) << 38)
+ | ((raw_array[:, i * n_pack + 18] & 0b11) << 36)
+ | ((raw_array[:, i * n_pack + 17] & 0b11) << 34)
+ | ((raw_array[:, i * n_pack + 16] & 0b11) << 32)
+ | ((raw_array[:, i * n_pack + 15] & 0b11) << 30)
+ | ((raw_array[:, i * n_pack + 14] & 0b11) << 28)
+ | ((raw_array[:, i * n_pack + 13] & 0b11) << 26)
+ | ((raw_array[:, i * n_pack + 12] & 0b11) << 24)
+ | ((raw_array[:, i * n_pack + 11] & 0b11) << 22)
+ | ((raw_array[:, i * n_pack + 10] & 0b11) << 20)
+ | ((raw_array[:, i * n_pack + 9] & 0b11) << 18)
+ | ((raw_array[:, i * n_pack + 8] & 0b11) << 16)
+ | ((raw_array[:, i * n_pack + 7] & 0b11) << 14)
+ | ((raw_array[:, i * n_pack + 6] & 0b11) << 12)
+ | ((raw_array[:, i * n_pack + 5] & 0b11) << 10)
+ | ((raw_array[:, i * n_pack + 4] & 0b11) << 8)
+ | ((raw_array[:, i * n_pack + 3] & 0b11) << 6)
+ | ((raw_array[:, i * n_pack + 2] & 0b11) << 4)
+ | ((raw_array[:, i * n_pack + 1] & 0b11) << 2)
+ | (raw_array[:, i * n_pack] & 0b11)
+ )
+ return packed_array
diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.py
index 8e9f0aa85..686105fdd 100644
--- a/auto_round/export/export_to_itrex/model_wrapper.py
+++ b/auto_round/export/export_to_itrex/model_wrapper.py
@@ -23,7 +23,7 @@ import torch
from packaging.version import Version
from torch.autograd import Function
from torch.nn import functional as F
-
+import numpy as np
from auto_round.utils import logger
NF4 = [
@@ -54,8 +54,18 @@ NF4_BIT = [7, 1, 2, 3, 4, 5, 6, 0, -8, -7, -6, -5, -4, -3, -2, -1]
FP4_BNB_BIT = [-5, -6, -3, -4, -1, -2, -7, 0, 1, 6, 7, 4, 5, 2, 3]
FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]
-FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}
-INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}
+FLOAT_MAPPING = {
+ "nf4": NF4,
+ "fp4": FP4_BNB,
+ "fp4_e2m1_bnb": FP4_BNB,
+ "fp4_e2m1": FP4_E2M1,
+}
+INT_MAPPING = {
+ "nf4": NF4_BIT,
+ "fp4": FP4_BNB_BIT,
+ "fp4_e2m1_bnb": FP4_BNB_BIT,
+ "fp4_e2m1": FP4_E2M1_BIT,
+}
def get_torch_version():
@@ -71,6 +81,7 @@ PT_VERSION = get_torch_version().release
class WeightOnlyLinear(torch.nn.Module):
+
def __init__(
self,
in_features,
@@ -85,6 +96,7 @@ class WeightOnlyLinear(torch.nn.Module):
compression_dim=1,
device="cpu",
use_optimum_format=True,
+ use_legacy_pack = False,
):
super().__init__()
self.use_optimum_format = use_optimum_format
@@ -186,6 +198,25 @@ class WeightOnlyLinear(torch.nn.Module):
self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))
else:
self.bias = None
+ self.pack_func = (
+ self.pack_tensor_native_impl if use_legacy_pack else self.pack_tensor
+ )
+
+ def pack_tensor_native_impl(self, raw_tensor):
+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)
+ target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)
+ packed_tensor = torch.zeros(
+ raw_tensor.shape[0], target_len, dtype=self.compression_dtype
+ ).to(raw_tensor.device)
+ for j in range(packed_tensor.shape[1]):
+ start = self.n_pack * j
+ end = self.n_pack * (j + 1)
+ tmp = raw_tensor[:, start:end].type(self.compression_dtype)
+ for e in range(tmp.shape[1]):
+ tmp[:, e] &= mask
+ tmp[:, e] = tmp[:, e] << (self.bits * e)
+ packed_tensor[:, j] |= tmp[:, e]
+ return packed_tensor
def pack(self, int_weight, scale, zp, bias):
if self.use_optimum_format:
@@ -203,7 +234,9 @@ class WeightOnlyLinear(torch.nn.Module):
if bias is not None:
assert hasattr(self, "bias"), "bias is not set when initializing."
self.bias = bias.type(self.float_type).to(self.device)
- assert scale.shape == self.scales.shape, "Scale shape is mismatched."
+ assert (
+ scale.shape == self.scales.shape
+ ), f"Scale shape is mismatched, expect {self.scales.shape}, got {scale.shape}"
self.scales = scale.type(self.float_type).to(self.device)
if not self.use_optimum_format and self.compression_dim == 0:
int_weight = int_weight.t_().contiguous()
@@ -211,17 +244,9 @@ class WeightOnlyLinear(torch.nn.Module):
origin_shape = int_weight.shape
target_shape = self.qweight.shape
assert origin_shape[0] == target_shape[0], "output channels mismatch, please check."
- mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)
# pack weight
- for j in range(target_shape[1]):
- start = self.n_pack * j
- end = self.n_pack * (j + 1)
- tmp = int_weight[:, start:end].type(self.compression_dtype)
- for e in range(tmp.shape[1]):
- tmp[:, e] &= mask
- tmp[:, e] = tmp[:, e] << (self.bits * e)
- self.qweight[:, j] |= tmp[:, e]
+ self.qweight.copy_(self.pack_func(int_weight))
if not self.use_optimum_format and self.compression_dim == 0:
self.qweight = self.qweight.t_().contiguous()
@@ -233,15 +258,7 @@ class WeightOnlyLinear(torch.nn.Module):
zp = zp.t_().contiguous()
self.qzeros = self.qzeros.t_().contiguous()
assert hasattr(self, "qzeros"), "zp is not set when initializing."
- target_shape = self.qzeros.shape
- for j in range(target_shape[1]):
- start = self.n_pack * j
- end = self.n_pack * (j + 1)
- tmp = zp[:, start:end].type(self.compression_dtype)
- for e in range(tmp.shape[1]):
- tmp[:, e] &= mask
- tmp[:, e] = tmp[:, e] << (self.bits * e)
- self.qzeros[:, j] |= tmp[:, e]
+ self.qzeros.copy_(self.pack_func(zp))
if self.use_optimum_format or self.compression_dim == 0:
self.qzeros = self.qzeros.t_().contiguous()
if self.use_optimum_format:
@@ -351,3 +368,198 @@ class WeightOnlyLinear(torch.nn.Module):
if self.use_optimum_format:
tmp_str += ", use_optimum_format=True"
return tmp_str
+
+ def pack_tensor_with_torch(self, raw_tensor):
+ """Pack the tensor with torch.
+
+ Args:
+ raw_tensor (tensor): raw tensor.
+
+ Returns:
+ tensor: packed tensor.
+ """
+ target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)
+ packed_tensor = torch.zeros(
+ raw_tensor.shape[0], target_len, dtype=self.compression_dtype
+ ).to(raw_tensor.device)
+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(
+ raw_tensor.device
+ )
+ for j in range(packed_tensor.shape[1]):
+ start = self.n_pack * j
+ end = self.n_pack * (j + 1)
+ tmp = raw_tensor[:, start:end].type(self.compression_dtype)
+ tmp &= mask
+ for e in range(tmp.shape[1]):
+ tmp[:, e] = tmp[:, e] << (self.bits * e)
+ packed_tensor[:, j] |= tmp[:, e]
+ return packed_tensor
+
+ def unpack_tensor_with_torch(self, packed_tensor):
+ """Unpack the tensor with torch.
+
+ Args:
+ packed_tensor (tensor): packed tensor.
+
+ Returns:
+ tensor: unpacked tensor.
+ """
+ target_dtype = torch.int16
+ target_len = packed_tensor.shape[1] * self.n_pack
+ unpacked_tensor = torch.zeros(
+ packed_tensor.shape[0], target_len, dtype=target_dtype
+ ).to(packed_tensor.device)
+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(
+ packed_tensor.device
+ )
+ for j in range(packed_tensor.shape[1]):
+ for e in range(self.n_pack):
+ index = j * self.n_pack + e
+ tmp = packed_tensor[:, j]
+ tmp = tmp << (self.compress_bits - self.bits * (e + 1))
+ tmp = tmp >> self.compress_bits - self.bits
+ if hasattr(self, "qzeros"):
+ tmp &= mask # remove sign bit
+ unpacked_tensor[:, index].copy_(tmp.type(target_dtype))
+ return unpacked_tensor
+
+ def pack_array_with_numba(
+ self,
+ raw_array: np.ndarray,
+ n_pack: int,
+ bits: int,
+ compress_bits: int,
+ compression_dtype=np.int32,
+ ) -> np.ndarray:
+ """Packs the input array by combining elements into a specified bit-width format using NumPy.
+
+ Args:
+ raw_array (np.ndarray): The array to be packed. Shape: [out_features, in_features] or [1, in_features].
+ n_pack (int): The number of elements to be packed together.
+ bits (int): The number of bits for each element.
+ compress_bits (int): The number of bits for each element of the compressed array, supported 2, 4, 8.
+ compression_dtype (np.dtype, optional): The data type of the compressed array. Defaults to np.int32.
+
+ Returns:
+ np.ndarray: The packed array.
+ """
+ # Try to pack with numba to accelerate the packing process.
+ # If numba is not availabll or the packing method is not supported,
+ # fallback to the torch implementation.
+ try:
+ import numba
+
+ numba.config.THREADING_LAYER = "safe"
+ except ImportError:
+ logger.warning(
+ "To accelerate packing, please install numba with `pip install numba tbb`."
+ )
+ return (
+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()
+ )
+ except Exception as e:
+ logger.warning(
+ f"Import numba failed with error: {e}, fallback to torch implementation."
+ )
+ return (
+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()
+ )
+ from auto_round.export.export_to_itrex.bit_packer import bit_packers
+
+ pack_func_name = (bits, compress_bits)
+ if pack_func_name not in bit_packers:
+ logger.warning(
+ (
+ f"Unsupported packing with bits: {bits}, compress_bits: {compress_bits} using numba, "
+ "fallback to torch implementation."
+ )
+ )
+ return (
+ self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()
+ )
+ out_features, in_features = raw_array.shape
+ new_in_features = (in_features + n_pack - 1) // n_pack
+ packed_array = np.zeros(
+ (out_features, new_in_features), dtype=compression_dtype
+ )
+ raw_array = raw_array.astype(compression_dtype)
+ pack_method = bit_packers[pack_func_name]
+ return pack_method(raw_array, packed_array, n_pack, new_in_features)
+
+ def pack_tensor_with_numpy_impl(self, raw_tensor):
+ """The implement of packing tensor with numpy."""
+ raw_array = raw_tensor.cpu().numpy()
+ target_len = np.ceil(raw_array.shape[1] / self.n_pack).astype(int)
+ target_dtype = torch.tensor(0, dtype=self.compression_dtype).numpy().dtype
+ packed_array = np.zeros((raw_array.shape[0], target_len), dtype=target_dtype)
+ mask = np.uint8(2**self.bits - 1)
+ for j in range(packed_array.shape[1]):
+ start = self.n_pack * j
+ end = self.n_pack * (j + 1)
+ tmp = raw_array[:, start:end].astype(target_dtype)
+ tmp &= mask
+ for e in range(tmp.shape[1]):
+ tmp[:, e] = np.left_shift(tmp[:, e], self.bits * e)
+ packed_array[:, j] |= tmp[:, e]
+ packed_tensor = torch.from_numpy(packed_array).to(device=raw_tensor.device)
+ return packed_tensor
+
+ def pack_tensor_with_numpy(self, raw_tensor):
+ """Pack the tensor with numpy."""
+ if self.bits == 8 and self.compression_dtype == torch.int8:
+ return raw_tensor
+ if self.bits not in [2, 4, 8]:
+ return self.pack_tensor_with_numpy_impl(raw_tensor)
+ compression_dtype = torch.tensor(0, dtype=self.compression_dtype).numpy().dtype
+ packed_array = self.pack_array_with_numba(
+ raw_tensor.cpu().numpy(),
+ self.n_pack,
+ self.bits,
+ self.compress_bits,
+ compression_dtype,
+ )
+ return torch.from_numpy(packed_array).to(device=raw_tensor.device)
+
+ def unpack_tensor_with_numpy(self, packed_tensor):
+ """Unpack the packed tensor with numpy."""
+ packed_array = packed_tensor.cpu().numpy()
+ target_dtype = np.int16
+ if (
+ self.bits == 8
+ and self.compression_dtype == torch.int8
+ and hasattr(self, "qzeros")
+ ):
+ # special case for unpacking uint8 date from int8 compression_dtype
+ target_dtype = np.uint8
+ target_len = packed_array.shape[1] * self.n_pack
+ unpacked_array = np.zeros(
+ (packed_array.shape[0], target_len), dtype=target_dtype
+ )
+ mask = np.uint8(2**self.bits - 1)
+ for j in range(packed_array.shape[1]):
+ for e in range(self.n_pack):
+ index = j * self.n_pack + e
+ tmp = packed_array[:, j]
+ tmp = np.left_shift(tmp, self.compress_bits - self.bits * (e + 1))
+ tmp = np.right_shift(tmp, self.compress_bits - self.bits)
+ if hasattr(self, "qzeros"):
+ tmp &= mask
+ unpacked_array[:, index] = tmp.astype(target_dtype)
+ unpacked_tensor = torch.from_numpy(unpacked_array).to(
+ device=packed_tensor.device
+ )
+ return unpacked_tensor
+
+ def pack_tensor(self, raw_tensor):
+ """Pack tensor."""
+ if "cuda" in raw_tensor.device.type:
+ return self.pack_tensor_with_torch(raw_tensor)
+ else:
+ return self.pack_tensor_with_numpy(raw_tensor)
+
+ def unpack_tensor(self, packed_tensor):
+ """Unpack tensor."""
+ if "cuda" in packed_tensor.device.type:
+ return self.unpack_tensor_with_torch(packed_tensor)
+ else:
+ return self.unpack_tensor_with_numpy(packed_tensor)
diff --git a/requirements.txt b/requirements.txt
index 455c60523..d010ea536 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,3 +12,5 @@ tqdm
packaging
auto-gptq>=0.7.1
pillow
+numba
+tbb
diff --git a/test/test_woq_linear.py b/test/test_woq_linear.py
new file mode 100644
index 000000000..527ddd8de
--- /dev/null
+++ b/test/test_woq_linear.py
@@ -0,0 +1,84 @@
+import pytest
+import torch
+
+from auto_round.export.export_to_itrex.model_wrapper import WeightOnlyLinear
+
+
+class TestWeightOnlyLinear:
+ @pytest.mark.parametrize(
+ "bits, compression_dtype",
+ [
+ (8, torch.int16),
+ (8, torch.int32),
+ (8, torch.int64),
+ (4, torch.int8),
+ (4, torch.int16),
+ (4, torch.int32),
+ (4, torch.int64),
+ (2, torch.int8),
+ (2, torch.int16),
+ (2, torch.int32),
+ (2, torch.int64),
+ ],
+ )
+ def test_pack_with_numba(self, bits, compression_dtype):
+ m = torch.nn.Linear(1024, 512)
+ dtype = "int"
+ weight = m.weight.detach()
+ group_size = 32
+ origin_shape = weight.shape
+ from auto_round.data_type.int import quant_tensor_sym
+ from auto_round.quantizer import quant_tensor, reshape_tensor
+
+ data = reshape_tensor(weight, group_size=group_size)
+ qdq, scale, zp = quant_tensor(
+ quant_tensor_sym, data=data, group_size=group_size
+ )
+ int_weight = (
+ qdq.div(scale)
+ .add(zp)
+ .clamp(0, 2 ** (bits) - 1)
+ .to(torch.int32)
+ .reshape(origin_shape)
+ )
+ scale = scale.reshape(weight.shape[0], -1)
+ zp = zp.reshape(weight.shape[0], -1).to(torch.int32).clamp(0, 2 ** (bits) - 1)
+ module_with_legacy_pack = WeightOnlyLinear(
+ in_features=m.in_features,
+ out_features=m.out_features,
+ dtype=dtype,
+ bits=bits,
+ groupsize=32,
+ zp=zp is not None,
+ bias=m.bias is not None,
+ use_optimum_format=False,
+ compression_dtype=compression_dtype,
+ use_legacy_pack=True,
+ )
+ module_with_legacy_pack.pack(
+ int_weight.clone(), scale.clone(), zp.clone(), m.bias
+ )
+ module_with_new_pack = WeightOnlyLinear(
+ in_features=m.in_features,
+ out_features=m.out_features,
+ dtype=dtype,
+ bits=bits,
+ groupsize=32,
+ zp=zp is not None,
+ bias=m.bias is not None,
+ use_optimum_format=False,
+ compression_dtype=compression_dtype,
+ use_legacy_pack=False,
+ )
+ module_with_new_pack.pack(int_weight.clone(), scale.clone(), zp.clone(), m.bias)
+
+ assert torch.equal(
+ module_with_new_pack.qweight, module_with_legacy_pack.qweight
+ )
+
+ assert torch.equal(module_with_new_pack.qzeros, module_with_legacy_pack.qzeros)
+ assert torch.equal(module_with_new_pack.scales, module_with_legacy_pack.scales)
+ unpacked_int_weight = module_with_new_pack.unpack_tensor(
+ module_with_legacy_pack.qweight
+ )
+ assert torch.equal(unpacked_int_weight, int_weight)