Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add ipex support for intel cpu (#292)

wenhuach21168a1f697b84
diff --git a/README.md b/README.md
index e0d76b868..c622ba4b7 100644
--- a/README.md
+++ b/README.md
@@ -204,7 +204,7 @@ Please run the quantization code first
### AutoRound format
-**CPU**: pip install intel-extension-for-transformers, auto_round version >0.3.1
+**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip install intel-extension-for-transformers,
**HPU**: docker image with Gaudi Software Stack is recommended. More details can be found
in [Gaudi Guide](https://docs.habana.ai/en/latest/).
@@ -217,7 +217,7 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRoundConfig
-backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 'pip install -v gptqmodel --no-build-isolation')
+backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 and 'pip install -v gptqmodel --no-build-isolation')
quantization_config = AutoRoundConfig(
backend=backend
)
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index c9d3d3d7f..859481302 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -48,6 +48,8 @@ from auto_round.backend import get_layer_backend, dynamic_import_inference_linea
import auto_round_extension.qbits.qlinear_qbits as qlinear_qbits
import auto_round_extension.qbits.qlinear_qbits_gptq as qlinear_qbits_gptq
+import auto_round_extension.ipex.qlinear_ipex_gptq as qlinear_ipex_gptq
+import auto_round_extension.ipex.qlinear_ipex_awq as qlinear_ipex_awq
from auto_round.backend import BackendInfos
from transformers.utils.versions import require_version
from enum import Enum
@@ -397,10 +399,11 @@ class AutoRoundQuantizer(HfQuantizer):
quantization_config.target_backend = quantization_config.backend
target_device = self.detect_device(quantization_config.target_backend, quantization_config.backend)
+ self.target_device = target_device
if hasattr(quantization_config, "backend"): # pragma: no cover
- if "hpu" == target_device and model.dtype != torch.bfloat16:
- logger.info("Change the dtype to `bfloat16` as HPU does not support float16")
+ if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:
+ logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")
model = model.to(torch.bfloat16)
bits = quantization_config.bits
@@ -576,14 +579,20 @@ class AutoRoundQuantizer(HfQuantizer):
new_layer.device = layer_device
set_module(module, layer_name, new_layer)
- def qbits_post_init(self, model):
+ def cpu_post_init(self, model):
dep_check = True
- for layer in model.modules():
+ message = "Repacking to CPU format"
+
+ for n, layer in tqdm(model.named_modules(), desc=message, total=len(list(model.named_modules())),
+ leave=True): ##not exit correctly
if isinstance(layer, (qlinear_qbits.QuantLinear, qlinear_qbits_gptq.QuantLinear)):
if dep_check:
layer.req_check()
layer.post_init()
dep_check = False
+ if isinstance(layer, (qlinear_ipex_gptq.QuantLinear, qlinear_ipex_awq.QuantLinear)):
+ layer.post_init()
+
return model
def repack_marlin(self, model):
@@ -692,8 +701,9 @@ class AutoRoundQuantizer(HfQuantizer):
self.repack_marlin(model)
model = autoround_post_init(model)
- # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.
- model = self.qbits_post_init(model)
+ # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.
+ if self.target_device == "cpu":
+ model = self.cpu_post_init(model)
return model
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4d0a8e05b..0cb2c3ff4 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -515,7 +515,7 @@ class AutoRound(object):
elif isinstance(data, str):
if self.tokenizer is None:
logger.error("please provide tokenizer for string input")
- exit()
+ exit(-1)
data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
data_new = {}
for key in data.keys():
@@ -553,7 +553,7 @@ class AutoRound(object):
f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
f"dataset or decease the sequence length"
)
- exit()
+ exit(-1)
elif total_cnt < nsamples:
logger.warning(
f"Insufficient number of samples collected may affect the quantification. "
@@ -1166,7 +1166,7 @@ class AutoRound(object):
format = format.split(":")[0]
if format not in EXPORT_FORMAT:
logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
- exit()
+ exit(-1)
save_quantized_as_format = EXPORT_FORMAT.get(format)
if "gptq" in format and not self.sym:
logger.warning(
@@ -1252,7 +1252,7 @@ class AutoRound(object):
layer = get_module(self.model, key)
if layer is None:
logger.error(f"could not find layer {key} in the model, exit...")
- exit()
+ exit(-1)
if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):
layer_names.append(key)
diff --git a/auto_round/backend.py b/auto_round/backend.py
index 16e898ab1..ea79ac850 100644
--- a/auto_round/backend.py
+++ b/auto_round/backend.py
@@ -13,13 +13,24 @@
# limitations under the License.
import functools
+import logging
from dataclasses import dataclass, field
from typing import List, Any, Optional
+from transformers.utils.versions import require_version
+
from auto_round.utils import get_library_version
BackendInfos = {}
+import cpuinfo
+
+def get_cpu_manufacturer():
+ cpu_info = cpuinfo.get_cpu_info()
+ if "brand_raw" in cpu_info and "intel" in cpu_info["brand_raw"].lower():
+ return "intel"
+ else:
+ return "others"
@dataclass
class BackendInfo:
@@ -55,6 +66,7 @@ class BackendInfo:
convertable_format: List[str] = field(default_factory=list)
feature_checks: List[Any] = field(default_factory=list)
alias: Optional[List[str]] = None
+ requirements: Optional[List[str]] = None
def feature_multiply_checker(in_feature, out_feature, in_feature_multiplier, out_feature_multiplier=None):
@@ -94,7 +106,8 @@ BackendInfos['gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],
priority=5,
feature_checks=[feature_multiply_checker_32],
alias=["auto_round:gptq:exllamav2", "auto_round:auto_gptq:exllamav2",
- 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"]
+ 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"],
+ requirements=["auto-gptq>=0.7.1"]
)
BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
@@ -102,28 +115,54 @@ BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
bits=[2, 4, 8], group_size=None,
priority=0, feature_checks=[feature_multiply_checker_32],
alias=["auto_round:gptq:tritonv2", "auto_round:auto_gptq:tritonv2",
- "auto_gptq:tritonv2"])
+ "auto_gptq:tritonv2"],
+ requirements=["auto-gptq>=0.7.1"]
+ )
BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm
packing_format="awq",
bits=[4], group_size=None,
priority=4, feature_checks=[feature_num_greater_checker_1024],
alias=["auto_awq:gemm", "auto_round:awq:gemm", "auto_round:auto_awq:gemm", "awq",
- "auto_awq", "auto_round:awq", "aut_round:auto_awq"])
+ "auto_awq", "auto_round:awq", "aut_round:auto_awq"],
+ requirements=["autoawq"]
+ )
BackendInfos['auto_round:qbits'] = BackendInfo(device=["cpu"], sym=[True, False],
packing_format="qbits",
bits=[2, 4, 8], group_size=None,
- priority=0,
+ priority=0 if "intel" in get_cpu_manufacturer() else 5,
feature_checks=[],
- convertable_format=["triton"])
+ convertable_format=["triton"],
+ requirements=["intel-extension-for-transformers"])
BackendInfos['auto_round:qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
packing_format="qbits",
bits=[2, 4, 8], group_size=None,
- priority=0,
+ priority=0 if "intel" in get_cpu_manufacturer() else 5,
feature_checks=[],
- convertable_format=["triton_zp+-1"])
+ convertable_format=["triton_zp+-1"],
+ requirements=["intel-extension-for-transformers"]
+ )
+
+BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu"], sym=[True, False],
+ packing_format="ipex_gptq",
+ bits=[4], group_size=None,
+ priority=5 if "intel" in get_cpu_manufacturer() else 5,
+ feature_checks=[],
+ convertable_format=["triton_zp+-1"],
+ requirements=["intel-extension-for-pytorch>=2.4"]
+ )
+
+BackendInfos['auto_round:ipex_awq'] = BackendInfo(device=["cpu"], sym=[True, False],
+ packing_format="ipex_awq",
+ bits=[4], group_size=None,
+ priority=5 if "intel" in get_cpu_manufacturer() else 5,
+ feature_checks=[],
+ ##convertable_format=["triton_zp+-1", "awq"],
+ convertable_format=["awq"],
+ requirements=["intel-extension-for-pytorch>=2.4"]
+ )
# BackendInfos['auto_round:marlin'] = BackendInfo(device=["gpu"], sym=[True],
# packing_format="marlin",
@@ -147,7 +186,8 @@ BackendInfos['auto_round:hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False
convertable_format=["triton_zp+-1"])
-def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features):
+def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features,
+ check_requirements=True):
"""Checks if the given configuration is compatible with the specified backend.
Args:
@@ -160,6 +200,7 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_format
packing_format (str): The packing format used by the backend (e.g., 'triton').
in_features (int): The number of input features for the model layer.
out_features (int): The number of output features for the model layer.
+ check_requirements (bool): Whether check the requirement
Returns:
bool: True if the configuration is compatible with the backend, False otherwise.
@@ -203,6 +244,13 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_format
if packing_format != backend.packing_format and packing_format not in backend.convertable_format:
return False
+ if check_requirements and backend.requirements is not None:
+ for requirement in backend.requirements:
+ try:
+ require_version(requirement)
+ except ImportError:
+ return False
+
return True
@@ -245,6 +293,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
else: # auto_round must be at the end
import auto_round_extension.qbits.qlinear_qbits as qlinear_qbits_autoround
return qlinear_qbits_autoround.QuantLinear
+ if "ipex_gptq" in backend:
+ from auto_round_extension.ipex.qlinear_ipex_gptq import QuantLinear
+ return QuantLinear
+
+ if "ipex_awq" in backend:
+ from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear
+ return QuantLinear
if "marlin" in backend:
from transformers.utils.versions import require_version
@@ -431,6 +486,21 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
# Raise an error if no compatible backends are found
if len(supported_backends) == 0:
+ supported_backends_need_package = []
+ for key in BackendInfos.keys():
+ if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,
+ check_requirements=False):
+ supported_backends_need_package.append(key)
+
+ if len(supported_backends_need_package) > 0:
+ supported_backends_need_package = sorted(supported_backends_need_package,
+ key=lambda support_backend: BackendInfos[support_backend].priority,
+ reverse=True)
+ backend_info = BackendInfos[supported_backends_need_package[0]]
+ str_info = ",".join(backend_info.requirements)
+ logging.error(f"`pip install {str_info}` to support inference")
+ exit(-1)
+
raise ValueError(f"None of the backends support this layer")
# Sort the compatible backends by priority and return the one with the highest priority
@@ -438,21 +508,3 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
reverse=True)
return supported_backends[0]
-
-
-if __name__ == "__main__":
- res = get_layer_backend("cuda", "gptq:exllamav2", "gptq:exllamav2", 4, 128, sym=False, in_features=128,
- out_features=128)
- assert res == "gptq:exllamav2"
-
- res = get_layer_backend("cuda", "gptq:exllamav2", "gptq:exllamav2", 2, 128, sym=False, in_features=128,
- out_features=128)
- assert res == "gptq:tritonv2"
-
- res = get_layer_backend("cpu", "auto_round:exllamav2", "auto_round:exllamav2", 4, 128, sym=False, in_features=128,
- out_features=128)
- assert res == "auto_round:qbits"
-
- res = get_layer_backend("cpu", "gptq:exllamav2", "gptq:exllamav2", 4, 128, sym=False, in_features=128,
- out_features=128)
- assert res == "auto_round:qbits_zp"
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
index cfb4a2a11..1cf084b2b 100644
--- a/auto_round/export/export_to_autogptq/qlinear_triton.py
+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py
@@ -135,7 +135,6 @@ class QuantLinear(nn.Module, TritonModuleMixin):
qzeros = qzeros.astype(np.int32)
self.qzeros = torch.from_numpy(qzeros)
- delattr(self, "g_idx")
__all__ = ["QuantLinear"]
diff --git a/auto_round_extension/cuda/qlinear_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.py
index 786bb091d..3bc415971 100644
--- a/auto_round_extension/cuda/qlinear_exllamav2.py
+++ b/auto_round_extension/cuda/qlinear_exllamav2.py
@@ -196,6 +196,10 @@ class QuantLinear(nn.Module):
dtype=torch.float16,
),
)
+ self.register_buffer(
+ "g_idx",
+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
+ )
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
@@ -210,6 +214,7 @@ class QuantLinear(nn.Module):
"qweight": self.qweight,
"qzeros": self.qzeros,
"scales": self.scales,
+ "g_idx": self.g_idx,
}
temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())
self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)
diff --git a/auto_round_extension/ipex/__init__.py b/auto_round_extension/ipex/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.py
new file mode 100644
index 000000000..c90e9861b
--- /dev/null
+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py
@@ -0,0 +1,87 @@
+import torch
+import torch.nn as nn
+
+class QuantLinear(nn.Module):
+
+ def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):
+ super().__init__()
+ assert w_bit == 4, "Only 4 bit are supported for now."
+
+ self.use_bf16 = True # Intel platform support bf16 even without amx.
+
+ self.in_features = in_features
+ self.out_features = out_features
+ self.w_bit = w_bit
+ self.group_size = group_size if group_size != -1 else in_features
+ self.scale_dtype = torch.float32
+
+ # quick sanity check (make sure alignment)
+ assert self.in_features % self.group_size == 0
+ assert out_features % (32 // self.w_bit) == 0
+ self.pack_num = 32 // self.w_bit
+
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (in_features // self.group_size, out_features // self.pack_num),
+ dtype=torch.int32,
+ device=dev,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (in_features // self.group_size, out_features),
+ dtype=torch.bfloat16 if self.use_bf16 else torch.float32,
+ device=dev,
+ ))
+ if bias:
+ self.register_buffer(
+ "bias",
+ torch.zeros((out_features), dtype=torch.bfloat16 if self.use_bf16 else torch.float32, device=dev),
+ )
+ else:
+ self.register_buffer(
+ "bias",
+ None,
+ )
+ qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)
+ self.register_buffer("qweight", qweight)
+
+ def post_init(self):
+ assert self.qweight.device.type == "cpu"
+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear
+ self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \
+ self.in_features, self.out_features, None, self.bias, \
+ self.group_size, None, 0, 1)
+
+ @classmethod
+ def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):
+ awq_linear = cls(
+ w_bit,
+ group_size,
+ linear.in_features,
+ linear.out_features,
+ linear.bias is not None,
+ linear.weight.device,
+ )
+ if init_only: # just prepare for loading sd
+ return awq_linear
+
+ raise NotImplementedError("Only inference is supported for IPEX kernels")
+
+ @torch.no_grad()
+ def forward(self, x):
+
+ outputs = self.ipex_linear(x)
+
+ return outputs
+
+ def extra_repr(self) -> str:
+ return ("in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(
+ self.in_features,
+ self.out_features,
+ self.bias is not None,
+ self.w_bit,
+ self.group_size,
+ ))
diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.py
new file mode 100644
index 000000000..c69dd06c6
--- /dev/null
+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py
@@ -0,0 +1,359 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+BITS_DTYPE_MAPPING = {
+ 8: "int8",
+}
+
+
+def convert_dtype_torch2str(dtype):
+ if dtype == torch.int8:
+ return "int8"
+ elif dtype == torch.float:
+ return "fp32"
+ elif dtype == torch.float16:
+ return "fp16"
+ elif dtype == torch.bfloat16:
+ return "bf16"
+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
+ return dtype
+ else:
+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)
+
+
+class QuantLinear(nn.Module):
+ QUANT_TYPE = "ipex_gptq"
+
+ def __init__(
+ self,
+ bits,
+ group_size,
+ infeatures,
+ outfeatures,
+ bias,
+ kernel_switch_threshold=128,
+ training=False,
+ weight_dtype=torch.bfloat16,
+ **kwargs,
+ ):
+ super().__init__()
+
+ if bits not in [4]:
+ raise NotImplementedError("Only 4-bits is supported for IPEX.")
+
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2 ** self.bits - 1
+ self.weight_dtype = weight_dtype
+ self.asym = True
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=weight_dtype,
+ ),
+ )
+ self.register_buffer(
+ "g_idx",
+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
+ )
+ if bias:
+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))
+ else:
+ self.bias = None
+
+ self.kernel_switch_threshold = kernel_switch_threshold
+
+ self.training = training
+
+ # for training forward
+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
+
+ def post_init(self):
+ assert self.qweight.device.type == "cpu"
+
+ # if not self.training and IPEX_AVAILABLE:
+ if not self.training:
+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear
+ from packaging import version
+ from auto_round.utils import get_library_version
+ ipex_version = get_library_version("intel_extension_for_pytorch")
+ if version.parse(ipex_version) >= version.parse("2.5"):
+ self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \
+ self.infeatures, self.outfeatures, None,
+ self.bias, \
+ self.group_size, self.g_idx, 0, 0)
+ else:
+ import intel_extension_for_pytorch as ipex
+ from intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_linear
+ from intel_extension_for_pytorch.utils.weight_only_quantization import (
+ _convert_optimum_format_to_desired,
+ )
+
+ qweight = self.qweight
+ scales = self.scales
+ qzeros = self.qzeros
+
+ qweight, scales, qzeros = _convert_optimum_format_to_desired(qweight, scales, qzeros)
+ weight_dtype = {
+ 4: ipex.quantization.WoqWeightDtype.INT4,
+ 8: ipex.quantization.WoqWeightDtype.INT8,
+ }
+ compute_dtype = {
+ "fp32": ipex.quantization.WoqLowpMode.NONE, # follow the activation datatype.
+ "bf16": ipex.quantization.WoqLowpMode.BF16,
+ "fp16": ipex.quantization.WoqLowpMode.FP16,
+ "int8": ipex.quantization.WoqLowpMode.INT8,
+ }
+ ipex_qconfig_mapping = ipex.quantization.get_weight_only_quant_qconfig_mapping(
+ weight_dtype=weight_dtype[4],
+ lowp_mode=compute_dtype["bf16"],
+ act_quant_mode=ipex.quantization.WoqActQuantMode.PER_IC_BLOCK,
+ group_size=self.group_size,
+ )
+ tmp_linear = torch.nn.Linear(
+ self.infeatures,
+ self.outfeatures,
+ True if hasattr(self, "bias") and self.bias is not None else False,
+ )
+ if tmp_linear.bias is not None and self.bias is not None:
+ tmp_linear.bias = torch.nn.Parameter(self.bias.float())
+
+ tmp_linear.qconfig = ipex_qconfig_mapping.global_qconfig
+ self.ipex_linear = ipex_linear.from_float_and_int4_weight(
+ mod=tmp_linear,
+ qweight=qweight,
+ scales=scales,
+ zero_points=qzeros,
+ bias=(self.bias.float() if hasattr(self, "bias") and self.bias is not None else None),
+ group_size=self.group_size,
+ g_idx=(self.g_idx if hasattr(self, "g_idx") else None)
+ )
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ W = linear.weight.data.clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
+ scales = scales.t().contiguous()
+ zeros = zeros.t().contiguous()
+ scale_zeros = zeros * scales
+ self.scales = scales.clone().to(dtype=linear.weight.dtype)
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().to(dtype=linear.weight.dtype)
+
+ intweight = []
+ for idx in range(self.infeatures):
+ intweight.append(
+ torch.round(
+ (W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[:, None])
+ intweight = torch.cat(intweight, dim=1)
+ intweight = intweight.t().contiguous()
+ intweight = intweight.numpy().astype(np.uint32)
+
+ i = 0
+ row = 0
+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
+ while row < qweight.shape[0]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qweight[row] |= intweight[j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ row += 1
+ elif self.bits == 3:
+ for j in range(i, i + 10):
+ qweight[row] |= intweight[j] << (3 * (j - i))
+ i += 10
+ qweight[row] |= intweight[i] << 30
+ row += 1
+ qweight[row] |= (intweight[i] >> 2) & 1
+ i += 1
+ for j in range(i, i + 10):
+ qweight[row] |= intweight[j] << (3 * (j - i) + 1)
+ i += 10
+ qweight[row] |= intweight[i] << 31
+ row += 1
+ qweight[row] |= (intweight[i] >> 1) & 0x3
+ i += 1
+ for j in range(i, i + 10):
+ qweight[row] |= intweight[j] << (3 * (j - i) + 2)
+ i += 10
+ row += 1
+ else:
+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+
+ qweight = qweight.astype(np.int32)
+ self.qweight = torch.from_numpy(qweight)
+
+ zeros -= 1
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ col += 1
+ elif self.bits == 3:
+ for j in range(i, i + 10):
+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i))
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 30
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1
+ i += 1
+ for j in range(i, i + 10):
+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 31
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
+ i += 1
+ for j in range(i, i + 10):
+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 2)
+ i += 10
+ col += 1
+ else:
+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+ def forward(self, x: torch.Tensor):
+ if not self.training and hasattr(self, "ipex_linear"):
+ return self.ipex_linear(x)
+
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ x = x.reshape(-1, x.shape[-1])
+ x_dtype = x.dtype
+ zeros = torch.bitwise_right_shift(
+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
+ self.wf.unsqueeze(0),
+ ).to(torch.int16)
+ zeros = torch.bitwise_and(zeros, (2 ** self.bits) - 1)
+
+ zeros = zeros + 1
+ zeros = zeros.reshape(self.scales.shape)
+
+ weight = torch.bitwise_right_shift(
+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),
+ self.wf.unsqueeze(-1),
+ ).to(torch.int16)
+ weight = torch.bitwise_and(weight, (2 ** self.bits) - 1)
+
+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
+ num_itr = self.g_idx.shape[0] // x.shape[-1]
+ if num_itr == 1:
+ weights = self.scales[self.g_idx.long()] * (weight - zeros[self.g_idx.long()])
+ else:
+ num_dim = self.g_idx.shape[0] // num_itr
+ weights = []
+ for i in range(num_itr):
+ scale_i = self.scales[:, i * num_dim: (i + 1) * num_dim]
+ weight_i = weight[:, i * num_dim: (i + 1) * num_dim]
+ zeros_i = zeros[:, i * num_dim: (i + 1) * num_dim]
+ g_idx_i = self.g_idx[i * num_dim: (i + 1) * num_dim]
+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))
+ weights = torch.cat(weights, dim=1)
+ out = torch.matmul(x, weights)
+ out = out.to(x_dtype)
+ out = out.reshape(out_shape)
+ out = out + self.bias if self.bias is not None else out
+
+ return out
+
+
+def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):
+ wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)
+ zeros = None
+ if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):
+ zp_shape = list(qzeros.shape)
+ zp_shape[1] = zp_shape[1] * (32 // bits)
+
+ zeros = torch.bitwise_right_shift(
+ torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)
+ ).to(torch.int16 if bits == 8 else torch.int8)
+ torch.bitwise_and(zeros, (2 ** bits) - 1, out=zeros)
+ if bits == 8:
+ zeros = zeros.to(torch.uint8)
+ zeros = zeros + 1
+ try:
+ zeros = zeros.reshape(zp_shape)
+ except:
+ # zeros and scales have different iteam numbers.
+ # remove 1 (due to 0 + 1 in line 252)
+ zeros = zeros[zeros != 1]
+ zeros = zeros.reshape(zp_shape)
+
+ weight = torch.bitwise_right_shift(
+ torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)
+ ).to(torch.int16 if bits == 8 else torch.int8)
+ weight.bitwise_and_((2 ** bits) - 1)
+ weight = weight.view(-1, weight.shape[-1])
+
+ if g_idx is not None:
+ group_size = weight.shape[0] // qzeros.shape[0]
+ weight2 = weight.clone()
+ group_dict = {}
+ for i in range(len(g_idx)):
+ group_idx = g_idx[i].item()
+ if group_idx not in group_dict:
+ target_idx = group_idx * group_size
+ group_dict[group_idx] = 0
+ else:
+ group_dict[group_idx] = group_dict[group_idx] + 1
+ target_idx = group_idx * group_size + group_dict[group_idx]
+ weight2[target_idx] = weight[i]
+ weight = weight2
+
+ return weight, zeros
+
+
+# Copied from qlinear_marlin.py
+def dequantize_weight(qweight, qzeros, scales, bits):
+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)
+ group_size = unpacked_qweight.shape[0] // scales.shape[0]
+ scales = scales.repeat_interleave(group_size, dim=0)
+ if unpacked_qzeros is not None:
+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)
+ else:
+ unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)
+ unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales
+
+ return unpacked_qweight, unpacked_qzeros
+
+
+__all__ = ["QuantLinear"]