PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add ipex support for intel cpu (#292)
wenhuach21168a1f697b84
diff --git a/README.md b/README.mdindex e0d76b868..c622ba4b7 100644--- a/README.md+++ b/README.md@@ -204,7 +204,7 @@ Please run the quantization code first### AutoRound format-**CPU**: pip install intel-extension-for-transformers, auto_round version >0.3.1+**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip install intel-extension-for-transformers,**HPU**: docker image with Gaudi Software Stack is recommended. More details can be foundin [Gaudi Guide](https://docs.habana.ai/en/latest/).@@ -217,7 +217,7 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundConfig-backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 'pip install -v gptqmodel --no-build-isolation')+backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 and 'pip install -v gptqmodel --no-build-isolation')quantization_config = AutoRoundConfig(backend=backend)diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex c9d3d3d7f..859481302 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -48,6 +48,8 @@ from auto_round.backend import get_layer_backend, dynamic_import_inference_lineaimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbitsimport auto_round_extension.qbits.qlinear_qbits_gptq as qlinear_qbits_gptq+import auto_round_extension.ipex.qlinear_ipex_gptq as qlinear_ipex_gptq+import auto_round_extension.ipex.qlinear_ipex_awq as qlinear_ipex_awqfrom auto_round.backend import BackendInfosfrom transformers.utils.versions import require_versionfrom enum import Enum@@ -397,10 +399,11 @@ class AutoRoundQuantizer(HfQuantizer):quantization_config.target_backend = quantization_config.backendtarget_device = self.detect_device(quantization_config.target_backend, quantization_config.backend)+ self.target_device = target_deviceif hasattr(quantization_config, "backend"): # pragma: no cover- if "hpu" == target_device and model.dtype != torch.bfloat16:- logger.info("Change the dtype to `bfloat16` as HPU does not support float16")+ if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:+ logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")model = model.to(torch.bfloat16)bits = quantization_config.bits@@ -576,14 +579,20 @@ class AutoRoundQuantizer(HfQuantizer):new_layer.device = layer_deviceset_module(module, layer_name, new_layer)- def qbits_post_init(self, model):+ def cpu_post_init(self, model):dep_check = True- for layer in model.modules():+ message = "Repacking to CPU format"++ for n, layer in tqdm(model.named_modules(), desc=message, total=len(list(model.named_modules())),+ leave=True): ##not exit correctlyif isinstance(layer, (qlinear_qbits.QuantLinear, qlinear_qbits_gptq.QuantLinear)):if dep_check:layer.req_check()layer.post_init()dep_check = False+ if isinstance(layer, (qlinear_ipex_gptq.QuantLinear, qlinear_ipex_awq.QuantLinear)):+ layer.post_init()+return modeldef repack_marlin(self, model):@@ -692,8 +701,9 @@ class AutoRoundQuantizer(HfQuantizer):self.repack_marlin(model)model = autoround_post_init(model)- # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.- model = self.qbits_post_init(model)+ # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.+ if self.target_device == "cpu":+ model = self.cpu_post_init(model)return modeldiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4d0a8e05b..0cb2c3ff4 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -515,7 +515,7 @@ class AutoRound(object):elif isinstance(data, str):if self.tokenizer is None:logger.error("please provide tokenizer for string input")- exit()+ exit(-1)data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").datadata_new = {}for key in data.keys():@@ -553,7 +553,7 @@ class AutoRound(object):f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "f"dataset or decease the sequence length")- exit()+ exit(-1)elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples collected may affect the quantification. "@@ -1166,7 +1166,7 @@ class AutoRound(object):format = format.split(":")[0]if format not in EXPORT_FORMAT:logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")- exit()+ exit(-1)save_quantized_as_format = EXPORT_FORMAT.get(format)if "gptq" in format and not self.sym:logger.warning(@@ -1252,7 +1252,7 @@ class AutoRound(object):layer = get_module(self.model, key)if layer is None:logger.error(f"could not find layer {key} in the model, exit...")- exit()+ exit(-1)if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):layer_names.append(key)diff --git a/auto_round/backend.py b/auto_round/backend.pyindex 16e898ab1..ea79ac850 100644--- a/auto_round/backend.py+++ b/auto_round/backend.py@@ -13,13 +13,24 @@# limitations under the License.import functools+import loggingfrom dataclasses import dataclass, fieldfrom typing import List, Any, Optional+from transformers.utils.versions import require_version+from auto_round.utils import get_library_versionBackendInfos = {}+import cpuinfo++def get_cpu_manufacturer():+ cpu_info = cpuinfo.get_cpu_info()+ if "brand_raw" in cpu_info and "intel" in cpu_info["brand_raw"].lower():+ return "intel"+ else:+ return "others"@dataclassclass BackendInfo:@@ -55,6 +66,7 @@ class BackendInfo:convertable_format: List[str] = field(default_factory=list)feature_checks: List[Any] = field(default_factory=list)alias: Optional[List[str]] = None+ requirements: Optional[List[str]] = Nonedef feature_multiply_checker(in_feature, out_feature, in_feature_multiplier, out_feature_multiplier=None):@@ -94,7 +106,8 @@ BackendInfos['gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],priority=5,feature_checks=[feature_multiply_checker_32],alias=["auto_round:gptq:exllamav2", "auto_round:auto_gptq:exllamav2",- 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"]+ 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"],+ requirements=["auto-gptq>=0.7.1"])BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],@@ -102,28 +115,54 @@ BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],bits=[2, 4, 8], group_size=None,priority=0, feature_checks=[feature_multiply_checker_32],alias=["auto_round:gptq:tritonv2", "auto_round:auto_gptq:tritonv2",- "auto_gptq:tritonv2"])+ "auto_gptq:tritonv2"],+ requirements=["auto-gptq>=0.7.1"]+ )BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemmpacking_format="awq",bits=[4], group_size=None,priority=4, feature_checks=[feature_num_greater_checker_1024],alias=["auto_awq:gemm", "auto_round:awq:gemm", "auto_round:auto_awq:gemm", "awq",- "auto_awq", "auto_round:awq", "aut_round:auto_awq"])+ "auto_awq", "auto_round:awq", "aut_round:auto_awq"],+ requirements=["autoawq"]+ )BackendInfos['auto_round:qbits'] = BackendInfo(device=["cpu"], sym=[True, False],packing_format="qbits",bits=[2, 4, 8], group_size=None,- priority=0,+ priority=0 if "intel" in get_cpu_manufacturer() else 5,feature_checks=[],- convertable_format=["triton"])+ convertable_format=["triton"],+ requirements=["intel-extension-for-transformers"])BackendInfos['auto_round:qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],packing_format="qbits",bits=[2, 4, 8], group_size=None,- priority=0,+ priority=0 if "intel" in get_cpu_manufacturer() else 5,feature_checks=[],- convertable_format=["triton_zp+-1"])+ convertable_format=["triton_zp+-1"],+ requirements=["intel-extension-for-transformers"]+ )++BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu"], sym=[True, False],+ packing_format="ipex_gptq",+ bits=[4], group_size=None,+ priority=5 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ convertable_format=["triton_zp+-1"],+ requirements=["intel-extension-for-pytorch>=2.4"]+ )++BackendInfos['auto_round:ipex_awq'] = BackendInfo(device=["cpu"], sym=[True, False],+ packing_format="ipex_awq",+ bits=[4], group_size=None,+ priority=5 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ ##convertable_format=["triton_zp+-1", "awq"],+ convertable_format=["awq"],+ requirements=["intel-extension-for-pytorch>=2.4"]+ )# BackendInfos['auto_round:marlin'] = BackendInfo(device=["gpu"], sym=[True],# packing_format="marlin",@@ -147,7 +186,8 @@ BackendInfos['auto_round:hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, Falseconvertable_format=["triton_zp+-1"])-def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features):+def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features,+ check_requirements=True):"""Checks if the given configuration is compatible with the specified backend.Args:@@ -160,6 +200,7 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_formatpacking_format (str): The packing format used by the backend (e.g., 'triton').in_features (int): The number of input features for the model layer.out_features (int): The number of output features for the model layer.+ check_requirements (bool): Whether check the requirementReturns:bool: True if the configuration is compatible with the backend, False otherwise.@@ -203,6 +244,13 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_formatif packing_format != backend.packing_format and packing_format not in backend.convertable_format:return False+ if check_requirements and backend.requirements is not None:+ for requirement in backend.requirements:+ try:+ require_version(requirement)+ except ImportError:+ return False+return True@@ -245,6 +293,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):else: # auto_round must be at the endimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbits_autoroundreturn qlinear_qbits_autoround.QuantLinear+ if "ipex_gptq" in backend:+ from auto_round_extension.ipex.qlinear_ipex_gptq import QuantLinear+ return QuantLinear++ if "ipex_awq" in backend:+ from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear+ return QuantLinearif "marlin" in backend:from transformers.utils.versions import require_version@@ -431,6 +486,21 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f# Raise an error if no compatible backends are foundif len(supported_backends) == 0:+ supported_backends_need_package = []+ for key in BackendInfos.keys():+ if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,+ check_requirements=False):+ supported_backends_need_package.append(key)++ if len(supported_backends_need_package) > 0:+ supported_backends_need_package = sorted(supported_backends_need_package,+ key=lambda support_backend: BackendInfos[support_backend].priority,+ reverse=True)+ backend_info = BackendInfos[supported_backends_need_package[0]]+ str_info = ",".join(backend_info.requirements)+ logging.error(f"`pip install {str_info}` to support inference")+ exit(-1)+raise ValueError(f"None of the backends support this layer")# Sort the compatible backends by priority and return the one with the highest priority@@ -438,21 +508,3 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_freverse=True)return supported_backends[0]---if __name__ == "__main__":- res = get_layer_backend("cuda", "gptq:exllamav2", "gptq:exllamav2", 4, 128, sym=False, in_features=128,- out_features=128)- assert res == "gptq:exllamav2"-- res = get_layer_backend("cuda", "gptq:exllamav2", "gptq:exllamav2", 2, 128, sym=False, in_features=128,- out_features=128)- assert res == "gptq:tritonv2"-- res = get_layer_backend("cpu", "auto_round:exllamav2", "auto_round:exllamav2", 4, 128, sym=False, in_features=128,- out_features=128)- assert res == "auto_round:qbits"-- res = get_layer_backend("cpu", "gptq:exllamav2", "gptq:exllamav2", 4, 128, sym=False, in_features=128,- out_features=128)- assert res == "auto_round:qbits_zp"diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex cfb4a2a11..1cf084b2b 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -135,7 +135,6 @@ class QuantLinear(nn.Module, TritonModuleMixin):qzeros = qzeros.astype(np.int32)self.qzeros = torch.from_numpy(qzeros)- delattr(self, "g_idx")__all__ = ["QuantLinear"]diff --git a/auto_round_extension/cuda/qlinear_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.pyindex 786bb091d..3bc415971 100644--- a/auto_round_extension/cuda/qlinear_exllamav2.py+++ b/auto_round_extension/cuda/qlinear_exllamav2.py@@ -196,6 +196,10 @@ class QuantLinear(nn.Module):dtype=torch.float16,),)+ self.register_buffer(+ "g_idx",+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),+ )if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))@@ -210,6 +214,7 @@ class QuantLinear(nn.Module):"qweight": self.qweight,"qzeros": self.qzeros,"scales": self.scales,+ "g_idx": self.g_idx,}temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)diff --git a/auto_round_extension/ipex/__init__.py b/auto_round_extension/ipex/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pynew file mode 100644index 000000000..c90e9861b--- /dev/null+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py@@ -0,0 +1,87 @@+import torch+import torch.nn as nn++class QuantLinear(nn.Module):++ def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):+ super().__init__()+ assert w_bit == 4, "Only 4 bit are supported for now."++ self.use_bf16 = True # Intel platform support bf16 even without amx.++ self.in_features = in_features+ self.out_features = out_features+ self.w_bit = w_bit+ self.group_size = group_size if group_size != -1 else in_features+ self.scale_dtype = torch.float32++ # quick sanity check (make sure alignment)+ assert self.in_features % self.group_size == 0+ assert out_features % (32 // self.w_bit) == 0+ self.pack_num = 32 // self.w_bit++ self.register_buffer(+ "qzeros",+ torch.zeros(+ (in_features // self.group_size, out_features // self.pack_num),+ dtype=torch.int32,+ device=dev,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (in_features // self.group_size, out_features),+ dtype=torch.bfloat16 if self.use_bf16 else torch.float32,+ device=dev,+ ))+ if bias:+ self.register_buffer(+ "bias",+ torch.zeros((out_features), dtype=torch.bfloat16 if self.use_bf16 else torch.float32, device=dev),+ )+ else:+ self.register_buffer(+ "bias",+ None,+ )+ qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)+ self.register_buffer("qweight", qweight)++ def post_init(self):+ assert self.qweight.device.type == "cpu"+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear+ self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \+ self.in_features, self.out_features, None, self.bias, \+ self.group_size, None, 0, 1)++ @classmethod+ def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):+ awq_linear = cls(+ w_bit,+ group_size,+ linear.in_features,+ linear.out_features,+ linear.bias is not None,+ linear.weight.device,+ )+ if init_only: # just prepare for loading sd+ return awq_linear++ raise NotImplementedError("Only inference is supported for IPEX kernels")++ @torch.no_grad()+ def forward(self, x):++ outputs = self.ipex_linear(x)++ return outputs++ def extra_repr(self) -> str:+ return ("in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ self.in_features,+ self.out_features,+ self.bias is not None,+ self.w_bit,+ self.group_size,+ ))diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pynew file mode 100644index 000000000..c69dd06c6--- /dev/null+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py@@ -0,0 +1,359 @@+import math++import numpy as np+import torch+import torch.nn as nn+import transformers++from logging import getLogger++logger = getLogger(__name__)++BITS_DTYPE_MAPPING = {+ 8: "int8",+}+++def convert_dtype_torch2str(dtype):+ if dtype == torch.int8:+ return "int8"+ elif dtype == torch.float:+ return "fp32"+ elif dtype == torch.float16:+ return "fp16"+ elif dtype == torch.bfloat16:+ return "bf16"+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:+ return dtype+ else:+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+++class QuantLinear(nn.Module):+ QUANT_TYPE = "ipex_gptq"++ def __init__(+ self,+ bits,+ group_size,+ infeatures,+ outfeatures,+ bias,+ kernel_switch_threshold=128,+ training=False,+ weight_dtype=torch.bfloat16,+ **kwargs,+ ):+ super().__init__()++ if bits not in [4]:+ raise NotImplementedError("Only 4-bits is supported for IPEX.")++ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2 ** self.bits - 1+ self.weight_dtype = weight_dtype+ self.asym = True++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=weight_dtype,+ ),+ )+ self.register_buffer(+ "g_idx",+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))+ else:+ self.bias = None++ self.kernel_switch_threshold = kernel_switch_threshold++ self.training = training++ # for training forward+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)++ def post_init(self):+ assert self.qweight.device.type == "cpu"++ # if not self.training and IPEX_AVAILABLE:+ if not self.training:+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear+ from packaging import version+ from auto_round.utils import get_library_version+ ipex_version = get_library_version("intel_extension_for_pytorch")+ if version.parse(ipex_version) >= version.parse("2.5"):+ self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \+ self.infeatures, self.outfeatures, None,+ self.bias, \+ self.group_size, self.g_idx, 0, 0)+ else:+ import intel_extension_for_pytorch as ipex+ from intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_linear+ from intel_extension_for_pytorch.utils.weight_only_quantization import (+ _convert_optimum_format_to_desired,+ )++ qweight = self.qweight+ scales = self.scales+ qzeros = self.qzeros++ qweight, scales, qzeros = _convert_optimum_format_to_desired(qweight, scales, qzeros)+ weight_dtype = {+ 4: ipex.quantization.WoqWeightDtype.INT4,+ 8: ipex.quantization.WoqWeightDtype.INT8,+ }+ compute_dtype = {+ "fp32": ipex.quantization.WoqLowpMode.NONE, # follow the activation datatype.+ "bf16": ipex.quantization.WoqLowpMode.BF16,+ "fp16": ipex.quantization.WoqLowpMode.FP16,+ "int8": ipex.quantization.WoqLowpMode.INT8,+ }+ ipex_qconfig_mapping = ipex.quantization.get_weight_only_quant_qconfig_mapping(+ weight_dtype=weight_dtype[4],+ lowp_mode=compute_dtype["bf16"],+ act_quant_mode=ipex.quantization.WoqActQuantMode.PER_IC_BLOCK,+ group_size=self.group_size,+ )+ tmp_linear = torch.nn.Linear(+ self.infeatures,+ self.outfeatures,+ True if hasattr(self, "bias") and self.bias is not None else False,+ )+ if tmp_linear.bias is not None and self.bias is not None:+ tmp_linear.bias = torch.nn.Parameter(self.bias.float())++ tmp_linear.qconfig = ipex_qconfig_mapping.global_qconfig+ self.ipex_linear = ipex_linear.from_float_and_int4_weight(+ mod=tmp_linear,+ qweight=qweight,+ scales=scales,+ zero_points=qzeros,+ bias=(self.bias.float() if hasattr(self, "bias") and self.bias is not None else None),+ group_size=self.group_size,+ g_idx=(self.g_idx if hasattr(self, "g_idx") else None)+ )++ def pack(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx++ scales = scales.t().contiguous()+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ self.scales = scales.clone().to(dtype=linear.weight.dtype)+ if linear.bias is not None:+ self.bias = linear.bias.clone().to(dtype=linear.weight.dtype)++ intweight = []+ for idx in range(self.infeatures):+ intweight.append(+ torch.round(+ (W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[:, None])+ intweight = torch.cat(intweight, dim=1)+ intweight = intweight.t().contiguous()+ intweight = intweight.numpy().astype(np.uint32)++ i = 0+ row = 0+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)+ while row < qweight.shape[0]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1+ elif self.bits == 3:+ for j in range(i, i + 10):+ qweight[row] |= intweight[j] << (3 * (j - i))+ i += 10+ qweight[row] |= intweight[i] << 30+ row += 1+ qweight[row] |= (intweight[i] >> 2) & 1+ i += 1+ for j in range(i, i + 10):+ qweight[row] |= intweight[j] << (3 * (j - i) + 1)+ i += 10+ qweight[row] |= intweight[i] << 31+ row += 1+ qweight[row] |= (intweight[i] >> 1) & 0x3+ i += 1+ for j in range(i, i + 10):+ qweight[row] |= intweight[j] << (3 * (j - i) + 2)+ i += 10+ row += 1+ else:+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ i += 32 // self.bits+ col += 1+ elif self.bits == 3:+ for j in range(i, i + 10):+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i))+ i += 10+ qzeros[:, col] |= zeros[:, i] << 30+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1+ i += 1+ for j in range(i, i + 10):+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 31+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3+ i += 1+ for j in range(i, i + 10):+ qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 2)+ i += 10+ col += 1+ else:+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++ def forward(self, x: torch.Tensor):+ if not self.training and hasattr(self, "ipex_linear"):+ return self.ipex_linear(x)++ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.reshape(-1, x.shape[-1])+ x_dtype = x.dtype+ zeros = torch.bitwise_right_shift(+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),+ self.wf.unsqueeze(0),+ ).to(torch.int16)+ zeros = torch.bitwise_and(zeros, (2 ** self.bits) - 1)++ zeros = zeros + 1+ zeros = zeros.reshape(self.scales.shape)++ weight = torch.bitwise_right_shift(+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),+ self.wf.unsqueeze(-1),+ ).to(torch.int16)+ weight = torch.bitwise_and(weight, (2 ** self.bits) - 1)++ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])+ num_itr = self.g_idx.shape[0] // x.shape[-1]+ if num_itr == 1:+ weights = self.scales[self.g_idx.long()] * (weight - zeros[self.g_idx.long()])+ else:+ num_dim = self.g_idx.shape[0] // num_itr+ weights = []+ for i in range(num_itr):+ scale_i = self.scales[:, i * num_dim: (i + 1) * num_dim]+ weight_i = weight[:, i * num_dim: (i + 1) * num_dim]+ zeros_i = zeros[:, i * num_dim: (i + 1) * num_dim]+ g_idx_i = self.g_idx[i * num_dim: (i + 1) * num_dim]+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))+ weights = torch.cat(weights, dim=1)+ out = torch.matmul(x, weights)+ out = out.to(x_dtype)+ out = out.reshape(out_shape)+ out = out + self.bias if self.bias is not None else out++ return out++[email protected]_grad()+def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):+ wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)+ zeros = None+ if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):+ zp_shape = list(qzeros.shape)+ zp_shape[1] = zp_shape[1] * (32 // bits)++ zeros = torch.bitwise_right_shift(+ torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)+ ).to(torch.int16 if bits == 8 else torch.int8)+ torch.bitwise_and(zeros, (2 ** bits) - 1, out=zeros)+ if bits == 8:+ zeros = zeros.to(torch.uint8)+ zeros = zeros + 1+ try:+ zeros = zeros.reshape(zp_shape)+ except:+ # zeros and scales have different iteam numbers.+ # remove 1 (due to 0 + 1 in line 252)+ zeros = zeros[zeros != 1]+ zeros = zeros.reshape(zp_shape)++ weight = torch.bitwise_right_shift(+ torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)+ ).to(torch.int16 if bits == 8 else torch.int8)+ weight.bitwise_and_((2 ** bits) - 1)+ weight = weight.view(-1, weight.shape[-1])++ if g_idx is not None:+ group_size = weight.shape[0] // qzeros.shape[0]+ weight2 = weight.clone()+ group_dict = {}+ for i in range(len(g_idx)):+ group_idx = g_idx[i].item()+ if group_idx not in group_dict:+ target_idx = group_idx * group_size+ group_dict[group_idx] = 0+ else:+ group_dict[group_idx] = group_dict[group_idx] + 1+ target_idx = group_idx * group_size + group_dict[group_idx]+ weight2[target_idx] = weight[i]+ weight = weight2++ return weight, zeros+++# Copied from qlinear_marlin.py[email protected]_grad()+def dequantize_weight(qweight, qzeros, scales, bits):+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)+ group_size = unpacked_qweight.shape[0] // scales.shape[0]+ scales = scales.repeat_interleave(group_size, dim=0)+ if unpacked_qzeros is not None:+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)+ else:+ unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)+ unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales++ return unpacked_qweight, unpacked_qzeros+++__all__ = ["QuantLinear"]