PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
speedup packing stage for autogptq and autoawq format (#407)
Wenhua Cheng937d0198b861
diff --git a/README.md b/README.mdindex 55bcb696c..b69a9c230 100644--- a/README.md+++ b/README.md@@ -6,7 +6,10 @@ AutoRound[](https://github.com/intel/auto-round)[](https://github.com/intel/auto-round)-[](https://github.com/intel/auto-round/blob/main/LICENSE)+[](https://github.com/intel/auto-round/blob/main/LICENSE)+<a href="https://huggingface.co/OPEA">+ <img alt="Model Checkpoints" src="https://img.shields.io/badge/%F0%9F%A4%97%20HF-Models-F57C00">+</a>---<div align="left">diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex f558ac941..0f629a027 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -25,6 +25,7 @@ MXFP_FORMAT_CACHE = {"mx_fp8": (4, 5, 8, 448.0, 0.015625),"mx_fp8e4m3": (4, 5, 8, 448.0, 0.015625),"mx_fp6e3m2": (3, 4, 4, 28.0, 0.25),+ "mx_fp6": (2, 5, 2, 7.5, 1.0),"mx_fp6e2m3": (2, 5, 2, 7.5, 1.0),"mx_fp4": (2, 3, 2, 6.0, 1.0),"mx_fp4e2m1": (2, 3, 2, 6.0, 1.0),diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex b0c116d9b..fe4707983 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -36,6 +36,7 @@# SOFTWARE.import torch+import auto_round.export.export_to_autogptq.qlinear_tritonfrom auto_round.utils import check_to_quantized, get_block_names, \get_module, logger, set_moduleimport copy@@ -102,7 +103,11 @@ def pack_layer(name, model, layer_config, backend, pbar):# so far can only pack layer on CPUqlayer.to("cpu")##force to float32 to be compatible with torch 2.0- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ if sym and isinstance(new_layer, auto_round.export.export_to_autogptq.qlinear_triton.QuantLinear):+ layer, scale = layer.to("cpu"), scale.to("cpu")+ zero = 2 ** (bits - 1)+ else:+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)sig = inspect.signature(qlayer.pack)param_count = len(sig.parameters)if param_count == 2:@@ -138,7 +143,6 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exlllogger.error(f"auto-gptq format may not support loading this quantized model")quantization_config['block_name_to_quantize'] = common_prefixquantization_config.pop("to_quant_block_names", None)-all_to_quantized = Truemodules_in_block_to_quantize = []@@ -215,6 +219,3 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)---diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex 1cf084b2b..8a10d06a5 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -18,6 +18,29 @@ import numpy as npimport torchimport torch.nn as nnimport transformers+import numba+++##TODO different bits+# @numba.jit(nopython=True, parallel=True)+# def pack_array_with_numba_b4_c32(+# raw_array: np.ndarray, packed_array: np.ndarray+# ) -> np.ndarray:+# """Pack the array with numba when bits=4 and compress_bits=32."""+# bits = 4+# n_pack = 32 // bits+#+# for row in range(packed_array.shape[0]):+# packed_array[row] = ((((raw_array[row * n_pack + 7]) << 28)+# | ((raw_array[row * n_pack + 6]) << 24)+# | ((raw_array[row * n_pack + 5]) << 20)+# | ((raw_array[row * n_pack + 4]) << 16)+# | ((raw_array[row * n_pack + 3]) << 12)+# | (raw_array[row * n_pack + 2]) << 8)+# | ((raw_array[row * n_pack + 1]) << 4)+# | ((raw_array[row * n_pack]) << 0))+#+# return packed_arrayclass TritonModuleMixin:@@ -76,7 +99,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ def pack_cpu(self, linear, scales, zeros, g_idx=None):W = linear.weight.data.clone()if isinstance(linear, nn.Conv2d):W = W.flatten(1)@@ -86,8 +109,11 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.g_idx = g_idx.clone() if g_idx is not None else self.g_idxscales = scales.t().contiguous()- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales+ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ else:+ scale_zeros = scales * zerosself.scales = scales.clone().half()if linear.bias is not None:self.bias = linear.bias.clone().half()@@ -107,34 +133,103 @@ class QuantLinear(nn.Module, TritonModuleMixin):row = 0qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)while row < qweight.shape[0]:- if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ if isinstance(zeros, torch.Tensor):+ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))i += 32 // self.bits- row += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")+ col += 1++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ zeros -= 1+ shape = scales.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++ def pack(self, linear, scales, zeros, g_idx):+ if torch.cuda.is_available():+ return self.pack_cuda(linear, scales, zeros, g_idx)+ else:+ return self.pack_cpu(linear, scales, zeros, g_idx)++ def pack_cuda(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()+ scales_t = scales.t().contiguous()+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()++ repeat_scales = scales.to("cuda:0").repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to("cuda:0").repeat_interleave(self.group_size, 1)+ else:+ repeat_zeros = zeros++ intweight = torch.round(W.to("cuda:0") / repeat_scales + repeat_zeros).to(torch.int).t().contiguous().to("cpu")+ intweight = intweight.numpy().astype(np.uint32)+ del repeat_scales++ i = 0+ row = 0+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)+ # pack_array_with_numba_b4_c32(intweight, qweight)+ while row < qweight.shape[0]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1qweight = qweight.astype(np.int32)self.qweight = torch.from_numpy(qweight)- zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:+ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:for j in range(i, i + (32 // self.bits)):qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))i += 32 // self.bitscol += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ zeros -= 1+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)__all__ = ["QuantLinear"]diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex b13d5d756..edcd61773 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -54,6 +54,8 @@ def pack_layer(name, model, layer_config, backend, pbar):bits = config["bits"]group_size = config["group_size"]linear_layer = get_module(model, name)+ if config["sym"] :+ zp = 2** (config["bits"]-1)q_linear = WQLinear_GEMM.from_linear(linear=linear_layer,w_bit=bits,@@ -62,10 +64,7 @@ def pack_layer(name, model, layer_config, backend, pbar):scales=scale,zeros=zp,)- linear_layer.cpu()- q_linear.to("cpu")set_module(model, name, q_linear)- clear_memory()pbar.update(1)diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex f4f45bf0e..a8c0c63ae 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -221,23 +221,29 @@ class WQLinear_GEMM(nn.Module):pack_num = 32 // awq_linear.w_bit- intweight = []- for idx in range(awq_linear.in_features):- intweight.append(- torch.round(- (linear.weight.data[:, idx] + scale_zeros[idx // group_size])- / awq_linear.scales[idx // group_size]- ).to(torch.int)[:, None]- )- intweight = torch.cat(intweight, dim=1)- intweight = intweight.t().contiguous()- intweight = intweight.to(dtype=torch.int32)-- best_device = get_best_device()+ if torch.cuda.is_available():- # Avoid: The operator 'aten::__lshift__.Scalar' is not currently implemented for the MPS device- if "mps" in best_device:- intweight = intweight.to("cpu")+ repeat_scales = scales.to("cuda:0").t().repeat_interleave(group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to("cuda:0").t().repeat_interleave(group_size, 1)+ else:+ repeat_zeros = zeros+ intweight = torch.round(linear.weight.to("cuda:0") / repeat_scales + repeat_zeros).to(+ torch.int).t().contiguous().to("cpu")+ intweight = intweight.to(dtype=torch.int32)+ del repeat_scales+ else:+ intweight = []+ for idx in range(awq_linear.in_features):+ intweight.append(+ torch.round(+ (linear.weight.data[:, idx] + scale_zeros[idx // group_size])+ / awq_linear.scales[idx // group_size]+ ).to(torch.int)[:, None]+ )+ intweight = torch.cat(intweight, dim=1)+ intweight = intweight.t().contiguous()+ intweight = intweight.to(dtype=torch.int32)qweight = torch.zeros((intweight.shape[0], intweight.shape[1] // 32 * awq_linear.w_bit),@@ -246,34 +252,35 @@ class WQLinear_GEMM(nn.Module):)for col in range(intweight.shape[1] // pack_num):- if awq_linear.w_bit == 4:- order_map = [0, 2, 4, 6, 1, 3, 5, 7]- else:- raise NotImplementedError("Only 4-bit are supported for now.")+ order_map = [0, 2, 4, 6, 1, 3, 5, 7]for i in range(pack_num):qweight_col = intweight[:, col * pack_num + order_map[i]]qweight[:, col] |= qweight_col << (i * awq_linear.w_bit)awq_linear.qweight = qweight- zeros = zeros.to(dtype=torch.int32, device=best_device)-- if "mps" in best_device:- zeros = zeros.to("cpu")-- qzeros = torch.zeros(- (zeros.shape[0], zeros.shape[1] // 32 * awq_linear.w_bit),- dtype=torch.int32,- device=zeros.device,- )+ if isinstance(zeros, torch.Tensor):+ qzeros = torch.zeros(+ (scales.shape[0], scales.shape[1] // 32 * awq_linear.w_bit),+ dtype=torch.int32,+ device=zeros.device,+ )+ zeros = zeros.to(dtype=torch.int32, device="cpu")- for col in range(zeros.shape[1] // pack_num):- if awq_linear.w_bit == 4:+ for col in range(zeros.shape[1] // pack_num):order_map = [0, 2, 4, 6, 1, 3, 5, 7]- else:- raise NotImplementedError("Only 4-bit are supported for now.")+ for i in range(pack_num):+ qzero_col = zeros[:, col * pack_num + order_map[i]]+ qzeros[:, col] |= qzero_col << (i * awq_linear.w_bit)+ else:+ value = 0for i in range(pack_num):- qzero_col = zeros[:, col * pack_num + order_map[i]]- qzeros[:, col] |= qzero_col << (i * awq_linear.w_bit)+ value |= zeros << (i * awq_linear.w_bit)+ qzeros = torch.ones(+ (scales.shape[0], scales.shape[1] // 32 * awq_linear.w_bit),+ dtype=torch.int32,+ device=qweight.device,+ ) * value+awq_linear.qzeros = qzerosreturn awq_lineardiff --git a/auto_round_extension/cuda/qlinear_triton.py b/auto_round_extension/cuda/qlinear_triton.pyindex 7686522c8..006911f2f 100644--- a/auto_round_extension/cuda/qlinear_triton.py+++ b/auto_round_extension/cuda/qlinear_triton.py@@ -127,7 +127,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ def pack_cpu(self, linear, scales, zeros, g_idx=None):W = linear.weight.data.clone()if isinstance(linear, nn.Conv2d):W = W.flatten(1)@@ -137,8 +137,11 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.g_idx = g_idx.clone() if g_idx is not None else self.g_idxscales = scales.t().contiguous()- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales+ if isinstance(zeros,torch.Tensor):+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ else:+ scale_zeros = scales*zerosself.scales = scales.clone().half()if linear.bias is not None:self.bias = linear.bias.clone().half()@@ -147,7 +150,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):for idx in range(self.infeatures):intweight.append(torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[- :, None+ :, None])intweight = torch.cat(intweight, dim=1)@@ -158,33 +161,101 @@ class QuantLinear(nn.Module, TritonModuleMixin):row = 0qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)while row < qweight.shape[0]:- if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))i += 32 // self.bits- row += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")+ col += 1++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ shape = scales.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++++ def pack(self, linear, scales, zeros, g_idx):+ if torch.cuda.is_available():+ return self.pack_cuda(linear, scales, zeros, g_idx)+ else:+ return self.pack_cpu(linear, scales, zeros, g_idx)++ def pack_cuda(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()+ scales_t = scales.t().contiguous()+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()++ repeat_scales = scales.to("cuda:0").repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to("cuda:0").repeat_interleave(self.group_size, 1)+ else:+ repeat_zeros = zeros++ intweight = torch.round(W.to("cuda:0") / repeat_scales + repeat_zeros).to(torch.int).t().contiguous().to("cpu")+ intweight = intweight.numpy().astype(np.uint32)+ del repeat_scales++ i = 0+ row = 0+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)+ # pack_array_with_numba_b4_c32(intweight, qweight)+ while row < qweight.shape[0]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1qweight = qweight.astype(np.int32)self.qweight = torch.from_numpy(qweight)- # zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:+ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:for j in range(i, i + (32 // self.bits)):qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))i += 32 // self.bitscol += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)