PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add torch backend (#555)
Weiweice8eedd763d1
diff --git a/README.md b/README.mdindex 447d3bdf5..54207c37f 100644--- a/README.md+++ b/README.md@@ -356,6 +356,7 @@ to the following table for the details and specify the backend you want.| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -513,3 +514,5 @@ If you find AutoRound useful for your research, please cite our paper:year={2023}}```++diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex de6a5dbdf..31361eb9e 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -539,7 +539,7 @@ class AutoRound(object):"The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"" particularly for 2-bit quantization and smaller models."" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/4/8 bits)."+ "the AutoRound format(2/3/4/8 bits).")save_format_ = format.replace(":", "-").replace("_", "-")save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir@@ -1665,7 +1665,7 @@ class AutoRound(object):)format = "auto_round"- if re.search("q\d_k", format) and not self.data_type.endswith("_dq"):+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):logger.error(f"datatype<{self.data_type}> not support to export {format} format."" Please change export format or data_type."diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 829314f0a..690533398 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -139,7 +139,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllif common_prefix not in BLOCK_PATTERNS:logger.error(f"auto-gptq format may not support loading this quantized model")quantization_config['block_name_to_quantize'] = common_prefix- quantization_config.pop("to_quant_block_names", None)+ quantization_config.pop("to_quant_block_names", None)## as layers maybe already packed, we need to check in layer_configlayer_config = kwargs["layer_config"]@@ -238,3 +238,5 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)++diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 476a91fe8..42fc63e6d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -20,10 +20,12 @@ import osimport torchimport torch.nn as nnimport transformers-+try:+ import auto_round_extension.triton.qlinear_tritonv2+except Exception as error:+ raise ImportError(error)+import auto_round_extension.torch.qlinear_torchimport auto_round.export.export_to_autoround.qlinear_triton_act--import auto_round_extension.triton.qlinear_tritonv2from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \filter_quantization_configimport threadpoolctl as tctl@@ -74,17 +76,17 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:if act_bits <= 8: ##easily have bug for other configuration, need to refine code laterreturn auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear-- from auto_round_extension.triton.qlinear_tritonv2 import QuantLinear+ from auto_round_extension.torch.qlinear_torch import QuantLinearreturn QuantLinear- elif "auto_round" in backend and "gptq" in backend and bits in (2, 4, 8):- from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx+ elif "auto_round" in backend and "gptq" in backend:+ from auto_round_extension.torch.qlinear_torch_zp import QuantLinearreturn QuantLinearelif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMMreturn WQLinear_GEMMelif "gptqmodel" in backend:- return auto_round_extension.triton.qlinear_tritonv2.QuantLinear+ from auto_round_extension.torch.qlinear_torch import QuantLinear+ return QuantLinearelif "gptq" in backend and not "gptqmodel" in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:@@ -190,14 +192,15 @@ def pack_layer(layer_name, model, backend):if "awq" not in backend:new_layer = QuantLinear( ##pylint: disable=E1123- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype+ bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype)new_layer.device = deviceset_module(model, layer_name, new_layer)qlayer = new_layerimport auto_round.export.export_to_autoround.qlinear_tritonif sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,- auto_round_extension.triton.qlinear_tritonv2.QuantLinear)):+ auto_round_extension.triton.qlinear_tritonv2.QuantLinear,+ auto_round_extension.torch.qlinear_torch.QuantLinear)):zp = int(zp.flatten()[0])qlayer.to("cpu")@@ -268,8 +271,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config = kwargs["serialization_dict"]quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)quantization_config["quant_method"] = "auto-round"- if quantization_config["bits"] == 3:- backend = "auto_round:auto_gptq"quantization_config["packing_format"] = backendtokenizer = kwargs.get("tokenizer", None)@@ -380,3 +381,5 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)++diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 127d76e1b..d1ad65f97 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -121,32 +121,50 @@ BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, FalBackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],packing_format="int32_zp",bits=[2, 3, 4, 8], group_size=None,- priority=0, feature_checks=[exllamav2_feature_check],+ priority=1, feature_checks=[exllamav2_feature_check],alias=["auto_gptq:cuda"],dtype=["float16"],convertable_format=["int32_zp"],requirements=["auto-gptq>=0.7.1"])-BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda","xpu"], sym=[True, False],+BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda", "xpu"], sym=[True, False],packing_format="int32",dtype=["float16", "bfloat16"],bits=[2, 4, 8],- priority=1, feature_checks=[feature_multiply_checker_32],+ priority=2, feature_checks=[feature_multiply_checker_32],alias=["auto_round", "tritonv2", "triton"],requirements=["triton>=2.0","auto-round>=0.5.0"])-BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda","xpu"], sym=[True], ## asym has accuracy+BackendInfos['auto_round:torch'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True, False],+ packing_format="int32",+ dtype=["float16", "bfloat16"],+ bits=[2, 3, 4, 8],+ priority=0, feature_checks=[feature_multiply_checker_32],+ alias=["auto_round", "torch"],+ requirements=["auto-round>=0.5.1"]+ )++BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda","xpu"], sym=[True], ## asym has accuracys# issuepacking_format="int32_zp",dtype=["float16", "bfloat16"],bits=[2, 4, 8],- priority=1, feature_checks=[feature_multiply_checker_32],+ priority=2, feature_checks=[feature_multiply_checker_32],alias=["tritonv2", "tritonv2_zp", "triton"],requirements=[ "triton>=2.0","auto-round>=0.5.0"])+BackendInfos['auto_round:torch_zp'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True],+ packing_format="int32_zp",+ dtype=["float16", "bfloat16"],+ bits=[2, 3, 4, 8],+ priority=0, feature_checks=[feature_multiply_checker_32],+ alias=["torch", "torch_zp"],+ requirements=["auto-round>=0.5.1"]+ )+BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],packing_format="int32",bits=[4, 8],@@ -189,7 +207,7 @@ BackendInfos['auto_awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False],BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],packing_format="qbits",bits=[2, 4, 8], group_size=None,- priority=0,+ priority=1,feature_checks=[],alias=["itrex", "qbits"],dtype=["float16", "bfloat16"],@@ -200,7 +218,7 @@ BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],packing_format="qbits_zp",bits=[2, 4, 8], group_size=None,dtype=["float16", "bfloat16"],- priority=0,+ priority=1,feature_checks=[],alias=["itrex", "qbits"],convertable_format=["int32_zp"],@@ -211,7 +229,7 @@ BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True, Fapacking_format="awq",bits=[2, 4, 8], group_size=None,dtype=["float16", "bfloat16"],- priority=0,+ priority=1,feature_checks=[],alias=["itrex", "qbits"],requirements=["intel-extension-for-transformers", "torch<2.7.0"]@@ -413,6 +431,14 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):from auto_round_extension.triton.qlinear_tritonv2_zp import QuantLinearreturn QuantLinear+ if backend == "auto_round:torch":+ from auto_round_extension.torch.qlinear_torch import QuantLinear+ return QuantLinear++ if backend == "auto_round:torch_zp":+ from auto_round_extension.torch.qlinear_torch_zp import QuantLinear+ return QuantLinear+raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")@@ -425,6 +451,8 @@ def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):return gptqmodel.nn_modules.qlinear.exllamav2.ExllamaV2QuantLinearelif "tritonv2" in backend:return gptqmodel.nn_modules.qlinear.tritonv2.TritonV2QuantLinear+ elif "torch" in backend:+ return gptqmodel.nn_modules.qlinear.torch.TorchQuantLinearelse:raise ValueError(f"Unsupported {backend}")@@ -737,3 +765,4 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="if logger_level == "error":exit(-1)+diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex d06c55144..a44b1fedd 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -263,6 +263,7 @@ def get_layer_config(model, quantization_config):for layer_name in layer_names:if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers+# Process AWQ format: exclude specified modules from quantizationextra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)@@ -557,3 +558,4 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):process_requirement(requirements, target_device,"warning")return model, used_backends+diff --git a/auto_round_extension/torch/__init__.py b/auto_round_extension/torch/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pynew file mode 100644index 000000000..831a23cfe--- /dev/null+++ b/auto_round_extension/torch/qlinear_torch.py@@ -0,0 +1,292 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math+from logging import getLogger++import numpy as np+import torch+import torch.nn as nn+import transformers+++logger = getLogger(__name__)+++class QuantLinear(nn.Module):+ """+ Torch quantized linear layer.+ """++ QUANT_TYPE = "torch"++ def __init__(+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs+ ):+ super().__init__()+ if bits not in [2, 3, 4, 8]:+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError(+ "in_feature and out_feature must be divisible by 32."+ )+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )+ if bias:+ self.register_buffer(+ "bias", torch.zeros((outfeatures), dtype=torch.float16)+ )+ else:+ self.bias = None++ self.trainable = trainable++ # is performed by unpacking the weights and using torch.matmul+ if self.bits in [2, 4, 8]:+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)+ elif self.bits == 3:+ self.wf = torch.tensor(+ [+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],+ ],+ dtype=torch.int32,+ ).reshape(1, 3, 12)++ self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8+++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"++ W = linear.weight.data.to(device).clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(+ torch.int32)+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(+ torch.int32)++ del repeat_scales++ if self.bits in [2, 4, 8]:+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight.to(torch.int32)+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)++ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")+ elif self.bits == 3:+ intweight = intweight.t().contiguous()+ intweight = intweight.cpu().numpy().astype(np.uint32)+ i = 0+ row = 0+ qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)+ while row < qweight.shape[0]:+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits+ shifted = (packed_weight << shifts)+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 30+ row += 1+ qweight[row] |= (intweight[i] >> 2) & 1+ i += 1+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 31+ row += 1+ qweight[row] |= (intweight[i] >> 1) & 0x3+ i += 1+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits + 2+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ row += 1++ self.qweight = qweight.cpu()++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)+ shifts = torch.arange(0, (32 // self.bits)) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 32 // self.bits+ col += 1+ elif self.bits == 3:+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] = shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 30+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 31+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits + 2+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ col += 1++ self.qzeros = qzeros.cpu()+ else:+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++ def forward(self, x):+ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.reshape(-1, x.shape[-1])+ x_dtype = x.dtype++ if self.bits in [2, 4, 8]:+ if self.wf.device != self.qzeros.device:+ self.wf = self.wf.to(self.qzeros.device)+ zeros = torch.bitwise_right_shift(+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),+ self.wf.unsqueeze(0),+ ).to(self.dequant_dtype)+ zeros = torch.bitwise_and(zeros, self.maxq).reshape(self.scales.shape)++ weight = torch.bitwise_and(+ torch.bitwise_right_shift(+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),+ self.wf.unsqueeze(-1),+ ).to(self.dequant_dtype),+ self.maxq+ )+ elif self.bits == 3:+ if self.wf.device != self.qzeros.device:+ self.wf = self.wf.to(self.qzeros.device)+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(+ -1, -1, -1, 12+ )+ zeros = zeros >> self.wf.unsqueeze(0)+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)+ zeros = zeros & 0x7+ zeros = torch.cat(+ [zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]],+ dim=2,+ )++ zeros = zeros.reshape(self.scales.shape)++ weight = self.qweight.reshape(self.qweight.shape[0] // 3, 3, 1, self.qweight.shape[1]).expand(+ -1, -1, 12, -1+ )+ weight = (weight >> self.wf.unsqueeze(-1)) & 0x7+ weight[:, 0, 10] = (weight[:, 0, 10] & 0x3) | ((weight[:, 1, 0] << 2) & 0x4)+ weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)+ weight = weight & 0x7+ weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)++ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])+ if hasattr(self, "g_idx"):+ num_itr = self.g_idx.shape[0] // x.shape[-1]+ num_dim = self.g_idx.shape[0] // num_itr+ weights = []+ for i in range(num_itr):+ scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]+ weight_i = weight[:, i * num_dim : (i + 1) * num_dim]+ zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]+ g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))+ weights = torch.cat(weights, dim=1)+ else:+ repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)+ repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)+ weights = repeat_scales * (weight - repeat_zeros)++ out = torch.matmul(x, weights)+ out = out.to(x_dtype)+ out = out.reshape(out_shape)+ out = out + self.bias if self.bias is not None else out+ return out+++__all__ = ["QuantLinear"]diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pynew file mode 100644index 000000000..0d4461d80--- /dev/null+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -0,0 +1,285 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++import math+from logging import getLogger++import numpy as np+import torch+import torch.nn as nn+import transformers+++logger = getLogger(__name__)+++class QuantLinear(nn.Module):+ """+ Torch quantized linear layer.+ """++ QUANT_TYPE = "torch"++ def __init__(+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs+ ):+ super().__init__()+ if bits not in [2, 3, 4, 8]:+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError(+ "in_feature and out_feature must be divisible by 32."+ )+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )+ if bias:+ self.register_buffer(+ "bias", torch.zeros((outfeatures), dtype=torch.float16)+ )+ else:+ self.bias = None++ self.trainable = trainable++ # is performed by unpacking the weights and using torch.matmul+ if self.bits in [2, 4, 8]:+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)+ elif self.bits == 3:+ self.wf = torch.tensor(+ [+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],+ ],+ dtype=torch.int32,+ ).reshape(1, 3, 12)++ self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8+++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"++ W = linear.weight.data.to(device).clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(+ torch.int32)+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(+ torch.int32)++ del repeat_scales++ if self.bits in [2, 4, 8]:+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight.to(torch.int32)+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)++ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")+ elif self.bits == 3:+ intweight = intweight.t().contiguous()+ intweight = intweight.cpu().numpy().astype(np.uint32)+ i = 0+ row = 0+ qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)+ while row < qweight.shape[0]:+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits+ shifted = (packed_weight << shifts)+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 30+ row += 1+ qweight[row] |= (intweight[i] >> 2) & 1+ i += 1+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 31+ row += 1+ qweight[row] |= (intweight[i] >> 1) & 0x3+ i += 1+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10) * self.bits + 2+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ row += 1++ self.qweight = qweight.cpu()++ zeros = zeros.t().contiguous()+ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)+ shifts = torch.arange(0, (32 // self.bits)) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 32 // self.bits+ col += 1+ elif self.bits == 3:+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] = shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 30+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 31+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10) * self.bits + 2+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ col += 1+ self.qzeros = qzeros.cpu()+++ def forward(self, x):+ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.reshape(-1, x.shape[-1])+ x_dtype = x.dtype++ if self.bits in [2, 4, 8]:+ if self.wf.device != self.qzeros.device:+ self.wf = self.wf.to(self.qzeros.device)+ zeros = torch.bitwise_right_shift(+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),+ self.wf.unsqueeze(0),+ ).to(self.dequant_dtype)+ zeros = torch.bitwise_and(zeros, self.maxq).reshape(self.scales.shape)++ weight = torch.bitwise_and(+ torch.bitwise_right_shift(+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),+ self.wf.unsqueeze(-1),+ ).to(self.dequant_dtype),+ self.maxq+ )+ elif self.bits == 3:+ if self.wf.device != self.qzeros.device:+ self.wf = self.wf.to(self.qzeros.device)+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(+ -1, -1, -1, 12+ )+ zeros = zeros >> self.wf.unsqueeze(0)+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)+ zeros = zeros & 0x7+ zeros = torch.cat(+ [zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]],+ dim=2,+ )++ zeros = zeros.reshape(self.scales.shape)++ weight = self.qweight.reshape(self.qweight.shape[0] // 3, 3, 1, self.qweight.shape[1]).expand(+ -1, -1, 12, -1+ )+ weight = (weight >> self.wf.unsqueeze(-1)) & 0x7+ weight[:, 0, 10] = (weight[:, 0, 10] & 0x3) | ((weight[:, 1, 0] << 2) & 0x4)+ weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)+ weight = weight & 0x7+ weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)++ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])+ if hasattr(self, "g_idx"):+ num_itr = self.g_idx.shape[0] // x.shape[-1]+ num_dim = self.g_idx.shape[0] // num_itr+ weights = []+ for i in range(num_itr):+ scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]+ weight_i = weight[:, i * num_dim : (i + 1) * num_dim]+ zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]+ g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))+ weights = torch.cat(weights, dim=1)+ else:+ repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)+ repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)+ weights = repeat_scales * (weight - repeat_zeros)++ out = torch.matmul(x, weights)+ out = out.to(x_dtype)+ out = out.reshape(out_shape)+ out = out + self.bias if self.bias is not None else out+ return out+++__all__ = ["QuantLinear"]diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex bc5d36595..725b4decb 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -141,8 +141,6 @@ autoround.quantize_and_save(output_dir, format='auto_gptq,auto_awq,auto_round')#### Mixed bits UsageAuto-GPTQ and Auto-AWQ only support a limited set of mixed-bit configurations. If you're unsure about the details, we recommend using the AutoRound format.-Also, avoid setting mixed bits to 3 for asymmetric quantization at this time, as models exported with this setting may not be compatible with future versions of the AutoRound format.-vLLM and SGLang fuse MoE and QKV layers, so it's recommended not to assign different bit widths to these layers.```python@@ -491,6 +489,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |### Convert GPTQ/AWQ to AutoRound@@ -556,3 +555,4 @@ If not explicitly specify '--task', the default value will be used (typically co* Random quantization results in tuning some models* ChatGlm-V1 is not supported+diff --git a/test/test_export.py b/test/test_export.pyindex 75c75c024..c1eb40268 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -22,13 +22,14 @@ class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):model_name = "facebook/opt-125m"+ self.save_dir = "./saved"self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@classmethoddef tearDownClass(self):- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(self.save_dir, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)def test_autogptq_format(self):@@ -46,7 +47,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_dirautoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)@@ -54,7 +55,7 @@ class TestAutoRound(unittest.TestCase):text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_autoround_format(self):bits, group_size, sym = 4, 128, True@@ -69,7 +70,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_dirautoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")@@ -84,7 +85,7 @@ class TestAutoRound(unittest.TestCase):text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_autoround_awq_format(self):bits, group_size, sym = 4, 128, False@@ -99,7 +100,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_dirautoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")@@ -112,7 +113,7 @@ class TestAutoRound(unittest.TestCase):text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)def test_autoawq_format(self):bits, group_size, sym = 4, 128, False@@ -127,7 +128,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_dirautoround.save_quantized(output_dir=quantized_model_path, inplace=False, \format="auto_awq", model_path="facebook/opt-125m")@@ -140,36 +141,59 @@ class TestAutoRound(unittest.TestCase):text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree("./saved", ignore_errors=True)-- # def test_autoround_marlin_format(self):- # if not torch.cuda.is_available():- # return- # try:- # import auto_gptq- # except:- # return- # bits, group_size, sym = 4, 128, True- # autoround = AutoRound(- # self.model,- # self.tokenizer,- # bits=bits,- # group_size=group_size,- # sym=sym,- # iters=2,- # seqlen=2,- # dataset=self.llm_dataloader,- # )- # autoround.quantize()- # quantized_model_path = "./saved"- #- # autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:marlin")- #- # from auto_round.auto_quantizer import AutoHfQuantizer- # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- # text = "There is a girl who likes adventure,"- # inputs = tokenizer(text, return_tensors="pt").to(model.device)- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- # shutil.rmtree("./saved", ignore_errors=True)- #+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++ def test_autoround_3bit_asym_format(self):+ bits, group_size, sym = 3, 128, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")+ device = "cpu" ##cpu, hpu, cuda+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++ def test_autoround_3bit_sym_format(self):+ bits, group_size, sym = 3, 128, True+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")+ device = "auto" ##cpu, hpu, cuda+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()+diff --git a/test_cuda/test_2_3bits.py b/test_cuda/test_2_3bits.pyindex c595bc5fc..2b31a6050 100644--- a/test_cuda/test_2_3bits.py+++ b/test_cuda/test_2_3bits.py@@ -54,6 +54,25 @@ class TestAutoRound(unittest.TestCase):accuracy = res['results']['lambada_openai']['acc,none']assert accuracy > 0.3shutil.rmtree("./saved", ignore_errors=True)++ def test_3bits_asym_autoround(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ bits, sym = 3, False+ autoround = AutoRound(model, tokenizer, bits=bits, sym=sym)+ autoround.quantize()+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ # tasks="arc_easy",+ tasks=self.tasks,+ batch_size="auto")++ ## 0.3423+ accuracy = res['results']['lambada_openai']['acc,none']+ assert accuracy > 0.32+ shutil.rmtree("./saved", ignore_errors=True)def test_norm_bias_tuning(self):model_name = "/models/opt-125m"@@ -103,4 +122,4 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test_cuda/test_export.py b/test_cuda/test_export.pyindex db68fa0dd..3663b538e 100644--- a/test_cuda/test_export.py+++ b/test_cuda/test_export.py@@ -174,7 +174,6 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)- #def test_autoawq_format(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -223,7 +222,7 @@ class TestAutoRound(unittest.TestCase):layer_config=layer_config)quantized_model_path = "/data5/wenhuach/test_export"- autoround.qsave(output_dir=quantized_model_path,+ autoround.quantize_and_save(output_dir=quantized_model_path,format="auto_awq")from auto_round import AutoRoundConfigmodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",@@ -242,3 +241,71 @@ class TestAutoRound(unittest.TestCase):print(res)shutil.rmtree("./saved", ignore_errors=True)+++ def test_autoround_3bit_asym_torch_format(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 3, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:gptqmodel")+++ device = "auto" ##cpu, hpu, cuda+ from auto_round import AutoRoundConfig+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+++ def test_autoround_3bit_sym_torch_format(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 3, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")+++ device = "auto" ##cpu, hpu, cuda+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(+ backend=device+ )+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()