Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add torch backend (#555)

Weiweice8eedd763d1
diff --git a/README.md b/README.md
index 447d3bdf5..54207c37f 100644
--- a/README.md
+++ b/README.md
@@ -356,6 +356,7 @@ to the following table for the details and specify the backend you want.
| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -513,3 +514,5 @@ If you find AutoRound useful for your research, please cite our paper:
year={2023}
}
```
+
+
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index de6a5dbdf..31361eb9e 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -539,7 +539,7 @@ class AutoRound(object):
"The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
" particularly for 2-bit quantization and smaller models."
" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
- "the AutoRound format(2/4/8 bits)."
+ "the AutoRound format(2/3/4/8 bits)."
)
save_format_ = format.replace(":", "-").replace("_", "-")
save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir
@@ -1665,7 +1665,7 @@ class AutoRound(object):
)
format = "auto_round"
- if re.search("q\d_k", format) and not self.data_type.endswith("_dq"):
+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
logger.error(
f"datatype<{self.data_type}> not support to export {format} format."
" Please change export format or data_type."
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 829314f0a..690533398 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -139,7 +139,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
if common_prefix not in BLOCK_PATTERNS:
logger.error(f"auto-gptq format may not support loading this quantized model")
quantization_config['block_name_to_quantize'] = common_prefix
- quantization_config.pop("to_quant_block_names", None)
+ quantization_config.pop("to_quant_block_names", None)
## as layers maybe already packed, we need to check in layer_config
layer_config = kwargs["layer_config"]
@@ -238,3 +238,5 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 476a91fe8..42fc63e6d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -20,10 +20,12 @@ import os
import torch
import torch.nn as nn
import transformers
-
+try:
+ import auto_round_extension.triton.qlinear_tritonv2
+except Exception as error:
+ raise ImportError(error)
+import auto_round_extension.torch.qlinear_torch
import auto_round.export.export_to_autoround.qlinear_triton_act
-
-import auto_round_extension.triton.qlinear_tritonv2
from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \
filter_quantization_config
import threadpoolctl as tctl
@@ -74,17 +76,17 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:
if act_bits <= 8: ##easily have bug for other configuration, need to refine code later
return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear
-
- from auto_round_extension.triton.qlinear_tritonv2 import QuantLinear
+ from auto_round_extension.torch.qlinear_torch import QuantLinear
return QuantLinear
- elif "auto_round" in backend and "gptq" in backend and bits in (2, 4, 8):
- from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx
+ elif "auto_round" in backend and "gptq" in backend:
+ from auto_round_extension.torch.qlinear_torch_zp import QuantLinear
return QuantLinear
elif "awq" in backend:
from ..export_to_awq.utils import WQLinear_GEMM
return WQLinear_GEMM
elif "gptqmodel" in backend:
- return auto_round_extension.triton.qlinear_tritonv2.QuantLinear
+ from auto_round_extension.torch.qlinear_torch import QuantLinear
+ return QuantLinear
elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
@@ -190,14 +192,15 @@ def pack_layer(layer_name, model, backend):
if "awq" not in backend:
new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
+ bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype
)
new_layer.device = device
set_module(model, layer_name, new_layer)
qlayer = new_layer
import auto_round.export.export_to_autoround.qlinear_triton
if sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,
- auto_round_extension.triton.qlinear_tritonv2.QuantLinear)):
+ auto_round_extension.triton.qlinear_tritonv2.QuantLinear,
+ auto_round_extension.torch.qlinear_torch.QuantLinear)):
zp = int(zp.flatten()[0])
qlayer.to("cpu")
@@ -268,8 +271,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config = kwargs["serialization_dict"]
quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
quantization_config["quant_method"] = "auto-round"
- if quantization_config["bits"] == 3:
- backend = "auto_round:auto_gptq"
quantization_config["packing_format"] = backend
tokenizer = kwargs.get("tokenizer", None)
@@ -380,3 +381,5 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
+
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 127d76e1b..d1ad65f97 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -121,32 +121,50 @@ BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, Fal
BackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],
packing_format="int32_zp",
bits=[2, 3, 4, 8], group_size=None,
- priority=0, feature_checks=[exllamav2_feature_check],
+ priority=1, feature_checks=[exllamav2_feature_check],
alias=["auto_gptq:cuda"],
dtype=["float16"],
convertable_format=["int32_zp"],
requirements=["auto-gptq>=0.7.1"]
)
-BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda","xpu"], sym=[True, False],
+BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda", "xpu"], sym=[True, False],
packing_format="int32",
dtype=["float16", "bfloat16"],
bits=[2, 4, 8],
- priority=1, feature_checks=[feature_multiply_checker_32],
+ priority=2, feature_checks=[feature_multiply_checker_32],
alias=["auto_round", "tritonv2", "triton"],
requirements=["triton>=2.0","auto-round>=0.5.0"]
)
-BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda","xpu"], sym=[True], ## asym has accuracy
+BackendInfos['auto_round:torch'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True, False],
+ packing_format="int32",
+ dtype=["float16", "bfloat16"],
+ bits=[2, 3, 4, 8],
+ priority=0, feature_checks=[feature_multiply_checker_32],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>=0.5.1"]
+ )
+
+BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda","xpu"], sym=[True], ## asym has accuracys
# issue
packing_format="int32_zp",
dtype=["float16", "bfloat16"],
bits=[2, 4, 8],
- priority=1, feature_checks=[feature_multiply_checker_32],
+ priority=2, feature_checks=[feature_multiply_checker_32],
alias=["tritonv2", "tritonv2_zp", "triton"],
requirements=[ "triton>=2.0","auto-round>=0.5.0"]
)
+BackendInfos['auto_round:torch_zp'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True],
+ packing_format="int32_zp",
+ dtype=["float16", "bfloat16"],
+ bits=[2, 3, 4, 8],
+ priority=0, feature_checks=[feature_multiply_checker_32],
+ alias=["torch", "torch_zp"],
+ requirements=["auto-round>=0.5.1"]
+ )
+
BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],
packing_format="int32",
bits=[4, 8],
@@ -189,7 +207,7 @@ BackendInfos['auto_awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False],
BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],
packing_format="qbits",
bits=[2, 4, 8], group_size=None,
- priority=0,
+ priority=1,
feature_checks=[],
alias=["itrex", "qbits"],
dtype=["float16", "bfloat16"],
@@ -200,7 +218,7 @@ BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
packing_format="qbits_zp",
bits=[2, 4, 8], group_size=None,
dtype=["float16", "bfloat16"],
- priority=0,
+ priority=1,
feature_checks=[],
alias=["itrex", "qbits"],
convertable_format=["int32_zp"],
@@ -211,7 +229,7 @@ BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True, Fa
packing_format="awq",
bits=[2, 4, 8], group_size=None,
dtype=["float16", "bfloat16"],
- priority=0,
+ priority=1,
feature_checks=[],
alias=["itrex", "qbits"],
requirements=["intel-extension-for-transformers", "torch<2.7.0"]
@@ -413,6 +431,14 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
from auto_round_extension.triton.qlinear_tritonv2_zp import QuantLinear
return QuantLinear
+ if backend == "auto_round:torch":
+ from auto_round_extension.torch.qlinear_torch import QuantLinear
+ return QuantLinear
+
+ if backend == "auto_round:torch_zp":
+ from auto_round_extension.torch.qlinear_torch_zp import QuantLinear
+ return QuantLinear
+
raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")
@@ -425,6 +451,8 @@ def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):
return gptqmodel.nn_modules.qlinear.exllamav2.ExllamaV2QuantLinear
elif "tritonv2" in backend:
return gptqmodel.nn_modules.qlinear.tritonv2.TritonV2QuantLinear
+ elif "torch" in backend:
+ return gptqmodel.nn_modules.qlinear.torch.TorchQuantLinear
else:
raise ValueError(f"Unsupported {backend}")
@@ -737,3 +765,4 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
if logger_level == "error":
exit(-1)
+
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index d06c55144..a44b1fedd 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -263,6 +263,7 @@ def get_layer_config(model, quantization_config):
for layer_name in layer_names:
if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):
extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers
+
# Process AWQ format: exclude specified modules from quantization
extra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)
@@ -557,3 +558,4 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
process_requirement(requirements, target_device,"warning")
return model, used_backends
+
diff --git a/auto_round_extension/torch/__init__.py b/auto_round_extension/torch/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
new file mode 100644
index 000000000..831a23cfe
--- /dev/null
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -0,0 +1,292 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from logging import getLogger
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+
+logger = getLogger(__name__)
+
+
+class QuantLinear(nn.Module):
+ """
+ Torch quantized linear layer.
+ """
+
+ QUANT_TYPE = "torch"
+
+ def __init__(
+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs
+ ):
+ super().__init__()
+ if bits not in [2, 3, 4, 8]:
+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError(
+ "in_feature and out_feature must be divisible by 32."
+ )
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+ if bias:
+ self.register_buffer(
+ "bias", torch.zeros((outfeatures), dtype=torch.float16)
+ )
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ # is performed by unpacking the weights and using torch.matmul
+ if self.bits in [2, 4, 8]:
+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
+ elif self.bits == 3:
+ self.wf = torch.tensor(
+ [
+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],
+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],
+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],
+ ],
+ dtype=torch.int32,
+ ).reshape(1, 3, 12)
+
+ self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
+
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+
+ W = linear.weight.data.to(device).clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
+ torch.int32)
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
+ torch.int32)
+
+ del repeat_scales
+
+ if self.bits in [2, 4, 8]:
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight.to(torch.int32)
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
+ elif self.bits == 3:
+ intweight = intweight.t().contiguous()
+ intweight = intweight.cpu().numpy().astype(np.uint32)
+ i = 0
+ row = 0
+ qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)
+ while row < qweight.shape[0]:
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits
+ shifted = (packed_weight << shifts)
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 30
+ row += 1
+ qweight[row] |= (intweight[i] >> 2) & 1
+ i += 1
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits + 1
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 31
+ row += 1
+ qweight[row] |= (intweight[i] >> 1) & 0x3
+ i += 1
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits + 2
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ row += 1
+
+ self.qweight = qweight.cpu()
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous()
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)
+ shifts = torch.arange(0, (32 // self.bits)) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 32 // self.bits
+ col += 1
+ elif self.bits == 3:
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] = shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 30
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits + 1
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 31
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits + 2
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ col += 1
+
+ self.qzeros = qzeros.cpu()
+ else:
+ shape = scales_t.shape
+ value = 0
+ for j in range(0, (32 // self.bits)):
+ value |= zeros << (self.bits * j)
+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+ def forward(self, x):
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ x = x.reshape(-1, x.shape[-1])
+ x_dtype = x.dtype
+
+ if self.bits in [2, 4, 8]:
+ if self.wf.device != self.qzeros.device:
+ self.wf = self.wf.to(self.qzeros.device)
+ zeros = torch.bitwise_right_shift(
+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
+ self.wf.unsqueeze(0),
+ ).to(self.dequant_dtype)
+ zeros = torch.bitwise_and(zeros, self.maxq).reshape(self.scales.shape)
+
+ weight = torch.bitwise_and(
+ torch.bitwise_right_shift(
+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),
+ self.wf.unsqueeze(-1),
+ ).to(self.dequant_dtype),
+ self.maxq
+ )
+ elif self.bits == 3:
+ if self.wf.device != self.qzeros.device:
+ self.wf = self.wf.to(self.qzeros.device)
+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(
+ -1, -1, -1, 12
+ )
+ zeros = zeros >> self.wf.unsqueeze(0)
+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)
+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)
+ zeros = zeros & 0x7
+ zeros = torch.cat(
+ [zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]],
+ dim=2,
+ )
+
+ zeros = zeros.reshape(self.scales.shape)
+
+ weight = self.qweight.reshape(self.qweight.shape[0] // 3, 3, 1, self.qweight.shape[1]).expand(
+ -1, -1, 12, -1
+ )
+ weight = (weight >> self.wf.unsqueeze(-1)) & 0x7
+ weight[:, 0, 10] = (weight[:, 0, 10] & 0x3) | ((weight[:, 1, 0] << 2) & 0x4)
+ weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)
+ weight = weight & 0x7
+ weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)
+
+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
+ if hasattr(self, "g_idx"):
+ num_itr = self.g_idx.shape[0] // x.shape[-1]
+ num_dim = self.g_idx.shape[0] // num_itr
+ weights = []
+ for i in range(num_itr):
+ scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]
+ weight_i = weight[:, i * num_dim : (i + 1) * num_dim]
+ zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]
+ g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]
+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))
+ weights = torch.cat(weights, dim=1)
+ else:
+ repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)
+ repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)
+ weights = repeat_scales * (weight - repeat_zeros)
+
+ out = torch.matmul(x, weights)
+ out = out.to(x_dtype)
+ out = out.reshape(out_shape)
+ out = out + self.bias if self.bias is not None else out
+ return out
+
+
+__all__ = ["QuantLinear"]
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
new file mode 100644
index 000000000..0d4461d80
--- /dev/null
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -0,0 +1,285 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import math
+from logging import getLogger
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+
+logger = getLogger(__name__)
+
+
+class QuantLinear(nn.Module):
+ """
+ Torch quantized linear layer.
+ """
+
+ QUANT_TYPE = "torch"
+
+ def __init__(
+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs
+ ):
+ super().__init__()
+ if bits not in [2, 3, 4, 8]:
+ raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError(
+ "in_feature and out_feature must be divisible by 32."
+ )
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+ if bias:
+ self.register_buffer(
+ "bias", torch.zeros((outfeatures), dtype=torch.float16)
+ )
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ # is performed by unpacking the weights and using torch.matmul
+ if self.bits in [2, 4, 8]:
+ self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
+ elif self.bits == 3:
+ self.wf = torch.tensor(
+ [
+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],
+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],
+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],
+ ],
+ dtype=torch.int32,
+ ).reshape(1, 3, 12)
+
+ self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
+
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+
+ W = linear.weight.data.to(device).clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
+ torch.int32)
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
+ torch.int32)
+
+ del repeat_scales
+
+ if self.bits in [2, 4, 8]:
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight.to(torch.int32)
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
+ elif self.bits == 3:
+ intweight = intweight.t().contiguous()
+ intweight = intweight.cpu().numpy().astype(np.uint32)
+ i = 0
+ row = 0
+ qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)
+ while row < qweight.shape[0]:
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits
+ shifted = (packed_weight << shifts)
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 30
+ row += 1
+ qweight[row] |= (intweight[i] >> 2) & 1
+ i += 1
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits + 1
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 31
+ row += 1
+ qweight[row] |= (intweight[i] >> 1) & 0x3
+ i += 1
+ packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10) * self.bits + 2
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ row += 1
+
+ self.qweight = qweight.cpu()
+
+ zeros = zeros.t().contiguous()
+ zeros -= 1
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)
+ shifts = torch.arange(0, (32 // self.bits)) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 32 // self.bits
+ col += 1
+ elif self.bits == 3:
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] = shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 30
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits + 1
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 31
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10) * self.bits + 2
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ col += 1
+ self.qzeros = qzeros.cpu()
+
+
+ def forward(self, x):
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ x = x.reshape(-1, x.shape[-1])
+ x_dtype = x.dtype
+
+ if self.bits in [2, 4, 8]:
+ if self.wf.device != self.qzeros.device:
+ self.wf = self.wf.to(self.qzeros.device)
+ zeros = torch.bitwise_right_shift(
+ torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
+ self.wf.unsqueeze(0),
+ ).to(self.dequant_dtype)
+ zeros = torch.bitwise_and(zeros, self.maxq).reshape(self.scales.shape)
+
+ weight = torch.bitwise_and(
+ torch.bitwise_right_shift(
+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),
+ self.wf.unsqueeze(-1),
+ ).to(self.dequant_dtype),
+ self.maxq
+ )
+ elif self.bits == 3:
+ if self.wf.device != self.qzeros.device:
+ self.wf = self.wf.to(self.qzeros.device)
+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(
+ -1, -1, -1, 12
+ )
+ zeros = zeros >> self.wf.unsqueeze(0)
+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)
+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)
+ zeros = zeros & 0x7
+ zeros = torch.cat(
+ [zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]],
+ dim=2,
+ )
+
+ zeros = zeros.reshape(self.scales.shape)
+
+ weight = self.qweight.reshape(self.qweight.shape[0] // 3, 3, 1, self.qweight.shape[1]).expand(
+ -1, -1, 12, -1
+ )
+ weight = (weight >> self.wf.unsqueeze(-1)) & 0x7
+ weight[:, 0, 10] = (weight[:, 0, 10] & 0x3) | ((weight[:, 1, 0] << 2) & 0x4)
+ weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)
+ weight = weight & 0x7
+ weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)
+
+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
+ if hasattr(self, "g_idx"):
+ num_itr = self.g_idx.shape[0] // x.shape[-1]
+ num_dim = self.g_idx.shape[0] // num_itr
+ weights = []
+ for i in range(num_itr):
+ scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]
+ weight_i = weight[:, i * num_dim : (i + 1) * num_dim]
+ zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]
+ g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]
+ weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))
+ weights = torch.cat(weights, dim=1)
+ else:
+ repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)
+ repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)
+ weights = repeat_scales * (weight - repeat_zeros)
+
+ out = torch.matmul(x, weights)
+ out = out.to(x_dtype)
+ out = out.reshape(out_shape)
+ out = out + self.bias if self.bias is not None else out
+ return out
+
+
+__all__ = ["QuantLinear"]
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index bc5d36595..725b4decb 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -141,8 +141,6 @@ autoround.quantize_and_save(output_dir, format='auto_gptq,auto_awq,auto_round')
#### Mixed bits Usage
Auto-GPTQ and Auto-AWQ only support a limited set of mixed-bit configurations. If you're unsure about the details, we recommend using the AutoRound format.
-Also, avoid setting mixed bits to 3 for asymmetric quantization at this time, as models exported with this setting may not be compatible with future versions of the AutoRound format.
-
vLLM and SGLang fuse MoE and QKV layers, so it's recommended not to assign different bit widths to these layers.
```python
@@ -491,6 +489,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
### Convert GPTQ/AWQ to AutoRound
@@ -556,3 +555,4 @@ If not explicitly specify '--task', the default value will be used (typically co
* Random quantization results in tuning some models
* ChatGlm-V1 is not supported
+
diff --git a/test/test_export.py b/test/test_export.py
index 75c75c024..c1eb40268 100644
--- a/test/test_export.py
+++ b/test/test_export.py
@@ -22,13 +22,14 @@ class TestAutoRound(unittest.TestCase):
@classmethod
def setUpClass(self):
model_name = "facebook/opt-125m"
+ self.save_dir = "./saved"
self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
@classmethod
def tearDownClass(self):
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(self.save_dir, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
def test_autogptq_format(self):
@@ -46,7 +47,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
@@ -54,7 +55,7 @@ class TestAutoRound(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
def test_autoround_format(self):
bits, group_size, sym = 4, 128, True
@@ -69,7 +70,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
@@ -84,7 +85,7 @@ class TestAutoRound(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
def test_autoround_awq_format(self):
bits, group_size, sym = 4, 128, False
@@ -99,7 +100,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")
@@ -112,7 +113,7 @@ class TestAutoRound(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
def test_autoawq_format(self):
bits, group_size, sym = 4, 128, False
@@ -127,7 +128,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \
format="auto_awq", model_path="facebook/opt-125m")
@@ -140,36 +141,59 @@ class TestAutoRound(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
-
- # def test_autoround_marlin_format(self):
- # if not torch.cuda.is_available():
- # return
- # try:
- # import auto_gptq
- # except:
- # return
- # bits, group_size, sym = 4, 128, True
- # autoround = AutoRound(
- # self.model,
- # self.tokenizer,
- # bits=bits,
- # group_size=group_size,
- # sym=sym,
- # iters=2,
- # seqlen=2,
- # dataset=self.llm_dataloader,
- # )
- # autoround.quantize()
- # quantized_model_path = "./saved"
- #
- # autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:marlin")
- #
- # from auto_round.auto_quantizer import AutoHfQuantizer
- # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- # text = "There is a girl who likes adventure,"
- # inputs = tokenizer(text, return_tensors="pt").to(model.device)
- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- # shutil.rmtree("./saved", ignore_errors=True)
- #
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+ def test_autoround_3bit_asym_format(self):
+ bits, group_size, sym = 3, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_dir
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
+ device = "cpu" ##cpu, hpu, cuda
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+ def test_autoround_3bit_sym_format(self):
+ bits, group_size, sym = 3, 128, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_dir
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
+ device = "auto" ##cpu, hpu, cuda
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
+
diff --git a/test_cuda/test_2_3bits.py b/test_cuda/test_2_3bits.py
index c595bc5fc..2b31a6050 100644
--- a/test_cuda/test_2_3bits.py
+++ b/test_cuda/test_2_3bits.py
@@ -54,6 +54,25 @@ class TestAutoRound(unittest.TestCase):
accuracy = res['results']['lambada_openai']['acc,none']
assert accuracy > 0.3
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_3bits_asym_autoround(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ bits, sym = 3, False
+ autoround = AutoRound(model, tokenizer, bits=bits, sym=sym)
+ autoround.quantize()
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ # tasks="arc_easy",
+ tasks=self.tasks,
+ batch_size="auto")
+
+ ## 0.3423
+ accuracy = res['results']['lambada_openai']['acc,none']
+ assert accuracy > 0.32
+ shutil.rmtree("./saved", ignore_errors=True)
def test_norm_bias_tuning(self):
model_name = "/models/opt-125m"
@@ -103,4 +122,4 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
if __name__ == "__main__":
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/test_cuda/test_export.py b/test_cuda/test_export.py
index db68fa0dd..3663b538e 100644
--- a/test_cuda/test_export.py
+++ b/test_cuda/test_export.py
@@ -174,7 +174,6 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
- #
def test_autoawq_format(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@@ -223,7 +222,7 @@ class TestAutoRound(unittest.TestCase):
layer_config=layer_config
)
quantized_model_path = "/data5/wenhuach/test_export"
- autoround.qsave(output_dir=quantized_model_path,
+ autoround.quantize_and_save(output_dir=quantized_model_path,
format="auto_awq")
from auto_round import AutoRoundConfig
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
@@ -242,3 +241,71 @@ class TestAutoRound(unittest.TestCase):
print(res)
shutil.rmtree("./saved", ignore_errors=True)
+
+
+ def test_autoround_3bit_asym_torch_format(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 3, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:gptqmodel")
+
+
+ device = "auto" ##cpu, hpu, cuda
+ from auto_round import AutoRoundConfig
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+ def test_autoround_3bit_sym_torch_format(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 3, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
+
+
+ device = "auto" ##cpu, hpu, cuda
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig(
+ backend=device
+ )
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()