Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix inference issue in ARK for AWQ format (#1170)

Luo Yufd5a17295017
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 43de447f1..057ecf5bd 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -177,6 +177,8 @@ gptqmodel_marlin_feature_checker = functools.partial(
mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)
+ark_feature_checker = functools.partial(in_feature_checker_group_size)
+
def fp8_static_scheme_checker(
in_feature: int,
@@ -438,13 +440,13 @@ BackendInfos["auto_awq:gemm"] = BackendInfo(
)
BackendInfos["auto_round_kernel"] = BackendInfo(
- device=["cpu", "xpu"],
- sym=[True],
+ device=["cpu"],
+ sym=[True, False],
packing_format=GPTQ_FORMAT_NO_ZP,
bits=[2, 4, 8],
group_size=None,
priority=6,
- checkers=[],
+ checkers=[ark_feature_checker],
alias=["ark"],
compute_dtype=["float32", "float16"],
data_type=["int"],
@@ -453,14 +455,46 @@ BackendInfos["auto_round_kernel"] = BackendInfo(
systems=["linux"],
)
-BackendInfos["auto_round_kernel_zp"] = BackendInfo(
- device=["cpu", "xpu"],
+BackendInfos["auto_round_kernel_xpu"] = BackendInfo(
+ device=["xpu"],
sym=[True],
+ packing_format=GPTQ_FORMAT_NO_ZP,
+ bits=[4, 8],
+ group_size=None,
+ priority=6,
+ checkers=[ark_feature_checker],
+ alias=["ark"],
+ compute_dtype=["float32", "float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ requirements=["torch>=2.9.0", "auto_round_kernel"],
+ systems=["linux"],
+)
+
+BackendInfos["auto_round_kernel_zp"] = BackendInfo(
+ device=["cpu"],
+ sym=[True, False],
packing_format=GPTQ_FORMAT,
bits=[2, 4, 8],
group_size=None,
priority=6,
- checkers=[],
+ checkers=[ark_feature_checker],
+ alias=["ark"],
+ compute_dtype=["float32", "float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ requirements=["torch>=2.9.0", "auto_round_kernel"],
+ systems=["linux"],
+)
+
+BackendInfos["auto_round_kernel_zp_xpu"] = BackendInfo(
+ device=["xpu"],
+ sym=[True],
+ packing_format=GPTQ_FORMAT,
+ bits=[4, 8],
+ group_size=None,
+ priority=6,
+ checkers=[ark_feature_checker],
alias=["ark"],
compute_dtype=["float32", "float16"],
data_type=["int"],
@@ -476,7 +510,23 @@ BackendInfos["auto_round_kernel_awq"] = BackendInfo(
bits=[2, 4, 8],
group_size=None,
priority=6,
- checkers=[],
+ checkers=[ark_feature_checker],
+ alias=["ark"],
+ compute_dtype=["float32", "float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ requirements=["torch>=2.9.0", "auto_round_kernel"],
+ systems=["linux"],
+)
+
+BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(
+ device=["xpu"],
+ sym=[True],
+ packing_format=AWQ_FORMAT,
+ bits=[4, 8],
+ group_size=None,
+ priority=6,
+ checkers=[ark_feature_checker],
alias=["ark"],
compute_dtype=["float32", "float16"],
data_type=["int"],
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index f4cd1c527..49aa42976 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -397,6 +397,16 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)
# Special handling for AWQ layers
from auto_round_extension.ark.qlinear import QuantLinearAWQ
+ if "auto_round_kernel" in layer_backend:
+ return QuantLinear(
+ bits=config["bits"],
+ group_size=config["group_size"],
+ sym=config["sym"],
+ in_features=in_features,
+ out_features=out_features,
+ bias=bias,
+ weight_dtype=layer.weight.dtype,
+ )
if "awq" in layer_backend and isinstance(QuantLinear, QuantLinearAWQ):
return QuantLinear.from_linear(
layer, config["bits"], config["group_size"], init_only=True, has_zero_points=not config["sym"]
diff --git a/auto_round_extension/ark/qlinear.py b/auto_round_extension/ark/qlinear.py
index 2fd3ee9d6..9e28b7347 100644
--- a/auto_round_extension/ark/qlinear.py
+++ b/auto_round_extension/ark/qlinear.py
@@ -17,6 +17,7 @@ import math
import torch
import torch.nn as nn
+from auto_round.export.export_to_awq.utils import dequantize_gemm, reverse_awq_order, unpack_awq
from auto_round.utils import convert_dtype_torch2str, logger
try:
@@ -32,176 +33,6 @@ BITS_DTYPE_MAPPING = {
8: "int8",
}
-AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
-
-
-def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
- shifts = torch.arange(0, 32, bits, device="cpu")
-
- # unpacking columnwise
- iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
- torch.int8 # smallest dtype available
- )
- iweights = iweights.view(iweights.shape[0], -1)
-
- # unpacking columnwise
- if qzeros is not None:
- izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
- torch.int8 # smallest dtype available
- )
- izeros = izeros.view(izeros.shape[0], -1)
- else:
- izeros = qzeros
-
- return iweights, izeros
-
-
-def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):
- reverse_order_tensor = torch.arange(
- iweights.shape[-1],
- dtype=torch.int32,
- device="cpu",
- )
- reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
- reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
- reverse_order_tensor = reverse_order_tensor.view(-1)
-
- if izeros is not None:
- izeros = izeros[:, reverse_order_tensor]
- iweights = iweights[:, reverse_order_tensor]
- return iweights, izeros
-
-
-class QuantLinearAWQ(nn.Module):
- QUANT_TYPE = "ark_awq"
-
- def __init__(self, w_bit, group_size, in_features, out_features, bias, zero_point, dev):
- super().__init__()
- assert ARK_INSTALLED, "Please install auto_round_kernel package."
-
- self.use_bf16 = ark.check_isa_supported("AMX")
-
- if w_bit not in [2, 3, 4, 8]:
- raise NotImplementedError("Only 2, 3, 4, 8 bits are supported for now.")
-
- self.in_features = in_features
- self.out_features = out_features
- self.w_bit = w_bit
- self.group_size = group_size if group_size != -1 else in_features
- self.zero_point = zero_point
- self.scale_dtype = torch.float32
-
- # quick sanity check (make sure alignment)
- assert self.in_features % self.group_size == 0
- assert out_features % (32 // self.w_bit) == 0
- self.pack_num = 32 // self.w_bit
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (in_features // self.group_size, out_features // self.pack_num),
- dtype=torch.int8,
- device=dev,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (in_features // self.group_size, out_features),
- dtype=torch.bfloat16 if self.use_bf16 else torch.float32,
- device=dev,
- ),
- )
- if bias:
- self.register_buffer(
- "bias",
- torch.zeros((out_features), dtype=torch.bfloat16 if self.use_bf16 else torch.float32, device=dev),
- )
- else:
- self.register_buffer(
- "bias",
- None,
- )
- qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)
- self.register_buffer("qweight", qweight)
-
- def post_init(self):
- assert self.qweight.device.type == "cpu"
-
- intweight, zeros = unpack_awq(self.qweight, self.qzeros, self.w_bit) # weight: k x n zeros: k / group_size x n
- intweight, zeros = reverse_awq_order(intweight, zeros, self.w_bit) # weight: k x n zeros: k / group_size x n
- if self.zero_point: ## asym has accuracy issue, have not root caused yet
- intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))
- zeros = torch.bitwise_and(zeros, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))
- else:
- ##symmetric, our default zp is 8
- intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))
- g_idx = torch.empty(0, dtype=torch.int32)
- self.qweight = ark.repack_quantized_weight(
- intweight,
- self.scales.float(),
- zeros,
- g_idx,
- BITS_DTYPE_MAPPING[self.w_bit],
- convert_dtype_torch2str(self.scale_dtype),
- convert_dtype_torch2str(self.scales.dtype),
- self.zero_point,
- self.group_size,
- )
-
- @classmethod
- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, has_zero_points=False):
- awq_linear = cls(
- w_bit,
- group_size,
- linear.in_features,
- linear.out_features,
- linear.bias is not None,
- has_zero_points,
- linear.weight.device,
- )
- if init_only: # just prepare for loading sd
- return awq_linear
-
- raise NotImplementedError("Only inference is supported for Exllama kernels")
-
- @torch.no_grad()
- def forward(self, x):
- assert ARK_INSTALLED, "ARK kernels could not be loaded. "
-
- input_dtype = x.dtype
- out_shape = x.shape[:-1] + (self.out_features,)
- x = x.view(-1, x.shape[-1]) # convert xd to 2d
- out_2d_shape = x.shape[:-1] + (self.out_features,)
-
- outputs = torch.zeros(out_2d_shape, dtype=input_dtype)
- bias = (
- self.bias
- if self.bias is not None
- else torch.empty(0, dtype=torch.bfloat16 if self.use_bf16 else torch.float32)
- )
-
- ark.woq_linear(
- x,
- self.qweight,
- bias,
- outputs,
- convert_dtype_torch2str(input_dtype),
- BITS_DTYPE_MAPPING[self.w_bit],
- convert_dtype_torch2str(self.scale_dtype),
- True,
- )
-
- return outputs.view(out_shape)
-
- def extra_repr(self) -> str:
- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(
- self.in_features,
- self.out_features,
- self.bias is not None,
- self.w_bit,
- self.group_size,
- )
-
class QuantLinear(nn.Module):
QUANT_TYPE = "ark_gptq_nozp"
@@ -211,38 +42,50 @@ class QuantLinear(nn.Module):
self,
bits,
group_size,
- infeatures,
- outfeatures,
+ sym,
+ in_features,
+ out_features,
bias,
- kernel_switch_threshold=128,
- trainable=False,
weight_dtype=torch.bfloat16,
**kwargs,
):
super().__init__()
-
if bits not in [2, 4, 8]:
- raise NotImplementedError("Only 2, 4,8 bits are supported for ARK.")
- assert ARK_INSTALLED, "Please install auto_round_kernel."
+ raise NotImplementedError("Only 2, 4, 8 bits are supported for ARK.")
+ if not ARK_INSTALLED:
+ raise ModuleNotFoundError(
+ "The 'auto_round_kernel' module is required but not installed. "
+ "Please install the 'auto-round-kernel' package, for example:\n"
+ " pip install auto-round-kernel"
+ )
- self.infeatures = infeatures
- self.outfeatures = outfeatures
+ self.infeatures = in_features
+ self.outfeatures = out_features
self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
+ self.group_size = group_size if group_size != -1 else self.infeatures
self.maxq = 2**self.bits - 1
+ self.qbias = 2 ** (self.bits - 1)
+ self.pack_num = 32 // self.bits
self.weight_dtype = weight_dtype
- self.asym = True
+ self.asym = not sym
ark.set_threads(torch.get_num_threads())
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
+ if not self.infeatures % self.group_size == 0:
+ raise NotImplementedError("in_features must be divisible by group_size")
+ if "awq" in self.QUANT_TYPE:
+ self.register_buffer(
+ "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)
+ )
+ else:
+ self.register_buffer(
+ "qweight",
+ torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),
+ )
self.register_buffer(
"qzeros",
torch.zeros(
(
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
+ math.ceil(self.infeatures / self.group_size),
+ self.outfeatures // 32 * self.bits,
),
dtype=torch.int32,
),
@@ -250,40 +93,54 @@ class QuantLinear(nn.Module):
self.register_buffer(
"scales",
torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
+ (math.ceil(self.infeatures / self.group_size), self.outfeatures),
dtype=weight_dtype,
),
)
if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))
+ self.register_buffer("bias", torch.zeros((self.outfeatures), dtype=torch.float))
else:
self.bias = None
- self.kernel_switch_threshold = kernel_switch_threshold
- self.trainable = trainable
+ def extra_repr(self) -> str:
+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(
+ self.infeatures,
+ self.outfeatures,
+ self.bias is not None,
+ self.bits,
+ self.group_size,
+ )
def post_init(self):
- assert self.qweight.device.type in ["cpu", "xpu"]
- # intweight: k x n, zeros: k / group_size x n
- intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS)
- if zeros is None:
- zeros = torch.empty(0, dtype=torch.int8)
- self.asym = False
+ if self.qweight.device.type not in ["cpu", "xpu"]:
+ raise NotImplementedError(
+ f"Device type {self.qweight.device.type} is not supported. Only CPU and XPU devices are supported."
+ )
+ if self.qweight.device.type != "cpu" and self.asym:
+ raise NotImplementedError("Asymmetric quantization is only supported on CPU devices")
+ if "awq" in self.QUANT_TYPE:
+ intweight, zeros = unpack_awq(
+ self.qweight, self.qzeros, self.bits
+ ) # weight: k x n zeros: k / group_size x n
+ intweight, zeros = reverse_awq_order(intweight, zeros, self.bits) # weight: k x n zeros: k / group_size x n
+ intweight = torch.bitwise_and(intweight, self.maxq)
+ zeros = torch.bitwise_and(zeros, self.maxq)
+
+ intweight = intweight - self.qbias # from uint8_t to int8_t
+ zeros = zeros - self.qbias # from uint8_t to int8_t
else:
- # change it to int8 with offset 128
- if self.bits == 8:
- zeros = (zeros.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)
+ # intweight: k x n, zeros: k / group_size x n
+ intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS, self.asym)
+ if zeros is None:
+ zeros = torch.empty(0, dtype=torch.int8)
else:
- zeros -= 2 ** (self.bits - 1)
- if self.qweight.device.type != "cpu":
- assert not self.asym
- if not self.asym:
- intweight -= 2 ** (self.bits - 1)
- intweight = intweight.to(torch.uint8 if self.asym else torch.int8)
- # due to asym return torch.uint8 but backend request int8,
- # change it to int8 with offset 128
- if self.asym:
- intweight = (intweight.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)
+ # change it to int8 with offset 128
+ zeros = (zeros.to(torch.int32) - self.qbias).to(torch.int8)
+ if self.asym:
+ intweight = (intweight.to(torch.int32) - self.qbias).to(torch.int8)
+ else:
+ intweight -= self.qbias
+ intweight = intweight.to(torch.int8)
logger.debug(
f"ARK repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}"
@@ -295,7 +152,9 @@ class QuantLinear(nn.Module):
scales = self.scales.to(torch.float16).contiguous()
else:
self.sdt = "fp32"
- self.cdt = "fp32"
+ self.cdt = "auto"
+ if self.asym and self.bits == 8:
+ self.cdt = "fp32"
scales = self.scales.float().contiguous()
self.wdt = BITS_DTYPE_MAPPING[self.bits]
@@ -303,7 +162,7 @@ class QuantLinear(nn.Module):
intweight.contiguous(),
scales,
zeros.contiguous(),
- torch.Tensor(),
+ torch.empty(0),
# compute_dtype
self.cdt,
# weight_dtype
@@ -314,10 +173,6 @@ class QuantLinear(nn.Module):
self.group_size,
)
- # self.revert_wei = torch.zeros(self.infeatures, self.outfeatures, dtype=scales.dtype, device=self.qweight.device)
- # # print(packw, packw.device, packw.dtype)
- # ark.dequantize_packed_weight(
- # self.qweight, self.revert_wei, False, self.cdt, self.wdt, self.sdt, self.group_size, self.outfeatures, self.infeatures)
# free mem
self.qzeros = torch.empty(0)
self.scales = torch.empty(0)
@@ -326,11 +181,14 @@ class QuantLinear(nn.Module):
self.bias = self.bias.to(torch.float32)
else:
self.bias = self.bias.to(torch.float16)
+ else:
+ self.bias = torch.empty(0)
def forward(self, x: torch.Tensor):
raw_input_dtype = x.dtype
if x.device.type == "cpu":
odt = torch.float32
+ self.bias = self.bias.to(torch.float32)
if raw_input_dtype != torch.float32:
x = x.to(torch.float32)
else:
@@ -340,12 +198,11 @@ class QuantLinear(nn.Module):
x = x.view(-1, x.shape[-1]) # convert xd to 2d
out_2d_shape = x.shape[:-1] + (self.outfeatures,)
outputs = torch.empty(out_2d_shape, device=x.device, dtype=odt)
- bias = self.bias if self.bias is not None else torch.empty(0, dtype=torch.float)
ark.woq_linear(
x,
self.qweight,
- bias,
+ self.bias,
outputs,
self.cdt, # compute_dtype
self.wdt, # weight_dtype
@@ -354,7 +211,7 @@ class QuantLinear(nn.Module):
self.group_size,
)
if x.device.type == "xpu":
- outputs = outputs + bias
+ outputs = outputs + self.bias
return outputs.to(raw_input_dtype).view(out_shape)
@@ -363,11 +220,15 @@ class QuantLinearGPTQ(QuantLinear):
ZP_BIAS = 1
+class QuantLinearAWQ(QuantLinear):
+ QUANT_TYPE = "ark_awq"
+
+
@torch.no_grad()
-def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias=1):
+def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias, asym):
wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32, device=qweight.device).unsqueeze(0)
zeros = None
- if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):
+ if asym:
zp_shape = list(qzeros.shape)
zp_shape[1] = zp_shape[1] * (32 // bits)
diff --git a/test/test_ark/test_model.py b/test/test_ark/test_model.py
index 911a186c0..09d8bf25a 100644
--- a/test/test_ark/test_model.py
+++ b/test/test_ark/test_model.py
@@ -1,6 +1,5 @@
import shutil
import sys
-import unittest
import pytest
@@ -23,14 +22,19 @@ class LLMDataLoader:
yield torch.ones([1, 10], dtype=torch.long)
-class TestAutoRoundTorchBackend(unittest.TestCase):
+class TestAutoRoundARKBackend:
@classmethod
- def setUpClass(self):
+ def setup_class(self):
self.model_name = "facebook/opt-125m"
self.save_folder = "./saved"
self.llm_dataloader = LLMDataLoader()
+ @classmethod
+ def teardown_class(self):
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
def model_infer(self, model, tokenizer):
prompts = [
"Hello,my name is",
@@ -57,71 +61,62 @@ class TestAutoRoundTorchBackend(unittest.TestCase):
print("-" * 50)
return decoded_outputs[0]
- @classmethod
- def tearDownClass(self):
- shutil.rmtree(self.save_folder, ignore_errors=True)
- shutil.rmtree("runs", ignore_errors=True)
-
- def test_torch_4bits_sym_cpu(self):
- model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = 4, 32, True
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=0,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")
-
- quantization_config = AutoRoundConfig(backend="ark")
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, dtype=torch.float16, device_map="cpu", quantization_config=quantization_config
- )
-
- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
- self.model_infer(model, tokenizer)
- result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=1000)
- print(result["results"]["lambada_openai"]["acc,none"])
- self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.28)
-
- shutil.rmtree("./saved", ignore_errors=True)
-
- def test_torch_4bits_sym_xpu(self):
- model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = 4, 32, True
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=0,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
+ def main_op(self, format, bits, group_size, sym, dtype, device, fast_cfg=True, tar_acc=0.28):
+ limit = 100
+ if device == "xpu":
+ limit = 1000
+ if not torch.xpu.is_available():
+ pytest.skip("No XPU device")
+ if sym is False:
+ pytest.skip("No asym support for XPU")
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto")
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+ if fast_cfg:
+ autoround = AutoRound(
+ model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, nsamples=1, disable_opt_rtn=True
+ )
+ else:
+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)
quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model
+ autoround.quantize_and_save(output_dir=quantized_model_path, format=format) ##will convert to gptq model
quantization_config = AutoRoundConfig(backend="ark")
model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, dtype=torch.float16, device_map="xpu", quantization_config=quantization_config
+ quantized_model_path, dtype=dtype, device_map=device, quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
self.model_infer(model, tokenizer)
- result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=1000)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=limit)
print(result["results"]["lambada_openai"]["acc,none"])
- self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.28)
+ assert result["results"]["lambada_openai"]["acc,none"] > tar_acc
torch.xpu.empty_cache()
shutil.rmtree(self.save_folder, ignore_errors=True)
+ @pytest.mark.parametrize("format", ["auto_round", "auto_round:gptqmodel"])
+ @pytest.mark.parametrize("bits, group_size, sym", [(4, 128, True), (8, 128, True)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("device", ["cpu", "xpu"])
+ def test_formats(self, format, bits, group_size, sym, dtype, device):
+ self.main_op(format, bits, group_size, sym, dtype, device)
+
+ @pytest.mark.parametrize("format", ["auto_round:auto_awq"])
+ @pytest.mark.parametrize("bits, group_size, sym", [(4, 32, True)])
+ @pytest.mark.parametrize("dtype", [torch.float16])
+ @pytest.mark.parametrize("device", ["cpu", "xpu"])
+ def test_awq_fp16(self, format, bits, group_size, sym, dtype, device):
+ self.main_op(format, bits, group_size, sym, dtype, device)
+
+ @pytest.mark.parametrize("format", ["auto_round"])
+ @pytest.mark.parametrize("bits, group_size, sym", [(2, 32, False)])
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("device", ["cpu"])
+ def test_other_bits(self, format, bits, group_size, sym, dtype, device):
+ self.main_op(format, bits, group_size, sym, dtype, device, False, 0.2)
+
if __name__ == "__main__":
- unittest.main()
+ p = TestAutoRoundARKBackend()
+ p.setup_class()
+ p.test_formats("auto_round:auto_awq", 4, 32, True, torch.bfloat16, "xpu")
+ p.teardown_class()