PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix inference issue in ARK for AWQ format (#1170)
Luo Yufd5a17295017
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 43de447f1..057ecf5bd 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -177,6 +177,8 @@ gptqmodel_marlin_feature_checker = functools.partial(mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)+ark_feature_checker = functools.partial(in_feature_checker_group_size)+def fp8_static_scheme_checker(in_feature: int,@@ -438,13 +440,13 @@ BackendInfos["auto_awq:gemm"] = BackendInfo()BackendInfos["auto_round_kernel"] = BackendInfo(- device=["cpu", "xpu"],- sym=[True],+ device=["cpu"],+ sym=[True, False],packing_format=GPTQ_FORMAT_NO_ZP,bits=[2, 4, 8],group_size=None,priority=6,- checkers=[],+ checkers=[ark_feature_checker],alias=["ark"],compute_dtype=["float32", "float16"],data_type=["int"],@@ -453,14 +455,46 @@ BackendInfos["auto_round_kernel"] = BackendInfo(systems=["linux"],)-BackendInfos["auto_round_kernel_zp"] = BackendInfo(- device=["cpu", "xpu"],+BackendInfos["auto_round_kernel_xpu"] = BackendInfo(+ device=["xpu"],sym=[True],+ packing_format=GPTQ_FORMAT_NO_ZP,+ bits=[4, 8],+ group_size=None,+ priority=6,+ checkers=[ark_feature_checker],+ alias=["ark"],+ compute_dtype=["float32", "float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ requirements=["torch>=2.9.0", "auto_round_kernel"],+ systems=["linux"],+)++BackendInfos["auto_round_kernel_zp"] = BackendInfo(+ device=["cpu"],+ sym=[True, False],packing_format=GPTQ_FORMAT,bits=[2, 4, 8],group_size=None,priority=6,- checkers=[],+ checkers=[ark_feature_checker],+ alias=["ark"],+ compute_dtype=["float32", "float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ requirements=["torch>=2.9.0", "auto_round_kernel"],+ systems=["linux"],+)++BackendInfos["auto_round_kernel_zp_xpu"] = BackendInfo(+ device=["xpu"],+ sym=[True],+ packing_format=GPTQ_FORMAT,+ bits=[4, 8],+ group_size=None,+ priority=6,+ checkers=[ark_feature_checker],alias=["ark"],compute_dtype=["float32", "float16"],data_type=["int"],@@ -476,7 +510,23 @@ BackendInfos["auto_round_kernel_awq"] = BackendInfo(bits=[2, 4, 8],group_size=None,priority=6,- checkers=[],+ checkers=[ark_feature_checker],+ alias=["ark"],+ compute_dtype=["float32", "float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ requirements=["torch>=2.9.0", "auto_round_kernel"],+ systems=["linux"],+)++BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(+ device=["xpu"],+ sym=[True],+ packing_format=AWQ_FORMAT,+ bits=[4, 8],+ group_size=None,+ priority=6,+ checkers=[ark_feature_checker],alias=["ark"],compute_dtype=["float32", "float16"],data_type=["int"],diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex f4cd1c527..49aa42976 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -397,6 +397,16 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)# Special handling for AWQ layersfrom auto_round_extension.ark.qlinear import QuantLinearAWQ+ if "auto_round_kernel" in layer_backend:+ return QuantLinear(+ bits=config["bits"],+ group_size=config["group_size"],+ sym=config["sym"],+ in_features=in_features,+ out_features=out_features,+ bias=bias,+ weight_dtype=layer.weight.dtype,+ )if "awq" in layer_backend and isinstance(QuantLinear, QuantLinearAWQ):return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True, has_zero_points=not config["sym"]diff --git a/auto_round_extension/ark/qlinear.py b/auto_round_extension/ark/qlinear.pyindex 2fd3ee9d6..9e28b7347 100644--- a/auto_round_extension/ark/qlinear.py+++ b/auto_round_extension/ark/qlinear.py@@ -17,6 +17,7 @@ import mathimport torchimport torch.nn as nn+from auto_round.export.export_to_awq.utils import dequantize_gemm, reverse_awq_order, unpack_awqfrom auto_round.utils import convert_dtype_torch2str, loggertry:@@ -32,176 +33,6 @@ BITS_DTYPE_MAPPING = {8: "int8",}-AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]---def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):- shifts = torch.arange(0, 32, bits, device="cpu")-- # unpacking columnwise- iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(- torch.int8 # smallest dtype available- )- iweights = iweights.view(iweights.shape[0], -1)-- # unpacking columnwise- if qzeros is not None:- izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(- torch.int8 # smallest dtype available- )- izeros = izeros.view(izeros.shape[0], -1)- else:- izeros = qzeros-- return iweights, izeros---def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):- reverse_order_tensor = torch.arange(- iweights.shape[-1],- dtype=torch.int32,- device="cpu",- )- reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)- reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]- reverse_order_tensor = reverse_order_tensor.view(-1)-- if izeros is not None:- izeros = izeros[:, reverse_order_tensor]- iweights = iweights[:, reverse_order_tensor]- return iweights, izeros---class QuantLinearAWQ(nn.Module):- QUANT_TYPE = "ark_awq"-- def __init__(self, w_bit, group_size, in_features, out_features, bias, zero_point, dev):- super().__init__()- assert ARK_INSTALLED, "Please install auto_round_kernel package."-- self.use_bf16 = ark.check_isa_supported("AMX")-- if w_bit not in [2, 3, 4, 8]:- raise NotImplementedError("Only 2, 3, 4, 8 bits are supported for now.")-- self.in_features = in_features- self.out_features = out_features- self.w_bit = w_bit- self.group_size = group_size if group_size != -1 else in_features- self.zero_point = zero_point- self.scale_dtype = torch.float32-- # quick sanity check (make sure alignment)- assert self.in_features % self.group_size == 0- assert out_features % (32 // self.w_bit) == 0- self.pack_num = 32 // self.w_bit- self.register_buffer(- "qzeros",- torch.zeros(- (in_features // self.group_size, out_features // self.pack_num),- dtype=torch.int8,- device=dev,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (in_features // self.group_size, out_features),- dtype=torch.bfloat16 if self.use_bf16 else torch.float32,- device=dev,- ),- )- if bias:- self.register_buffer(- "bias",- torch.zeros((out_features), dtype=torch.bfloat16 if self.use_bf16 else torch.float32, device=dev),- )- else:- self.register_buffer(- "bias",- None,- )- qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)- self.register_buffer("qweight", qweight)-- def post_init(self):- assert self.qweight.device.type == "cpu"-- intweight, zeros = unpack_awq(self.qweight, self.qzeros, self.w_bit) # weight: k x n zeros: k / group_size x n- intweight, zeros = reverse_awq_order(intweight, zeros, self.w_bit) # weight: k x n zeros: k / group_size x n- if self.zero_point: ## asym has accuracy issue, have not root caused yet- intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))- zeros = torch.bitwise_and(zeros, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))- else:- ##symmetric, our default zp is 8- intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))- g_idx = torch.empty(0, dtype=torch.int32)- self.qweight = ark.repack_quantized_weight(- intweight,- self.scales.float(),- zeros,- g_idx,- BITS_DTYPE_MAPPING[self.w_bit],- convert_dtype_torch2str(self.scale_dtype),- convert_dtype_torch2str(self.scales.dtype),- self.zero_point,- self.group_size,- )-- @classmethod- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, has_zero_points=False):- awq_linear = cls(- w_bit,- group_size,- linear.in_features,- linear.out_features,- linear.bias is not None,- has_zero_points,- linear.weight.device,- )- if init_only: # just prepare for loading sd- return awq_linear-- raise NotImplementedError("Only inference is supported for Exllama kernels")-- @torch.no_grad()- def forward(self, x):- assert ARK_INSTALLED, "ARK kernels could not be loaded. "-- input_dtype = x.dtype- out_shape = x.shape[:-1] + (self.out_features,)- x = x.view(-1, x.shape[-1]) # convert xd to 2d- out_2d_shape = x.shape[:-1] + (self.out_features,)-- outputs = torch.zeros(out_2d_shape, dtype=input_dtype)- bias = (- self.bias- if self.bias is not None- else torch.empty(0, dtype=torch.bfloat16 if self.use_bf16 else torch.float32)- )-- ark.woq_linear(- x,- self.qweight,- bias,- outputs,- convert_dtype_torch2str(input_dtype),- BITS_DTYPE_MAPPING[self.w_bit],- convert_dtype_torch2str(self.scale_dtype),- True,- )-- return outputs.view(out_shape)-- def extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.in_features,- self.out_features,- self.bias is not None,- self.w_bit,- self.group_size,- )-class QuantLinear(nn.Module):QUANT_TYPE = "ark_gptq_nozp"@@ -211,38 +42,50 @@ class QuantLinear(nn.Module):self,bits,group_size,- infeatures,- outfeatures,+ sym,+ in_features,+ out_features,bias,- kernel_switch_threshold=128,- trainable=False,weight_dtype=torch.bfloat16,**kwargs,):super().__init__()-if bits not in [2, 4, 8]:- raise NotImplementedError("Only 2, 4,8 bits are supported for ARK.")- assert ARK_INSTALLED, "Please install auto_round_kernel."+ raise NotImplementedError("Only 2, 4, 8 bits are supported for ARK.")+ if not ARK_INSTALLED:+ raise ModuleNotFoundError(+ "The 'auto_round_kernel' module is required but not installed. "+ "Please install the 'auto-round-kernel' package, for example:\n"+ " pip install auto-round-kernel"+ )- self.infeatures = infeatures- self.outfeatures = outfeatures+ self.infeatures = in_features+ self.outfeatures = out_featuresself.bits = bits- self.group_size = group_size if group_size != -1 else infeatures+ self.group_size = group_size if group_size != -1 else self.infeaturesself.maxq = 2**self.bits - 1+ self.qbias = 2 ** (self.bits - 1)+ self.pack_num = 32 // self.bitsself.weight_dtype = weight_dtype- self.asym = True+ self.asym = not symark.set_threads(torch.get_num_threads())- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )+ if not self.infeatures % self.group_size == 0:+ raise NotImplementedError("in_features must be divisible by group_size")+ if "awq" in self.QUANT_TYPE:+ self.register_buffer(+ "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)+ )+ else:+ self.register_buffer(+ "qweight",+ torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),+ )self.register_buffer("qzeros",torch.zeros((- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,+ math.ceil(self.infeatures / self.group_size),+ self.outfeatures // 32 * self.bits,),dtype=torch.int32,),@@ -250,40 +93,54 @@ class QuantLinear(nn.Module):self.register_buffer("scales",torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),+ (math.ceil(self.infeatures / self.group_size), self.outfeatures),dtype=weight_dtype,),)if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))+ self.register_buffer("bias", torch.zeros((self.outfeatures), dtype=torch.float))else:self.bias = None- self.kernel_switch_threshold = kernel_switch_threshold- self.trainable = trainable+ def extra_repr(self) -> str:+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ self.infeatures,+ self.outfeatures,+ self.bias is not None,+ self.bits,+ self.group_size,+ )def post_init(self):- assert self.qweight.device.type in ["cpu", "xpu"]- # intweight: k x n, zeros: k / group_size x n- intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS)- if zeros is None:- zeros = torch.empty(0, dtype=torch.int8)- self.asym = False+ if self.qweight.device.type not in ["cpu", "xpu"]:+ raise NotImplementedError(+ f"Device type {self.qweight.device.type} is not supported. Only CPU and XPU devices are supported."+ )+ if self.qweight.device.type != "cpu" and self.asym:+ raise NotImplementedError("Asymmetric quantization is only supported on CPU devices")+ if "awq" in self.QUANT_TYPE:+ intweight, zeros = unpack_awq(+ self.qweight, self.qzeros, self.bits+ ) # weight: k x n zeros: k / group_size x n+ intweight, zeros = reverse_awq_order(intweight, zeros, self.bits) # weight: k x n zeros: k / group_size x n+ intweight = torch.bitwise_and(intweight, self.maxq)+ zeros = torch.bitwise_and(zeros, self.maxq)++ intweight = intweight - self.qbias # from uint8_t to int8_t+ zeros = zeros - self.qbias # from uint8_t to int8_telse:- # change it to int8 with offset 128- if self.bits == 8:- zeros = (zeros.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)+ # intweight: k x n, zeros: k / group_size x n+ intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS, self.asym)+ if zeros is None:+ zeros = torch.empty(0, dtype=torch.int8)else:- zeros -= 2 ** (self.bits - 1)- if self.qweight.device.type != "cpu":- assert not self.asym- if not self.asym:- intweight -= 2 ** (self.bits - 1)- intweight = intweight.to(torch.uint8 if self.asym else torch.int8)- # due to asym return torch.uint8 but backend request int8,- # change it to int8 with offset 128- if self.asym:- intweight = (intweight.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)+ # change it to int8 with offset 128+ zeros = (zeros.to(torch.int32) - self.qbias).to(torch.int8)+ if self.asym:+ intweight = (intweight.to(torch.int32) - self.qbias).to(torch.int8)+ else:+ intweight -= self.qbias+ intweight = intweight.to(torch.int8)logger.debug(f"ARK repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}"@@ -295,7 +152,9 @@ class QuantLinear(nn.Module):scales = self.scales.to(torch.float16).contiguous()else:self.sdt = "fp32"- self.cdt = "fp32"+ self.cdt = "auto"+ if self.asym and self.bits == 8:+ self.cdt = "fp32"scales = self.scales.float().contiguous()self.wdt = BITS_DTYPE_MAPPING[self.bits]@@ -303,7 +162,7 @@ class QuantLinear(nn.Module):intweight.contiguous(),scales,zeros.contiguous(),- torch.Tensor(),+ torch.empty(0),# compute_dtypeself.cdt,# weight_dtype@@ -314,10 +173,6 @@ class QuantLinear(nn.Module):self.group_size,)- # self.revert_wei = torch.zeros(self.infeatures, self.outfeatures, dtype=scales.dtype, device=self.qweight.device)- # # print(packw, packw.device, packw.dtype)- # ark.dequantize_packed_weight(- # self.qweight, self.revert_wei, False, self.cdt, self.wdt, self.sdt, self.group_size, self.outfeatures, self.infeatures)# free memself.qzeros = torch.empty(0)self.scales = torch.empty(0)@@ -326,11 +181,14 @@ class QuantLinear(nn.Module):self.bias = self.bias.to(torch.float32)else:self.bias = self.bias.to(torch.float16)+ else:+ self.bias = torch.empty(0)def forward(self, x: torch.Tensor):raw_input_dtype = x.dtypeif x.device.type == "cpu":odt = torch.float32+ self.bias = self.bias.to(torch.float32)if raw_input_dtype != torch.float32:x = x.to(torch.float32)else:@@ -340,12 +198,11 @@ class QuantLinear(nn.Module):x = x.view(-1, x.shape[-1]) # convert xd to 2dout_2d_shape = x.shape[:-1] + (self.outfeatures,)outputs = torch.empty(out_2d_shape, device=x.device, dtype=odt)- bias = self.bias if self.bias is not None else torch.empty(0, dtype=torch.float)ark.woq_linear(x,self.qweight,- bias,+ self.bias,outputs,self.cdt, # compute_dtypeself.wdt, # weight_dtype@@ -354,7 +211,7 @@ class QuantLinear(nn.Module):self.group_size,)if x.device.type == "xpu":- outputs = outputs + bias+ outputs = outputs + self.biasreturn outputs.to(raw_input_dtype).view(out_shape)@@ -363,11 +220,15 @@ class QuantLinearGPTQ(QuantLinear):ZP_BIAS = 1+class QuantLinearAWQ(QuantLinear):+ QUANT_TYPE = "ark_awq"++@torch.no_grad()-def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias=1):+def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias, asym):wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32, device=qweight.device).unsqueeze(0)zeros = None- if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):+ if asym:zp_shape = list(qzeros.shape)zp_shape[1] = zp_shape[1] * (32 // bits)diff --git a/test/test_ark/test_model.py b/test/test_ark/test_model.pyindex 911a186c0..09d8bf25a 100644--- a/test/test_ark/test_model.py+++ b/test/test_ark/test_model.py@@ -1,6 +1,5 @@import shutilimport sys-import unittestimport pytest@@ -23,14 +22,19 @@ class LLMDataLoader:yield torch.ones([1, 10], dtype=torch.long)-class TestAutoRoundTorchBackend(unittest.TestCase):+class TestAutoRoundARKBackend:@classmethod- def setUpClass(self):+ def setup_class(self):self.model_name = "facebook/opt-125m"self.save_folder = "./saved"self.llm_dataloader = LLMDataLoader()+ @classmethod+ def teardown_class(self):+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+def model_infer(self, model, tokenizer):prompts = ["Hello,my name is",@@ -57,71 +61,62 @@ class TestAutoRoundTorchBackend(unittest.TestCase):print("-" * 50)return decoded_outputs[0]- @classmethod- def tearDownClass(self):- shutil.rmtree(self.save_folder, ignore_errors=True)- shutil.rmtree("runs", ignore_errors=True)-- def test_torch_4bits_sym_cpu(self):- model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = 4, 32, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=0,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")-- quantization_config = AutoRoundConfig(backend="ark")- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, dtype=torch.float16, device_map="cpu", quantization_config=quantization_config- )-- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)- self.model_infer(model, tokenizer)- result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=1000)- print(result["results"]["lambada_openai"]["acc,none"])- self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.28)-- shutil.rmtree("./saved", ignore_errors=True)-- def test_torch_4bits_sym_xpu(self):- model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = 4, 32, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=0,- seqlen=2,- dataset=self.llm_dataloader,- )+ def main_op(self, format, bits, group_size, sym, dtype, device, fast_cfg=True, tar_acc=0.28):+ limit = 100+ if device == "xpu":+ limit = 1000+ if not torch.xpu.is_available():+ pytest.skip("No XPU device")+ if sym is False:+ pytest.skip("No asym support for XPU")+ model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto")+ tokenizer = AutoTokenizer.from_pretrained(self.model_name)+ if fast_cfg:+ autoround = AutoRound(+ model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, nsamples=1, disable_opt_rtn=True+ )+ else:+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model+ autoround.quantize_and_save(output_dir=quantized_model_path, format=format) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="ark")model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, dtype=torch.float16, device_map="xpu", quantization_config=quantization_config+ quantized_model_path, dtype=dtype, device_map=device, quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)- result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=1000)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=32, tasks="lambada_openai", limit=limit)print(result["results"]["lambada_openai"]["acc,none"])- self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.28)+ assert result["results"]["lambada_openai"]["acc,none"] > tar_acctorch.xpu.empty_cache()shutil.rmtree(self.save_folder, ignore_errors=True)+ @pytest.mark.parametrize("format", ["auto_round", "auto_round:gptqmodel"])+ @pytest.mark.parametrize("bits, group_size, sym", [(4, 128, True), (8, 128, True)])+ @pytest.mark.parametrize("dtype", [torch.bfloat16])+ @pytest.mark.parametrize("device", ["cpu", "xpu"])+ def test_formats(self, format, bits, group_size, sym, dtype, device):+ self.main_op(format, bits, group_size, sym, dtype, device)++ @pytest.mark.parametrize("format", ["auto_round:auto_awq"])+ @pytest.mark.parametrize("bits, group_size, sym", [(4, 32, True)])+ @pytest.mark.parametrize("dtype", [torch.float16])+ @pytest.mark.parametrize("device", ["cpu", "xpu"])+ def test_awq_fp16(self, format, bits, group_size, sym, dtype, device):+ self.main_op(format, bits, group_size, sym, dtype, device)++ @pytest.mark.parametrize("format", ["auto_round"])+ @pytest.mark.parametrize("bits, group_size, sym", [(2, 32, False)])+ @pytest.mark.parametrize("dtype", [torch.bfloat16])+ @pytest.mark.parametrize("device", ["cpu"])+ def test_other_bits(self, format, bits, group_size, sym, dtype, device):+ self.main_op(format, bits, group_size, sym, dtype, device, False, 0.2)+if __name__ == "__main__":- unittest.main()+ p = TestAutoRoundARKBackend()+ p.setup_class()+ p.test_formats("auto_round:auto_awq", 4, 32, True, torch.bfloat16, "xpu")+ p.teardown_class()