PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update xpu format exporting (#214)
WeiweiZhang190d9fc07b8db
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 1adf65987..19833b217 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -20,7 +20,7 @@ from typing import Dict, List, Optional, Unionimport torchimport transformersfrom auto_round.export.register import register_format-from auto_round.utils import get_module, logger, set_module+from auto_round.utils import get_module, logger, set_module, detect_devicefrom .config import QuantConfigfrom .model_wrapper import WeightOnlyLinear@@ -109,7 +109,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):@register_format("itrex_xpu")def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):- """Save configure file and weights for CPU backend inference."""+ """Save configure file and weights for XPU backend inference."""model = kwargs["model"]layer_config = kwargs["layer_config"]sym = kwargs["sym"]@@ -185,15 +185,11 @@ def pack_model(5. zeros is always needed even for sym.inplace (bool, optional): Compress the model in place, or copy the model and compress it.- xpu args:- compression_dtype=torch.int8,- compression_dim=0,- use_optimum_format=False,- scale_dtype=convert_dtype_str2torch(config.scale_dtype),- device="xpu","""+ # Due to XPU doesn't support tuning yet+ device = "cpu" if device == "xpu" else deviceif model.device.type == 'meta':- model = model.to("cuda" if device == "xpu" else device)+ model = model.to(device)if inplace:compressed_model = modelelse:@@ -203,6 +199,7 @@ def pack_model(q_config = json.load(f)else:q_config = layer_config+for k, v in q_config.items():if "float" in v["data_type"]:continue@@ -219,17 +216,10 @@ def pack_model(if not isinstance(scale, torch.Tensor):scale = torch.tensor(scale, dtype=convert_dtype)zp = torch.tensor(zp, dtype=torch.int32)- if device == "xpu":- scale = torch.tensor(v["scale"], dtype=torch.float32)- zp = None if sym else torch.tensor(v["zp"], dtype=torch.int32)else:if not inplace:scale = scale.clone()zp = zp.clone()- if device == "xpu":- # Please note that for XPU, the scale data type is forcibly set to fp32- scale = scale.to(dtype=torch.float32)- zp = None if sym else zp.to(dtype=torch.int32)else:scale = scale.to(dtype=convert_dtype)zp = zp.to(dtype=torch.int32)@@ -252,7 +242,7 @@ def pack_model(scale_dtype=scale_dtype,zp=zp is not None,bias=m.bias is not None,- device="cuda" if device == "xpu" else device,+ device=device,compression_dtype=compression_dtype,compression_dim=compression_dim,use_optimum_format=use_optimum_format, # xpu is False@@ -261,3 +251,4 @@ def pack_model(set_module(compressed_model, k, new_module)return compressed_model+diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 0b0548332..f71d5c436 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -346,9 +346,7 @@ if __name__ == '__main__':autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)if 'xpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,- compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,- device="xpu")+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)if "cpu" in deployment_device:autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)if "fake" in deployment_device:diff --git a/test/test_autoround_export_to_itrex.py b/test/test_autoround_export_to_itrex.pyindex 3c005d224..e2b1fe1e5 100644--- a/test/test_autoround_export_to_itrex.py+++ b/test/test_autoround_export_to_itrex.py@@ -90,7 +90,28 @@ class TestAutoroundExport(unittest.TestCase):self.assertEqual(config.desc_act, loaded_config.desc_act)self.assertEqual(config.bits, loaded_config.bits)self.assertEqual(config.sym, loaded_config.sym)++ def test_xpu_export(self):+ model = copy.deepcopy(self.gptj)+ out1 = model(self.lm_input)+ round = AutoRound+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)+ q_model, layer_config1 = optq_1.quantize()+ from auto_round.export.export_to_itrex import pack_model++ compressed_model_xpu = pack_model(model=q_model, layer_config=layer_config1, device="xpu", inplace=False)+ compressed_model_cpu = pack_model(model=q_model, layer_config=layer_config1, inplace=False)+ out2 = model(self.lm_input)+ out3 = q_model(self.lm_input)+ out4 = compressed_model_xpu(self.lm_input)+ out5 = compressed_model_cpu(self.lm_input)+ self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))+ self.assertFalse(torch.all(out1[0] == out2[0]))+ self.assertTrue(torch.all(out2[0] == out3[0]))+ self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-3)))+ self.assertTrue(torch.all(torch.isclose(out4[0], out5[0], atol=1e-5)))if __name__ == "__main__":unittest.main()+