Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update xpu format exporting (#214)

WeiweiZhang190d9fc07b8db
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 1adf65987..19833b217 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -20,7 +20,7 @@ from typing import Dict, List, Optional, Union
import torch
import transformers
from auto_round.export.register import register_format
-from auto_round.utils import get_module, logger, set_module
+from auto_round.utils import get_module, logger, set_module, detect_device
from .config import QuantConfig
from .model_wrapper import WeightOnlyLinear
@@ -109,7 +109,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
@register_format("itrex_xpu")
def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):
- """Save configure file and weights for CPU backend inference."""
+ """Save configure file and weights for XPU backend inference."""
model = kwargs["model"]
layer_config = kwargs["layer_config"]
sym = kwargs["sym"]
@@ -185,15 +185,11 @@ def pack_model(
5. zeros is always needed even for sym.
inplace (bool, optional): Compress the model in place, or copy the model and compress it.
- xpu args:
- compression_dtype=torch.int8,
- compression_dim=0,
- use_optimum_format=False,
- scale_dtype=convert_dtype_str2torch(config.scale_dtype),
- device="xpu",
"""
+ # Due to XPU doesn't support tuning yet
+ device = "cpu" if device == "xpu" else device
if model.device.type == 'meta':
- model = model.to("cuda" if device == "xpu" else device)
+ model = model.to(device)
if inplace:
compressed_model = model
else:
@@ -203,6 +199,7 @@ def pack_model(
q_config = json.load(f)
else:
q_config = layer_config
+
for k, v in q_config.items():
if "float" in v["data_type"]:
continue
@@ -219,17 +216,10 @@ def pack_model(
if not isinstance(scale, torch.Tensor):
scale = torch.tensor(scale, dtype=convert_dtype)
zp = torch.tensor(zp, dtype=torch.int32)
- if device == "xpu":
- scale = torch.tensor(v["scale"], dtype=torch.float32)
- zp = None if sym else torch.tensor(v["zp"], dtype=torch.int32)
else:
if not inplace:
scale = scale.clone()
zp = zp.clone()
- if device == "xpu":
- # Please note that for XPU, the scale data type is forcibly set to fp32
- scale = scale.to(dtype=torch.float32)
- zp = None if sym else zp.to(dtype=torch.int32)
else:
scale = scale.to(dtype=convert_dtype)
zp = zp.to(dtype=torch.int32)
@@ -252,7 +242,7 @@ def pack_model(
scale_dtype=scale_dtype,
zp=zp is not None,
bias=m.bias is not None,
- device="cuda" if device == "xpu" else device,
+ device=device,
compression_dtype=compression_dtype,
compression_dim=compression_dim,
use_optimum_format=use_optimum_format, # xpu is False
@@ -261,3 +251,4 @@ def pack_model(
set_module(compressed_model, k, new_module)
return compressed_model
+
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 0b0548332..f71d5c436 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -346,9 +346,7 @@ if __name__ == '__main__':
autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)
if 'xpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
- compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,
- device="xpu")
+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)
if "cpu" in deployment_device:
autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
if "fake" in deployment_device:
diff --git a/test/test_autoround_export_to_itrex.py b/test/test_autoround_export_to_itrex.py
index 3c005d224..e2b1fe1e5 100644
--- a/test/test_autoround_export_to_itrex.py
+++ b/test/test_autoround_export_to_itrex.py
@@ -90,7 +90,28 @@ class TestAutoroundExport(unittest.TestCase):
self.assertEqual(config.desc_act, loaded_config.desc_act)
self.assertEqual(config.bits, loaded_config.bits)
self.assertEqual(config.sym, loaded_config.sym)
+
+ def test_xpu_export(self):
+ model = copy.deepcopy(self.gptj)
+ out1 = model(self.lm_input)
+ round = AutoRound
+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)
+ q_model, layer_config1 = optq_1.quantize()
+ from auto_round.export.export_to_itrex import pack_model
+
+ compressed_model_xpu = pack_model(model=q_model, layer_config=layer_config1, device="xpu", inplace=False)
+ compressed_model_cpu = pack_model(model=q_model, layer_config=layer_config1, inplace=False)
+ out2 = model(self.lm_input)
+ out3 = q_model(self.lm_input)
+ out4 = compressed_model_xpu(self.lm_input)
+ out5 = compressed_model_cpu(self.lm_input)
+ self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))
+ self.assertFalse(torch.all(out1[0] == out2[0]))
+ self.assertTrue(torch.all(out2[0] == out3[0]))
+ self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-3)))
+ self.assertTrue(torch.all(torch.isclose(out4[0], out5[0], atol=1e-5)))
if __name__ == "__main__":
unittest.main()
+