PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support xpu in tuning and inference (#481)
Wenhua Cheng1603573ac0e9
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex da78e5698..0fb39e6b3 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -126,7 +126,7 @@ class AutoHfQuantizer:else:quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101quant_method = quantization_config.quant_method-+# Again, we need a special care for bnb as we have a single quantization config# class for both 4-bit and 8-bit quantizationif quant_method == QuantizationMethod.BITS_AND_BYTES:@@ -170,15 +170,15 @@ class AutoHfQuantizer:warning_msg = ""loading_attr_dict = quantization_config_from_args.get_loading_attributes() \- if quantization_config_from_args is not None else None+ if quantization_config_from_args is not None else Noneif isinstance(quantization_config, dict):if "auto-round" in quantization_config["quant_method"]:quantization_config = AutoRoundConfig.from_dict(quantization_config)else:- if isinstance(quantization_config_from_args, (AutoRoundConfig)):+ if isinstance(quantization_config_from_args, (AutoRoundConfig)):logger.info(f"Loading quantized model in auto_round format.")tmp_backend = quantization_config["quant_method"]- if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:+ if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or ""`auto-round` format")exit(-1)@@ -187,7 +187,7 @@ class AutoHfQuantizer:target_backend = loading_attr_dict["backend"]loading_attr_dict.pop("backend")if "auto_round" not in target_backend:- target_backend = f"auto_round:{tmp_backend}" #+ target_backend = f"auto_round:{tmp_backend}" #quantization_config = AutoRoundConfig.from_dict(quantization_config)setattr(quantization_config, "backend", target_backend)else:@@ -196,7 +196,7 @@ class AutoHfQuantizer:if isinstance(quantization_config,(GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:# special case for GPTQ / AWQ config collision-+for attr, val in loading_attr_dict.items():setattr(quantization_config, attr, val)warning_msg += (@@ -208,7 +208,7 @@ class AutoHfQuantizer:warnings.warn(warning_msg)return quantization_config-+@staticmethoddef supports_quant_method(quantization_config_dict):from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING@@ -220,7 +220,7 @@ class AutoHfQuantizer:quant_method = QuantizationMethod.BITS_AND_BYTES + suffixelif quant_method is None:raise ValueError(- "The model's quantization config from the arguments has no `quant_method` attribute."\+ "The model's quantization config from the arguments has no `quant_method` attribute." \"Make sure that the model has been correctly quantized")@@ -287,15 +287,11 @@ class AutoRoundConfig(QuantizationConfigMixin):raise ValueError("group_size must be greater than 0 or equal to -1")def get_loading_attributes(self):- # attributes_dict = copy.deepcopy(self.__dict__)- loading_attibutes_dict = {"backend": self.backend}- # loading_attributes = ["backend"]- # loading_attibutes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}+ loading_attibutes_dict = {"target_backend": self.backend}return loading_attibutes_dictdef to_dict(self):config_dict = super().to_dict()- config_dict.pop("disable_exllama", None)return config_dict@@ -355,7 +351,18 @@ class AutoRoundQuantizer(HfQuantizer):# Return None if no matching backend or alias is foundreturn None+ def detect_auto_device(self):+ if torch.cuda.is_available():+ return "cuda"+ elif is_hpu_supported():+ return "hpu"+ elif torch.xpu.is_available():+ return "xpu"+ else:+ return "cpu"+def detect_device(self, target_backend, orig_backend):+ ##TODO need to refine later"""Detects the appropriate device for the specified backend.This function determines the device type based on the target backend. If the target backend is@@ -385,29 +392,29 @@ class AutoRoundQuantizer(HfQuantizer):return "cuda"elif "hpu" in target_backend:return "hpu"+ elif "xpu" in target_backend:+ return "xpu"elif "cpu" in target_backend:return "cpu"# Determine the device automatically based on availabilityif target_backend.split(":")[0] == "auto":- if torch.cuda.is_available():- return "cuda"- elif is_hpu_supported():- return "hpu"- else:- return "cpu"+ return self.detect_auto_device()# Find the backend and determine the device type from BackendInfosbackend = self.find_backend(target_backend)if backend is None:- raise ValueError("Backend not found, please set it to 'auto' to have a try ")+ raise ValueError("Backend is not found, please set it to 'auto' to have a try ")device = BackendInfos[backend].device[0]if "cuda" in device and torch.cuda.is_available():return deviceelif "hpu" in device and is_hpu_supported():return device+ elif "xpu" in device and torch.xpu.is_available():+ return deviceelse:+ ## trickreturn "cpu"def convert_model(self, model: nn.Module):@@ -436,7 +443,6 @@ class AutoRoundQuantizer(HfQuantizer):if not hasattr(quantization_config, "target_backend"):quantization_config.target_backend = quantization_config.backend- target_device = self.detect_device(quantization_config.target_backend, quantization_config.backend)target_device = self.detect_device(None, quantization_config.backend)self.target_device = target_device@@ -451,13 +457,13 @@ class AutoRoundQuantizer(HfQuantizer):data_type = quantization_config.data_type if hasattr(quantization_config,"data_type") else "int" # pragma: no coversym = quantization_config.sym-+quant_block_list = quantization_config.quant_block_list if hasattr(quantization_config,- "quant_block_list") else None+ "quant_block_list") else Noneif quant_block_list is None:to_quant_block_names = quantization_config.to_quant_block_names if hasattr(quantization_config,- "to_quant_block_names") else None+ "to_quant_block_names") else Noneif to_quant_block_names is not None:if isinstance(to_quant_block_names, (list, tuple)):quant_block_list = to_quant_block_names@@ -474,22 +480,20 @@ class AutoRoundQuantizer(HfQuantizer):extra_config = {}if hasattr(quantization_config, "extra_config"):extra_config = quantization_config.extra_config- if hasattr(quantization_config, "modules_in_block_to_quantize"):##gptq format+ if hasattr(quantization_config, "modules_in_block_to_quantize"): ##gptq formatmodules_in_block_to_quantize_tmp = quantization_config.modules_in_block_to_quantizemodules_in_block_to_quantize = [item for sublist in modules_in_block_to_quantize_tmp for item in sublist]for layer_name in layer_names:quantized = Falsefor qname in modules_in_block_to_quantize:if qname in layer_name:- quantized=True+ quantized = Truebreakif not quantized:- extra_config[layer_name]={"bits":16}+ extra_config[layer_name] = {"bits": 16}if hasattr(quantization_config, "modules_to_not_convert"):for layer_name in quantization_config.modules_to_not_convert:- extra_config[layer_name]={"bits":16}--+ extra_config[layer_name] = {"bits": 16}layer_names += extra_config.keys()layer_names = list(set(layer_names))@@ -514,6 +518,8 @@ class AutoRoundQuantizer(HfQuantizer):backend = quantization_config.backendelif 'gptq' in quantization_config.quant_method: # pragma: no coverbackend = 'gptq'+ elif "awq" in quantization_config.quant_method:+ backend = "awq"else: # pragma: no coverlogger.error("Please specify quantization backend")raise ValueError("Quantization backend must be specified.")@@ -670,8 +676,19 @@ class AutoRoundQuantizer(HfQuantizer):for n, layer in tqdm(layers, desc=message, total=len(layers),leave=True):layer.post_init()+ return model-+ def xpu_post_init(self, model):+ message = "Repacking to XPU format"+ from auto_round_extension.ipex import ipex_qlinear_classes+ cpu_layers = tuple(list(ipex_qlinear_classes))+ layers = [] ## ipex post_init will add one more layer+ for n, m in model.named_modules():+ if isinstance(m, cpu_layers):+ layers.append((n, m))+ for n, layer in tqdm(layers, desc=message, total=len(layers),+ leave=True):+ layer.post_init()return modeldef repack_marlin(self, model):@@ -783,6 +800,8 @@ class AutoRoundQuantizer(HfQuantizer):# there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.if self.target_device == "cpu":model = self.cpu_post_init(model)+ elif self.target_device == "xpu":+ model = self.xpu_post_init(model)return model@@ -816,5 +835,3 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer--diff --git a/auto_round/backend.py b/auto_round/backend.pyindex 07e19427c..fd43af788 100644--- a/auto_round/backend.py+++ b/auto_round/backend.py@@ -137,7 +137,7 @@ BackendInfos['gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],packing_format="triton_zp+-1",bits=[2, 3, 4, 8], group_size=None,priority=1, feature_checks=[feature_multiply_checker_32],- alias=["auto_round:auto_gptq:cuda","auto_gptq:cuda","auto_round:gptq:cuda"],+ alias=["auto_round:auto_gptq:cuda", "auto_gptq:cuda", "auto_round:gptq:cuda"],convertable_format=["triton_zp+-1"],requirements=["auto-gptq>=0.7.1"])@@ -169,31 +169,31 @@ BackendInfos['auto_round:qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, Fal)BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True],- packing_format="awq",- bits=[2, 4, 8], group_size=None,- priority=0 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- requirements=["intel-extension-for-transformers"]- )+ packing_format="awq",+ bits=[2, 4, 8], group_size=None,+ priority=0 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ requirements=["intel-extension-for-transformers"]+ )-BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu"], sym=[True, False],+BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],packing_format="ipex_gptq",bits=[4], group_size=None,priority=5 if "intel" in get_cpu_manufacturer() else 5,feature_checks=[],convertable_format=["triton_zp+-1"],- requirements=["intel-extension-for-pytorch>=2.4"]+ requirements=["intel-extension-for-pytorch>=2.5"])-BackendInfos['auto_round:ipex_awq'] = BackendInfo(device=["cpu"], sym=[True, False],- packing_format="ipex_awq",- bits=[4], group_size=None,- priority=5 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- ##convertable_format=["triton_zp+-1", "awq"],- convertable_format=["awq"],- requirements=["intel-extension-for-pytorch>=2.4"]- )+BackendInfos['auto_round:ipex_awq_xpu'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],+ packing_format="ipex_awq",+ bits=[4], group_size=None,+ priority=1,+ feature_checks=[],+ ##convertable_format=["triton_zp+-1", "awq"],+ convertable_format=["awq"],+ requirements=["intel-extension-for-pytorch>=2.6"]+ )# BackendInfos['auto_round:marlin'] = BackendInfo(device=["gpu"], sym=[True],# packing_format="marlin",@@ -554,4 +554,3 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_freverse=True)return supported_backends[0]-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex da9082cb9..0f6f544ce 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -588,6 +588,8 @@ def detect_device(device=None):elif is_optimum_habana_available(): # pragma: no coverdevice = torch.device("hpu")# logger.info("Using HPU device")+ elif torch.xpu.is_available(): # pragma: no cover+ device = torch.device("xpu")# Use CPU as a fallbackelse:device = torch.device("cpu")@@ -1159,9 +1161,13 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nondef get_device_and_parallelism(device):from auto_round.utils import detect_devicedevices = device.replace(" ", "").split(',')- if all(s.isdigit() for s in devices) and len(devices) > 1:+ if all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():device = "cuda"parallelism = True+ elif all(s.isdigit() for s in devices) and len(devices) > 1 and torch.xpu.is_available():+ device = "xpu"+ parallelism = False+ # pragma: no coverelif device == "auto":device = detect_device(device)parallelism = Truediff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pyindex 46a42d898..d224234e7 100644--- a/auto_round_extension/ipex/qlinear_ipex_awq.py+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py@@ -6,9 +6,7 @@ class QuantLinear(nn.Module):def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):super().__init__()assert w_bit == 4, "Only 4 bit are supported for now."-- self.use_bf16 = True # Intel platform support bf16 even without amx.-+ self.compute_dtype = torch.float16 if torch.xpu.is_available() else torch.bfloat16self.in_features = in_featuresself.out_features = out_featuresself.w_bit = w_bit@@ -32,13 +30,13 @@ class QuantLinear(nn.Module):"scales",torch.zeros((in_features // self.group_size, out_features),- dtype=torch.bfloat16 if self.use_bf16 else torch.float32,+ dtype=self.compute_dtype,device=dev,))if bias:self.register_buffer("bias",- torch.zeros((out_features), dtype=torch.bfloat16 if self.use_bf16 else torch.float32, device=dev),+ torch.zeros((out_features), dtype=self.compute_dtype, device=dev),)else:self.register_buffer(@@ -49,19 +47,21 @@ class QuantLinear(nn.Module):self.register_buffer("qweight", qweight)def post_init(self):- assert self.qweight.device.type == "cpu"- from packaging import version- from auto_round.auto_quantizer import importlib_metadata- from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear- version_ipex = version.parse(importlib_metadata.version("intel_extension_for_pytorch"))- if version_ipex < version.parse("2.6.0"):- self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \- self.in_features, self.out_features, None, self.bias, \- self.group_size, None, 0, 1)- else:- self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \- self.in_features, self.out_features, None, self.bias, \- self.group_size, None, 1, 0)+ assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"+ import intel_extension_for_pytorch as ipex++ self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(self.qweight,+ self.scales,+ self.qzeros, \+ self.in_features,+ self.out_features,+ None,+ self.bias, \+ self.group_size,+ None,+ 1,+ 0+ )@classmethoddef from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pyindex c69dd06c6..c486638ae 100644--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py@@ -95,19 +95,29 @@ class QuantLinear(nn.Module):self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)def post_init(self):- assert self.qweight.device.type == "cpu"+ assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"# if not self.training and IPEX_AVAILABLE:if not self.training:- from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear+ import intel_extension_for_pytorch as ipex+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear, \+ QuantDtype, QuantMethodfrom packaging import versionfrom auto_round.utils import get_library_versionipex_version = get_library_version("intel_extension_for_pytorch")if version.parse(ipex_version) >= version.parse("2.5"):- self.ipex_linear = WeightOnlyQuantizedLinear.from_weight(self.qweight, self.scales, self.qzeros, \- self.infeatures, self.outfeatures, None,- self.bias, \- self.group_size, self.g_idx, 0, 0)+ self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(self.qweight,+ self.scales,+ self.qzeros, \+ self.infeatures,+ self.outfeatures,+ None,+ self.bias, \+ self.group_size,+ None,+ ipex.llm.quantization.QuantMethod.GPTQ_GEMM,+ ipex.llm.quantization.QuantDtype.INT4+ )else:import intel_extension_for_pytorch as ipexfrom intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_lineardiff --git a/test/test_export.py b/test/test_export.pyindex 238f41158..0394886bf 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -118,14 +118,14 @@ class TestAutoRound(unittest.TestCase):quantization_config = AutoRoundConfig(backend=device)- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device, quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,+ quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_awq_format(self):bits, group_size, sym = 4, 128, Falseautoround = AutoRound(@@ -142,12 +142,13 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")- try:- import awq- except:- return+from auto_round import AutoRoundConfig- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ quantization_config = AutoRoundConfig(+ backend="cpu"+ )+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",+ quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -170,7 +171,7 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \- format="auto_awq", model_path="facebook/opt-125m")+ format="auto_awq", model_path="facebook/opt-125m")try:import awqexcept:diff --git a/test_xpu/test_autoround.py b/test_xpu/test_autoround.pynew file mode 100644index 000000000..d0ab1dd99--- /dev/null+++ b/test_xpu/test_autoround.py@@ -0,0 +1,106 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRoundConfig+from auto_round import AutoRound++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(3):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundXPU(unittest.TestCase):+ @classmethod+ def setUpClass(self):++ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+ pass++ def test_gptq_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True++ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path)++ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(+ backend="auto"+ )++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map="auto", quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++++ def test_awq_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="xpu")+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")++ quantized_model_path = "./saved"+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(+ backend="auto"+ )++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map="auto", quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++if __name__ == "__main__":+ unittest.main()\ No newline at end of file