PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support autoawq format (#115)
yintong-lub08b967b70da
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex 6b0fd04be..ec3047930 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -16,5 +16,5 @@ from .register import EXPORT_FORMATfrom auto_round.export.export_to_autogptq.export import save_quantized_as_autogptqfrom .export_to_itrex import save_quantized_as_itrex, QuantConfigfrom .export_to_autoround.export import save_quantized_as_autoround-+from .export_to_awq.export import save_quantized_as_autoawqdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex bc1cee2d2..7a0061d17 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -41,16 +41,12 @@ def check_neq_config(config, data_type, bits, group_size, sym):Returns:list: A list of strings indicating which configuration parameters do not match."""- res = []- if data_type != config["data_type"]:- res.append("data_type")- if bits != config["bits"]:- res.append("bits")- if group_size != config["group_size"]:- res.append("group_size")- if sym != config["sym"]:- res.append("sym")- return res+ expected_config = {"data_type": data_type,+ "bits": bits,+ "group_size": group_size,+ "sym": sym+ }+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):@@ -89,6 +85,7 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):disable_exllamav2 = Truedisable_exllamav1 = True+from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401QuantLinear = dynamically_import_QuantLinear(use_triton=use_triton,@@ -119,16 +116,22 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):Raises:AssertionError: If the backend is not supported."""- if "auto_round" in backend:+ if "auto_round" in backend and "awq" not in backend:##only support triton and exllamav2if not ("triton" in backend or "exllamav2" in backend):logger.warning_once(f"autoround format does not support {backend}, try to pack with autogptq")return get_autogptq_packing_qlinear(backend, bits, group_size, sym)from auto_round_extension.cuda.qliner_triton import QuantLinearreturn QuantLinear+ elif "awq" in backend:+ try:+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401+ return WQLinear_GEMM+ except:+ logger.error("autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")+ returnelif "gptq" in backend:return get_autogptq_packing_qlinear(backend, bits, group_size, sym)-else:assert False, f"only support gptq and autoround backend"@@ -160,7 +163,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exbackend = "autoround:exllamav2"backend = backend.replace("autoround", "auto_round")backend = backend.replace("auto-round", "auto_round")- if not ("triton" in backend or "exllamav2" in backend):+ if not ("triton" in backend or "exllamav2" in backend or "awq" in backend):logger.info(f"autoround format does not support {backend}, try to pack with autogptq")backend = backend.replace("auto_round", "auto_gptq")@@ -174,7 +177,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]quantization_config["quant_method"] = "intel/auto-round"- quantization_config["backend"] = backend+ if "awq" not in backend:+ quantization_config["backend"] = backendextra_config = {}for layer_name in layer_config:if layer_name not in layer_names_in_block and layer_config[layer_name]["bits"] <= 8: ##lm head@@ -198,9 +202,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif len(extra_config) > 0:quantization_config["extra_config"] = extra_configwith tctl.threadpool_limits(limits=1):+ modules_to_not_convert = []for name in layer_config.keys():config = kwargs["layer_config"][name]if config["bits"] > 8:+ if "awq" in backend:+ modules_to_not_convert.append(name)continuelogger.info(f"packing {name}")@@ -224,33 +231,55 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exout_features = layer.weight.shape[1]bias = layer.bias is not None and torch.any(layer.bias)- new_layer = QuantLinear( ##pylint: disable=E1123+ if "awq" not in backend:+ new_layer = QuantLinear( ##pylint: disable=E1123bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype- )-- new_layer.device = device- set_module(model, name, new_layer)- qlayer = new_layer- scale = layer_config[name]["scale"]- zero = layer_config[name]["zp"]- # so far can only pack layer on CPU- qlayer.to("cpu")- ##force to float32 to be compatible with torch 2.0- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)- sig = inspect.signature(qlayer.pack)- param_count = len(sig.parameters)- if param_count == 2:- qlayer.pack(layer, scale)+ )+ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer_config[name]["scale"]+ zero = layer_config[name]["zp"]+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ sig = inspect.signature(qlayer.pack)+ param_count = len(sig.parameters)+ if param_count == 2:+ qlayer.pack(layer, scale)+ else:+ qlayer.pack(layer, scale, zero, None)+ qlayer.to(device)else:- qlayer.pack(layer, scale, zero, None)- qlayer.to(device)+ from awq.utils.utils import clear_memory # pylint: disable=E0401+ scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)+ scale = scale.t().contiguous()+ zp = zp.t().contiguous()+ if bits != 4:+ logger.error("AutoAWQ format only supports 4-bits quantization.")+ qlayer = QuantLinear.from_linear(+ linear=layer,+ w_bit=bits,+ group_size=group_size,+ init_only=False,+ scales=scale,+ zeros=zp,+ )+ qlayer.to(device)+ set_module(model, name, qlayer)+ clear_memory()+if hasattr(model, "config"):model.config.quantization_config = quantization_configtokenizer = kwargs["tokenizer"]if tokenizer is not None:tokenizer.save_pretrained(output_dir)- save(model, output_dir, safe_serialization=safe_serialization)+ if "awq" not in backend:+ save(model, output_dir, safe_serialization=safe_serialization)+ else:+ save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):@@ -282,3 +311,41 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri+def save_awq(+ model: nn.Module,+ save_dir: str,+ max_shard_size: str = "5GB",+ safe_serialization: bool = True,+ modules_to_not_convert: list = [],+):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ os.makedirs(save_dir, exist_ok=True)+ quantization_config = model.config.quantization_config+ model.config.quantization_config["quant_method"] = "awq"+ model.config.quantization_config["modules_to_not_convert"] = None if not modules_to_not_convert \+ else modules_to_not_convert+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_file = "quantization_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(quantization_config, f, indent=2)++diff --git a/auto_round/export/export_to_awq/__init__.py b/auto_round/export/export_to_awq/__init__.pynew file mode 100644index 000000000..0bdb4d35e--- /dev/null+++ b/auto_round/export/export_to_awq/__init__.py@@ -0,0 +1,17 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from .export import save_quantized_as_autoawq++diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pynew file mode 100644index 000000000..79aab82e9--- /dev/null+++ b/auto_round/export/export_to_awq/export.py@@ -0,0 +1,227 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.+# MIT License+# Copyright (c) 2023 MIT HAN Lab+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+++import os+from os.path import isdir, isfile, join+import torch+import torch.nn as nn+from auto_round.export.register import register_format+from auto_round.utils import convert_dtype_torch2str_hf, logger+import copy+import json+from typing import Dict, List, Optional, Union+++@register_format("auto_awq")+def save_quantized_as_autoawq(output_dir, model_path, inplace=True, **kwargs):+ """Export the model to autogptq format to easily leverage cuda kernel."""+ model = kwargs["model"]+ layer_config = kwargs["layer_config"]+ sym = kwargs["sym"]+ bits = kwargs["bits"]+ group_size = kwargs["group_size"]+ iters = kwargs["iters"]+ lr = kwargs["lr"]+ minmax_lr = kwargs["minmax_lr"]+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]+ enable_quanted_input = kwargs["enable_quanted_input"]+ scale_dtype = kwargs["scale_dtype"]+ tokenizer = kwargs["tokenizer"]++ logger.info("Saving quantized model to auto_awq format")+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ ##check module quantized in block, this may have bug for mixed precision quantization+ modules_to_not_convert = []+ if inplace:+ compressed_model = model.to("cpu")+ else:+ compressed_model = copy.deepcopy(model.to("cpu"))++ try:+ from awq import AutoAWQForCausalLM # pylint: disable=E0401+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401+ from awq.utils.utils import clear_memory # pylint: disable=E0401+ except:+ logger.error("autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")++ q_linear_module = WQLinear_GEMM+ awq_model = AutoAWQForCausalLM.from_pretrained(model_path)+ self_modules = awq_model.get_model_layers(compressed_model)+ del awq_model # release memory+ for i in range(len(self_modules)):+ module = self_modules[i]+ named_linears = get_named_linears(module)+ for name, linear_layer in named_linears.items():+ key = get_module_name(compressed_model, linear_layer)+ logger.info(f"packing {name}")+ config = layer_config[key]+ if config["bits"] > 8:+ modules_to_not_convert.append(name)+ continue+ config["zp"] = config["zp"].to(torch.float32)+ scale, zp = config["scale"], config["zp"]+ scale = scale.t().contiguous()+ zp = zp.t().contiguous()+ q_linear = q_linear_module.from_linear(+ linear=linear_layer,+ w_bit=bits,+ group_size=group_size,+ init_only=False,+ scales=scale,+ zeros=zp,+ )+ linear_layer.cpu()+ q_linear.to(next(module.parameters()).device)+ set_op_by_name(module, name, q_linear)+ clear_memory()++ quant_config = {}+ quant_config["quant_method"] = "awq"+ quant_config["modules_to_not_convert"] = None+ quant_config["version"] = "gemm"+ quant_config["iters"] = iters+ quant_config["lr"] = lr+ quant_config["minmax_lr"] = minmax_lr+ quant_config["enable_minmax_tuning"] = enable_minmax_tuning+ quant_config["enable_quanted_input"] = enable_quanted_input+ quant_config["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)+ quant_config["sym"] = sym+ quant_config["bits"] = bits+ quant_config["group_size"] = group_size+ quant_config["zero_point"] = not sym++ save_quantized(compressed_model, save_dir=output_dir, quant_config=quant_config)+++from safetensors.torch import save_file+from transformers.modeling_utils import shard_checkpoint+++def save_quantized(+ model,+ save_dir,+ quant_config,+ safetensors=True,+ shard_size="5GB",+):+ save_dir = save_dir[:-1] if save_dir[-1] == "/" else save_dir++ # Save model+ class EmptyModule(nn.Module):+ def __init__(self):+ super(EmptyModule, self).__init__()++ def forward(self, x):+ return x++ # Save model and config files with empty state dict+ awq_quant_config = {+ "quant_method": "awq",+ "zero_point": quant_config["zero_point"],+ "group_size": quant_config["group_size"],+ "bits": quant_config["bits"],+ "version": "gemm",+ "modules_to_not_convert": quant_config["modules_to_not_convert"],+ }++ model.config.quantization_config = awq_quant_config+ model.generation_config.do_sample = True+ model.save_pretrained(save_dir, state_dict=EmptyModule().state_dict())++ # Remove empty state dict+ default_paths = [+ f"{save_dir}/model.safetensors",+ f"{save_dir}/pytorch_model.bin",+ ]+ for path in default_paths:+ if os.path.exists(path):+ os.remove(path)++ # model_name has no extension, add it when saving state_dict+ model_name = "model.safetensors" if safetensors else "pytorch_model.bin"++ # shard checkpoint into chunks (10GB default)+ shards, index = shard_checkpoint(model.state_dict(), max_shard_size=shard_size, weights_name=model_name)++ for shard_file, shard in shards.items():+ if safetensors:+ # safetensors must be in the same memory, so we duplicate and use contiguous memory+ shard = {k: v.clone().contiguous() for k, v in shard.items()}+ save_file(shard, os.path.join(save_dir, shard_file), metadata={"format": "pt"})+ else:+ torch.save(shard, os.path.join(save_dir, shard_file))++ # save shard index+ if index is not None:+ with open(f"{save_dir}/{model_name}.index.json", "w+") as file:+ file.write(json.dumps(index, indent=4))++ # save quantize_config+ with open(join(save_dir, "quantization_config.json"), "w", encoding="utf-8") as f:+ json.dump(quant_config, f, indent=2)+++def get_named_linears(module):+ """Get the name, linear_op pairs of a given module.+ Args:+ module: A module to be searched.+ """+ return {name: m for name, m in module.named_modules() if isinstance(m, torch.nn.Linear)}+++def set_op_by_name(layer, name, new_module):+ levels = name.split(".")+ if len(levels) > 1:+ mod_ = layer+ for l_idx in range(len(levels) - 1):+ if levels[l_idx].isdigit():+ mod_ = mod_[int(levels[l_idx])]+ else:+ mod_ = getattr(mod_, levels[l_idx])+ setattr(mod_, levels[-1], new_module)+ else:+ setattr(layer, name, new_module)+++def get_module_name(model, module_to_find):+ """Get the name of a given module in a model.+ Args:+ model: The model.+ module_to_find: A module to be found.+ Returns:+ name: The corresponding name of the given module.+ """+ for name, module in model.named_modules():+ if module is module_to_find:+ return name+ return Nonediff --git a/auto_round/utils.py b/auto_round/utils.pyindex 11c69ac19..a3f2a5e18 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -812,6 +812,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):else:from auto_round_extension.hpu.qlinear_hpu import QuantLinearreturn QuantLinear+ if "awq" in backend: # pragma: no cover+ try:+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401+ except:+ raise ImportError("autoawq is required. Please install it by 'pip install autoawq' to \+ support auto_awq format.")+ return WQLinear_GEMMif bits == 4 and exllama2_available and "exllamav2" in backend:from auto_round_extension.cuda.qliner_exllamav2 import QuantLinearelif bits == 4 and "exllamav2" in backend:diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex e8bb8ae79..733447600 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -317,7 +317,7 @@ if __name__ == '__main__':deployment_device = args.deployment_device.split(',')gpu_formats = []for item in deployment_device:- if "gpu" in item or "auto_gptq" in item or "auto_round" in item:+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item or "auto_awq" in item:gpu_formats.append(item)if 'gpu' in deployment_device:@@ -336,6 +336,9 @@ if __name__ == '__main__':elif "gptq" in gpu_format:eval_folder = f'{export_dir}-gpu'autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ elif "auto_awq" in gpu_format: # pragma: no cover+ eval_folder = f'{export_dir}-awq'+ autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)if 'xpu' in deployment_device:autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,diff --git a/test/test_export.py b/test/test_export.pyindex 7a747837b..aa23ff589 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -120,6 +120,67 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)++ def test_autoround_awq_format(self):+ try:+ import awq+ except:+ return+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")++ from auto_round.auto_quantizer import AutoHfQuantizer+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+++ def test_autoawq_format(self):+ try:+ import awq+ except:+ return+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \+ format="auto_awq", model_path="facebook/opt-125m")++ from auto_round.auto_quantizer import AutoHfQuantizer+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+# def test_autoround_marlin_format(self):# if not torch.cuda.is_available():# return@@ -150,5 +211,4 @@ class TestAutoRound(unittest.TestCase):# inputs = tokenizer(text, return_tensors="pt").to(model.device)# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))# shutil.rmtree("./saved", ignore_errors=True)- #-+ #\ No newline at end of file