Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support autoawq format (#115)

yintong-lub08b967b70da
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index 6b0fd04be..ec3047930 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -16,5 +16,5 @@ from .register import EXPORT_FORMAT
from auto_round.export.export_to_autogptq.export import save_quantized_as_autogptq
from .export_to_itrex import save_quantized_as_itrex, QuantConfig
from .export_to_autoround.export import save_quantized_as_autoround
-
+from .export_to_awq.export import save_quantized_as_autoawq
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index bc1cee2d2..7a0061d17 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -41,16 +41,12 @@ def check_neq_config(config, data_type, bits, group_size, sym):
Returns:
list: A list of strings indicating which configuration parameters do not match.
"""
- res = []
- if data_type != config["data_type"]:
- res.append("data_type")
- if bits != config["bits"]:
- res.append("bits")
- if group_size != config["group_size"]:
- res.append("group_size")
- if sym != config["sym"]:
- res.append("sym")
- return res
+ expected_config = {"data_type": data_type,
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym
+ }
+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
@@ -89,6 +85,7 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
disable_exllamav2 = True
disable_exllamav1 = True
+
from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
QuantLinear = dynamically_import_QuantLinear(
use_triton=use_triton,
@@ -119,16 +116,22 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):
Raises:
AssertionError: If the backend is not supported.
"""
- if "auto_round" in backend:
+ if "auto_round" in backend and "awq" not in backend:
##only support triton and exllamav2
if not ("triton" in backend or "exllamav2" in backend):
logger.warning_once(f"autoround format does not support {backend}, try to pack with autogptq")
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
from auto_round_extension.cuda.qliner_triton import QuantLinear
return QuantLinear
+ elif "awq" in backend:
+ try:
+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401
+ return WQLinear_GEMM
+ except:
+ logger.error("autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")
+ return
elif "gptq" in backend:
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
-
else:
assert False, f"only support gptq and autoround backend"
@@ -160,7 +163,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
backend = "autoround:exllamav2"
backend = backend.replace("autoround", "auto_round")
backend = backend.replace("auto-round", "auto_round")
- if not ("triton" in backend or "exllamav2" in backend):
+ if not ("triton" in backend or "exllamav2" in backend or "awq" in backend):
logger.info(f"autoround format does not support {backend}, try to pack with autogptq")
backend = backend.replace("auto_round", "auto_gptq")
@@ -174,7 +177,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
quantization_config["quant_method"] = "intel/auto-round"
- quantization_config["backend"] = backend
+ if "awq" not in backend:
+ quantization_config["backend"] = backend
extra_config = {}
for layer_name in layer_config:
if layer_name not in layer_names_in_block and layer_config[layer_name]["bits"] <= 8: ##lm head
@@ -198,9 +202,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
with tctl.threadpool_limits(limits=1):
+ modules_to_not_convert = []
for name in layer_config.keys():
config = kwargs["layer_config"][name]
if config["bits"] > 8:
+ if "awq" in backend:
+ modules_to_not_convert.append(name)
continue
logger.info(f"packing {name}")
@@ -224,33 +231,55 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
out_features = layer.weight.shape[1]
bias = layer.bias is not None and torch.any(layer.bias)
- new_layer = QuantLinear( ##pylint: disable=E1123
+ if "awq" not in backend:
+ new_layer = QuantLinear( ##pylint: disable=E1123
bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
-
- new_layer.device = device
- set_module(model, name, new_layer)
- qlayer = new_layer
- scale = layer_config[name]["scale"]
- zero = layer_config[name]["zp"]
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- ##force to float32 to be compatible with torch 2.0
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
- sig = inspect.signature(qlayer.pack)
- param_count = len(sig.parameters)
- if param_count == 2:
- qlayer.pack(layer, scale)
+ )
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer_config[name]["scale"]
+ zero = layer_config[name]["zp"]
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ ##force to float32 to be compatible with torch 2.0
+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
+ sig = inspect.signature(qlayer.pack)
+ param_count = len(sig.parameters)
+ if param_count == 2:
+ qlayer.pack(layer, scale)
+ else:
+ qlayer.pack(layer, scale, zero, None)
+ qlayer.to(device)
else:
- qlayer.pack(layer, scale, zero, None)
- qlayer.to(device)
+ from awq.utils.utils import clear_memory # pylint: disable=E0401
+ scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)
+ scale = scale.t().contiguous()
+ zp = zp.t().contiguous()
+ if bits != 4:
+ logger.error("AutoAWQ format only supports 4-bits quantization.")
+ qlayer = QuantLinear.from_linear(
+ linear=layer,
+ w_bit=bits,
+ group_size=group_size,
+ init_only=False,
+ scales=scale,
+ zeros=zp,
+ )
+ qlayer.to(device)
+ set_module(model, name, qlayer)
+ clear_memory()
+
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
tokenizer = kwargs["tokenizer"]
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
- save(model, output_dir, safe_serialization=safe_serialization)
+ if "awq" not in backend:
+ save(model, output_dir, safe_serialization=safe_serialization)
+ else:
+ save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)
def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
@@ -282,3 +311,41 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
+def save_awq(
+ model: nn.Module,
+ save_dir: str,
+ max_shard_size: str = "5GB",
+ safe_serialization: bool = True,
+ modules_to_not_convert: list = [],
+):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ os.makedirs(save_dir, exist_ok=True)
+ quantization_config = model.config.quantization_config
+ model.config.quantization_config["quant_method"] = "awq"
+ model.config.quantization_config["modules_to_not_convert"] = None if not modules_to_not_convert \
+ else modules_to_not_convert
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_file = "quantization_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(quantization_config, f, indent=2)
+
+
diff --git a/auto_round/export/export_to_awq/__init__.py b/auto_round/export/export_to_awq/__init__.py
new file mode 100644
index 000000000..0bdb4d35e
--- /dev/null
+++ b/auto_round/export/export_to_awq/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .export import save_quantized_as_autoawq
+
+
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
new file mode 100644
index 000000000..79aab82e9
--- /dev/null
+++ b/auto_round/export/export_to_awq/export.py
@@ -0,0 +1,227 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# MIT License
+# Copyright (c) 2023 MIT HAN Lab
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+
+import os
+from os.path import isdir, isfile, join
+import torch
+import torch.nn as nn
+from auto_round.export.register import register_format
+from auto_round.utils import convert_dtype_torch2str_hf, logger
+import copy
+import json
+from typing import Dict, List, Optional, Union
+
+
+@register_format("auto_awq")
+def save_quantized_as_autoawq(output_dir, model_path, inplace=True, **kwargs):
+ """Export the model to autogptq format to easily leverage cuda kernel."""
+ model = kwargs["model"]
+ layer_config = kwargs["layer_config"]
+ sym = kwargs["sym"]
+ bits = kwargs["bits"]
+ group_size = kwargs["group_size"]
+ iters = kwargs["iters"]
+ lr = kwargs["lr"]
+ minmax_lr = kwargs["minmax_lr"]
+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]
+ enable_quanted_input = kwargs["enable_quanted_input"]
+ scale_dtype = kwargs["scale_dtype"]
+ tokenizer = kwargs["tokenizer"]
+
+ logger.info("Saving quantized model to auto_awq format")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ ##check module quantized in block, this may have bug for mixed precision quantization
+ modules_to_not_convert = []
+ if inplace:
+ compressed_model = model.to("cpu")
+ else:
+ compressed_model = copy.deepcopy(model.to("cpu"))
+
+ try:
+ from awq import AutoAWQForCausalLM # pylint: disable=E0401
+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401
+ from awq.utils.utils import clear_memory # pylint: disable=E0401
+ except:
+ logger.error("autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")
+
+ q_linear_module = WQLinear_GEMM
+ awq_model = AutoAWQForCausalLM.from_pretrained(model_path)
+ self_modules = awq_model.get_model_layers(compressed_model)
+ del awq_model # release memory
+ for i in range(len(self_modules)):
+ module = self_modules[i]
+ named_linears = get_named_linears(module)
+ for name, linear_layer in named_linears.items():
+ key = get_module_name(compressed_model, linear_layer)
+ logger.info(f"packing {name}")
+ config = layer_config[key]
+ if config["bits"] > 8:
+ modules_to_not_convert.append(name)
+ continue
+ config["zp"] = config["zp"].to(torch.float32)
+ scale, zp = config["scale"], config["zp"]
+ scale = scale.t().contiguous()
+ zp = zp.t().contiguous()
+ q_linear = q_linear_module.from_linear(
+ linear=linear_layer,
+ w_bit=bits,
+ group_size=group_size,
+ init_only=False,
+ scales=scale,
+ zeros=zp,
+ )
+ linear_layer.cpu()
+ q_linear.to(next(module.parameters()).device)
+ set_op_by_name(module, name, q_linear)
+ clear_memory()
+
+ quant_config = {}
+ quant_config["quant_method"] = "awq"
+ quant_config["modules_to_not_convert"] = None
+ quant_config["version"] = "gemm"
+ quant_config["iters"] = iters
+ quant_config["lr"] = lr
+ quant_config["minmax_lr"] = minmax_lr
+ quant_config["enable_minmax_tuning"] = enable_minmax_tuning
+ quant_config["enable_quanted_input"] = enable_quanted_input
+ quant_config["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)
+ quant_config["sym"] = sym
+ quant_config["bits"] = bits
+ quant_config["group_size"] = group_size
+ quant_config["zero_point"] = not sym
+
+ save_quantized(compressed_model, save_dir=output_dir, quant_config=quant_config)
+
+
+from safetensors.torch import save_file
+from transformers.modeling_utils import shard_checkpoint
+
+
+def save_quantized(
+ model,
+ save_dir,
+ quant_config,
+ safetensors=True,
+ shard_size="5GB",
+):
+ save_dir = save_dir[:-1] if save_dir[-1] == "/" else save_dir
+
+ # Save model
+ class EmptyModule(nn.Module):
+ def __init__(self):
+ super(EmptyModule, self).__init__()
+
+ def forward(self, x):
+ return x
+
+ # Save model and config files with empty state dict
+ awq_quant_config = {
+ "quant_method": "awq",
+ "zero_point": quant_config["zero_point"],
+ "group_size": quant_config["group_size"],
+ "bits": quant_config["bits"],
+ "version": "gemm",
+ "modules_to_not_convert": quant_config["modules_to_not_convert"],
+ }
+
+ model.config.quantization_config = awq_quant_config
+ model.generation_config.do_sample = True
+ model.save_pretrained(save_dir, state_dict=EmptyModule().state_dict())
+
+ # Remove empty state dict
+ default_paths = [
+ f"{save_dir}/model.safetensors",
+ f"{save_dir}/pytorch_model.bin",
+ ]
+ for path in default_paths:
+ if os.path.exists(path):
+ os.remove(path)
+
+ # model_name has no extension, add it when saving state_dict
+ model_name = "model.safetensors" if safetensors else "pytorch_model.bin"
+
+ # shard checkpoint into chunks (10GB default)
+ shards, index = shard_checkpoint(model.state_dict(), max_shard_size=shard_size, weights_name=model_name)
+
+ for shard_file, shard in shards.items():
+ if safetensors:
+ # safetensors must be in the same memory, so we duplicate and use contiguous memory
+ shard = {k: v.clone().contiguous() for k, v in shard.items()}
+ save_file(shard, os.path.join(save_dir, shard_file), metadata={"format": "pt"})
+ else:
+ torch.save(shard, os.path.join(save_dir, shard_file))
+
+ # save shard index
+ if index is not None:
+ with open(f"{save_dir}/{model_name}.index.json", "w+") as file:
+ file.write(json.dumps(index, indent=4))
+
+ # save quantize_config
+ with open(join(save_dir, "quantization_config.json"), "w", encoding="utf-8") as f:
+ json.dump(quant_config, f, indent=2)
+
+
+def get_named_linears(module):
+ """Get the name, linear_op pairs of a given module.
+ Args:
+ module: A module to be searched.
+ """
+ return {name: m for name, m in module.named_modules() if isinstance(m, torch.nn.Linear)}
+
+
+def set_op_by_name(layer, name, new_module):
+ levels = name.split(".")
+ if len(levels) > 1:
+ mod_ = layer
+ for l_idx in range(len(levels) - 1):
+ if levels[l_idx].isdigit():
+ mod_ = mod_[int(levels[l_idx])]
+ else:
+ mod_ = getattr(mod_, levels[l_idx])
+ setattr(mod_, levels[-1], new_module)
+ else:
+ setattr(layer, name, new_module)
+
+
+def get_module_name(model, module_to_find):
+ """Get the name of a given module in a model.
+ Args:
+ model: The model.
+ module_to_find: A module to be found.
+ Returns:
+ name: The corresponding name of the given module.
+ """
+ for name, module in model.named_modules():
+ if module is module_to_find:
+ return name
+ return None
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 11c69ac19..a3f2a5e18 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -812,6 +812,13 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
else:
from auto_round_extension.hpu.qlinear_hpu import QuantLinear
return QuantLinear
+ if "awq" in backend: # pragma: no cover
+ try:
+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401
+ except:
+ raise ImportError("autoawq is required. Please install it by 'pip install autoawq' to \
+ support auto_awq format.")
+ return WQLinear_GEMM
if bits == 4 and exllama2_available and "exllamav2" in backend:
from auto_round_extension.cuda.qliner_exllamav2 import QuantLinear
elif bits == 4 and "exllamav2" in backend:
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index e8bb8ae79..733447600 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -317,7 +317,7 @@ if __name__ == '__main__':
deployment_device = args.deployment_device.split(',')
gpu_formats = []
for item in deployment_device:
- if "gpu" in item or "auto_gptq" in item or "auto_round" in item:
+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item or "auto_awq" in item:
gpu_formats.append(item)
if 'gpu' in deployment_device:
@@ -336,6 +336,9 @@ if __name__ == '__main__':
elif "gptq" in gpu_format:
eval_folder = f'{export_dir}-gpu'
autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ elif "auto_awq" in gpu_format: # pragma: no cover
+ eval_folder = f'{export_dir}-awq'
+ autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)
if 'xpu' in deployment_device:
autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
diff --git a/test/test_export.py b/test/test_export.py
index 7a747837b..aa23ff589 100644
--- a/test/test_export.py
+++ b/test/test_export.py
@@ -120,6 +120,67 @@ class TestAutoRound(unittest.TestCase):
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_autoround_awq_format(self):
+ try:
+ import awq
+ except:
+ return
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")
+
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+ def test_autoawq_format(self):
+ try:
+ import awq
+ except:
+ return
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \
+ format="auto_awq", model_path="facebook/opt-125m")
+
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
# def test_autoround_marlin_format(self):
# if not torch.cuda.is_available():
# return
@@ -150,5 +211,4 @@ class TestAutoRound(unittest.TestCase):
# inputs = tokenizer(text, return_tensors="pt").to(model.device)
# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
# shutil.rmtree("./saved", ignore_errors=True)
- #
-
+ #
\ No newline at end of file