PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support llmcompressor format (#646)
Xin Hebba62c2bb207
diff --git a/README.md b/README.mdindex 6ed389084..604bbdcc5 100644--- a/README.md+++ b/README.md@@ -277,6 +277,8 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widelyadopted within the community, **only 4-bits quantization is supported**.+**llmcompressor Format**: This format is for reusing llmcompressor format, **only INT8 W8A8 dynamic quantization is supported**.+**GGUF** Format: Experimental feature. This format is well-suited for CPU devices and is widely adopted by the community.### Quantization Costsdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex e1fa1d238..9d265072d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -498,31 +498,20 @@ class AutoRound(object):if self.group_size == 0 and "fp8" not in self.data_type:logger.warning("group_size of 0 is not supported for data_type other than fp8 ")- def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):- """Quantizes the model and saves it in the specified format(s).+ def parse_format_to_list(self, format: str) -> list:+ """Parses the format string into a list of formats.- This function checks the validity of the requested format(s), quantizes- the model accordingly, and saves it to the specified output directory.- If multiple formats are provided, the model is saved separately for each format.+ This method checks the requested format(s) against the model's+ quantization settings and adjusts them if necessary. It ensures that+ the formats are compatible with the model's data type, bit width,+ and activation quantization settings.Args:- output_dir (str, optional): The directory where the quantized model- will be saved. Defaults to "tmp_autoround".- format (str, optional): The quantization format(s) to use, separated- by commas if multiple. Defaults to "auto_round".- inplace (bool, optional): Whether to modify the model in place if only- one format is used. Defaults to True.- **kwargs: Additional arguments for the quantization and saving process.+ format (str): The requested format(s) for quantization, separated by commas.Returns:- model: A qdq model or packed model based on the configurations- folders: The folder paths where the quantized models are saved.-- Raises:- ValueError: If an unsupported format is specified.+ list: A list of validated and updated formats."""- # Validate and process the specified formats- self.orig_output_dir = output_dir_gguf_args_check(self, format)formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')@@ -561,12 +550,6 @@ class AutoRound(object):)formats = ["auto_round"]- # If multiple formats are specified, enforce inplace=False- if len(formats) > 1:- inplace = False- self.inplace = kwargs.get("inplace", inplace)- kwargs.pop("inplace", None)-# Adjust format settings based on compatibilityfor index in range(len(formats)):format = formats[index]@@ -591,14 +574,98 @@ class AutoRound(object):return [x for x in lst if not (x in seen or seen.add(x))]formats = remove_duplicates(formats)- self.formats = formats+ for format in formats:+ self._check_supported_format(format)+ return formats++ def _check_supported_format(self, format: str) -> bool:+ """Checks if the specified format is supported.++ This method validates the requested format against the model's bit width,+ group size, symmetry, and activation quantization settings. It raises an+ error if the format is incompatible with the current model configuration.++ Args:+ format (str): The requested format for quantization.++ Returns:+ bool: True if the format is supported, False otherwise.+ """+ format = format.replace("q*_", f"q{self.bits}_")+ # only support to export afp8+ if self.act_bits <= 8:+ if "fp8" not in self.act_data_type or self.act_dynamic:+ if format == "llmcompressor":+ bits, group_size, sym, act_bits = 8, -1, True, 8+ assert self.bits == bits and self.group_size == group_size and self.sym == sym \+ and self.act_bits == act_bits and self.act_dynamic, \+ f"Currently only support to export llmcompressor format for dynamic quantized" \+ f" W{self.bits}A{self.act_bits} model, but got bits={self.bits}," \+ f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"+ elif format != "fake":+ logger.warning(+ f"Currently only support to export auto_round format quantized model"+ " with fp8 dtype activation for activation quantization."+ " Change format to fake and save."+ )+ format = "fake"+ else:+ if format != "auto_round":+ logger.warning(+ f"Currently only support to export auto_round format for static W{self.bits}AFP8 model,"+ " change format to auto_round"+ )+ format = "auto_round"++ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):+ logger.error(+ f"datatype<{self.data_type}> not support to export {format} format."+ " Please change export format or `data_type`."+ )+ sys.exit(-1)++ def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):+ """Quantizes the model and saves it in the specified format(s).++ This function checks the validity of the requested format(s), quantizes+ the model accordingly, and saves it to the specified output directory.+ If multiple formats are provided, the model is saved separately for each format.++ Args:+ output_dir (str, optional): The directory where the quantized model+ will be saved. Defaults to "tmp_autoround".+ format (str, optional): The quantization format(s) to use, separated+ by commas if multiple. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place if only+ one format is used. Defaults to True.+ **kwargs: Additional arguments for the quantization and saving process.++ Returns:+ model: A qdq model or packed model based on the configurations+ folders: The folder paths where the quantized models are saved.++ Raises:+ ValueError: If an unsupported format is specified.+ """+ # Validate and process the specified formats+ self.orig_output_dir = output_dir++ # check and update the format based on the current configuration+ format_list = self.parse_format_to_list(format)+ self.formats = format_list++ # If multiple formats are specified, enforce inplace=False+ if len(format_list) > 1:+ inplace = False+ self.inplace = kwargs.get("inplace", inplace)+ kwargs.pop("inplace", None)# Perform model quantizationmodel, _ = self.quantize()- # Save the quantized model in the specified formats+ # Save the quantized model in the specified format_listfolders = []- for format in formats:+ for format in format_list:if "gptq" in format and not self.sym:logger.warning("The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"@@ -2287,31 +2354,7 @@ class AutoRound(object):Returns:object: The compressed model object."""- format = format.replace("q*_", f"q{self.bits}_")- # only support to export afp8- if self.act_bits <= 8:- if "fp8" not in self.act_data_type or self.act_dynamic:- if format != "fake":- logger.warning(- f"Currently only support to export auto_round format quantized model"- " with fp8 dtype activation for activation quantization."- " Change format to fake and save."- )- format = "fake"- else:- if format != "auto_round":- logger.warning(- f"Currently only support to export auto_round format for static W{self.bits}AFP8 model,"- " change format to auto_round"- )- format = "auto_round"-- if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):- logger.error(- f"datatype<{self.data_type}> not support to export {format} format."- " Please change export format or `data_type`."- )- sys.exit(-1)+ self._check_supported_format(format)if self.low_cpu_mem_usage:self.model = self.model.to('cpu')diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex d1012123e..42196d115 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -76,4 +76,9 @@ def _packing_layer_with_autoawq(*args, **kwargs):return pack_layer(*args, **kwargs)+@register_format("llmcompressor")+def _save_quantized_as_llmcompressor(*args, **kwargs):+ from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor++ return save_quantized_as_llmcompressor(*args, **kwargs)diff --git a/auto_round/export/export_to_llmcompressor/__init__.py b/auto_round/export/export_to_llmcompressor/__init__.pynew file mode 100644index 000000000..0d2740cdb--- /dev/null+++ b/auto_round/export/export_to_llmcompressor/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pynew file mode 100644index 000000000..1e1b9a6a1--- /dev/null+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -0,0 +1,56 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++quantization_config = {+"config_groups": {+ "group_0": {+ "input_activations": {+ "actorder": None,+ "block_structure": None,+ "dynamic": True,+ "group_size": None,+ "num_bits": 8,+ "observer": None,+ "observer_kwargs": {},+ "strategy": "token",+ "symmetric": True,+ "type": "int"+ },+ "output_activations": None,+ "targets": [+ "Linear"+ ],+ "weights": {+ "actorder": None,+ "block_structure": None,+ "dynamic": False,+ "group_size": None,+ "num_bits": 8,+ "observer": "minmax",+ "observer_kwargs": {},+ "strategy": "channel",+ "symmetric": True,+ "type": "int"+ }+ }+},+"format": "int-quantized",+"global_compression_ratio": 1.5, # not exactly correct, but a placeholder+"ignore": [],+"kv_cache_scheme": None,+"quant_method": "compressed-tensors",+"quantization_status": "compressed",+"provider": "auto-round" # This is the key to indicate the model provider+}diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pynew file mode 100644index 000000000..a60338752--- /dev/null+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -0,0 +1,73 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+import torch+from auto_round.utils import get_module, logger, set_module, detect_device+from auto_round.wrapper import WrapperWALayer+from auto_round.export.export_to_llmcompressor.config import quantization_config++[email protected]_grad()+def recover_qweight(qdq_weight, scale):+ """+ Recover the quantized weight to its original floating-point representation.++ Args:+ qdq_weight (torch.Tensor): The quantized weight tensor.+ scale (float): The scale factor used for quantization.++ Returns:+ torch.Tensor: The recovered floating-point weight tensor.+ """+ return (qdq_weight / scale).to(torch.int8)+++[email protected]_grad()+def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):+ safe_serialization = kwargs.get("safe_serialization", True)+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)+ if output_dir is not None and os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")++ # save tokenizer, processor+ if output_dir is not None and tokenizer is not None:+ tokenizer.save_pretrained(output_dir)+ if output_dir is not None and processor is not None:+ processor.save_pretrained(output_dir)++ # generate q_weight+ device = detect_device()+ for n, m in model.named_modules():+ if isinstance(m, WrapperWALayer):+ m = m.orig_layer+ q_weight = recover_qweight(m.weight.to(device), m.scale.to(device)).to("cpu")+ delattr(m, 'weight')+ setattr(m, 'weight', torch.nn.Buffer(q_weight))+ setattr(m, "weight_scale", torch.nn.Buffer(m.scale))++ # replace WrapperWALayer with orig_layer+ candidates = []+ for n, m in model.named_modules():+ if isinstance(m, WrapperWALayer):+ candidates.append(n)+ for n in candidates:+ set_module(model, n, get_module(model, n).orig_layer)++ # save model.config, model.state_dict()+ model.config.quantization_config = quantization_config+ model.config.save_pretrained(output_dir)+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)diff --git a/test/test_cpu/test_llmcompressor_w8a8.py b/test/test_cpu/test_llmcompressor_w8a8.pynew file mode 100644index 000000000..7156b6a07--- /dev/null+++ b/test/test_cpu/test_llmcompressor_w8a8.py@@ -0,0 +1,39 @@+import os+import sys+import unittest+import shutil+sys.path.insert(0, "../..")++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+++class TestLLMC(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "stas/tiny-random-llama-2"+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+++ def test_llmcompressor_w8a8(self):+ bits, group_size, sym, act_bits = 8, -1, True, 8+ ## quantize the model+ autoround = AutoRound(+ self.model, self.tokenizer,+ bits=bits, group_size=group_size, sym=sym, act_bits=act_bits,+ iters=0,+ )+ autoround.quantize()+ autoround.save_quantized("./saved", format='llmcompressor', inplace=True)+++if __name__ == "__main__":+ unittest.main()