Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support llmcompressor format (#646)

Xin Hebba62c2bb207
diff --git a/README.md b/README.md
index 6ed389084..604bbdcc5 100644
--- a/README.md
+++ b/README.md
@@ -277,6 +277,8 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.
**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widely
adopted within the community, **only 4-bits quantization is supported**.
+**llmcompressor Format**: This format is for reusing llmcompressor format, **only INT8 W8A8 dynamic quantization is supported**.
+
**GGUF** Format: Experimental feature. This format is well-suited for CPU devices and is widely adopted by the community.
### Quantization Costs
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index e1fa1d238..9d265072d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -498,31 +498,20 @@ class AutoRound(object):
if self.group_size == 0 and "fp8" not in self.data_type:
logger.warning("group_size of 0 is not supported for data_type other than fp8 ")
- def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):
- """Quantizes the model and saves it in the specified format(s).
+ def parse_format_to_list(self, format: str) -> list:
+ """Parses the format string into a list of formats.
- This function checks the validity of the requested format(s), quantizes
- the model accordingly, and saves it to the specified output directory.
- If multiple formats are provided, the model is saved separately for each format.
+ This method checks the requested format(s) against the model's
+ quantization settings and adjusts them if necessary. It ensures that
+ the formats are compatible with the model's data type, bit width,
+ and activation quantization settings.
Args:
- output_dir (str, optional): The directory where the quantized model
- will be saved. Defaults to "tmp_autoround".
- format (str, optional): The quantization format(s) to use, separated
- by commas if multiple. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place if only
- one format is used. Defaults to True.
- **kwargs: Additional arguments for the quantization and saving process.
+ format (str): The requested format(s) for quantization, separated by commas.
Returns:
- model: A qdq model or packed model based on the configurations
- folders: The folder paths where the quantized models are saved.
-
- Raises:
- ValueError: If an unsupported format is specified.
+ list: A list of validated and updated formats.
"""
- # Validate and process the specified formats
- self.orig_output_dir = output_dir
_gguf_args_check(self, format)
formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
@@ -561,12 +550,6 @@ class AutoRound(object):
)
formats = ["auto_round"]
- # If multiple formats are specified, enforce inplace=False
- if len(formats) > 1:
- inplace = False
- self.inplace = kwargs.get("inplace", inplace)
- kwargs.pop("inplace", None)
-
# Adjust format settings based on compatibility
for index in range(len(formats)):
format = formats[index]
@@ -591,14 +574,98 @@ class AutoRound(object):
return [x for x in lst if not (x in seen or seen.add(x))]
formats = remove_duplicates(formats)
- self.formats = formats
+ for format in formats:
+ self._check_supported_format(format)
+ return formats
+
+ def _check_supported_format(self, format: str) -> bool:
+ """Checks if the specified format is supported.
+
+ This method validates the requested format against the model's bit width,
+ group size, symmetry, and activation quantization settings. It raises an
+ error if the format is incompatible with the current model configuration.
+
+ Args:
+ format (str): The requested format for quantization.
+
+ Returns:
+ bool: True if the format is supported, False otherwise.
+ """
+ format = format.replace("q*_", f"q{self.bits}_")
+ # only support to export afp8
+ if self.act_bits <= 8:
+ if "fp8" not in self.act_data_type or self.act_dynamic:
+ if format == "llmcompressor":
+ bits, group_size, sym, act_bits = 8, -1, True, 8
+ assert self.bits == bits and self.group_size == group_size and self.sym == sym \
+ and self.act_bits == act_bits and self.act_dynamic, \
+ f"Currently only support to export llmcompressor format for dynamic quantized" \
+ f" W{self.bits}A{self.act_bits} model, but got bits={self.bits}," \
+ f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"
+ elif format != "fake":
+ logger.warning(
+ f"Currently only support to export auto_round format quantized model"
+ " with fp8 dtype activation for activation quantization."
+ " Change format to fake and save."
+ )
+ format = "fake"
+ else:
+ if format != "auto_round":
+ logger.warning(
+ f"Currently only support to export auto_round format for static W{self.bits}AFP8 model,"
+ " change format to auto_round"
+ )
+ format = "auto_round"
+
+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
+ logger.error(
+ f"datatype<{self.data_type}> not support to export {format} format."
+ " Please change export format or `data_type`."
+ )
+ sys.exit(-1)
+
+ def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):
+ """Quantizes the model and saves it in the specified format(s).
+
+ This function checks the validity of the requested format(s), quantizes
+ the model accordingly, and saves it to the specified output directory.
+ If multiple formats are provided, the model is saved separately for each format.
+
+ Args:
+ output_dir (str, optional): The directory where the quantized model
+ will be saved. Defaults to "tmp_autoround".
+ format (str, optional): The quantization format(s) to use, separated
+ by commas if multiple. Defaults to "auto_round".
+ inplace (bool, optional): Whether to modify the model in place if only
+ one format is used. Defaults to True.
+ **kwargs: Additional arguments for the quantization and saving process.
+
+ Returns:
+ model: A qdq model or packed model based on the configurations
+ folders: The folder paths where the quantized models are saved.
+
+ Raises:
+ ValueError: If an unsupported format is specified.
+ """
+ # Validate and process the specified formats
+ self.orig_output_dir = output_dir
+
+ # check and update the format based on the current configuration
+ format_list = self.parse_format_to_list(format)
+ self.formats = format_list
+
+ # If multiple formats are specified, enforce inplace=False
+ if len(format_list) > 1:
+ inplace = False
+ self.inplace = kwargs.get("inplace", inplace)
+ kwargs.pop("inplace", None)
# Perform model quantization
model, _ = self.quantize()
- # Save the quantized model in the specified formats
+ # Save the quantized model in the specified format_list
folders = []
- for format in formats:
+ for format in format_list:
if "gptq" in format and not self.sym:
logger.warning(
"The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
@@ -2287,31 +2354,7 @@ class AutoRound(object):
Returns:
object: The compressed model object.
"""
- format = format.replace("q*_", f"q{self.bits}_")
- # only support to export afp8
- if self.act_bits <= 8:
- if "fp8" not in self.act_data_type or self.act_dynamic:
- if format != "fake":
- logger.warning(
- f"Currently only support to export auto_round format quantized model"
- " with fp8 dtype activation for activation quantization."
- " Change format to fake and save."
- )
- format = "fake"
- else:
- if format != "auto_round":
- logger.warning(
- f"Currently only support to export auto_round format for static W{self.bits}AFP8 model,"
- " change format to auto_round"
- )
- format = "auto_round"
-
- if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
- logger.error(
- f"datatype<{self.data_type}> not support to export {format} format."
- " Please change export format or `data_type`."
- )
- sys.exit(-1)
+ self._check_supported_format(format)
if self.low_cpu_mem_usage:
self.model = self.model.to('cpu')
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index d1012123e..42196d115 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -76,4 +76,9 @@ def _packing_layer_with_autoawq(*args, **kwargs):
return pack_layer(*args, **kwargs)
+@register_format("llmcompressor")
+def _save_quantized_as_llmcompressor(*args, **kwargs):
+ from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor
+
+ return save_quantized_as_llmcompressor(*args, **kwargs)
diff --git a/auto_round/export/export_to_llmcompressor/__init__.py b/auto_round/export/export_to_llmcompressor/__init__.py
new file mode 100644
index 000000000..0d2740cdb
--- /dev/null
+++ b/auto_round/export/export_to_llmcompressor/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.py
new file mode 100644
index 000000000..1e1b9a6a1
--- /dev/null
+++ b/auto_round/export/export_to_llmcompressor/config.py
@@ -0,0 +1,56 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+quantization_config = {
+"config_groups": {
+ "group_0": {
+ "input_activations": {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": True,
+ "group_size": None,
+ "num_bits": 8,
+ "observer": None,
+ "observer_kwargs": {},
+ "strategy": "token",
+ "symmetric": True,
+ "type": "int"
+ },
+ "output_activations": None,
+ "targets": [
+ "Linear"
+ ],
+ "weights": {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": False,
+ "group_size": None,
+ "num_bits": 8,
+ "observer": "minmax",
+ "observer_kwargs": {},
+ "strategy": "channel",
+ "symmetric": True,
+ "type": "int"
+ }
+ }
+},
+"format": "int-quantized",
+"global_compression_ratio": 1.5, # not exactly correct, but a placeholder
+"ignore": [],
+"kv_cache_scheme": None,
+"quant_method": "compressed-tensors",
+"quantization_status": "compressed",
+"provider": "auto-round" # This is the key to indicate the model provider
+}
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
new file mode 100644
index 000000000..a60338752
--- /dev/null
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import torch
+from auto_round.utils import get_module, logger, set_module, detect_device
+from auto_round.wrapper import WrapperWALayer
+from auto_round.export.export_to_llmcompressor.config import quantization_config
+
+
+def recover_qweight(qdq_weight, scale):
+ """
+ Recover the quantized weight to its original floating-point representation.
+
+ Args:
+ qdq_weight (torch.Tensor): The quantized weight tensor.
+ scale (float): The scale factor used for quantization.
+
+ Returns:
+ torch.Tensor: The recovered floating-point weight tensor.
+ """
+ return (qdq_weight / scale).to(torch.int8)
+
+
+
+def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):
+ safe_serialization = kwargs.get("safe_serialization", True)
+ tokenizer = kwargs.get("tokenizer", None)
+ processor = kwargs.get("processor", None)
+ if output_dir is not None and os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+
+ # save tokenizer, processor
+ if output_dir is not None and tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ if output_dir is not None and processor is not None:
+ processor.save_pretrained(output_dir)
+
+ # generate q_weight
+ device = detect_device()
+ for n, m in model.named_modules():
+ if isinstance(m, WrapperWALayer):
+ m = m.orig_layer
+ q_weight = recover_qweight(m.weight.to(device), m.scale.to(device)).to("cpu")
+ delattr(m, 'weight')
+ setattr(m, 'weight', torch.nn.Buffer(q_weight))
+ setattr(m, "weight_scale", torch.nn.Buffer(m.scale))
+
+ # replace WrapperWALayer with orig_layer
+ candidates = []
+ for n, m in model.named_modules():
+ if isinstance(m, WrapperWALayer):
+ candidates.append(n)
+ for n in candidates:
+ set_module(model, n, get_module(model, n).orig_layer)
+
+ # save model.config, model.state_dict()
+ model.config.quantization_config = quantization_config
+ model.config.save_pretrained(output_dir)
+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)
diff --git a/test/test_cpu/test_llmcompressor_w8a8.py b/test/test_cpu/test_llmcompressor_w8a8.py
new file mode 100644
index 000000000..7156b6a07
--- /dev/null
+++ b/test/test_cpu/test_llmcompressor_w8a8.py
@@ -0,0 +1,39 @@
+import os
+import sys
+import unittest
+import shutil
+sys.path.insert(0, "../..")
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+
+class TestLLMC(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "stas/tiny-random-llama-2"
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ def test_llmcompressor_w8a8(self):
+ bits, group_size, sym, act_bits = 8, -1, True, 8
+ ## quantize the model
+ autoround = AutoRound(
+ self.model, self.tokenizer,
+ bits=bits, group_size=group_size, sym=sym, act_bits=act_bits,
+ iters=0,
+ )
+ autoround.quantize()
+ autoround.save_quantized("./saved", format='llmcompressor', inplace=True)
+
+
+if __name__ == "__main__":
+ unittest.main()