Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

use torch.compile by default for PyTorch versions 2.6 and above (#295)

wenhuach21c922f5b316ac
diff --git a/README.md b/README.md
index 2b31a85c0..46329b35f 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,9 @@ more accuracy data and recipes across various models.
<div align="left">
## What's New
-* [2024/10] AutoRound has been integrated to [torch/ao](https://github.com/pytorch/ao), check out their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)
+
+* [2024/10] AutoRound has been integrated to [torch/ao](https://github.com/pytorch/ao), check out
+ their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)
* [2024/10] Important update: We now support full-range symmetric quantization and have made it the default
configuration. This configuration is typically better or comparable to asymmetric quantization and significantly
outperforms other symmetric variants, especially at low bit-widths like 2-bit.
@@ -37,7 +39,6 @@ more accuracy data and recipes across various models.
* [2024/08] AutoRound introduces several experimental features, including fast tuning of norm/bias parameters (for 2-bit
and W4A4), activation quantization, and the mx_fp data type.
-
## Installation
### Build from Source
@@ -57,7 +58,8 @@ pip install auto-round
### Basic Usage (Gaudi2/CPU/GPU)
A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the terminal.
-Alternatively, you can use ```auto_round``` instead of ```auto-round```. Set the format you want in `format` and multiple formats exporting has been supported.
+Alternatively, you can use ```auto_round``` instead of ```auto-round```. Set the format you want in `format` and
+multiple formats exporting has been supported.
```bash
CUDA_VISIBLE_DEVICES=0 auto-round \
@@ -198,13 +200,36 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)
</details>
+### Quantization Costs
+
+Testing was conducted on the Nvidia A100 80G using the nightly version of PyTorch 2.6.0.dev20241029+cu124. Please note
+that data
+loading and packing costs have been excluded from the evaluation. **We enable torch.compile for Torch 2.6, but not for
+2.5
+due to encountered issues.**
+
+To optimize GPU memory usage, in addition to activating `low_gpu_mem_usage`, you can set `gradient_accumulate_steps=8`
+and a
+`batch_size=1`, though this may increase tuning time.
+
+The 3B and 14B models were evaluated on Qwen 2.5, the 8X7B model is Mixtral, while the remaining models utilized LLaMA
+3.1.
+
+| Torch version/Config W4G128 | 3B | 8B | 14B | 70B | 8X7B |
+|---------------------------------------------------------------------------------------------|---------------|----------------|----------------|-----------------|----------------|
+| 2.6 with torch compile | 7min<br/>10GB | 12min<br/>18GB | 23min<br/>22GB | 120min<br/>42GB | 28min<br/>46GB |
+| 2.6 with torch compile <br/> low_gpu_mem_usage=True | 12min<br/>6GB | 19min<br/>10GB | 33min<br/>11GB | 140min<br/>25GB | 38min<br/>36GB |
+| 2.6 with torch compile <br/> low_gpu_mem_usage=True <br/> gradient_accumulate_steps=8,bs=1 | 15min<br/>3GB | 25min<br/>6GB | 45min<br/>7GB | 187min<br/>19GB | 75min<br/>36GB |
+| 2.5 w/o torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |
+
## Model Inference
Please run the quantization code first
### AutoRound format
-**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip install intel-extension-for-transformers,
+**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip
+install intel-extension-for-transformers,
**HPU**: docker image with Gaudi Software Stack is recommended. More details can be found
in [Gaudi Guide](https://docs.habana.ai/en/latest/).
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 731974ec1..f82f3c4bd 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -23,7 +23,7 @@ from transformers import set_seed
from torch import autocast
from tqdm import tqdm
import accelerate
-
+from packaging import version
from .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \
WrapperTransformerConv1d
from .special_model_handler import (check_hidden_state_dim,
@@ -49,7 +49,7 @@ from .utils import (
get_layer_names_in_block,
mv_module_from_gpu,
unsupport_meta_device, detect_device_count, clear_memory,
- get_multimodal_block_names,
+ get_multimodal_block_names, get_library_version,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -290,7 +290,6 @@ class AutoRound(object):
self.train_bs = total_samples
logger.warning(f"force the train batch size to {total_samples}")
-
self.quant_blocks(
self.model,
inputs,
@@ -382,12 +381,17 @@ class AutoRound(object):
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
+ torch_version = get_library_version("torch")
+ if version.parse(torch_version) >= version.parse("2.5.99"):
+ quant_layer = torch.compile(self.quant_layer)
+ else:
+ quant_layer = self.quant_layer
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
q_layer_input = q_layer_inputs[layer_name] if enable_quanted_input else None
q_layer_input = to_device(q_layer_input, self.cache_device)
- self.quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
+ quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
del layer_input
clear_memory(q_layer_input)
@@ -564,7 +568,6 @@ class AutoRound(object):
for n, m in embed_layers:
m = m.to("meta")
-
@torch.no_grad()
def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
"""Attempts to cache intermediate data on GPU, if failed, then using CPU.
@@ -599,9 +602,9 @@ class AutoRound(object):
logger.info("switch to cpu to cache inputs")
if "lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] < 8:
logger.warning(f"we strongly recommend using additional CUDA/HPU devices,e.g. "
- f"'CUDA_VISIBLE_DEVICES=0,1 python xxx',"
- f" for optimal performance during calibration when enabling lm-head quantization. "
- f"Otherwise, the process may be significantly slower.")
+ f"'CUDA_VISIBLE_DEVICES=0,1 python xxx',"
+ f" for optimal performance during calibration when enabling lm-head quantization. "
+ f"Otherwise, the process may be significantly slower.")
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
all_inputs = self.cache_inter_data(
@@ -909,7 +912,7 @@ class AutoRound(object):
unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
mv_module_from_gpu(layer, self.low_cpu_mem_usage)
dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- logger.info(dump_info)
+ logger.debug(dump_info)
def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
"""Quantize the weights of a given block of the model.
@@ -1049,6 +1052,7 @@ class AutoRound(object):
if self.enable_quanted_input:
if self.low_cpu_mem_usage:
block = block.to(device)
+ clear_memory()
q_outputs = self.get_block_outputs(
block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,
cache_device=self.cache_device
@@ -1106,6 +1110,12 @@ class AutoRound(object):
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
to_dtype(input_others[key][i], tmp_dtype)
+ torch_version = get_library_version("torch")
+ if version.parse(torch_version) >= version.parse("2.5.99"):
+ quant_block = torch.compile(self.quant_block)
+ else:
+ quant_block = self.quant_block
+
pbar = tqdm(range(0, len(block_names), nblocks))
for i in pbar:
if nblocks == 1:
@@ -1121,7 +1131,7 @@ class AutoRound(object):
if not self.model.device.type == "meta" or self.low_cpu_mem_usage:
m = m.to(device)
- q_input, input_ids = self.quant_block(
+ q_input, input_ids = quant_block(
m,
input_ids,
input_others,