PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
use torch.compile by default for PyTorch versions 2.6 and above (#295)
wenhuach21c922f5b316ac
diff --git a/README.md b/README.mdindex 2b31a85c0..46329b35f 100644--- a/README.md+++ b/README.md@@ -26,7 +26,9 @@ more accuracy data and recipes across various models.<div align="left">## What's New-* [2024/10] AutoRound has been integrated to [torch/ao](https://github.com/pytorch/ao), check out their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)++* [2024/10] AutoRound has been integrated to [torch/ao](https://github.com/pytorch/ao), check out+ their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)* [2024/10] Important update: We now support full-range symmetric quantization and have made it the defaultconfiguration. This configuration is typically better or comparable to asymmetric quantization and significantlyoutperforms other symmetric variants, especially at low bit-widths like 2-bit.@@ -37,7 +39,6 @@ more accuracy data and recipes across various models.* [2024/08] AutoRound introduces several experimental features, including fast tuning of norm/bias parameters (for 2-bitand W4A4), activation quantization, and the mx_fp data type.-## Installation### Build from Source@@ -57,7 +58,8 @@ pip install auto-round### Basic Usage (Gaudi2/CPU/GPU)A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the terminal.-Alternatively, you can use ```auto_round``` instead of ```auto-round```. Set the format you want in `format` and multiple formats exporting has been supported.+Alternatively, you can use ```auto_round``` instead of ```auto-round```. Set the format you want in `format` and+multiple formats exporting has been supported.```bashCUDA_VISIBLE_DEVICES=0 auto-round \@@ -198,13 +200,36 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)</details>+### Quantization Costs++Testing was conducted on the Nvidia A100 80G using the nightly version of PyTorch 2.6.0.dev20241029+cu124. Please note+that data+loading and packing costs have been excluded from the evaluation. **We enable torch.compile for Torch 2.6, but not for+2.5+due to encountered issues.**++To optimize GPU memory usage, in addition to activating `low_gpu_mem_usage`, you can set `gradient_accumulate_steps=8`+and a+`batch_size=1`, though this may increase tuning time.++The 3B and 14B models were evaluated on Qwen 2.5, the 8X7B model is Mixtral, while the remaining models utilized LLaMA+3.1.++| Torch version/Config W4G128 | 3B | 8B | 14B | 70B | 8X7B |+|---------------------------------------------------------------------------------------------|---------------|----------------|----------------|-----------------|----------------|+| 2.6 with torch compile | 7min<br/>10GB | 12min<br/>18GB | 23min<br/>22GB | 120min<br/>42GB | 28min<br/>46GB |+| 2.6 with torch compile <br/> low_gpu_mem_usage=True | 12min<br/>6GB | 19min<br/>10GB | 33min<br/>11GB | 140min<br/>25GB | 38min<br/>36GB |+| 2.6 with torch compile <br/> low_gpu_mem_usage=True <br/> gradient_accumulate_steps=8,bs=1 | 15min<br/>3GB | 25min<br/>6GB | 45min<br/>7GB | 187min<br/>19GB | 75min<br/>36GB |+| 2.5 w/o torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |+## Model InferencePlease run the quantization code first### AutoRound format-**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip install intel-extension-for-transformers,+**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip+install intel-extension-for-transformers,**HPU**: docker image with Gaudi Software Stack is recommended. More details can be foundin [Gaudi Guide](https://docs.habana.ai/en/latest/).diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 731974ec1..f82f3c4bd 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -23,7 +23,7 @@ from transformers import set_seedfrom torch import autocastfrom tqdm import tqdmimport accelerate-+from packaging import versionfrom .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \WrapperTransformerConv1dfrom .special_model_handler import (check_hidden_state_dim,@@ -49,7 +49,7 @@ from .utils import (get_layer_names_in_block,mv_module_from_gpu,unsupport_meta_device, detect_device_count, clear_memory,- get_multimodal_block_names,+ get_multimodal_block_names, get_library_version,)from .low_cpu_mem.utils import get_layers_before_block@@ -290,7 +290,6 @@ class AutoRound(object):self.train_bs = total_sampleslogger.warning(f"force the train batch size to {total_samples}")-self.quant_blocks(self.model,inputs,@@ -382,12 +381,17 @@ class AutoRound(object):self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()+ torch_version = get_library_version("torch")+ if version.parse(torch_version) >= version.parse("2.5.99"):+ quant_layer = torch.compile(self.quant_layer)+ else:+ quant_layer = self.quant_layerfor layer_name in layer_names:layer_input = layer_inputs[layer_name]layer_input = to_device(layer_input, self.cache_device)q_layer_input = q_layer_inputs[layer_name] if enable_quanted_input else Noneq_layer_input = to_device(q_layer_input, self.cache_device)- self.quant_layer(layer_name, layer_input, q_layer_input, device=self.device)+ quant_layer(layer_name, layer_input, q_layer_input, device=self.device)del layer_inputclear_memory(q_layer_input)@@ -564,7 +568,6 @@ class AutoRound(object):for n, m in embed_layers:m = m.to("meta")-@torch.no_grad()def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):"""Attempts to cache intermediate data on GPU, if failed, then using CPU.@@ -599,9 +602,9 @@ class AutoRound(object):logger.info("switch to cpu to cache inputs")if "lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] < 8:logger.warning(f"we strongly recommend using additional CUDA/HPU devices,e.g. "- f"'CUDA_VISIBLE_DEVICES=0,1 python xxx',"- f" for optimal performance during calibration when enabling lm-head quantization. "- f"Otherwise, the process may be significantly slower.")+ f"'CUDA_VISIBLE_DEVICES=0,1 python xxx',"+ f" for optimal performance during calibration when enabling lm-head quantization. "+ f"Otherwise, the process may be significantly slower.")self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()all_inputs = self.cache_inter_data(@@ -909,7 +912,7 @@ class AutoRound(object):unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)mv_module_from_gpu(layer, self.low_cpu_mem_usage)dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"- logger.info(dump_info)+ logger.debug(dump_info)def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):"""Quantize the weights of a given block of the model.@@ -1049,6 +1052,7 @@ class AutoRound(object):if self.enable_quanted_input:if self.low_cpu_mem_usage:block = block.to(device)+ clear_memory()q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,cache_device=self.cache_device@@ -1106,6 +1110,12 @@ class AutoRound(object):elif isinstance(input_others[key], list):for i in range(len(input_others[key])):to_dtype(input_others[key][i], tmp_dtype)+ torch_version = get_library_version("torch")+ if version.parse(torch_version) >= version.parse("2.5.99"):+ quant_block = torch.compile(self.quant_block)+ else:+ quant_block = self.quant_block+pbar = tqdm(range(0, len(block_names), nblocks))for i in pbar:if nblocks == 1:@@ -1121,7 +1131,7 @@ class AutoRound(object):if not self.model.device.type == "meta" or self.low_cpu_mem_usage:m = m.to(device)- q_input, input_ids = self.quant_block(+ q_input, input_ids = quant_block(m,input_ids,input_others,