PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add torch compile arg (#314)
wenhuach218a9eed0b1d97
diff --git a/README.md b/README.mdindex d364ba3bb..f5031df8a 100644--- a/README.md+++ b/README.md@@ -31,13 +31,13 @@ more accuracy data and recipes across various models.their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)* [2024/10] Important update: We now support full-range symmetric quantization and have made it the defaultconfiguration. This configuration is typically better or comparable to asymmetric quantization and significantly- outperforms other symmetric variants, especially at low bit-widths like 2-bit.+ outperforms other symmetric variants, especially at low bit-widths like 2-bit, check out [some accuracy data](./docs/full_range_sym.md).* [2024/09] AutoRound format supports several LVM models, check out theexamples [Qwen2-Vl](./examples/multimodal-modeling/Qwen-VL),[Phi-3-vision](./examples/multimodal-modeling/Phi-3-vision), [Llava](./examples/multimodal-modeling/Llava)* [2024/08] AutoRound format supports Intel Gaudi2 devices. Please referto [Intel/Qwen2-7B-int4-inc](https://huggingface.co/Intel/Qwen2-7B-int4-inc).* [2024/08] AutoRound introduces several experimental features, including fast tuning of norm/bias parameters (for 2-bit- and W4A4), activation quantization, and the mx_fp data type.+ and W4A4, check out [more details](./docs/tuning_norm_bias.md)), activation quantization, and the mx_fp data type.## Installation@@ -248,7 +248,8 @@ quantization_config = AutoRoundConfig()quantized_model_path = "./tmp_autoround"model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map=backend.split(':')[0], quantization_config=quantization_config)+ device_map=backend.split(':')[0],+ quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 621e8b877..361666338 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -49,8 +49,7 @@ from .utils import (to_dtype,get_layer_names_in_block,mv_module_from_gpu,- unsupport_meta_device, detect_device_count, clear_memory,- get_multimodal_block_names, get_library_version,+ unsupport_meta_device, clear_memory,compile_func,)from .low_cpu_mem.utils import get_layers_before_block@@ -110,6 +109,9 @@ class AutoRound(object):act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True+Returns:The quantized model."""@@ -150,6 +152,7 @@ class AutoRound(object):act_dynamic: bool = True,quant_block_list: list = None,enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = None,**kwargs,):self.quantized = False@@ -213,6 +216,7 @@ class AutoRound(object):torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()logger.info(f"using {self.model.dtype} for quantization tuning")+ self.enable_torch_compile = enable_torch_compileif is_optimum_habana_available():logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401@@ -395,7 +399,7 @@ class AutoRound(object):self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()device = next(self.model.parameters()).device- quant_layer = compile_func(self.quant_layer, device)+ quant_layer = compile_func(self.quant_layer, device, self.enable_torch_compile)for layer_name in layer_names:layer_input = layer_inputs[layer_name]layer_input = to_device(layer_input, self.cache_device)@@ -613,7 +617,7 @@ class AutoRound(object):if "CUDA out of memory" in str(e):logger.info("switch to cpu to cache inputs")if (("lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] <= 16) or- self.__class__.__name__=="AutoRoundMLLM") :+ self.__class__.__name__ == "AutoRoundMLLM"):logger.warning(f"we strongly recommend using additional CUDA/HPU devices,e.g. "f"set `--device '0,1'` in our cmd line usage or "f"load the model with `device_mapping=auto`,"@@ -724,7 +728,7 @@ class AutoRound(object):if self.batch_dim is None:self.batch_dim = 0- if hidden_states is not None and self.batch_size>1:+ if hidden_states is not None and self.batch_size > 1:if hidden_states.shape[0] > self.batch_size:self.batch_dim = 1if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:@@ -1027,7 +1031,7 @@ class AutoRound(object):if self.sampler == "rand":whole_indices = torch.randperm(nsamples)[:pick_samples]##we assume the block input and output shape is same- if self.gradient_accumulate_steps!=1:+ if self.gradient_accumulate_steps != 1:current_input_ids = [input_ids[i] for i in whole_indices]num_elm = sum(id.numel() for id in current_input_ids)for tmp_step in range(self.gradient_accumulate_steps):@@ -1151,7 +1155,7 @@ class AutoRound(object):elif isinstance(input_others[key], list):for i in range(len(input_others[key])):to_dtype(input_others[key][i], tmp_dtype)- quant_block = compile_func(self.quant_block, device)+ quant_block = compile_func(self.quant_block, device, self.enable_torch_compile)pbar = tqdm(range(0, len(block_names), nblocks))for i in pbar:@@ -1417,6 +1421,8 @@ class AutoRoundOPT(AutoRound):act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function**kwargs: Additional keyword arguments.Returns:@@ -1457,8 +1463,9 @@ class AutoRoundOPT(AutoRound):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- enable_norm_bias_tuning: bool = False,quant_block_list: list = None,+ enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = None,optimizer="AdamW",**kwargs,):@@ -1495,8 +1502,9 @@ class AutoRoundOPT(AutoRound):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- enable_norm_bias_tuning=enable_norm_bias_tuning,quant_block_list=quant_block_list,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_torch_compile=enable_torch_compile,**kwargs,)@@ -1583,6 +1591,8 @@ class AutoRoundAdam(AutoRoundOPT):act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer functionReturns:The quantized model."""@@ -1621,8 +1631,9 @@ class AutoRoundAdam(AutoRoundOPT):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- enable_norm_bias_tuning: bool = False,quant_block_list: list = None,+ enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = None,optimizer="AdamW",**kwargs,):@@ -1659,9 +1670,9 @@ class AutoRoundAdam(AutoRoundOPT):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- enable_norm_bias_tuning=enable_norm_bias_tuning,quant_block_list=quant_block_list,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_torch_compile=enable_torch_compile,optimizer=optimizer,**kwargs,)-diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex fa20a04c0..ffa68d39c 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -71,8 +71,12 @@ class AutoRoundMLLM(AutoRound):act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True**kwargs: Additional keyword arguments.++"""+def __init__(self,model,@@ -113,6 +117,7 @@ class AutoRoundMLLM(AutoRound):act_dynamic: bool = True,quant_block_list: list = None,enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = None,**kwargs,):if quant_block_list is None:@@ -159,11 +164,11 @@ class AutoRoundMLLM(AutoRound):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- enable_norm_bias_tuning=enable_norm_bias_tuning,quant_block_list=quant_block_list,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_torch_compile=enable_torch_compile,**kwargs,)-def calib(self, nsamples, bs):"""Perform calibration for quantization.@@ -192,7 +197,7 @@ class AutoRoundMLLM(AutoRound):)else:self.dataloader = self.dataset- total_cnt = 0+ total_cnt = 0if self.low_cpu_mem_usage:embed_layers = get_layers_before_block(self.model)@@ -201,7 +206,7 @@ class AutoRoundMLLM(AutoRound):for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):if data is None:- continue+ continueif isinstance(data, torch.Tensor):input_ids = data.to(self.device)data_new = input_ids@@ -215,7 +220,7 @@ class AutoRoundMLLM(AutoRound):images=None,max_length=self.seqlen,squeeze=False,- )+ )data_new = {}for key in data.keys():data_new[key] = data[key].to(self.device)@@ -230,7 +235,7 @@ class AutoRoundMLLM(AutoRound):images=data["image"],max_length=self.seqlen,squeeze=False,- )+ )data_new = {}for key in data.keys():data_new[key] = to_device(data[key], self.model.device)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 438841204..e5497c3d9 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -31,6 +31,7 @@ from packaging import versionimport gcfrom .special_model_handler import shareable_keywords+@lru_cache(None)def warning_once(self, msg: str):self.warning(msg)@@ -753,6 +754,7 @@ def is_autoround_exllamav2_available():res = Falsereturn res+@lru_cache(None)def is_hpu_supported(): # pragma: no covertry:@@ -888,11 +890,11 @@ TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")def check_hpu_compile_mode():assert (- os.environ["PT_HPU_LAZY_MODE"] == "0"+ os.environ["PT_HPU_LAZY_MODE"] == "0"), "Please set `PT_HPU_LAZY_MODE=0` to use HPU compile mode"# Note: this is a temporary solution, will be removed in the futureassert (- os.environ["PT_ENABLE_INT64_SUPPORT"] == "1"+ os.environ["PT_ENABLE_INT64_SUPPORT"] == "1"), "Please set `PT_ENABLE_INT64_SUPPORT=1` to use HPU compile mode"@@ -903,15 +905,15 @@ def compile_func_on_hpu(func):return func-def compile_func_on_cuda_or_cpu(func):- if TORCH_VERSION_AT_LEAST_2_6_PRE_RELEASE:+def compile_func_on_cuda_or_cpu(func, enable_torch_compile):+ if enable_torch_compile or TORCH_VERSION_AT_LEAST_2_6_PRE_RELEASE:return torch.compile(func)else:return func-def compile_func(fun, device):+def compile_func(fun, device, enable_torch_compile):if "hpu" in str(device):- return compile_func_on_hpu(fun)+ return compile_func_on_hpu(fun) ## use auto by defaultelse:- return compile_func_on_cuda_or_cpu(fun)+ return compile_func_on_cuda_or_cpu(fun, enable_torch_compile)