Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add torch compile arg (#314)

wenhuach218a9eed0b1d97
diff --git a/README.md b/README.md
index d364ba3bb..f5031df8a 100644
--- a/README.md
+++ b/README.md
@@ -31,13 +31,13 @@ more accuracy data and recipes across various models.
their [release note](https://github.com/pytorch/ao/releases/tag/v0.6.1)
* [2024/10] Important update: We now support full-range symmetric quantization and have made it the default
configuration. This configuration is typically better or comparable to asymmetric quantization and significantly
- outperforms other symmetric variants, especially at low bit-widths like 2-bit.
+ outperforms other symmetric variants, especially at low bit-widths like 2-bit, check out [some accuracy data](./docs/full_range_sym.md).
* [2024/09] AutoRound format supports several LVM models, check out the
examples [Qwen2-Vl](./examples/multimodal-modeling/Qwen-VL),[Phi-3-vision](./examples/multimodal-modeling/Phi-3-vision), [Llava](./examples/multimodal-modeling/Llava)
* [2024/08] AutoRound format supports Intel Gaudi2 devices. Please refer
to [Intel/Qwen2-7B-int4-inc](https://huggingface.co/Intel/Qwen2-7B-int4-inc).
* [2024/08] AutoRound introduces several experimental features, including fast tuning of norm/bias parameters (for 2-bit
- and W4A4), activation quantization, and the mx_fp data type.
+ and W4A4, check out [more details](./docs/tuning_norm_bias.md)), activation quantization, and the mx_fp data type.
## Installation
@@ -248,7 +248,8 @@ quantization_config = AutoRoundConfig(
)
quantized_model_path = "./tmp_autoround"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map=backend.split(':')[0], quantization_config=quantization_config)
+ device_map=backend.split(':')[0],
+ quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 621e8b877..361666338 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -49,8 +49,7 @@ from .utils import (
to_dtype,
get_layer_names_in_block,
mv_module_from_gpu,
- unsupport_meta_device, detect_device_count, clear_memory,
- get_multimodal_block_names, get_library_version,
+ unsupport_meta_device, clear_memory,
compile_func,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -110,6 +109,9 @@ class AutoRound(object):
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True
+
Returns:
The quantized model.
"""
@@ -150,6 +152,7 @@ class AutoRound(object):
act_dynamic: bool = True,
quant_block_list: list = None,
enable_norm_bias_tuning: bool = False,
+ enable_torch_compile: bool = None,
**kwargs,
):
self.quantized = False
@@ -213,6 +216,7 @@ class AutoRound(object):
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
logger.info(f"using {self.model.dtype} for quantization tuning")
+ self.enable_torch_compile = enable_torch_compile
if is_optimum_habana_available():
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
@@ -395,7 +399,7 @@ class AutoRound(object):
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
device = next(self.model.parameters()).device
- quant_layer = compile_func(self.quant_layer, device)
+ quant_layer = compile_func(self.quant_layer, device, self.enable_torch_compile)
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
@@ -613,7 +617,7 @@ class AutoRound(object):
if "CUDA out of memory" in str(e):
logger.info("switch to cpu to cache inputs")
if (("lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] <= 16) or
- self.__class__.__name__=="AutoRoundMLLM") :
+ self.__class__.__name__ == "AutoRoundMLLM"):
logger.warning(f"we strongly recommend using additional CUDA/HPU devices,e.g. "
f"set `--device '0,1'` in our cmd line usage or "
f"load the model with `device_mapping=auto`,"
@@ -724,7 +728,7 @@ class AutoRound(object):
if self.batch_dim is None:
self.batch_dim = 0
- if hidden_states is not None and self.batch_size>1:
+ if hidden_states is not None and self.batch_size > 1:
if hidden_states.shape[0] > self.batch_size:
self.batch_dim = 1
if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:
@@ -1027,7 +1031,7 @@ class AutoRound(object):
if self.sampler == "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
##we assume the block input and output shape is same
- if self.gradient_accumulate_steps!=1:
+ if self.gradient_accumulate_steps != 1:
current_input_ids = [input_ids[i] for i in whole_indices]
num_elm = sum(id.numel() for id in current_input_ids)
for tmp_step in range(self.gradient_accumulate_steps):
@@ -1151,7 +1155,7 @@ class AutoRound(object):
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
to_dtype(input_others[key][i], tmp_dtype)
- quant_block = compile_func(self.quant_block, device)
+ quant_block = compile_func(self.quant_block, device, self.enable_torch_compile)
pbar = tqdm(range(0, len(block_names), nblocks))
for i in pbar:
@@ -1417,6 +1421,8 @@ class AutoRoundOPT(AutoRound):
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
**kwargs: Additional keyword arguments.
Returns:
@@ -1457,8 +1463,9 @@ class AutoRoundOPT(AutoRound):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- enable_norm_bias_tuning: bool = False,
quant_block_list: list = None,
+ enable_norm_bias_tuning: bool = False,
+ enable_torch_compile: bool = None,
optimizer="AdamW",
**kwargs,
):
@@ -1495,8 +1502,9 @@ class AutoRoundOPT(AutoRound):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
quant_block_list=quant_block_list,
+ enable_norm_bias_tuning=enable_norm_bias_tuning,
+ enable_torch_compile=enable_torch_compile,
**kwargs,
)
@@ -1583,6 +1591,8 @@ class AutoRoundAdam(AutoRoundOPT):
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
Returns:
The quantized model.
"""
@@ -1621,8 +1631,9 @@ class AutoRoundAdam(AutoRoundOPT):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- enable_norm_bias_tuning: bool = False,
quant_block_list: list = None,
+ enable_norm_bias_tuning: bool = False,
+ enable_torch_compile: bool = None,
optimizer="AdamW",
**kwargs,
):
@@ -1659,9 +1670,9 @@ class AutoRoundAdam(AutoRoundOPT):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
quant_block_list=quant_block_list,
+ enable_norm_bias_tuning=enable_norm_bias_tuning,
+ enable_torch_compile=enable_torch_compile,
optimizer=optimizer,
**kwargs,
)
-
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index fa20a04c0..ffa68d39c 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -71,8 +71,12 @@ class AutoRoundMLLM(AutoRound):
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True
**kwargs: Additional keyword arguments.
+
+
"""
+
def __init__(
self,
model,
@@ -113,6 +117,7 @@ class AutoRoundMLLM(AutoRound):
act_dynamic: bool = True,
quant_block_list: list = None,
enable_norm_bias_tuning: bool = False,
+ enable_torch_compile: bool = None,
**kwargs,
):
if quant_block_list is None:
@@ -159,11 +164,11 @@ class AutoRoundMLLM(AutoRound):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
quant_block_list=quant_block_list,
+ enable_norm_bias_tuning=enable_norm_bias_tuning,
+ enable_torch_compile=enable_torch_compile,
**kwargs,
)
-
def calib(self, nsamples, bs):
"""Perform calibration for quantization.
@@ -192,7 +197,7 @@ class AutoRoundMLLM(AutoRound):
)
else:
self.dataloader = self.dataset
- total_cnt = 0
+ total_cnt = 0
if self.low_cpu_mem_usage:
embed_layers = get_layers_before_block(self.model)
@@ -201,7 +206,7 @@ class AutoRoundMLLM(AutoRound):
for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):
if data is None:
- continue
+ continue
if isinstance(data, torch.Tensor):
input_ids = data.to(self.device)
data_new = input_ids
@@ -215,7 +220,7 @@ class AutoRoundMLLM(AutoRound):
images=None,
max_length=self.seqlen,
squeeze=False,
- )
+ )
data_new = {}
for key in data.keys():
data_new[key] = data[key].to(self.device)
@@ -230,7 +235,7 @@ class AutoRoundMLLM(AutoRound):
images=data["image"],
max_length=self.seqlen,
squeeze=False,
- )
+ )
data_new = {}
for key in data.keys():
data_new[key] = to_device(data[key], self.model.device)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 438841204..e5497c3d9 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -31,6 +31,7 @@ from packaging import version
import gc
from .special_model_handler import shareable_keywords
+
@lru_cache(None)
def warning_once(self, msg: str):
self.warning(msg)
@@ -753,6 +754,7 @@ def is_autoround_exllamav2_available():
res = False
return res
+
@lru_cache(None)
def is_hpu_supported(): # pragma: no cover
try:
@@ -888,11 +890,11 @@ TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")
def check_hpu_compile_mode():
assert (
- os.environ["PT_HPU_LAZY_MODE"] == "0"
+ os.environ["PT_HPU_LAZY_MODE"] == "0"
), "Please set `PT_HPU_LAZY_MODE=0` to use HPU compile mode"
# Note: this is a temporary solution, will be removed in the future
assert (
- os.environ["PT_ENABLE_INT64_SUPPORT"] == "1"
+ os.environ["PT_ENABLE_INT64_SUPPORT"] == "1"
), "Please set `PT_ENABLE_INT64_SUPPORT=1` to use HPU compile mode"
@@ -903,15 +905,15 @@ def compile_func_on_hpu(func):
return func
-def compile_func_on_cuda_or_cpu(func):
- if TORCH_VERSION_AT_LEAST_2_6_PRE_RELEASE:
+def compile_func_on_cuda_or_cpu(func, enable_torch_compile):
+ if enable_torch_compile or TORCH_VERSION_AT_LEAST_2_6_PRE_RELEASE:
return torch.compile(func)
else:
return func
-def compile_func(fun, device):
+def compile_func(fun, device, enable_torch_compile):
if "hpu" in str(device):
- return compile_func_on_hpu(fun)
+ return compile_func_on_hpu(fun) ## use auto by default
else:
- return compile_func_on_cuda_or_cpu(fun)
+ return compile_func_on_cuda_or_cpu(fun, enable_torch_compile)