PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix some torch compile warnings (#755)
Wenhua Cheng1f1d0a9c433c
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex aedcd860d..c30d2280a 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -31,10 +31,12 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.sign_sgd import SignSGDfrom auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,+ SUPPORTED_FORMATS,SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,CpuInfo,@@ -75,6 +77,7 @@ from auto_round.utils import (logger,mv_module_from_gpu,reset_params,+ set_amax_for_all_moe_layers,set_module,to_device,to_dtype,@@ -629,7 +632,6 @@ class AutoRound(object):_gguf_args_check(self, format, model_type=ModelType.MMPROJ)formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")- from auto_round.utils import SUPPORTED_FORMATSfor format_ in formats:if format_ not in SUPPORTED_FORMATS:@@ -926,7 +928,7 @@ class AutoRound(object):logger.info("start to compute imatrix for GGUF quantization.")# Load dataset- from .calib_dataset import get_dataloader+ from auto_round.calib_dataset import get_dataloaderif hasattr(self.model, "is_fp8"):convert_fp8_model_to_16b_model(self.model, self.amp_dtype)@@ -1004,8 +1006,6 @@ class AutoRound(object):accelerate.hooks.remove_hook_from_submodules(model)# Perform quantization using RTN- from tqdm import tqdm-pbar = tqdm(all_to_quantized_module_names)block_names_cnt = len(flatten_list(get_block_names(self.model, True)))clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt@@ -2224,12 +2224,14 @@ class AutoRound(object):unwrapper_layer(self.model, wrapper_linear, layer_name, {})mv_module_from_gpu(layer, self.low_cpu_mem_usage)+ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)if self.enable_minmax_tuning:optimizer = self.optimizer(- [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0)else:- optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)if self.lr_scheduler is None:lr_schedule = torch.optim.lr_scheduler.LinearLR(@@ -2450,12 +2452,14 @@ class AutoRound(object):else:round_params.append(m.params[key])+ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)if self.enable_minmax_tuning:optimizer = self.optimizer(- [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0)else:- optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)if len(round_params) + len(minmax_params) <= 0:dump_info = (@@ -2463,7 +2467,7 @@ class AutoRound(object):f"layers in the block")logger.info(dump_info)- unwrapper_block(block, {}) ## TODO Quant layer should change+ unwrapper_block(block, {}) # TODO Quant layer should changemv_module_from_gpu(block, self.low_cpu_mem_usage)return output, output@@ -2490,15 +2494,15 @@ class AutoRound(object):init_loss = Nonebest_params = {}total_loss = 0-for i in range(self.iters):total_loss = 0if self.sampler == "rand":whole_indices = torch.randperm(nsamples)[:pick_samples]- ##we assume the block input and output shape is same+ # We assume the block input and output shape is sameif self.gradient_accumulate_steps != 1:current_input_ids = [input_ids[i] for i in whole_indices]num_elm = sum(id.numel() for id in current_input_ids)+for tmp_step in range(self.gradient_accumulate_steps):indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]current_input_ids, current_input_others = AutoRound.sampling_inputs(@@ -2512,7 +2516,6 @@ class AutoRound(object):current_output = [output[x] for x in indices]current_output = torch.cat(current_output, dim=self.batch_dim)-current_output = to_device(current_output, device)output_q = block_forward(@@ -2561,11 +2564,10 @@ class AutoRound(object):logger.info(f"{unquantized_layer_names} have not been quantized")with torch.no_grad():unwrapper_block(block, best_params)+if self.enable_quanted_input:if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):- from auto_round.utils import set_amax_for_all_moe_layers-- # enable moe experts act_max automatic generation for WrapperWALayer+ # Enable moe experts act_max automatic generation for WrapperWALayerset_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")if self.low_cpu_mem_usage:block = block.to(device)@@ -2616,7 +2618,7 @@ class AutoRound(object):clear_memory()input_ids = to_device(input_ids, self.cache_device)input_others = to_device(input_others, self.cache_device)- ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+ # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usagetmp_dtype = self.amp_dtype if self.amp else torch.float32for i in range(len(input_ids)):input_ids[i] = input_ids[i].to(tmp_dtype)@@ -2871,8 +2873,6 @@ class AutoRound(object):Returns:The specified optimizer."""- from auto_round.sign_sgd import SignSGD-return SignSGDdef get_scaler(self):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 40fbcc5de..fec2177e2 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -172,6 +172,8 @@ auto_gptq = LazyImport("auto_gptq")htcore = LazyImport("habana_frameworks.torch.core")+@torch._dynamo.disable()+@lru_cache(None)def is_optimum_habana_available():from transformers.utils.import_utils import is_optimum_available@@ -920,6 +922,7 @@ def _clear_memory_for_cpu_and_cuda(tensor=None):torch.xpu.empty_cache()+@torch._dynamo.disable()def clear_memory(tensor=None):if is_hpu_supported():# hpu does not have empty_cache@@ -1117,8 +1120,6 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nondef get_device_and_parallelism(device: Union[str, torch.device, int]) -> Tuple[str, bool]:- from auto_round.utils import detect_device-if isinstance(device, str):devices = device.replace(" ", "").split(",")elif isinstance(device, int):@@ -2457,33 +2458,34 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_namodel = get_module(model, moe_name)# Handle input quantizers of experts that are not calibratedfor name, sub_module in model.named_modules():- if is_moe(sub_module) and hasattr(sub_module, "experts"):- expert_linear_names = get_expert_linear_names(sub_module)- for linear_name in expert_linear_names:- if isinstance(sub_module.experts, collections.abc.Iterable):- # For other MoE models (like Mixtral) with iterable experts- try:- set_amax_for_uncalibrated_experts(- [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name- )- except AttributeError as e:- # Provide more helpful debugging information- expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))- raise AttributeError(- f"Failed to access attribute '{linear_name}' on experts. "- f"MoE module type: {type(sub_module).__name__}, "- f"Expert types: {expert_types}, "- f"Expected linear names: {expert_linear_names}. "- f"This suggests the get_expert_linear_names function may need "- f"to be updated for this model architecture. "- f"Original error: {e}"- ) from e- else:- # Unsupported MoE model structure- raise NotImplementedError(- f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."- f"Please file an issue or add support for this model architecture."+ if not (is_moe(sub_module) and hasattr(sub_module, "experts")):+ continue+ expert_linear_names = get_expert_linear_names(sub_module)+ for linear_name in expert_linear_names:+ if isinstance(sub_module.experts, collections.abc.Iterable):+ # For other MoE models (like Mixtral) with iterable experts+ try:+ set_amax_for_uncalibrated_experts(+ [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name)+ except AttributeError as e:+ # Provide more helpful debugging information+ expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))+ raise AttributeError(+ f"Failed to access attribute '{linear_name}' on experts. "+ f"MoE module type: {type(sub_module).__name__}, "+ f"Expert types: {expert_types}, "+ f"Expected linear names: {expert_linear_names}. "+ f"This suggests the get_expert_linear_names function may need "+ f"to be updated for this model architecture. "+ f"Original error: {e}"+ ) from e+ else:+ # Unsupported MoE model structure+ raise NotImplementedError(+ f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."+ f"Please file an issue or add support for this model architecture."+ )class BackendDataType(str, Enum):