PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Reduce peak gpu memory usage and support moe estimation (#981)
Xin He84e9a77635ac
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 39602680c..0a682079a 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1441,7 +1441,7 @@ class BaseCompressor(object):if is_complex_device_mapping(self.device_map):set_auto_device_map_for_block_with_tuning(- block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size+ block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, self.device)# Dispatch model if neededif is_complex_device_mapping(self.device_map):@@ -2332,10 +2332,10 @@ class BaseCompressor(object):if total_loss < best_loss:best_loss = total_lossif not self.not_use_best_mse:- best_params = collect_best_params(wrapper_linear)+ best_params = collect_best_params(wrapper_linear, self.cache_device)last_best_iter = iif self.not_use_best_mse and i == self.iters - 1:- best_params = collect_best_params(wrapper_linear)+ best_params = collect_best_params(wrapper_linear, self.cache_device)if not self.not_use_best_mse:if 0 < self.dynamic_max_gap <= i - last_best_iter:@@ -2413,6 +2413,7 @@ class BaseCompressor(object):input_others: dict,indices: list[int],device: str,+ cache_device: str = "cpu",) -> torch.Tensor:current_input_ids, current_input_others = self._sampling_inputs(input_ids,@@ -2423,7 +2424,7 @@ class BaseCompressor(object):share_cache_keys=self.shared_cache_keys,)output_q = self.block_forward(block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device)- return output_q+ return output_q.to(cache_device)def _get_current_num_elm(self,@@ -2458,13 +2459,15 @@ class BaseCompressor(object):if is_fp8_linear(m):new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to(device)set_module(block, n, new_layer)-- if is_complex_device_mapping(self.device_map):- set_auto_device_map_for_block_with_tuning(+ # card_0_in_high_risk indicates that card_0 memory is already in high usage (90%) w/o any weights+ # loss_device is used to calculate loss on the second device if available and card_0_in_high_risk+ if self.device_map == "auto" or ((isinstance(self.device_map, str) and "," in self.device_map)):+ card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, device)else:block = block.to(device)+ card_0_in_high_risk, loss_device = False, deviceif is_complex_device_mapping(self.device_map):for n, m in block.named_modules():@@ -2594,13 +2597,13 @@ class BaseCompressor(object):current_output = self._get_current_output(output, indices)- current_output = to_device(current_output, device)+ current_output = to_device(current_output, loss_device)- output_q = self._get_current_q_output(block, input_ids, input_others, indices, device)+ output_q = self._get_current_q_output(block, input_ids, input_others, indices, device, loss_device)if self.attention_mask:tmp_attention_mask = [self.attention_mask[i] for i in indices]- tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)+ tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(loss_device)tmp_attention_mask.unsqueeze_(-1)num_elm = torch.sum(tmp_attention_mask).item()if num_elm == 0:@@ -2608,7 +2611,7 @@ class BaseCompressor(object):else:tmp_attention_mask = 1.0if self.amp:- with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):+ with autocast(device_type=loss_device.split(":")[0], dtype=self.amp_dtype):loss = mse_loss( # pylint: disable=not-callableoutput_q * tmp_attention_mask, current_output * tmp_attention_mask)@@ -2619,20 +2622,29 @@ class BaseCompressor(object):)total_loss += loss.item() / num_elm++ if self.low_gpu_mem_usage and card_0_in_high_risk:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.5)+self._scale_loss_and_backward(scaler, loss)+ if self.low_gpu_mem_usage and card_0_in_high_risk:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.8)+if i == 0:init_loss = total_lossif total_loss < best_loss:best_loss = total_lossif not self.not_use_best_mse:- best_params = collect_best_params(block)+ best_params = collect_best_params(block, self.cache_device)# print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)last_best_iter = iif self.not_use_best_mse and i == self.iters - 1:- best_params = collect_best_params(block)+ best_params = collect_best_params(block, self.cache_device)if not self.not_use_best_mse:if 0 < self.dynamic_max_gap <= i - last_best_iter:@@ -2649,6 +2661,8 @@ class BaseCompressor(object):f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}")logger.info(dump_info)+ if self.low_gpu_mem_usage:+ clear_memory() # clear cached memory during trainingif len(unquantized_layer_names) != 0:logger.info(f"{unquantized_layer_names} have not been quantized")with torch.no_grad():@@ -2659,8 +2673,6 @@ class BaseCompressor(object):set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")if self.enable_quanted_input:- if self.low_gpu_mem_usage:- clear_memory()q_outputs = self._get_block_outputs(block,input_ids,@@ -2786,6 +2798,7 @@ class BaseCompressor(object):modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)+ m.config = model.config if hasattr(model, "config") else Noneq_input, input_ids = quantize_block(m,input_ids,@@ -2793,6 +2806,8 @@ class BaseCompressor(object):q_input=q_input,device=device,)+ if hasattr(model, "config"):+ del m.configif self.is_packing_immediate:from auto_round.export import PACKING_LAYER_WITH_FORMATdiff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 7d6f28806..ba5db377c 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -199,13 +199,14 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nonreturn True, ""-def collect_best_params(block):+def collect_best_params(block, cache_device="cpu"):+ """Collect the best parameters from the block to the specified device."""params = {}for n, m in block.named_modules():if hasattr(m, "orig_layer"):params[n] = {}for key in m.params.keys():- params[n][key] = copy.deepcopy(m.params[key].data)+ params[n][key] = m.params[key].data.to(cache_device, copy=True)return paramsdiff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 3cfcde3dc..b0ecf9019 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -440,31 +440,34 @@ def clear_memory_if_reached_threshold(threshold=0.85):"""# Detect CUDA/XPU devicesif torch.cuda.is_available():- name, device_api = "CUDA", torch.cuda+ name, device_api = "cuda", torch.cudaelif hasattr(torch, "xpu") and torch.xpu.is_available():- name, device_api = "XPU", torch.xpu+ name, device_api = "xpu", torch.xpuelse:- return+ return Falsenum_devices = device_api.device_count()for i in range(num_devices):try:total_memory = device_api.get_device_properties(i).total_memory- allocated_memory = device_api.memory_reserved(i) if name == "CUDA" else device_api.memory_allocated(i)- memory_usage_ratio = allocated_memory / total_memory+ reserved_memory = device_api.memory_reserved(i)+ memory_usage_ratio = reserved_memory / total_memoryif memory_usage_ratio >= threshold:logger.warning_once(- f"{name} device {i}: Memory usage {memory_usage_ratio*100:.2f}% "- f"exceeds threshold {threshold*100:.2f}%. Clearing memory..."+ f"Major device ({name}:{i}) has reached memory threshold. "+ + "Memory clearing operation will be called during each iteration, which "+ + "will result in more time consumption."+ )+ logger.warning_once(+ "To alleviate high memory usage on the major device, consider reducing the `batch_size` "+ + "(and correspondingly increasing `gradient_accumulation_steps) or shortening the seqlen.")clear_memory()- allocated_memory = device_api.memory_reserved(i) if name == "CUDA" else device_api.memory_allocated(i)- memory_usage_ratio = allocated_memory / total_memory- logger.warning_once(f"Cleared memory. {name} device {i}: Memory usage {memory_usage_ratio*100:.2f}%")return Trueexcept Exception as e:- logger.warning_once(f"Failed to check memory for {name} device {i}: {e}")+ logger.warning_once(f"Failed to check memory for {name}:{i}: {e}")+ return Falsedef check_memory_availability(device, inputs, weight, org_seqlen, org_bs):@@ -479,7 +482,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):Returns:tuple: A tuple containing availability status (bool), modified sequence length (int),- and modified batch size (int).+ and modified batch size (int)."""weight_memory = weight.numel() * weight.element_size()if "cuda" in device:@@ -741,8 +744,93 @@ def _allocate_layers_to_devices(return ordered_device_map, names+def get_first_available_attr(obj, attr_names: list[str], default=None):+ """+ Get the first available attribute from a list of attribute names.++ Args:+ obj: The object to get the attribute from.+ attr_names (list[str]): List of attribute names to try in order.+ default: Default value to return if none of the attributes exist.++ Returns:+ The value of the first available attribute, or default if none exist.+ """+ for attr_name in attr_names:+ value = getattr(obj, attr_name, None)+ if value is not None:+ return value+ return default+++def get_moe_memory_ratio(block: torch.nn.Module) -> float:+ """+ Calculate the memory ratio for MoE (Mixture of Experts) models.++ For MoE models, only num_experts_per_tok experts are activated per token,+ not all experts. This function returns the ratio of active experts to total experts.++ Args:+ block (torch.nn.Module): The model block to analyze.++ Returns:+ float: Memory ratio (num_experts_per_tok / num_experts).+ Returns 1.0 for non-MoE models.+ bool: True if the model is MoE, False otherwise.++ Examples:+ - Non-MoE model: returns 1.0+ - Mixtral (2/8 experts): returns 0.25+ - Qwen2MoE (4/60 experts): returns ~0.067+ """+ from auto_round.utils.model import is_moe++ for name, module in block.named_modules():+ if not is_moe(module):+ continue++ config = getattr(block, "config", None)+ if config is None:+ break++ # Try to get num_experts_per_tok (active experts count)+ # Mixtral, Qwen2MoE, DeepSeek, GPT-OSS, Llama4, LLaDAMoE, SmallThinker+ num_experts_per_tok = get_first_available_attr(+ config, ["num_experts_per_tok", "moe_num_active_primary_experts"]+ )++ # HunYuan MoE uses moe_topk (array), get first element+ if num_experts_per_tok is None:+ moe_topk = getattr(config, "moe_topk", None) # HunYuan MoE V1+ if moe_topk is not None and isinstance(moe_topk, (list, tuple)) and len(moe_topk) > 0:+ num_experts_per_tok = moe_topk[0]+ elif moe_topk is not None:+ num_experts_per_tok = moe_topk++ if num_experts_per_tok is None:+ break++ # Get total number of experts+ # Mixtral, PhiMoE, Grok, Llama4, Qwen2MoE, Olmo, BailingMoE, GroveMoE, HunYuan, LLaDAMoE, SmallThinker, DeepSeek+ num_experts = get_first_available_attr(+ config, ["num_local_experts", "num_experts", "moe_num_primary_experts", "n_routed_experts"]+ )++ if num_experts is not None and num_experts > 0:+ moe_ratio = num_experts_per_tok / num_experts+ logger.debug(+ f"MoE detected: {num_experts_per_tok}/{num_experts} experts active per token, "+ f"activation memory ratio: {moe_ratio:.2f}"+ )+ logger.debug(f"Using MoE memory ratio: {moe_ratio:.4f}")+ return moe_ratio, True+ break # Only check once per block++ return 1.0, False # Default ratio for non-MoE models++def estimate_tuning_block_mem(- block: torch.nn.Module, input_ids: list[torch.Tensor], pick_samples: int+ block: torch.nn.Module, input_ids: list[torch.Tensor], batch_size: int) -> tuple[dict, float]:"""Calculates the memory consumption of a specific block in the model.@@ -750,26 +838,27 @@ def estimate_tuning_block_mem(Args:block (torch.nn.Module): The block of the model to analyze.input_ids (list[torch.Tensor]): A list of input tensors for the block.- pick_samples (int): Number of samples to consider for memory estimation.+ batch_size (int): Number of samples to consider for memory estimation.Returns:tuple: A tuple containing the following:- layer_memory_dict (dict): A dictionary mapping layer names to their memory consumption (in GB).- Format: {layer_name: {"param_memory": float, "output_memory": float}}+ Format: {layer_name: {"param_memory": float, "output_memory": float}}- input_output_memory (float): The memory consumption (in GB) for input and outputtensors of the block.- additional_memory (float): Additional memory overhead (in GB) for operations like attention."""# Calculate all block parameters memory and build layer-wise memory dict- from auto_round.utils.model import get_layer_features+ from auto_round.utils.model import get_layer_features, is_moelayer_memory_dict = {}- total_param_mem = 0# Calculate batch_size and sequence_length from input_ids for output memory estimationseq_len = input_ids[0].shape[1] if input_ids and len(input_ids[0].shape) >= 2 else 1element_size = input_ids[0].element_size() if input_ids else 2 # Default to 2 bytes (fp16/bf16)+ moe_ratio, has_moe = get_moe_memory_ratio(block) # Get MoE memory ratio (1.0 for non-MoE models)+for name, module in block.named_modules():if check_to_quantized(module):enable_act_quant = module.act_bits <= 8@@ -782,45 +871,75 @@ def estimate_tuning_block_mem(in_features, out_features = get_layer_features(module)if in_features is not None and out_features is not None:# Output tensor size: batch_size * seq_len * out_features * element_size- output_size = pick_samples * seq_len * out_features * element_size+ output_size = batch_size * seq_len * out_features * element_sizeoutput_memory_gb = output_size / 1024**3# If enable_act_quant, add input tensor memory to param_memoryif enable_act_quant:- input_size = pick_samples * seq_len * in_features * element_size+ input_size = batch_size * seq_len * in_features * element_sizeinput_memory_gb = input_size / 1024**3param_memory_gb += input_memory_gbelse:output_memory_gb = 0.0+ if has_moe:+ pparent_module = get_module(block, layer_name.rsplit(".", 2)[0]) if "." in layer_name else block+ is_moe_expert = "expert" in layer_name.lower() and isinstance(pparent_module, torch.nn.ModuleList)+ else:+ is_moe_expert = False+# memory * 2, because it contains grad tensor.- layer_memory_dict[layer_name] = {"param_memory": param_memory_gb * 2, "output_memory": output_memory_gb * 2}+ layer_memory_dict[layer_name] = {+ "param_memory": param_memory_gb * 2,+ "output_memory": output_memory_gb * 2,+ "is_moe_expert": is_moe_expert,+ }# Assuming bfloat16 or float32, input and outputblock_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_ids) / 1024**3# Roughly estimate additional memory for attention and other operations- additional_activation_memory = sum(info["output_memory"] for info in layer_memory_dict.values())+ # For MoE expert layers, multiply activation memory by the ratio of active experts+ # For non-MoE layers (attention, norm, etc.), use full activation memory+ layer_activation_memory = 0.0+ for layer_name, info in layer_memory_dict.items():+ if info.get("is_moe_expert", False):+ # MoE expert layer: only a fraction of experts are active+ layer_activation_memory += info["output_memory"] * moe_ratio+ else:+ # Non-MoE layer: use full activation memory+ layer_activation_memory += info["output_memory"]++ # layer_activation_memory considers other ops activation memory# 1GB considers norm weight, sdpa, reference_output, etc.- additional_memory = additional_activation_memory + 1 # GB+ additional_memory = layer_activation_memory + 1 # GB+ if has_moe:+ # TODO: Cannot estimate the memory usage correctly for MoE models yet.+ # For MoE models, additional memory usage can be higher due to routing, gating,+ # and multiple expert activations. Here we use a conservative estimate.+ moe_additional_memory = additional_memory * 6 # GB+ additional_memory += moe_additional_memoryif torch.xpu.is_available():# https://github.com/intel/torch-xpu-ops/issues/2232# TODO: XPU takes more memory than expected. for llama 8B, it's about 12 GBxpu_additional_memory = 12 # GBadditional_memory += xpu_additional_memory- logger.warning_once("XPU additional memory usage of SDPA is estimated to be 12 GB.")- logger.warning_once("Remove it after https://github.com/intel/torch-xpu-ops/issues/2232 is fixed.")+ logger.warning_once(+ "[Memory Estimation]: If there is an abnormal memory issue, please collect log with "+ + "AR_LOG_LEVEL=debug and raise issue to us."+ )- return layer_memory_dict, block_input_output_memory, additional_memory+ return layer_memory_dict, layer_activation_memory, block_input_output_memory, additional_memorydef set_auto_device_map_for_block_with_tuning(block: torch.nn.Module,device_map,input_ids: list[torch.Tensor],- low_gpu_mem_usage=False,- pick_samples=8,- output_device=None,+ low_gpu_mem_usage: bool = False,+ batch_size: int = 8,+ output_device: str | torch.device = None,+ card_0_threshold: float = 0.9,):"""Automatically sets the device map for the block based on available GPUs and memory constraints.@@ -830,11 +949,16 @@ def set_auto_device_map_for_block_with_tuning(device_map (str | int | dict): Specifies the device mapping.input_ids (list[torch.Tensor]): List of input tensors used for estimating memory requirements.low_gpu_mem_usage (bool, optional): If True, ignoring input/output memory. Defaults to False.- pick_samples (int, optional): Number of samples to consider for memory estimation. Defaults to 8.+ batch_size (int, optional): Number of samples to consider for memory estimation. Defaults to 8.output_device (str | torch.device, optional): Device to move unassigned modules to. Defaults to None.+ card_0_threshold (float, optional): Threshold ratio to determine if the first device is at high risk of+ running out of memory. Defaults to 0.9 (90%).Returns:- None+ card_0_in_high_risk (bool): True if the first device is at risk of running out of memory, False otherwise.+ card_0_in_high_risk = card_0_used_memory / device_0_memory > card_0_threshold+ card_0_used_memory = card_0_left_memory + block_input_output_memory + additional_memory+ We may need to clear card 0 memory more frequently during training/inference in that case.Raises:RuntimeError: If no CUDA or XPU devices are found.@@ -842,6 +966,12 @@ def set_auto_device_map_for_block_with_tuning(Note:This function is intended for internal use in device memory management and tuning."""+ if not (device_map == "auto" or ((isinstance(device_map, str) and "," in device_map))):+ block = block.to(output_device)+ card_0_in_high_risk = False # card 0 contains weight, clear_memory will not help much+ loss_device = output_device+ return card_0_in_high_risk, loss_device+if torch.cuda.is_available():num_devices = torch.cuda.device_count()device_name = "cuda"@@ -857,35 +987,39 @@ def set_auto_device_map_for_block_with_tuning(if device_list:gpu_devices = [f"{device_name}:{i}" for i in device_list]device_0 = gpu_devices[0]+ device_1 = gpu_devices[1]else:gpu_devices = [f"{device_name}:{i}" for i in range(num_devices)]device_0 = f"{device_name}:0"+ device_1 = f"{device_name}:1"device_0_memory = get_device_memory(device_list[0] if device_list else 0)- layer_memory_dict, block_input_output_memory, additional_memory = estimate_tuning_block_mem(- block, input_ids, pick_samples+ device_1_memory = get_device_memory(device_list[1] if device_list else 1)+ layer_memory_dict, layer_activation_memory, block_input_output_memory, additional_memory = (+ estimate_tuning_block_mem(block, input_ids, batch_size))+ loss_memory = block_input_output_memory / 2 # GB, rough estimate for loss tensor memoryif low_gpu_mem_usage:block_input_output_memory = 0- # Calculate total block memory from layer memory dict (including both param and output memory)total_block_param_memory = sum(info["param_memory"] for info in layer_memory_dict.values())- total_block_output_memory = sum(info["output_memory"] for info in layer_memory_dict.values())# Average dispatch strategy# card_0_left_memory = card_0_mem - block_input_output_memory - additional_memory - layer_outputs_memory- logger.debug("Card 0 used memory details [Estimated]:")+ card_0_used_memory = block_input_output_memory + layer_activation_memory + additional_memory+ logger.debug(f"Card 0 used memory details [Estimated]: {card_0_used_memory} GB")logger.debug(f" Block input output cache memory: {block_input_output_memory} GB")- logger.debug(f" Quantized layer outputs memory: {total_block_output_memory} GB")+ logger.debug(f" Quantized layer outputs memory: {layer_activation_memory} GB")logger.debug(f" Additional_memory from other ops: {additional_memory} GB")- card_0_left_memory = max(- 0, device_0_memory - block_input_output_memory - total_block_output_memory - additional_memory- )+ card_0_left_memory = max(0, (device_0_memory - card_0_used_memory))+ card_0_in_high_risk = card_0_used_memory / device_0_memory >= card_0_threshold+ card_1_left_memory = max(0, device_1_memory - loss_memory) if card_0_in_high_risk else device_1_memory+ loss_device = device_1 if card_0_in_high_risk else output_device# Calculate total available memory across all devices- total_available_memory = card_0_left_memory- for i in range(1, len(gpu_devices)):+ total_available_memory = card_0_left_memory + card_1_left_memory+ for i in range(2, len(gpu_devices)):device_idx = device_list[i] if device_list else itotal_available_memory += get_device_memory(device_idx)@@ -916,6 +1050,8 @@ def set_auto_device_map_for_block_with_tuning(if has_params or has_buffers:module = module.to(output_device)+ return card_0_in_high_risk, loss_device+def partition_dict_numbers(number_dict, n):"""@@ -986,18 +1122,18 @@ def partition_dict_numbers(number_dict, n):def set_avg_auto_device_map(model: torch.nn.Module, device_map):block_name_list = get_block_names(model)device_list = None+ if torch.cuda.is_available():+ num_devices = torch.cuda.device_count()+ device_name = "cuda"+ elif torch.xpu.is_available():+ num_devices = torch.xpu.device_count()+ device_name = "xpu"+ else:+ return+if isinstance(device_map, str) and "," in device_map:device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]num_devices = len(device_list)- else:- if torch.cuda.is_available():- num_devices = torch.cuda.device_count()- device_name = "cuda"- elif torch.xpu.is_available():- num_devices = torch.xpu.device_count()- device_name = "xpu"- else:- returnif device_list:gpu_devices = [f"{device_name}:{i}" for i in device_list]diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex f8b42ec7a..4d2c7d5fd 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -550,7 +550,7 @@ class WrapperLayerNorm(torch.nn.Module):def unwrapper(self, best_params):if best_params is None:return self.orig_layer- v = best_params["v"]+ v = best_params["v"].to(self.device)weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh)@@ -601,7 +601,7 @@ class WrapperLlamaNorm(torch.nn.Module):def unwrapper(self, best_params):if best_params is None:return self.orig_layer- v = best_params["v"]+ v = best_params["v"].to(self.device)weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh)