PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add device list to clear memory (#1009)
Wenhua Cheng1d02da16dba2
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 45ed0c14f..8f398e7a1 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1459,12 +1459,12 @@ class BaseCompressor(object):if is_fp8_model(self.model):convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype, device=self.device)- if is_auto_device_mapping(self.device_map):+ if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:set_auto_device_map_for_block_with_tuning(block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, self.device)# Dispatch model if needed- if len(self.device_list) > 0:+ if len(self.device_list) > 1:from accelerate.hooks import AlignDevicesHook, add_hook_to_modulefor _, m in block.named_modules():@@ -2498,7 +2498,7 @@ class BaseCompressor(object):set_module(block, n, new_layer)# card_0_in_high_risk indicates that card_0 memory is already in high usage (90%) w/o any weights# loss_device is used to calculate loss on the second device if available and card_0_in_high_risk- if self.device_map == "auto" or ((isinstance(self.device_map, str) and "," in self.device_map)):+ if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, device)@@ -2699,7 +2699,7 @@ class BaseCompressor(object):)logger.info(dump_info)if self.low_gpu_mem_usage:- clear_memory() # clear cached memory during training+ clear_memory(self.device_list) # clear cached memory during trainingif len(unquantized_layer_names) != 0:logger.info(f"{unquantized_layer_names} have not been quantized")with torch.no_grad():@@ -2721,7 +2721,7 @@ class BaseCompressor(object):if len(self.device_list) > 1:accelerate.hooks.remove_hook_from_submodules(block)mv_module_from_gpu(block)- clear_memory(input_ids)+ clear_memory(input_ids, self.device_list)return q_outputs, output@@ -2729,7 +2729,7 @@ class BaseCompressor(object):if len(self.device_list) > 1:accelerate.hooks.remove_hook_from_submodules(block)mv_module_from_gpu(block)- clear_memory(input_ids)+ clear_memory(input_ids, self.device_list)return None, outputdef _split_inputs(self, inputs: dict) -> tuple[torch.Tensor, dict]: