PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add blockwise way in pure rtn to ease module replacements (#1320)
Wenhua Cheng57d1a8acfbe9
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 81d74e325..fe2d2d092 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1283,11 +1283,7 @@ class BaseCompressor(object):for handle in hook_handles:handle.remove()else:- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt- cnt = 0- pbar = tqdm(all_to_quantized_module_names)- # A workaround to trigger garbage collection in time+ use_blockwise_quantization = Falsetied_weight_keys = getattr(self.model, "_tied_weight_keys", {})tied_weight_values = list(tied_weight_keys.values())# In fact, we should detect whether it is is_separate_lm_head, to simplify, we don't do it@@ -1295,21 +1291,61 @@ class BaseCompressor(object):lm_head_name = get_lm_head_name(self.model)if lm_head_name is not None:tied_weight_values.append(lm_head_name)- for n, m in self.model.named_modules():- if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:- pbar.set_description(f"Quantizing {m.global_name}")- self._quantize_layer_via_rtn(m.global_name)++ if use_blockwise_quantization: # The ram usage is a little higher+ all_to_quantized_module_names = list(set(all_to_quantized_module_names))++ all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)+ pbar = tqdm(range(sum(len(block) for block in all_blocks)))+ for block_names in all_blocks:+ for block_name in block_names:+ pbar.set_description(f"Quantizing {block_name}")+ block = get_module(self.model, block_name)+ for name, m in block.named_modules():+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:+ self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)+ all_to_quantized_module_names.remove(m.global_name)+ elif (+ not any(m.children())+ and len(m.state_dict()) > 0+ and m.global_name not in tied_weight_values+ ):+ set_module(self.model, m.global_name, copy.deepcopy(m))+ if self.is_immediate_saving:+ shard_writer(self, name=m.global_name)+ m.to("meta")+ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()+ pbar.update(1)+ cnt = 1+ for name in all_to_quantized_module_names:+ logger.info(f"Quantizing remaining layer {name} on CPU.")+ self._quantize_layer_via_rtn(name, to_cpu=True)cnt += 1- pbar.update()- if cnt % clear_mem_freq == 0:+ if cnt % 10 == 0:clear_memory(device_list=self.device_list)memory_monitor.log_summary()+ else:+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ cnt = 0+ pbar = tqdm(all_to_quantized_module_names)- elif not any(m.children()) and len(m.state_dict()) > 0 and n not in tied_weight_values:- set_module(self.model, n, copy.deepcopy(m))- if self.is_immediate_saving:- shard_writer(self, name=n)- m.to("meta")+ for n, m in self.model.named_modules():+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:+ pbar.set_description(f"Quantizing {m.global_name}")+ self._quantize_layer_via_rtn(m.global_name)+ cnt += 1+ pbar.update()+ if cnt % clear_mem_freq == 0:+ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()++ elif not any(m.children()) and len(m.state_dict()) > 0 and n not in tied_weight_values:+ set_module(self.model, n, copy.deepcopy(m))+ if self.is_immediate_saving:+ shard_writer(self, name=n)+ m.to("meta")# Convert remaining fp8if is_fp8_model(self.model):