PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix low cpu speed issue (#1251)
Wenhua Cheng90db4445b236
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex aec2bbee1..5605393e6 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1170,7 +1170,10 @@ class BaseCompressor(object):m = m.to("cpu")set_module(self.model, name, m)if self.immediate_saving:- all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]+ if hasattr(self, "all_to_quantized_module_names"):+ all_to_quantized_module_names = self.all_to_quantized_module_names+ else:+ all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])m = get_module(self.model, name)immediate_saving(self, m, name, last_module)@@ -1204,6 +1207,7 @@ class BaseCompressor(object):self.model.to(self.amp_dtype)all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]+ self.all_to_quantized_module_names = all_to_quantized_module_namesif is_nv_fp(self.data_type):from auto_round.data_type.nvfp import calculate_gparamfrom auto_round.data_type.utils import update_fused_layer_global_scales@@ -1278,6 +1282,7 @@ class BaseCompressor(object):self._quantize_layer_via_rtn(name)if cnt % clear_mem_freq == 0:clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()cnt = 1cnt += 1# Convert remaining fp8diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex b5ea27dea..4a4d4e1ff 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -966,7 +966,9 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last# User configurable (can be preset on rounder)max_shard_size = getattr(rounder, "max_shard_size", "5GB")safe_serialization = getattr(rounder, "safe_serialization", True)- layer_names = rounder._get_quantized_layer_names_outside_blocks()+ if not hasattr(rounder, "quantized_layer_names_outside_blocks"):+ rounder.quantized_layer_names_outside_blocks = rounder._get_quantized_layer_names_outside_blocks()+ layer_names = rounder.quantized_layer_names_outside_blocksif len(layer_names) > 0 and name != layer_names[-1]:last_group = False@@ -1046,7 +1048,6 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, lastdel rounder._current_shard_tensors[param]rounder._current_shard_tensors = OrderedDict()rounder._current_shard_size = 0- clear_memory()for pname, tensor in flat_tensors.items():t_elems = tensor.numel()@@ -1177,7 +1178,6 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, lastfor _attr in attrs_to_cleanup:if hasattr(rounder, _attr):delattr(rounder, _attr)- clear_memory()except Exception as _cleanup_err:logger.warning(f"shard cleanup warning: {_cleanup_err}")