Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix low cpu speed issue (#1251)

Wenhua Cheng90db4445b236
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index aec2bbee1..5605393e6 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1170,7 +1170,10 @@ class BaseCompressor(object):
m = m.to("cpu")
set_module(self.model, name, m)
if self.immediate_saving:
- all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
+ if hasattr(self, "all_to_quantized_module_names"):
+ all_to_quantized_module_names = self.all_to_quantized_module_names
+ else:
+ all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])
m = get_module(self.model, name)
immediate_saving(self, m, name, last_module)
@@ -1204,6 +1207,7 @@ class BaseCompressor(object):
self.model.to(self.amp_dtype)
all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
+ self.all_to_quantized_module_names = all_to_quantized_module_names
if is_nv_fp(self.data_type):
from auto_round.data_type.nvfp import calculate_gparam
from auto_round.data_type.utils import update_fused_layer_global_scales
@@ -1278,6 +1282,7 @@ class BaseCompressor(object):
self._quantize_layer_via_rtn(name)
if cnt % clear_mem_freq == 0:
clear_memory(device_list=self.device_list)
+ memory_monitor.log_summary()
cnt = 1
cnt += 1
# Convert remaining fp8
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index b5ea27dea..4a4d4e1ff 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -966,7 +966,9 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last
# User configurable (can be preset on rounder)
max_shard_size = getattr(rounder, "max_shard_size", "5GB")
safe_serialization = getattr(rounder, "safe_serialization", True)
- layer_names = rounder._get_quantized_layer_names_outside_blocks()
+ if not hasattr(rounder, "quantized_layer_names_outside_blocks"):
+ rounder.quantized_layer_names_outside_blocks = rounder._get_quantized_layer_names_outside_blocks()
+ layer_names = rounder.quantized_layer_names_outside_blocks
if len(layer_names) > 0 and name != layer_names[-1]:
last_group = False
@@ -1046,7 +1048,6 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last
del rounder._current_shard_tensors[param]
rounder._current_shard_tensors = OrderedDict()
rounder._current_shard_size = 0
- clear_memory()
for pname, tensor in flat_tensors.items():
t_elems = tensor.numel()
@@ -1177,7 +1178,6 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last
for _attr in attrs_to_cleanup:
if hasattr(rounder, _attr):
delattr(rounder, _attr)
- clear_memory()
except Exception as _cleanup_err:
logger.warning(f"shard cleanup warning: {_cleanup_err}")