PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add memory monitor and import auto-scheme on demand (#1049)
Wenhua Cheng247a9a7bcffa
diff --git a/auto_round/auto_scheme/__init__.py b/auto_round/auto_scheme/__init__.pyindex f2682d376..583086c6b 100644--- a/auto_round/auto_scheme/__init__.py+++ b/auto_round/auto_scheme/__init__.py@@ -2,6 +2,7 @@## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0@@ -11,12 +12,21 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from auto_round.logger import loggerfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme-from auto_round.auto_scheme.register import AUTO_SCHEME_METHODS-try:- import auto_round.auto_scheme.default_alg-except ImportError:- logger.warning("AutoScheme is currently supported only on Linux.")++def __getattr__(name):+ if name == "AUTO_SCHEME_METHODS":+ try:+ import auto_round.auto_scheme.default_alg+ except ImportError:+ logger.warning("AutoScheme is currently supported only on Linux.")++ from auto_round.auto_scheme.register import AUTO_SCHEME_METHODS++ return AUTO_SCHEME_METHODS++ raise AttributeError(f"auto-scheme has no attribute '{name}'")diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex 8e72db35b..bc76a5543 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -17,7 +17,6 @@ from typing import Iterable, Optional, Unionimport torch-from auto_round.auto_scheme.register import AUTO_SCHEME_METHODSfrom auto_round.auto_scheme.utils import compute_avg_bits_for_schemefrom auto_round.compressors.utils import gguf_type_fallbackfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG@@ -103,7 +102,9 @@ class GenScheme:def get_layer_config(self) -> dict[str, dict]:method_name = self.auto_scheme.method- method_func = AUTO_SCHEME_METHODS[method_name]+ from auto_round import auto_scheme++ method_func = auto_scheme.AUTO_SCHEME_METHODS[method_name]if self.auto_scheme.low_gpu_mem_usage:self.enable_torch_compile = Falsediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 7e9288f8d..31683eb7c 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -87,6 +87,7 @@ from auto_round.utils import (is_fp8_model,is_hpex_available,llm_load_model,+ memory_monitor,mv_module_from_gpu,normalize_input,set_amax_for_all_moe_layers,@@ -1025,6 +1026,7 @@ class BaseCompressor(object):self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)folders.append(save_folder)+ memory_monitor.log_summary()return model, folders@@ -1513,6 +1515,7 @@ class BaseCompressor(object):all_to_quantized_module_names.remove(m.tmp_name)if not self.immediate_saving:mv_module_from_gpu(block)+ memory_monitor.log_summary()pbar.update(1)pbar.close()@@ -1752,6 +1755,8 @@ class BaseCompressor(object):layer.cpu()layer_names.remove(layer_name)if len(layer_names) == 0:+ memory_monitor.update()+ memory_monitor.log_summary()returnq_layer_inputs = Noneenable_quanted_input = self.enable_quanted_input@@ -1770,7 +1775,7 @@ class BaseCompressor(object):if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(self.model- ) ##self.model.hf_device_map has not been changed+ ) # self.model.hf_device_map has not been changedif not self.immediate_saving:self.model = mv_module_from_gpu(self.model)clear_memory(device_list=self.device_list)@@ -1789,13 +1794,14 @@ class BaseCompressor(object):immediate_saving(self, m, name=layer_name, last_group=True)del layer_inputclear_memory(q_layer_input, device_list=self.device_list)+ memory_monitor.log_summary()@torch.no_grad()def _get_block_outputs(self,block: torch.nn.Module,- input_ids: torch.Tensor,- input_others: torch.Tensor,+ input_ids: torch.Tensor | list[torch.Tensor],+ input_others: torch.Tensor | dict,bs: int,device: Union[str, torch.device],cache_device: Union[str, torch.device],@@ -2805,7 +2811,7 @@ class BaseCompressor(object):f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}")- logger.info(dump_info)+if self.low_gpu_mem_usage:clear_memory(device_list=self.device_list) # clear cached memory during trainingif len(unquantized_layer_names) != 0:@@ -2833,6 +2839,8 @@ class BaseCompressor(object):mv_module_from_gpu(block)clear_memory(input_ids)+ memory_info_summary = memory_monitor.get_summary()+ logger.infoclean(dump_info + "," + memory_info_summary)return q_outputs, outputelse:@@ -2841,6 +2849,8 @@ class BaseCompressor(object):if auto_offload:mv_module_from_gpu(block)clear_memory(input_ids)+ memory_info_summary = memory_monitor.get_summary()+ logger.infoclean(dump_info + "," + memory_info_summary)return None, output@@ -3174,7 +3184,7 @@ class BaseCompressor(object):cls,input_ids: Union[list[torch.Tensor], dict],input_others: dict,- indices: list[int],+ indices: list[int] | torch.Tensor,seqlen: int,batch_dim: int = 0,share_cache_keys: tuple = (),diff --git a/auto_round/logger.py b/auto_round/logger.pyindex a43be3167..afcce709d 100644--- a/auto_round/logger.py+++ b/auto_round/logger.py@@ -53,7 +53,18 @@ def trace(self, message, *args):# Add the trace method to the Logger class+logging.Logger.trace = trace+INFOCLEAN_LEVEL = 21+logging.addLevelName(INFOCLEAN_LEVEL, "INFOCLEAN")+++def infoclean(self, message, *args, **kwargs):+ if self.isEnabledFor(INFOCLEAN_LEVEL):+ self._log(INFOCLEAN_LEVEL, message, args, **kwargs)+++logging.Logger.infoclean = infocleanclass AutoRoundFormatter(logging.Formatter):@@ -65,10 +76,11 @@ class AutoRoundFormatter(logging.Formatter):cyan = "\x1b[36;1m"blue = "\x1b[34;1m"_format = "%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s"-+ _format_clean = "%(message)s"FORMATS = {logging.DEBUG: blue + _format + reset,logging.INFO: grey + _format + reset,+ INFOCLEAN_LEVEL: grey + _format_clean + reset,logging.WARNING: yellow + _format + reset,logging.ERROR: bold_red + _format + reset,logging.CRITICAL: bold_red + _format + reset,diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 2f63a3a2d..38674c508 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -16,9 +16,11 @@ import osimport refrom functools import lru_cachefrom itertools import combinations+from threading import Lockfrom typing import Callable, Unionimport cpuinfo+import psutilimport torchfrom auto_round.logger import logger@@ -442,6 +444,7 @@ def _clear_memory_for_cpu_and_cuda(def clear_memory(tensor: torch.Tensor | None | list[torch.Tensor] = None, device_list: list | tuple | None = None):from auto_round.utils.device import is_hpex_available+ memory_monitor.update(device_list=device_list)if is_hpex_available():# hpu does not have empty_cachereturn@@ -1308,3 +1311,95 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, Nonereturn sorted(devices)raise TypeError(f"Unsupported device_map type: {type(device_map)}")+++class MemoryMonitor:+ """Global memory monitor for tracking peak RAM and VRAM usage."""++ _instance = None+ _lock = Lock()+ _initialized = False++ def __new__(cls):+ if cls._instance is None:+ with cls._lock:+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ cls._instance._initialized = False+ return cls._instance++ def __init__(self):+ if self._initialized:+ return+ self._initialized = True+ self.peak_ram = 0.0 # GB+ self.peak_vram = {} # {device_id: peak_mb}+ self.enabled = True++ def update(self, device_list=None):+ """Update current memory usage and track peaks."""+ if not self.enabled:+ return+ # Track RAM+ process = psutil.Process()+ current_ram = process.memory_info().rss / 1024**3 # GB+ self.peak_ram = max(self.peak_ram, current_ram)+ if device_list is None: # TODO this have issue, wait for clean memory all pass device_list+ device_list = [0]+ if device_list is not None:+ if not isinstance(device_list, (list, tuple)):+ device_list = [device_list]+ else:+ if torch.cuda.is_available():+ device_list = list(range(torch.cuda.device_count()))+ elif torch.xpu.is_available():+ device_list = list(range(torch.xpu.device_count()))++ for device in device_list:+ if device == "cpu":+ continue+ if torch.cuda.is_available():+ current_vram = torch.cuda.memory_reserved(device) / 1024**3 # GB+ elif torch.xpu.is_available():+ current_vram = torch.xpu.memory_reserved(device) / 1024**3 # GB+ else:+ return++ device = str(device).split(":")[-1]+ if current_vram > 0:+ if device not in self.peak_vram:+ self.peak_vram[device] = 0.0++ self.peak_vram[device] = max(self.peak_vram[device], current_vram)++ def update_cpu(self):+ if not self.enabled:+ return+ process = psutil.Process()+ current_ram = process.memory_info().rss / 1024**3 # GB+ self.peak_ram = max(self.peak_ram, current_ram)++ def reset(self):+ """Reset all statistics."""+ self.peak_ram = 0.0+ self.peak_vram = {}++ def get_summary(self):+ """Get summary of peak memory usage."""+ summary = f"'peak_ram': {round(self.peak_ram, 2)}GB"+ if len(self.peak_vram) > 0:+ sorted_items = sorted(self.peak_vram.items())+ items_str = ", ".join([f"'{k}': {round(v, 2)}GB" for k, v in sorted_items])+ summary += f", 'peak_vram': {{{items_str}}}"+ return summary++ def log_summary(self):+ """Log memory usage summary."""+ summary = self.get_summary()+ logger.info(summary)++ return summary+++# Global singleton instance+memory_monitor = MemoryMonitor()diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex a434d9010..29924bc79 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -725,7 +725,7 @@ If not explicitly specify '--task', the default value will be used (typically co~~~The last format will be used in evaluation if multiple formats have been exported.-Note: To use the vllm backend, please add `--vllm` into the upper command.+Note: To use the vllm backend, please add `--eval_backend vllm` to the command above.### Eval the Quantized model