PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
dispatch model with real max memory (#1022)
Xin He255322fba89d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex a2b61eae3..c098320f9 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -25,7 +25,7 @@ from typing import Any, Callable, Optional, Unionimport accelerateimport torchfrom accelerate.big_modeling import dispatch_model, infer_auto_device_map-from accelerate.utils import get_balanced_memory+from accelerate.utils import get_max_memoryfrom torch import autocastfrom tqdm import tqdmfrom transformers import set_seed@@ -1992,11 +1992,7 @@ class BaseCompressor(object):if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):no_split_modules = getattr(self.model, "_no_split_modules", [])devices = parse_available_devices(self.device_map)- max_memory = get_balanced_memory(- self.model,- max_memory=None,- no_split_module_classes=no_split_modules,- )+ max_memory = get_max_memory()new_max_memory = {}for device in devices:if ":" in device: