Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support for immediate saving to reduce ram usage (#965)

Tang Kaihuidaeb3bb78814
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index d52914447..f2d2b1346 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -85,6 +85,7 @@ class AutoRound:
enable_adam: bool = False,
# for MLLM and Diffusion
extra_config: ExtraConfig = None,
+ low_cpu_mem_usage: bool = False,
**kwargs,
) -> BaseCompressor:
"""Initialize AutoRound with quantization and tuning configuration.
@@ -105,6 +106,7 @@ class AutoRound:
lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.
iters (int, optional): Optimization iterations. Defaults to 200.
seqlen (int, optional): Calibration sequence length. Defaults to 2048.
nsamples (int, optional): Number of calibration samples. Defaults to 128.
@@ -186,6 +188,7 @@ class AutoRound:
device_map=device_map,
enable_torch_compile=enable_torch_compile,
seed=seed,
+ low_cpu_mem_usage=low_cpu_mem_usage,
**kwargs,
)
return ar
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 0a682079a..8763e528b 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -39,6 +39,7 @@ from auto_round.compressors.utils import (
collect_best_params,
get_shared_keys,
gguf_args_check,
+ immediate_saving,
infer_bits_by_data_type,
init_cache,
is_mx_fp,
@@ -148,6 +149,7 @@ class BaseCompressor(object):
enable_alg_ext: bool = False,
disable_opt_rtn: bool = False,
seed: int = 42,
+ low_cpu_mem_usage: bool = False,
**kwargs,
):
"""Initialize AutoRound with quantization and tuning configuration.
@@ -168,6 +170,7 @@ class BaseCompressor(object):
lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.
iters (int, optional): Optimization iterations. Defaults to 200.
seqlen (int, optional): Calibration sequence length. Defaults to 2048.
nsamples (int, optional): Number of calibration samples. Defaults to 128.
@@ -244,6 +247,7 @@ class BaseCompressor(object):
self.supported_types = SUPPORTED_LAYER_TYPES
self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
+ self.low_cpu_mem_usage = low_cpu_mem_usage
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
@@ -336,7 +340,10 @@ class BaseCompressor(object):
self.lr_scheduler = lr_scheduler
self.optimizer = self._get_optimizer(None)
self.disable_opt_rtn = disable_opt_rtn
- self.is_packing_immediate = False # whether to pack the layer immediately after tuning
+
+ # Whether to pack the layer immediately after tuning
+ self.immediate_packing = False
+ self.immediate_saving = False
# KV cache, this one does not affect tuning but will collect some infos during tuning
self.static_kv_dtype = static_kv_dtype
@@ -1205,7 +1212,7 @@ class BaseCompressor(object):
`rtn_*` version if supported, then wraps and unwraps the module to apply
quantization. If GPU memory is insufficient, it falls back to CPU.
- If packing is enabled (`is_packing_immediate`), the function will also export
+ If packing is enabled (`immediate_packing`), the function will also export
the quantized layer to the appropriate backend format.
Args:
@@ -1222,7 +1229,7 @@ class BaseCompressor(object):
# Step 1: Try quantization on GPU first, fall back to CPU if OOM
# if only export gguf, using gguf-packing instead of rtn
- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:
+ if self.immediate_packing and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:
m.scale = None
m.zp = None
else:
@@ -1259,34 +1266,45 @@ class BaseCompressor(object):
raise
# Step 2: Optional immediate packing/export
- if self.is_packing_immediate:
- from auto_round.export import PACKING_LAYER_WITH_FORMAT
-
- if check_to_quantized(m):
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- has_gguf = any("gguf" in fmt for fmt in self.formats)
-
- if has_gguf:
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- output_dir = self._get_save_folder_name(self.formats[0])
- model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT
- pack_gguf_layer(
- name,
- self.model,
- self.formats[0],
- output_dir,
- self.layer_config,
- self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- model_type=model_type,
- )
- else:
- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
+ if self.immediate_packing:
+ self._immediate_pack(name)
else:
set_module(self.model, name, m)
+ if self.immediate_saving:
+ all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
+ last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])
+ m = get_module(self.model, name)
+ immediate_saving(self, m, name, last_module)
+
+ def _immediate_pack(self, name: str):
+ m = get_module(self.model, name)
+ if not check_to_quantized(m):
+ return
+ from auto_round.export import PACKING_LAYER_WITH_FORMAT
+
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ has_gguf = any("gguf" in fmt for fmt in self.formats)
+
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+
+ output_dir = self._get_save_folder_name(self.formats[0])
+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT
+ pack_gguf_layer(
+ name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type,
+ )
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
+
@torch.inference_mode()
def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
@@ -1301,7 +1319,6 @@ class BaseCompressor(object):
self.model.to(self.amp_dtype)
all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
-
if is_nv_fp(self.data_type):
from auto_round.data_type.nvfp import calculate_gparam
from auto_round.data_type.utils import update_fused_layer_global_scales
@@ -1477,8 +1494,8 @@ class BaseCompressor(object):
if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
self._quantize_layer_via_rtn(m.tmp_name)
all_to_quantized_module_names.remove(m.tmp_name)
-
- mv_module_from_gpu(block)
+ if not self.immediate_saving:
+ mv_module_from_gpu(block)
pbar.update(1)
pbar.close()
@@ -1556,7 +1573,12 @@ class BaseCompressor(object):
)
and self.inplace
):
- self.is_packing_immediate = True
+ self.immediate_packing = True
+ if "gguf" not in formats[0] and self.low_cpu_mem_usage:
+ self.immediate_saving = True
+ if self.immediate_saving and "int" not in self.data_type:
+ logger.warning("immediate_saving is only supported for int quantization, set to False")
+ self.immediate_saving = False
if self.iters == 0:
return self._quantize_rtn()
@@ -1628,15 +1650,14 @@ class BaseCompressor(object):
device=self.device,
pbar=pbar,
)
- if self.is_packing_immediate and len(self.formats) != 1:
+ if self.immediate_packing and len(self.formats) != 1:
raise ValueError(
- f"Expected exactly one packing format when 'is_packing_immediate' is True, "
+ f"Expected exactly one packing format when 'immediate_packing' is True, "
f"but got {len(self.formats)} formats."
)
pbar.set_description("Quantizing done")
pbar.close()
-
- self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately
+ self._quantize_layers(layer_names, all_inputs)
if is_fp8_model(self.model):
for n, m in self.model.named_modules():
@@ -1714,7 +1735,7 @@ class BaseCompressor(object):
has_gguf = False
if hasattr(self, "formats"):
has_gguf = any("gguf" in format_ for format_ in self.formats)
- if has_gguf and self.is_packing_immediate:
+ if has_gguf and self.immediate_packing:
enable_quanted_input = False
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
@@ -1727,8 +1748,8 @@ class BaseCompressor(object):
accelerate.hooks.remove_hook_from_submodules(
self.model
) ##self.model.hf_device_map has not been changed
-
- self.model = mv_module_from_gpu(self.model)
+ if not self.immediate_saving:
+ self.model = mv_module_from_gpu(self.model)
clear_memory()
quant_layer = self._quantize_layer
for layer_name in layer_names:
@@ -1737,6 +1758,12 @@ class BaseCompressor(object):
q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
q_layer_input = to_device(q_layer_input, self.cache_device)
quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
+ if self.immediate_packing:
+ self._immediate_pack(layer_name)
+
+ if self.immediate_saving:
+ m = get_module(self.model, layer_name)
+ immediate_saving(self, m, name=layer_name, last_group=True)
del layer_input
clear_memory(q_layer_input)
@@ -1937,7 +1964,6 @@ class BaseCompressor(object):
layer_names = []
if layer_names is None:
layer_names = []
-
if self.low_gpu_mem_usage or (
len(block_names) == 1
and len(layer_names) == 0
@@ -2732,6 +2758,7 @@ class BaseCompressor(object):
input_ids = to_device(input_ids, self.cache_device)
input_others = to_device(input_others, self.cache_device)
# As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
+
tmp_dtype = self.amp_dtype if self.amp else torch.float32
input_ids = to_dtype(input_ids, tmp_dtype)
@@ -2808,38 +2835,20 @@ class BaseCompressor(object):
)
if hasattr(model, "config"):
del m.config
- if self.is_packing_immediate:
- from auto_round.export import PACKING_LAYER_WITH_FORMAT
-
+ if self.immediate_packing:
for _, tmp_m in m.named_modules():
if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
continue
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- has_gguf = any("gguf" in format_ for format_ in self.formats)
- if has_gguf:
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- output_dir = self._get_save_folder_name(self.formats[0])
- model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT
- pack_gguf_layer(
- tmp_m.tmp_name,
- self.model,
- self.formats[0],
- output_dir,
- self.layer_config,
- self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- model_type=model_type,
- )
- else:
- PACKING_LAYER_WITH_FORMAT[target_backend](
- tmp_m.tmp_name, self.model, self.formats[0], device=self.device
- )
+ self._immediate_pack(tmp_m.tmp_name)
+
+ if self.immediate_saving:
+ last_group = (i + nblocks) >= len(block_names)
+ immediate_saving(self, m, last_group=last_group)
if pbar is not None:
pbar.update(1)
- self.model = mv_module_from_gpu(self.model)
+ if not self.immediate_saving:
+ self.model = mv_module_from_gpu(self.model)
for n, m in self.model.named_modules():
if hasattr(m, "name"):
delattr(m, "name")
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index ba5db377c..c4cf83395 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -26,7 +26,7 @@ from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
-from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_model
+from auto_round.utils import check_to_quantized, copy_python_files_from_model_cache, is_fp8_linear, is_fp8_model
class BackendDataType(str, Enum):
@@ -1031,3 +1031,241 @@ def reset_params(inputs):
"""
if "use_cache" in inputs.keys(): # Not storing kv cache
inputs["use_cache"] = False
+
+
+def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last_group: bool = False):
+ """
+ Shard-saves the parameters of a model block (or group of blocks) immediately into disk,
+ accumulating tensors into size-limited shards, optionally finalizing all remaining
+ model weights when processing the last group.
+
+ Args:
+ rounder (object): The object of compressor.
+ m (torch.nn.Module): The current block (or composite module) whose parameters will be added to the shard set.
+ name (str): Override module name used as prefix for saved parameter keys. If None, falls back to m.tmp_name.
+ last_group (bool): If True, triggers final pass over the entire model to include unsaved weights,
+ writes shard index, renames shard files, copies source files, and releases temporary state.
+ """
+ import json
+ import os
+ from collections import OrderedDict
+
+ from auto_round.utils import clear_memory, get_module
+
+ # User configurable (can be preset on rounder)
+ max_shard_size = getattr(rounder, "max_shard_size", "5GB")
+ safe_serialization = getattr(rounder, "safe_serialization", True)
+ layer_names = rounder._get_quantized_layer_names_outside_blocks()
+ if len(layer_names) > 0 and name != layer_names[-1]:
+ last_group = False
+
+ def _parse_size(size_str: str) -> int:
+ s = size_str.strip().upper()
+ if s.endswith("GB"):
+ return int(s[:-2]) * (1024**3)
+ if s.endswith("MB"):
+ return int(s[:-2]) * (1024**2)
+ if s.endswith("KB"):
+ return int(s[:-2]) * 1024
+ return int(s)
+
+ # Init global accumulators (once)
+ if not hasattr(rounder, "_shard_init_done"):
+ rounder._shard_init_done = True
+ rounder._max_shard_bytes = _parse_size(str(max_shard_size))
+ rounder._use_safetensors = False
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file as _sf # noqa
+
+ rounder._use_safetensors = True
+ except ImportError:
+ logger.warning("safe_serialization=True but safetensors not installed; fallback to torch.save.")
+ rounder._current_shard_tensors = OrderedDict()
+ rounder._current_shard_size = 0
+ rounder._shard_meta = [] # list of dicts: {file, params}
+ rounder._global_weight_map = {} # param_name -> final shard file (filled after finalize)
+ rounder._shard_counter = 0
+ rounder._shard_suffix = "safetensors" if rounder._use_safetensors else "bin"
+ # new global counters
+ rounder._total_param_elems = 0
+ rounder._total_param_size_bytes = 0
+ # Directory
+ rounder._packed_blocks_root = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")
+ os.makedirs(rounder._packed_blocks_root, exist_ok=True)
+
+ # Collect tensors directly from current (multi)block `m`
+ flat_tensors = OrderedDict()
+ for k, v in m.state_dict().items():
+ tmp_name = name if name is not None else m.tmp_name
+ if isinstance(v, torch.Tensor):
+ flat_tensors[f"{tmp_name}.{k}"] = v
+
+ # Append tensors into the running shard(s)
+ def _flush_current_shard():
+ if len(rounder._current_shard_tensors) == 0:
+ return
+ rounder._shard_counter += 1
+ tmp_name = f"model-shard-{rounder._shard_counter:05d}.{rounder._shard_suffix}" # temporary name
+ tmp_path = os.path.join(rounder._packed_blocks_root, tmp_name)
+ if rounder._use_safetensors:
+ from safetensors.torch import save_file
+
+ save_file(rounder._current_shard_tensors, tmp_path)
+ else:
+ torch.save(rounder._current_shard_tensors, tmp_path)
+ params = list(rounder._current_shard_tensors.keys())
+ rounder._shard_meta.append({"tmp_file": tmp_name, "params": params})
+ for param in params:
+ free_module_name = param.rsplit(".", 1)[0]
+ free_module = get_module(rounder.model, free_module_name)
+
+ # free module only when all its parameters have been saved
+ free_flag = True
+ free_module_state_dict = free_module.state_dict()
+ already_saved_name = []
+ for _meta in rounder._shard_meta:
+ already_saved_name += _meta.get("params", [])
+ for free_module_key in free_module_state_dict:
+ free_module_key_full_name = f"{free_module_name}.{free_module_key}"
+ if free_module_key_full_name not in already_saved_name:
+ free_flag = False
+ if free_flag:
+ free_module.to("meta")
+ del rounder._current_shard_tensors[param]
+ rounder._current_shard_tensors = OrderedDict()
+ rounder._current_shard_size = 0
+ clear_memory()
+
+ for pname, tensor in flat_tensors.items():
+ t_elems = tensor.numel()
+ t_size = t_elems * tensor.element_size()
+ # accumulate global stats
+ rounder._total_param_elems += t_elems
+ rounder._total_param_size_bytes += t_size
+ if t_size > rounder._max_shard_bytes:
+ _flush_current_shard()
+ rounder._current_shard_tensors[pname] = tensor
+ rounder._current_shard_size = t_size
+ _flush_current_shard()
+ continue
+ if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:
+ _flush_current_shard()
+ rounder._current_shard_tensors[pname] = tensor
+ rounder._current_shard_size += t_size
+
+ if last_group:
+
+ # 1) Add the remaining (unsaved) model weights into new shard(s),
+ # do not overwrite the already saved weights.
+ try:
+ full_sd = rounder.model.state_dict()
+ except Exception as e:
+ logger.warning(f"failed to obtain full state_dict for remaining weights: {e}")
+ full_sd = {}
+ tie_word_embeddings: bool = getattr(getattr(rounder.model, "config", None), "tie_word_embeddings", True)
+ for pname, tensor in full_sd.items():
+ if "lm_head" in pname and tie_word_embeddings:
+ continue
+ if not isinstance(tensor, torch.Tensor):
+ continue
+ # Check whether pname already stored in previous shards via _shard_meta
+ already_saved = False
+ for _meta in rounder._shard_meta:
+ if pname in _meta.get("params", []):
+ already_saved = True
+ break
+ if already_saved:
+ continue # skip weights already saved
+ # Size accounting
+ t_elems = tensor.numel()
+ t_size = t_elems * tensor.element_size()
+
+ # Update global stats (these counters may already include earlier packed weights)
+ rounder._total_param_elems += t_elems
+ rounder._total_param_size_bytes += t_size
+
+ # If this tensor alone exceeds shard size -> dedicated shard
+ if t_size > rounder._max_shard_bytes:
+ _flush_current_shard()
+ rounder._current_shard_tensors[pname] = tensor.detach().cpu()
+ rounder._current_shard_size = t_size
+ _flush_current_shard()
+ continue
+
+ # If adding this tensor would overflow current shard -> flush current first
+ if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:
+ _flush_current_shard()
+
+ # Add to current shard
+ rounder._current_shard_tensors[pname] = tensor.detach().cpu()
+ rounder._current_shard_size += t_size
+
+ # 2) Flush any remaining unsaved leftover tensors
+ _flush_current_shard()
+
+ # 3) Finalize: rename temp shard files to HF-style names and build index
+ total_shards = rounder._shard_counter
+ if total_shards == 0:
+ logger.warning("no tensors saved across all blocks")
+ else:
+ final_names = []
+ for idx, meta in enumerate(rounder._shard_meta, start=1):
+ old_tmp = meta["tmp_file"]
+ old_path = os.path.join(rounder._packed_blocks_root, old_tmp)
+ if total_shards == 1:
+ new_name = f"model.{rounder._shard_suffix}"
+ else:
+ new_name = f"model-{idx:05d}-of-{total_shards:05d}.{rounder._shard_suffix}"
+ new_path = os.path.join(rounder._packed_blocks_root, new_name)
+ os.rename(old_path, new_path)
+ final_names.append(new_name)
+ for p in meta["params"]:
+ rounder._global_weight_map[p] = new_name
+
+ index_fname = "model.safetensors.index.json" if rounder._use_safetensors else "model.bin.index.json"
+ index_path = os.path.join(rounder._packed_blocks_root, index_fname)
+ index_data = {
+ "metadata": {
+ "format": "safetensors" if rounder._use_safetensors else "pytorch",
+ "total_shards": total_shards,
+ "total_parameters": int(getattr(rounder, "_total_param_elems", 0)),
+ "total_size": int(getattr(rounder, "_total_param_size_bytes", 0)),
+ },
+ "weight_map": rounder._global_weight_map,
+ }
+ if total_shards > 1:
+ with open(index_path, "w", encoding="utf-8") as f:
+ json.dump(index_data, f, indent=2)
+ logger.info(
+ f"saved {total_shards} shard(s) (HF-style, including remaining unsaved weights) to "
+ f"{rounder._packed_blocks_root} (index: {index_fname})"
+ )
+
+ try:
+ copy_python_files_from_model_cache(rounder.model, rounder._get_save_folder_name(rounder.formats[0]))
+ except Exception as e:
+ logger.warning("Skipping source model Python file copy due to error: %s", e)
+
+ # 4) Cleanup attributes to release memory after final shard is written
+ try:
+ attrs_to_cleanup = [
+ "_current_shard_tensors",
+ "_current_shard_size",
+ "_shard_counter",
+ "_max_shard_bytes",
+ "_use_safetensors",
+ "_shard_suffix",
+ "_packed_blocks_root",
+ "_total_param_elems",
+ "_total_param_size_bytes",
+ "_shard_init_done",
+ "_shard_meta",
+ "_global_weight_map",
+ ]
+ for _attr in attrs_to_cleanup:
+ if hasattr(rounder, _attr):
+ delattr(rounder, _attr)
+ clear_memory()
+ except Exception as _cleanup_err:
+ logger.warning(f"shard cleanup warning: {_cleanup_err}")
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index c5cf926ae..99c86fd67 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -70,6 +70,7 @@ from auto_round.utils import (
matches_any_regex,
set_module,
to_standard_regex,
+ unsupported_meta_device,
)
BLOCK_PATTERNS = [ ## copy from transformers optimum
@@ -283,18 +284,18 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ if not unsupported_meta_device(model):
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, backend, device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
if output_dir is None:
return model
quantization_config["lm_head"] = lm_head_quantized
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index d587efa2d..a67dc7c21 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -41,6 +41,7 @@ from auto_round.utils import (
get_module,
set_module,
to_standard_regex,
+ unsupported_meta_device,
)
@@ -350,17 +351,18 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ if not unsupported_meta_device(model):
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, backend, device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
filter_quantization_config(quantization_config)
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 2785a0e55..f821c2d56 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -41,6 +41,7 @@ from auto_round.utils import (
get_block_names,
get_module,
set_module,
+ unsupported_meta_device,
)
@@ -110,10 +111,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if image_processor is not None and output_dir is not None:
image_processor.save_pretrained(output_dir)
- if inplace:
- compressed_model = model.to("cpu")
+ if not unsupported_meta_device(model):
+ if inplace:
+ compressed_model = model.to("cpu")
+ else:
+ compressed_model = copy.deepcopy(model.to("cpu"))
else:
- compressed_model = copy.deepcopy(model.to("cpu"))
+ compressed_model = model
names = list(layer_config.keys())
@@ -121,17 +125,18 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, compressed_model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ if not unsupported_meta_device(model):
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, compressed_model, backend, device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
if output_dir is None:
return model
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 31766af21..af9702484 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -38,6 +38,7 @@ from auto_round.utils import (
get_module,
set_amax_for_all_moe_layers,
set_module,
+ unsupported_meta_device,
)
from auto_round.wrapper import WrapperWALayer
@@ -188,17 +189,18 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
max_workers = 1
if not torch.cuda.is_available() or not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ if not unsupported_meta_device(model):
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, backend, device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 1b0b48b35..2b8243845 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -36,6 +36,7 @@ from auto_round.utils import (
get_packing_device,
logger,
set_module,
+ unsupported_meta_device,
)
@@ -147,17 +148,18 @@ def save_quantized_as_static_fp(output_dir: str, inplace: bool = True, **kwargs)
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, kwargs.get("data_type", "fp8"), device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ if not unsupported_meta_device(model):
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, kwargs.get("data_type", "fp8"), device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
# Get llm-compressor format config
check_compressed_tensors_supported()
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 78e2f43e6..519077529 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -16,7 +16,7 @@ import os
import torch.nn as nn
-from auto_round.utils import copy_python_files_from_model_cache, logger
+from auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_device
def save_model(
@@ -47,12 +47,19 @@ def save_model(
Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
"""
os.makedirs(save_dir, exist_ok=True)
- try:
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
- except ValueError as e:
- if hasattr(model, "generation_config"):
- setattr(model.generation_config, "do_sample", True)
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ if unsupported_meta_device(model):
+ if hasattr(model, "config") and model.config is not None:
+ model.config.save_pretrained(save_dir)
+
+ if hasattr(model, "generation_config") and model.generation_config is not None:
+ model.generation_config.save_pretrained(save_dir)
+ else:
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except ValueError as e:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 7dfd4b479..924e68922 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -808,6 +808,26 @@ class TestAutoRound(unittest.TestCase):
ar = AutoRound(model_name, iters=1, dataset=data, seqlen=8)
ar.quantize()
+ def test_low_cpu_mem_usage(self):
+ bits, group_size = 4, 32
+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ quantized_model_path = self.save_folder
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ iters=2,
+ seqlen=10,
+ dataset=self.llm_dataloader,
+ low_cpu_mem_usage=True,
+ device_map="cpu",
+ )
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
def test_create_adam(self):
model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-0.6B"
from auto_round import AutoRound
diff --git a/test/test_cuda/test_main_func.py b/test/test_cuda/test_main_func.py
index 681387285..523fe6784 100644
--- a/test/test_cuda/test_main_func.py
+++ b/test/test_cuda/test_main_func.py
@@ -199,6 +199,25 @@ class TestMainFunc(unittest.TestCase):
ar = AutoRound(model_name, iters=1, dataset=data, seqlen=8, quant_lm_head=True)
ar.quantize()
+ def test_low_cpu_mem_usage(self):
+ bits, group_size = 4, 32
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ quantized_model_path = "./saved"
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ iters=2,
+ seqlen=10,
+ dataset=self.llm_dataloader,
+ low_cpu_mem_usage=True,
+ )
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()