PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support for immediate saving to reduce ram usage (#965)
Tang Kaihuidaeb3bb78814
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex d52914447..f2d2b1346 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -85,6 +85,7 @@ class AutoRound:enable_adam: bool = False,# for MLLM and Diffusionextra_config: ExtraConfig = None,+ low_cpu_mem_usage: bool = False,**kwargs,) -> BaseCompressor:"""Initialize AutoRound with quantization and tuning configuration.@@ -105,6 +106,7 @@ class AutoRound:lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.iters (int, optional): Optimization iterations. Defaults to 200.seqlen (int, optional): Calibration sequence length. Defaults to 2048.nsamples (int, optional): Number of calibration samples. Defaults to 128.@@ -186,6 +188,7 @@ class AutoRound:device_map=device_map,enable_torch_compile=enable_torch_compile,seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,**kwargs,)return ardiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 0a682079a..8763e528b 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -39,6 +39,7 @@ from auto_round.compressors.utils import (collect_best_params,get_shared_keys,gguf_args_check,+ immediate_saving,infer_bits_by_data_type,init_cache,is_mx_fp,@@ -148,6 +149,7 @@ class BaseCompressor(object):enable_alg_ext: bool = False,disable_opt_rtn: bool = False,seed: int = 42,+ low_cpu_mem_usage: bool = False,**kwargs,):"""Initialize AutoRound with quantization and tuning configuration.@@ -168,6 +170,7 @@ class BaseCompressor(object):lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.iters (int, optional): Optimization iterations. Defaults to 200.seqlen (int, optional): Calibration sequence length. Defaults to 2048.nsamples (int, optional): Number of calibration samples. Defaults to 128.@@ -244,6 +247,7 @@ class BaseCompressor(object):self.supported_types = SUPPORTED_LAYER_TYPESself.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESself.scale_dtype = convert_dtype_str2torch(scale_dtype)+ self.low_cpu_mem_usage = low_cpu_mem_usageif kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")@@ -336,7 +340,10 @@ class BaseCompressor(object):self.lr_scheduler = lr_schedulerself.optimizer = self._get_optimizer(None)self.disable_opt_rtn = disable_opt_rtn- self.is_packing_immediate = False # whether to pack the layer immediately after tuning++ # Whether to pack the layer immediately after tuning+ self.immediate_packing = False+ self.immediate_saving = False# KV cache, this one does not affect tuning but will collect some infos during tuningself.static_kv_dtype = static_kv_dtype@@ -1205,7 +1212,7 @@ class BaseCompressor(object):`rtn_*` version if supported, then wraps and unwraps the module to applyquantization. If GPU memory is insufficient, it falls back to CPU.- If packing is enabled (`is_packing_immediate`), the function will also export+ If packing is enabled (`immediate_packing`), the function will also exportthe quantized layer to the appropriate backend format.Args:@@ -1222,7 +1229,7 @@ class BaseCompressor(object):# Step 1: Try quantization on GPU first, fall back to CPU if OOM# if only export gguf, using gguf-packing instead of rtn- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:+ if self.immediate_packing and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:m.scale = Nonem.zp = Noneelse:@@ -1259,34 +1266,45 @@ class BaseCompressor(object):raise# Step 2: Optional immediate packing/export- if self.is_packing_immediate:- from auto_round.export import PACKING_LAYER_WITH_FORMAT-- if check_to_quantized(m):- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in fmt for fmt in self.formats)-- if has_gguf:- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- output_dir = self._get_save_folder_name(self.formats[0])- model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT- pack_gguf_layer(- name,- self.model,- self.formats[0],- output_dir,- self.layer_config,- self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type,- )- else:- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)+ if self.immediate_packing:+ self._immediate_pack(name)else:set_module(self.model, name, m)+ if self.immediate_saving:+ all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]+ last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])+ m = get_module(self.model, name)+ immediate_saving(self, m, name, last_module)++ def _immediate_pack(self, name: str):+ m = get_module(self.model, name)+ if not check_to_quantized(m):+ return+ from auto_round.export import PACKING_LAYER_WITH_FORMAT++ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in fmt for fmt in self.formats)++ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer++ output_dir = self._get_save_folder_name(self.formats[0])+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT+ pack_gguf_layer(+ name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type,+ )+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)+@torch.inference_mode()def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.@@ -1301,7 +1319,6 @@ class BaseCompressor(object):self.model.to(self.amp_dtype)all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]-if is_nv_fp(self.data_type):from auto_round.data_type.nvfp import calculate_gparamfrom auto_round.data_type.utils import update_fused_layer_global_scales@@ -1477,8 +1494,8 @@ class BaseCompressor(object):if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:self._quantize_layer_via_rtn(m.tmp_name)all_to_quantized_module_names.remove(m.tmp_name)-- mv_module_from_gpu(block)+ if not self.immediate_saving:+ mv_module_from_gpu(block)pbar.update(1)pbar.close()@@ -1556,7 +1573,12 @@ class BaseCompressor(object):)and self.inplace):- self.is_packing_immediate = True+ self.immediate_packing = True+ if "gguf" not in formats[0] and self.low_cpu_mem_usage:+ self.immediate_saving = True+ if self.immediate_saving and "int" not in self.data_type:+ logger.warning("immediate_saving is only supported for int quantization, set to False")+ self.immediate_saving = Falseif self.iters == 0:return self._quantize_rtn()@@ -1628,15 +1650,14 @@ class BaseCompressor(object):device=self.device,pbar=pbar,)- if self.is_packing_immediate and len(self.formats) != 1:+ if self.immediate_packing and len(self.formats) != 1:raise ValueError(- f"Expected exactly one packing format when 'is_packing_immediate' is True, "+ f"Expected exactly one packing format when 'immediate_packing' is True, "f"but got {len(self.formats)} formats.")pbar.set_description("Quantizing done")pbar.close()-- self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately+ self._quantize_layers(layer_names, all_inputs)if is_fp8_model(self.model):for n, m in self.model.named_modules():@@ -1714,7 +1735,7 @@ class BaseCompressor(object):has_gguf = Falseif hasattr(self, "formats"):has_gguf = any("gguf" in format_ for format_ in self.formats)- if has_gguf and self.is_packing_immediate:+ if has_gguf and self.immediate_packing:enable_quanted_input = Falseif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:@@ -1727,8 +1748,8 @@ class BaseCompressor(object):accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed-- self.model = mv_module_from_gpu(self.model)+ if not self.immediate_saving:+ self.model = mv_module_from_gpu(self.model)clear_memory()quant_layer = self._quantize_layerfor layer_name in layer_names:@@ -1737,6 +1758,12 @@ class BaseCompressor(object):q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else Noneq_layer_input = to_device(q_layer_input, self.cache_device)quant_layer(layer_name, layer_input, q_layer_input, device=self.device)+ if self.immediate_packing:+ self._immediate_pack(layer_name)++ if self.immediate_saving:+ m = get_module(self.model, layer_name)+ immediate_saving(self, m, name=layer_name, last_group=True)del layer_inputclear_memory(q_layer_input)@@ -1937,7 +1964,6 @@ class BaseCompressor(object):layer_names = []if layer_names is None:layer_names = []-if self.low_gpu_mem_usage or (len(block_names) == 1and len(layer_names) == 0@@ -2732,6 +2758,7 @@ class BaseCompressor(object):input_ids = to_device(input_ids, self.cache_device)input_others = to_device(input_others, self.cache_device)# As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+tmp_dtype = self.amp_dtype if self.amp else torch.float32input_ids = to_dtype(input_ids, tmp_dtype)@@ -2808,38 +2835,20 @@ class BaseCompressor(object):)if hasattr(model, "config"):del m.config- if self.is_packing_immediate:- from auto_round.export import PACKING_LAYER_WITH_FORMAT-+ if self.immediate_packing:for _, tmp_m in m.named_modules():if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):continue- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in format_ for format_ in self.formats)- if has_gguf:- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- output_dir = self._get_save_folder_name(self.formats[0])- model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT- pack_gguf_layer(- tmp_m.tmp_name,- self.model,- self.formats[0],- output_dir,- self.layer_config,- self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type,- )- else:- PACKING_LAYER_WITH_FORMAT[target_backend](- tmp_m.tmp_name, self.model, self.formats[0], device=self.device- )+ self._immediate_pack(tmp_m.tmp_name)++ if self.immediate_saving:+ last_group = (i + nblocks) >= len(block_names)+ immediate_saving(self, m, last_group=last_group)if pbar is not None:pbar.update(1)- self.model = mv_module_from_gpu(self.model)+ if not self.immediate_saving:+ self.model = mv_module_from_gpu(self.model)for n, m in self.model.named_modules():if hasattr(m, "name"):delattr(m, "name")diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex ba5db377c..c4cf83395 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -26,7 +26,7 @@ from torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme-from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_model+from auto_round.utils import check_to_quantized, copy_python_files_from_model_cache, is_fp8_linear, is_fp8_modelclass BackendDataType(str, Enum):@@ -1031,3 +1031,241 @@ def reset_params(inputs):"""if "use_cache" in inputs.keys(): # Not storing kv cacheinputs["use_cache"] = False+++def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last_group: bool = False):+ """+ Shard-saves the parameters of a model block (or group of blocks) immediately into disk,+ accumulating tensors into size-limited shards, optionally finalizing all remaining+ model weights when processing the last group.++ Args:+ rounder (object): The object of compressor.+ m (torch.nn.Module): The current block (or composite module) whose parameters will be added to the shard set.+ name (str): Override module name used as prefix for saved parameter keys. If None, falls back to m.tmp_name.+ last_group (bool): If True, triggers final pass over the entire model to include unsaved weights,+ writes shard index, renames shard files, copies source files, and releases temporary state.+ """+ import json+ import os+ from collections import OrderedDict++ from auto_round.utils import clear_memory, get_module++ # User configurable (can be preset on rounder)+ max_shard_size = getattr(rounder, "max_shard_size", "5GB")+ safe_serialization = getattr(rounder, "safe_serialization", True)+ layer_names = rounder._get_quantized_layer_names_outside_blocks()+ if len(layer_names) > 0 and name != layer_names[-1]:+ last_group = False++ def _parse_size(size_str: str) -> int:+ s = size_str.strip().upper()+ if s.endswith("GB"):+ return int(s[:-2]) * (1024**3)+ if s.endswith("MB"):+ return int(s[:-2]) * (1024**2)+ if s.endswith("KB"):+ return int(s[:-2]) * 1024+ return int(s)++ # Init global accumulators (once)+ if not hasattr(rounder, "_shard_init_done"):+ rounder._shard_init_done = True+ rounder._max_shard_bytes = _parse_size(str(max_shard_size))+ rounder._use_safetensors = False+ if safe_serialization:+ try:+ from safetensors.torch import save_file as _sf # noqa++ rounder._use_safetensors = True+ except ImportError:+ logger.warning("safe_serialization=True but safetensors not installed; fallback to torch.save.")+ rounder._current_shard_tensors = OrderedDict()+ rounder._current_shard_size = 0+ rounder._shard_meta = [] # list of dicts: {file, params}+ rounder._global_weight_map = {} # param_name -> final shard file (filled after finalize)+ rounder._shard_counter = 0+ rounder._shard_suffix = "safetensors" if rounder._use_safetensors else "bin"+ # new global counters+ rounder._total_param_elems = 0+ rounder._total_param_size_bytes = 0+ # Directory+ rounder._packed_blocks_root = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")+ os.makedirs(rounder._packed_blocks_root, exist_ok=True)++ # Collect tensors directly from current (multi)block `m`+ flat_tensors = OrderedDict()+ for k, v in m.state_dict().items():+ tmp_name = name if name is not None else m.tmp_name+ if isinstance(v, torch.Tensor):+ flat_tensors[f"{tmp_name}.{k}"] = v++ # Append tensors into the running shard(s)+ def _flush_current_shard():+ if len(rounder._current_shard_tensors) == 0:+ return+ rounder._shard_counter += 1+ tmp_name = f"model-shard-{rounder._shard_counter:05d}.{rounder._shard_suffix}" # temporary name+ tmp_path = os.path.join(rounder._packed_blocks_root, tmp_name)+ if rounder._use_safetensors:+ from safetensors.torch import save_file++ save_file(rounder._current_shard_tensors, tmp_path)+ else:+ torch.save(rounder._current_shard_tensors, tmp_path)+ params = list(rounder._current_shard_tensors.keys())+ rounder._shard_meta.append({"tmp_file": tmp_name, "params": params})+ for param in params:+ free_module_name = param.rsplit(".", 1)[0]+ free_module = get_module(rounder.model, free_module_name)++ # free module only when all its parameters have been saved+ free_flag = True+ free_module_state_dict = free_module.state_dict()+ already_saved_name = []+ for _meta in rounder._shard_meta:+ already_saved_name += _meta.get("params", [])+ for free_module_key in free_module_state_dict:+ free_module_key_full_name = f"{free_module_name}.{free_module_key}"+ if free_module_key_full_name not in already_saved_name:+ free_flag = False+ if free_flag:+ free_module.to("meta")+ del rounder._current_shard_tensors[param]+ rounder._current_shard_tensors = OrderedDict()+ rounder._current_shard_size = 0+ clear_memory()++ for pname, tensor in flat_tensors.items():+ t_elems = tensor.numel()+ t_size = t_elems * tensor.element_size()+ # accumulate global stats+ rounder._total_param_elems += t_elems+ rounder._total_param_size_bytes += t_size+ if t_size > rounder._max_shard_bytes:+ _flush_current_shard()+ rounder._current_shard_tensors[pname] = tensor+ rounder._current_shard_size = t_size+ _flush_current_shard()+ continue+ if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:+ _flush_current_shard()+ rounder._current_shard_tensors[pname] = tensor+ rounder._current_shard_size += t_size++ if last_group:++ # 1) Add the remaining (unsaved) model weights into new shard(s),+ # do not overwrite the already saved weights.+ try:+ full_sd = rounder.model.state_dict()+ except Exception as e:+ logger.warning(f"failed to obtain full state_dict for remaining weights: {e}")+ full_sd = {}+ tie_word_embeddings: bool = getattr(getattr(rounder.model, "config", None), "tie_word_embeddings", True)+ for pname, tensor in full_sd.items():+ if "lm_head" in pname and tie_word_embeddings:+ continue+ if not isinstance(tensor, torch.Tensor):+ continue+ # Check whether pname already stored in previous shards via _shard_meta+ already_saved = False+ for _meta in rounder._shard_meta:+ if pname in _meta.get("params", []):+ already_saved = True+ break+ if already_saved:+ continue # skip weights already saved+ # Size accounting+ t_elems = tensor.numel()+ t_size = t_elems * tensor.element_size()++ # Update global stats (these counters may already include earlier packed weights)+ rounder._total_param_elems += t_elems+ rounder._total_param_size_bytes += t_size++ # If this tensor alone exceeds shard size -> dedicated shard+ if t_size > rounder._max_shard_bytes:+ _flush_current_shard()+ rounder._current_shard_tensors[pname] = tensor.detach().cpu()+ rounder._current_shard_size = t_size+ _flush_current_shard()+ continue++ # If adding this tensor would overflow current shard -> flush current first+ if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:+ _flush_current_shard()++ # Add to current shard+ rounder._current_shard_tensors[pname] = tensor.detach().cpu()+ rounder._current_shard_size += t_size++ # 2) Flush any remaining unsaved leftover tensors+ _flush_current_shard()++ # 3) Finalize: rename temp shard files to HF-style names and build index+ total_shards = rounder._shard_counter+ if total_shards == 0:+ logger.warning("no tensors saved across all blocks")+ else:+ final_names = []+ for idx, meta in enumerate(rounder._shard_meta, start=1):+ old_tmp = meta["tmp_file"]+ old_path = os.path.join(rounder._packed_blocks_root, old_tmp)+ if total_shards == 1:+ new_name = f"model.{rounder._shard_suffix}"+ else:+ new_name = f"model-{idx:05d}-of-{total_shards:05d}.{rounder._shard_suffix}"+ new_path = os.path.join(rounder._packed_blocks_root, new_name)+ os.rename(old_path, new_path)+ final_names.append(new_name)+ for p in meta["params"]:+ rounder._global_weight_map[p] = new_name++ index_fname = "model.safetensors.index.json" if rounder._use_safetensors else "model.bin.index.json"+ index_path = os.path.join(rounder._packed_blocks_root, index_fname)+ index_data = {+ "metadata": {+ "format": "safetensors" if rounder._use_safetensors else "pytorch",+ "total_shards": total_shards,+ "total_parameters": int(getattr(rounder, "_total_param_elems", 0)),+ "total_size": int(getattr(rounder, "_total_param_size_bytes", 0)),+ },+ "weight_map": rounder._global_weight_map,+ }+ if total_shards > 1:+ with open(index_path, "w", encoding="utf-8") as f:+ json.dump(index_data, f, indent=2)+ logger.info(+ f"saved {total_shards} shard(s) (HF-style, including remaining unsaved weights) to "+ f"{rounder._packed_blocks_root} (index: {index_fname})"+ )++ try:+ copy_python_files_from_model_cache(rounder.model, rounder._get_save_folder_name(rounder.formats[0]))+ except Exception as e:+ logger.warning("Skipping source model Python file copy due to error: %s", e)++ # 4) Cleanup attributes to release memory after final shard is written+ try:+ attrs_to_cleanup = [+ "_current_shard_tensors",+ "_current_shard_size",+ "_shard_counter",+ "_max_shard_bytes",+ "_use_safetensors",+ "_shard_suffix",+ "_packed_blocks_root",+ "_total_param_elems",+ "_total_param_size_bytes",+ "_shard_init_done",+ "_shard_meta",+ "_global_weight_map",+ ]+ for _attr in attrs_to_cleanup:+ if hasattr(rounder, _attr):+ delattr(rounder, _attr)+ clear_memory()+ except Exception as _cleanup_err:+ logger.warning(f"shard cleanup warning: {_cleanup_err}")diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex c5cf926ae..99c86fd67 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -70,6 +70,7 @@ from auto_round.utils import (matches_any_regex,set_module,to_standard_regex,+ unsupported_meta_device,)BLOCK_PATTERNS = [ ## copy from transformers optimum@@ -283,18 +284,18 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllmax_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally-- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ if not unsupported_meta_device(model):+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, backend, device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ passif output_dir is None:return modelquantization_config["lm_head"] = lm_head_quantizeddiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex d587efa2d..a67dc7c21 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -41,6 +41,7 @@ from auto_round.utils import (get_module,set_module,to_standard_regex,+ unsupported_meta_device,)@@ -350,17 +351,18 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exmax_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ if not unsupported_meta_device(model):+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, backend, device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ passfilter_quantization_config(quantization_config)if hasattr(model, "config"):model.config.quantization_config = quantization_configdiff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 2785a0e55..f821c2d56 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -41,6 +41,7 @@ from auto_round.utils import (get_block_names,get_module,set_module,+ unsupported_meta_device,)@@ -110,10 +111,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if image_processor is not None and output_dir is not None:image_processor.save_pretrained(output_dir)- if inplace:- compressed_model = model.to("cpu")+ if not unsupported_meta_device(model):+ if inplace:+ compressed_model = model.to("cpu")+ else:+ compressed_model = copy.deepcopy(model.to("cpu"))else:- compressed_model = copy.deepcopy(model.to("cpu"))+ compressed_model = modelnames = list(layer_config.keys())@@ -121,17 +125,18 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):max_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, compressed_model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ if not unsupported_meta_device(model):+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, compressed_model, backend, device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ passif output_dir is None:return modeldiff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 31766af21..af9702484 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -38,6 +38,7 @@ from auto_round.utils import (get_module,set_amax_for_all_moe_layers,set_module,+ unsupported_meta_device,)from auto_round.wrapper import WrapperWALayer@@ -188,17 +189,18 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):max_workers = 1if not torch.cuda.is_available() or not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ if not unsupported_meta_device(model):+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, backend, device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ passignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 1b0b48b35..2b8243845 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -36,6 +36,7 @@ from auto_round.utils import (get_packing_device,logger,set_module,+ unsupported_meta_device,)@@ -147,17 +148,18 @@ def save_quantized_as_static_fp(output_dir: str, inplace: bool = True, **kwargs)max_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, kwargs.get("data_type", "fp8"), device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ if not unsupported_meta_device(model):+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, kwargs.get("data_type", "fp8"), device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ pass# Get llm-compressor format configcheck_compressed_tensors_supported()diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 78e2f43e6..519077529 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -16,7 +16,7 @@ import osimport torch.nn as nn-from auto_round.utils import copy_python_files_from_model_cache, logger+from auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_devicedef save_model(@@ -47,12 +47,19 @@ def save_model(Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`)."""os.makedirs(save_dir, exist_ok=True)- try:- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)- except ValueError as e:- if hasattr(model, "generation_config"):- setattr(model.generation_config, "do_sample", True)- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ if unsupported_meta_device(model):+ if hasattr(model, "config") and model.config is not None:+ model.config.save_pretrained(save_dir)++ if hasattr(model, "generation_config") and model.generation_config is not None:+ model.generation_config.save_pretrained(save_dir)+ else:+ try:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ except ValueError as e:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 7dfd4b479..924e68922 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -808,6 +808,26 @@ class TestAutoRound(unittest.TestCase):ar = AutoRound(model_name, iters=1, dataset=data, seqlen=8)ar.quantize()+ def test_low_cpu_mem_usage(self):+ bits, group_size = 4, 32+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ quantized_model_path = self.save_folder+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ iters=2,+ seqlen=10,+ dataset=self.llm_dataloader,+ low_cpu_mem_usage=True,+ device_map="cpu",+ )+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ shutil.rmtree(quantized_model_path, ignore_errors=True)+def test_create_adam(self):model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-0.6B"from auto_round import AutoRounddiff --git a/test/test_cuda/test_main_func.py b/test/test_cuda/test_main_func.pyindex 681387285..523fe6784 100644--- a/test/test_cuda/test_main_func.py+++ b/test/test_cuda/test_main_func.py@@ -199,6 +199,25 @@ class TestMainFunc(unittest.TestCase):ar = AutoRound(model_name, iters=1, dataset=data, seqlen=8, quant_lm_head=True)ar.quantize()+ def test_low_cpu_mem_usage(self):+ bits, group_size = 4, 32+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ quantized_model_path = "./saved"+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ iters=2,+ seqlen=10,+ dataset=self.llm_dataloader,+ low_cpu_mem_usage=True,+ )+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()