Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix low_cpu and set it to True as default (#1288)

Wenhua Chengf654e52deac2
diff --git a/README.md b/README.md
index c8c0f3bfd..bd1eb06ff 100644
--- a/README.md
+++ b/README.md
@@ -212,7 +212,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")
##### Device/Speed Configuration
- **`enable_torch_compile` (bool)**: If no exception is raised, typically we recommend setting it to True for faster quantization with lower resource.
- **`low_gpu_mem_usage` (bool)**: Whether to offload intermediate features to CPU at the cost of ~20% more tuning time (default is `False`).
-- **`low_cpu_mem_usage` (bool)**: [Experimental Feature]Whether to enable saving immediately to reduce ram usage (default is `False`).
+- **`low_cpu_mem_usage` (bool)**: [Experimental Feature]Whether to enable saving immediately to reduce ram usage (default is `True`).
- **`device_map` (str|dict|int)**: The device to be used for tuning, e.g., `auto`, `cpu`, `cuda`, `0,1,2` (default is `0`). When using `auto`, it will try to use all available GPUs.
</details>
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index e3592e406..1540d254d 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -135,7 +135,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="Enable memory-efficient mode by offloading intermediate features to CPU. "
"Useful when working with large models that don't fit in GPU memory.",
)
- basic.add_argument("--low_cpu_mem_usage", action="store_true", help="Lower CPU memory mode. Defaults to False.")
+ basic.add_argument(
+ "--low_cpu_mem_usage", action="store_true", help="Deprecated, Lower CPU memory mode. Defaults to False."
+ )
+ basic.add_argument(
+ "--disable_low_cpu_mem_usage", action="store_true", help="disable lower CPU memory mode. Defaults to False."
+ )
basic.add_argument(
"--format",
"--formats",
@@ -259,8 +264,8 @@ class BasicArgumentParser(argparse.ArgumentParser):
action="store_true",
help="Enable PyTorch deterministic algorithms for reproducible results. ",
)
- group = tuning.add_mutually_exclusive_group()
- group.add_argument(
+ group_opt_rtn = tuning.add_mutually_exclusive_group()
+ group_opt_rtn.add_argument(
"--disable_opt_rtn",
action="store_const",
const=True,
@@ -269,7 +274,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="Disable optimization for RTN (Round-To-Nearest) mode when iters=0. "
"RTN is fast but less accurate; keeping optimization enabled is recommended.",
)
- group.add_argument(
+ group_opt_rtn.add_argument(
"--enable_opt_rtn",
action="store_const",
const=False,
@@ -513,6 +518,12 @@ def tune(args):
from auto_round.utils import detect_device, get_library_version, logger
+ if args.low_cpu_mem_usage:
+ logger.warning(
+ "`low_cpu_mem_usage` is deprecated and is now enabled by default. "
+ "To disable it, use `--disable_low_cpu_mem_usage`."
+ )
+
if args.format is None:
args.format = "auto_round"
@@ -639,6 +650,9 @@ def tune(args):
shared_layers=args.shared_layers,
ignore_scale_zp_bits=args.ignore_scale_zp_bits,
)
+ low_cpu_mem_usage = True
+ if args.disable_low_cpu_mem_usage:
+ low_cpu_mem_usage = False
autoround: BaseCompressor = AutoRound(
model=model_name,
@@ -651,7 +665,7 @@ def tune(args):
batch_size=args.batch_size,
gradient_accumulate_steps=args.gradient_accumulate_steps,
low_gpu_mem_usage=args.low_gpu_mem_usage,
- low_cpu_mem_usage=args.low_cpu_mem_usage,
+ low_cpu_mem_usage=low_cpu_mem_usage,
device_map=args.device_map,
enable_torch_compile=enable_torch_compile,
seed=args.seed,
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index b4c8f8f49..17082430e 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -298,7 +298,7 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterab
return []
# Retrieve all high-level block names (for example, transformer blocks)
for n, m in model.named_modules():
- m.tmp_name = n # attach global name
+ m.global_name = n # attach global name
block_names = get_block_names(model, quant_vision=True)
block_names = [item for sublist in block_names for item in sublist]
@@ -327,7 +327,7 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterab
block_layer_names = []
for name in block_layer_local_names:
module = get_module(block_module, name)
- block_layer_names.append(module.tmp_name)
+ block_layer_names.append(module.global_name)
for group in fuzzy_match_groups:
matched_layers = set()
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index dd88bad84..33ca1ccd7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -86,7 +86,7 @@ class AutoRound:
extra_config: ExtraConfig = None,
enable_alg_ext: bool = None,
disable_opt_rtn: bool | None = None,
- low_cpu_mem_usage: bool = False,
+ low_cpu_mem_usage: bool = True,
**kwargs,
) -> BaseCompressor:
"""Initialize AutoRound with quantization and tuning configuration.
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index f6714db16..5776a1a1d 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -45,8 +45,8 @@ def register_dataset(name):
names = name
else:
names = [name]
- for tmp_name in names:
- CALIB_DATASETS[tmp_name] = dataset
+ for global_name in names:
+ CALIB_DATASETS[global_name] = dataset
return dataset
return register
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 4bba244ed..c30502f9e 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -14,7 +14,6 @@
import copy
import os
-import re
import sys
import time
import traceback
@@ -34,6 +33,7 @@ from transformers import set_seed
from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+from auto_round.compressors.shard_writer import shard_writer
from auto_round.compressors.utils import (
IndexSampler,
block_forward,
@@ -41,12 +41,10 @@ from auto_round.compressors.utils import (
check_skippable_keywords,
collect_best_params,
get_shared_keys,
- immediate_saving,
infer_bits_by_data_type,
init_cache,
is_mx_fp,
is_nv_fp,
- is_standard_fp,
is_static_wfp8afp8,
is_wfp8afp8,
reset_params,
@@ -78,13 +76,13 @@ from auto_round.utils import (
compile_func,
convert_dtype_str2torch,
convert_fp8_layer_to_linear,
- convert_fp8_model_to_16b_model,
- copy_python_files_from_model_cache,
+ convert_fp8_module_to_16b,
detect_device,
find_matching_blocks,
flatten_list,
get_block_names,
get_layer_names_in_block,
+ get_lm_head_name,
get_module,
htcore,
is_auto_device_mapping,
@@ -191,7 +189,7 @@ class BaseCompressor(object):
enable_alg_ext: bool = False,
disable_opt_rtn: bool | None = None,
seed: int = 42,
- low_cpu_mem_usage: bool = False,
+ low_cpu_mem_usage: bool = True,
**kwargs,
):
"""Initialize AutoRound with quantization and tuning configuration.
@@ -232,7 +230,7 @@ class BaseCompressor(object):
- enable_alg_ext, quant_lm_head, lr, lr_scheduler, not_use_best_mse, dynamic_max_gap,
super_group_size, super_bits, scale_dtype ("fp16" etc.),
nblocks, to_quant_block_names,
- enable_norm_bias_tuning, enable_quanted_input,
+ enable_norm_bias_tuning, enable_quanted_input, enable_opt_rtn,
disable_deterministic_algorithms, mllm, static_kv_dtype,enable_deterministic_algorithms,momentum
Raises:
ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
@@ -308,6 +306,7 @@ class BaseCompressor(object):
self.momentum = kwargs.pop("momentum", 0.0)
static_kv_dtype = kwargs.pop("static_kv_dtype", None)
static_attention_dtype = kwargs.pop("static_attention_dtype", None)
+ enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)
device = kwargs.pop("device", None)
if envs.AR_USE_MODELSCOPE:
platform = "model_scope"
@@ -389,7 +388,12 @@ class BaseCompressor(object):
# Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.
# To avoid None issue, we keep a copy though it's a little ugly
+ if enable_opt_rtn and disable_opt_rtn:
+ raise ValueError("`enable_opt_rtn` and `disable_opt_rtn` are mutually exclusive; " "only one can be set.")
+ if enable_opt_rtn:
+ disable_opt_rtn = False
self.orig_disable_opt_rtn = disable_opt_rtn
+
if self.iters != 0 and self.orig_disable_opt_rtn is not None:
logger.warning("`disable_opt_rtn` only works when `iters` is set to 0, ignore it now.")
disable_opt_rtn = True
@@ -420,8 +424,8 @@ class BaseCompressor(object):
self.disable_opt_rtn = disable_opt_rtn
# Whether to pack the layer immediately after tuning
- self.immediate_packing = False
- self.immediate_saving = False
+ self.is_immediate_packing = False
+ self.is_immediate_saving = False
# KV cache, this one does not affect tuning but will collect some infos during tuning
self.static_kv_dtype = static_kv_dtype
@@ -1111,8 +1115,8 @@ class BaseCompressor(object):
m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)
set_module(self.model, name, m)
tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device
- # Step 1: Try quantization on GPU first, fall back to CPU if OOM
- if self.immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:
+ # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic
+ if self.is_immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:
m = m.to(tuning_device)
m.scale = None
m.zp = None
@@ -1123,8 +1127,8 @@ class BaseCompressor(object):
not disable_opt_rtn
and self.orig_disable_opt_rtn is None
and self.is_moe_model
- and "expert" in m.tmp_name
- and "shared_expert" not in m.tmp_name
+ and "expert" in m.global_name
+ and "shared_expert" not in m.global_name
and self.super_bits is None # GGUF still uses the optimized RTN for MoE layers
):
disable_opt_rtn = True
@@ -1162,7 +1166,7 @@ class BaseCompressor(object):
raise
# Step 2: Optional immediate packing/export
- if self.immediate_packing: # For gguf, packing conducts on block level
+ if self.is_immediate_packing: # For gguf, packing conducts on block level
self._immediate_pack(name)
if to_cpu:
m = m.to("cpu")
@@ -1172,17 +1176,13 @@ class BaseCompressor(object):
if to_cpu:
m = m.to("cpu")
set_module(self.model, name, m)
- if self.immediate_saving:
- if hasattr(self, "all_to_quantized_module_names"):
- all_to_quantized_module_names = self.all_to_quantized_module_names
- else:
- all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
- last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])
+ if self.is_immediate_saving:
m = get_module(self.model, name)
- immediate_saving(self, m, name, last_module)
+ m.to("cpu")
+ shard_writer(self, m, name, False)
def _immediate_pack(self, name: str):
- if not self.immediate_packing:
+ if not self.is_immediate_packing:
return
self.formats[0].immediate_pack(
name=name,
@@ -1276,21 +1276,38 @@ class BaseCompressor(object):
else:
block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
- if clear_mem_freq == 0:
- clear_mem_freq = 1
+ cnt = 0
pbar = tqdm(all_to_quantized_module_names)
- cnt = 1
- for name in pbar:
- pbar.set_description(f"Quantizing {name}")
- self._quantize_layer_via_rtn(name)
- if cnt % clear_mem_freq == 0:
- clear_memory(device_list=self.device_list)
- memory_monitor.log_summary()
- cnt = 1
- cnt += 1
+ # A workaround to trigger garbage collection in time
+ tied_weight_keys = getattr(self.model, "_tied_weight_keys", {})
+ tied_weight_values = list(tied_weight_keys.values())
+ # In fact, we should detect whether it is is_separate_lm_head, to simplify, we don't do it
+ if hasattr(self, "formats") and self.formats[0].is_gguf():
+ lm_head_name = get_lm_head_name(self.model)
+ if lm_head_name is not None:
+ tied_weight_values.append(lm_head_name)
+ for n, m in self.model.named_modules():
+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
+ pbar.set_description(f"Quantizing {m.global_name}")
+ self._quantize_layer_via_rtn(m.global_name)
+ cnt += 1
+ pbar.update()
+ if cnt % clear_mem_freq == 0:
+ clear_memory(device_list=self.device_list)
+ memory_monitor.log_summary()
+
+ elif not any(m.children()) and len(m.state_dict()) > 0 and n not in tied_weight_values:
+ set_module(self.model, n, copy.deepcopy(m))
+ if self.is_immediate_saving:
+ shard_writer(self, name=n)
+ m.to("meta")
+
# Convert remaining fp8
if is_fp8_model(self.model):
- convert_fp8_model_to_16b_model(self.model, self.amp_dtype, self.device)
+ convert_fp8_module_to_16b(self.model, self.amp_dtype, self.device)
+ if self.is_immediate_saving:
+ shard_writer(self, is_finalize=True)
+
self.quantized = True
return self.model, self.layer_config
@@ -1357,7 +1374,7 @@ class BaseCompressor(object):
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
if is_fp8_model(self.model):
- convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype, device=self.device)
+ convert_fp8_module_to_16b(block, dtype=self.amp_dtype, device=self.device)
if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:
set_auto_device_map_for_block_with_tuning(
@@ -1394,15 +1411,17 @@ class BaseCompressor(object):
block.to("cpu")
clear_memory(device_list=self.device_list)
- for _, m in block.named_modules():
+ for name, m in block.named_modules():
# fix issue: Ling-flash-2.0-q2_k_s fail infer on cuda but well on cpu
# https://huggingface.co/Intel/Ling-flash-2.0-gguf-q2ks-mixed-AutoRound/discussions/1
if hasattr(m, "imatrix"):
m.imatrix /= m.imatrix_cnt
- if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
- self._quantize_layer_via_rtn(m.tmp_name, to_cpu=self.low_gpu_mem_usage)
- all_to_quantized_module_names.remove(m.tmp_name)
- if not self.immediate_saving:
+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
+ self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)
+ all_to_quantized_module_names.remove(m.global_name)
+
+ if not self.is_immediate_saving:
+ # some modules may have been flushed and set to meta, so we could not move to gpu
mv_module_from_gpu(block)
if block_name == block_names[-1]:
clear_memory(input_ids, device_list=self.device_list)
@@ -1419,6 +1438,8 @@ class BaseCompressor(object):
dtype = torch.float32
self._quantize_layer_via_rtn(name, dtype=dtype)
# clear_memory(device_list=self.device_list)
+ # if self.is_immediate_saving:
+ # shard_writer(self, is_finalize=True)
def _update_inputs(self, inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:
keys = inputs.keys()
@@ -1452,6 +1473,69 @@ class BaseCompressor(object):
fill_default_value=fill_default_value,
)
+ def _adjust_immediate_packing_and_saving(self):
+ formats = getattr(self, "formats", [])
+ if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
+ self.is_immediate_packing = True
+
+ if self.has_qlayer_outside_block and self.iters != 0:
+ self.is_immediate_packing = False
+
+ if not ("causallm" in self.model.__class__.__name__.lower() and not self.mllm):
+ # TODO For tied keys, there may some issues, we haven't not verified this
+ tied_weight_keys = getattr(self.model, "_tied_weight_keys", {})
+ if len(tied_weight_keys) > 1:
+ self.is_immediate_saving = False
+ if self.low_cpu_mem_usage:
+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")
+ return
+ if len(tied_weight_keys) == 1:
+ key = tied_weight_keys.keys[0]
+ if "lm_head" not in key:
+ self.is_immediate_saving = False
+ if self.low_cpu_mem_usage:
+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")
+ return
+
+ if self.low_cpu_mem_usage and self.is_immediate_packing:
+ self.is_immediate_saving = True
+
+ if self.low_cpu_mem_usage and not self.is_immediate_packing:
+ logger.warning(
+ "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "
+ "Setting `low_cpu_mem_usage` to False."
+ )
+ self.low_cpu_mem_usage = False
+ self.is_immediate_saving = False
+
+ if self.low_cpu_mem_usage and self.is_immediate_packing:
+ if self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:
+ logger.warning(
+ "`low_cpu_mem_usage` is not fully supported "
+ "when there are quantized layers outside blocks and optimized RTN is disabled. "
+ "Setting `low_cpu_mem_usage` to False."
+ )
+ self.low_cpu_mem_usage = False
+ self.is_immediate_saving = False
+ elif self.has_qlayer_outside_block and self.iters > 0:
+ logger.warning(
+ "`low_cpu_mem_usage` is not fully supported "
+ "when there are quantized layers outside blocks and optimized RTN is disabled. "
+ "Setting low_cpu_mem_usage to False."
+ )
+ self.low_cpu_mem_usage = False
+ self.is_immediate_saving = False
+ elif formats[0].is_gguf():
+ logger.warning(
+ "`low_cpu_mem_usage` is not fully supported for gguf format" "Setting `low_cpu_mem_usage `to False."
+ )
+ self.low_cpu_mem_usage = False
+ self.is_immediate_saving = False
+
+ if self.is_immediate_saving and "int" not in self.data_type:
+ logger.warning("immediate_saving is only supported for int quantization, set to False")
+ self.is_immediate_saving = False
+
def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
Returns:
@@ -1467,7 +1551,7 @@ class BaseCompressor(object):
# Temporary names must be assigned after handle_moe_model;
# placing them earlier would cause them to be removed when the module is replaced.
for n, m in self.model.named_modules():
- m.tmp_name = n
+ m.global_name = n
if not self.is_auto_scheme:
enable_gguf_official_mixed = True
@@ -1475,19 +1559,20 @@ class BaseCompressor(object):
enable_gguf_official_mixed = False
self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)
+
+ def _should_disable_inplace_due_to_layers_outside_block() -> bool:
+ return self.has_qlayer_outside_block and (self.iters != 0 or (self.iters == 0 and not self.disable_opt_rtn))
+
+ # Disable inplace mode when there are quantized layers outside blocks
+ # under specific iteration/optimization settings.
+ if _should_disable_inplace_due_to_layers_outside_block():
+ self.inplace = False
if not hasattr(self, "formats"):
logger.warning("this API is deprecated, please use `quantize_and_save` instead")
else:
# Determine if immediate packing is required
- formats = self.formats
- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
- self.immediate_packing = True
- if not formats[0].is_gguf() and self.low_cpu_mem_usage:
- self.immediate_saving = True
+ self._adjust_immediate_packing_and_saving()
- if self.immediate_saving and "int" not in self.data_type:
- logger.warning("immediate_saving is only supported for int quantization, set to False")
- self.immediate_saving = False
if self.iters == 0:
return self._quantize_rtn()
@@ -1504,7 +1589,7 @@ class BaseCompressor(object):
self.model = self.model.to(self.amp_dtype)
layer_names = self._get_quantized_layer_names_outside_blocks()
- self.start_time = time.time()
+ start_time = time.time()
all_first_block_names = [block[0] for block in all_blocks]
if len(layer_names) > 0:
logger.info(
@@ -1559,7 +1644,7 @@ class BaseCompressor(object):
device=self.device,
pbar=pbar,
)
- if self.immediate_packing and len(self.formats) != 1:
+ if self.is_immediate_packing and len(self.formats) != 1:
raise ValueError(
f"Expected exactly one packing format when 'immediate_packing' is True, "
f"but got {len(self.formats)} formats."
@@ -1573,9 +1658,11 @@ class BaseCompressor(object):
if is_fp8_linear(m):
new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to("cpu")
set_module(self.model, n, new_layer)
+ if self.is_immediate_saving:
+ shard_writer(self, is_finalize=True)
end_time = time.time()
- cost_time = end_time - self.start_time
+ cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
# Dump a summary
@@ -1587,7 +1674,7 @@ class BaseCompressor(object):
quantized_layers.append(n)
else:
unquantized_layers.append(n)
- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately
+ elif hasattr(m, "scales") or hasattr(m, "scale"): # packing_immediately
quantized_layers.append(n)
summary_info = (
f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
@@ -1662,7 +1749,7 @@ class BaseCompressor(object):
if hasattr(self, "formats"):
has_gguf = any(format_.is_gguf() for format_ in self.formats)
- if has_gguf and self.immediate_packing:
+ if has_gguf and self.is_immediate_packing:
enable_quanted_input = False
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
@@ -1675,7 +1762,7 @@ class BaseCompressor(object):
accelerate.hooks.remove_hook_from_submodules(
self.model
) # self.model.hf_device_map has not been changed
- if not self.immediate_saving:
+ if not self.is_immediate_saving:
self.model = mv_module_from_gpu(self.model)
clear_memory(device_list=self.device_list)
quant_layer = self._quantize_layer
@@ -1685,12 +1772,12 @@ class BaseCompressor(object):
q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
q_layer_input = to_device(q_layer_input, self.cache_device)
quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
- if self.immediate_packing:
+ if self.is_immediate_packing:
self._immediate_pack(layer_name)
- if self.immediate_saving:
+ if self.is_immediate_saving:
m = get_module(self.model, layer_name)
- immediate_saving(self, m, name=layer_name, last_group=True)
+ shard_writer(self, m, name=layer_name, is_finalize=False)
del layer_input
clear_memory(q_layer_input, device_list=self.device_list)
memory_monitor.log_summary()
@@ -1762,7 +1849,7 @@ class BaseCompressor(object):
The capture hook look like:
def input_capture_hook(module, *args, **kwargs):
- _all_module_input[module._tmp_name].append((args, kwargs))
+ _all_module_input[module._global_name].append((args, kwargs))
"""
first_block_name = self.quant_block_list[0][0]
@@ -2925,19 +3012,18 @@ class BaseCompressor(object):
)
if hasattr(model, "config"):
del m.config
- if self.immediate_packing:
- for _, tmp_m in m.named_modules():
+ if self.is_immediate_packing:
+ for n, tmp_m in m.named_modules():
if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
continue
- self._immediate_pack(tmp_m.tmp_name)
+ self._immediate_pack(tmp_m.global_name)
- if self.immediate_saving:
- last_group = (i + nblocks) >= len(block_names)
- immediate_saving(self, m, last_group=last_group)
+ if self.is_immediate_saving:
+ shard_writer(self, m, is_finalize=False)
if pbar is not None:
pbar.update(1)
- if not self.immediate_saving:
+ if not self.is_immediate_saving:
self.model = mv_module_from_gpu(self.model)
for n, m in self.model.named_modules():
if hasattr(m, "name"):
diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.py
index 50e807320..6d9580e4f 100644
--- a/auto_round/compressors/diffusion/compressor.py
+++ b/auto_round/compressors/diffusion/compressor.py
@@ -94,7 +94,7 @@ class DiffusionCompressor(BaseCompressor):
nsamples: int = 128,
batch_size: int = 8,
gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
+ low_gpu_mem_usage: bool = True,
device_map: Union[str, torch.device, int, dict] = 0,
enable_torch_compile: bool = False,
seed: int = 42,
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
index 67180555a..462df089a 100644
--- a/auto_round/compressors/mllm/compressor.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -160,7 +160,7 @@ class MLLMCompressor(BaseCompressor):
nsamples: int = 128,
batch_size: int = 8,
gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
+ low_gpu_mem_usage: bool = True,
device_map: Union[str, torch.device, int, dict] = 0,
enable_torch_compile: bool = False,
seed: int = 42,
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
new file mode 100644
index 000000000..7641895b0
--- /dev/null
+++ b/auto_round/compressors/shard_writer.py
@@ -0,0 +1,226 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from collections import OrderedDict
+
+import torch
+
+from auto_round.logger import logger
+from auto_round.utils import get_lm_head_name, get_module
+
+
+class ShardWriter:
+ """
+ Handles shard-saving of model parameters to disk with memory management.
+ """
+
+ def __init__(self, rounder):
+ self.model = rounder.model
+ self.lm_head_name = get_lm_head_name(self.model)
+ total_params = sum(p.numel() for p in self.model.parameters())
+ # Heuristic estimate of model size in GB used to choose a default max_shard_size:
+ # - total_params * rounder.bits -> total number of bits in all parameters
+ # - // 8 -> convert bits to bytes
+ # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)
+ # - final // 10 -> apply a safety margin so default shards are
+ # smaller than the full model; this intentionally
+ # underestimates size before clamping below.
+ model_size = int(total_params * rounder.bits // 1e9 // 8) // 10
+ model_size = max(1, min(int(model_size), 5))
+
+ # Configuration
+ self.max_shard_size = self._parse_size(getattr(rounder, "max_shard_size", f"{model_size}GB"))
+ self.safe_serialization = getattr(rounder, "safe_serialization", True)
+
+ # Internal State
+ self.use_safetensors = self._check_safetensors()
+ self.shard_suffix = "safetensors" if self.use_safetensors else "bin"
+ self.current_shard_tensors = OrderedDict()
+ self.current_shard_size = 0
+ self.shard_meta = [] # List of {tmp_file: str, params: list}
+ self.global_weight_map = {}
+ self.shard_counter = 0
+
+ # Stats
+ self.total_param_elems = 0
+ self.total_param_size_bytes = 0
+
+ # Directory Setup
+ self.output_dir = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")
+ os.makedirs(self.output_dir, exist_ok=True)
+
+ def _parse_size(self, size_str: str) -> int:
+ if isinstance(size_str, int):
+ return size_str
+ s = size_str.strip().upper()
+ units = {"GB": 1024**3, "MB": 1024**2, "KB": 1024, "B": 1}
+ for unit, mult in units.items():
+ if s.endswith(unit):
+ return int(float(s[: -len(unit)]) * mult)
+ return int(s)
+
+ def _check_safetensors(self) -> bool:
+ if self.safe_serialization:
+ try:
+ import safetensors.torch
+
+ return True
+ except ImportError:
+ logger.warning("safetensors not installed; falling back to torch.save.")
+ return False
+
+ def save_module(self, m: torch.nn.Module, name: str = None):
+ """Extracts and accumulates tensors from a module."""
+ prefix = name if name is not None else getattr(m, "global_name", "model")
+ sd = m.state_dict()
+
+ for k, v in sd.items():
+ if not isinstance(v, torch.Tensor):
+ continue
+ param_name = f"{prefix}.{k}"
+ self._add_tensor(param_name, v)
+
+ def _add_tensor(self, name: str, tensor: torch.Tensor):
+ t_size = tensor.nbytes
+ self.total_param_elems += tensor.numel()
+ self.total_param_size_bytes += t_size
+ tensor = tensor.detach().cpu()
+ # If single tensor exceeds limit, flush current, save it solo, then continue
+ if t_size > self.max_shard_size:
+ self._flush_shard()
+ self.current_shard_tensors[name] = tensor
+ self.current_shard_size = t_size
+ self._flush_shard()
+ # If adding exceeds limit, flush first
+ elif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:
+ self._flush_shard()
+ self.current_shard_tensors[name] = tensor
+ self.current_shard_size = t_size
+ else:
+ self.current_shard_tensors[name] = tensor
+ self.current_shard_size += t_size
+
+ def _flush_shard(self):
+ if not self.current_shard_tensors:
+ return
+
+ self.shard_counter += 1
+ tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"
+ tmp_path = os.path.join(self.output_dir, tmp_name)
+
+ if self.use_safetensors:
+ from safetensors.torch import save_file
+
+ save_file(self.current_shard_tensors, tmp_path)
+ else:
+ torch.save(self.current_shard_tensors, tmp_path)
+
+ saved_params = list(self.current_shard_tensors.keys())
+ self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params})
+
+ # Offload logic: move modules to meta device once all params are saved
+ self._offload_to_meta(saved_params)
+
+ self.current_shard_tensors = OrderedDict()
+ self.current_shard_size = 0
+
+ def _offload_to_meta(self, saved_params):
+ """Attempts to move fully saved modules to the 'meta' device to free RAM."""
+ # Using a set for faster lookup of all saved parameters
+ all_saved = {p for meta in self.shard_meta for p in meta["params"]}
+
+ for param_full_name in saved_params:
+ module_path = param_full_name.rsplit(".", 1)[0]
+
+ module = get_module(self.model, module_path)
+ # Check if all parameters of this module are now in 'all_saved'
+ if module is not None and all(f"{module_path}.{k}" in all_saved for k in module.state_dict().keys()):
+ module.to("meta")
+
+ def finalize(self):
+ """Saves remaining weights, renames files, and writes the index JSON."""
+ # 1. Capture remaining weights not yet saved
+ full_sd = self.model.state_dict()
+ tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
+ all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}
+
+ for pname, tensor in full_sd.items():
+ if pname in all_saved_names:
+ continue
+ layer_name = ".".join(pname.split(".")[:-1])
+ if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:
+ lm_head_module = get_module(self.model, self.lm_head_name)
+ lm_head_module.to("meta") # Must to meta, otherwise model's saver will dump it again
+ continue
+ self._add_tensor(pname, tensor.detach().to("cpu"))
+
+ self._flush_shard()
+
+ # 2. Rename temp files to HF standard and map weights
+ if self.shard_counter == 0:
+ logger.warning("No tensors saved.")
+ return
+
+ for idx, meta in enumerate(self.shard_meta, start=1):
+ old_path = os.path.join(self.output_dir, meta["tmp_file"])
+ new_name = (
+ f"model.{self.shard_suffix}"
+ if self.shard_counter == 1
+ else f"model-{idx:05d}-of-{self.shard_counter:05d}.{self.shard_suffix}"
+ )
+
+ os.rename(old_path, os.path.join(self.output_dir, new_name))
+ for p in meta["params"]:
+ self.global_weight_map[p] = new_name
+
+ # 3. Write Index JSON
+ index_ext = "safetensors.index.json" if self.use_safetensors else "bin.index.json"
+ index_path = os.path.join(self.output_dir, f"model.{index_ext}")
+
+ index_data = {
+ "metadata": {
+ "format": "safetensors" if self.use_safetensors else "pytorch",
+ "total_shards": self.shard_counter,
+ "total_parameters": int(self.total_param_elems),
+ "total_size": int(self.total_param_size_bytes),
+ },
+ "weight_map": self.global_weight_map,
+ }
+
+ if self.shard_counter > 1:
+ with open(index_path, "w", encoding="utf-8") as f:
+ json.dump(index_data, f, indent=2)
+
+ logger.info(f"model has been saved to {self.output_dir}")
+
+
+def shard_writer(rounder: object, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):
+ if m is None and name is None and not is_finalize and not is_finalize:
+ raise ValueError("Must specify either name or m")
+ if not hasattr(rounder, "_shard_writer"):
+ rounder._shard_writer = ShardWriter(rounder)
+
+ if m is None and name is not None:
+ m = get_module(rounder.model, name)
+ # Perform the save
+ if m is not None:
+ rounder._shard_writer.save_module(m, name)
+
+ if is_finalize:
+ rounder._shard_writer.finalize()
+ # Optional: cleanup the saver object from rounder
+ del rounder._shard_writer
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index db95d9547..17d3c9f34 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -382,7 +382,7 @@ def set_layer_config(
if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
layer_config.setdefault(n, copy.deepcopy(default_dict))
layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
- logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")
+ # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")
# enforce shape divisibility for mxfp/nvfp
if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
for n, m in model.named_modules():
@@ -941,243 +941,6 @@ def reset_params(inputs):
inputs["use_cache"] = False
-def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last_group: bool = False):
- """
- Shard-saves the parameters of a model block (or group of blocks) immediately into disk,
- accumulating tensors into size-limited shards, optionally finalizing all remaining
- model weights when processing the last group.
-
- Args:
- rounder (object): The object of compressor.
- m (torch.nn.Module): The current block (or composite module) whose parameters will be added to the shard set.
- name (str): Override module name used as prefix for saved parameter keys. If None, falls back to m.tmp_name.
- last_group (bool): If True, triggers final pass over the entire model to include unsaved weights,
- writes shard index, renames shard files, copies source files, and releases temporary state.
- """
- import json
- from collections import OrderedDict
-
- from auto_round.utils import clear_memory, get_module
-
- # User configurable (can be preset on rounder)
- max_shard_size = getattr(rounder, "max_shard_size", "5GB")
- safe_serialization = getattr(rounder, "safe_serialization", True)
- if not hasattr(rounder, "quantized_layer_names_outside_blocks"):
- rounder.quantized_layer_names_outside_blocks = rounder._get_quantized_layer_names_outside_blocks()
- layer_names = rounder.quantized_layer_names_outside_blocks
- if len(layer_names) > 0 and name != layer_names[-1]:
- last_group = False
-
- def _parse_size(size_str: str) -> int:
- s = size_str.strip().upper()
- if s.endswith("GB"):
- return int(s[:-2]) * (1024**3)
- if s.endswith("MB"):
- return int(s[:-2]) * (1024**2)
- if s.endswith("KB"):
- return int(s[:-2]) * 1024
- return int(s)
-
- # Init global accumulators (once)
- if not hasattr(rounder, "_shard_init_done"):
- rounder._shard_init_done = True
- rounder._max_shard_bytes = _parse_size(str(max_shard_size))
- rounder._use_safetensors = False
- if safe_serialization:
- try:
- from safetensors.torch import save_file as _sf # noqa
-
- rounder._use_safetensors = True
- except ImportError:
- logger.warning("safe_serialization=True but safetensors not installed; fallback to torch.save.")
- rounder._current_shard_tensors = OrderedDict()
- rounder._current_shard_size = 0
- rounder._shard_meta = [] # list of dicts: {file, params}
- rounder._global_weight_map = {} # param_name -> final shard file (filled after finalize)
- rounder._shard_counter = 0
- rounder._shard_suffix = "safetensors" if rounder._use_safetensors else "bin"
- # new global counters
- rounder._total_param_elems = 0
- rounder._total_param_size_bytes = 0
- # Directory
- rounder._packed_blocks_root = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")
- os.makedirs(rounder._packed_blocks_root, exist_ok=True)
-
- # Collect tensors directly from current (multi)block `m`
- flat_tensors = OrderedDict()
- for k, v in m.state_dict().items():
- tmp_name = name if name is not None else m.tmp_name
- if isinstance(v, torch.Tensor):
- flat_tensors[f"{tmp_name}.{k}"] = v
-
- # Append tensors into the running shard(s)
- def _flush_current_shard():
- if len(rounder._current_shard_tensors) == 0:
- return
- rounder._shard_counter += 1
- tmp_name = f"model-shard-{rounder._shard_counter:05d}.{rounder._shard_suffix}" # temporary name
- tmp_path = os.path.join(rounder._packed_blocks_root, tmp_name)
- if rounder._use_safetensors:
- from safetensors.torch import save_file
-
- save_file(rounder._current_shard_tensors, tmp_path)
- else:
- torch.save(rounder._current_shard_tensors, tmp_path)
- params = list(rounder._current_shard_tensors.keys())
- rounder._shard_meta.append({"tmp_file": tmp_name, "params": params})
- for param in params:
- free_module_name = param.rsplit(".", 1)[0]
- free_module = get_module(rounder.model, free_module_name)
-
- # free module only when all its parameters have been saved
- free_flag = True
- free_module_state_dict = free_module.state_dict()
- already_saved_name = []
- for _meta in rounder._shard_meta:
- already_saved_name += _meta.get("params", [])
- for free_module_key in free_module_state_dict:
- free_module_key_full_name = f"{free_module_name}.{free_module_key}"
- if free_module_key_full_name not in already_saved_name:
- free_flag = False
- if free_flag:
- free_module.to("meta")
- del rounder._current_shard_tensors[param]
- rounder._current_shard_tensors = OrderedDict()
- rounder._current_shard_size = 0
-
- for pname, tensor in flat_tensors.items():
- t_elems = tensor.numel()
- t_size = t_elems * tensor.element_size()
- # accumulate global stats
- rounder._total_param_elems += t_elems
- rounder._total_param_size_bytes += t_size
- if t_size > rounder._max_shard_bytes:
- _flush_current_shard()
- rounder._current_shard_tensors[pname] = tensor
- rounder._current_shard_size = t_size
- _flush_current_shard()
- continue
- if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:
- _flush_current_shard()
- rounder._current_shard_tensors[pname] = tensor
- rounder._current_shard_size += t_size
-
- if last_group:
-
- # 1) Add the remaining (unsaved) model weights into new shard(s),
- # do not overwrite the already saved weights.
- try:
- full_sd = rounder.model.state_dict()
- except Exception as e:
- logger.warning(f"failed to obtain full state_dict for remaining weights: {e}")
- full_sd = {}
- tie_word_embeddings: bool = getattr(getattr(rounder.model, "config", None), "tie_word_embeddings", True)
- for pname, tensor in full_sd.items():
- if "lm_head" in pname and tie_word_embeddings:
- continue
- if not isinstance(tensor, torch.Tensor):
- continue
- # Check whether pname already stored in previous shards via _shard_meta
- already_saved = False
- for _meta in rounder._shard_meta:
- if pname in _meta.get("params", []):
- already_saved = True
- break
- if already_saved:
- continue # skip weights already saved
- # Size accounting
- t_elems = tensor.numel()
- t_size = t_elems * tensor.element_size()
-
- # Update global stats (these counters may already include earlier packed weights)
- rounder._total_param_elems += t_elems
- rounder._total_param_size_bytes += t_size
-
- # If this tensor alone exceeds shard size -> dedicated shard
- if t_size > rounder._max_shard_bytes:
- _flush_current_shard()
- rounder._current_shard_tensors[pname] = tensor.detach().cpu()
- rounder._current_shard_size = t_size
- _flush_current_shard()
- continue
-
- # If adding this tensor would overflow current shard -> flush current first
- if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:
- _flush_current_shard()
-
- # Add to current shard
- rounder._current_shard_tensors[pname] = tensor.detach().cpu()
- rounder._current_shard_size += t_size
-
- # 2) Flush any remaining unsaved leftover tensors
- _flush_current_shard()
-
- # 3) Finalize: rename temp shard files to HF-style names and build index
- total_shards = rounder._shard_counter
- if total_shards == 0:
- logger.warning("no tensors saved across all blocks")
- else:
- final_names = []
- for idx, meta in enumerate(rounder._shard_meta, start=1):
- old_tmp = meta["tmp_file"]
- old_path = os.path.join(rounder._packed_blocks_root, old_tmp)
- if total_shards == 1:
- new_name = f"model.{rounder._shard_suffix}"
- else:
- new_name = f"model-{idx:05d}-of-{total_shards:05d}.{rounder._shard_suffix}"
- new_path = os.path.join(rounder._packed_blocks_root, new_name)
- os.rename(old_path, new_path)
- final_names.append(new_name)
- for p in meta["params"]:
- rounder._global_weight_map[p] = new_name
-
- index_fname = "model.safetensors.index.json" if rounder._use_safetensors else "model.bin.index.json"
- index_path = os.path.join(rounder._packed_blocks_root, index_fname)
- index_data = {
- "metadata": {
- "format": "safetensors" if rounder._use_safetensors else "pytorch",
- "total_shards": total_shards,
- "total_parameters": int(getattr(rounder, "_total_param_elems", 0)),
- "total_size": int(getattr(rounder, "_total_param_size_bytes", 0)),
- },
- "weight_map": rounder._global_weight_map,
- }
- if total_shards > 1:
- with open(index_path, "w", encoding="utf-8") as f:
- json.dump(index_data, f, indent=2)
- logger.info(
- f"saved {total_shards} shard(s) (HF-style, including remaining unsaved weights) to "
- f"{rounder._packed_blocks_root} (index: {index_fname})"
- )
-
- try:
- copy_python_files_from_model_cache(rounder.model, rounder._get_save_folder_name(rounder.formats[0]))
- except Exception as e:
- logger.warning("Skipping source model Python file copy due to error: %s", e)
-
- # 4) Cleanup attributes to release memory after final shard is written
- try:
- attrs_to_cleanup = [
- "_current_shard_tensors",
- "_current_shard_size",
- "_shard_counter",
- "_max_shard_bytes",
- "_use_safetensors",
- "_shard_suffix",
- "_packed_blocks_root",
- "_total_param_elems",
- "_total_param_size_bytes",
- "_shard_init_done",
- "_shard_meta",
- "_global_weight_map",
- ]
- for _attr in attrs_to_cleanup:
- if hasattr(rounder, _attr):
- delattr(rounder, _attr)
- except Exception as _cleanup_err:
- logger.warning(f"shard cleanup warning: {_cleanup_err}")
-
-
class IndexSampler:
"""A cyclic sampler that returns shuffled index batches.
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index d16a28f3b..bc19dd32b 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -261,7 +261,7 @@ def save_quantized_as_autogptq(
# --- Block-wise quantization verification ---
for n, m in model.named_modules():
- m.tmp_name = n
+ m.global_name = n
all_to_quantized = True
modules_in_block_to_quantize = []
@@ -269,9 +269,9 @@ def save_quantized_as_autogptq(
for block_names in all_blocks:
first_block = get_module(model, block_names[0])
for n, m in first_block.named_modules():
- if m.tmp_name not in layer_config:
+ if m.global_name not in layer_config:
continue
- if not check_to_quantized(layer_config[m.tmp_name]):
+ if not check_to_quantized(layer_config[m.global_name]):
all_to_quantized = False
else:
modules_in_block_to_quantize.append(n)
@@ -281,7 +281,7 @@ def save_quantized_as_autogptq(
modules_in_block_to_quantize = None
for _, m in model.named_modules():
- delattr(m, "tmp_name")
+ delattr(m, "global_name")
if not inplace:
model = copy.deepcopy(model.to("cpu"))
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index ee3d4ecee..0e190184f 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -57,6 +57,7 @@ from auto_round.utils import (
get_module,
get_packing_device,
is_fp8_model,
+ is_separate_tensor,
logger,
)
@@ -372,7 +373,8 @@ def prepare_tensors(cls):
device = get_packing_device(cls.device)
for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):
-
+ if name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, name):
+ continue
if data_torch is None or data_torch.numel() == 0:
continue
# we don't need these
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 70317cf28..4fa88ffb1 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -193,7 +193,7 @@ def pack_gguf_layer(
block = get_module(model, block_name)
for n, m in block.named_modules():
if check_to_quantized(m):
- names_in_blocks.append(m.tmp_name)
+ names_in_blocks.append(m.global_name)
if name in model.last_layer_name_to_block_name:
# Packing block
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 6ec6e9b15..a81b34bb7 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -146,7 +146,7 @@ def _check_divisible_by_32(ar):
continue
ar.layer_config.setdefault(n, copy.deepcopy(default_dict))
ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
- logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")
+ logger.warning_once("some layers are skipped quantization (shape not divisible by 32).")
class OutputFormat(ABC):
@@ -353,7 +353,7 @@ class LLMCompressorFormat(OutputFormat):
self.output_format = f"llm_compressor:{format}"
self.backend = None
- def check_and_reset_format(self, ar: BaseCompressor) -> str:
+ def check_and_reset_format(self, ar: BaseCompressor) -> str | None:
if self.backend is not None:
new_format = self.backend.check_and_reset_format(ar)
self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index 8fbcc09fd..fc98eda31 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -463,16 +463,29 @@ def _clear_memory_for_cpu_and_cuda(
torch.xpu.empty_cache()
-@torch._dynamo.disable()
-def clear_memory(tensor: torch.Tensor | None | list[torch.Tensor] = None, device_list: list | tuple | None = None):
- from auto_round.utils.device import is_hpex_available
+class ClearMemory:
+ def __init__(self, device_list: list | tuple | None = None):
+ self.device_list = device_list
+
+ def __call__(
+ self,
+ tensor: torch.Tensor | None | list[torch.Tensor | dict] = None,
+ device_list: list | tuple | None = None,
+ ):
+ from auto_round.utils.device import is_hpex_available
+
+ if is_hpex_available():
+ memory_monitor.update_cpu()
+ return
+ else:
+ if device_list is not None:
+ self.device_list = device_list
+ final_device_list = self.device_list
+ memory_monitor.update(final_device_list)
+ _clear_memory_for_cpu_and_cuda(tensor, final_device_list)
- memory_monitor.update(device_list=device_list)
- if is_hpex_available():
- # hpu does not have empty_cache
- return
- else:
- _clear_memory_for_cpu_and_cuda(tensor, device_list)
+
+clear_memory = torch._dynamo.disable()(ClearMemory(device_list=[0]))
def clear_memory_if_reached_threshold(threshold=0.85, device_list=None):
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index f4c6e7b75..f14ad9539 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -840,7 +840,7 @@ def get_layer_names_in_block(
class_names = []
for n, m in model.named_modules():
if type(m) in supported_types or (class_names is not None and m.__class__.__name__ in class_names):
- m.bk_tmp_name = n
+ m.bk_global_name = n
layers_in_block = []
if bool(quant_block_list):
all_blocks = quant_block_list
@@ -850,9 +850,9 @@ def get_layer_names_in_block(
for block_name in block_names:
block = get_module(model, block_name)
for n, m in block.named_modules():
- if hasattr(m, "bk_tmp_name"):
- layers_in_block.append(m.bk_tmp_name)
- delattr(m, "bk_tmp_name")
+ if hasattr(m, "bk_global_name"):
+ layers_in_block.append(m.bk_global_name)
+ delattr(m, "bk_global_name")
return layers_in_block
@@ -1006,7 +1006,7 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"
if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
scheme_keys = (f.name for f in fields(QuantizationScheme))
- keys = tuple(scheme_keys) + ("tmp_name", "scale_dtype")
+ keys = tuple(scheme_keys) + ("global_name", "scale_dtype")
for key in keys:
setattr(new_layer, key, getattr(layer, key, None))
@@ -1020,7 +1020,7 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"
return new_layer
-def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16, device: str = "cpu"):
+def convert_fp8_module_to_16b(model, dtype=torch.bfloat16, device: str = "cpu"):
"""
Convert a model with FP8 quantized layers to a model with 16-bit linear layers.
This is useful for compatibility with other frameworks or for further processing.
@@ -1429,3 +1429,27 @@ def is_separate_lm_head(model: torch.nn.Module) -> bool:
return True
else:
return False
+
+
+def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:
+ dir_path = model.name_or_path
+ if not os.path.isdir(dir_path):
+ dir_path = download_hf_model(dir_path)
+ if not tensor_name.endswith(".weight"):
+ tensor_name += ".weight"
+
+ if "model.safetensors.index.json" in os.listdir(dir_path):
+ with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:
+ index_mapping = json.load(f)
+ if tensor_name in index_mapping["weight_map"]:
+ return True
+ else:
+ return False
+ else:
+ from safetensors import safe_open
+
+ f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
+ if tensor_name in f.keys():
+ return True
+ else:
+ return False
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 3383d6385..24836d85b 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -470,7 +470,7 @@ class WrapperLinear(torch.nn.Module):
Returns:
torch.Tensor: Output tensor after applying the wrapped layer.
"""
- # logger.info(self.orig_layer.tmp_name)
+ # logger.info(self.orig_layer.global_name)
x = x.to(self.device)
weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)
diff --git a/test/test_cpu/quantization/test_asym.py b/test/test_cpu/quantization/test_asym.py
index 525daa09e..7b0b90100 100644
--- a/test/test_cpu/quantization/test_asym.py
+++ b/test/test_cpu/quantization/test_asym.py
@@ -38,9 +38,9 @@ class TestAutoRoundAsym(unittest.TestCase):
def test_asym_group_size(self):
model_name = self.model_name
- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
- tokenizer = AutoTokenizer.from_pretrained(model_name)
for group_size in [32, 64, 128]:
+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
bits, sym = 4, False
ar = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)
ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)
@@ -59,9 +59,9 @@ class TestAutoRoundAsym(unittest.TestCase):
def test_asym_bits(self):
model_name = self.model_name
- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
- tokenizer = AutoTokenizer.from_pretrained(model_name)
for bits in [2, 8]:
+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
group_size, sym = 128, False
ar = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)
ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)
@@ -81,9 +81,10 @@ class TestAutoRoundAsym(unittest.TestCase):
# use parameters later
def test_asym_format(self):
model_name = self.model_name
- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
- tokenizer = AutoTokenizer.from_pretrained(model_name)
+
for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:
+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
bits, group_size, sym = 4, 128, False
ar = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)
# TODO when ark is ready, uncomment the following lines to do inference test