PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix low_cpu and set it to True as default (#1288)
Wenhua Chengf654e52deac2
diff --git a/README.md b/README.mdindex c8c0f3bfd..bd1eb06ff 100644--- a/README.md+++ b/README.md@@ -212,7 +212,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")##### Device/Speed Configuration- **`enable_torch_compile` (bool)**: If no exception is raised, typically we recommend setting it to True for faster quantization with lower resource.- **`low_gpu_mem_usage` (bool)**: Whether to offload intermediate features to CPU at the cost of ~20% more tuning time (default is `False`).-- **`low_cpu_mem_usage` (bool)**: [Experimental Feature]Whether to enable saving immediately to reduce ram usage (default is `False`).+- **`low_cpu_mem_usage` (bool)**: [Experimental Feature]Whether to enable saving immediately to reduce ram usage (default is `True`).- **`device_map` (str|dict|int)**: The device to be used for tuning, e.g., `auto`, `cpu`, `cuda`, `0,1,2` (default is `0`). When using `auto`, it will try to use all available GPUs.</details>diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex e3592e406..1540d254d 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -135,7 +135,12 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Enable memory-efficient mode by offloading intermediate features to CPU. ""Useful when working with large models that don't fit in GPU memory.",)- basic.add_argument("--low_cpu_mem_usage", action="store_true", help="Lower CPU memory mode. Defaults to False.")+ basic.add_argument(+ "--low_cpu_mem_usage", action="store_true", help="Deprecated, Lower CPU memory mode. Defaults to False."+ )+ basic.add_argument(+ "--disable_low_cpu_mem_usage", action="store_true", help="disable lower CPU memory mode. Defaults to False."+ )basic.add_argument("--format","--formats",@@ -259,8 +264,8 @@ class BasicArgumentParser(argparse.ArgumentParser):action="store_true",help="Enable PyTorch deterministic algorithms for reproducible results. ",)- group = tuning.add_mutually_exclusive_group()- group.add_argument(+ group_opt_rtn = tuning.add_mutually_exclusive_group()+ group_opt_rtn.add_argument("--disable_opt_rtn",action="store_const",const=True,@@ -269,7 +274,7 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Disable optimization for RTN (Round-To-Nearest) mode when iters=0. ""RTN is fast but less accurate; keeping optimization enabled is recommended.",)- group.add_argument(+ group_opt_rtn.add_argument("--enable_opt_rtn",action="store_const",const=False,@@ -513,6 +518,12 @@ def tune(args):from auto_round.utils import detect_device, get_library_version, logger+ if args.low_cpu_mem_usage:+ logger.warning(+ "`low_cpu_mem_usage` is deprecated and is now enabled by default. "+ "To disable it, use `--disable_low_cpu_mem_usage`."+ )+if args.format is None:args.format = "auto_round"@@ -639,6 +650,9 @@ def tune(args):shared_layers=args.shared_layers,ignore_scale_zp_bits=args.ignore_scale_zp_bits,)+ low_cpu_mem_usage = True+ if args.disable_low_cpu_mem_usage:+ low_cpu_mem_usage = Falseautoround: BaseCompressor = AutoRound(model=model_name,@@ -651,7 +665,7 @@ def tune(args):batch_size=args.batch_size,gradient_accumulate_steps=args.gradient_accumulate_steps,low_gpu_mem_usage=args.low_gpu_mem_usage,- low_cpu_mem_usage=args.low_cpu_mem_usage,+ low_cpu_mem_usage=low_cpu_mem_usage,device_map=args.device_map,enable_torch_compile=enable_torch_compile,seed=args.seed,diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex b4c8f8f49..17082430e 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -298,7 +298,7 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterabreturn []# Retrieve all high-level block names (for example, transformer blocks)for n, m in model.named_modules():- m.tmp_name = n # attach global name+ m.global_name = n # attach global nameblock_names = get_block_names(model, quant_vision=True)block_names = [item for sublist in block_names for item in sublist]@@ -327,7 +327,7 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterabblock_layer_names = []for name in block_layer_local_names:module = get_module(block_module, name)- block_layer_names.append(module.tmp_name)+ block_layer_names.append(module.global_name)for group in fuzzy_match_groups:matched_layers = set()diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex dd88bad84..33ca1ccd7 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -86,7 +86,7 @@ class AutoRound:extra_config: ExtraConfig = None,enable_alg_ext: bool = None,disable_opt_rtn: bool | None = None,- low_cpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = True,**kwargs,) -> BaseCompressor:"""Initialize AutoRound with quantization and tuning configuration.diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex f6714db16..5776a1a1d 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -45,8 +45,8 @@ def register_dataset(name):names = nameelse:names = [name]- for tmp_name in names:- CALIB_DATASETS[tmp_name] = dataset+ for global_name in names:+ CALIB_DATASETS[global_name] = datasetreturn datasetreturn registerdiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 4bba244ed..c30502f9e 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -14,7 +14,6 @@import copyimport os-import reimport sysimport timeimport traceback@@ -34,6 +33,7 @@ from transformers import set_seedfrom auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.compressors.shard_writer import shard_writerfrom auto_round.compressors.utils import (IndexSampler,block_forward,@@ -41,12 +41,10 @@ from auto_round.compressors.utils import (check_skippable_keywords,collect_best_params,get_shared_keys,- immediate_saving,infer_bits_by_data_type,init_cache,is_mx_fp,is_nv_fp,- is_standard_fp,is_static_wfp8afp8,is_wfp8afp8,reset_params,@@ -78,13 +76,13 @@ from auto_round.utils import (compile_func,convert_dtype_str2torch,convert_fp8_layer_to_linear,- convert_fp8_model_to_16b_model,- copy_python_files_from_model_cache,+ convert_fp8_module_to_16b,detect_device,find_matching_blocks,flatten_list,get_block_names,get_layer_names_in_block,+ get_lm_head_name,get_module,htcore,is_auto_device_mapping,@@ -191,7 +189,7 @@ class BaseCompressor(object):enable_alg_ext: bool = False,disable_opt_rtn: bool | None = None,seed: int = 42,- low_cpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = True,**kwargs,):"""Initialize AutoRound with quantization and tuning configuration.@@ -232,7 +230,7 @@ class BaseCompressor(object):- enable_alg_ext, quant_lm_head, lr, lr_scheduler, not_use_best_mse, dynamic_max_gap,super_group_size, super_bits, scale_dtype ("fp16" etc.),nblocks, to_quant_block_names,- enable_norm_bias_tuning, enable_quanted_input,+ enable_norm_bias_tuning, enable_quanted_input, enable_opt_rtn,disable_deterministic_algorithms, mllm, static_kv_dtype,enable_deterministic_algorithms,momentumRaises:ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.@@ -308,6 +306,7 @@ class BaseCompressor(object):self.momentum = kwargs.pop("momentum", 0.0)static_kv_dtype = kwargs.pop("static_kv_dtype", None)static_attention_dtype = kwargs.pop("static_attention_dtype", None)+ enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)device = kwargs.pop("device", None)if envs.AR_USE_MODELSCOPE:platform = "model_scope"@@ -389,7 +388,12 @@ class BaseCompressor(object):# Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.# To avoid None issue, we keep a copy though it's a little ugly+ if enable_opt_rtn and disable_opt_rtn:+ raise ValueError("`enable_opt_rtn` and `disable_opt_rtn` are mutually exclusive; " "only one can be set.")+ if enable_opt_rtn:+ disable_opt_rtn = Falseself.orig_disable_opt_rtn = disable_opt_rtn+if self.iters != 0 and self.orig_disable_opt_rtn is not None:logger.warning("`disable_opt_rtn` only works when `iters` is set to 0, ignore it now.")disable_opt_rtn = True@@ -420,8 +424,8 @@ class BaseCompressor(object):self.disable_opt_rtn = disable_opt_rtn# Whether to pack the layer immediately after tuning- self.immediate_packing = False- self.immediate_saving = False+ self.is_immediate_packing = False+ self.is_immediate_saving = False# KV cache, this one does not affect tuning but will collect some infos during tuningself.static_kv_dtype = static_kv_dtype@@ -1111,8 +1115,8 @@ class BaseCompressor(object):m = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device)set_module(self.model, name, m)tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device- # Step 1: Try quantization on GPU first, fall back to CPU if OOM- if self.immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:+ # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic+ if self.is_immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:m = m.to(tuning_device)m.scale = Nonem.zp = None@@ -1123,8 +1127,8 @@ class BaseCompressor(object):not disable_opt_rtnand self.orig_disable_opt_rtn is Noneand self.is_moe_model- and "expert" in m.tmp_name- and "shared_expert" not in m.tmp_name+ and "expert" in m.global_name+ and "shared_expert" not in m.global_nameand self.super_bits is None # GGUF still uses the optimized RTN for MoE layers):disable_opt_rtn = True@@ -1162,7 +1166,7 @@ class BaseCompressor(object):raise# Step 2: Optional immediate packing/export- if self.immediate_packing: # For gguf, packing conducts on block level+ if self.is_immediate_packing: # For gguf, packing conducts on block levelself._immediate_pack(name)if to_cpu:m = m.to("cpu")@@ -1172,17 +1176,13 @@ class BaseCompressor(object):if to_cpu:m = m.to("cpu")set_module(self.model, name, m)- if self.immediate_saving:- if hasattr(self, "all_to_quantized_module_names"):- all_to_quantized_module_names = self.all_to_quantized_module_names- else:- all_to_quantized_module_names = [n for n, m in self.model.named_modules() if check_to_quantized(m)]- last_module = (len(all_to_quantized_module_names) == 0) or (name == all_to_quantized_module_names[-1])+ if self.is_immediate_saving:m = get_module(self.model, name)- immediate_saving(self, m, name, last_module)+ m.to("cpu")+ shard_writer(self, m, name, False)def _immediate_pack(self, name: str):- if not self.immediate_packing:+ if not self.is_immediate_packing:returnself.formats[0].immediate_pack(name=name,@@ -1276,21 +1276,38 @@ class BaseCompressor(object):else:block_names_cnt = len(flatten_list(get_block_names(self.model, True)))clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt- if clear_mem_freq == 0:- clear_mem_freq = 1+ cnt = 0pbar = tqdm(all_to_quantized_module_names)- cnt = 1- for name in pbar:- pbar.set_description(f"Quantizing {name}")- self._quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:- clear_memory(device_list=self.device_list)- memory_monitor.log_summary()- cnt = 1- cnt += 1+ # A workaround to trigger garbage collection in time+ tied_weight_keys = getattr(self.model, "_tied_weight_keys", {})+ tied_weight_values = list(tied_weight_keys.values())+ # In fact, we should detect whether it is is_separate_lm_head, to simplify, we don't do it+ if hasattr(self, "formats") and self.formats[0].is_gguf():+ lm_head_name = get_lm_head_name(self.model)+ if lm_head_name is not None:+ tied_weight_values.append(lm_head_name)+ for n, m in self.model.named_modules():+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:+ pbar.set_description(f"Quantizing {m.global_name}")+ self._quantize_layer_via_rtn(m.global_name)+ cnt += 1+ pbar.update()+ if cnt % clear_mem_freq == 0:+ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()++ elif not any(m.children()) and len(m.state_dict()) > 0 and n not in tied_weight_values:+ set_module(self.model, n, copy.deepcopy(m))+ if self.is_immediate_saving:+ shard_writer(self, name=n)+ m.to("meta")+# Convert remaining fp8if is_fp8_model(self.model):- convert_fp8_model_to_16b_model(self.model, self.amp_dtype, self.device)+ convert_fp8_module_to_16b(self.model, self.amp_dtype, self.device)+ if self.is_immediate_saving:+ shard_writer(self, is_finalize=True)+self.quantized = Truereturn self.model, self.layer_config@@ -1357,7 +1374,7 @@ class BaseCompressor(object):pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)if is_fp8_model(self.model):- convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype, device=self.device)+ convert_fp8_module_to_16b(block, dtype=self.amp_dtype, device=self.device)if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:set_auto_device_map_for_block_with_tuning(@@ -1394,15 +1411,17 @@ class BaseCompressor(object):block.to("cpu")clear_memory(device_list=self.device_list)- for _, m in block.named_modules():+ for name, m in block.named_modules():# fix issue: Ling-flash-2.0-q2_k_s fail infer on cuda but well on cpu# https://huggingface.co/Intel/Ling-flash-2.0-gguf-q2ks-mixed-AutoRound/discussions/1if hasattr(m, "imatrix"):m.imatrix /= m.imatrix_cnt- if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:- self._quantize_layer_via_rtn(m.tmp_name, to_cpu=self.low_gpu_mem_usage)- all_to_quantized_module_names.remove(m.tmp_name)- if not self.immediate_saving:+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:+ self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)+ all_to_quantized_module_names.remove(m.global_name)++ if not self.is_immediate_saving:+ # some modules may have been flushed and set to meta, so we could not move to gpumv_module_from_gpu(block)if block_name == block_names[-1]:clear_memory(input_ids, device_list=self.device_list)@@ -1419,6 +1438,8 @@ class BaseCompressor(object):dtype = torch.float32self._quantize_layer_via_rtn(name, dtype=dtype)# clear_memory(device_list=self.device_list)+ # if self.is_immediate_saving:+ # shard_writer(self, is_finalize=True)def _update_inputs(self, inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:keys = inputs.keys()@@ -1452,6 +1473,69 @@ class BaseCompressor(object):fill_default_value=fill_default_value,)+ def _adjust_immediate_packing_and_saving(self):+ formats = getattr(self, "formats", [])+ if len(formats) == 1 and not formats[0].is_fake() and self.inplace:+ self.is_immediate_packing = True++ if self.has_qlayer_outside_block and self.iters != 0:+ self.is_immediate_packing = False++ if not ("causallm" in self.model.__class__.__name__.lower() and not self.mllm):+ # TODO For tied keys, there may some issues, we haven't not verified this+ tied_weight_keys = getattr(self.model, "_tied_weight_keys", {})+ if len(tied_weight_keys) > 1:+ self.is_immediate_saving = False+ if self.low_cpu_mem_usage:+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")+ return+ if len(tied_weight_keys) == 1:+ key = tied_weight_keys.keys[0]+ if "lm_head" not in key:+ self.is_immediate_saving = False+ if self.low_cpu_mem_usage:+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")+ return++ if self.low_cpu_mem_usage and self.is_immediate_packing:+ self.is_immediate_saving = True++ if self.low_cpu_mem_usage and not self.is_immediate_packing:+ logger.warning(+ "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "+ "Setting `low_cpu_mem_usage` to False."+ )+ self.low_cpu_mem_usage = False+ self.is_immediate_saving = False++ if self.low_cpu_mem_usage and self.is_immediate_packing:+ if self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:+ logger.warning(+ "`low_cpu_mem_usage` is not fully supported "+ "when there are quantized layers outside blocks and optimized RTN is disabled. "+ "Setting `low_cpu_mem_usage` to False."+ )+ self.low_cpu_mem_usage = False+ self.is_immediate_saving = False+ elif self.has_qlayer_outside_block and self.iters > 0:+ logger.warning(+ "`low_cpu_mem_usage` is not fully supported "+ "when there are quantized layers outside blocks and optimized RTN is disabled. "+ "Setting low_cpu_mem_usage to False."+ )+ self.low_cpu_mem_usage = False+ self.is_immediate_saving = False+ elif formats[0].is_gguf():+ logger.warning(+ "`low_cpu_mem_usage` is not fully supported for gguf format" "Setting `low_cpu_mem_usage `to False."+ )+ self.low_cpu_mem_usage = False+ self.is_immediate_saving = False++ if self.is_immediate_saving and "int" not in self.data_type:+ logger.warning("immediate_saving is only supported for int quantization, set to False")+ self.is_immediate_saving = False+def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.Returns:@@ -1467,7 +1551,7 @@ class BaseCompressor(object):# Temporary names must be assigned after handle_moe_model;# placing them earlier would cause them to be removed when the module is replaced.for n, m in self.model.named_modules():- m.tmp_name = n+ m.global_name = nif not self.is_auto_scheme:enable_gguf_official_mixed = True@@ -1475,19 +1559,20 @@ class BaseCompressor(object):enable_gguf_official_mixed = Falseself.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)++ def _should_disable_inplace_due_to_layers_outside_block() -> bool:+ return self.has_qlayer_outside_block and (self.iters != 0 or (self.iters == 0 and not self.disable_opt_rtn))++ # Disable inplace mode when there are quantized layers outside blocks+ # under specific iteration/optimization settings.+ if _should_disable_inplace_due_to_layers_outside_block():+ self.inplace = Falseif not hasattr(self, "formats"):logger.warning("this API is deprecated, please use `quantize_and_save` instead")else:# Determine if immediate packing is required- formats = self.formats- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:- self.immediate_packing = True- if not formats[0].is_gguf() and self.low_cpu_mem_usage:- self.immediate_saving = True+ self._adjust_immediate_packing_and_saving()- if self.immediate_saving and "int" not in self.data_type:- logger.warning("immediate_saving is only supported for int quantization, set to False")- self.immediate_saving = Falseif self.iters == 0:return self._quantize_rtn()@@ -1504,7 +1589,7 @@ class BaseCompressor(object):self.model = self.model.to(self.amp_dtype)layer_names = self._get_quantized_layer_names_outside_blocks()- self.start_time = time.time()+ start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]if len(layer_names) > 0:logger.info(@@ -1559,7 +1644,7 @@ class BaseCompressor(object):device=self.device,pbar=pbar,)- if self.immediate_packing and len(self.formats) != 1:+ if self.is_immediate_packing and len(self.formats) != 1:raise ValueError(f"Expected exactly one packing format when 'immediate_packing' is True, "f"but got {len(self.formats)} formats."@@ -1573,9 +1658,11 @@ class BaseCompressor(object):if is_fp8_linear(m):new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype, self.device).to("cpu")set_module(self.model, n, new_layer)+ if self.is_immediate_saving:+ shard_writer(self, is_finalize=True)end_time = time.time()- cost_time = end_time - self.start_time+ cost_time = end_time - start_timelogger.info(f"quantization tuning time {cost_time}")# Dump a summary@@ -1587,7 +1674,7 @@ class BaseCompressor(object):quantized_layers.append(n)else:unquantized_layers.append(n)- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately+ elif hasattr(m, "scales") or hasattr(m, "scale"): # packing_immediatelyquantized_layers.append(n)summary_info = (f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"@@ -1662,7 +1749,7 @@ class BaseCompressor(object):if hasattr(self, "formats"):has_gguf = any(format_.is_gguf() for format_ in self.formats)- if has_gguf and self.immediate_packing:+ if has_gguf and self.is_immediate_packing:enable_quanted_input = Falseif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:@@ -1675,7 +1762,7 @@ class BaseCompressor(object):accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed- if not self.immediate_saving:+ if not self.is_immediate_saving:self.model = mv_module_from_gpu(self.model)clear_memory(device_list=self.device_list)quant_layer = self._quantize_layer@@ -1685,12 +1772,12 @@ class BaseCompressor(object):q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else Noneq_layer_input = to_device(q_layer_input, self.cache_device)quant_layer(layer_name, layer_input, q_layer_input, device=self.device)- if self.immediate_packing:+ if self.is_immediate_packing:self._immediate_pack(layer_name)- if self.immediate_saving:+ if self.is_immediate_saving:m = get_module(self.model, layer_name)- immediate_saving(self, m, name=layer_name, last_group=True)+ shard_writer(self, m, name=layer_name, is_finalize=False)del layer_inputclear_memory(q_layer_input, device_list=self.device_list)memory_monitor.log_summary()@@ -1762,7 +1849,7 @@ class BaseCompressor(object):The capture hook look like:def input_capture_hook(module, *args, **kwargs):- _all_module_input[module._tmp_name].append((args, kwargs))+ _all_module_input[module._global_name].append((args, kwargs))"""first_block_name = self.quant_block_list[0][0]@@ -2925,19 +3012,18 @@ class BaseCompressor(object):)if hasattr(model, "config"):del m.config- if self.immediate_packing:- for _, tmp_m in m.named_modules():+ if self.is_immediate_packing:+ for n, tmp_m in m.named_modules():if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):continue- self._immediate_pack(tmp_m.tmp_name)+ self._immediate_pack(tmp_m.global_name)- if self.immediate_saving:- last_group = (i + nblocks) >= len(block_names)- immediate_saving(self, m, last_group=last_group)+ if self.is_immediate_saving:+ shard_writer(self, m, is_finalize=False)if pbar is not None:pbar.update(1)- if not self.immediate_saving:+ if not self.is_immediate_saving:self.model = mv_module_from_gpu(self.model)for n, m in self.model.named_modules():if hasattr(m, "name"):diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.pyindex 50e807320..6d9580e4f 100644--- a/auto_round/compressors/diffusion/compressor.py+++ b/auto_round/compressors/diffusion/compressor.py@@ -94,7 +94,7 @@ class DiffusionCompressor(BaseCompressor):nsamples: int = 128,batch_size: int = 8,gradient_accumulate_steps: int = 1,- low_gpu_mem_usage: bool = False,+ low_gpu_mem_usage: bool = True,device_map: Union[str, torch.device, int, dict] = 0,enable_torch_compile: bool = False,seed: int = 42,diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 67180555a..462df089a 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -160,7 +160,7 @@ class MLLMCompressor(BaseCompressor):nsamples: int = 128,batch_size: int = 8,gradient_accumulate_steps: int = 1,- low_gpu_mem_usage: bool = False,+ low_gpu_mem_usage: bool = True,device_map: Union[str, torch.device, int, dict] = 0,enable_torch_compile: bool = False,seed: int = 42,diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pynew file mode 100644index 000000000..7641895b0--- /dev/null+++ b/auto_round/compressors/shard_writer.py@@ -0,0 +1,226 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import json+import os+from collections import OrderedDict++import torch++from auto_round.logger import logger+from auto_round.utils import get_lm_head_name, get_module+++class ShardWriter:+ """+ Handles shard-saving of model parameters to disk with memory management.+ """++ def __init__(self, rounder):+ self.model = rounder.model+ self.lm_head_name = get_lm_head_name(self.model)+ total_params = sum(p.numel() for p in self.model.parameters())+ # Heuristic estimate of model size in GB used to choose a default max_shard_size:+ # - total_params * rounder.bits -> total number of bits in all parameters+ # - // 8 -> convert bits to bytes+ # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)+ # - final // 10 -> apply a safety margin so default shards are+ # smaller than the full model; this intentionally+ # underestimates size before clamping below.+ model_size = int(total_params * rounder.bits // 1e9 // 8) // 10+ model_size = max(1, min(int(model_size), 5))++ # Configuration+ self.max_shard_size = self._parse_size(getattr(rounder, "max_shard_size", f"{model_size}GB"))+ self.safe_serialization = getattr(rounder, "safe_serialization", True)++ # Internal State+ self.use_safetensors = self._check_safetensors()+ self.shard_suffix = "safetensors" if self.use_safetensors else "bin"+ self.current_shard_tensors = OrderedDict()+ self.current_shard_size = 0+ self.shard_meta = [] # List of {tmp_file: str, params: list}+ self.global_weight_map = {}+ self.shard_counter = 0++ # Stats+ self.total_param_elems = 0+ self.total_param_size_bytes = 0++ # Directory Setup+ self.output_dir = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")+ os.makedirs(self.output_dir, exist_ok=True)++ def _parse_size(self, size_str: str) -> int:+ if isinstance(size_str, int):+ return size_str+ s = size_str.strip().upper()+ units = {"GB": 1024**3, "MB": 1024**2, "KB": 1024, "B": 1}+ for unit, mult in units.items():+ if s.endswith(unit):+ return int(float(s[: -len(unit)]) * mult)+ return int(s)++ def _check_safetensors(self) -> bool:+ if self.safe_serialization:+ try:+ import safetensors.torch++ return True+ except ImportError:+ logger.warning("safetensors not installed; falling back to torch.save.")+ return False++ def save_module(self, m: torch.nn.Module, name: str = None):+ """Extracts and accumulates tensors from a module."""+ prefix = name if name is not None else getattr(m, "global_name", "model")+ sd = m.state_dict()++ for k, v in sd.items():+ if not isinstance(v, torch.Tensor):+ continue+ param_name = f"{prefix}.{k}"+ self._add_tensor(param_name, v)++ def _add_tensor(self, name: str, tensor: torch.Tensor):+ t_size = tensor.nbytes+ self.total_param_elems += tensor.numel()+ self.total_param_size_bytes += t_size+ tensor = tensor.detach().cpu()+ # If single tensor exceeds limit, flush current, save it solo, then continue+ if t_size > self.max_shard_size:+ self._flush_shard()+ self.current_shard_tensors[name] = tensor+ self.current_shard_size = t_size+ self._flush_shard()+ # If adding exceeds limit, flush first+ elif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:+ self._flush_shard()+ self.current_shard_tensors[name] = tensor+ self.current_shard_size = t_size+ else:+ self.current_shard_tensors[name] = tensor+ self.current_shard_size += t_size++ def _flush_shard(self):+ if not self.current_shard_tensors:+ return++ self.shard_counter += 1+ tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"+ tmp_path = os.path.join(self.output_dir, tmp_name)++ if self.use_safetensors:+ from safetensors.torch import save_file++ save_file(self.current_shard_tensors, tmp_path)+ else:+ torch.save(self.current_shard_tensors, tmp_path)++ saved_params = list(self.current_shard_tensors.keys())+ self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params})++ # Offload logic: move modules to meta device once all params are saved+ self._offload_to_meta(saved_params)++ self.current_shard_tensors = OrderedDict()+ self.current_shard_size = 0++ def _offload_to_meta(self, saved_params):+ """Attempts to move fully saved modules to the 'meta' device to free RAM."""+ # Using a set for faster lookup of all saved parameters+ all_saved = {p for meta in self.shard_meta for p in meta["params"]}++ for param_full_name in saved_params:+ module_path = param_full_name.rsplit(".", 1)[0]++ module = get_module(self.model, module_path)+ # Check if all parameters of this module are now in 'all_saved'+ if module is not None and all(f"{module_path}.{k}" in all_saved for k in module.state_dict().keys()):+ module.to("meta")++ def finalize(self):+ """Saves remaining weights, renames files, and writes the index JSON."""+ # 1. Capture remaining weights not yet saved+ full_sd = self.model.state_dict()+ tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)+ all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}++ for pname, tensor in full_sd.items():+ if pname in all_saved_names:+ continue+ layer_name = ".".join(pname.split(".")[:-1])+ if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:+ lm_head_module = get_module(self.model, self.lm_head_name)+ lm_head_module.to("meta") # Must to meta, otherwise model's saver will dump it again+ continue+ self._add_tensor(pname, tensor.detach().to("cpu"))++ self._flush_shard()++ # 2. Rename temp files to HF standard and map weights+ if self.shard_counter == 0:+ logger.warning("No tensors saved.")+ return++ for idx, meta in enumerate(self.shard_meta, start=1):+ old_path = os.path.join(self.output_dir, meta["tmp_file"])+ new_name = (+ f"model.{self.shard_suffix}"+ if self.shard_counter == 1+ else f"model-{idx:05d}-of-{self.shard_counter:05d}.{self.shard_suffix}"+ )++ os.rename(old_path, os.path.join(self.output_dir, new_name))+ for p in meta["params"]:+ self.global_weight_map[p] = new_name++ # 3. Write Index JSON+ index_ext = "safetensors.index.json" if self.use_safetensors else "bin.index.json"+ index_path = os.path.join(self.output_dir, f"model.{index_ext}")++ index_data = {+ "metadata": {+ "format": "safetensors" if self.use_safetensors else "pytorch",+ "total_shards": self.shard_counter,+ "total_parameters": int(self.total_param_elems),+ "total_size": int(self.total_param_size_bytes),+ },+ "weight_map": self.global_weight_map,+ }++ if self.shard_counter > 1:+ with open(index_path, "w", encoding="utf-8") as f:+ json.dump(index_data, f, indent=2)++ logger.info(f"model has been saved to {self.output_dir}")++[email protected]_grad()+def shard_writer(rounder: object, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):+ if m is None and name is None and not is_finalize and not is_finalize:+ raise ValueError("Must specify either name or m")+ if not hasattr(rounder, "_shard_writer"):+ rounder._shard_writer = ShardWriter(rounder)++ if m is None and name is not None:+ m = get_module(rounder.model, name)+ # Perform the save+ if m is not None:+ rounder._shard_writer.save_module(m, name)++ if is_finalize:+ rounder._shard_writer.finalize()+ # Optional: cleanup the saver object from rounder+ del rounder._shard_writerdiff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex db95d9547..17d3c9f34 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -382,7 +382,7 @@ def set_layer_config(if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:layer_config.setdefault(n, copy.deepcopy(default_dict))layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")+ # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")# enforce shape divisibility for mxfp/nvfpif (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:for n, m in model.named_modules():@@ -941,243 +941,6 @@ def reset_params(inputs):inputs["use_cache"] = False-def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, last_group: bool = False):- """- Shard-saves the parameters of a model block (or group of blocks) immediately into disk,- accumulating tensors into size-limited shards, optionally finalizing all remaining- model weights when processing the last group.-- Args:- rounder (object): The object of compressor.- m (torch.nn.Module): The current block (or composite module) whose parameters will be added to the shard set.- name (str): Override module name used as prefix for saved parameter keys. If None, falls back to m.tmp_name.- last_group (bool): If True, triggers final pass over the entire model to include unsaved weights,- writes shard index, renames shard files, copies source files, and releases temporary state.- """- import json- from collections import OrderedDict-- from auto_round.utils import clear_memory, get_module-- # User configurable (can be preset on rounder)- max_shard_size = getattr(rounder, "max_shard_size", "5GB")- safe_serialization = getattr(rounder, "safe_serialization", True)- if not hasattr(rounder, "quantized_layer_names_outside_blocks"):- rounder.quantized_layer_names_outside_blocks = rounder._get_quantized_layer_names_outside_blocks()- layer_names = rounder.quantized_layer_names_outside_blocks- if len(layer_names) > 0 and name != layer_names[-1]:- last_group = False-- def _parse_size(size_str: str) -> int:- s = size_str.strip().upper()- if s.endswith("GB"):- return int(s[:-2]) * (1024**3)- if s.endswith("MB"):- return int(s[:-2]) * (1024**2)- if s.endswith("KB"):- return int(s[:-2]) * 1024- return int(s)-- # Init global accumulators (once)- if not hasattr(rounder, "_shard_init_done"):- rounder._shard_init_done = True- rounder._max_shard_bytes = _parse_size(str(max_shard_size))- rounder._use_safetensors = False- if safe_serialization:- try:- from safetensors.torch import save_file as _sf # noqa-- rounder._use_safetensors = True- except ImportError:- logger.warning("safe_serialization=True but safetensors not installed; fallback to torch.save.")- rounder._current_shard_tensors = OrderedDict()- rounder._current_shard_size = 0- rounder._shard_meta = [] # list of dicts: {file, params}- rounder._global_weight_map = {} # param_name -> final shard file (filled after finalize)- rounder._shard_counter = 0- rounder._shard_suffix = "safetensors" if rounder._use_safetensors else "bin"- # new global counters- rounder._total_param_elems = 0- rounder._total_param_size_bytes = 0- # Directory- rounder._packed_blocks_root = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")- os.makedirs(rounder._packed_blocks_root, exist_ok=True)-- # Collect tensors directly from current (multi)block `m`- flat_tensors = OrderedDict()- for k, v in m.state_dict().items():- tmp_name = name if name is not None else m.tmp_name- if isinstance(v, torch.Tensor):- flat_tensors[f"{tmp_name}.{k}"] = v-- # Append tensors into the running shard(s)- def _flush_current_shard():- if len(rounder._current_shard_tensors) == 0:- return- rounder._shard_counter += 1- tmp_name = f"model-shard-{rounder._shard_counter:05d}.{rounder._shard_suffix}" # temporary name- tmp_path = os.path.join(rounder._packed_blocks_root, tmp_name)- if rounder._use_safetensors:- from safetensors.torch import save_file-- save_file(rounder._current_shard_tensors, tmp_path)- else:- torch.save(rounder._current_shard_tensors, tmp_path)- params = list(rounder._current_shard_tensors.keys())- rounder._shard_meta.append({"tmp_file": tmp_name, "params": params})- for param in params:- free_module_name = param.rsplit(".", 1)[0]- free_module = get_module(rounder.model, free_module_name)-- # free module only when all its parameters have been saved- free_flag = True- free_module_state_dict = free_module.state_dict()- already_saved_name = []- for _meta in rounder._shard_meta:- already_saved_name += _meta.get("params", [])- for free_module_key in free_module_state_dict:- free_module_key_full_name = f"{free_module_name}.{free_module_key}"- if free_module_key_full_name not in already_saved_name:- free_flag = False- if free_flag:- free_module.to("meta")- del rounder._current_shard_tensors[param]- rounder._current_shard_tensors = OrderedDict()- rounder._current_shard_size = 0-- for pname, tensor in flat_tensors.items():- t_elems = tensor.numel()- t_size = t_elems * tensor.element_size()- # accumulate global stats- rounder._total_param_elems += t_elems- rounder._total_param_size_bytes += t_size- if t_size > rounder._max_shard_bytes:- _flush_current_shard()- rounder._current_shard_tensors[pname] = tensor- rounder._current_shard_size = t_size- _flush_current_shard()- continue- if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:- _flush_current_shard()- rounder._current_shard_tensors[pname] = tensor- rounder._current_shard_size += t_size-- if last_group:-- # 1) Add the remaining (unsaved) model weights into new shard(s),- # do not overwrite the already saved weights.- try:- full_sd = rounder.model.state_dict()- except Exception as e:- logger.warning(f"failed to obtain full state_dict for remaining weights: {e}")- full_sd = {}- tie_word_embeddings: bool = getattr(getattr(rounder.model, "config", None), "tie_word_embeddings", True)- for pname, tensor in full_sd.items():- if "lm_head" in pname and tie_word_embeddings:- continue- if not isinstance(tensor, torch.Tensor):- continue- # Check whether pname already stored in previous shards via _shard_meta- already_saved = False- for _meta in rounder._shard_meta:- if pname in _meta.get("params", []):- already_saved = True- break- if already_saved:- continue # skip weights already saved- # Size accounting- t_elems = tensor.numel()- t_size = t_elems * tensor.element_size()-- # Update global stats (these counters may already include earlier packed weights)- rounder._total_param_elems += t_elems- rounder._total_param_size_bytes += t_size-- # If this tensor alone exceeds shard size -> dedicated shard- if t_size > rounder._max_shard_bytes:- _flush_current_shard()- rounder._current_shard_tensors[pname] = tensor.detach().cpu()- rounder._current_shard_size = t_size- _flush_current_shard()- continue-- # If adding this tensor would overflow current shard -> flush current first- if rounder._current_shard_size + t_size > rounder._max_shard_bytes and rounder._current_shard_size > 0:- _flush_current_shard()-- # Add to current shard- rounder._current_shard_tensors[pname] = tensor.detach().cpu()- rounder._current_shard_size += t_size-- # 2) Flush any remaining unsaved leftover tensors- _flush_current_shard()-- # 3) Finalize: rename temp shard files to HF-style names and build index- total_shards = rounder._shard_counter- if total_shards == 0:- logger.warning("no tensors saved across all blocks")- else:- final_names = []- for idx, meta in enumerate(rounder._shard_meta, start=1):- old_tmp = meta["tmp_file"]- old_path = os.path.join(rounder._packed_blocks_root, old_tmp)- if total_shards == 1:- new_name = f"model.{rounder._shard_suffix}"- else:- new_name = f"model-{idx:05d}-of-{total_shards:05d}.{rounder._shard_suffix}"- new_path = os.path.join(rounder._packed_blocks_root, new_name)- os.rename(old_path, new_path)- final_names.append(new_name)- for p in meta["params"]:- rounder._global_weight_map[p] = new_name-- index_fname = "model.safetensors.index.json" if rounder._use_safetensors else "model.bin.index.json"- index_path = os.path.join(rounder._packed_blocks_root, index_fname)- index_data = {- "metadata": {- "format": "safetensors" if rounder._use_safetensors else "pytorch",- "total_shards": total_shards,- "total_parameters": int(getattr(rounder, "_total_param_elems", 0)),- "total_size": int(getattr(rounder, "_total_param_size_bytes", 0)),- },- "weight_map": rounder._global_weight_map,- }- if total_shards > 1:- with open(index_path, "w", encoding="utf-8") as f:- json.dump(index_data, f, indent=2)- logger.info(- f"saved {total_shards} shard(s) (HF-style, including remaining unsaved weights) to "- f"{rounder._packed_blocks_root} (index: {index_fname})"- )-- try:- copy_python_files_from_model_cache(rounder.model, rounder._get_save_folder_name(rounder.formats[0]))- except Exception as e:- logger.warning("Skipping source model Python file copy due to error: %s", e)-- # 4) Cleanup attributes to release memory after final shard is written- try:- attrs_to_cleanup = [- "_current_shard_tensors",- "_current_shard_size",- "_shard_counter",- "_max_shard_bytes",- "_use_safetensors",- "_shard_suffix",- "_packed_blocks_root",- "_total_param_elems",- "_total_param_size_bytes",- "_shard_init_done",- "_shard_meta",- "_global_weight_map",- ]- for _attr in attrs_to_cleanup:- if hasattr(rounder, _attr):- delattr(rounder, _attr)- except Exception as _cleanup_err:- logger.warning(f"shard cleanup warning: {_cleanup_err}")--class IndexSampler:"""A cyclic sampler that returns shuffled index batches.diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex d16a28f3b..bc19dd32b 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -261,7 +261,7 @@ def save_quantized_as_autogptq(# --- Block-wise quantization verification ---for n, m in model.named_modules():- m.tmp_name = n+ m.global_name = nall_to_quantized = Truemodules_in_block_to_quantize = []@@ -269,9 +269,9 @@ def save_quantized_as_autogptq(for block_names in all_blocks:first_block = get_module(model, block_names[0])for n, m in first_block.named_modules():- if m.tmp_name not in layer_config:+ if m.global_name not in layer_config:continue- if not check_to_quantized(layer_config[m.tmp_name]):+ if not check_to_quantized(layer_config[m.global_name]):all_to_quantized = Falseelse:modules_in_block_to_quantize.append(n)@@ -281,7 +281,7 @@ def save_quantized_as_autogptq(modules_in_block_to_quantize = Nonefor _, m in model.named_modules():- delattr(m, "tmp_name")+ delattr(m, "global_name")if not inplace:model = copy.deepcopy(model.to("cpu"))diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex ee3d4ecee..0e190184f 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -57,6 +57,7 @@ from auto_round.utils import (get_module,get_packing_device,is_fp8_model,+ is_separate_tensor,logger,)@@ -372,7 +373,8 @@ def prepare_tensors(cls):device = get_packing_device(cls.device)for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):-+ if name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, name):+ continueif data_torch is None or data_torch.numel() == 0:continue# we don't need thesediff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 70317cf28..4fa88ffb1 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -193,7 +193,7 @@ def pack_gguf_layer(block = get_module(model, block_name)for n, m in block.named_modules():if check_to_quantized(m):- names_in_blocks.append(m.tmp_name)+ names_in_blocks.append(m.global_name)if name in model.last_layer_name_to_block_name:# Packing blockdiff --git a/auto_round/formats.py b/auto_round/formats.pyindex 6ec6e9b15..a81b34bb7 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -146,7 +146,7 @@ def _check_divisible_by_32(ar):continuear.layer_config.setdefault(n, copy.deepcopy(default_dict))ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")+ logger.warning_once("some layers are skipped quantization (shape not divisible by 32).")class OutputFormat(ABC):@@ -353,7 +353,7 @@ class LLMCompressorFormat(OutputFormat):self.output_format = f"llm_compressor:{format}"self.backend = None- def check_and_reset_format(self, ar: BaseCompressor) -> str:+ def check_and_reset_format(self, ar: BaseCompressor) -> str | None:if self.backend is not None:new_format = self.backend.check_and_reset_format(ar)self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backenddiff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 8fbcc09fd..fc98eda31 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -463,16 +463,29 @@ def _clear_memory_for_cpu_and_cuda(torch.xpu.empty_cache()-@torch._dynamo.disable()-def clear_memory(tensor: torch.Tensor | None | list[torch.Tensor] = None, device_list: list | tuple | None = None):- from auto_round.utils.device import is_hpex_available+class ClearMemory:+ def __init__(self, device_list: list | tuple | None = None):+ self.device_list = device_list++ def __call__(+ self,+ tensor: torch.Tensor | None | list[torch.Tensor | dict] = None,+ device_list: list | tuple | None = None,+ ):+ from auto_round.utils.device import is_hpex_available++ if is_hpex_available():+ memory_monitor.update_cpu()+ return+ else:+ if device_list is not None:+ self.device_list = device_list+ final_device_list = self.device_list+ memory_monitor.update(final_device_list)+ _clear_memory_for_cpu_and_cuda(tensor, final_device_list)- memory_monitor.update(device_list=device_list)- if is_hpex_available():- # hpu does not have empty_cache- return- else:- _clear_memory_for_cpu_and_cuda(tensor, device_list)++clear_memory = torch._dynamo.disable()(ClearMemory(device_list=[0]))def clear_memory_if_reached_threshold(threshold=0.85, device_list=None):diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex f4c6e7b75..f14ad9539 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -840,7 +840,7 @@ def get_layer_names_in_block(class_names = []for n, m in model.named_modules():if type(m) in supported_types or (class_names is not None and m.__class__.__name__ in class_names):- m.bk_tmp_name = n+ m.bk_global_name = nlayers_in_block = []if bool(quant_block_list):all_blocks = quant_block_list@@ -850,9 +850,9 @@ def get_layer_names_in_block(for block_name in block_names:block = get_module(model, block_name)for n, m in block.named_modules():- if hasattr(m, "bk_tmp_name"):- layers_in_block.append(m.bk_tmp_name)- delattr(m, "bk_tmp_name")+ if hasattr(m, "bk_global_name"):+ layers_in_block.append(m.bk_global_name)+ delattr(m, "bk_global_name")return layers_in_block@@ -1006,7 +1006,7 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"if layer.bias is not None:new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))scheme_keys = (f.name for f in fields(QuantizationScheme))- keys = tuple(scheme_keys) + ("tmp_name", "scale_dtype")+ keys = tuple(scheme_keys) + ("global_name", "scale_dtype")for key in keys:setattr(new_layer, key, getattr(layer, key, None))@@ -1020,7 +1020,7 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"return new_layer-def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16, device: str = "cpu"):+def convert_fp8_module_to_16b(model, dtype=torch.bfloat16, device: str = "cpu"):"""Convert a model with FP8 quantized layers to a model with 16-bit linear layers.This is useful for compatibility with other frameworks or for further processing.@@ -1429,3 +1429,27 @@ def is_separate_lm_head(model: torch.nn.Module) -> bool:return Trueelse:return False+++def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:+ dir_path = model.name_or_path+ if not os.path.isdir(dir_path):+ dir_path = download_hf_model(dir_path)+ if not tensor_name.endswith(".weight"):+ tensor_name += ".weight"++ if "model.safetensors.index.json" in os.listdir(dir_path):+ with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:+ index_mapping = json.load(f)+ if tensor_name in index_mapping["weight_map"]:+ return True+ else:+ return False+ else:+ from safetensors import safe_open++ f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")+ if tensor_name in f.keys():+ return True+ else:+ return Falsediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 3383d6385..24836d85b 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -470,7 +470,7 @@ class WrapperLinear(torch.nn.Module):Returns:torch.Tensor: Output tensor after applying the wrapped layer."""- # logger.info(self.orig_layer.tmp_name)+ # logger.info(self.orig_layer.global_name)x = x.to(self.device)weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)diff --git a/test/test_cpu/quantization/test_asym.py b/test/test_cpu/quantization/test_asym.pyindex 525daa09e..7b0b90100 100644--- a/test/test_cpu/quantization/test_asym.py+++ b/test/test_cpu/quantization/test_asym.py@@ -38,9 +38,9 @@ class TestAutoRoundAsym(unittest.TestCase):def test_asym_group_size(self):model_name = self.model_name- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")- tokenizer = AutoTokenizer.from_pretrained(model_name)for group_size in [32, 64, 128]:+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)bits, sym = 4, Falsear = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)@@ -59,9 +59,9 @@ class TestAutoRoundAsym(unittest.TestCase):def test_asym_bits(self):model_name = self.model_name- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")- tokenizer = AutoTokenizer.from_pretrained(model_name)for bits in [2, 8]:+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)group_size, sym = 128, Falsear = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)@@ -81,9 +81,10 @@ class TestAutoRoundAsym(unittest.TestCase):# use parameters laterdef test_asym_format(self):model_name = self.model_name- model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")- tokenizer = AutoTokenizer.from_pretrained(model_name)+for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:+ model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)bits, group_size, sym = 4, 128, Falsear = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)# TODO when ark is ready, uncomment the following lines to do inference test