Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support packing immediately for gguf to reduce ram usage (#638)

Wenhua Cheng3b282dc9dd86
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4d9b299a2..e1fa1d238 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -19,7 +19,7 @@ import sys
import torch
import copy
import time
-from typing import Optional, Union, List
+from typing import Union, Any
from transformers import set_seed
from torch import autocast
from tqdm import tqdm
@@ -56,9 +56,10 @@ from auto_round.utils import (
init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,
_gguf_args_check,
check_seqlen_compatible,
- get_layer_config_by_gguf_format, get_lm_head_name
+ get_layer_config_by_gguf_format, get_lm_head_name, flatten_list
)
-from .low_cpu_mem.utils import get_layers_before_block
+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+from auto_round.low_cpu_mem.utils import get_layers_before_block
class AutoRound(object):
@@ -446,12 +447,8 @@ class AutoRound(object):
f" than gradient_accumulate_steps * batch_size")
self._dq_check()
- # def _check_format_compatibility(self, format): ##TODO
- # ##check lm_head, mixed_bits, bits, each layer supporting, etc
- # pass
def _check_compatibility(self):
- # Check the legitimacy of seqlen
##check gguf and others
has_gguf = False
@@ -525,6 +522,7 @@ class AutoRound(object):
ValueError: If an unsupported format is specified.
"""
# Validate and process the specified formats
+ self.orig_output_dir = output_dir
_gguf_args_check(self, format)
formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
@@ -595,15 +593,11 @@ class AutoRound(object):
formats = remove_duplicates(formats)
self.formats = formats
- # # Check format compatibility
- # self._check_format_compatibility(formats)
-
# Perform model quantization
model, _ = self.quantize()
# Save the quantized model in the specified formats
folders = []
- low_cpu_mem_usage = len(formats) == 1 and "gguf" in formats[0]
for format in formats:
if "gptq" in format and not self.sym:
logger.warning(
@@ -612,350 +606,531 @@ class AutoRound(object):
" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
"the AutoRound format(2/3/4/8 bits)."
)
- save_format_ = format.replace(":", "-").replace("_", "-")
- save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir
- self.save_quantized(
- save_folder, format=format, inplace=inplace, low_cpu_mem_usage=low_cpu_mem_usage, **kwargs)
+ save_folder = self.get_save_folder_name(format)
+ self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
folders.append(save_folder)
return model, folders
- @torch.inference_mode
+ def get_save_folder_name(self, format_str: str) -> str:
+ """Generates the save folder name based on the provided format string.
+
+ If there are multiple formats to handle, the function creates a subfolder
+ named after the format string with special characters replaced. If there's
+ only one format, it returns the original output directory directly.
+
+ Args:
+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
+
+ Returns:
+ str: The path to the folder where results should be saved.
+ """
+ # Replace special characters to make the folder name filesystem-safe
+ sanitized_format = format_str.replace(":", "-").replace("_", "-")
+
+ # Use a subfolder only if there are multiple formats
+ if len(self.formats) > 1:
+ return os.path.join(self.orig_output_dir, sanitized_format)
+
+ return self.orig_output_dir
+
+ @torch.inference_mode()
def quantize_embedding_layer(self):
- for n, m in self.model.named_modules():
- if not isinstance(m, torch.nn.Embedding) or n not in self.layer_config:
+ """Quantizes embedding layers in the model according to the configuration.
+
+ This method iterates through all modules in the model, identifies embedding
+ layers specified in `self.layer_config`, and applies the appropriate quantization
+ function based on bit precision, grouping strategy, and dtype.
+
+ Returns:
+ bool: True if the quantization process completes without critical errors.
+ """
+ for name, module in self.model.named_modules():
+ # Skip non-Embedding modules or layers not in config
+ if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:
continue
- config = self.layer_config[n]
+ config = self.layer_config[name]
+
+ # Skip layers that are not marked for quantization
if not check_to_quantized(config):
continue
- layer = m
- layer_name = n
config["scale_dtype"] = self.scale_dtype
-
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
dtype = config["data_type"]
+
+ # Determine quantization function key with symmetry/asymmetry
if dtype not in QUANT_FUNC_WITH_DTYPE:
- if config['sym']:
- dtype = dtype + "_sym"
- else:
- dtype = dtype + "_asym"
+ dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"
+
+ # Optionally use optimized rounding (RTN) variant
+ if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:
+ dtype = f"rtn_{dtype}"
- if not self.disable_opt_rtn and "rtn_" + dtype in QUANT_FUNC_WITH_DTYPE:
- dtype = "rtn_" + dtype
quant_func = QUANT_FUNC_WITH_DTYPE[dtype]
+ # Attempt quantization on GPU, fall back to CPU if OOM
try:
- weight, scale, zp = quant_func(layer.weight.to(self.device), **{k: config[k] for k in
- ["bits", "group_size", "super_bits",
- "super_group_size", "scale_dtype"]})
+ weight, scale, zp = quant_func(
+ module.weight.to(self.device),
+ **{k: config[k] for k in [
+ "bits", "group_size", "super_bits",
+ "super_group_size", "scale_dtype"
+ ]}
+ )
except RuntimeError as e:
if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
- logger.info("out of vram, fallback to cpu")
- weight, scale, zp = quant_func(layer.weight.to("cpu"), **{k: config[k] for k in
- ["bits", "group_size", "super_bits",
- "super_group_size", "scale_dtype"]})
+ logger.info("out of VRAM, falling back to CPU.")
+ weight, scale, zp = quant_func(
+ module.weight.to("cpu"),
+ **{k: config[k] for k in [
+ "bits", "group_size", "super_bits",
+ "super_group_size", "scale_dtype"
+ ]}
+ )
else:
raise
- layer.weight.data.copy_(weight.cpu())
- for name, val in zip(["scale", "zp"], [scale, zp]):
- if isinstance(val, dict):
- for k, v in val.items():
- setattr(layer, k if k == "scale" else "w_" + k, v.cpu())
+ # Overwrite the module's weights with the quantized version
+ module.weight.data.copy_(weight.cpu())
+
+ # Attach scale and zero point (zp) to the module
+ for param_name, value in zip(["scale", "zp"], [scale, zp]):
+ if isinstance(value, dict):
+ for k, v in value.items():
+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
else:
- setattr(layer, name, val.cpu())
+ setattr(module, param_name, value.cpu())
- self.layer_config.setdefault(layer_name, {}).update(config)
+ # Update config
+ self.layer_config.setdefault(name, {}).update(config)
+
+ # Release memory
clear_memory()
+
return True
- def get_imatrix(self):
- ##TODO switch to blockwise way when oom
- logger.info("start to get imatrix for gguf quantization")
+ def quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
+ """Performs RTN quantization using input activation statistics (imatrix).
+
+ This method accumulates per-channel second-moment activation statistics (imatrix)
+ via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,
+ it falls back to CPU-based blockwise quantization.
+
+ Args:
+ all_to_quantized_module_names (list[str]):
+ A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.
+
+ Returns:
+ None
+ """
+ logger.info("start to compute imatrix for GGUF quantization.")
+
+ # Load dataset
from .calib_dataset import get_dataloader
if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "") ##remove all whitespaces
-
+ dataset_name = self.dataset.replace(" ", "")
self.dataloader = get_dataloader(
- self.tokenizer,
- self.seqlen,
- dataset,
- self.seed,
- self.batch_size,
- self.nsamples,
+ self.tokenizer, self.seqlen, dataset_name,
+ self.seed, self.batch_size, self.nsamples
)
else:
self.dataloader = self.dataset
+
model = self.model
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+
+ # Dispatch multi-GPU model if necessary
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
from accelerate.big_modeling import dispatch_model
dispatch_model(model, model.hf_device_map)
-
-
def register_act_hook(model):
+ """Registers hooks to accumulate activation squared norms into `imatrix`."""
+
def get_act_max_hook(module, input, output):
- if isinstance(input, (tuple, list)):
- input = input[0]
+ input = input[0] if isinstance(input, (tuple, list)) else input
+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
+ squared = torch.sum(flattened ** 2, dim=0).to(torch.float32)
+
if not hasattr(module, "imatrix"):
- module.imatrix = (torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(
- torch.float32)
+ module.imatrix = squared
+ module.imatrix_cnt = 1
else:
- module.imatrix = module.imatrix + (
- torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(torch.float32)
+ module.imatrix += squared
+ module.imatrix_cnt += 1
hook_handles = []
-
- for n, m in model.named_modules():
- if isinstance(m, self.supported_types) and check_to_quantized(m):
- hook = m.register_forward_hook(get_act_max_hook)
+ for name, module in model.named_modules():
+ if isinstance(module, self.supported_types) and check_to_quantized(module):
+ hook = module.register_forward_hook(get_act_max_hook)
hook_handles.append(hook)
return hook_handles
- hooks = register_act_hook(self.model)
+ hooks = register_act_hook(model)
try:
- model = self.model.to(self.device)
+ # Move model to target device
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 :
+ from accelerate.big_modeling import dispatch_model
+
+ dispatch_model(self.model, self.model.hf_device_map)
+ else:
+ model = model.to(self.device)
cnt = 0
+
+ # Run forward pass to accumulate imatrix
for data in self.dataloader:
cnt += data["input_ids"].shape[0]
- data = to_device(data, self.model.device)
+ data = to_device(data, self.device)
model(**data)
- if cnt > self.nsamples:
+ if cnt >= self.nsamples:
break
+
+ # Remove hooks after data collection
+ for hook in hooks:
+ hook.remove()
+
+ # Normalize imatrix by count
+ for _, module in model.named_modules():
+ if hasattr(module, "imatrix"):
+ module.imatrix /= module.imatrix_cnt
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+ accelerate.hooks.remove_hook_from_submodules(model)
+ # Perform quantization using RTN
+ from tqdm import tqdm
+ pbar = tqdm(all_to_quantized_module_names)
+ for name in pbar:
+ pbar.set_description(f"Quantizing {name}")
+ self.quantize_layer_via_rtn(name)
except RuntimeError as e:
try:
- model=self.model.to("cpu")
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+ accelerate.hooks.remove_hook_from_submodules(model)
+ # Fallback: out-of-memory → try CPU blockwise quantization
+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
+ model = model.to("cpu")
clear_memory()
- logger.warning("out of vram, fallback to blockwise way to get imatrix, this may affect accuracy")
- self.model_forward_blockwise()
- except Exception as e:
- model=self.model.to("cpu")
+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ except Exception:
+ # Final fallback: warn and use CPU-only quantization
+ logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
+ model = model.to("cpu")
clear_memory()
- logger.warning("fallback to cpu, pleaser use more gpus to get imatrix, by setting `--device 0,1,2,3`")
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model) ##self.model.hf_device_map has not been changed
- cnt = 0
- for data in self.dataloader:
- cnt += data["input_ids"].shape[0]
- data = to_device(data, self.model.device)
- model(**data)
- if cnt > self.nsamples:
- break
-
- for hook in hooks:
- hook.remove()
- for n, m in model.named_modules():
- if hasattr(m, "imatrix"):
- m.imatrix /= cnt
-
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+ accelerate.hooks.remove_hook_from_submodules(model)
+
+ orig_device = self.device
+ self.device = "cpu"
+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self.device = orig_device
+ finally:
+ # Always remove hooks
+ for hook in hooks:
+ hook.remove()
+
+ # Move back to CPU and free memory
model.to("cpu")
clear_memory()
- def check_need_to_quantize_lm_head_embedding(self):
+ def check_need_to_quantize_lm_head_embedding(self) -> bool:
+ """Checks if LM head and embedding layers need quantization for GGUF format.
+
+ This function inspects the current model's formats and determines whether
+ it needs to apply quantization settings to the embedding and LM head layers.
+ The function modifies `self.layer_config` in-place and updates the model modules.
+
+ Returns:
+ bool: True if the LM head needs quantization, otherwise False.
+
+ Raises:
+ NotImplementedError: If multiple non-fake GGUF formats are specified.
+ """
if not hasattr(self, "formats"):
return False
- has_gguf = False
- for format_ in self.formats:
- if "gguf" in format_:
- has_gguf = True
+
+ has_gguf: bool = any("gguf" in fmt for fmt in self.formats)
if not has_gguf:
return False
- formats = [f for f in self.formats if "fake" not in f]
+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]
if not (len(formats) == 1 and "gguf" in formats[0]):
raise NotImplementedError("Only one GGUF format can be set at a time.")
- target_gguf_format = formats[0]
- tie_word_embeddings = True
- if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):
- tie_word_embeddings = self.model.config.tie_word_embeddings
- for n, m in self.model.named_modules():
- if isinstance(m, torch.nn.Embedding):
- embedding_name = n
- key = "lm_head" if tie_word_embeddings else "embedding"
- config = GGUF_INNER_CONFIG[GGUF_CONFIG[target_gguf_format][key]]
- act_bits = 16
- scale_dtype = self.scale_dtype
- keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
- self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})
- for key in keys:
- if key in self.layer_config[embedding_name]:
- self.layer_config["fixed_by_user"] = True
- continue
- self.layer_config[embedding_name][key] = getattr(config, "get")(key)
- setattr(get_module(self.model, embedding_name), key, config.get(key))
- self.layer_config[embedding_name]["act_bits"] = act_bits
- self.layer_config[embedding_name]["scale_dtype"] = scale_dtype
- setattr(get_module(self.model, embedding_name), "act_bits", act_bits)
- setattr(get_module(self.model, embedding_name), "scale_dtype", scale_dtype)
- if not tie_word_embeddings:
- # TODO: If the user has set this explicitly, do not override
+ target_format: str = formats[0]
+ tie_word_embeddings: bool = getattr(
+ getattr(self.model, "config", None),
+ "tie_word_embeddings",
+ True
+ )
- lm_head_name = get_lm_head_name(self.model)
- config = GGUF_CONFIG[GGUF_CONFIG[formats[0]]["lm_head"]]
+ for name, module in self.model.named_modules():
+ if isinstance(module, torch.nn.Embedding):
+ key: str = "lm_head" if tie_word_embeddings else "embedding"
+ config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]
+ self._apply_config_to_layer(name, config,True)
- act_bits = 16
- scale_dtype = self.scale_dtype
- keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
- self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})
- for key in keys:
- if key in self.layer_config[lm_head_name] and self.layer_config[lm_head_name].get("fixed_by_user",
- False):
- continue
- self.layer_config[lm_head_name][key] = getattr(config, "get")(key)
- setattr(get_module(self.model, lm_head_name), key, config.get(key))
- self.layer_config[lm_head_name]["act_bits"] = act_bits
- self.layer_config[lm_head_name]["scale_dtype"] = scale_dtype
-
- setattr(get_module(self.model, lm_head_name), "act_bits", act_bits)
- setattr(get_module(self.model, lm_head_name), "scale_dtype", scale_dtype)
+ if not tie_word_embeddings:
+ lm_head_name: str = get_lm_head_name(self.model)
+ config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]
+ check_fixed_by_user = self.layer_config[lm_head_name].get("fixed_by_user", False)
+ self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)
return True
+
return False
- @torch.inference_mode
- def quantize_rtn(self):
+ def _apply_config_to_layer(
+ self,
+ layer_name: str,
+ config: dict[str, Any],
+ check_fixed_by_user: bool = False,
+ ) -> None:
+ """Applies GGUF quantization configuration to a given layer.
+
+ Args:
+ layer_name (str): Name of the layer to configure.
+ config (dict[str, Any]): GGUF layer configuration.
+ check_fixed_by_user (bool): If True, preserve user-defined settings.
+ """
+ act_bits: int = 16
+ scale_dtype: Any = self.scale_dtype
+ keys: list[str] = [
+ "bits", "group_size", "super_bits",
+ "super_group_size", "data_type", "sym"
+ ]
+
+ self.layer_config[layer_name] = self.layer_config.get(layer_name, {})
+
+ for key in keys:
+ if (
+ key in self.layer_config[layer_name]
+ and check_fixed_by_user
+ # and self.layer_config[layer_name].get("fixed_by_user", False)
+ ):
+ continue
+ self.layer_config[layer_name][key] = config.get(key)
+ setattr(get_module(self.model, layer_name), key, config.get(key))
+
+ self.layer_config[layer_name]["act_bits"] = act_bits
+ self.layer_config[layer_name]["scale_dtype"] = scale_dtype
+ setattr(get_module(self.model, layer_name), "act_bits", act_bits)
+ setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)
+
+ def quantize_layer_via_rtn(self, name: str) -> None:
+ """Quantizes a layer using RTN (Round-To-Nearest) if available.
+
+ This function attempts to quantize a layer by switching its data type to a
+ `rtn_*` version if supported, then wraps and unwraps the module to apply
+ quantization. If GPU memory is insufficient, it falls back to CPU.
+
+ If packing is enabled (`is_packing_immediate`), the function will also export
+ the quantized layer to the appropriate backend format.
+
+ Args:
+ name (str): Name of the layer to quantize.
+
+ Raises:
+ RuntimeError: If quantization fails for reasons unrelated to memory.
+ """
+ m = get_module(self.model, name)
+
+ # Step 1: Use optimized RTN data type if available
+ if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):
+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+ rtn_dtype = "rtn_" + m.data_type
+ if rtn_dtype in QUANT_FUNC_WITH_DTYPE:
+ m.data_type = rtn_dtype
+ self.layer_config[name]["data_type"] = m.data_type
+
+ # Step 2: Try quantization on GPU first, fall back to CPU if OOM
+ try:
+ m.to(self.device)
+ m = WrapperLinear(
+ m,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ )
+ m = m.unwrapper({})
+ m.to("cpu")
+ except RuntimeError as e:
+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ logger.warning("Out of VRAM, falling back to CPU.")
+ m.to("cpu")
+ m = WrapperLinear(
+ m,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ )
+ m = m.unwrapper({})
+ else:
+ raise
+
+ if self.low_gpu_mem_usage:
+ clear_memory()
+
+ # Step 3: Optional immediate packing/export
+ if self.is_packing_immediate:
+ from auto_round.export import PACKING_LAYER_WITH_FORMAT
+
+ if check_to_quantized(m):
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ has_gguf = any("gguf" in fmt for fmt in self.formats)
+
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+ output_dir = self.get_save_folder_name(self.formats[0])
+ pack_gguf_layer(
+ name, self.model, self.formats[0],
+ output_dir, self.layer_config, self.tokenizer
+ )
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](
+ name, self.model, self.formats[0]
+ )
+
+ if self.low_gpu_mem_usage:
+ clear_memory()
+ else:
+ set_module(self.model, name, m)
+
+ @torch.inference_mode()
+ def quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
+ """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
+
+ If the target format includes GGUF with `k`, and optimized RTN is enabled,
+ blockwise quantization with input caching and imatrix is used.
+
+ Returns:
+ tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.
+ """
if self.amp:
self.model.to(self.amp_dtype)
self.model.to("cpu")
- all_to_quantized_module_names = []
- for n, m in self.model.named_modules():
- if check_to_quantized(m):
- all_to_quantized_module_names.append(n)
- if hasattr(self, "formats"):
- has_gguf_k = False
- for format_ in self.formats:
- if "gguf" in format_ and "k" in format_:
- has_gguf_k = True
- if has_gguf_k and not self.disable_opt_rtn:
- self.get_imatrix()
- self.quantize_embedding_layer()
- pbar = tqdm(all_to_quantized_module_names)
-
- for name in pbar:
- pbar.set_description(f"Quantizing {name}")
- m = get_module(self.model, name)
- if not self.disable_opt_rtn and not (m.data_type.startswith("rtn_")): ## use rtn version first
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
- if "rtn_" + m.data_type in QUANT_FUNC_WITH_DTYPE:
- m.data_type = "rtn_" + m.data_type
- self.layer_config[name]["data_type"] = m.data_type
- try:
- m.to(self.device)
- m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,
- enable_round_tuning=False)
- m = m.unwrapper({})
- m.to("cpu")
- except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
- logger.warning("out of vram, fallback to cpu")
- m.to("cpu")
- m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,
- enable_round_tuning=False)
- m = m.unwrapper({})
- else:
- raise
- if self.low_gpu_mem_usage:
- clear_memory()
- if self.is_packing_immediate:
- from auto_round.export import PACKING_LAYER_WITH_FORMAT
- if check_to_quantized(m):
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0])
- if self.low_gpu_mem_usage:
- clear_memory()
- else:
- set_module(self.model, name, m)
+ all_to_quantized_module_names: list[str] = [
+ n for n, m in self.model.named_modules() if check_to_quantized(m)
+ ]
+
+ has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))
+
+ self.quantize_embedding_layer()
+
+ if has_gguf_k and not self.disable_opt_rtn:
+ self.quant_rtn_with_imatrix(all_to_quantized_module_names)
+ else:
+ pbar = tqdm(all_to_quantized_module_names)
+ for name in pbar:
+ pbar.set_description(f"Quantizing {name}")
+ self.quantize_layer_via_rtn(name)
self.quantized = True
return self.model, self.layer_config
- def model_forward_blockwise(self):
- if bool(self.quant_block_list):
- all_blocks = self.quant_block_list
- else:
- all_blocks = get_block_names(self.model)
- if len(all_blocks) == 0:
- raise ValueError("could not find any blocks, please check the model or quant_block_list")
+ def quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
+ """Quantize model layers block by block using cached inputs and imatrix.
+
+ Args:
+ all_to_quantized_module_names (list[str]): Names of layers to be quantized.
+ """
+ all_to_quantized_module_names = list(set(all_to_quantized_module_names))
+
+ all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)
+ if not all_blocks:
+ raise ValueError("Could not find any blocks. Check the model or quant_block_list.")
- ##ignore layers
all_first_block_names = [block[0] for block in all_blocks]
- all_inputs = self.cache_inter_data(
- all_first_block_names, self.nsamples)
+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)
+ # Clear hooks for multi-GPU setups
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+
+ pbar = tqdm(range(sum(len(block) for block in all_blocks)))
for block_names in all_blocks:
- inputs = all_inputs[block_names[0]]
- all_inputs.pop(block_names[0])
- keys = inputs.keys()
- input_id_str = [key for key in keys if key.startswith('hidden_state')]
- if len(input_id_str) != 1:
- raise RuntimeError(f"hidden_states arg mismatch error,"
- "please raise an issue in https://github.com/intel/auto-round/issues")
- inputs["input_ids"] = inputs.pop(input_id_str[0], None)
+ first_block = block_names[0]
+ inputs = all_inputs.pop(first_block)
+ input_keys = [k for k in inputs if k.startswith("hidden_state")]
+ if len(input_keys) != 1:
+ raise RuntimeError(
+ "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"
+ )
+ inputs["input_ids"] = inputs.pop(input_keys[0])
clear_memory(self.inputs)
- if "input_ids" in inputs.keys():
- total_samples = len(inputs["input_ids"])
- if total_samples < self.batch_size:
- self.batch_size = total_samples
- logger.warning(f"force the train batch size to {total_samples}")
+ total_samples = len(inputs["input_ids"])
+ if total_samples < self.batch_size:
+ self.batch_size = total_samples
+ logger.warning(f"Forcing batch size to {total_samples}")
- input_ids = inputs["input_ids"]
- inputs.pop("input_ids", None)
- for block_name in block_names:
- input_others = inputs
- clear_memory()
- input_ids = to_device(input_ids, self.cache_device)
- input_others = to_device(input_others, self.cache_device)
- ## as in the calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
- for i in range(len(input_ids)):
- input_ids[i] = input_ids[i].to(tmp_dtype)
-
- for key in input_others.keys():
- if isinstance(input_others[key], torch.Tensor) and (
- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
- ):
- input_others[key] = input_others[key].to(tmp_dtype)
- elif isinstance(input_others[key], list):
- for i in range(len(input_others[key])):
- to_dtype(input_others[key][i], tmp_dtype)
+ input_ids = to_device(inputs.pop("input_ids"), self.cache_device)
+ input_others = to_device(inputs, self.cache_device)
+
+ tmp_dtype = self.amp_dtype if self.amp else torch.float32
+ input_ids = [id_.to(tmp_dtype) for id_ in input_ids]
+ for key, val in input_others.items():
+ if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):
+ input_others[key] = val.to(tmp_dtype)
+ elif isinstance(val, list):
+ input_others[key] = [to_dtype(v, tmp_dtype) for v in val]
+
+ for block_name in block_names:
+ pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
+ # Dispatch model if needed
if self.device_map is not None:
from accelerate import dispatch_model
- for n, m in block.named_modules():
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ for _, m in block.named_modules():
if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
continue
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
add_hook_to_module(m, hook, True)
- input_ids = self.get_block_outputs(block, input_ids, input_others,
- self.batch_size * self.infer_bs_coeff,
- self.device,
- self.cache_device)
+ input_ids = self.get_block_outputs(
+ block,
+ input_ids,
+ input_others,
+ self.batch_size * self.infer_bs_coeff,
+ self.device,
+ self.cache_device,
+ )
+
+ accelerate.hooks.remove_hook_from_submodules(block)
+
+ # Normalize imatrix and quantize layers
+ for _, m in block.named_modules():
+ if hasattr(m, "imatrix"):
+ m.imatrix /= m.imatrix_cnt
+ if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
+ self.quantize_layer_via_rtn(m.tmp_name)
+ all_to_quantized_module_names.remove(m.tmp_name)
- if self.device_map is not None:
- accelerate.hooks.remove_hook_from_submodules(
- block)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ pbar.update(1)
+
+ pbar.close()
+
+ # Process remaining layers not in blocks
+ for name in all_to_quantized_module_names:
+ self.quantize_layer_via_rtn(name)
+
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
Returns:
The quantized model and layer configurations.
"""
+ for n, m in self.model.named_modules():
+ m.tmp_name = n
self._check_compatibility()
self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)
if not hasattr(self, "formats"):
@@ -974,8 +1149,8 @@ class AutoRound(object):
# Determine if immediate packing is required
formats = self.formats
if (len(formats) == 1 and
- ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0]) and
- not self.has_qlayer_outside_block and self.inplace): # TODO: Support more formats
+ ("awq" in formats[0] or "gptq" in formats[0] or
+ "auto_round" in formats[0] or "gguf" in formats[0]) and self.inplace):
self.is_packing_immediate = True
if self.iters == 0:
return self.quantize_rtn()
@@ -1086,7 +1261,7 @@ class AutoRound(object):
"""Quantizes specified layers based on inputs and configuration.
Args:
- layer_names (list): List of layer names to quantize.
+ layer_names (list): list of layer names to quantize.
layer_inputs (dict): Dictionary mapping layer names to input data.
Returns:
@@ -1115,6 +1290,12 @@ class AutoRound(object):
return
q_layer_inputs = None
enable_quanted_input = self.enable_quanted_input
+ has_gguf = False
+ if hasattr(self, "formats"):
+ has_gguf = any("gguf" in format_ for format_ in self.formats)
+ if has_gguf and self.is_packing_immediate:
+ enable_quanted_input = False
+
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
from accelerate.big_modeling import dispatch_model
@@ -1138,7 +1319,7 @@ class AutoRound(object):
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
- q_layer_input = q_layer_inputs[layer_name] if enable_quanted_input else None
+ q_layer_input = q_layer_inputs[layer_name] if q_layer_inputs is not None else None
q_layer_input = to_device(q_layer_input, self.cache_device)
quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
del layer_input
@@ -2070,7 +2251,14 @@ class AutoRound(object):
for _, tmp_m in m.named_modules():
if hasattr(tmp_m, "bits") and check_to_quantized(tmp_m):
target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])
+ has_gguf = any("gguf" in format_ for format_ in self.formats)
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+ output_dir = self.get_save_folder_name(self.formats[0])
+ pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,
+ self.tokenizer)
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])
pbar.set_description(f"Quantizing done")
pbar.update(1)
pbar.close()
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index 845544017..d1012123e 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -50,7 +50,7 @@ def _save_quantized_as_autoawq(*args, **kwargs):
return save_quantized_as_autoawq(*args, **kwargs)
@register_format("gguf")
-def _save_quantized_as_autoawq(*args, **kwargs):
+def _save_quantized_as_gguf(*args, **kwargs):
from auto_round.export.export_to_gguf.export import save_quantized_as_gguf
return save_quantized_as_gguf(*args, **kwargs)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 1f5f7fc66..27ea57aba 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -35,6 +35,7 @@
from __future__ import annotations
import ast
+import gc
import logging
import argparse
import contextlib
@@ -56,6 +57,7 @@ import torch
from auto_round.utils import logger, LazyImport, clear_memory, get_module, clean_module_parameter
from auto_round.export.export_to_gguf.packing import ggml_quant_gpu
+
gguf = LazyImport("gguf")
if TYPE_CHECKING:
@@ -318,8 +320,10 @@ class OriModel:
logger.info(f"gguf: experts used count = {n_experts_used}")
if (head_dim := self.hparams.get("head_dim")) is not None:
- self.gguf_writer.add_key_length(head_dim)
- self.gguf_writer.add_value_length(head_dim)
+ # Workaround for incorrect AutoConfig value for DeepSeekV3 (is set correctly in DeepSeekV2Model class)
+ if self.hparams.get("model_type") != "deepseek_v3":
+ self.gguf_writer.add_key_length(head_dim)
+ self.gguf_writer.add_value_length(head_dim)
self.gguf_writer.add_file_type(self.ftype)
logger.info(f"gguf: file type = {self.ftype}")
@@ -1137,14 +1141,14 @@ class Model(OriModel):
if not low_cpu_mem_usage:
return False
- process = psutil.Process(os.getpid())
- mem_usage = process.memory_info().rss
- memory_info = psutil.virtual_memory()
- if memory_info.available > mem_usage / 3:
- return False
- else:
- logger.info("use low cpu memory mode.")
- return True
+ # process = psutil.Process(os.getpid())
+ # mem_usage = process.memory_info().rss
+ # memory_info = psutil.virtual_memory()
+ # if memory_info.available > mem_usage / 3:
+ # return False
+ # else:
+ # logger.info("use low cpu memory mode.")
+ return True
def get_moe_name(self, name, new_name):
type_mapping = {
@@ -1323,9 +1327,17 @@ class Model(OriModel):
max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):
+ if data_torch is None:
+ continue
# we don't need these
if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
continue
+ if hasattr(self,"current_packing_block") and self.current_packing_block is not None: # pylint: disable=E1101
+ current_packing_block_split = self.current_packing_block.split('.') # pylint: disable=E1101
+ name_split = name.split('.')
+ if (len(name_split)<len(current_packing_block_split) or
+ name_split[:len(current_packing_block_split)]!=current_packing_block_split):
+ continue
old_dtype = data_torch.dtype
@@ -1339,7 +1351,8 @@ class Model(OriModel):
if part.isdecimal():
bid = int(part)
break
-
+
+ clean_weight_list = []
for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
data = data_torch.squeeze().cpu().numpy()
@@ -1383,7 +1396,7 @@ class Model(OriModel):
# get name by new_name (for experts),
name = self.get_moe_name(name, new_name)
-
+ clean_weight_list.append(name)
# get data_qtype by layer_config
data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)
@@ -1491,11 +1504,11 @@ class Model(OriModel):
self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)
- # save cpu memory, but slow
+ # # save cpu memory, but slow
if self.low_cpu_mem_usage:
- module = get_module(self.model, ".".join(name.split(".")[:-1]))
- clean_module_parameter(module, name.split(".")[-1])
- clear_memory()
+ for weight_name in clean_weight_list:
+ module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))
+ clean_module_parameter(module, weight_name.split(".")[-1])
@Model.register("GPTNeoXForCausalLM")
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index db8ec381e..a950c7b11 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -16,51 +16,38 @@ import os
import sys
import shutil
import torch
-from .convert import Model
-from auto_round.utils import logger, LazyImport
from pathlib import Path
import time
+from auto_round.export.export_to_gguf.convert import Model
+from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module
gguf = LazyImport("gguf")
FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
- "f32": gguf.LlamaFileType.ALL_F32,
- "f16": gguf.LlamaFileType.MOSTLY_F16,
- "bf16": gguf.LlamaFileType.MOSTLY_BF16,
- "q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,
- "q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,
- "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,
- "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,
- "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
- "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
- "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,
- "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,
- "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,
- "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
- "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,
- "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,
- "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,
- "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,
- "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,
- "auto": gguf.LlamaFileType.GUESSED,
- }
-
-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, low_cpu_mem_usage=False, **kwargs):
- """Export the model to gguf format."""
- if output_dir is not None and os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ "f32": gguf.LlamaFileType.ALL_F32,
+ "f16": gguf.LlamaFileType.MOSTLY_F16,
+ "bf16": gguf.LlamaFileType.MOSTLY_BF16,
+ "q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,
+ "q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,
+ "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,
+ "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,
+ "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,
+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,
+ "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,
+ "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,
+ "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,
+ "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,
+ "auto": gguf.LlamaFileType.GUESSED,
+}
- st = time.time()
-
- model = kwargs["model"]
- tokenizer = kwargs.get("tokenizer", None)
- config = model.config
+def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False):
tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
- if tokenizer is not None:
- tokenizer.save_pretrained(tmp_work_dir)
- config.save_pretrained(tmp_work_dir)
-
with torch.inference_mode():
hparams = Model.load_hparams(tmp_work_dir)
model_architecture = hparams["architectures"][0]
@@ -81,26 +68,102 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, l
raise TypeError(f"{output_type} type is not supported")
output_type = FTYPE_MAP.get(output_type.lower())
-
model_instance = model_class(
model,
layer_config,
dir_model=tmp_work_dir,
ftype=output_type,
fname_out=Path(output_dir),
- low_cpu_mem_usage=low_cpu_mem_usage,
+ low_cpu_mem_usage=low_cpu_mem_usage, # pylint: disable=E0401
is_big_endian=False,
model_name=model_name,
split_max_tensors=False,
split_max_size=0,
dry_run=False,
- small_first_shard=False,
- )
- model_instance.write()
- rt = time.time() - st
- logger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")
+ small_first_shard=False)
+ return model_instance
+
+
+def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):
+ """Export the model to gguf format."""
+ global gguf_model_instance_global
+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):
+ logger.warning_once(f"{output_dir} already exists, this may cause model conflict")
+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ if "gguf_model_instance_global" not in globals():
+ config = model.config
+ if tokenizer is not None:
+ tokenizer.save_pretrained(tmp_work_dir)
+ config.save_pretrained(tmp_work_dir)
+
+ gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend,
+ low_cpu_mem_usage=True)
+
+ if not hasattr(model, "last_layer_name_to_block_name"):
+ block_name_to_last_layer_name = {}
+ block_names = get_block_names(model, quant_vision=True)
+ block_names_flatten = flatten_list(block_names)
+ for n, m in model.named_modules():
+ if not check_to_quantized(m):
+ continue
+ for block_name in block_names_flatten:
+ block_name_split = block_name.split('.')
+ name_split = n.split('.')
+ if (len(name_split) < len(block_name_split) or
+ name_split[:len(block_name_split)] != block_name_split):
+ continue
+ block_name_to_last_layer_name[block_name] = n
+ last_layer_name_to_block_name = {v: k for k, v in block_name_to_last_layer_name.items()}
+ model.last_layer_name_to_block_name = last_layer_name_to_block_name
+ if name in model.last_layer_name_to_block_name:
+ ##packing block
+ gguf_model_instance_global.current_packing_block = model.last_layer_name_to_block_name[name]
+ gguf_model_instance_global.prepare_tensors()
+ block = get_module(model, model.last_layer_name_to_block_name[name])
+ for n, m in block.named_modules():
+ if hasattr(m, "weight"):
+ m.weight = None
+ if hasattr(m, "bias"):
+ m.bias = None
+ import gc
+ gc.collect()
+ model.last_layer_name_to_block_name.pop(name)
+ if len(model.last_layer_name_to_block_name) == 0:
+ gguf_model_instance_global.current_packing_block = None
+
+
+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):
+ """Export the model to gguf format."""
+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+
+ st = time.time()
+ global gguf_model_instance_global
+
+ model = kwargs["model"]
+ if "gguf_model_instance_global" not in globals():
+ tokenizer = kwargs.get("tokenizer", None)
+ config = model.config
+ if tokenizer is not None:
+ tokenizer.save_pretrained(tmp_work_dir)
+ config.save_pretrained(tmp_work_dir)
+
+ config = model.config
+ if tokenizer is not None:
+ tokenizer.save_pretrained(tmp_work_dir)
+ config.save_pretrained(tmp_work_dir)
+
+ gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend)
+
+ gguf_model_instance_global.write()
+ rt = time.time() - st
+ logger.info(f"Model successfully exported to {gguf_model_instance_global.fname_out}, running time={rt}")
+ del gguf_model_instance_global
shutil.rmtree(tmp_work_dir, ignore_errors=True)
return model
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index d4b92508e..ce2345161 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -782,9 +782,6 @@ def get_layer_names_in_block(model, supported_types=(torch.nn.Linear,
for n, m in block.named_modules():
if hasattr(m, "tmp_name"):
layers_in_block.append(m.tmp_name)
- for n, m in model.named_modules():
- if hasattr(m, "tmp_name"):
- delattr(m, "tmp_name")
return layers_in_block
@@ -1592,7 +1589,7 @@ def _search_gguf_type(gguf_type):
if not bits:
raise KeyError(f"{gguf_type} is not a correct gguf type, please check")
- for suffix in ["_0", "_1", "_k"]:
+ for suffix in ["_k", "_0", "_1"]:
if gguf_type.endswith(suffix):
continue
if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:
@@ -1894,6 +1891,14 @@ def get_gguf_qtype_by_layer_config(layer_config):
return gguf.GGMLQuantizationType.Q8_0
raise ValueError(f"Unknown layer config")
+def flatten_list(nested_list):
+ flattened = []
+ for item in nested_list:
+ if isinstance(item, (list, tuple)):
+ flattened.extend(flatten_list(item))
+ else:
+ flattened.append(item)
+ return flattened
def clean_module_parameter(submodule, parameter):
is_buffer = parameter in submodule._buffers
@@ -1901,5 +1906,4 @@ def clean_module_parameter(submodule, parameter):
if is_buffer:
submodule._buffers[parameter] = None
else:
- submodule._parameters[parameter] = None
- gc.collect()
\ No newline at end of file
+ submodule._parameters[parameter] = None
\ No newline at end of file
diff --git a/auto_round/version.py b/auto_round/version.py
index 41f5016dc..d9f21f781 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -14,4 +14,4 @@
"""Intel® auto-round: An open-source Python library
supporting popular model weight only compression based on signround."""
-__version__ = "0.5.1"
+__version__ = "0.6.0"