PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support packing immediately for gguf to reduce ram usage (#638)
Wenhua Cheng3b282dc9dd86
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4d9b299a2..e1fa1d238 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -19,7 +19,7 @@ import sysimport torchimport copyimport time-from typing import Optional, Union, List+from typing import Union, Anyfrom transformers import set_seedfrom torch import autocastfrom tqdm import tqdm@@ -56,9 +56,10 @@ from auto_round.utils import (init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,_gguf_args_check,check_seqlen_compatible,- get_layer_config_by_gguf_format, get_lm_head_name+ get_layer_config_by_gguf_format, get_lm_head_name, flatten_list)-from .low_cpu_mem.utils import get_layers_before_block+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.low_cpu_mem.utils import get_layers_before_blockclass AutoRound(object):@@ -446,12 +447,8 @@ class AutoRound(object):f" than gradient_accumulate_steps * batch_size")self._dq_check()- # def _check_format_compatibility(self, format): ##TODO- # ##check lm_head, mixed_bits, bits, each layer supporting, etc- # passdef _check_compatibility(self):- # Check the legitimacy of seqlen##check gguf and othershas_gguf = False@@ -525,6 +522,7 @@ class AutoRound(object):ValueError: If an unsupported format is specified."""# Validate and process the specified formats+ self.orig_output_dir = output_dir_gguf_args_check(self, format)formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')@@ -595,15 +593,11 @@ class AutoRound(object):formats = remove_duplicates(formats)self.formats = formats- # # Check format compatibility- # self._check_format_compatibility(formats)-# Perform model quantizationmodel, _ = self.quantize()# Save the quantized model in the specified formatsfolders = []- low_cpu_mem_usage = len(formats) == 1 and "gguf" in formats[0]for format in formats:if "gptq" in format and not self.sym:logger.warning(@@ -612,350 +606,531 @@ class AutoRound(object):" We recommend exporting to either the AutoAWQ format ( only 4 bits) or ""the AutoRound format(2/3/4/8 bits).")- save_format_ = format.replace(":", "-").replace("_", "-")- save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir- self.save_quantized(- save_folder, format=format, inplace=inplace, low_cpu_mem_usage=low_cpu_mem_usage, **kwargs)+ save_folder = self.get_save_folder_name(format)+ self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)folders.append(save_folder)return model, folders- @torch.inference_mode+ def get_save_folder_name(self, format_str: str) -> str:+ """Generates the save folder name based on the provided format string.++ If there are multiple formats to handle, the function creates a subfolder+ named after the format string with special characters replaced. If there's+ only one format, it returns the original output directory directly.++ Args:+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').++ Returns:+ str: The path to the folder where results should be saved.+ """+ # Replace special characters to make the folder name filesystem-safe+ sanitized_format = format_str.replace(":", "-").replace("_", "-")++ # Use a subfolder only if there are multiple formats+ if len(self.formats) > 1:+ return os.path.join(self.orig_output_dir, sanitized_format)++ return self.orig_output_dir++ @torch.inference_mode()def quantize_embedding_layer(self):- for n, m in self.model.named_modules():- if not isinstance(m, torch.nn.Embedding) or n not in self.layer_config:+ """Quantizes embedding layers in the model according to the configuration.++ This method iterates through all modules in the model, identifies embedding+ layers specified in `self.layer_config`, and applies the appropriate quantization+ function based on bit precision, grouping strategy, and dtype.++ Returns:+ bool: True if the quantization process completes without critical errors.+ """+ for name, module in self.model.named_modules():+ # Skip non-Embedding modules or layers not in config+ if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:continue- config = self.layer_config[n]+ config = self.layer_config[name]++ # Skip layers that are not marked for quantizationif not check_to_quantized(config):continue- layer = m- layer_name = nconfig["scale_dtype"] = self.scale_dtype-- from auto_round.data_type import QUANT_FUNC_WITH_DTYPEdtype = config["data_type"]++ # Determine quantization function key with symmetry/asymmetryif dtype not in QUANT_FUNC_WITH_DTYPE:- if config['sym']:- dtype = dtype + "_sym"- else:- dtype = dtype + "_asym"+ dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"++ # Optionally use optimized rounding (RTN) variant+ if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:+ dtype = f"rtn_{dtype}"- if not self.disable_opt_rtn and "rtn_" + dtype in QUANT_FUNC_WITH_DTYPE:- dtype = "rtn_" + dtypequant_func = QUANT_FUNC_WITH_DTYPE[dtype]+ # Attempt quantization on GPU, fall back to CPU if OOMtry:- weight, scale, zp = quant_func(layer.weight.to(self.device), **{k: config[k] for k in- ["bits", "group_size", "super_bits",- "super_group_size", "scale_dtype"]})+ weight, scale, zp = quant_func(+ module.weight.to(self.device),+ **{k: config[k] for k in [+ "bits", "group_size", "super_bits",+ "super_group_size", "scale_dtype"+ ]}+ )except RuntimeError as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):- logger.info("out of vram, fallback to cpu")- weight, scale, zp = quant_func(layer.weight.to("cpu"), **{k: config[k] for k in- ["bits", "group_size", "super_bits",- "super_group_size", "scale_dtype"]})+ logger.info("out of VRAM, falling back to CPU.")+ weight, scale, zp = quant_func(+ module.weight.to("cpu"),+ **{k: config[k] for k in [+ "bits", "group_size", "super_bits",+ "super_group_size", "scale_dtype"+ ]}+ )else:raise- layer.weight.data.copy_(weight.cpu())- for name, val in zip(["scale", "zp"], [scale, zp]):- if isinstance(val, dict):- for k, v in val.items():- setattr(layer, k if k == "scale" else "w_" + k, v.cpu())+ # Overwrite the module's weights with the quantized version+ module.weight.data.copy_(weight.cpu())++ # Attach scale and zero point (zp) to the module+ for param_name, value in zip(["scale", "zp"], [scale, zp]):+ if isinstance(value, dict):+ for k, v in value.items():+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())else:- setattr(layer, name, val.cpu())+ setattr(module, param_name, value.cpu())- self.layer_config.setdefault(layer_name, {}).update(config)+ # Update config+ self.layer_config.setdefault(name, {}).update(config)++ # Release memoryclear_memory()+return True- def get_imatrix(self):- ##TODO switch to blockwise way when oom- logger.info("start to get imatrix for gguf quantization")+ def quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:+ """Performs RTN quantization using input activation statistics (imatrix).++ This method accumulates per-channel second-moment activation statistics (imatrix)+ via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,+ it falls back to CPU-based blockwise quantization.++ Args:+ all_to_quantized_module_names (list[str]):+ A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.++ Returns:+ None+ """+ logger.info("start to compute imatrix for GGUF quantization.")++ # Load datasetfrom .calib_dataset import get_dataloaderif isinstance(self.dataset, str):- dataset = self.dataset.replace(" ", "") ##remove all whitespaces-+ dataset_name = self.dataset.replace(" ", "")self.dataloader = get_dataloader(- self.tokenizer,- self.seqlen,- dataset,- self.seed,- self.batch_size,- self.nsamples,+ self.tokenizer, self.seqlen, dataset_name,+ self.seed, self.batch_size, self.nsamples)else:self.dataloader = self.dataset+model = self.model- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:++ # Dispatch multi-GPU model if necessary+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_modeldispatch_model(model, model.hf_device_map)--def register_act_hook(model):+ """Registers hooks to accumulate activation squared norms into `imatrix`."""+def get_act_max_hook(module, input, output):- if isinstance(input, (tuple, list)):- input = input[0]+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(flattened ** 2, dim=0).to(torch.float32)+if not hasattr(module, "imatrix"):- module.imatrix = (torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(- torch.float32)+ module.imatrix = squared+ module.imatrix_cnt = 1else:- module.imatrix = module.imatrix + (- torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(torch.float32)+ module.imatrix += squared+ module.imatrix_cnt += 1hook_handles = []-- for n, m in model.named_modules():- if isinstance(m, self.supported_types) and check_to_quantized(m):- hook = m.register_forward_hook(get_act_max_hook)+ for name, module in model.named_modules():+ if isinstance(module, self.supported_types) and check_to_quantized(module):+ hook = module.register_forward_hook(get_act_max_hook)hook_handles.append(hook)return hook_handles- hooks = register_act_hook(self.model)+ hooks = register_act_hook(model)try:- model = self.model.to(self.device)+ # Move model to target device+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 :+ from accelerate.big_modeling import dispatch_model++ dispatch_model(self.model, self.model.hf_device_map)+ else:+ model = model.to(self.device)cnt = 0++ # Run forward pass to accumulate imatrixfor data in self.dataloader:cnt += data["input_ids"].shape[0]- data = to_device(data, self.model.device)+ data = to_device(data, self.device)model(**data)- if cnt > self.nsamples:+ if cnt >= self.nsamples:break++ # Remove hooks after data collection+ for hook in hooks:+ hook.remove()++ # Normalize imatrix by count+ for _, module in model.named_modules():+ if hasattr(module, "imatrix"):+ module.imatrix /= module.imatrix_cnt+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)+ # Perform quantization using RTN+ from tqdm import tqdm+ pbar = tqdm(all_to_quantized_module_names)+ for name in pbar:+ pbar.set_description(f"Quantizing {name}")+ self.quantize_layer_via_rtn(name)except RuntimeError as e:try:- model=self.model.to("cpu")+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)+ # Fallback: out-of-memory → try CPU blockwise quantization+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")+ model = model.to("cpu")clear_memory()- logger.warning("out of vram, fallback to blockwise way to get imatrix, this may affect accuracy")- self.model_forward_blockwise()- except Exception as e:- model=self.model.to("cpu")+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ except Exception:+ # Final fallback: warn and use CPU-only quantization+ logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")+ model = model.to("cpu")clear_memory()- logger.warning("fallback to cpu, pleaser use more gpus to get imatrix, by setting `--device 0,1,2,3`")- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(- self.model) ##self.model.hf_device_map has not been changed- cnt = 0- for data in self.dataloader:- cnt += data["input_ids"].shape[0]- data = to_device(data, self.model.device)- model(**data)- if cnt > self.nsamples:- break-- for hook in hooks:- hook.remove()- for n, m in model.named_modules():- if hasattr(m, "imatrix"):- m.imatrix /= cnt-+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)++ orig_device = self.device+ self.device = "cpu"+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self.device = orig_device+ finally:+ # Always remove hooks+ for hook in hooks:+ hook.remove()++ # Move back to CPU and free memorymodel.to("cpu")clear_memory()- def check_need_to_quantize_lm_head_embedding(self):+ def check_need_to_quantize_lm_head_embedding(self) -> bool:+ """Checks if LM head and embedding layers need quantization for GGUF format.++ This function inspects the current model's formats and determines whether+ it needs to apply quantization settings to the embedding and LM head layers.+ The function modifies `self.layer_config` in-place and updates the model modules.++ Returns:+ bool: True if the LM head needs quantization, otherwise False.++ Raises:+ NotImplementedError: If multiple non-fake GGUF formats are specified.+ """if not hasattr(self, "formats"):return False- has_gguf = False- for format_ in self.formats:- if "gguf" in format_:- has_gguf = True++ has_gguf: bool = any("gguf" in fmt for fmt in self.formats)if not has_gguf:return False- formats = [f for f in self.formats if "fake" not in f]+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]if not (len(formats) == 1 and "gguf" in formats[0]):raise NotImplementedError("Only one GGUF format can be set at a time.")- target_gguf_format = formats[0]- tie_word_embeddings = True- if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):- tie_word_embeddings = self.model.config.tie_word_embeddings- for n, m in self.model.named_modules():- if isinstance(m, torch.nn.Embedding):- embedding_name = n- key = "lm_head" if tie_word_embeddings else "embedding"- config = GGUF_INNER_CONFIG[GGUF_CONFIG[target_gguf_format][key]]- act_bits = 16- scale_dtype = self.scale_dtype- keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]- self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})- for key in keys:- if key in self.layer_config[embedding_name]:- self.layer_config["fixed_by_user"] = True- continue- self.layer_config[embedding_name][key] = getattr(config, "get")(key)- setattr(get_module(self.model, embedding_name), key, config.get(key))- self.layer_config[embedding_name]["act_bits"] = act_bits- self.layer_config[embedding_name]["scale_dtype"] = scale_dtype- setattr(get_module(self.model, embedding_name), "act_bits", act_bits)- setattr(get_module(self.model, embedding_name), "scale_dtype", scale_dtype)- if not tie_word_embeddings:- # TODO: If the user has set this explicitly, do not override+ target_format: str = formats[0]+ tie_word_embeddings: bool = getattr(+ getattr(self.model, "config", None),+ "tie_word_embeddings",+ True+ )- lm_head_name = get_lm_head_name(self.model)- config = GGUF_CONFIG[GGUF_CONFIG[formats[0]]["lm_head"]]+ for name, module in self.model.named_modules():+ if isinstance(module, torch.nn.Embedding):+ key: str = "lm_head" if tie_word_embeddings else "embedding"+ config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]+ self._apply_config_to_layer(name, config,True)- act_bits = 16- scale_dtype = self.scale_dtype- keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]- self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})- for key in keys:- if key in self.layer_config[lm_head_name] and self.layer_config[lm_head_name].get("fixed_by_user",- False):- continue- self.layer_config[lm_head_name][key] = getattr(config, "get")(key)- setattr(get_module(self.model, lm_head_name), key, config.get(key))- self.layer_config[lm_head_name]["act_bits"] = act_bits- self.layer_config[lm_head_name]["scale_dtype"] = scale_dtype-- setattr(get_module(self.model, lm_head_name), "act_bits", act_bits)- setattr(get_module(self.model, lm_head_name), "scale_dtype", scale_dtype)+ if not tie_word_embeddings:+ lm_head_name: str = get_lm_head_name(self.model)+ config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]+ check_fixed_by_user = self.layer_config[lm_head_name].get("fixed_by_user", False)+ self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)return True+return False- @torch.inference_mode- def quantize_rtn(self):+ def _apply_config_to_layer(+ self,+ layer_name: str,+ config: dict[str, Any],+ check_fixed_by_user: bool = False,+ ) -> None:+ """Applies GGUF quantization configuration to a given layer.++ Args:+ layer_name (str): Name of the layer to configure.+ config (dict[str, Any]): GGUF layer configuration.+ check_fixed_by_user (bool): If True, preserve user-defined settings.+ """+ act_bits: int = 16+ scale_dtype: Any = self.scale_dtype+ keys: list[str] = [+ "bits", "group_size", "super_bits",+ "super_group_size", "data_type", "sym"+ ]++ self.layer_config[layer_name] = self.layer_config.get(layer_name, {})++ for key in keys:+ if (+ key in self.layer_config[layer_name]+ and check_fixed_by_user+ # and self.layer_config[layer_name].get("fixed_by_user", False)+ ):+ continue+ self.layer_config[layer_name][key] = config.get(key)+ setattr(get_module(self.model, layer_name), key, config.get(key))++ self.layer_config[layer_name]["act_bits"] = act_bits+ self.layer_config[layer_name]["scale_dtype"] = scale_dtype+ setattr(get_module(self.model, layer_name), "act_bits", act_bits)+ setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)++ def quantize_layer_via_rtn(self, name: str) -> None:+ """Quantizes a layer using RTN (Round-To-Nearest) if available.++ This function attempts to quantize a layer by switching its data type to a+ `rtn_*` version if supported, then wraps and unwraps the module to apply+ quantization. If GPU memory is insufficient, it falls back to CPU.++ If packing is enabled (`is_packing_immediate`), the function will also export+ the quantized layer to the appropriate backend format.++ Args:+ name (str): Name of the layer to quantize.++ Raises:+ RuntimeError: If quantization fails for reasons unrelated to memory.+ """+ m = get_module(self.model, name)++ # Step 1: Use optimized RTN data type if available+ if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+ rtn_dtype = "rtn_" + m.data_type+ if rtn_dtype in QUANT_FUNC_WITH_DTYPE:+ m.data_type = rtn_dtype+ self.layer_config[name]["data_type"] = m.data_type++ # Step 2: Try quantization on GPU first, fall back to CPU if OOM+ try:+ m.to(self.device)+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ )+ m = m.unwrapper({})+ m.to("cpu")+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ logger.warning("Out of VRAM, falling back to CPU.")+ m.to("cpu")+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ )+ m = m.unwrapper({})+ else:+ raise++ if self.low_gpu_mem_usage:+ clear_memory()++ # Step 3: Optional immediate packing/export+ if self.is_packing_immediate:+ from auto_round.export import PACKING_LAYER_WITH_FORMAT++ if check_to_quantized(m):+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in fmt for fmt in self.formats)++ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer+ output_dir = self.get_save_folder_name(self.formats[0])+ pack_gguf_layer(+ name, self.model, self.formats[0],+ output_dir, self.layer_config, self.tokenizer+ )+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](+ name, self.model, self.formats[0]+ )++ if self.low_gpu_mem_usage:+ clear_memory()+ else:+ set_module(self.model, name, m)++ @torch.inference_mode()+ def quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.++ If the target format includes GGUF with `k`, and optimized RTN is enabled,+ blockwise quantization with input caching and imatrix is used.++ Returns:+ tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.+ """if self.amp:self.model.to(self.amp_dtype)self.model.to("cpu")- all_to_quantized_module_names = []- for n, m in self.model.named_modules():- if check_to_quantized(m):- all_to_quantized_module_names.append(n)- if hasattr(self, "formats"):- has_gguf_k = False- for format_ in self.formats:- if "gguf" in format_ and "k" in format_:- has_gguf_k = True- if has_gguf_k and not self.disable_opt_rtn:- self.get_imatrix()- self.quantize_embedding_layer()- pbar = tqdm(all_to_quantized_module_names)-- for name in pbar:- pbar.set_description(f"Quantizing {name}")- m = get_module(self.model, name)- if not self.disable_opt_rtn and not (m.data_type.startswith("rtn_")): ## use rtn version first- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE- if "rtn_" + m.data_type in QUANT_FUNC_WITH_DTYPE:- m.data_type = "rtn_" + m.data_type- self.layer_config[name]["data_type"] = m.data_type- try:- m.to(self.device)- m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,- enable_round_tuning=False)- m = m.unwrapper({})- m.to("cpu")- except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):- logger.warning("out of vram, fallback to cpu")- m.to("cpu")- m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,- enable_round_tuning=False)- m = m.unwrapper({})- else:- raise- if self.low_gpu_mem_usage:- clear_memory()- if self.is_packing_immediate:- from auto_round.export import PACKING_LAYER_WITH_FORMAT- if check_to_quantized(m):- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0])- if self.low_gpu_mem_usage:- clear_memory()- else:- set_module(self.model, name, m)+ all_to_quantized_module_names: list[str] = [+ n for n, m in self.model.named_modules() if check_to_quantized(m)+ ]++ has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))++ self.quantize_embedding_layer()++ if has_gguf_k and not self.disable_opt_rtn:+ self.quant_rtn_with_imatrix(all_to_quantized_module_names)+ else:+ pbar = tqdm(all_to_quantized_module_names)+ for name in pbar:+ pbar.set_description(f"Quantizing {name}")+ self.quantize_layer_via_rtn(name)self.quantized = Truereturn self.model, self.layer_config- def model_forward_blockwise(self):- if bool(self.quant_block_list):- all_blocks = self.quant_block_list- else:- all_blocks = get_block_names(self.model)- if len(all_blocks) == 0:- raise ValueError("could not find any blocks, please check the model or quant_block_list")+ def quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:+ """Quantize model layers block by block using cached inputs and imatrix.++ Args:+ all_to_quantized_module_names (list[str]): Names of layers to be quantized.+ """+ all_to_quantized_module_names = list(set(all_to_quantized_module_names))++ all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)+ if not all_blocks:+ raise ValueError("Could not find any blocks. Check the model or quant_block_list.")- ##ignore layersall_first_block_names = [block[0] for block in all_blocks]- all_inputs = self.cache_inter_data(- all_first_block_names, self.nsamples)+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)+ # Clear hooks for multi-GPU setupsif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed+ accelerate.hooks.remove_hook_from_submodules(self.model)++ pbar = tqdm(range(sum(len(block) for block in all_blocks)))for block_names in all_blocks:- inputs = all_inputs[block_names[0]]- all_inputs.pop(block_names[0])- keys = inputs.keys()- input_id_str = [key for key in keys if key.startswith('hidden_state')]- if len(input_id_str) != 1:- raise RuntimeError(f"hidden_states arg mismatch error,"- "please raise an issue in https://github.com/intel/auto-round/issues")- inputs["input_ids"] = inputs.pop(input_id_str[0], None)+ first_block = block_names[0]+ inputs = all_inputs.pop(first_block)+ input_keys = [k for k in inputs if k.startswith("hidden_state")]+ if len(input_keys) != 1:+ raise RuntimeError(+ "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"+ )+ inputs["input_ids"] = inputs.pop(input_keys[0])clear_memory(self.inputs)- if "input_ids" in inputs.keys():- total_samples = len(inputs["input_ids"])- if total_samples < self.batch_size:- self.batch_size = total_samples- logger.warning(f"force the train batch size to {total_samples}")+ total_samples = len(inputs["input_ids"])+ if total_samples < self.batch_size:+ self.batch_size = total_samples+ logger.warning(f"Forcing batch size to {total_samples}")- input_ids = inputs["input_ids"]- inputs.pop("input_ids", None)- for block_name in block_names:- input_others = inputs- clear_memory()- input_ids = to_device(input_ids, self.cache_device)- input_others = to_device(input_others, self.cache_device)- ## as in the calibration phase, we may use bf16 for calibration due to low_gpu_memory usage- tmp_dtype = self.amp_dtype if self.amp else torch.float32- for i in range(len(input_ids)):- input_ids[i] = input_ids[i].to(tmp_dtype)-- for key in input_others.keys():- if isinstance(input_others[key], torch.Tensor) and (- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16- ):- input_others[key] = input_others[key].to(tmp_dtype)- elif isinstance(input_others[key], list):- for i in range(len(input_others[key])):- to_dtype(input_others[key][i], tmp_dtype)+ input_ids = to_device(inputs.pop("input_ids"), self.cache_device)+ input_others = to_device(inputs, self.cache_device)++ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ input_ids = [id_.to(tmp_dtype) for id_ in input_ids]+ for key, val in input_others.items():+ if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):+ input_others[key] = val.to(tmp_dtype)+ elif isinstance(val, list):+ input_others[key] = [to_dtype(v, tmp_dtype) for v in val]++ for block_name in block_names:+ pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)+ # Dispatch model if neededif self.device_map is not None:from accelerate import dispatch_model- for n, m in block.named_modules():+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ for _, m in block.named_modules():if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):continue- from accelerate.hooks import AlignDevicesHook, add_hook_to_modulehook = AlignDevicesHook(m.tuning_device, io_same_device=True)add_hook_to_module(m, hook, True)- input_ids = self.get_block_outputs(block, input_ids, input_others,- self.batch_size * self.infer_bs_coeff,- self.device,- self.cache_device)+ input_ids = self.get_block_outputs(+ block,+ input_ids,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ self.device,+ self.cache_device,+ )++ accelerate.hooks.remove_hook_from_submodules(block)++ # Normalize imatrix and quantize layers+ for _, m in block.named_modules():+ if hasattr(m, "imatrix"):+ m.imatrix /= m.imatrix_cnt+ if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:+ self.quantize_layer_via_rtn(m.tmp_name)+ all_to_quantized_module_names.remove(m.tmp_name)- if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(- block)mv_module_from_gpu(block, self.low_cpu_mem_usage)+ pbar.update(1)++ pbar.close()++ # Process remaining layers not in blocks+ for name in all_to_quantized_module_names:+ self.quantize_layer_via_rtn(name)+def quantize(self):"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.Returns:The quantized model and layer configurations."""+ for n, m in self.model.named_modules():+ m.tmp_name = nself._check_compatibility()self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)if not hasattr(self, "formats"):@@ -974,8 +1149,8 @@ class AutoRound(object):# Determine if immediate packing is requiredformats = self.formatsif (len(formats) == 1 and- ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0]) and- not self.has_qlayer_outside_block and self.inplace): # TODO: Support more formats+ ("awq" in formats[0] or "gptq" in formats[0] or+ "auto_round" in formats[0] or "gguf" in formats[0]) and self.inplace):self.is_packing_immediate = Trueif self.iters == 0:return self.quantize_rtn()@@ -1086,7 +1261,7 @@ class AutoRound(object):"""Quantizes specified layers based on inputs and configuration.Args:- layer_names (list): List of layer names to quantize.+ layer_names (list): list of layer names to quantize.layer_inputs (dict): Dictionary mapping layer names to input data.Returns:@@ -1115,6 +1290,12 @@ class AutoRound(object):returnq_layer_inputs = Noneenable_quanted_input = self.enable_quanted_input+ has_gguf = False+ if hasattr(self, "formats"):+ has_gguf = any("gguf" in format_ for format_ in self.formats)+ if has_gguf and self.is_packing_immediate:+ enable_quanted_input = False+if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:from accelerate.big_modeling import dispatch_model@@ -1138,7 +1319,7 @@ class AutoRound(object):for layer_name in layer_names:layer_input = layer_inputs[layer_name]layer_input = to_device(layer_input, self.cache_device)- q_layer_input = q_layer_inputs[layer_name] if enable_quanted_input else None+ q_layer_input = q_layer_inputs[layer_name] if q_layer_inputs is not None else Noneq_layer_input = to_device(q_layer_input, self.cache_device)quant_layer(layer_name, layer_input, q_layer_input, device=self.device)del layer_input@@ -2070,7 +2251,14 @@ class AutoRound(object):for _, tmp_m in m.named_modules():if hasattr(tmp_m, "bits") and check_to_quantized(tmp_m):target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])+ has_gguf = any("gguf" in format_ for format_ in self.formats)+ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer+ output_dir = self.get_save_folder_name(self.formats[0])+ pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,+ self.tokenizer)+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.name, self.model, self.formats[0])pbar.set_description(f"Quantizing done")pbar.update(1)pbar.close()diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex 845544017..d1012123e 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -50,7 +50,7 @@ def _save_quantized_as_autoawq(*args, **kwargs):return save_quantized_as_autoawq(*args, **kwargs)@register_format("gguf")-def _save_quantized_as_autoawq(*args, **kwargs):+def _save_quantized_as_gguf(*args, **kwargs):from auto_round.export.export_to_gguf.export import save_quantized_as_ggufreturn save_quantized_as_gguf(*args, **kwargs)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 1f5f7fc66..27ea57aba 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -35,6 +35,7 @@from __future__ import annotationsimport ast+import gcimport loggingimport argparseimport contextlib@@ -56,6 +57,7 @@ import torchfrom auto_round.utils import logger, LazyImport, clear_memory, get_module, clean_module_parameterfrom auto_round.export.export_to_gguf.packing import ggml_quant_gpu+gguf = LazyImport("gguf")if TYPE_CHECKING:@@ -318,8 +320,10 @@ class OriModel:logger.info(f"gguf: experts used count = {n_experts_used}")if (head_dim := self.hparams.get("head_dim")) is not None:- self.gguf_writer.add_key_length(head_dim)- self.gguf_writer.add_value_length(head_dim)+ # Workaround for incorrect AutoConfig value for DeepSeekV3 (is set correctly in DeepSeekV2Model class)+ if self.hparams.get("model_type") != "deepseek_v3":+ self.gguf_writer.add_key_length(head_dim)+ self.gguf_writer.add_value_length(head_dim)self.gguf_writer.add_file_type(self.ftype)logger.info(f"gguf: file type = {self.ftype}")@@ -1137,14 +1141,14 @@ class Model(OriModel):if not low_cpu_mem_usage:return False- process = psutil.Process(os.getpid())- mem_usage = process.memory_info().rss- memory_info = psutil.virtual_memory()- if memory_info.available > mem_usage / 3:- return False- else:- logger.info("use low cpu memory mode.")- return True+ # process = psutil.Process(os.getpid())+ # mem_usage = process.memory_info().rss+ # memory_info = psutil.virtual_memory()+ # if memory_info.available > mem_usage / 3:+ # return False+ # else:+ # logger.info("use low cpu memory mode.")+ return Truedef get_moe_name(self, name, new_name):type_mapping = {@@ -1323,9 +1327,17 @@ class Model(OriModel):max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):+ if data_torch is None:+ continue# we don't need theseif name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):continue+ if hasattr(self,"current_packing_block") and self.current_packing_block is not None: # pylint: disable=E1101+ current_packing_block_split = self.current_packing_block.split('.') # pylint: disable=E1101+ name_split = name.split('.')+ if (len(name_split)<len(current_packing_block_split) or+ name_split[:len(current_packing_block_split)]!=current_packing_block_split):+ continueold_dtype = data_torch.dtype@@ -1339,7 +1351,8 @@ class Model(OriModel):if part.isdecimal():bid = int(part)break-++ clean_weight_list = []for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):data = data_torch.squeeze().cpu().numpy()@@ -1383,7 +1396,7 @@ class Model(OriModel):# get name by new_name (for experts),name = self.get_moe_name(name, new_name)-+ clean_weight_list.append(name)# get data_qtype by layer_configdata_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)@@ -1491,11 +1504,11 @@ class Model(OriModel):self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)- # save cpu memory, but slow+ # # save cpu memory, but slowif self.low_cpu_mem_usage:- module = get_module(self.model, ".".join(name.split(".")[:-1]))- clean_module_parameter(module, name.split(".")[-1])- clear_memory()+ for weight_name in clean_weight_list:+ module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))+ clean_module_parameter(module, weight_name.split(".")[-1])@Model.register("GPTNeoXForCausalLM")diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex db8ec381e..a950c7b11 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -16,51 +16,38 @@ import osimport sysimport shutilimport torch-from .convert import Model-from auto_round.utils import logger, LazyImportfrom pathlib import Pathimport time+from auto_round.export.export_to_gguf.convert import Model+from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_modulegguf = LazyImport("gguf")FTYPE_MAP: dict[str, gguf.LlamaFileType] = {- "f32": gguf.LlamaFileType.ALL_F32,- "f16": gguf.LlamaFileType.MOSTLY_F16,- "bf16": gguf.LlamaFileType.MOSTLY_BF16,- "q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,- "q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,- "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,- "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,- "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,- "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,- "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,- "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,- "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,- "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,- "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,- "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,- "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,- "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,- "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,- "auto": gguf.LlamaFileType.GUESSED,- }--def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, low_cpu_mem_usage=False, **kwargs):- """Export the model to gguf format."""- if output_dir is not None and os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")+ "f32": gguf.LlamaFileType.ALL_F32,+ "f16": gguf.LlamaFileType.MOSTLY_F16,+ "bf16": gguf.LlamaFileType.MOSTLY_BF16,+ "q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,+ "q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,+ "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,+ "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,+ "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,+ "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,+ "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,+ "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,+ "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,+ "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,+ "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,+ "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,+ "auto": gguf.LlamaFileType.GUESSED,+}- st = time.time()-- model = kwargs["model"]- tokenizer = kwargs.get("tokenizer", None)- config = model.config+def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False):tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))- if tokenizer is not None:- tokenizer.save_pretrained(tmp_work_dir)- config.save_pretrained(tmp_work_dir)-with torch.inference_mode():hparams = Model.load_hparams(tmp_work_dir)model_architecture = hparams["architectures"][0]@@ -81,26 +68,102 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, lraise TypeError(f"{output_type} type is not supported")output_type = FTYPE_MAP.get(output_type.lower())-model_instance = model_class(model,layer_config,dir_model=tmp_work_dir,ftype=output_type,fname_out=Path(output_dir),- low_cpu_mem_usage=low_cpu_mem_usage,+ low_cpu_mem_usage=low_cpu_mem_usage, # pylint: disable=E0401is_big_endian=False,model_name=model_name,split_max_tensors=False,split_max_size=0,dry_run=False,- small_first_shard=False,- )- model_instance.write()- rt = time.time() - st- logger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")+ small_first_shard=False)+ return model_instance++[email protected]_mode()+def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):+ """Export the model to gguf format."""+ global gguf_model_instance_global+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):+ logger.warning_once(f"{output_dir} already exists, this may cause model conflict")+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ if "gguf_model_instance_global" not in globals():+ config = model.config+ if tokenizer is not None:+ tokenizer.save_pretrained(tmp_work_dir)+ config.save_pretrained(tmp_work_dir)++ gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend,+ low_cpu_mem_usage=True)++ if not hasattr(model, "last_layer_name_to_block_name"):+ block_name_to_last_layer_name = {}+ block_names = get_block_names(model, quant_vision=True)+ block_names_flatten = flatten_list(block_names)+ for n, m in model.named_modules():+ if not check_to_quantized(m):+ continue+ for block_name in block_names_flatten:+ block_name_split = block_name.split('.')+ name_split = n.split('.')+ if (len(name_split) < len(block_name_split) or+ name_split[:len(block_name_split)] != block_name_split):+ continue+ block_name_to_last_layer_name[block_name] = n+ last_layer_name_to_block_name = {v: k for k, v in block_name_to_last_layer_name.items()}+ model.last_layer_name_to_block_name = last_layer_name_to_block_name+ if name in model.last_layer_name_to_block_name:+ ##packing block+ gguf_model_instance_global.current_packing_block = model.last_layer_name_to_block_name[name]+ gguf_model_instance_global.prepare_tensors()+ block = get_module(model, model.last_layer_name_to_block_name[name])+ for n, m in block.named_modules():+ if hasattr(m, "weight"):+ m.weight = None+ if hasattr(m, "bias"):+ m.bias = None+ import gc+ gc.collect()+ model.last_layer_name_to_block_name.pop(name)+ if len(model.last_layer_name_to_block_name) == 0:+ gguf_model_instance_global.current_packing_block = None++[email protected]_mode()+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):+ """Export the model to gguf format."""+ tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")++ st = time.time()+ global gguf_model_instance_global++ model = kwargs["model"]+ if "gguf_model_instance_global" not in globals():+ tokenizer = kwargs.get("tokenizer", None)+ config = model.config+ if tokenizer is not None:+ tokenizer.save_pretrained(tmp_work_dir)+ config.save_pretrained(tmp_work_dir)++ config = model.config+ if tokenizer is not None:+ tokenizer.save_pretrained(tmp_work_dir)+ config.save_pretrained(tmp_work_dir)++ gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend)++ gguf_model_instance_global.write()+ rt = time.time() - st+ logger.info(f"Model successfully exported to {gguf_model_instance_global.fname_out}, running time={rt}")+ del gguf_model_instance_globalshutil.rmtree(tmp_work_dir, ignore_errors=True)return model-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex d4b92508e..ce2345161 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -782,9 +782,6 @@ def get_layer_names_in_block(model, supported_types=(torch.nn.Linear,for n, m in block.named_modules():if hasattr(m, "tmp_name"):layers_in_block.append(m.tmp_name)- for n, m in model.named_modules():- if hasattr(m, "tmp_name"):- delattr(m, "tmp_name")return layers_in_block@@ -1592,7 +1589,7 @@ def _search_gguf_type(gguf_type):if not bits:raise KeyError(f"{gguf_type} is not a correct gguf type, please check")- for suffix in ["_0", "_1", "_k"]:+ for suffix in ["_k", "_0", "_1"]:if gguf_type.endswith(suffix):continueif (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:@@ -1894,6 +1891,14 @@ def get_gguf_qtype_by_layer_config(layer_config):return gguf.GGMLQuantizationType.Q8_0raise ValueError(f"Unknown layer config")+def flatten_list(nested_list):+ flattened = []+ for item in nested_list:+ if isinstance(item, (list, tuple)):+ flattened.extend(flatten_list(item))+ else:+ flattened.append(item)+ return flatteneddef clean_module_parameter(submodule, parameter):is_buffer = parameter in submodule._buffers@@ -1901,5 +1906,4 @@ def clean_module_parameter(submodule, parameter):if is_buffer:submodule._buffers[parameter] = Noneelse:- submodule._parameters[parameter] = None- gc.collect()\ No newline at end of file+ submodule._parameters[parameter] = None\ No newline at end of filediff --git a/auto_round/version.py b/auto_round/version.pyindex 41f5016dc..d9f21f781 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -14,4 +14,4 @@"""Intel® auto-round: An open-source Python librarysupporting popular model weight only compression based on signround."""-__version__ = "0.5.1"+__version__ = "0.6.0"