PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[BUG FIX] fix bug of deepseek gguf:q*k (#637)
Heng Guoba5e35c1d10b
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 25f832b6b..4d9b299a2 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -603,6 +603,7 @@ class AutoRound(object):# Save the quantized model in the specified formatsfolders = []+ low_cpu_mem_usage = len(formats) == 1 and "gguf" in formats[0]for format in formats:if "gptq" in format and not self.sym:logger.warning(@@ -613,7 +614,8 @@ class AutoRound(object):)save_format_ = format.replace(":", "-").replace("_", "-")save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir- self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)+ self.save_quantized(+ save_folder, format=format, inplace=inplace, low_cpu_mem_usage=low_cpu_mem_usage, **kwargs)folders.append(save_folder)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 4a3b180c4..1f5f7fc66 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -42,6 +42,7 @@ import jsonimport osimport reimport sys+import psutilfrom enum import IntEnumfrom pathlib import Pathfrom hashlib import sha256@@ -52,8 +53,8 @@ import mathimport numpy as npimport torch-from auto_round.utils import logger, LazyImport-from .packing import ggml_quant_gpu+from auto_round.utils import logger, LazyImport, clear_memory, get_module, clean_module_parameter+from auto_round.export.export_to_gguf.packing import ggml_quant_gpugguf = LazyImport("gguf")@@ -1096,6 +1097,7 @@ class Model(OriModel):dir_model: Path,ftype: gguf.LlamaFileType,fname_out: Path,+ low_cpu_mem_usage: bool = False,is_big_endian: bool = False,use_temp_file: bool = False,eager: bool = False,@@ -1108,6 +1110,7 @@ class Model(OriModel):hparams: dict[str, Any] | None = None):self.model = modelself.layer_config = layer_config+ self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)super().__init__(dir_model=dir_model,ftype=ftype,@@ -1130,6 +1133,19 @@ class Model(OriModel):if "model_arch" not in cls.__dict__:raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")+ def _need_low_cpu_mem(self, low_cpu_mem_usage):+ if not low_cpu_mem_usage:+ return False++ process = psutil.Process(os.getpid())+ mem_usage = process.memory_info().rss+ memory_info = psutil.virtual_memory()+ if memory_info.available > mem_usage / 3:+ return False+ else:+ logger.info("use low cpu memory mode.")+ return True+def get_moe_name(self, name, new_name):type_mapping = {"FFN_GATE_EXP": ["gate_proj", "w1", "linear"],@@ -1157,9 +1173,34 @@ class Model(OriModel):def get_tensors(self) -> Iterator[tuple[str, Tensor]]:for name, tensor in self.model.named_parameters():yield name, tensor++ def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp+ d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale+ d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin+ wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin++ data = ggml_quant_gpu(+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ device=device+ )+ return datadef _quant_data(self, data_torch, data_qtype, name, bid):- from auto_round.utils import get_modulesuffix = '.weight'if suffix in name:layer_name = name[:-len(suffix)]@@ -1174,7 +1215,7 @@ class Model(OriModel):attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]attr_tensor = attr_tensor.reshape(ori_shape)setattr(module, attr, attr_tensor)- scale = module.scale+ scale = module.scale if hasattr(module, "scale") else Nonezp = module.zp if hasattr(module, "zp") else Noneif torch.cuda.is_available():device = "cuda"@@ -1186,7 +1227,7 @@ class Model(OriModel):scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scalezp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zpif data_qtype.name.lower().endswith("_k"):- d_scale = module.w_d_scale.to(torch.float32)+ d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else Noned_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else Nonewmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None@@ -1389,8 +1430,40 @@ class Model(OriModel):data_qtype = gguf.GGMLQuantizationType.F16data = gguf.quants.quantize(data, data_qtype)else:+ # for deepseek v2+ if name.endswith("kv_b_proj.weight") and self.model_arch.name == 'DEEPSEEK2':+ layer_name = name[:-len('.weight')]+ module = get_module(self.model, layer_name)+ n_head_kv = self.hparams["num_key_value_heads"]+ v_head_dim = self.hparams["v_head_dim"]+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]++ attr_list = {+ "scale": None,+ "zp": None,+ "d_scale": None,+ "wmin": None,+ "d_wmin": None+ }+ for attr in attr_list:+ if hasattr(module, attr) or hasattr(module, "w_" + attr):+ if attr in ['scale', "zp"]:+ attr_tensor = getattr(module, attr)+ else:+ attr_tensor = getattr(module, "w_" + attr)+ kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)+ k_b = k_b.transpose(1, 2)++ name_kb = name.replace("kv_b_proj", "k_b_proj")+ if new_name == self.map_tensor_name(name_kb):+ attr_list[attr] = k_b+ else:+ attr_list[attr] = v_b+ data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)+# for MOE model- if len(data_torch.shape) == 3:+ elif len(data_torch.shape) == 3:new_data = []for idx, arr in enumerate(data_torch):arr_name = name.split('.')@@ -1412,14 +1485,18 @@ class Model(OriModel):shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"# n_dims is implicit in the shape- if isinstance(data_qtype, bool):- breakpoint()logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"f" --> {data_qtype.name}, shape = {shape_str}")self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)+ # save cpu memory, but slow+ if self.low_cpu_mem_usage:+ module = get_module(self.model, ".".join(name.split(".")[:-1]))+ clean_module_parameter(module, name.split(".")[-1])+ clear_memory()+@Model.register("GPTNeoXForCausalLM")class GPTNeoXModel(Model):diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 527085e5e..db8ec381e 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -45,7 +45,7 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {"auto": gguf.LlamaFileType.GUESSED,}-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, low_cpu_mem_usage=False, **kwargs):"""Export the model to gguf format."""if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")@@ -88,12 +88,14 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, *dir_model=tmp_work_dir,ftype=output_type,fname_out=Path(output_dir),+ low_cpu_mem_usage=low_cpu_mem_usage,is_big_endian=False,model_name=model_name,split_max_tensors=False,split_max_size=0,dry_run=False,- small_first_shard=False)+ small_first_shard=False,+ )model_instance.write()rt = time.time() - stlogger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex a923a37ec..d4b92508e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -29,7 +29,7 @@ from packaging import versionimport gcfrom auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYSimport transformers-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_KSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")@@ -1219,7 +1219,7 @@ def _gguf_args_check(args_or_ar, format_str=None):logger.error(f"Model {model_architecture} is not supported to export GGUF format")sys.exit(1)- if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):+ if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % QK_K != 0):model_name = args_or_ar.model.split('/')model_name = model_name[-1] if model_name[-1] else model_name[-2]hidden_size = hparams["hidden_size"]@@ -1599,6 +1599,15 @@ def _search_gguf_type(gguf_type):return tmp_typereturn None+def _gguf_type_fallback(gguf_type):+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q5_k":+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q6_k":+ gguf_type = "gguf:q8_0"+ return gguf_type+##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129def get_layer_config_by_gguf_format(layer_config, gguf_format, model):# TODO: support for other format later@@ -1804,18 +1813,30 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):new_type = "gguf:q5_k"new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]if input_features % new_block_size != 0:- if new_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):- new_type = "gguf:q5_0"- elif new_type == "gguf:q5_k":- new_type = "gguf:q5_0"- elif new_type == "gguf:q6_k":- new_type = "gguf:q8_0"+ new_type = _gguf_type_fallback(new_type)new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]if input_features % new_block_size != 0:new_type = "gguf:bf16"logger.warning(f"fallback {layer_name} to {new_type}, "f"because input_features({input_features}) % block_size({block_size}) != 0")+ # for deepseek v2+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") \+ and 'Deepseek' in model.config.architectures[0]:+ fallback = False++ # calc if need fallback+ qk_nope_head_dim = model.config.qk_nope_head_dim+ kv_b_shape = get_module(model, layer_name).weight.shape++ if qk_nope_head_dim < QK_K or qk_nope_head_dim % QK_K != 0 \+ or kv_b_shape[-1] < QK_K or kv_b_shape[-1] % QK_K != 0:+ fallback = True+ if fallback:+ tmp_type = _gguf_type_fallback(new_type)+ logger.warning_once(+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}")+ new_type = tmp_typetarget_config = GGUF_INNER_CONFIG[new_type]@@ -1872,3 +1893,13 @@ def get_gguf_qtype_by_layer_config(layer_config):if bits == 8 and sym and group_size == 32:return gguf.GGMLQuantizationType.Q8_0raise ValueError(f"Unknown layer config")+++def clean_module_parameter(submodule, parameter):+ is_buffer = parameter in submodule._buffers+ with torch.no_grad():+ if is_buffer:+ submodule._buffers[parameter] = None+ else:+ submodule._parameters[parameter] = None+ gc.collect()\ No newline at end of file