Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[BUG FIX] fix bug of deepseek gguf:q*k (#637)

Heng Guoba5e35c1d10b
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 25f832b6b..4d9b299a2 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -603,6 +603,7 @@ class AutoRound(object):
# Save the quantized model in the specified formats
folders = []
+ low_cpu_mem_usage = len(formats) == 1 and "gguf" in formats[0]
for format in formats:
if "gptq" in format and not self.sym:
logger.warning(
@@ -613,7 +614,8 @@ class AutoRound(object):
)
save_format_ = format.replace(":", "-").replace("_", "-")
save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir
- self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
+ self.save_quantized(
+ save_folder, format=format, inplace=inplace, low_cpu_mem_usage=low_cpu_mem_usage, **kwargs)
folders.append(save_folder)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 4a3b180c4..1f5f7fc66 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -42,6 +42,7 @@ import json
import os
import re
import sys
+import psutil
from enum import IntEnum
from pathlib import Path
from hashlib import sha256
@@ -52,8 +53,8 @@ import math
import numpy as np
import torch
-from auto_round.utils import logger, LazyImport
-from .packing import ggml_quant_gpu
+from auto_round.utils import logger, LazyImport, clear_memory, get_module, clean_module_parameter
+from auto_round.export.export_to_gguf.packing import ggml_quant_gpu
gguf = LazyImport("gguf")
@@ -1096,6 +1097,7 @@ class Model(OriModel):
dir_model: Path,
ftype: gguf.LlamaFileType,
fname_out: Path,
+ low_cpu_mem_usage: bool = False,
is_big_endian: bool = False,
use_temp_file: bool = False,
eager: bool = False,
@@ -1108,6 +1110,7 @@ class Model(OriModel):
hparams: dict[str, Any] | None = None):
self.model = model
self.layer_config = layer_config
+ self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)
super().__init__(
dir_model=dir_model,
ftype=ftype,
@@ -1130,6 +1133,19 @@ class Model(OriModel):
if "model_arch" not in cls.__dict__:
raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")
+ def _need_low_cpu_mem(self, low_cpu_mem_usage):
+ if not low_cpu_mem_usage:
+ return False
+
+ process = psutil.Process(os.getpid())
+ mem_usage = process.memory_info().rss
+ memory_info = psutil.virtual_memory()
+ if memory_info.available > mem_usage / 3:
+ return False
+ else:
+ logger.info("use low cpu memory mode.")
+ return True
+
def get_moe_name(self, name, new_name):
type_mapping = {
"FFN_GATE_EXP": ["gate_proj", "w1", "linear"],
@@ -1157,9 +1173,34 @@ class Model(OriModel):
def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
for name, tensor in self.model.named_parameters():
yield name, tensor
+
+ def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):
+ if torch.cuda.is_available():
+ device = "cuda"
+ elif torch.xpu.is_available():
+ device = "xpu"
+ else:
+ device = "cpu"
+ data_torch = data_torch.to(torch.float32)
+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
+ d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale
+ d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin
+ wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin
+
+ data = ggml_quant_gpu(
+ data_torch,
+ data_qtype.name.lower(),
+ scale,
+ zp,
+ wmin=wmin,
+ d_scale=d_scale,
+ d_wmin=d_wmin,
+ device=device
+ )
+ return data
def _quant_data(self, data_torch, data_qtype, name, bid):
- from auto_round.utils import get_module
suffix = '.weight'
if suffix in name:
layer_name = name[:-len(suffix)]
@@ -1174,7 +1215,7 @@ class Model(OriModel):
attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]
attr_tensor = attr_tensor.reshape(ori_shape)
setattr(module, attr, attr_tensor)
- scale = module.scale
+ scale = module.scale if hasattr(module, "scale") else None
zp = module.zp if hasattr(module, "zp") else None
if torch.cuda.is_available():
device = "cuda"
@@ -1186,7 +1227,7 @@ class Model(OriModel):
scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
if data_qtype.name.lower().endswith("_k"):
- d_scale = module.w_d_scale.to(torch.float32)
+ d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None
d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None
wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None
@@ -1389,8 +1430,40 @@ class Model(OriModel):
data_qtype = gguf.GGMLQuantizationType.F16
data = gguf.quants.quantize(data, data_qtype)
else:
+ # for deepseek v2
+ if name.endswith("kv_b_proj.weight") and self.model_arch.name == 'DEEPSEEK2':
+ layer_name = name[:-len('.weight')]
+ module = get_module(self.model, layer_name)
+ n_head_kv = self.hparams["num_key_value_heads"]
+ v_head_dim = self.hparams["v_head_dim"]
+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+
+ attr_list = {
+ "scale": None,
+ "zp": None,
+ "d_scale": None,
+ "wmin": None,
+ "d_wmin": None
+ }
+ for attr in attr_list:
+ if hasattr(module, attr) or hasattr(module, "w_" + attr):
+ if attr in ['scale', "zp"]:
+ attr_tensor = getattr(module, attr)
+ else:
+ attr_tensor = getattr(module, "w_" + attr)
+ kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)
+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+ k_b = k_b.transpose(1, 2)
+
+ name_kb = name.replace("kv_b_proj", "k_b_proj")
+ if new_name == self.map_tensor_name(name_kb):
+ attr_list[attr] = k_b
+ else:
+ attr_list[attr] = v_b
+ data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)
+
# for MOE model
- if len(data_torch.shape) == 3:
+ elif len(data_torch.shape) == 3:
new_data = []
for idx, arr in enumerate(data_torch):
arr_name = name.split('.')
@@ -1412,14 +1485,18 @@ class Model(OriModel):
shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"
# n_dims is implicit in the shape
- if isinstance(data_qtype, bool):
- breakpoint()
logger.info(
f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"
f" --> {data_qtype.name}, shape = {shape_str}")
self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)
+ # save cpu memory, but slow
+ if self.low_cpu_mem_usage:
+ module = get_module(self.model, ".".join(name.split(".")[:-1]))
+ clean_module_parameter(module, name.split(".")[-1])
+ clear_memory()
+
@Model.register("GPTNeoXForCausalLM")
class GPTNeoXModel(Model):
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 527085e5e..db8ec381e 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -45,7 +45,7 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
"auto": gguf.LlamaFileType.GUESSED,
}
-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):
+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, low_cpu_mem_usage=False, **kwargs):
"""Export the model to gguf format."""
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
@@ -88,12 +88,14 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, *
dir_model=tmp_work_dir,
ftype=output_type,
fname_out=Path(output_dir),
+ low_cpu_mem_usage=low_cpu_mem_usage,
is_big_endian=False,
model_name=model_name,
split_max_tensors=False,
split_max_size=0,
dry_run=False,
- small_first_shard=False)
+ small_first_shard=False,
+ )
model_instance.write()
rt = time.time() - st
logger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")
diff --git a/auto_round/utils.py b/auto_round/utils.py
index a923a37ec..d4b92508e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -29,7 +29,7 @@ from packaging import version
import gc
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS
import transformers
-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG
+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K
SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
@@ -1219,7 +1219,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
logger.error(f"Model {model_architecture} is not supported to export GGUF format")
sys.exit(1)
- if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):
+ if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % QK_K != 0):
model_name = args_or_ar.model.split('/')
model_name = model_name[-1] if model_name[-1] else model_name[-2]
hidden_size = hparams["hidden_size"]
@@ -1599,6 +1599,15 @@ def _search_gguf_type(gguf_type):
return tmp_type
return None
+def _gguf_type_fallback(gguf_type):
+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q5_k":
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q6_k":
+ gguf_type = "gguf:q8_0"
+ return gguf_type
+
##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
# TODO: support for other format later
@@ -1804,18 +1813,30 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
new_type = "gguf:q5_k"
new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
if input_features % new_block_size != 0:
- if new_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
- new_type = "gguf:q5_0"
- elif new_type == "gguf:q5_k":
- new_type = "gguf:q5_0"
- elif new_type == "gguf:q6_k":
- new_type = "gguf:q8_0"
+ new_type = _gguf_type_fallback(new_type)
new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
if input_features % new_block_size != 0:
new_type = "gguf:bf16"
logger.warning(
f"fallback {layer_name} to {new_type}, "
f"because input_features({input_features}) % block_size({block_size}) != 0")
+ # for deepseek v2
+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") \
+ and 'Deepseek' in model.config.architectures[0]:
+ fallback = False
+
+ # calc if need fallback
+ qk_nope_head_dim = model.config.qk_nope_head_dim
+ kv_b_shape = get_module(model, layer_name).weight.shape
+
+ if qk_nope_head_dim < QK_K or qk_nope_head_dim % QK_K != 0 \
+ or kv_b_shape[-1] < QK_K or kv_b_shape[-1] % QK_K != 0:
+ fallback = True
+ if fallback:
+ tmp_type = _gguf_type_fallback(new_type)
+ logger.warning_once(
+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}")
+ new_type = tmp_type
target_config = GGUF_INNER_CONFIG[new_type]
@@ -1872,3 +1893,13 @@ def get_gguf_qtype_by_layer_config(layer_config):
if bits == 8 and sym and group_size == 32:
return gguf.GGMLQuantizationType.Q8_0
raise ValueError(f"Unknown layer config")
+
+
+def clean_module_parameter(submodule, parameter):
+ is_buffer = parameter in submodule._buffers
+ with torch.no_grad():
+ if is_buffer:
+ submodule._buffers[parameter] = None
+ else:
+ submodule._parameters[parameter] = None
+ gc.collect()
\ No newline at end of file