Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[WIP] support for glm4.5 gguf (#735)

Heng Guo4f740ccbd510
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 854f3f460..319be6d70 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -911,7 +911,7 @@ class AutoRound(object):
module.imatrix = squared
module.imatrix_cnt = input.shape[0]
else:
- module.imatrix += squared
+ module.imatrix += squared.to(module.imatrix.device)
module.imatrix_cnt += input.shape[0]
hook_handles = []
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index b8240b894..c204cf1e8 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -48,6 +48,7 @@ import requests
import torch
from transformers import AutoConfig
+from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.packing import ggml_quant
from auto_round.utils import LazyImport, clean_module_parameter, get_module, logger
@@ -108,7 +109,7 @@ def get_moe_name(cls, name, new_name):
"FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],
"FFN_UP_EXP": ["up_proj", "w3", "linear_v"],
}
- nums = re.findall("\d+", name)
+ nums = re.findall(r"\d+", name)
if len(nums) != 2:
return name
name_tmp = name[: -len(".weight")].replace(f".{nums[1]}", "")
@@ -127,7 +128,46 @@ def get_moe_name(cls, name, new_name):
def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
+ if not hasattr(cls.model, "tensor_name_list"):
+ cls.model.tensor_name_list = []
+
+ reverse_key_mapping = {v: k for k, v in cls.model._checkpoint_conversion_mapping.items()}
for name, tensor in cls.model.named_parameters():
+ if name not in cls.model.tensor_name_list:
+ cls.model.tensor_name_list.append(name)
+ for pattern, replacement in reverse_key_mapping.items():
+ replacement = replacement.lstrip("^") # strip off un-needed chars and patterns
+ replacement = re.sub(r"\(.*\)", "", replacement)
+ key, n_replace = re.subn(pattern, replacement, name)
+ cls.model.tensor_name_list.append(key)
+ if n_replace > 0:
+ break
+ yield name, tensor
+
+ extra_tensor = {}
+ if hasattr(cls.model, "name_or_path"):
+ from safetensors import safe_open
+
+ from auto_round.export.export_to_gguf.special_handle import get_tensor_from_file
+ from auto_round.utils import download_hf_model
+
+ dir_path = cls.model.name_or_path
+ if not os.path.isdir(dir_path):
+ dir_path = download_hf_model(dir_path)
+ INDEX_FILE = "model.safetensors.index.json"
+ if INDEX_FILE in os.listdir(dir_path):
+ with open(os.path.join(dir_path, INDEX_FILE)) as f:
+ tensor_index = json.load(f)
+ for tensor_name in tensor_index["weight_map"]:
+ if tensor_name not in cls.model.tensor_name_list:
+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
+ else:
+ f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
+ for tensor_name in f.keys():
+ if tensor_name not in cls.model.tensor_name_list:
+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
+
+ for name, tensor in extra_tensor.items():
yield name, tensor
@@ -356,6 +396,14 @@ def prepare_tensors(cls):
modify_name = _special_name_handle(cls, name)
for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):
+ skip = False
+ for tensor_info in cls.gguf_writer.tensors:
+ if new_name in tensor_info:
+ print("new_name already add to gguf_writer, skip")
+ skip = True
+ break
+ if skip:
+ continue
data = data_torch.squeeze().cpu().numpy()
# if data ends up empty, it means data_torch was a scalar tensor -> restore
@@ -513,7 +561,7 @@ def prepare_tensors(cls):
data = _quant_data_with_args(data_torch, data_qtype, **attr_list)
# for MOE model
- elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:
+ elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:
new_data = []
for idx, arr in enumerate(data_torch):
arr_name = name.split(".")
diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.py
index 490788d6d..f8a463e9b 100644
--- a/auto_round/export/export_to_gguf/special_handle.py
+++ b/auto_round/export/export_to_gguf/special_handle.py
@@ -36,8 +36,9 @@ def get_tensor_from_file(dir_path, tensor_name):
INDEX_FILE = "model.safetensors.index.json"
# get filename
if INDEX_FILE in os.listdir(dir_path):
- tensor_index = json.load(open(os.path.join(dir_path, INDEX_FILE)))
- filename = tensor_index["weight_map"][tensor_name]
+ with open(os.path.join(dir_path, INDEX_FILE)) as f:
+ tensor_index = json.load(f)
+ filename = tensor_index["weight_map"][tensor_name]
else:
filename = "model.safetensors"
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 9c3933610..3e25a32d3 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -26,6 +26,7 @@ SUPPORT_ONLY_TEXT_MODELS = [
"idefics3",
"llama4",
"internvl_chat",
+ "glm4v_moe",
]
NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index 9e3fc4b26..d05273b69 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"
+ gguf_file = os.listdir(quantized_model_path)[0]
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"
+ gguf_file = os.listdir(quantized_model_path)[0]
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)