PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[WIP] support for glm4.5 gguf (#735)
Heng Guo4f740ccbd510
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 854f3f460..319be6d70 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -911,7 +911,7 @@ class AutoRound(object):module.imatrix = squaredmodule.imatrix_cnt = input.shape[0]else:- module.imatrix += squared+ module.imatrix += squared.to(module.imatrix.device)module.imatrix_cnt += input.shape[0]hook_handles = []diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex b8240b894..c204cf1e8 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -48,6 +48,7 @@ import requestsimport torchfrom transformers import AutoConfig+from auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.export.export_to_gguf.packing import ggml_quantfrom auto_round.utils import LazyImport, clean_module_parameter, get_module, logger@@ -108,7 +109,7 @@ def get_moe_name(cls, name, new_name):"FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],"FFN_UP_EXP": ["up_proj", "w3", "linear_v"],}- nums = re.findall("\d+", name)+ nums = re.findall(r"\d+", name)if len(nums) != 2:return namename_tmp = name[: -len(".weight")].replace(f".{nums[1]}", "")@@ -127,7 +128,46 @@ def get_moe_name(cls, name, new_name):def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:+ if not hasattr(cls.model, "tensor_name_list"):+ cls.model.tensor_name_list = []++ reverse_key_mapping = {v: k for k, v in cls.model._checkpoint_conversion_mapping.items()}for name, tensor in cls.model.named_parameters():+ if name not in cls.model.tensor_name_list:+ cls.model.tensor_name_list.append(name)+ for pattern, replacement in reverse_key_mapping.items():+ replacement = replacement.lstrip("^") # strip off un-needed chars and patterns+ replacement = re.sub(r"\(.*\)", "", replacement)+ key, n_replace = re.subn(pattern, replacement, name)+ cls.model.tensor_name_list.append(key)+ if n_replace > 0:+ break+ yield name, tensor++ extra_tensor = {}+ if hasattr(cls.model, "name_or_path"):+ from safetensors import safe_open++ from auto_round.export.export_to_gguf.special_handle import get_tensor_from_file+ from auto_round.utils import download_hf_model++ dir_path = cls.model.name_or_path+ if not os.path.isdir(dir_path):+ dir_path = download_hf_model(dir_path)+ INDEX_FILE = "model.safetensors.index.json"+ if INDEX_FILE in os.listdir(dir_path):+ with open(os.path.join(dir_path, INDEX_FILE)) as f:+ tensor_index = json.load(f)+ for tensor_name in tensor_index["weight_map"]:+ if tensor_name not in cls.model.tensor_name_list:+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)+ else:+ f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")+ for tensor_name in f.keys():+ if tensor_name not in cls.model.tensor_name_list:+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)++ for name, tensor in extra_tensor.items():yield name, tensor@@ -356,6 +396,14 @@ def prepare_tensors(cls):modify_name = _special_name_handle(cls, name)for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):+ skip = False+ for tensor_info in cls.gguf_writer.tensors:+ if new_name in tensor_info:+ print("new_name already add to gguf_writer, skip")+ skip = True+ break+ if skip:+ continuedata = data_torch.squeeze().cpu().numpy()# if data ends up empty, it means data_torch was a scalar tensor -> restore@@ -513,7 +561,7 @@ def prepare_tensors(cls):data = _quant_data_with_args(data_torch, data_qtype, **attr_list)# for MOE model- elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:+ elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:new_data = []for idx, arr in enumerate(data_torch):arr_name = name.split(".")diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.pyindex 490788d6d..f8a463e9b 100644--- a/auto_round/export/export_to_gguf/special_handle.py+++ b/auto_round/export/export_to_gguf/special_handle.py@@ -36,8 +36,9 @@ def get_tensor_from_file(dir_path, tensor_name):INDEX_FILE = "model.safetensors.index.json"# get filenameif INDEX_FILE in os.listdir(dir_path):- tensor_index = json.load(open(os.path.join(dir_path, INDEX_FILE)))- filename = tensor_index["weight_map"][tensor_name]+ with open(os.path.join(dir_path, INDEX_FILE)) as f:+ tensor_index = json.load(f)+ filename = tensor_index["weight_map"][tensor_name]else:filename = "model.safetensors"diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 9c3933610..3e25a32d3 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -26,6 +26,7 @@ SUPPORT_ONLY_TEXT_MODELS = ["idefics3","llama4","internvl_chat",+ "glm4v_moe",]NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 9e3fc4b26..d05273b69 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"+ gguf_file = os.listdir(quantized_model_path)[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"+ gguf_file = os.listdir(quantized_model_path)[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)