Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bugs of export deepseek gguf format when iters=0 and q3k accuracy (#665)

Heng Guo923683ae7a9e
diff --git a/README.md b/README.md
index 68da05f8e..dbd6b54fc 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## What's New
-* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0).
+<!-- * [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0). -->
* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)
and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) for
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index cd27447e0..57d18ecbc 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1204,6 +1204,7 @@ class AutoRound(object):
all_to_quantized_module_names.remove(m.tmp_name)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ clear_memory()
pbar.update(1)
pbar.close()
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 669f81d22..0d07b3cc6 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -602,7 +602,8 @@ class ModelBase(OriModel):
for name, tensor in self.model.named_parameters():
yield name, tensor
- def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):
+ def _quant_data_with_args(
+ self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):
if torch.cuda.is_available():
device = "cuda"
elif torch.xpu.is_available():
@@ -615,9 +616,18 @@ class ModelBase(OriModel):
d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale
d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin
wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin
+ imatrix = imatrix.to(torch.float32) if isinstance(imatrix, torch.Tensor) else imatrix
data = ggml_quant(
- data_torch, data_qtype.name.lower(), scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, device=device)
+ data_torch,
+ data_qtype.name.lower(),
+ scale,
+ zp,
+ wmin=wmin,
+ d_scale=d_scale,
+ d_wmin=d_wmin,
+ imatrix=imatrix,
+ device=device)
return data
def _quant_data(self, data_torch, data_qtype, name, modify_name, bid):
@@ -927,6 +937,8 @@ class ModelBase(OriModel):
attr_tensor = getattr(module, attr)
else:
attr_tensor = getattr(module, "w_" + attr)
+ if attr_tensor is None:
+ continue
kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
k_b = k_b.transpose(1, 2)
@@ -936,6 +948,7 @@ class ModelBase(OriModel):
attr_list[attr] = k_b
else:
attr_list[attr] = v_b
+ attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None
data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)
# for MOE model
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index e83a0c7f6..7b71bca86 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -161,7 +161,8 @@ def make_q3_quants(data, bits, do_rmse=False):
L[:,:, i][replace_idx] = new_L[replace_idx]
sumlx = slx
suml2 = sl2
- return sumlx / suml2, L.to(torch.uint8)
+ isuml2 = torch.where(suml2 == 0, 0, 1 / suml2)
+ return sumlx * isuml2, L.to(torch.uint8)
L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmax
scales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])
diff --git a/auto_round/utils.py b/auto_round/utils.py
index be03caeb5..2a044c825 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1311,10 +1311,20 @@ def llm_load_model(
trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
)
else:
- model = model_cls.from_pretrained(
- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
- trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
- )
+ try:
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
+ )
+ except OSError as e:
+ logger.warning(
+ f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path,
+ low_cpu_mem_usage=True,
+ torch_dtype=torch_dtype,
+ trust_remote_code=False,
+ device_map="auto" if use_auto_mapping else None)
model = model.eval()
model = _to_model_dtype(model, model_dtype)