PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bugs of export deepseek gguf format when iters=0 and q3k accuracy (#665)
Heng Guo923683ae7a9e
diff --git a/README.md b/README.mdindex 68da05f8e..dbd6b54fc 100644--- a/README.md+++ b/README.md@@ -34,7 +34,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).## What's New-* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0).+<!-- * [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0). -->* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please referto [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) fordiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex cd27447e0..57d18ecbc 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1204,6 +1204,7 @@ class AutoRound(object):all_to_quantized_module_names.remove(m.tmp_name)mv_module_from_gpu(block, self.low_cpu_mem_usage)+ clear_memory()pbar.update(1)pbar.close()diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 669f81d22..0d07b3cc6 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -602,7 +602,8 @@ class ModelBase(OriModel):for name, tensor in self.model.named_parameters():yield name, tensor- def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):+ def _quant_data_with_args(+ self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):if torch.cuda.is_available():device = "cuda"elif torch.xpu.is_available():@@ -615,9 +616,18 @@ class ModelBase(OriModel):d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scaled_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wminwmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin+ imatrix = imatrix.to(torch.float32) if isinstance(imatrix, torch.Tensor) else imatrixdata = ggml_quant(- data_torch, data_qtype.name.lower(), scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, device=device)+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ imatrix=imatrix,+ device=device)return datadef _quant_data(self, data_torch, data_qtype, name, modify_name, bid):@@ -927,6 +937,8 @@ class ModelBase(OriModel):attr_tensor = getattr(module, attr)else:attr_tensor = getattr(module, "w_" + attr)+ if attr_tensor is None:+ continuekv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)k_b = k_b.transpose(1, 2)@@ -936,6 +948,7 @@ class ModelBase(OriModel):attr_list[attr] = k_belse:attr_list[attr] = v_b+ attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else Nonedata = self._quant_data_with_args(data_torch, data_qtype, **attr_list)# for MOE modeldiff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex e83a0c7f6..7b71bca86 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -161,7 +161,8 @@ def make_q3_quants(data, bits, do_rmse=False):L[:,:, i][replace_idx] = new_L[replace_idx]sumlx = slxsuml2 = sl2- return sumlx / suml2, L.to(torch.uint8)+ isuml2 = torch.where(suml2 == 0, 0, 1 / suml2)+ return sumlx * isuml2, L.to(torch.uint8)L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmaxscales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])diff --git a/auto_round/utils.py b/auto_round/utils.pyindex be03caeb5..2a044c825 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1311,10 +1311,20 @@ def llm_load_model(trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None)else:- model = model_cls.from_pretrained(- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None- )+ try:+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None+ )+ except OSError as e:+ logger.warning(+ f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path,+ low_cpu_mem_usage=True,+ torch_dtype=torch_dtype,+ trust_remote_code=False,+ device_map="auto" if use_auto_mapping else None)model = model.eval()model = _to_model_dtype(model, model_dtype)