PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
handle zeros in imatrix (#667)
Wenhua Chengc1ea0d985389
diff --git a/README.md b/README.mdindex dbd6b54fc..7e520b1f3 100644--- a/README.md+++ b/README.md@@ -34,7 +34,11 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).## What's New-<!-- * [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0). -->+* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for+ all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in+ v0.6.1. Example+ models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v1](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v1)+ and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v1](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v1).* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please referto [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) fordiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 57d18ecbc..fabc1ef2e 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -261,7 +261,6 @@ class AutoRound(object):self.disable_opt_rtn = disable_opt_rtn-torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()if self.act_bits <= 8 and self.amp_dtype == torch.float16:@@ -454,7 +453,6 @@ class AutoRound(object):self._dq_check()def _check_compatibility(self):-##check gguf and othershas_gguf = Falseif hasattr(self, "formats"):@@ -605,7 +603,7 @@ class AutoRound(object):if format == "llmcompressor":bits, group_size, sym, act_bits = 8, -1, True, 8assert self.bits == bits and self.group_size == group_size and self.sym == sym \- and self.act_bits == act_bits and self.act_dynamic, \+ and self.act_bits == act_bits and self.act_dynamic, \f"Currently only support to export llmcompressor format for dynamic quantized" \f" W{self.bits}A{self.act_bits} model, but got bits={self.bits}," \f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"@@ -846,7 +844,7 @@ class AutoRound(object):try:# Move model to target device- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 :+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_modeldispatch_model(self.model, self.model.hf_device_map)@@ -876,9 +874,14 @@ class AutoRound(object):# Perform quantization using RTNfrom tqdm import tqdmpbar = tqdm(all_to_quantized_module_names)+ cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)+ if self.low_gpu_mem_usage and cnt % 10 == 0:+ clear_memory()+ cnt = 1+ cnt += 1except RuntimeError as e:try:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:@@ -891,7 +894,8 @@ class AutoRound(object):self.quantize_via_rtn_blockwise(all_to_quantized_module_names)except Exception:# Final fallback: warn and use CPU-only quantization- logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")+ logger.warning("Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")model = model.to("cpu")clear_memory()if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:@@ -946,7 +950,7 @@ class AutoRound(object):if isinstance(module, torch.nn.Embedding):key: str = "lm_head" if tie_word_embeddings else "embedding"config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]- self._apply_config_to_layer(name, config,True)+ self._apply_config_to_layer(name, config, True)if not tie_word_embeddings:lm_head_name: str = get_lm_head_name(self.model)@@ -1023,7 +1027,7 @@ class AutoRound(object):# Step 2: Try quantization on GPU first, fall back to CPU if OOM# if only export gguf, using gguf-packing instead of rtn- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0]:+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and self.disable_opt_rtn:m.scale = Nonem.zp = Noneelse:@@ -1051,9 +1055,6 @@ class AutoRound(object):else:raise- if self.low_gpu_mem_usage:- clear_memory()-# Step 3: Optional immediate packing/exportif self.is_packing_immediate:from auto_round.export import PACKING_LAYER_WITH_FORMAT@@ -1081,8 +1082,8 @@ class AutoRound(object):name, self.model, self.formats[0])- if self.low_gpu_mem_usage:- clear_memory()+ # if self.low_gpu_mem_usage:+ # clear_memory()else:set_module(self.model, name, m)@@ -1112,9 +1113,14 @@ class AutoRound(object):self.quant_rtn_with_imatrix(all_to_quantized_module_names)else:pbar = tqdm(all_to_quantized_module_names)+ cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)+ if self.low_gpu_mem_usage and cnt % 10 == 0:+ clear_memory()+ cnt = 1+ cnt += 1self.quantized = Truereturn self.model, self.layer_config@@ -1208,11 +1214,15 @@ class AutoRound(object):pbar.update(1)pbar.close()+ cnt = 1# Process remaining layers not in blocksfor name in all_to_quantized_module_names:self.quantize_layer_via_rtn(name)-+ if self.low_gpu_mem_usage and cnt % 10 == 0:+ clear_memory()+ cnt = 1+ cnt += 1def quantize(self):"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.@@ -1960,7 +1970,6 @@ class AutoRound(object):unwrapper_layer(self.model, wrapper_linear, layer_name, {})mv_module_from_gpu(layer, self.low_cpu_mem_usage)-if self.enable_minmax_tuning:optimizer = self.optimizer([{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0@@ -2150,7 +2159,7 @@ class AutoRound(object):f"layers in the block")logger.info(dump_info)- unwrapper_block(block, {}) ## TODO Quant layer should change+ unwrapper_block(block, {}) ## TODO Quant layer should changemv_module_from_gpu(block, self.low_cpu_mem_usage)return output, outputdiff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 2d26c97ca..ae016c914 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -13,7 +13,7 @@# limitations under the License.import torch-from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, loggerfrom auto_round.data_type.register import register_dtype@@ -263,7 +263,6 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_qdq_result = (scale * q - wmin).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- # zp = round_ste(wmin / scale) # remove this laterreturn qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}@@ -303,11 +302,11 @@ def quant_tensor_gguf_asym_dq("""orig_dtype = tensor.dtypemaxq = 2 ** bits - 1- group_size = 16 if bits ==2 else 32- super_bits = 4 if bits == 2 else 6+ group_size = 16 if bits == 2 else 32+ super_bits = 4 if bits == 2 else 6super_group_size = 16 if bits == 2 else 8tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- tensor =tensor.to(torch.float32)+ tensor = tensor.to(torch.float32)if bits not in [2, 4, 5]:raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")quant_weights = None@@ -345,6 +344,33 @@ def quant_tensor_gguf_asym_dq(5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},}+ weights = imatrix.reshape(1, -1)++ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)++ if torch.min(quant_weights) == 0:+ logger.warning_once(+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")++ zero_cnt = torch.sum(quant_weights == 0, dim=-1)+ replace_index = zero_cnt > group_size // 2+ if torch.sum(replace_index) > 0:+ ## fallback to no imatrix+ if bits == 2:+ tmp_quant_weights = torch.abs(tensor)+ elif bits == 4 or bits == 5:+ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ## Note 32 is different from QK_K+ av_x = torch.sqrt(sigma2)+ tmp_quant_weights = torch.abs(tensor) + av_x+ quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]+ mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)+ if torch.sum(mean_replace_index) > 0:+ ## use mean values to fill zero values+ tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[1] - zero_cnt)+ tmp_quant_weights = tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1])+ quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]+# sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K# if imatrix is None:# av_x = torch.sqrt(sigma2)@@ -353,10 +379,6 @@ def quant_tensor_gguf_asym_dq(# imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)# quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)-params = search_kwargs[bits]scale, wmin_0 = iterative_wls_quant_search(@@ -372,7 +394,7 @@ def quant_tensor_gguf_asym_dq(d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)-+d_scale = d_scale.unsqueeze(-1)d_wmin = d_wmin.unsqueeze(-1)scale = (d_scale * q_scale).view(-1, 1)@@ -488,7 +510,7 @@ def quant_tensor_gguf_sym_dq(if bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")-+maxq = 2 ** (bits - 1)group_size = 16super_bits = 6 if bits == 3 else 8@@ -503,28 +525,50 @@ def quant_tensor_gguf_sym_dq(n_blocks = tensor.nelement() // block_size# (nb, 16, 16)tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):if bits == 3:- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)elif bits == 6:scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)else:imatrix = imatrix.to(tensor.device)- if bits == 3:- # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K- # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)- # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)- # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)- elif bits == 6:- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)++ # if bits == 3:+ # # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ # # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ # # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ # # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)+ # weights = imatrix.reshape(1, -1)+ # weights = weights.expand(tensor.numel() // weights.numel(), -1)+ # quant_weights = weights.reshape(tensor.shape)+ # elif bits == 6:++ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)+ if torch.min(quant_weights) == 0:+ logger.warning_once(+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")++ zero_cnt = torch.sum(quant_weights == 0, dim=-1)+ replace_index = zero_cnt > group_size // 2+ if torch.sum(replace_index) > 0:+ if bits == 6:+ quant_weights[replace_index, :] = 1.0+ else:+ sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ tmp_quant_weights = torch.sqrt(sigma2 + tensor * tensor)+ quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]+ mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)+ if torch.sum(mean_replace_index) > 0:+ ## use mean values to fill zero values+ tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[1] - zero_cnt)+ tmp_quant_weights = tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1])+ quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]++ scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)# conduct double quantscale, d_scale = double_quant_tensor_sym(scale, super_bits)