Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

handle zeros in imatrix (#667)

Wenhua Chengc1ea0d985389
diff --git a/README.md b/README.md
index dbd6b54fc..7e520b1f3 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,11 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## What's New
-<!-- * [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0). -->
+* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for
+ all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in
+ v0.6.1. Example
+ models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v1](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v1)
+ and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v1](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v1).
* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)
and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) for
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 57d18ecbc..fabc1ef2e 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -261,7 +261,6 @@ class AutoRound(object):
self.disable_opt_rtn = disable_opt_rtn
-
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
@@ -454,7 +453,6 @@ class AutoRound(object):
self._dq_check()
def _check_compatibility(self):
-
##check gguf and others
has_gguf = False
if hasattr(self, "formats"):
@@ -605,7 +603,7 @@ class AutoRound(object):
if format == "llmcompressor":
bits, group_size, sym, act_bits = 8, -1, True, 8
assert self.bits == bits and self.group_size == group_size and self.sym == sym \
- and self.act_bits == act_bits and self.act_dynamic, \
+ and self.act_bits == act_bits and self.act_dynamic, \
f"Currently only support to export llmcompressor format for dynamic quantized" \
f" W{self.bits}A{self.act_bits} model, but got bits={self.bits}," \
f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"
@@ -846,7 +844,7 @@ class AutoRound(object):
try:
# Move model to target device
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 :
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
from accelerate.big_modeling import dispatch_model
dispatch_model(self.model, self.model.hf_device_map)
@@ -876,9 +874,14 @@ class AutoRound(object):
# Perform quantization using RTN
from tqdm import tqdm
pbar = tqdm(all_to_quantized_module_names)
+ cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
self.quantize_layer_via_rtn(name)
+ if self.low_gpu_mem_usage and cnt % 10 == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
except RuntimeError as e:
try:
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
@@ -891,7 +894,8 @@ class AutoRound(object):
self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
except Exception:
# Final fallback: warn and use CPU-only quantization
- logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
+ logger.warning("Fallback to CPU. "
+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")
model = model.to("cpu")
clear_memory()
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
@@ -946,7 +950,7 @@ class AutoRound(object):
if isinstance(module, torch.nn.Embedding):
key: str = "lm_head" if tie_word_embeddings else "embedding"
config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]
- self._apply_config_to_layer(name, config,True)
+ self._apply_config_to_layer(name, config, True)
if not tie_word_embeddings:
lm_head_name: str = get_lm_head_name(self.model)
@@ -1023,7 +1027,7 @@ class AutoRound(object):
# Step 2: Try quantization on GPU first, fall back to CPU if OOM
# if only export gguf, using gguf-packing instead of rtn
- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0]:
+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and self.disable_opt_rtn:
m.scale = None
m.zp = None
else:
@@ -1051,9 +1055,6 @@ class AutoRound(object):
else:
raise
- if self.low_gpu_mem_usage:
- clear_memory()
-
# Step 3: Optional immediate packing/export
if self.is_packing_immediate:
from auto_round.export import PACKING_LAYER_WITH_FORMAT
@@ -1081,8 +1082,8 @@ class AutoRound(object):
name, self.model, self.formats[0]
)
- if self.low_gpu_mem_usage:
- clear_memory()
+ # if self.low_gpu_mem_usage:
+ # clear_memory()
else:
set_module(self.model, name, m)
@@ -1112,9 +1113,14 @@ class AutoRound(object):
self.quant_rtn_with_imatrix(all_to_quantized_module_names)
else:
pbar = tqdm(all_to_quantized_module_names)
+ cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
self.quantize_layer_via_rtn(name)
+ if self.low_gpu_mem_usage and cnt % 10 == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
self.quantized = True
return self.model, self.layer_config
@@ -1208,11 +1214,15 @@ class AutoRound(object):
pbar.update(1)
pbar.close()
+ cnt = 1
# Process remaining layers not in blocks
for name in all_to_quantized_module_names:
self.quantize_layer_via_rtn(name)
-
+ if self.low_gpu_mem_usage and cnt % 10 == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
@@ -1960,7 +1970,6 @@ class AutoRound(object):
unwrapper_layer(self.model, wrapper_linear, layer_name, {})
mv_module_from_gpu(layer, self.low_cpu_mem_usage)
-
if self.enable_minmax_tuning:
optimizer = self.optimizer(
[{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0
@@ -2150,7 +2159,7 @@ class AutoRound(object):
f"layers in the block"
)
logger.info(dump_info)
- unwrapper_block(block, {}) ## TODO Quant layer should change
+ unwrapper_block(block, {}) ## TODO Quant layer should change
mv_module_from_gpu(block, self.low_cpu_mem_usage)
return output, output
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 2d26c97ca..ae016c914 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -13,7 +13,7 @@
# limitations under the License.
import torch
-from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger
from auto_round.data_type.register import register_dtype
@@ -263,7 +263,6 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_
qdq_result = (scale * q - wmin).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
- # zp = round_ste(wmin / scale) # remove this later
return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
@@ -303,11 +302,11 @@ def quant_tensor_gguf_asym_dq(
"""
orig_dtype = tensor.dtype
maxq = 2 ** bits - 1
- group_size = 16 if bits ==2 else 32
- super_bits = 4 if bits == 2 else 6
+ group_size = 16 if bits == 2 else 32
+ super_bits = 4 if bits == 2 else 6
super_group_size = 16 if bits == 2 else 8
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- tensor =tensor.to(torch.float32)
+ tensor = tensor.to(torch.float32)
if bits not in [2, 4, 5]:
raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")
quant_weights = None
@@ -345,6 +344,33 @@ def quant_tensor_gguf_asym_dq(
5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},
}
+ weights = imatrix.reshape(1, -1)
+
+ weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ quant_weights = weights.reshape(tensor.shape)
+
+ if torch.min(quant_weights) == 0:
+ logger.warning_once(
+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")
+
+ zero_cnt = torch.sum(quant_weights == 0, dim=-1)
+ replace_index = zero_cnt > group_size // 2
+ if torch.sum(replace_index) > 0:
+ ## fallback to no imatrix
+ if bits == 2:
+ tmp_quant_weights = torch.abs(tensor)
+ elif bits == 4 or bits == 5:
+ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ## Note 32 is different from QK_K
+ av_x = torch.sqrt(sigma2)
+ tmp_quant_weights = torch.abs(tensor) + av_x
+ quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]
+ mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)
+ if torch.sum(mean_replace_index) > 0:
+ ## use mean values to fill zero values
+ tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[1] - zero_cnt)
+ tmp_quant_weights = tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1])
+ quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]
+
# sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
# if imatrix is None:
# av_x = torch.sqrt(sigma2)
@@ -353,10 +379,6 @@ def quant_tensor_gguf_asym_dq(
# imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
# quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
- weights = imatrix.reshape(1, -1)
- weights = weights.expand(tensor.numel() // weights.numel(), -1)
- quant_weights = weights.reshape(tensor.shape)
-
params = search_kwargs[bits]
scale, wmin_0 = iterative_wls_quant_search(
@@ -372,7 +394,7 @@ def quant_tensor_gguf_asym_dq(
d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)
d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)
-
+
d_scale = d_scale.unsqueeze(-1)
d_wmin = d_wmin.unsqueeze(-1)
scale = (d_scale * q_scale).view(-1, 1)
@@ -488,7 +510,7 @@ def quant_tensor_gguf_sym_dq(
if bits not in [3, 6]:
raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")
-
+
maxq = 2 ** (bits - 1)
group_size = 16
super_bits = 6 if bits == 3 else 8
@@ -503,28 +525,50 @@ def quant_tensor_gguf_sym_dq(
n_blocks = tensor.nelement() // block_size
# (nb, 16, 16)
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+
if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
if bits == 3:
- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug
+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)
##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
elif bits == 6:
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
else:
imatrix = imatrix.to(tensor.device)
- if bits == 3:
- # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
- # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
- # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
- # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
- weights = imatrix.reshape(1, -1)
- weights = weights.expand(tensor.numel() // weights.numel(), -1)
- quant_weights = weights.reshape(tensor.shape)
- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
- elif bits == 6:
- weights = imatrix.reshape(1, -1)
- weights = weights.expand(tensor.numel() // weights.numel(), -1)
- quant_weights = weights.reshape(tensor.shape)
- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
+
+ # if bits == 3:
+ # # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+ # # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
+ # # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
+ # # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
+ # weights = imatrix.reshape(1, -1)
+ # weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ # quant_weights = weights.reshape(tensor.shape)
+ # elif bits == 6:
+
+ weights = imatrix.reshape(1, -1)
+ weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ quant_weights = weights.reshape(tensor.shape)
+ if torch.min(quant_weights) == 0:
+ logger.warning_once(
+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")
+
+ zero_cnt = torch.sum(quant_weights == 0, dim=-1)
+ replace_index = zero_cnt > group_size // 2
+ if torch.sum(replace_index) > 0:
+ if bits == 6:
+ quant_weights[replace_index, :] = 1.0
+ else:
+ sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+ tmp_quant_weights = torch.sqrt(sigma2 + tensor * tensor)
+ quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]
+ mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)
+ if torch.sum(mean_replace_index) > 0:
+ ## use mean values to fill zero values
+ tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[1] - zero_cnt)
+ tmp_quant_weights = tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1])
+ quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]
+
+ scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
# conduct double quant
scale, d_scale = double_quant_tensor_sym(scale, super_bits)