Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix not using imatrix for gguf at rtn mode (#623)

Wenhua Cheng5181fa022fbc
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 1eab0d739..e34fbde4d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -464,11 +464,11 @@ class AutoRound(object):
has_besides_gguf = True
if has_gguf and has_besides_gguf:
raise ValueError("gguf format is not compatible with other formats, please choose only one of them")
- if has_gguf and self.iters != 0 and self.bits >= 4:
+ if has_gguf and self.iters != 0 and self.bits != 3:
logger.warning(
"We recommend setting `iters=0` when exporting to GGUF format,"
" as we have optimized the RTN method for this case."
- " We are likely to release new algorithms for certain configurations in the future."
+ " We are likely to release new algorithm for certain configurations in the future."
)
##check group_size 32 for auto_round
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 6cf41da65..4355bc994 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -340,13 +340,13 @@ def quant_tensor_gguf_asym_dq(
5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},
}
- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
- if imatrix is None:
- av_x = torch.sqrt(sigma2)
- quant_weights = torch.abs(av_x + tensor * tensor)
- else:
- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
+ # sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+ # if imatrix is None:
+ # av_x = torch.sqrt(sigma2)
+ # quant_weights = torch.abs(av_x + tensor * tensor)
+ # else:
+ # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
+ # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
weights = imatrix.reshape(1, -1)
weights = weights.expand(tensor.numel() // weights.numel(), -1)
@@ -393,7 +393,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
Returns:
Tuple: (Optimal scale tensor, optimal minimum value tensor)
"""
- dtype = torch.bfloat16
+ dtype = torch.float32
data = data.to(dtype)
maxq = 2 ** bits - 1
minq = 0
@@ -408,7 +408,8 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))
quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)
diff = scale * quant_data + rmin - data
- best_mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)
+
+ best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights*diff ** 2, dim=1, keepdim=True)
for is_ in range(nstep):
factor = rrmin + rdelta * is_ + maxq - minq
@@ -426,7 +427,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
quant_data = torch.clamp(torch.round((1 / this_scale) * (data - this_min)), minq, maxq)
diff = this_scale * quant_data + this_min - data
- mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)
+ mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)
idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]
best_mad[idx_to_replace] = mad[idx_to_replace]
@@ -494,9 +495,13 @@ def quant_tensor_gguf_sym_dq(
else:
imatrix = imatrix.to(tensor.device)
if bits == 3:
- sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
+ # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+ # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
+ # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
+ # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
+ weights = imatrix.reshape(1, -1)
+ weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ quant_weights = weights.reshape(tensor.shape)
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
elif bits == 6:
weights = imatrix.reshape(1, -1)