PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix not using imatrix for gguf at rtn mode (#623)
Wenhua Cheng5181fa022fbc
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 1eab0d739..e34fbde4d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -464,11 +464,11 @@ class AutoRound(object):has_besides_gguf = Trueif has_gguf and has_besides_gguf:raise ValueError("gguf format is not compatible with other formats, please choose only one of them")- if has_gguf and self.iters != 0 and self.bits >= 4:+ if has_gguf and self.iters != 0 and self.bits != 3:logger.warning("We recommend setting `iters=0` when exporting to GGUF format,"" as we have optimized the RTN method for this case."- " We are likely to release new algorithms for certain configurations in the future."+ " We are likely to release new algorithm for certain configurations in the future.")##check group_size 32 for auto_rounddiff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 6cf41da65..4355bc994 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -340,13 +340,13 @@ def quant_tensor_gguf_asym_dq(5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},}- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K- if imatrix is None:- av_x = torch.sqrt(sigma2)- quant_weights = torch.abs(av_x + tensor * tensor)- else:- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ # sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ # if imatrix is None:+ # av_x = torch.sqrt(sigma2)+ # quant_weights = torch.abs(av_x + tensor * tensor)+ # else:+ # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)weights = imatrix.reshape(1, -1)weights = weights.expand(tensor.numel() // weights.numel(), -1)@@ -393,7 +393,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uReturns:Tuple: (Optimal scale tensor, optimal minimum value tensor)"""- dtype = torch.bfloat16+ dtype = torch.float32data = data.to(dtype)maxq = 2 ** bits - 1minq = 0@@ -408,7 +408,8 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uscale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)diff = scale * quant_data + rmin - data- best_mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)++ best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights*diff ** 2, dim=1, keepdim=True)for is_ in range(nstep):factor = rrmin + rdelta * is_ + maxq - minq@@ -426,7 +427,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uquant_data = torch.clamp(torch.round((1 / this_scale) * (data - this_min)), minq, maxq)diff = this_scale * quant_data + this_min - data- mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)+ mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]best_mad[idx_to_replace] = mad[idx_to_replace]@@ -494,9 +495,13 @@ def quant_tensor_gguf_sym_dq(else:imatrix = imatrix.to(tensor.device)if bits == 3:- sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ # sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ # imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ # quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)+ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)elif bits == 6:weights = imatrix.reshape(1, -1)