PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine gguf rtn algorithm and fix bugs (#630)
Wenhua Chengef6413b2c700
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex e34fbde4d..fc3f74b80 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -774,8 +774,10 @@ class AutoRound(object):act_bits = 16scale_dtype = self.scale_dtypekeys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]- self.layer_config[embedding_name] = {}+ self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})for key in keys:+ if key in self.layer_config[embedding_name]:+ continueself.layer_config[embedding_name][key] = getattr(config, "get")(key)setattr(get_module(self.model, embedding_name), key, config.get(key))self.layer_config[embedding_name]["act_bits"] = act_bits@@ -792,7 +794,10 @@ class AutoRound(object):act_bits = 16scale_dtype = self.scale_dtypekeys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]+ self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})for key in keys:+ if key in self.layer_config[lm_head_name]:+ continueself.layer_config[lm_head_name][key] = getattr(config, "get")(key)setattr(get_module(self.model, lm_head_name), key, config.get(key))self.layer_config[lm_head_name]["act_bits"] = act_bits@@ -1072,7 +1077,6 @@ class AutoRound(object):"""# Get the names of layers in quantization blockslayers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)-##process regex in layer_configall_supported_layer_names = []# List of configuration keys@@ -1098,7 +1102,9 @@ class AutoRound(object):delattr(m, key)# Skip unsupported types- if not isinstance(m, tuple(self.supported_types)):+ supported_types = self.supported_types++ if not isinstance(m,supported_types):continueall_supported_layer_names.append(n)@@ -1116,7 +1122,9 @@ class AutoRound(object):for match_name in matched_names:layer_config[match_name] = valelse:- raise ValueError(f"key {name} in layer_config is invalid, please have a double check")+ tmp_m = get_module(self.model,name)+ if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)@@ -1124,7 +1132,7 @@ class AutoRound(object):for n, m in self.model.named_modules():# Skip unsupported types- if not isinstance(m, tuple(self.supported_types)):+ if not isinstance(m, supported_types):continue# If the layer is not in the config and is part of a quantization block, use default configuration@@ -1152,7 +1160,8 @@ class AutoRound(object):layer_config[n]["in_blocks"] = False# If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block- if n not in layers_in_blocks and check_to_quantized(layer_config[n]):+ if (n not in layers_in_blocks and check_to_quantized(layer_config[n])+ and not isinstance(m,torch.nn.Embedding)):has_qlayer_outside_block = Truein_features, out_features = get_layer_features(m)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 4355bc994..0ab57d9b9 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -16,6 +16,7 @@ import torchfrom auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.data_type.register import register_dtype+@register_dtype("int_sym_dq")def quant_tensor_sym_dq(tensor,@@ -149,8 +150,8 @@ def double_quant_tensor_sym(tensor, bits):imax = abs(tensor).argmax(axis=-1, keepdims=True)wmax = torch.take_along_dim(tensor, imax, dim=-1)scale = wmax / -maxq- inverse_scale = torch.where(scale == 0, 0, 1 / scale)- qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)),-maxq, maxq - 1) * scale+ inverse_scale = torch.where(scale == 0, 0, 1 / scale) ##1e-40+ qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)), -maxq, maxq - 1) * scalereturn qdq_tensor, scale@@ -169,7 +170,7 @@ def make_qp_quants(nmax, data, quant_weights):if _is == 0:continuescale_is = group_max / (0.1 * _is + nmax)- iscale_is = torch.where(scale == 0, 0, 1 / scale_is)+ iscale_is = torch.where(scale_is == 0, 0, 1 / scale_is)tmp_L = torch.round(iscale_is * data).clip(max=nmax)diffs = data - scale_is * tmp_L@@ -182,25 +183,25 @@ def make_qp_quants(nmax, data, quant_weights):L = torch.round(iscale * data).clip(max=nmax)sumlx = torch.sum(quant_weights * data * L, dim=-1)suml2 = torch.sum(quant_weights * L * L, dim=-1)-- for _ in range(5):- n_changed = 0- for i in range(data.shape[-1]):- slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]- sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]- replace_idx = (slx > 0) & (sl2 > 0)- new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)- replace_idx &= new_L != L[:, i]- slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]- sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]-- replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2- L[:, i][replace_idx] = new_L[replace_idx]- sumlx[replace_idx] = slx[replace_idx]- suml2[replace_idx] = sl2[replace_idx]- n_changed = replace_idx.sum()- if n_changed == 0:- break+ #+ # for _ in range(5):+ # n_changed = 0+ # for i in range(data.shape[-1]):+ # slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]+ # sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]+ # replace_idx = (slx > 0) & (sl2 > 0)+ # new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)+ # replace_idx &= new_L != L[:, i]+ # slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]+ # sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]+ #+ # replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2+ # L[:, i][replace_idx] = new_L[replace_idx]+ # sumlx[replace_idx] = slx[replace_idx]+ # suml2[replace_idx] = sl2[replace_idx]+ # n_changed = replace_idx.sum()+ # if n_changed == 0:+ # breakreturn sumlx / suml2, L@@ -304,8 +305,9 @@ def quant_tensor_gguf_asym_dq("""orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor =tensor.to(torch.float32)if bits not in [2, 4, 5]:- raise ValueError(f"bits={bits} not supported by gguf_int_asym_dq")+ raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")maxq = 2 ** bits - 1QK_K = 256quant_weights = None@@ -326,9 +328,12 @@ def quant_tensor_gguf_asym_dq(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],use_mad=params["use_mad"], weights=quant_weights)+ scale = scale.to(scale_dtype)+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)scale = scale.reshape(-1, super_group_size)wmin = wmin_m.reshape(-1, super_group_size)scale, d_scale = double_quant_tensor(scale, super_bits)+ wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)wmin_m = wmin_m.view(-1, 1)scale = scale.view(-1, 1)@@ -358,7 +363,8 @@ def quant_tensor_gguf_asym_dq(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],use_mad=params["use_mad"], weights=quant_weights)-+ scale = scale.to(scale_dtype)+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)nmax = 2 ** super_bits - 1scale = scale.reshape(-1, super_group_size)wmin_m = wmin_m_0.reshape(-1, super_group_size)@@ -379,7 +385,7 @@ def quant_tensor_gguf_asym_dq(def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None):- """Performs iterative weighted least squares quantization search.+ """Adapted from Llamacpp. Performs iterative weighted least squares quantization search.Args:data (torch.Tensor): Input tensor to quantize.@@ -409,7 +415,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uquant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)diff = scale * quant_data + rmin - data- best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights*diff ** 2, dim=1, keepdim=True)+ best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)for is_ in range(nstep):factor = rrmin + rdelta * is_ + maxq - minq@@ -488,7 +494,7 @@ def quant_tensor_gguf_sym_dq(tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):if bits == 3:- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)elif bits == 6:scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)@@ -508,7 +514,7 @@ def quant_tensor_gguf_sym_dq(weights = weights.expand(tensor.numel() // weights.numel(), -1)quant_weights = weights.reshape(tensor.shape)scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)-+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)# conduct double quantscale, d_scale = double_quant_tensor_sym(scale, super_bits)@@ -520,5 +526,3 @@ def quant_tensor_gguf_sym_dq(qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, {"scale": scale, "d_scale": d_scale}, zp--diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex f45244e67..c5633e996 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -59,6 +59,9 @@ def torch_roundf(n):def make_qx_quants(data, bits, rmse_type=0, qw=None):+ """+ adapted from llmacpp+ """nmax = pow(2, bits - 1)imax = abs(data).argmax(axis=-1, keepdims=True)group_max = torch.take_along_dim(data, imax, dim=-1)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 326529201..4a2781a82 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1168,6 +1168,8 @@ def get_layer_features(layer):return layer.in_features, layer.out_featureselif isinstance(layer, transformers.pytorch_utils.Conv1D): # TODO: Verify correctnessreturn layer.weight.shape[0], layer.weight.shape[1]+ elif isinstance(layer, torch.nn.Embedding):+ return layer.num_embeddings, layer.embedding_dimreturn None, None # Unsupported layer typediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex dd8856b5a..54f3a4565 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -78,7 +78,10 @@ class WrapperLinear(torch.nn.Module):self.enable_round_tuning = enable_round_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8- self.q_scale_thresh = 1e-5+ if (hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32):+ self.q_scale_thresh = 1e-8+ else:+ self.q_scale_thresh = 1e-5self._init_tuning_params_and_quant_func()self.orig_forward = self.linear_forward if isinstance(self.orig_layer, torch.nn.Linear) else self.conv1d_forward