Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine gguf rtn algorithm and fix bugs (#630)

Wenhua Chengef6413b2c700
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index e34fbde4d..fc3f74b80 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -774,8 +774,10 @@ class AutoRound(object):
act_bits = 16
scale_dtype = self.scale_dtype
keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
- self.layer_config[embedding_name] = {}
+ self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})
for key in keys:
+ if key in self.layer_config[embedding_name]:
+ continue
self.layer_config[embedding_name][key] = getattr(config, "get")(key)
setattr(get_module(self.model, embedding_name), key, config.get(key))
self.layer_config[embedding_name]["act_bits"] = act_bits
@@ -792,7 +794,10 @@ class AutoRound(object):
act_bits = 16
scale_dtype = self.scale_dtype
keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
+ self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})
for key in keys:
+ if key in self.layer_config[lm_head_name]:
+ continue
self.layer_config[lm_head_name][key] = getattr(config, "get")(key)
setattr(get_module(self.model, lm_head_name), key, config.get(key))
self.layer_config[lm_head_name]["act_bits"] = act_bits
@@ -1072,7 +1077,6 @@ class AutoRound(object):
"""
# Get the names of layers in quantization blocks
layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
-
##process regex in layer_config
all_supported_layer_names = []
# List of configuration keys
@@ -1098,7 +1102,9 @@ class AutoRound(object):
delattr(m, key)
# Skip unsupported types
- if not isinstance(m, tuple(self.supported_types)):
+ supported_types = self.supported_types
+
+ if not isinstance(m,supported_types):
continue
all_supported_layer_names.append(n)
@@ -1116,7 +1122,9 @@ class AutoRound(object):
for match_name in matched_names:
layer_config[match_name] = val
else:
- raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
+ tmp_m = get_module(self.model,name)
+ if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style
+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
@@ -1124,7 +1132,7 @@ class AutoRound(object):
for n, m in self.model.named_modules():
# Skip unsupported types
- if not isinstance(m, tuple(self.supported_types)):
+ if not isinstance(m, supported_types):
continue
# If the layer is not in the config and is part of a quantization block, use default configuration
@@ -1152,7 +1160,8 @@ class AutoRound(object):
layer_config[n]["in_blocks"] = False
# If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block
- if n not in layers_in_blocks and check_to_quantized(layer_config[n]):
+ if (n not in layers_in_blocks and check_to_quantized(layer_config[n])
+ and not isinstance(m,torch.nn.Embedding)):
has_qlayer_outside_block = True
in_features, out_features = get_layer_features(m)
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 4355bc994..0ab57d9b9 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -16,6 +16,7 @@ import torch
from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.data_type.register import register_dtype
+
@register_dtype("int_sym_dq")
def quant_tensor_sym_dq(
tensor,
@@ -149,8 +150,8 @@ def double_quant_tensor_sym(tensor, bits):
imax = abs(tensor).argmax(axis=-1, keepdims=True)
wmax = torch.take_along_dim(tensor, imax, dim=-1)
scale = wmax / -maxq
- inverse_scale = torch.where(scale == 0, 0, 1 / scale)
- qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)),-maxq, maxq - 1) * scale
+ inverse_scale = torch.where(scale == 0, 0, 1 / scale) ##1e-40
+ qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)), -maxq, maxq - 1) * scale
return qdq_tensor, scale
@@ -169,7 +170,7 @@ def make_qp_quants(nmax, data, quant_weights):
if _is == 0:
continue
scale_is = group_max / (0.1 * _is + nmax)
- iscale_is = torch.where(scale == 0, 0, 1 / scale_is)
+ iscale_is = torch.where(scale_is == 0, 0, 1 / scale_is)
tmp_L = torch.round(iscale_is * data).clip(max=nmax)
diffs = data - scale_is * tmp_L
@@ -182,25 +183,25 @@ def make_qp_quants(nmax, data, quant_weights):
L = torch.round(iscale * data).clip(max=nmax)
sumlx = torch.sum(quant_weights * data * L, dim=-1)
suml2 = torch.sum(quant_weights * L * L, dim=-1)
-
- for _ in range(5):
- n_changed = 0
- for i in range(data.shape[-1]):
- slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]
- sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]
- replace_idx = (slx > 0) & (sl2 > 0)
- new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)
- replace_idx &= new_L != L[:, i]
- slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]
- sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]
-
- replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2
- L[:, i][replace_idx] = new_L[replace_idx]
- sumlx[replace_idx] = slx[replace_idx]
- suml2[replace_idx] = sl2[replace_idx]
- n_changed = replace_idx.sum()
- if n_changed == 0:
- break
+ #
+ # for _ in range(5):
+ # n_changed = 0
+ # for i in range(data.shape[-1]):
+ # slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]
+ # sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]
+ # replace_idx = (slx > 0) & (sl2 > 0)
+ # new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)
+ # replace_idx &= new_L != L[:, i]
+ # slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]
+ # sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]
+ #
+ # replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2
+ # L[:, i][replace_idx] = new_L[replace_idx]
+ # sumlx[replace_idx] = slx[replace_idx]
+ # suml2[replace_idx] = sl2[replace_idx]
+ # n_changed = replace_idx.sum()
+ # if n_changed == 0:
+ # break
return sumlx / suml2, L
@@ -304,8 +305,9 @@ def quant_tensor_gguf_asym_dq(
"""
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ tensor =tensor.to(torch.float32)
if bits not in [2, 4, 5]:
- raise ValueError(f"bits={bits} not supported by gguf_int_asym_dq")
+ raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")
maxq = 2 ** bits - 1
QK_K = 256
quant_weights = None
@@ -326,9 +328,12 @@ def quant_tensor_gguf_asym_dq(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
use_mad=params["use_mad"], weights=quant_weights
)
+ scale = scale.to(scale_dtype)
+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
scale = scale.reshape(-1, super_group_size)
wmin = wmin_m.reshape(-1, super_group_size)
scale, d_scale = double_quant_tensor(scale, super_bits)
+ wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)
wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)
wmin_m = wmin_m.view(-1, 1)
scale = scale.view(-1, 1)
@@ -358,7 +363,8 @@ def quant_tensor_gguf_asym_dq(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
use_mad=params["use_mad"], weights=quant_weights
)
-
+ scale = scale.to(scale_dtype)
+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
nmax = 2 ** super_bits - 1
scale = scale.reshape(-1, super_group_size)
wmin_m = wmin_m_0.reshape(-1, super_group_size)
@@ -379,7 +385,7 @@ def quant_tensor_gguf_asym_dq(
def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None):
- """Performs iterative weighted least squares quantization search.
+ """Adapted from Llamacpp. Performs iterative weighted least squares quantization search.
Args:
data (torch.Tensor): Input tensor to quantize.
@@ -409,7 +415,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)
diff = scale * quant_data + rmin - data
- best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights*diff ** 2, dim=1, keepdim=True)
+ best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)
for is_ in range(nstep):
factor = rrmin + rdelta * is_ + maxq - minq
@@ -488,7 +494,7 @@ def quant_tensor_gguf_sym_dq(
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
if bits == 3:
- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug
+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug
##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
elif bits == 6:
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
@@ -508,7 +514,7 @@ def quant_tensor_gguf_sym_dq(
weights = weights.expand(tensor.numel() // weights.numel(), -1)
quant_weights = weights.reshape(tensor.shape)
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
-
+ scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
# conduct double quant
scale, d_scale = double_quant_tensor_sym(scale, super_bits)
@@ -520,5 +526,3 @@ def quant_tensor_gguf_sym_dq(
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
return qdq_result, {"scale": scale, "d_scale": d_scale}, zp
-
-
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index f45244e67..c5633e996 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -59,6 +59,9 @@ def torch_roundf(n):
def make_qx_quants(data, bits, rmse_type=0, qw=None):
+ """
+ adapted from llmacpp
+ """
nmax = pow(2, bits - 1)
imax = abs(data).argmax(axis=-1, keepdims=True)
group_max = torch.take_along_dim(data, imax, dim=-1)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 326529201..4a2781a82 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1168,6 +1168,8 @@ def get_layer_features(layer):
return layer.in_features, layer.out_features
elif isinstance(layer, transformers.pytorch_utils.Conv1D): # TODO: Verify correctness
return layer.weight.shape[0], layer.weight.shape[1]
+ elif isinstance(layer, torch.nn.Embedding):
+ return layer.num_embeddings, layer.embedding_dim
return None, None # Unsupported layer type
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index dd8856b5a..54f3a4565 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -78,7 +78,10 @@ class WrapperLinear(torch.nn.Module):
self.enable_round_tuning = enable_round_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
- self.q_scale_thresh = 1e-5
+ if (hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32):
+ self.q_scale_thresh = 1e-8
+ else:
+ self.q_scale_thresh = 1e-5
self._init_tuning_params_and_quant_func()
self.orig_forward = self.linear_forward if isinstance(self.orig_layer, torch.nn.Linear) else self.conv1d_forward