Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

change the scale thresh generally (#229)

WeiweiZhang1b2f160c260ce
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 545eebd0a..d39bf5224 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -59,8 +59,6 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
wmax[tmp] = +1
scale = ((wmax - wmin) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
- if (scale == 0.).any():
- scale = torch.clamp(scale, min=1e-5)
zp = round_ste(-wmin / scale) # pylint: disable=E1130
scale = scale.unsqueeze(dim=-1)
zp = zp.unsqueeze(dim=-1)
@@ -117,8 +115,6 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
wmax_new[tmp] = +1
scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
- if (scale == 0.).any():
- scale = torch.clamp(scale, min=1e-5)
scale = scale.unsqueeze(dim=-1)
zp = torch.full_like(scale, (maxq + 1) / 2)
@@ -175,3 +171,4 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0
q = torch.clamp(int_w + zp, 0, maxq)
return scale * (q - zp), scale, zp
+
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 58a3858d0..481cce89a 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -46,7 +46,7 @@ def reshape_tensor(v, group_size=-1):
def quant_tensor(
quant_func, data, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs,
+ weight_min=None, weight_max=None, q_scale_thresh=1e-5, **kwargs,
):
"""Quantizes and dequantizes weight, handing the group size issue .
@@ -126,6 +126,7 @@ class WrapperLayerNorm(torch.nn.Module):
self.group_size = group_size
self.device = device
weight_dtype = torch.float32
+ self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
reshape_tensor(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),
@@ -141,13 +142,14 @@ class WrapperLayerNorm(torch.nn.Module):
return self.orig_layer
v = best_params['v']
weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,
- v)
+ v, q_scale_thresh=self.q_scale_thresh)
+ self.orig_layer.q_scale_thresh = self.q_scale_thresh
self.orig_layer.weight.data.copy_(weight_q)
return self.orig_layer
def forward(self, input):
weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,
- self.v)
+ self.v, q_scale_thresh=self.q_scale_thresh)
import torch.nn.functional as F
return F.layer_norm(
input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps)
@@ -168,6 +170,7 @@ class WrapperLlamaNorm(torch.nn.Module):
self.group_size = group_size
self.device = device
weight_dtype = torch.float32
+ self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
reshape_tensor(
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),
@@ -182,13 +185,14 @@ class WrapperLlamaNorm(torch.nn.Module):
return self.orig_layer
v = best_params['v']
weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,
- v)
+ v, q_scale_thresh=self.q_scale_thresh)
+ self.orig_layer.q_scale_thresh = self.q_scale_thresh
self.orig_layer.weight.data.copy_(weight_q)
return self.orig_layer
def forward(self, hidden_states):
weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,
- self.v)
+ self.v, q_scale_thresh=self.q_scale_thresh)
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
@@ -314,7 +318,7 @@ class WrapperLinear(torch.nn.Module):
qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.orig_layer.weight, self.bits,
self.group_size, v,
min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max,
- data_type=self.data_type)
+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
@@ -326,7 +330,7 @@ class WrapperLinear(torch.nn.Module):
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"]
bias, _, _ = quant_tensor(self.bias_quant_func, self.orig_layer.bias, self.bias_bits, self.bias_group_size,
- bias_v)
+ bias_v, q_scale_thresh=self.q_scale_thresh)
self.orig_layer.bias.grad = None
self.orig_layer.bias.data.copy_(bias)
@@ -362,7 +366,7 @@ class WrapperLinear(torch.nn.Module):
weight_q, _, _ = quant_tensor(self.weight_quant_func, weight, self.bits, self.group_size, self.value,
self.min_scale,
self.max_scale, self.scale_dtype, self.weight_min, self.weight_max,
- data_type=self.data_type)
+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)
weight_q = weight_q.to(weight.dtype)
if self.act_quant:
x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,
@@ -373,7 +377,8 @@ class WrapperLinear(torch.nn.Module):
if bias is not None and bias.device.type == 'meta':
bias = self.orig_layer.get_bias().to(self.device)
if self.enable_norm_bias_tuning:
- bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v)
+ bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,
+ q_scale_thresh=self.q_scale_thresh)
return F.linear(x, weight_q, bias)
@@ -494,7 +499,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.weight_t, self.bits, self.group_size, v,
min_scale,
max_scale, self.scale_dtype, self.weight_min, self.weight_max,
- data_type=self.data_type)
+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
@@ -506,7 +511,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"]
bias, _, _ = quant_tensor(self.bias_quant_func, self.orig_layer.bias, self.bias_bits, self.bias_group_size,
- bias_v)
+ bias_v, q_scale_thresh=self.q_scale_thresh)
self.orig_layer.bias.grad = None
self.orig_layer.bias.data.copy_(bias)
@@ -537,7 +542,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.max_scale.clamp_(0, 1.0)
weight_q, _, _ = quant_tensor(self.weight_quant_func, self.weight_t, self.bits, self.group_size, self.value,
self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,
- self.weight_max, data_type=self.data_type)
+ self.weight_max, data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)
weight_q = weight_q.to(self.weight_t.dtype)
size_out = x.size()[:-1] + (self.orig_layer.nf,)
if self.act_quant:
@@ -546,7 +551,8 @@ class WrapperTransformerConv1d(torch.nn.Module):
data_type=self.data_type)
bias = self.orig_layer.bias
if self.enable_norm_bias_tuning:
- bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v)
+ bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,
+ q_scale_thresh=self.q_scale_thresh)
x = torch.addmm(bias, x.view(-1, x.size(-1)), weight_q.t())
x = x.view(*size_out)
return x
@@ -650,3 +656,4 @@ def unwrapper_block(block, best_params):
best_param = None
orig_layer = m.unwrapper(best_param)
set_module(block, n, orig_layer)
+