Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine the code and the speedup is notable (#240)

wenhuach2146f23b9688fb
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index d39bf5224..349aed27a 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -35,28 +35,18 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
+ if weight_min is None or weight_max is None:
+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)
+ else:
+ wmin_tmp = weight_min
+ wmax_tmp = weight_max
if isinstance(min_scale, torch.Tensor):
- if weight_min is None or weight_max is None:
- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin_tmp = weight_min
- wmax_tmp = weight_max
- wmin_tmp = wmin_tmp * min_scale
- wmax_tmp = wmax_tmp * max_scale
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
else:
- if weight_min is None or weight_max is None:
- wmin = torch.clamp(weight.min(1)[0], max=0)
- wmax = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin = weight_min
- wmax = weight_max
-
- tmp = (wmin == 0) & (wmax == 0)
- wmin[tmp] = -1
- wmax[tmp] = +1
+ wmin = wmin_tmp
+ wmax = wmax_tmp
scale = ((wmax - wmin) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
zp = round_ste(-wmin / scale) # pylint: disable=E1130
@@ -69,7 +59,7 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
@register_dtype("int_sym")
def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
- weight_max=None, q_scale_thresh=0.0, **kargs):
+ weight_max=None, q_scale_thresh=0.0, **kwargs):
"""Quantizes and dequantizes weight symmetrically.
Args:
@@ -85,24 +75,18 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
+ if weight_min is None or weight_max is None:
+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)
+ else:
+ wmin_tmp = weight_min
+ wmax_tmp = weight_max
if isinstance(min_scale, torch.Tensor):
- if weight_min is None or weight_max is None:
- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin_tmp = weight_min
- wmax_tmp = weight_max
- wmin_tmp = wmin_tmp * min_scale
- wmax_tmp = wmax_tmp * max_scale
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
else:
- if weight_min is None or weight_max is None:
- wmin = torch.clamp(weight.min(1)[0], max=0)
- wmax = torch.clamp(weight.max(1)[0], min=0)
- else:
- wmin = weight_min
- wmax = weight_max
+ wmin = wmin_tmp
+ wmax = wmax_tmp
wmax_new = torch.max(wmin.abs(), wmax)
tmp = wmin < 0
@@ -110,9 +94,6 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
if torch.any(tmp):
wmin_new[tmp] = -wmax_new[tmp]
- tmp = (wmin_new == 0) & (wmax_new == 0)
- wmin_new[tmp] = -1
- wmax_new[tmp] = +1
scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
scale = scale.unsqueeze(dim=-1)
@@ -140,28 +121,19 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
+ if weight_min is None or weight_max is None:
+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)
+ else:
+ wmin_tmp = weight_min
+ wmax_tmp = weight_max
if isinstance(min_scale, torch.Tensor):
- if weight_min is None or weight_max is None:
- wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)
- wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)
- else:
- wmin_tmp = weight_min
- wmax_tmp = weight_max
- wmin_tmp = wmin_tmp * min_scale
- wmax_tmp = wmax_tmp * max_scale
- wmax = torch.maximum(wmax_tmp, wmin_tmp)
- wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
else:
- if weight_min is None or weight_max is None:
- wmin = torch.clamp(weight.min(-1)[0], max=0)
- wmax = torch.clamp(weight.max(-1)[0], min=0)
- else:
- wmin = weight_min
- wmax = weight_max
-
- tmp = (wmin == 0) & (wmax == 0)
- wmin[tmp] = -1
- wmax[tmp] = +1
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+
scale = ((wmax - wmin) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
zp = -wmin / scale # pylint: disable=E1130
@@ -170,5 +142,3 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0
int_w = weight / scale + v
q = torch.clamp(int_w + zp, 0, maxq)
return scale * (q - zp), scale, zp
-
-
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 481cce89a..00dbc4f49 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -378,7 +378,7 @@ class WrapperLinear(torch.nn.Module):
bias = self.orig_layer.get_bias().to(self.device)
if self.enable_norm_bias_tuning:
bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,
- q_scale_thresh=self.q_scale_thresh)
+ q_scale_thresh=self.q_scale_thresh)
return F.linear(x, weight_q, bias)
@@ -552,7 +552,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
bias = self.orig_layer.bias
if self.enable_norm_bias_tuning:
bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,
- q_scale_thresh=self.q_scale_thresh)
+ q_scale_thresh=self.q_scale_thresh)
x = torch.addmm(bias, x.view(-1, x.size(-1)), weight_q.t())
x = x.view(*size_out)
return x
@@ -656,4 +656,3 @@ def unwrapper_block(block, best_params):
best_param = None
orig_layer = m.unwrapper(best_param)
set_module(block, n, orig_layer)
-