PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine the code and the speedup is notable (#240)
wenhuach2146f23b9688fb
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex d39bf5224..349aed27a 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -35,28 +35,18 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dQuantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** bits - 1)+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_maxif isinstance(min_scale, torch.Tensor):- if weight_min is None or weight_max is None:- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- else:- wmin_tmp = weight_min- wmax_tmp = weight_max- wmin_tmp = wmin_tmp * min_scale- wmax_tmp = wmax_tmp * max_scale- wmax = torch.maximum(wmax_tmp, wmin_tmp)- wmin = torch.minimum(wmax_tmp, wmin_tmp)+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scaleelse:- if weight_min is None or weight_max is None:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)- else:- wmin = weight_min- wmax = weight_max-- tmp = (wmin == 0) & (wmax == 0)- wmin[tmp] = -1- wmax[tmp] = +1+ wmin = wmin_tmp+ wmax = wmax_tmpscale = ((wmax - wmin) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)zp = round_ste(-wmin / scale) # pylint: disable=E1130@@ -69,7 +59,7 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d@register_dtype("int_sym")def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,- weight_max=None, q_scale_thresh=0.0, **kargs):+ weight_max=None, q_scale_thresh=0.0, **kwargs):"""Quantizes and dequantizes weight symmetrically.Args:@@ -85,24 +75,18 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtQuantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** bits - 1)+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_maxif isinstance(min_scale, torch.Tensor):- if weight_min is None or weight_max is None:- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- else:- wmin_tmp = weight_min- wmax_tmp = weight_max- wmin_tmp = wmin_tmp * min_scale- wmax_tmp = wmax_tmp * max_scale- wmax = torch.maximum(wmax_tmp, wmin_tmp)- wmin = torch.minimum(wmax_tmp, wmin_tmp)+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scaleelse:- if weight_min is None or weight_max is None:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)- else:- wmin = weight_min- wmax = weight_max+ wmin = wmin_tmp+ wmax = wmax_tmpwmax_new = torch.max(wmin.abs(), wmax)tmp = wmin < 0@@ -110,9 +94,6 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtif torch.any(tmp):wmin_new[tmp] = -wmax_new[tmp]- tmp = (wmin_new == 0) & (wmax_new == 0)- wmin_new[tmp] = -1- wmax_new[tmp] = +1scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)scale = scale.unsqueeze(dim=-1)@@ -140,28 +121,19 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0Quantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** bits - 1)+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_maxif isinstance(min_scale, torch.Tensor):- if weight_min is None or weight_max is None:- wmin_tmp = torch.clamp(weight.min(-1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(-1)[0], min=0)- else:- wmin_tmp = weight_min- wmax_tmp = weight_max- wmin_tmp = wmin_tmp * min_scale- wmax_tmp = wmax_tmp * max_scale- wmax = torch.maximum(wmax_tmp, wmin_tmp)- wmin = torch.minimum(wmax_tmp, wmin_tmp)+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scaleelse:- if weight_min is None or weight_max is None:- wmin = torch.clamp(weight.min(-1)[0], max=0)- wmax = torch.clamp(weight.max(-1)[0], min=0)- else:- wmin = weight_min- wmax = weight_max-- tmp = (wmin == 0) & (wmax == 0)- wmin[tmp] = -1- wmax[tmp] = +1+ wmin = wmin_tmp+ wmax = wmax_tmp+scale = ((wmax - wmin) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)zp = -wmin / scale # pylint: disable=E1130@@ -170,5 +142,3 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0int_w = weight / scale + vq = torch.clamp(int_w + zp, 0, maxq)return scale * (q - zp), scale, zp--diff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 481cce89a..00dbc4f49 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -378,7 +378,7 @@ class WrapperLinear(torch.nn.Module):bias = self.orig_layer.get_bias().to(self.device)if self.enable_norm_bias_tuning:bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,- q_scale_thresh=self.q_scale_thresh)+ q_scale_thresh=self.q_scale_thresh)return F.linear(x, weight_q, bias)@@ -552,7 +552,7 @@ class WrapperTransformerConv1d(torch.nn.Module):bias = self.orig_layer.biasif self.enable_norm_bias_tuning:bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,- q_scale_thresh=self.q_scale_thresh)+ q_scale_thresh=self.q_scale_thresh)x = torch.addmm(bias, x.view(-1, x.size(-1)), weight_q.t())x = x.view(*size_out)return x@@ -656,4 +656,3 @@ def unwrapper_block(block, best_params):best_param = Noneorig_layer = m.unwrapper(best_param)set_module(block, n, orig_layer)-