PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
change the scale thresh generally (#229)
WeiweiZhang1b2f160c260ce
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 545eebd0a..d39bf5224 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -59,8 +59,6 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dwmax[tmp] = +1scale = ((wmax - wmin) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)- if (scale == 0.).any():- scale = torch.clamp(scale, min=1e-5)zp = round_ste(-wmin / scale) # pylint: disable=E1130scale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)@@ -117,8 +115,6 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtwmax_new[tmp] = +1scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)- if (scale == 0.).any():- scale = torch.clamp(scale, min=1e-5)scale = scale.unsqueeze(dim=-1)zp = torch.full_like(scale, (maxq + 1) / 2)@@ -175,3 +171,4 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0q = torch.clamp(int_w + zp, 0, maxq)return scale * (q - zp), scale, zp+diff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 58a3858d0..481cce89a 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -46,7 +46,7 @@ def reshape_tensor(v, group_size=-1):def quant_tensor(quant_func, data, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs,+ weight_min=None, weight_max=None, q_scale_thresh=1e-5, **kwargs,):"""Quantizes and dequantizes weight, handing the group size issue .@@ -126,6 +126,7 @@ class WrapperLayerNorm(torch.nn.Module):self.group_size = group_sizeself.device = deviceweight_dtype = torch.float32+ self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(reshape_tensor(torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),@@ -141,13 +142,14 @@ class WrapperLayerNorm(torch.nn.Module):return self.orig_layerv = best_params['v']weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,- v)+ v, q_scale_thresh=self.q_scale_thresh)+ self.orig_layer.q_scale_thresh = self.q_scale_threshself.orig_layer.weight.data.copy_(weight_q)return self.orig_layerdef forward(self, input):weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,- self.v)+ self.v, q_scale_thresh=self.q_scale_thresh)import torch.nn.functional as Freturn F.layer_norm(input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps)@@ -168,6 +170,7 @@ class WrapperLlamaNorm(torch.nn.Module):self.group_size = group_sizeself.device = deviceweight_dtype = torch.float32+ self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(reshape_tensor(torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),@@ -182,13 +185,14 @@ class WrapperLlamaNorm(torch.nn.Module):return self.orig_layerv = best_params['v']weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,- v)+ v, q_scale_thresh=self.q_scale_thresh)+ self.orig_layer.q_scale_thresh = self.q_scale_threshself.orig_layer.weight.data.copy_(weight_q)return self.orig_layerdef forward(self, hidden_states):weight_q, _, _ = quant_tensor(self.quant_func, self.orig_layer.weight, self.bits, self.group_size,- self.v)+ self.v, q_scale_thresh=self.q_scale_thresh)input_dtype = hidden_states.dtypehidden_states = hidden_states.to(torch.float32)variance = hidden_states.pow(2).mean(-1, keepdim=True)@@ -314,7 +318,7 @@ class WrapperLinear(torch.nn.Module):qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.orig_layer.weight, self.bits,self.group_size, v,min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max,- data_type=self.data_type)+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)@@ -326,7 +330,7 @@ class WrapperLinear(torch.nn.Module):if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"]bias, _, _ = quant_tensor(self.bias_quant_func, self.orig_layer.bias, self.bias_bits, self.bias_group_size,- bias_v)+ bias_v, q_scale_thresh=self.q_scale_thresh)self.orig_layer.bias.grad = Noneself.orig_layer.bias.data.copy_(bias)@@ -362,7 +366,7 @@ class WrapperLinear(torch.nn.Module):weight_q, _, _ = quant_tensor(self.weight_quant_func, weight, self.bits, self.group_size, self.value,self.min_scale,self.max_scale, self.scale_dtype, self.weight_min, self.weight_max,- data_type=self.data_type)+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)weight_q = weight_q.to(weight.dtype)if self.act_quant:x, _, _ = quant_tensor(self.act_quant_func, x, self.act_bits, self.act_group_size,@@ -373,7 +377,8 @@ class WrapperLinear(torch.nn.Module):if bias is not None and bias.device.type == 'meta':bias = self.orig_layer.get_bias().to(self.device)if self.enable_norm_bias_tuning:- bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v)+ bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,+ q_scale_thresh=self.q_scale_thresh)return F.linear(x, weight_q, bias)@@ -494,7 +499,7 @@ class WrapperTransformerConv1d(torch.nn.Module):qdq_weight, scale, zp = quant_tensor(self.weight_quant_func, self.weight_t, self.bits, self.group_size, v,min_scale,max_scale, self.scale_dtype, self.weight_min, self.weight_max,- data_type=self.data_type)+ data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)scale = scale.reshape(qdq_weight.shape[0], -1)if zp is not None:zp = zp.reshape(qdq_weight.shape[0], -1)@@ -506,7 +511,7 @@ class WrapperTransformerConv1d(torch.nn.Module):if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"]bias, _, _ = quant_tensor(self.bias_quant_func, self.orig_layer.bias, self.bias_bits, self.bias_group_size,- bias_v)+ bias_v, q_scale_thresh=self.q_scale_thresh)self.orig_layer.bias.grad = Noneself.orig_layer.bias.data.copy_(bias)@@ -537,7 +542,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.max_scale.clamp_(0, 1.0)weight_q, _, _ = quant_tensor(self.weight_quant_func, self.weight_t, self.bits, self.group_size, self.value,self.min_scale, self.max_scale, self.scale_dtype, self.weight_min,- self.weight_max, data_type=self.data_type)+ self.weight_max, data_type=self.data_type, q_scale_thresh=self.q_scale_thresh)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)if self.act_quant:@@ -546,7 +551,8 @@ class WrapperTransformerConv1d(torch.nn.Module):data_type=self.data_type)bias = self.orig_layer.biasif self.enable_norm_bias_tuning:- bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v)+ bias, _, _ = quant_tensor(self.bias_quant_func, bias, self.bias_bits, self.bias_group_size, self.bias_v,+ q_scale_thresh=self.q_scale_thresh)x = torch.addmm(bias, x.view(-1, x.size(-1)), weight_q.t())x = x.view(*size_out)return x@@ -650,3 +656,4 @@ def unwrapper_block(block, best_params):best_param = Noneorig_layer = m.unwrapper(best_param)set_module(block, n, orig_layer)+