PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
20% speedup by removing new zero tensor (#110)
wenhuach21c7434b63dbef
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ec277077d..943729954 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -69,22 +69,25 @@ class WrapperLinear(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.scale_dtype = self.orig_layer.scale_dtypeself.sym = self.orig_layer.sym+ weight_dtype = self.orig_layer.weight.dtype+ weight_dtype = torch.float32 ##TODO revert the change to check the accuracyself.value = torch.nn.Parameter(- torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True+ torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),+ requires_grad=True,)self.enable_minmax_tuning = enable_minmax_tuningshape = get_scale_shape(self.orig_layer.weight, self.group_size)-+ weight_dtype = self.orig_layer.weight.dtypeif self.enable_minmax_tuning:self.min_scale = torch.nn.Parameter(- torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True+ torch.zeros(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True)self.max_scale = torch.nn.Parameter(- torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True+ torch.zeros(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True)else:- self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device)- self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device)+ self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device, dtype=weight_dtype)+ self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device, dtype=weight_dtype)def unwrapper(self, v, min_scale, max_scale):"""Unwrapper the layer to the original layer.@@ -174,17 +177,26 @@ class WrapperTransformerConv1d(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.sym = self.orig_layer.symself.scale_dtype = self.orig_layer.scale_dtype+ weight_dtype = self.orig_layer.weight.dtype+ weight_dtype = torch.float32 ##TODO revert the change to check the accuracy+device = self.orig_layer.weight.deviceself.weight_t = self.orig_layer.weight.t()- self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)+ self.value = torch.nn.Parameter(+ torch.zeros(self.weight_t.shape, device=device, dtype=weight_dtype), requires_grad=True+ )shape = get_scale_shape(self.weight_t, self.group_size)if enable_minmax_tuning:- self.min_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)- self.max_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)+ self.min_scale = torch.nn.Parameter(+ torch.zeros(shape, device=device, dtype=weight_dtype), requires_grad=True+ )+ self.max_scale = torch.nn.Parameter(+ torch.zeros(shape, device=device, dtype=weight_dtype), requires_grad=True+ )else:- self.min_scale = torch.tensor(0, device=device)- self.max_scale = torch.tensor(0, device=device)+ self.min_scale = torch.tensor(0, device=device, dtype=weight_dtype)+ self.max_scale = torch.tensor(0, device=device, dtype=weight_dtype)def unwrapper(self, v=0, min_scale=0, max_scale=0):"""Unwrapper the layer to the original conv1d layer.@@ -563,7 +575,7 @@ class AutoRound(object):logger.info(summary_info)self.quantized = True- self.model = self.model.to(self.model_orig_dtype)+ ##self.model = self.model.to(self.model_orig_dtype)##keep it as amp dtypereturn self.model, self.weight_configdef dump_data_to_weight_config(self):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 0e2b832f6..c830b8d8b 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -105,18 +105,16 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dQuantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2**num_bits - 1)- zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)- # zeros = torch.zeros(weight.shape[0], device=weight.device)if isinstance(min_scale, torch.Tensor):- wmin_tmp = torch.minimum(weight.min(1)[0], zeros)- wmax_tmp = torch.maximum(weight.max(1)[0], zeros)+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)wmin_tmp *= min_scale + 1.0wmax_tmp *= max_scale + 1.0wmax = torch.maximum(wmax_tmp, wmin_tmp)wmin = torch.minimum(wmax_tmp, wmin_tmp)else:- wmin = torch.minimum(weight.min(1)[0], zeros)- wmax = torch.maximum(weight.max(1)[0], zeros)+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)tmp = (wmin == 0) & (wmax == 0)wmin[tmp] = -1@@ -144,17 +142,16 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtQuantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2**num_bits - 1)- zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)if isinstance(min_scale, torch.Tensor):- wmin_tmp = torch.minimum(weight.min(1)[0], zeros)- wmax_tmp = torch.maximum(weight.max(1)[0], zeros)+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)wmin_tmp *= min_scale + 1.0wmax_tmp *= max_scale + 1.0wmax = torch.maximum(wmax_tmp, wmin_tmp)wmin = torch.minimum(wmax_tmp, wmin_tmp)else:- wmin = torch.minimum(weight.min(1)[0], zeros)- wmax = torch.maximum(weight.max(1)[0], zeros)+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)wmax_new = torch.max(wmin.abs(), wmax)tmp = wmin < 0wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occurdiff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex 6ff8c0c04..c7ab82d72 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -80,13 +80,19 @@ pip install -r requirements.txtCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128```- **Reduced GPU Memory Usage:**-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8-```+set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)++reduce the train bs to 4(potential accuracy drop)- **Speedup the tuning:**-disable_low_gpu_mem_usage(more gpu memory) or set the n_sample to 128( little accuracy drop) or change the train bs to 4(little accuracy drop) or combine them+disable_low_gpu_mem_usage(more gpu memory)++reduce the train bs to 4(little accuracy drop)++reduce the seqlen to 512(potential large accuracy drop)++or combine them- **Enable quantized lm-head:**