Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

20% speedup by removing new zero tensor (#110)

wenhuach21c7434b63dbef
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index ec277077d..943729954 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -69,22 +69,25 @@ class WrapperLinear(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.scale_dtype = self.orig_layer.scale_dtype
self.sym = self.orig_layer.sym
+ weight_dtype = self.orig_layer.weight.dtype
+ weight_dtype = torch.float32 ##TODO revert the change to check the accuracy
self.value = torch.nn.Parameter(
- torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True
+ torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),
+ requires_grad=True,
)
self.enable_minmax_tuning = enable_minmax_tuning
shape = get_scale_shape(self.orig_layer.weight, self.group_size)
-
+ weight_dtype = self.orig_layer.weight.dtype
if self.enable_minmax_tuning:
self.min_scale = torch.nn.Parameter(
- torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True
+ torch.zeros(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True
)
self.max_scale = torch.nn.Parameter(
- torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True
+ torch.zeros(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True
)
else:
- self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device)
- self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device)
+ self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device, dtype=weight_dtype)
+ self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device, dtype=weight_dtype)
def unwrapper(self, v, min_scale, max_scale):
"""Unwrapper the layer to the original layer.
@@ -174,17 +177,26 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.group_size = self.orig_layer.group_size
self.sym = self.orig_layer.sym
self.scale_dtype = self.orig_layer.scale_dtype
+ weight_dtype = self.orig_layer.weight.dtype
+ weight_dtype = torch.float32 ##TODO revert the change to check the accuracy
+
device = self.orig_layer.weight.device
self.weight_t = self.orig_layer.weight.t()
- self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)
+ self.value = torch.nn.Parameter(
+ torch.zeros(self.weight_t.shape, device=device, dtype=weight_dtype), requires_grad=True
+ )
shape = get_scale_shape(self.weight_t, self.group_size)
if enable_minmax_tuning:
- self.min_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)
- self.max_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)
+ self.min_scale = torch.nn.Parameter(
+ torch.zeros(shape, device=device, dtype=weight_dtype), requires_grad=True
+ )
+ self.max_scale = torch.nn.Parameter(
+ torch.zeros(shape, device=device, dtype=weight_dtype), requires_grad=True
+ )
else:
- self.min_scale = torch.tensor(0, device=device)
- self.max_scale = torch.tensor(0, device=device)
+ self.min_scale = torch.tensor(0, device=device, dtype=weight_dtype)
+ self.max_scale = torch.tensor(0, device=device, dtype=weight_dtype)
def unwrapper(self, v=0, min_scale=0, max_scale=0):
"""Unwrapper the layer to the original conv1d layer.
@@ -563,7 +575,7 @@ class AutoRound(object):
logger.info(summary_info)
self.quantized = True
- self.model = self.model.to(self.model_orig_dtype)
+ ##self.model = self.model.to(self.model_orig_dtype)##keep it as amp dtype
return self.model, self.weight_config
def dump_data_to_weight_config(self):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 0e2b832f6..c830b8d8b 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -105,18 +105,16 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_d
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2**num_bits - 1)
- zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
- # zeros = torch.zeros(weight.shape[0], device=weight.device)
if isinstance(min_scale, torch.Tensor):
- wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
- wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
wmin_tmp *= min_scale + 1.0
wmax_tmp *= max_scale + 1.0
wmax = torch.maximum(wmax_tmp, wmin_tmp)
wmin = torch.minimum(wmax_tmp, wmin_tmp)
else:
- wmin = torch.minimum(weight.min(1)[0], zeros)
- wmax = torch.maximum(weight.max(1)[0], zeros)
+ wmin = torch.clamp(weight.min(1)[0], max=0)
+ wmax = torch.clamp(weight.max(1)[0], min=0)
tmp = (wmin == 0) & (wmax == 0)
wmin[tmp] = -1
@@ -144,17 +142,16 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dt
Quantized and dequantized weight, scale, zero-point
"""
maxq = torch.tensor(2**num_bits - 1)
- zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
if isinstance(min_scale, torch.Tensor):
- wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
- wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)
+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)
wmin_tmp *= min_scale + 1.0
wmax_tmp *= max_scale + 1.0
wmax = torch.maximum(wmax_tmp, wmin_tmp)
wmin = torch.minimum(wmax_tmp, wmin_tmp)
else:
- wmin = torch.minimum(weight.min(1)[0], zeros)
- wmax = torch.maximum(weight.max(1)[0], zeros)
+ wmin = torch.clamp(weight.min(1)[0], max=0)
+ wmax = torch.clamp(weight.max(1)[0], min=0)
wmax_new = torch.max(wmin.abs(), wmax)
tmp = wmin < 0
wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 6ff8c0c04..c7ab82d72 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -80,13 +80,19 @@ pip install -r requirements.txt
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128
```
- **Reduced GPU Memory Usage:**
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8
-```
+set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)
+
+reduce the train bs to 4(potential accuracy drop)
- **Speedup the tuning:**
-disable_low_gpu_mem_usage(more gpu memory) or set the n_sample to 128( little accuracy drop) or change the train bs to 4(little accuracy drop) or combine them
+disable_low_gpu_mem_usage(more gpu memory)
+
+reduce the train bs to 4(little accuracy drop)
+
+reduce the seqlen to 512(potential large accuracy drop)
+
+or combine them
- **Enable quantized lm-head:**