PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
speedup the tuning a little (#175)
wenhuach21473f474df665
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 9a57b20d0..51159c14c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -149,14 +149,13 @@ class AutoRound(object):self.device = detect_device(device)self.scale_dtype = convert_dtype_str2torch(scale_dtype)self.set_amp_dtype()- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else device- logger.info(f"using {self.model.dtype} for quantization tuning")+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.deviceself.dataset = dataset+self.iters = itersif self.iters <= 0:logger.warning("iters must be positive, reset it to 200")self.iters = 200-self.lr = lr or (1.0 / self.iters)self.minmax_lr = minmax_lr or self.lr@@ -172,8 +171,8 @@ class AutoRound(object):self.hidden_dim_flag = Noneself.infer_bs_coeff = 1torch.set_printoptions(precision=3, sci_mode=True)-self.check_configs()+ logger.info(f"using {self.model.dtype} for quantization tuning")if is_optimum_habana_available():logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401@@ -1216,7 +1215,7 @@ class AutoOPTRound(AutoRound):enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,- device="auto",+ device=None,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,@@ -1364,7 +1363,7 @@ class AutoAdamRound(AutoOPTRound):enable_full_range: bool = False,batch_size: int = 8,amp: bool = True,- device="auto",+ device=None,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,diff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 5fce47d40..9d8738331 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -20,6 +20,8 @@ from .utils import (get_scale_shape,set_module)++def round_ste(x: torch.Tensor):"""Straight-Through Estimator for rounding.This function is adapted from omniquant.@@ -33,7 +35,8 @@ def round_ste(x: torch.Tensor):return (x.round() - x).detach() + x-def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16):+def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None):"""Quantizes and dequantizes weight asymmetrically.Args:@@ -42,27 +45,37 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scav: Rounding value perturbationmin_scale: Minimum scale coefficient for weightmax_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.Returns:Quantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** num_bits - 1)if isinstance(min_scale, torch.Tensor):- wmin_tmp = torch.clamp(weight.min(1)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- wmin_tmp *= min_scale- wmax_tmp *= max_scale+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_max+ wmin_tmp = wmin_tmp * min_scale+ wmax_tmp = wmax_tmp * max_scalewmax = torch.maximum(wmax_tmp, wmin_tmp)wmin = torch.minimum(wmax_tmp, wmin_tmp)else:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)+ if weight_min is None or weight_max is None:+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin = weight_min+ wmax = weight_maxtmp = (wmin == 0) & (wmax == 0)wmin[tmp] = -1wmax[tmp] = +1scale = ((wmax - wmin) / maxq).to(scale_dtype)- zp = round_ste(-wmin / scale)+ zp = round_ste(-wmin / scale) # pylint: disable=E1130scale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)int_w = round_ste(weight / scale + v)@@ -70,7 +83,8 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scareturn scale * (q - zp), scale, zp-def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16):+def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,+ weight_max=None):"""Quantizes and dequantizes weight symmetrically.Args:@@ -79,21 +93,32 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scalv: Rounding value perturbationmin_scale: Minimum scale coefficient for weightmax_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.Returns:Quantized and dequantized weight, scale, zero-point"""maxq = torch.tensor(2 ** num_bits - 1)if isinstance(min_scale, torch.Tensor):- wmin_tmp = torch.clamp(weight.min(1,)[0], max=0)- wmax_tmp = torch.clamp(weight.max(1)[0], min=0)- wmin_tmp *= min_scale- wmax_tmp *= max_scale+ if weight_min is None or weight_max is None:+ wmin_tmp = torch.clamp(weight.min(1)[0], max=0)+ wmax_tmp = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin_tmp = weight_min+ wmax_tmp = weight_max+ wmin_tmp = wmin_tmp * min_scale+ wmax_tmp = wmax_tmp * max_scalewmax = torch.maximum(wmax_tmp, wmin_tmp)wmin = torch.minimum(wmax_tmp, wmin_tmp)else:- wmin = torch.clamp(weight.min(1)[0], max=0)- wmax = torch.clamp(weight.max(1)[0], min=0)+ if weight_min is None or weight_max is None:+ wmin = torch.clamp(weight.min(1)[0], max=0)+ wmax = torch.clamp(weight.max(1)[0], min=0)+ else:+ wmin = weight_min+ wmax = weight_max+wmax_new = torch.max(wmin.abs(), wmax)tmp = wmin < 0wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur@@ -113,7 +138,8 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=1.0, max_scale=1.0, scalreturn scale * (q - zp), scale, zp-def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16):+def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtype=torch.float16, weight_min=None,+ weight_max=None):"""Quantizes and dequantizes weight symmetrically or asymmetrically .Args:@@ -123,19 +149,42 @@ def quant_weight_actor(weight, num_bits, sym, v, min_scale, max_scale, scale_dtyv: Rounding value perturbationmin_scale: Minimum scale coefficient for weightmax_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.Returns:Quantized and dequantized weight, scale, zero-point"""assert num_bits > 0, "num_bits should be larger than 0"if sym:- return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype)+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)+ else:+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype, weight_min, weight_max)+++def reshape_tensor(v, group_size=-1):+ """Reshapes the tensor based on the group size.++ Args:+ v (torch.Tensor): The input tensor to be reshaped.+ group_size (int, optional): The number of elements to group together.++ Returns:+ torch.Tensor: The reshaped tensor. If padding is applied, the padded tensor is returned.+ """+ if group_size == -1 or v.shape[1] < group_size:+ return v+ if v.shape[1] % group_size == 0:+ v = v.reshape(-1, group_size)else:- return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)+ pad_len = (v.shape[1] + group_size - 1) // group_size * group_size - v.shape[1]+ v = torch.nn.functional.pad(v, (0, pad_len))+ return vdef quant_weight(- weight, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16+ weight, num_bits=4, group_size=-1, sym=False, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ weight_min=None, weight_max=None):"""Quantizes and dequantizes weight, handing the group size issue .@@ -147,45 +196,37 @@ def quant_weight(v: Rounding value perturbationmin_scale: Minimum scale coefficient for weightmax_scale: Maximum scale coefficient for weight+ weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.+ weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.Returns:Quantized and dequantized weight, scale, zero-point"""if group_size == -1 or weight.shape[1] < group_size:return quant_weight_actor(- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype+ weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,+ weight_min=weight_min, weight_max=weight_max)orig_shape = weight.shapeif weight.shape[1] % group_size == 0:weight = weight.reshape(-1, group_size)- if isinstance(v, torch.Tensor):- v = v.reshape(-1, group_size)-weight, scale, zp = quant_weight_actor(- weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype+ weight, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,+ weight_min=weight_min, weight_max=weight_max)weight = weight.reshape(orig_shape)- scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d- if zp is not None:- zp = zp.reshape(weight.shape[0], -1)return weight, scale, zpelse:pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]weight_new = torch.nn.functional.pad(weight, (0, pad_len))- v = torch.nn.functional.pad(v, (0, pad_len))weight_new = weight_new.reshape(-1, group_size)- if isinstance(v, torch.Tensor):- v = v.reshape(-1, group_size)weight_new, scale, zp = quant_weight_actor(- weight_new, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype+ weight_new, num_bits, sym=sym, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype,+ weight_min=weight_min, weight_max=weight_max)weight_new = weight_new.reshape(orig_shape[0], -1)-weight_new = weight_new[:, :-pad_len]- scale = scale.reshape(weight_new.shape[0], -1) ##only for linear, conv1d- if zp is not None:- zp = zp.reshape(weight_new.shape[0], -1)return weight_new, scale, zp@@ -213,12 +254,16 @@ class WrapperLinear(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.scale_dtype = self.orig_layer.scale_dtypeself.sym = self.orig_layer.sym- weight_dtype = self.orig_layer.weight.dtypeweight_dtype = torch.float32self.value = torch.nn.Parameter(- torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),- requires_grad=True,- )+ reshape_tensor(+ torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),+ self.group_size),+ requires_grad=True)+ weight_reshape = reshape_tensor(self.orig_layer.weight.data, self.group_size)+ self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)+ self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)+self.enable_minmax_tuning = enable_minmax_tuningshape = get_scale_shape(self.orig_layer.weight, self.group_size)if self.enable_minmax_tuning:@@ -246,7 +291,7 @@ class WrapperLinear(torch.nn.Module):min_scale.clamp_(0, 1.0)max_scale.clamp_(0, 1.0)- q_dq_weight, scale, zp = quant_weight(+ qdq_weight, scale, zp = quant_weight(self.orig_layer.weight,self.num_bits,self.group_size,@@ -255,8 +300,13 @@ class WrapperLinear(torch.nn.Module):min_scale,max_scale,self.scale_dtype,+ self.weight_min,+ self.weight_max,)- self.orig_layer.weight.data.copy_(q_dq_weight)+ scale = scale.reshape(qdq_weight.shape[0], -1)+ if zp is not None:+ zp = zp.reshape(qdq_weight.shape[0], -1)+ self.orig_layer.weight.data.copy_(qdq_weight)self.orig_layer.weight.grad = None ##clear gradself.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu") if zp is not None else None@@ -285,6 +335,8 @@ class WrapperLinear(torch.nn.Module):self.min_scale,self.max_scale,self.scale_dtype,+ self.weight_min,+ self.weight_max)weight_q = weight_q.to(weight.dtype)# pylint: disable=not-callable@@ -320,13 +372,18 @@ class WrapperTransformerConv1d(torch.nn.Module):self.group_size = self.orig_layer.group_sizeself.sym = self.orig_layer.symself.scale_dtype = self.orig_layer.scale_dtype- weight_dtype = self.orig_layer.weight.dtypeweight_dtype = torch.float32device = self.orig_layer.weight.deviceself.weight_t = self.orig_layer.weight.t()self.value = torch.nn.Parameter(- torch.zeros(self.weight_t.shape, device=device, dtype=weight_dtype), requires_grad=True+ reshape_tensor(torch.zeros(self.weight_t.shape, device=device, dtype=weight_dtype),+ group_size=self.group_size),+ requires_grad=True)+ weight_reshape = reshape_tensor(self.weight_t, self.group_size)+ self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)+ self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)+shape = get_scale_shape(self.weight_t, self.group_size)if enable_minmax_tuning:@@ -353,10 +410,14 @@ class WrapperTransformerConv1d(torch.nn.Module):"""min_scale.clamp_(0, 1.0)max_scale.clamp_(0, 1.0)- weight_q, scale, zp = quant_weight(- self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale, max_scale, self.scale_dtype+ qdq_weight, scale, zp = quant_weight(+ self.weight_t, self.num_bits, self.group_size, self.sym, v, min_scale, max_scale, self.scale_dtype,+ self.weight_min, self.weight_max)- self.orig_layer.weight.data.copy_(weight_q.t())+ scale = scale.reshape(qdq_weight.shape[0], -1)+ if zp is not None:+ zp = zp.reshape(qdq_weight.shape[0], -1)+ self.orig_layer.weight.data.copy_(qdq_weight.t())self.orig_layer.weight.grad = Noneself.orig_layer.scale = scale.to("cpu")self.orig_layer.zp = zp.to("cpu")@@ -383,6 +444,8 @@ class WrapperTransformerConv1d(torch.nn.Module):self.min_scale,self.max_scale,self.scale_dtype,+ self.weight_min,+ self.weight_max)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)@@ -493,4 +556,4 @@ def unwrapper_block(block, vs, min_scales, max_scales):max_scale = max_scales[n]max_scale = torch.clamp(max_scale, 0, 1.0)orig_layer = m.unwrapper(v, min_scale, max_scale)- set_module(block, n, orig_layer)\ No newline at end of file+ set_module(block, n, orig_layer)diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex 02ae14ae7..44e8bddae 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -52,43 +52,41 @@ pip install -r requirements.txtCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128```- **Reduced GPU Memory Usage:**+-+enable low_gpu_mem_usage(more tuning cost)set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)reduce the train bs to 4(potential accuracy drop)-reduce the nsamples to 128 (potential accuracy drop)+reduce the seqlen to 512 (potential accuracy drop)or combine them- **Speedup the tuning:**-disable_low_gpu_mem_usage(more gpu memory)-reduce the train bs to 4(little accuracy drop)reduce the seqlen to 512(potential large accuracy drop)-reduce the nsamples to 128 (potential accuracy drop)-or combine them- **Enable quantized lm-head:**-Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.+Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios.```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head```- **Enable marlin kernel:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --sym --bits 4 --group_size 128 --disable_low_gpu_mem_usage --deployment_device "auto_round:marlin"+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --sym --bits 4 --group_size 128 --deployment_device "auto_round:marlin"```for AutoGPTQ format, enable `--sym` in the quantization and set the deployment_device to "gpu" and then follow the API in AutoGPTQ-- **Utilizing the AdamW Optimizer:**+- **Utilize the AdamW Optimizer:**Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.@@ -128,7 +126,7 @@ CUDA_VISIBLE_DEVICES=1,2 python3 eval/evaluation.py --model_name /save_model_patYou can also utilize the official lm_eval [link](https://github.com/EleutherAI/lm-evaluation-harness/tree/main?tab=readme-ov-file#basic-usage).## 4. Known Issues-* Random issues in tuning some models+* Random quantization results in tuning some models* ChatGlm-V1 is not supporteddiff --git a/test/test_autoround_export.py b/test/test_autoround_export.pyindex 591064594..d90ca15f1 100644--- a/test/test_autoround_export.py+++ b/test/test_autoround_export.py@@ -67,7 +67,7 @@ class TestAutoroundExport(unittest.TestCase):self.assertTrue(torch.all(torch.isclose(out1[0], out2[0], atol=1e-1)))self.assertFalse(torch.all(out1[0] == out2[0]))self.assertTrue(torch.all(out2[0] == out3[0]))- self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))+ self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-3)))self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())model = copy.deepcopy(self.gptj)@@ -78,7 +78,7 @@ class TestAutoroundExport(unittest.TestCase):out4 = q_model(self.lm_input)out5 = compressed_model(self.lm_input)self.assertTrue(torch.all(out1[0] == out6[0]))- self.assertTrue(torch.all(torch.isclose(out4[0], out5[0], atol=1e-5)))+ self.assertTrue(torch.all(torch.isclose(out4[0], out5[0], atol=1e-3)))def test_config(self):from auto_round.export import QuantConfig