PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support deepspeed LinearLayer and LinearAllreduce (#698)
Xin He8c2df83a9a1d
diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.shindex a1ed65ad4..7fb4d813d 100644--- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh+++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh@@ -24,7 +24,7 @@ pip install pipdeptreepipdeptreepython -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \---ignored-modules=tensorflow,keras,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers \+--ignored-modules=tensorflow,keras,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers,deepspeed,deepspeed.module_inject \/auto-round/${scan_module} > $log_dir/pylint.jsonexit_code=$?diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 3ab23e08b..635648299 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -38,6 +38,11 @@ from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")+deepspeed_exists = False+if importlib.util.find_spec("deepspeed"): # check if deepspeed is installed+ deepspeed_exists = True++class SupportedFormats:def __init__(self):@@ -67,16 +72,18 @@ class SupportedFormats:return self._support_list[key]+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")SUPPORTED_FORMATS = SupportedFormats()-SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)##changed to str as it relies triton or others lib to load thisINNER_SUPPORTED_LAYER_TYPES = ("FP8Linear",)-# INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)-SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")+if deepspeed_exists:+ from deepspeed.module_inject import LinearAllreduce, LinearLayer++ SUPPORTED_LAYER_TYPES = SUPPORTED_LAYER_TYPES + (LinearLayer, LinearAllreduce)def infer_bits_by_data_type(data_type: str):@@ -1172,6 +1179,8 @@ def get_layer_features(layer):return layer.weight.shape[0], layer.weight.shape[1]elif isinstance(layer, torch.nn.Embedding):return layer.num_embeddings, layer.embedding_dim+ elif deepspeed_exists and isinstance(layer, (LinearLayer, LinearAllreduce)):+ return layer.weight.shape[1], layer.weight.shape[0] # (input_dim, output_dim)return None, None # Unsupported layer typediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex c3dfc0765..8942a2c0d 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -18,7 +18,20 @@ from torch.functional import Ffrom auto_round.data_type import get_quant_func-from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, is_mx_fp, is_nv_fp, logger, set_module+from .utils import (+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ deepspeed_exists,+ get_scale_shape,+ is_mx_fp,+ is_nv_fp,+ logger,+ set_module,+)++if deepspeed_exists:+ from deepspeed import comm as dist+ from deepspeed.module_inject import LinearAllreduce, LinearLayerdef reshape_and_pad_tensor(v, group_size=-1):@@ -94,7 +107,18 @@ class WrapperLinear(torch.nn.Module):else:self.q_scale_thresh = 1e-5self._init_tuning_params_and_quant_func()- self.orig_forward = self.linear_forward if isinstance(self.orig_layer, torch.nn.Linear) else self.conv1d_forward+ if deepspeed_exists:+ if isinstance(self.orig_layer, (torch.nn.Linear, LinearLayer)):+ self.orig_forward = self.linear_forward+ elif isinstance(self.orig_layer, LinearAllreduce):+ self.orig_forward = self.all_reduce_linear_forward+ self.mp_group = self.orig_layer.mp_group+ else:+ self.orig_forward = self.conv1d_forward+ else:+ self.orig_forward = (+ self.linear_forward if isinstance(self.orig_layer, torch.nn.Linear) else self.conv1d_forward+ )def _init_tuning_params_and_quant_func(self):"""Initializes tuning parameters and quantization functions.@@ -367,6 +391,24 @@ class WrapperLinear(torch.nn.Module):"""return F.linear(x, weight, bias)+ def all_reduce_linear_forward(self, x, weight, bias):+ """Performs the forward pass for a linear layer.++ Args:+ x (torch.Tensor): Input tensor.+ weight (torch.Tensor): Weight tensor for the linear layer.+ bias (torch.Tensor): Bias tensor for the linear layer.++ Returns:+ torch.Tensor: Output tensor after applying the linear layer.+ """+ output = torch.matmul(x, weight.transpose(-1, -2))+ if self.mp_group is not None:+ dist.inference_all_reduce(output, group=self.mp_group)+ if bias is not None:+ output += bias+ return output+def conv1d_forward(self, x, weight, bias):"""Performs the forward pass for a Conv1D layer.@@ -414,7 +456,10 @@ class WrapperWALayer(torch.nn.Module):def __init__(self, orig_layer):super(WrapperWALayer, self).__init__()self.orig_layer = orig_layer+ self.data_type = orig_layer.data_type if hasattr(orig_layer, "data_type") else None+ self.act_data_type = orig_layer.act_data_type if hasattr(orig_layer, "act_data_type") else Noneself.act_quant_func = self.orig_layer.act_quant_func+ self.extra_repr_org = orig_layer.extra_reprdef forward(self, x):act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None@@ -429,6 +474,9 @@ class WrapperWALayer(torch.nn.Module):)return self.orig_layer.forward(x)+ def extra_repr(self):+ return f"{self.extra_repr_org()}, weight_type={self.data_type}, act_data_type={self.act_data_type}"+class WrapperLayerNorm(torch.nn.Module):"""A wrapper for layer normalization with quantized weights.