PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix fp8 model input issue (#724)
Wenhua Cheng9113fa8eb547
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex f6d6b6ea2..ed2ab46bd 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -251,6 +251,9 @@ class AutoRound(object):self.bits = tmp_bitsself.supported_types = SUPPORTED_LAYER_TYPESself.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ if "hpu" in str(self.device):+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")+self.model = model.eval()self.tokenizer = tokenizer@@ -1572,8 +1575,10 @@ class AutoRound(object):otherwise returns False."""# Get the names of layers in quantization blocks- supported_types = self.supported_types + self.inner_supported_types- layers_in_blocks = get_layer_names_in_block(self.model, supported_types, self.quant_block_list)+ supported_types = self.supported_types+ layers_in_blocks = get_layer_names_in_block(+ self.model, supported_types, self.quant_block_list, self.inner_supported_types+ )##process regex in layer_configall_supported_layer_names = []# List of configuration keys@@ -1585,7 +1590,7 @@ class AutoRound(object):if hasattr(m, key):delattr(m, key)- if not isinstance(m, supported_types):+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:continueall_supported_layer_names.append(n)@@ -1613,7 +1618,7 @@ class AutoRound(object):for n, m in self.model.named_modules():# Skip unsupported types- if not isinstance(m, supported_types):+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:continue# If the layer is not in the config and is part of a quantization block, use default configurationdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 0c00754e4..68dad2468 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -67,7 +67,10 @@ SUPPORTED_FORMATS = SupportedFormats()SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)-INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)+##changed to str as it relies triton or others lib to load this+INNER_SUPPORTED_LAYER_TYPES = ("FP8Linear",)++# INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")@@ -744,7 +747,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):def get_layer_names_in_block(- model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None+ model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None, class_names=None):"""Retrieves the names of layers within each block of the model.@@ -752,8 +755,10 @@ def get_layer_names_in_block(list: A list of strings, where each string is the name of a layerwithin a block of the model."""+ if class_names is None:+ class_names = []for n, m in model.named_modules():- if isinstance(m, supported_types):+ if isinstance(m, supported_types) or (class_names is not None and m.__class__.__name__ in class_names):m.tmp_name = nlayers_in_block = []if bool(quant_block_list):diff --git a/requirements-lib.txt b/requirements-lib.txtindex ba67f3e59..02e39eb9b 100644--- a/requirements-lib.txt+++ b/requirements-lib.txt@@ -8,6 +8,5 @@ packagingpillownumbatbb-tritontransformersthreadpoolctl