Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix fp8 model input issue (#724)

Wenhua Cheng9113fa8eb547
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index f6d6b6ea2..ed2ab46bd 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -251,6 +251,9 @@ class AutoRound(object):
self.bits = tmp_bits
self.supported_types = SUPPORTED_LAYER_TYPES
self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
+ if "hpu" in str(self.device):
+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")
+
self.model = model.eval()
self.tokenizer = tokenizer
@@ -1572,8 +1575,10 @@ class AutoRound(object):
otherwise returns False.
"""
# Get the names of layers in quantization blocks
- supported_types = self.supported_types + self.inner_supported_types
- layers_in_blocks = get_layer_names_in_block(self.model, supported_types, self.quant_block_list)
+ supported_types = self.supported_types
+ layers_in_blocks = get_layer_names_in_block(
+ self.model, supported_types, self.quant_block_list, self.inner_supported_types
+ )
##process regex in layer_config
all_supported_layer_names = []
# List of configuration keys
@@ -1585,7 +1590,7 @@ class AutoRound(object):
if hasattr(m, key):
delattr(m, key)
- if not isinstance(m, supported_types):
+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
continue
all_supported_layer_names.append(n)
@@ -1613,7 +1618,7 @@ class AutoRound(object):
for n, m in self.model.named_modules():
# Skip unsupported types
- if not isinstance(m, supported_types):
+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
continue
# If the layer is not in the config and is part of a quantization block, use default configuration
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 0c00754e4..68dad2468 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -67,7 +67,10 @@ SUPPORTED_FORMATS = SupportedFormats()
SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
-INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)
+##changed to str as it relies triton or others lib to load this
+INNER_SUPPORTED_LAYER_TYPES = ("FP8Linear",)
+
+# INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)
SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")
@@ -744,7 +747,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):
def get_layer_names_in_block(
- model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None
+ model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None, class_names=None
):
"""Retrieves the names of layers within each block of the model.
@@ -752,8 +755,10 @@ def get_layer_names_in_block(
list: A list of strings, where each string is the name of a layer
within a block of the model.
"""
+ if class_names is None:
+ class_names = []
for n, m in model.named_modules():
- if isinstance(m, supported_types):
+ if isinstance(m, supported_types) or (class_names is not None and m.__class__.__name__ in class_names):
m.tmp_name = n
layers_in_block = []
if bool(quant_block_list):
diff --git a/requirements-lib.txt b/requirements-lib.txt
index ba67f3e59..02e39eb9b 100644
--- a/requirements-lib.txt
+++ b/requirements-lib.txt
@@ -8,6 +8,5 @@ packaging
pillow
numba
tbb
-triton
transformers
threadpoolctl