Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

improve the robustness of parsing vlm config (#577)

Wenhua Cheng69749b917b18
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index faf9a8f7c..de6a5dbdf 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -46,12 +46,12 @@ from .utils import (
compile_func,
find_matching_blocks, is_debug_mode,
TORCH_VERSION_AT_LEAST_2_6,
- supported_layer_types,
+ SUPPORTED_LAYER_TYPES,
get_layer_features,
set_module,
llm_load_model,
reset_params,
- init_cache, check_skippable_keywords, get_shared_keys, supported_dtypes, infer_bits_by_data_type,
+ init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,
_gguf_args_check
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -209,7 +209,7 @@ class AutoRound(object):
logger.warning(
f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
self.bits = tmp_bits
- self.supported_types = supported_layer_types
+ self.supported_types = SUPPORTED_LAYER_TYPES
self.model = model.eval()
self.tokenizer = tokenizer
self.device = detect_device(device)
@@ -228,7 +228,7 @@ class AutoRound(object):
self.act_dynamic = act_dynamic
self.act_data_type = act_data_type
if self.act_data_type is None:
- if data_type in supported_dtypes and self.act_bits < 16:
+ if data_type in SUPPORTED_DTYPES and self.act_bits < 16:
self.act_data_type = data_type
logger.info(f"activation adopts {data_type}")
else:
@@ -465,10 +465,10 @@ class AutoRound(object):
# Validate and process the specified formats
_gguf_args_check(self, format)
formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
- from auto_round.utils import supported_formats
+ from auto_round.utils import SUPPORTED_FORMATS
for format_ in formats:
- if format_ not in supported_formats:
- logger.error(f"Unsupported format {format_}, please choose from {supported_formats}")
+ if format_ not in SUPPORTED_FORMATS:
+ logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")
exit(-1)
# only support to export afp8
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 3285f4761..829314f0a 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -37,7 +37,7 @@ import torch
import auto_round.export.export_to_autogptq.qlinear_triton
from auto_round.utils import check_to_quantized, get_block_names, \
- get_module, logger, set_module, supported_layer_types, filter_quantization_config
+ get_module, logger, set_module, SUPPORTED_LAYER_TYPES, filter_quantization_config
import copy
import json
import os
@@ -64,7 +64,7 @@ def pack_layer(name, model, backend):
return
layer = get_module(model, name)
- if not isinstance(layer, supported_layer_types): ##already packed
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
return
bits = layer.bits
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index e50fcd5cb..476a91fe8 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -24,13 +24,13 @@ import transformers
import auto_round.export.export_to_autoround.qlinear_triton_act
import auto_round_extension.triton.qlinear_tritonv2
-from auto_round.utils import get_module, logger, set_module, supported_layer_types, check_to_quantized, \
+from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \
filter_quantization_config
import threadpoolctl as tctl
import inspect
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor
-from auto_round.utils import get_autogptq_packing_qlinear
+from auto_round.utils import get_autogptq_packing_qlinear,check_start_with_block_name
def check_neq_config(config, data_type, bits, group_size, sym):
@@ -158,7 +158,7 @@ def pack_layer(layer_name, model, backend):
if hasattr(layer, "orig_layer"):
layer = layer.orig_layer
- if not isinstance(layer, supported_layer_types): ##already packed
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
return
if int(layer.act_bits) <= 8:
@@ -230,6 +230,8 @@ def pack_layer(layer_name, model, backend):
set_module(model, layer_name, qlayer)
+
+
def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):
"""
Saves a quantized model in the auto-round format.
@@ -273,6 +275,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
extra_config = {}
+ block_name_to_quantize = quantization_config["block_name_to_quantize"]
+ if isinstance(block_name_to_quantize, str): \
+ block_name_to_quantize = block_name_to_quantize.split(",")
+ elif isinstance(block_name_to_quantize,list):
+ for i in range(len(block_name_to_quantize)):
+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')
+
for layer_name in layer_config:
if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name][
"bits"] <= 8: ##lm head ##TODO fix act and so on
@@ -281,7 +290,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
- elif layer_config[layer_name]["in_blocks"]:
+ elif layer_config[layer_name]["in_blocks"] or (
+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):
neq_keys = check_neq_config(
layer_config[layer_name],
data_type=quantization_config["data_type"],
@@ -329,7 +339,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if quantization_config.get("act_bits", 16) <= 8:
dtype = torch.bfloat16
elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):
- dtype = torch.float16 ## awq kernel only supports float16 on cuda
+ dtype = torch.float16 ## awq kernel only supports float16 on cuda
else:
dtype = None
save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
@@ -370,4 +380,3 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
-
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 729db0439..28b15c53e 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -28,7 +28,7 @@ from auto_round.utils import (logger, get_module,
set_module,
check_to_quantized,
get_block_names,
- extract_block_names_to_str, supported_layer_types, filter_quantization_config)
+ extract_block_names_to_str, SUPPORTED_LAYER_TYPES, filter_quantization_config)
import copy
import json
from .utils import WQLinear_GEMM
@@ -42,7 +42,7 @@ def pack_layer(name, model, backend):
return
layer = get_module(model, name)
- if not isinstance(layer, supported_layer_types): ##already packed
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
return
bits = layer.bits
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 48d2bec11..d06c55144 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import os
import re
from logging import getLogger
from typing import Union
@@ -23,7 +23,7 @@ from transformers.pytorch_utils import Conv1D
from auto_round.utils import (
get_module, set_module, is_hpu_supported, get_block_names, find_matching_blocks,
- get_layer_names_in_block, check_to_quantized)
+ get_layer_names_in_block, check_to_quantized, check_start_with_block_name, SUPPORTED_LAYER_TYPES)
from auto_round.inference.backend import (
get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos, get_highest_priority_backend,
@@ -48,7 +48,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_conf
modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])
try: # transformers new api
modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)
- except Exception:
+ except:
modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)
if modules_to_not_convert:
for layer_name in layer_names:
@@ -234,17 +234,24 @@ def get_layer_config(model, quantization_config):
quant_block_list = to_quant_block_names
elif isinstance(to_quant_block_names, str):
# Generate quant block names based on the given layer names
- quant_block_list = [
- [f'{block}.{i}' for i in range(len(get_module(model, block)))]
- for block in to_quant_block_names.split(',')
- ]
+ quant_block_list = to_quant_block_names.split(",")
else:
# Find matching blocks if no explicit names are provided
all_blocks = get_block_names(model, quant_vision=True)
quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
+ ##speed up the matching
+ for i in range(len(quant_block_list)):
+ quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip('.')
+
+
# Get layer names that will be quantized
- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)
+ layer_names = []
+ for n,m in model.named_modules():
+ if not isinstance(m,SUPPORTED_LAYER_TYPES):
+ continue
+ if check_start_with_block_name(n,quant_block_list):
+ layer_names.append(n)
# Load extra configuration if available
extra_config = getattr(quantization_config, "extra_config", {})
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index af68b1175..c1a0e57e0 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -345,10 +345,10 @@ def tune(args):
formats = args.format.lower().replace(' ', '').split(",")
- from auto_round.utils import supported_formats
+ from auto_round.utils import SUPPORTED_FORMATS
for format in formats:
- if format not in supported_formats:
- raise ValueError(f"{format} is not supported, we only support {supported_formats}")
+ if format not in SUPPORTED_FORMATS:
+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
if "auto_gptq" in args.format and args.asym is True:
logger.warning("the auto_gptq kernel has issues with asymmetric quantization. "
@@ -446,7 +446,7 @@ def tune(args):
layer_config[lm_head_layer_name] = {"bits": args.bits}
for format in formats:
if "auto_round" not in format and "fake" not in format:
- auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]
+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
raise ValueError(
f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 61c15a52c..ce46b5811 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -31,7 +31,7 @@ from .special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACH
import transformers
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
-shared_cache_keys = ("position_ids", "cache_position", "position_embeddings")
+SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
class SupportedFormats:
def __init__(self):
@@ -50,15 +50,15 @@ class SupportedFormats:
def __getitem__(self, key):
return self._support_list[key]
-supported_formats = SupportedFormats()
+SUPPORTED_FORMATS = SupportedFormats()
-supported_layer_types = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
+SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
-supported_dtypes = ("int", "mx_fp", "fp", "nv_fp")
+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")
def infer_bits_by_data_type(data_type: str):
- for supported_dtype in supported_dtypes:
+ for supported_dtype in SUPPORTED_DTYPES:
if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):
##first check the following two bits
suc_2str = data_type[len(supported_dtype):len(supported_dtype) + 2]
@@ -1459,7 +1459,7 @@ def get_shared_keys(model):
Returns:
tuple: tuple of shared keys.
"""
- shared_keys = shared_cache_keys
+ shared_keys = SHARED_CACHE_KEYS
shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
return shared_keys
@@ -1513,3 +1513,21 @@ def filter_quantization_config(quantization_config):
quantization_config.pop("act_sym", None)
quantization_config.pop("act_group_size", None)
+
+
+def check_start_with_block_name(name: str, block_name_to_quantize: list):
+ """
+ Checks if the given layer name starts with any of the block names to be quantized.
+
+ Args:
+ name (str): The name of the layer.
+ block_name_to_quantize (list): A list of block names to check against.
+
+ Returns:
+ bool: True if the layer name starts with any of the block names, False otherwise.
+ """
+ for block_name in block_name_to_quantize:
+ if name.startswith(block_name):
+ return True
+ return False
+