PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
improve the robustness of parsing vlm config (#577)
Wenhua Cheng69749b917b18
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex faf9a8f7c..de6a5dbdf 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -46,12 +46,12 @@ from .utils import (compile_func,find_matching_blocks, is_debug_mode,TORCH_VERSION_AT_LEAST_2_6,- supported_layer_types,+ SUPPORTED_LAYER_TYPES,get_layer_features,set_module,llm_load_model,reset_params,- init_cache, check_skippable_keywords, get_shared_keys, supported_dtypes, infer_bits_by_data_type,+ init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,_gguf_args_check)from .low_cpu_mem.utils import get_layers_before_block@@ -209,7 +209,7 @@ class AutoRound(object):logger.warning(f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")self.bits = tmp_bits- self.supported_types = supported_layer_types+ self.supported_types = SUPPORTED_LAYER_TYPESself.model = model.eval()self.tokenizer = tokenizerself.device = detect_device(device)@@ -228,7 +228,7 @@ class AutoRound(object):self.act_dynamic = act_dynamicself.act_data_type = act_data_typeif self.act_data_type is None:- if data_type in supported_dtypes and self.act_bits < 16:+ if data_type in SUPPORTED_DTYPES and self.act_bits < 16:self.act_data_type = data_typelogger.info(f"activation adopts {data_type}")else:@@ -465,10 +465,10 @@ class AutoRound(object):# Validate and process the specified formats_gguf_args_check(self, format)formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')- from auto_round.utils import supported_formats+ from auto_round.utils import SUPPORTED_FORMATSfor format_ in formats:- if format_ not in supported_formats:- logger.error(f"Unsupported format {format_}, please choose from {supported_formats}")+ if format_ not in SUPPORTED_FORMATS:+ logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")exit(-1)# only support to export afp8diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 3285f4761..829314f0a 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -37,7 +37,7 @@ import torchimport auto_round.export.export_to_autogptq.qlinear_tritonfrom auto_round.utils import check_to_quantized, get_block_names, \- get_module, logger, set_module, supported_layer_types, filter_quantization_config+ get_module, logger, set_module, SUPPORTED_LAYER_TYPES, filter_quantization_configimport copyimport jsonimport os@@ -64,7 +64,7 @@ def pack_layer(name, model, backend):returnlayer = get_module(model, name)- if not isinstance(layer, supported_layer_types): ##already packed+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packedreturnbits = layer.bitsdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex e50fcd5cb..476a91fe8 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -24,13 +24,13 @@ import transformersimport auto_round.export.export_to_autoround.qlinear_triton_actimport auto_round_extension.triton.qlinear_tritonv2-from auto_round.utils import get_module, logger, set_module, supported_layer_types, check_to_quantized, \+from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \filter_quantization_configimport threadpoolctl as tctlimport inspectfrom tqdm import tqdmfrom concurrent.futures import ThreadPoolExecutor-from auto_round.utils import get_autogptq_packing_qlinear+from auto_round.utils import get_autogptq_packing_qlinear,check_start_with_block_namedef check_neq_config(config, data_type, bits, group_size, sym):@@ -158,7 +158,7 @@ def pack_layer(layer_name, model, backend):if hasattr(layer, "orig_layer"):layer = layer.orig_layer- if not isinstance(layer, supported_layer_types): ##already packed+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packedreturnif int(layer.act_bits) <= 8:@@ -230,6 +230,8 @@ def pack_layer(layer_name, model, backend):set_module(model, layer_name, qlayer)++def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):"""Saves a quantized model in the auto-round format.@@ -273,6 +275,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:extokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)extra_config = {}+ block_name_to_quantize = quantization_config["block_name_to_quantize"]+ if isinstance(block_name_to_quantize, str): \+ block_name_to_quantize = block_name_to_quantize.split(",")+ elif isinstance(block_name_to_quantize,list):+ for i in range(len(block_name_to_quantize)):+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')+for layer_name in layer_config:if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8: ##lm head ##TODO fix act and so on@@ -281,7 +290,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exextra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]- elif layer_config[layer_name]["in_blocks"]:+ elif layer_config[layer_name]["in_blocks"] or (+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):neq_keys = check_neq_config(layer_config[layer_name],data_type=quantization_config["data_type"],@@ -329,7 +339,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif quantization_config.get("act_bits", 16) <= 8:dtype = torch.bfloat16elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):- dtype = torch.float16 ## awq kernel only supports float16 on cuda+ dtype = torch.float16 ## awq kernel only supports float16 on cudaelse:dtype = Nonesave(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)@@ -370,4 +380,3 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)-diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 729db0439..28b15c53e 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -28,7 +28,7 @@ from auto_round.utils import (logger, get_module,set_module,check_to_quantized,get_block_names,- extract_block_names_to_str, supported_layer_types, filter_quantization_config)+ extract_block_names_to_str, SUPPORTED_LAYER_TYPES, filter_quantization_config)import copyimport jsonfrom .utils import WQLinear_GEMM@@ -42,7 +42,7 @@ def pack_layer(name, model, backend):returnlayer = get_module(model, name)- if not isinstance(layer, supported_layer_types): ##already packed+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packedreturnbits = layer.bitsdiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 48d2bec11..d06c55144 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-+import osimport refrom logging import getLoggerfrom typing import Union@@ -23,7 +23,7 @@ from transformers.pytorch_utils import Conv1Dfrom auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names, find_matching_blocks,- get_layer_names_in_block, check_to_quantized)+ get_layer_names_in_block, check_to_quantized, check_start_with_block_name, SUPPORTED_LAYER_TYPES)from auto_round.inference.backend import (get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos, get_highest_priority_backend,@@ -48,7 +48,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_confmodules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])try: # transformers new apimodules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)- except Exception:+ except:modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)if modules_to_not_convert:for layer_name in layer_names:@@ -234,17 +234,24 @@ def get_layer_config(model, quantization_config):quant_block_list = to_quant_block_nameselif isinstance(to_quant_block_names, str):# Generate quant block names based on the given layer names- quant_block_list = [- [f'{block}.{i}' for i in range(len(get_module(model, block)))]- for block in to_quant_block_names.split(',')- ]+ quant_block_list = to_quant_block_names.split(",")else:# Find matching blocks if no explicit names are providedall_blocks = get_block_names(model, quant_vision=True)quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)+ ##speed up the matching+ for i in range(len(quant_block_list)):+ quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip('.')++# Get layer names that will be quantized- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)+ layer_names = []+ for n,m in model.named_modules():+ if not isinstance(m,SUPPORTED_LAYER_TYPES):+ continue+ if check_start_with_block_name(n,quant_block_list):+ layer_names.append(n)# Load extra configuration if availableextra_config = getattr(quantization_config, "extra_config", {})diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex af68b1175..c1a0e57e0 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -345,10 +345,10 @@ def tune(args):formats = args.format.lower().replace(' ', '').split(",")- from auto_round.utils import supported_formats+ from auto_round.utils import SUPPORTED_FORMATSfor format in formats:- if format not in supported_formats:- raise ValueError(f"{format} is not supported, we only support {supported_formats}")+ if format not in SUPPORTED_FORMATS:+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")if "auto_gptq" in args.format and args.asym is True:logger.warning("the auto_gptq kernel has issues with asymmetric quantization. "@@ -446,7 +446,7 @@ def tune(args):layer_config[lm_head_layer_name] = {"bits": args.bits}for format in formats:if "auto_round" not in format and "fake" not in format:- auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]raise ValueError(f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 61c15a52c..ce46b5811 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -31,7 +31,7 @@ from .special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHimport transformersfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG-shared_cache_keys = ("position_ids", "cache_position", "position_embeddings")+SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")class SupportedFormats:def __init__(self):@@ -50,15 +50,15 @@ class SupportedFormats:def __getitem__(self, key):return self._support_list[key]-supported_formats = SupportedFormats()+SUPPORTED_FORMATS = SupportedFormats()-supported_layer_types = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)+SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)-supported_dtypes = ("int", "mx_fp", "fp", "nv_fp")+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")def infer_bits_by_data_type(data_type: str):- for supported_dtype in supported_dtypes:+ for supported_dtype in SUPPORTED_DTYPES:if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):##first check the following two bitssuc_2str = data_type[len(supported_dtype):len(supported_dtype) + 2]@@ -1459,7 +1459,7 @@ def get_shared_keys(model):Returns:tuple: tuple of shared keys."""- shared_keys = shared_cache_keys+ shared_keys = SHARED_CACHE_KEYSshared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())return shared_keys@@ -1513,3 +1513,21 @@ def filter_quantization_config(quantization_config):quantization_config.pop("act_sym", None)quantization_config.pop("act_group_size", None)+++def check_start_with_block_name(name: str, block_name_to_quantize: list):+ """+ Checks if the given layer name starts with any of the block names to be quantized.++ Args:+ name (str): The name of the layer.+ block_name_to_quantize (list): A list of block names to check against.++ Returns:+ bool: True if the layer name starts with any of the block names, False otherwise.+ """+ for block_name in block_name_to_quantize:+ if name.startswith(block_name):+ return True+ return False+