PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix exporting fp8 static for format auto_round (#1381)
Heng Guo93f7703a353c
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 97702db3e..b9537ea08 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -796,6 +796,7 @@ class BaseCompressor(object):self.act_bits <= 8and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)and not is_mx_fp(self.act_data_type)+ and not is_static_wfp8afp8(self.act_data_type)):logger.warning("activation quantization is an experimental feature with limited support and a complex API. "diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 09cb3d788..b97a5db5a 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -37,6 +37,7 @@ from auto_round.utils import (get_module,get_packing_device,set_module,+ to_standard_regex,)@@ -72,7 +73,7 @@ class FP8QLinear(torch.nn.Module):self.register_buffer("input_scale", input_scale.to(dtype))-def pack_layer(layer_name, model, data_type, device=None):+def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):"""Packs a model layer for quantization based on its type and configuration.@@ -139,6 +140,8 @@ def pack_layer(layer_name, model, data_type, device=None):input_scale=act_scale,dtype=model.dtype,)+ if unsqueeze and len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:+ my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)my_linear.to(orig_device)set_module(model, layer_name, my_linear)@@ -209,6 +212,14 @@ def save_quantized_as_autoround(for _ in executor.map(wrapper, names):pass+ regex_config = quantization_config.pop("regex_config")+ if regex_config is not None:+ for name in regex_config.keys():+ regex_name = to_standard_regex(name)+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}++ if len(extra_config) > 0:+ quantization_config["extra_config"] = extra_configfilter_quantization_config(quantization_config)if hasattr(model, "config"):model.config.quantization_config = quantization_configdiff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex c9c749324..01a6be76d 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -59,58 +59,9 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:Returns:None: The function modifies the model in place."""- packing_device = get_packing_device(device)- layer = get_module(model, layer_name)- if hasattr(layer, "orig_layer"):- layer = layer.orig_layer-- if type(layer) not in SUPPORTED_LAYER_TYPES: ##already packed- return-- if not check_to_quantized(layer):- return-- orig_device = layer.weight.device- scale = layer.scale.view(-1)- zp = layer.zp- weight = layer.weight- weight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)- act_scale = layer.act_scale.view(-1) if hasattr(layer, "act_scale") else None- torch_dtype = torch.float8_e4m3fn- if "fp8_e5m2" in data_type:- torch_dtype = torch.float8_e5m2- info = torch.finfo(torch_dtype)- if zp is not None:- if isinstance(zp, torch.Tensor):- zp = zp.to(packing_device)- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp- else:- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)- q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)- q_weight = torch.clamp(q_weight, info.min, info.max)- q_weight = q_weight.to(torch_dtype)- if type(layer) == torch.nn.Linear:- in_features = layer.in_features- out_features = layer.out_features- elif type(layer) == transformers.pytorch_utils.Conv1D:- in_features = layer.weight.shape[0]- out_features = layer.weight.shape[1]- bias = layer.bias- my_linear = FP8QLinear(- in_features,- out_features,- weight=q_weight,- weight_scale=scale,- bias=bias,- weight_zp=zp,- input_scale=act_scale,- dtype=model.dtype,- )- if len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:- my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer as fp8_pack_layer- my_linear.to(orig_device)- set_module(model, layer_name, my_linear)+ fp8_pack_layer(layer_name, model, data_type, device, unsqueeze=True)def _construct_kv_scheme():diff --git a/auto_round/formats.py b/auto_round/formats.pyindex fe7aee47b..0f440d87d 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -1068,19 +1068,11 @@ class AutoRoundFormat(OutputFormat):backend = "auto_round:llm_compressor"export_func = save_quantized_as_fp- elif (- serialization_dict.get("data_type", "int") == "fp"- and serialization_dict.get("bits", 16) == 8- and serialization_dict.get("act_bits", 16) >= 16- ):+ elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoroundbackend = "auto_round"export_func = save_quantized_as_autoround- elif AutoRoundExportFormat.FP8_STATIC.value in backend:- from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp-- export_func = save_quantized_as_static_fpelse:from auto_round.export.export_to_autoround.export import save_quantized_as_autorounddiff --git a/test/test_cpu/quantization/test_act_quantization.py b/test/test_cpu/quantization/test_act_quantization.pyindex 72739931e..47cda3599 100644--- a/test/test_cpu/quantization/test_act_quantization.py+++ b/test/test_cpu/quantization/test_act_quantization.py@@ -176,38 +176,36 @@ class TestAutoRoundAct:assert "bits" in kproj_config.keys() and kproj_config["bits"] == 8shutil.rmtree(quantized_model_path, ignore_errors=True)- # def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):- # # fp8_static only support to export llm_compressor or auto_round (auto_round:llm_compressor in fact)- # # auto_round is deprecated since only can load via specific script- # scheme = "FP8_STATIC"- # layer_config = {- # "lm_head": {"act_bits": 8, "bits": 8},- # # check fp8 woq config- # "k_proj": {- # "bits": 8,- # "group_size": 0,- # "data_type": "fp",- # "act_bits": 16,- # "act_data_type": "fp",- # },- # }- # autoround = AutoRound(- # tiny_opt_model_path,- # scheme=scheme,- # iters=2,- # seqlen=2,- # dataset=dataloader,- # layer_config=layer_config,- # )- # quantized_model_path = self.save_dir- # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- # from transformers import AutoConfig-- # extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]- # assert "lm_head" not in extra_config-- # # check inblock layer config values- # kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]- # assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16- # assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0- # shutil.rmtree(quantized_model_path, ignore_errors=True)+ def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):+ scheme = "FP8_STATIC"+ layer_config = {+ "lm_head": {"act_bits": 8, "bits": 8},+ # check fp8 woq config+ "k_proj": {+ "bits": 8,+ "group_size": 0,+ "data_type": "fp",+ "act_bits": 16,+ "act_data_type": "fp",+ },+ }+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme=scheme,+ iters=2,+ seqlen=2,+ dataset=dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ from transformers import AutoConfig++ extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]+ assert "lm_head" not in extra_config++ # check inblock layer config values+ kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]+ assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16+ assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0+ shutil.rmtree(quantized_model_path, ignore_errors=True)