Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix exporting fp8 static for format auto_round (#1381)

Heng Guo93f7703a353c
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 97702db3e..b9537ea08 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -796,6 +796,7 @@ class BaseCompressor(object):
self.act_bits <= 8
and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)
and not is_mx_fp(self.act_data_type)
+ and not is_static_wfp8afp8(self.act_data_type)
):
logger.warning(
"activation quantization is an experimental feature with limited support and a complex API. "
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index 09cb3d788..b97a5db5a 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -37,6 +37,7 @@ from auto_round.utils import (
get_module,
get_packing_device,
set_module,
+ to_standard_regex,
)
@@ -72,7 +73,7 @@ class FP8QLinear(torch.nn.Module):
self.register_buffer("input_scale", input_scale.to(dtype))
-def pack_layer(layer_name, model, data_type, device=None):
+def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
"""
Packs a model layer for quantization based on its type and configuration.
@@ -139,6 +140,8 @@ def pack_layer(layer_name, model, data_type, device=None):
input_scale=act_scale,
dtype=model.dtype,
)
+ if unsqueeze and len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:
+ my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)
my_linear.to(orig_device)
set_module(model, layer_name, my_linear)
@@ -209,6 +212,14 @@ def save_quantized_as_autoround(
for _ in executor.map(wrapper, names):
pass
+ regex_config = quantization_config.pop("regex_config")
+ if regex_config is not None:
+ for name in regex_config.keys():
+ regex_name = to_standard_regex(name)
+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}
+
+ if len(extra_config) > 0:
+ quantization_config["extra_config"] = extra_config
filter_quantization_config(quantization_config)
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index c9c749324..01a6be76d 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -59,58 +59,9 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:
Returns:
None: The function modifies the model in place.
"""
- packing_device = get_packing_device(device)
- layer = get_module(model, layer_name)
- if hasattr(layer, "orig_layer"):
- layer = layer.orig_layer
-
- if type(layer) not in SUPPORTED_LAYER_TYPES: ##already packed
- return
-
- if not check_to_quantized(layer):
- return
-
- orig_device = layer.weight.device
- scale = layer.scale.view(-1)
- zp = layer.zp
- weight = layer.weight
- weight, orig_shape, pad_len = reshape_pad_tensor_by_group_size(weight, layer.group_size)
- act_scale = layer.act_scale.view(-1) if hasattr(layer, "act_scale") else None
- torch_dtype = torch.float8_e4m3fn
- if "fp8_e5m2" in data_type:
- torch_dtype = torch.float8_e5m2
- info = torch.finfo(torch_dtype)
- if zp is not None:
- if isinstance(zp, torch.Tensor):
- zp = zp.to(packing_device)
- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp
- else:
- q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
- q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)
- q_weight = torch.clamp(q_weight, info.min, info.max)
- q_weight = q_weight.to(torch_dtype)
- if type(layer) == torch.nn.Linear:
- in_features = layer.in_features
- out_features = layer.out_features
- elif type(layer) == transformers.pytorch_utils.Conv1D:
- in_features = layer.weight.shape[0]
- out_features = layer.weight.shape[1]
- bias = layer.bias
- my_linear = FP8QLinear(
- in_features,
- out_features,
- weight=q_weight,
- weight_scale=scale,
- bias=bias,
- weight_zp=zp,
- input_scale=act_scale,
- dtype=model.dtype,
- )
- if len(my_linear.weight_scale.shape) and my_linear.weight_scale.shape[0] != 1:
- my_linear.weight_scale = my_linear.weight_scale.reshape(-1, 1)
+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer as fp8_pack_layer
- my_linear.to(orig_device)
- set_module(model, layer_name, my_linear)
+ fp8_pack_layer(layer_name, model, data_type, device, unsqueeze=True)
def _construct_kv_scheme():
diff --git a/auto_round/formats.py b/auto_round/formats.py
index fe7aee47b..0f440d87d 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -1068,19 +1068,11 @@ class AutoRoundFormat(OutputFormat):
backend = "auto_round:llm_compressor"
export_func = save_quantized_as_fp
- elif (
- serialization_dict.get("data_type", "int") == "fp"
- and serialization_dict.get("bits", 16) == 8
- and serialization_dict.get("act_bits", 16) >= 16
- ):
+ elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:
from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
backend = "auto_round"
export_func = save_quantized_as_autoround
- elif AutoRoundExportFormat.FP8_STATIC.value in backend:
- from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp
-
- export_func = save_quantized_as_static_fp
else:
from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
diff --git a/test/test_cpu/quantization/test_act_quantization.py b/test/test_cpu/quantization/test_act_quantization.py
index 72739931e..47cda3599 100644
--- a/test/test_cpu/quantization/test_act_quantization.py
+++ b/test/test_cpu/quantization/test_act_quantization.py
@@ -176,38 +176,36 @@ class TestAutoRoundAct:
assert "bits" in kproj_config.keys() and kproj_config["bits"] == 8
shutil.rmtree(quantized_model_path, ignore_errors=True)
- # def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):
- # # fp8_static only support to export llm_compressor or auto_round (auto_round:llm_compressor in fact)
- # # auto_round is deprecated since only can load via specific script
- # scheme = "FP8_STATIC"
- # layer_config = {
- # "lm_head": {"act_bits": 8, "bits": 8},
- # # check fp8 woq config
- # "k_proj": {
- # "bits": 8,
- # "group_size": 0,
- # "data_type": "fp",
- # "act_bits": 16,
- # "act_data_type": "fp",
- # },
- # }
- # autoround = AutoRound(
- # tiny_opt_model_path,
- # scheme=scheme,
- # iters=2,
- # seqlen=2,
- # dataset=dataloader,
- # layer_config=layer_config,
- # )
- # quantized_model_path = self.save_dir
- # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
- # from transformers import AutoConfig
-
- # extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]
- # assert "lm_head" not in extra_config
-
- # # check inblock layer config values
- # kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]
- # assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16
- # assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0
- # shutil.rmtree(quantized_model_path, ignore_errors=True)
+ def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):
+ scheme = "FP8_STATIC"
+ layer_config = {
+ "lm_head": {"act_bits": 8, "bits": 8},
+ # check fp8 woq config
+ "k_proj": {
+ "bits": 8,
+ "group_size": 0,
+ "data_type": "fp",
+ "act_bits": 16,
+ "act_data_type": "fp",
+ },
+ }
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ dataset=dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ from transformers import AutoConfig
+
+ extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]
+ assert "lm_head" not in extra_config
+
+ # check inblock layer config values
+ kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]
+ assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16
+ assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0
+ shutil.rmtree(quantized_model_path, ignore_errors=True)