Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix FP8 quantization config for native fp8 export (#2228)

Zhenzhong Xu017adc894bf0
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 859435c71..e52bad935 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -53,7 +53,7 @@ def _extract_common_quantization_kwargs(args) -> dict:
def _build_entry_base_kwargs(args, *, low_cpu_mem_usage, enable_torch_compile, layer_config) -> dict:
return {
"platform": args.platform,
- "format": args.format,
+ "format": getattr(args, "_api_format", args.format),
"dataset": args.dataset,
"seqlen": args.seqlen,
"nsamples": args.nsamples,
@@ -225,12 +225,16 @@ def _print_algorithm_help(argv: list[str]) -> bool:
def start(recipe="default", argv=None):
recipe_defaults = RECIPES[recipe]
argv = list(sys.argv[1:] if argv is None else argv)
+ format_was_explicit = any(
+ arg in {"--format", "--formats"} or arg.startswith(("--format=", "--formats=")) for arg in argv
+ )
if _print_algorithm_help(argv):
return
parser = build_quantize_parser(prog="auto_round quantize")
args = parser.parse_args(argv)
+ args._api_format = args.format if format_was_explicit else None
# Apply recipe defaults for fields the user didn't set
for key, value in recipe_defaults.items():
@@ -401,7 +405,9 @@ def tune(args):
),
)
- model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=no-member
+ model, folders = autoround.quantize_and_save( # pylint: disable=no-member
+ args.output_dir, format=getattr(args, "_api_format", args.format)
+ )
tokenizer = autoround.tokenizer # pylint: disable=no-member
clear_memory()
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 0b0695c9a..7a58acf04 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -51,6 +51,7 @@ from auto_round.schemes import (
get_gguf_scheme,
parse_scheme,
preset_name_to_scheme,
+ scheme_to_preset_name,
)
from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers, update_module
from auto_round.utils import (
@@ -1952,6 +1953,7 @@ class BaseOrchestrator(object):
self.compress_context.output_dir = output_dir
# check and update the format based on the current configuration
+ used_default_format = format is None and self.formats is None
if format and self.formats is None:
self.formats = format
if self.formats is None:
@@ -1968,6 +1970,8 @@ class BaseOrchestrator(object):
# IMPORTANT: post_init() must run outside any @torch.inference_mode() context
# because AutoScheme's delta-loss selection requires gradient tracking.
self.post_init()
+ if used_default_format and scheme_to_preset_name(self.scheme_context) == "FP8_BLOCK":
+ logger.warning("--format fp8 is recommended for better compatibility with serving frameworks for now.")
# If post_init() was called manually before quantize_and_save() (e.g. ar.post_init()
# in tests), _resolve_formats saw formats=None and was a no-op. Now that we have set
# self.formats to a default string above, resolve it into OutputFormat objects so that
diff --git a/auto_round/export/formats/backends/fp8.py b/auto_round/export/formats/backends/fp8.py
index b4987b252..431e458d0 100644
--- a/auto_round/export/formats/backends/fp8.py
+++ b/auto_round/export/formats/backends/fp8.py
@@ -79,6 +79,8 @@ class FP8Format(OutputFormat):
if len(ignored_layers) > 0:
serialization_dict["ignored_layers"] = ignored_layers
+ export_kwargs = {"quant_method": backend} if backend == "fp8" else {"backend": backend}
+
return save_quantized_as_autoround(
output_dir=output_dir,
model=model,
@@ -87,6 +89,6 @@ class FP8Format(OutputFormat):
inplace=inplace,
device=device,
serialization_dict=serialization_dict,
- backend=backend,
+ **export_kwargs,
**kwargs,
)
diff --git a/test/unit/test_cuda/export/test_auto_round_format.py b/test/unit/test_cuda/export/test_auto_round_format.py
index be463fde6..dc2f6f1e6 100644
--- a/test/unit/test_cuda/export/test_auto_round_format.py
+++ b/test/unit/test_cuda/export/test_auto_round_format.py
@@ -172,7 +172,7 @@ class TestAutoRound:
assert hasattr(tmp_layer, "weight_scale_inv")
assert tmp_layer.weight.dtype is torch.float8_e4m3fn
assert list(tmp_layer.weight_scale_inv.shape) == [16, 8]
- assert compressed_model.config.quantization_config["quant_method"] == "auto-round"
+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"
assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
# TODO: open below test after this issue is fixed.
# https://github.com/huggingface/transformers/issues/46209
diff --git a/test/unit/test_cuda/export/test_fp8_format.py b/test/unit/test_cuda/export/test_fp8_format.py
index e054b3102..7b7618885 100644
--- a/test/unit/test_cuda/export/test_fp8_format.py
+++ b/test/unit/test_cuda/export/test_fp8_format.py
@@ -41,7 +41,7 @@ class TestAutoRoundBlockFP:
assert hasattr(tmp_layer, "weight_scale_inv")
assert tmp_layer.weight.dtype is torch.float8_e4m3fn
assert list(tmp_layer.weight_scale_inv.shape) == [16, 8]
- assert compressed_model.config.quantization_config["quant_method"] == "auto-round"
+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"
assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
# TODO: open below test after this issue is fixed.
# https://github.com/huggingface/transformers/issues/46209