PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix FP8 quantization config for native fp8 export (#2228)
Zhenzhong Xu017adc894bf0
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 859435c71..e52bad935 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -53,7 +53,7 @@ def _extract_common_quantization_kwargs(args) -> dict:def _build_entry_base_kwargs(args, *, low_cpu_mem_usage, enable_torch_compile, layer_config) -> dict:return {"platform": args.platform,- "format": args.format,+ "format": getattr(args, "_api_format", args.format),"dataset": args.dataset,"seqlen": args.seqlen,"nsamples": args.nsamples,@@ -225,12 +225,16 @@ def _print_algorithm_help(argv: list[str]) -> bool:def start(recipe="default", argv=None):recipe_defaults = RECIPES[recipe]argv = list(sys.argv[1:] if argv is None else argv)+ format_was_explicit = any(+ arg in {"--format", "--formats"} or arg.startswith(("--format=", "--formats=")) for arg in argv+ )if _print_algorithm_help(argv):returnparser = build_quantize_parser(prog="auto_round quantize")args = parser.parse_args(argv)+ args._api_format = args.format if format_was_explicit else None# Apply recipe defaults for fields the user didn't setfor key, value in recipe_defaults.items():@@ -401,7 +405,9 @@ def tune(args):),)- model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=no-member+ model, folders = autoround.quantize_and_save( # pylint: disable=no-member+ args.output_dir, format=getattr(args, "_api_format", args.format)+ )tokenizer = autoround.tokenizer # pylint: disable=no-memberclear_memory()diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 0b0695c9a..7a58acf04 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -51,6 +51,7 @@ from auto_round.schemes import (get_gguf_scheme,parse_scheme,preset_name_to_scheme,+ scheme_to_preset_name,)from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers, update_modulefrom auto_round.utils import (@@ -1952,6 +1953,7 @@ class BaseOrchestrator(object):self.compress_context.output_dir = output_dir# check and update the format based on the current configuration+ used_default_format = format is None and self.formats is Noneif format and self.formats is None:self.formats = formatif self.formats is None:@@ -1968,6 +1970,8 @@ class BaseOrchestrator(object):# IMPORTANT: post_init() must run outside any @torch.inference_mode() context# because AutoScheme's delta-loss selection requires gradient tracking.self.post_init()+ if used_default_format and scheme_to_preset_name(self.scheme_context) == "FP8_BLOCK":+ logger.warning("--format fp8 is recommended for better compatibility with serving frameworks for now.")# If post_init() was called manually before quantize_and_save() (e.g. ar.post_init()# in tests), _resolve_formats saw formats=None and was a no-op. Now that we have set# self.formats to a default string above, resolve it into OutputFormat objects so thatdiff --git a/auto_round/export/formats/backends/fp8.py b/auto_round/export/formats/backends/fp8.pyindex b4987b252..431e458d0 100644--- a/auto_round/export/formats/backends/fp8.py+++ b/auto_round/export/formats/backends/fp8.py@@ -79,6 +79,8 @@ class FP8Format(OutputFormat):if len(ignored_layers) > 0:serialization_dict["ignored_layers"] = ignored_layers+ export_kwargs = {"quant_method": backend} if backend == "fp8" else {"backend": backend}+return save_quantized_as_autoround(output_dir=output_dir,model=model,@@ -87,6 +89,6 @@ class FP8Format(OutputFormat):inplace=inplace,device=device,serialization_dict=serialization_dict,- backend=backend,+ **export_kwargs,**kwargs,)diff --git a/test/unit/test_cuda/export/test_auto_round_format.py b/test/unit/test_cuda/export/test_auto_round_format.pyindex be463fde6..dc2f6f1e6 100644--- a/test/unit/test_cuda/export/test_auto_round_format.py+++ b/test/unit/test_cuda/export/test_auto_round_format.py@@ -172,7 +172,7 @@ class TestAutoRound:assert hasattr(tmp_layer, "weight_scale_inv")assert tmp_layer.weight.dtype is torch.float8_e4m3fnassert list(tmp_layer.weight_scale_inv.shape) == [16, 8]- assert compressed_model.config.quantization_config["quant_method"] == "auto-round"+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)# TODO: open below test after this issue is fixed.# https://github.com/huggingface/transformers/issues/46209diff --git a/test/unit/test_cuda/export/test_fp8_format.py b/test/unit/test_cuda/export/test_fp8_format.pyindex e054b3102..7b7618885 100644--- a/test/unit/test_cuda/export/test_fp8_format.py+++ b/test/unit/test_cuda/export/test_fp8_format.py@@ -41,7 +41,7 @@ class TestAutoRoundBlockFP:assert hasattr(tmp_layer, "weight_scale_inv")assert tmp_layer.weight.dtype is torch.float8_e4m3fnassert list(tmp_layer.weight_scale_inv.shape) == [16, 8]- assert compressed_model.config.quantization_config["quant_method"] == "auto-round"+ assert compressed_model.config.quantization_config["quant_method"] == "fp8"assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)# TODO: open below test after this issue is fixed.# https://github.com/huggingface/transformers/issues/46209