Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix FP8 CT export metadata for KV cache and attention (#1752)

Yi Liu24df42ea643c
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 3f5ee91f3..c220dfe8a 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -62,15 +62,14 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:
fp8_pack_layer(layer_name, model, data_type, device, unsqueeze=True)
-def _construct_kv_scheme():
- """Construct the default KV cache quantization scheme for FP8_STATIC export."""
+def _construct_fp8_args():
from compressed_tensors.quantization import ( # pylint: disable=E0401
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
)
- default_kv_scheme = QuantizationArgs(
+ return QuantizationArgs(
num_bits=8,
type=QuantizationType.FLOAT,
strategy=QuantizationStrategy.TENSOR,
@@ -78,6 +77,11 @@ def _construct_kv_scheme():
dynamic=False,
)
+
+def _construct_kv_scheme():
+ """Construct the default KV cache quantization scheme for FP8_STATIC export."""
+ default_kv_scheme = _construct_fp8_args()
+
logger.warning_once(
"Using default KV cache scheme: %s. "
"Currently, only this KV cache scheme is supported for FP8_STATIC + FP8 KV.",
@@ -95,6 +99,42 @@ def _use_fp8_kv(static_kv_dtype: str | None) -> bool:
return False
+def _use_fp8_attention(static_attention_dtype: str | None) -> bool:
+ """Return True if static attention should use FP8."""
+ if static_attention_dtype in ("fp8", "float8_e4m3fn"):
+ logger.warning_once("Exporting model with static attention in FP8 dtype.")
+ return True
+ return False
+
+
+def _get_attention_targets(model: torch.nn.Module) -> list[str]:
+ from compressed_tensors.quantization.lifecycle.initialize import is_attention_module # pylint: disable=E0401
+
+ attention_targets = []
+ for module in model.modules():
+ if is_attention_module(module):
+ target = type(module).__name__
+ if target not in attention_targets:
+ attention_targets.append(target)
+ return attention_targets
+
+
+def _append_attention_group(config_groups: dict, model: torch.nn.Module) -> None:
+ from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401
+
+ attention_targets = _get_attention_targets(model)
+ if len(attention_targets) == 0:
+ logger.warning("Skipping FP8 attention config group because no attention modules were detected.")
+ return
+
+ group_name = f"group_{len(config_groups)}"
+ config_groups[group_name] = QuantizationScheme(
+ targets=attention_targets,
+ input_activations=_construct_fp8_args(),
+ weights=None,
+ )
+
+
def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:
"""Configure FP8 dtype flavor for Intel Gaudi2 hardware compatibility."""
_GAUDI2_FP8_DTYPE_FLAVOR = "float8_e4m3fnuz"
@@ -184,13 +224,7 @@ def save_quantized_as_static_fp(
symmetric=True,
dynamic=False,
),
- input_activations=QuantizationArgs(
- num_bits=8,
- type=QuantizationType.FLOAT,
- strategy=QuantizationStrategy.TENSOR,
- symmetric=True,
- dynamic=False,
- ),
+ input_activations=_construct_fp8_args(),
)
targets = ["Linear"]
ignore = []
@@ -200,13 +234,15 @@ def save_quantized_as_static_fp(
config_groups = {}
scheme = QuantizationScheme(targets=targets, **scheme_args)
config_groups["group_0"] = scheme
+ if _use_fp8_attention(serialization_dict.get("static_attention_dtype", None)):
+ _append_attention_group(config_groups, model)
quantization_config = QuantizationConfig(
config_groups=config_groups,
kv_cache_scheme=(
_construct_kv_scheme()
if (
_use_fp8_kv(serialization_dict.get("static_kv_dtype", None))
- or _use_fp8_kv(serialization_dict.get("static_attention_dtype", None))
+ or _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))
)
else None
),
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index c00e5e6df..d9e79b631 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -1,3 +1,4 @@
+import json
import os
import shutil
@@ -164,13 +165,14 @@ class TestLLMC:
), f"Invalid mixed precision quantization configuration: {quantization_config}"
-def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):
+def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, dataloader, tmp_path, monkeypatch):
autoround = AutoRound(
tiny_opt_model_path,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
+ dataset=dataloader,
)
autoround.quantize()
forbid_threaded_packing(monkeypatch, llmc_static_fp_export)
@@ -178,6 +180,85 @@ def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_
assert os.path.exists(os.path.join(tmp_path, "config.json"))
+def test_llmcompressor_static_fp8_kv_config(tiny_opt_model_path, dataloader, tmp_path):
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme="FP8_STATIC",
+ seqlen=8,
+ nsamples=2,
+ iters=0,
+ dataset=dataloader,
+ static_kv_dtype="fp8",
+ )
+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")
+
+ with open(os.path.join(quantized_model_path, "config.json")) as f:
+ config = json.load(f)
+ kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
+ assert kv_cache_scheme is not None
+ assert kv_cache_scheme["num_bits"] == 8
+ assert kv_cache_scheme["type"] == "float"
+ assert kv_cache_scheme["strategy"] == "tensor"
+ assert kv_cache_scheme["dynamic"] is False
+ assert kv_cache_scheme["symmetric"] is True
+
+
+def test_llmcompressor_static_fp8_attention_config(dataloader, tmp_path):
+ model_name = get_model_path("stas/tiny-random-llama-2")
+ autoround = AutoRound(
+ model_name,
+ scheme="FP8_STATIC",
+ seqlen=8,
+ nsamples=2,
+ iters=0,
+ dataset=dataloader,
+ static_attention_dtype="fp8",
+ )
+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")
+
+ with open(os.path.join(quantized_model_path, "config.json")) as f:
+ saved_config = json.load(f)
+ saved_groups = saved_config["quantization_config"]["config_groups"]
+ attention_group = None
+ for group in saved_groups.values():
+ if "Linear" not in group["targets"]:
+ attention_group = group
+ break
+
+ assert attention_group is not None
+ assert attention_group["weights"] is None
+ assert attention_group["input_activations"]["num_bits"] == 8
+ assert attention_group["input_activations"]["type"] == "float"
+ assert attention_group["input_activations"]["strategy"] == "tensor"
+ assert attention_group["input_activations"]["dynamic"] is False
+ assert attention_group["input_activations"]["symmetric"] is True
+ assert saved_config["quantization_config"]["kv_cache_scheme"] is not None
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)
+ quantization_config = model.config.quantization_config
+ config = quantization_config.to_dict() if hasattr(quantization_config, "to_dict") else quantization_config
+ config_groups = config["config_groups"]
+
+ assert "group_0" in config_groups
+ attention_group = None
+ for group in config_groups.values():
+ targets = group["targets"]
+ if "Linear" not in targets:
+ attention_group = group
+ break
+
+ assert attention_group is not None
+ assert attention_group["targets"] == [model.model.layers[0].self_attn.__class__.__name__]
+ assert attention_group["weights"] is None
+ assert attention_group["input_activations"]["num_bits"] == 8
+ assert attention_group["input_activations"]["type"] == "float"
+ assert attention_group["input_activations"]["strategy"] == "tensor"
+ assert attention_group["input_activations"]["dynamic"] is False
+ assert attention_group["input_activations"]["symmetric"] is True
+ assert getattr(model.model.layers[0].self_attn, "q_scale", None) is not None
+ assert config["kv_cache_scheme"] is not None
+
+
def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):
autoround = AutoRound(
model=opt_name_or_path,