Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix: skip exporting fp8 attention q_max (#2103)

Yi Liu2a267e721bc2
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index 1ca0395c7..f2bcf78bd 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -24,6 +24,7 @@ from auto_round import envs
from auto_round.compressors.utils import _get_save_folder_name
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
+from auto_round.experimental.attention import is_attention_calibration_tensor_name
from auto_round.logger import logger
from auto_round.utils import (
get_lm_head_name,
@@ -228,6 +229,8 @@ class ShardWriter:
for k, v in sd.items():
if not isinstance(v, torch.Tensor):
continue
+ if is_attention_calibration_tensor_name(k):
+ continue
param_name = f"{prefix}.{k}"
self._add_tensor(param_name, v)
@@ -267,6 +270,9 @@ class ShardWriter:
return list(expanded.items())
def _add_tensor(self, name: str, tensor: torch.Tensor):
+ if is_attention_calibration_tensor_name(name):
+ return
+
if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":
self.skipped_meta_tensors.append(name)
return
diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.py
index 0a2361423..ea293873d 100644
--- a/auto_round/experimental/attention.py
+++ b/auto_round/experimental/attention.py
@@ -41,6 +41,7 @@ from auto_round.utils import logger
__all__ = [
"QuantizedAttentionImpl",
"init_hooked_attention",
+ "is_attention_calibration_tensor_name",
"attention_quant_ctx",
]
@@ -51,6 +52,11 @@ QUERY_SCALE_NAME = "q_scale"
QUERY_MAX_NAME = "q_max"
+def is_attention_calibration_tensor_name(name: str) -> bool:
+ """Return True when a serialized tensor name is the transient attention q_max."""
+ return name.rsplit(".", 1)[-1] == QUERY_MAX_NAME
+
+
class QuantizedAttentionImpl(torch.nn.Module):
"""
QuantizedAttentionImpl module which wraps the functionality of the original
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index baa1394f9..e23d2afbd 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -318,6 +318,7 @@ class TestAutoRound:
assert weight_name in f.keys()
assert f.get_tensor(weight_name).shape == torch.Size([1])
assert f.get_tensor(weight_name).dtype == torch.float32 or f.get_tensor(weight_name).dtype == torch.bfloat16
+ assert "model.decoder.layers.8.self_attn.q_max" not in f.keys()
def test_awq_lmhead_export(self, dataloader):
bits, sym, group_size = 4, False, 128
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index 8a6e6d95f..0a3c3282a 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -151,6 +151,8 @@ class TestLLMC:
assert kv_cache_scheme["symmetric"] is True
def test_mxfp8_llmcompressor_attention_config(self, tiny_opt_model_path, tmp_path):
+ from safetensors import safe_open
+
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
@@ -185,6 +187,11 @@ class TestLLMC:
assert quantization_config["kv_cache_scheme"] is not None
assert getattr(compressed_model.model.decoder.layers[0].self_attn, "q_scale", None) is not None
+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as checkpoint:
+ keys = list(checkpoint.keys())
+ assert any(key.endswith(".self_attn.q_scale") for key in keys), "q_scale not found in checkpoint"
+ assert not any(key.endswith(".q_max") for key in keys), "q_max should not be exported"
+
def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
scheme = AutoScheme(
avg_bits=7,