PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix: skip exporting fp8 attention q_max (#2103)
Yi Liu2a267e721bc2
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 1ca0395c7..f2bcf78bd 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -24,6 +24,7 @@ from auto_round import envsfrom auto_round.compressors.utils import _get_save_folder_namefrom auto_round.context.compress import CompressContextfrom auto_round.context.model import ModelContext+from auto_round.experimental.attention import is_attention_calibration_tensor_namefrom auto_round.logger import loggerfrom auto_round.utils import (get_lm_head_name,@@ -228,6 +229,8 @@ class ShardWriter:for k, v in sd.items():if not isinstance(v, torch.Tensor):continue+ if is_attention_calibration_tensor_name(k):+ continueparam_name = f"{prefix}.{k}"self._add_tensor(param_name, v)@@ -267,6 +270,9 @@ class ShardWriter:return list(expanded.items())def _add_tensor(self, name: str, tensor: torch.Tensor):+ if is_attention_calibration_tensor_name(name):+ return+if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":self.skipped_meta_tensors.append(name)returndiff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex 0a2361423..ea293873d 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -41,6 +41,7 @@ from auto_round.utils import logger__all__ = ["QuantizedAttentionImpl","init_hooked_attention",+ "is_attention_calibration_tensor_name","attention_quant_ctx",]@@ -51,6 +52,11 @@ QUERY_SCALE_NAME = "q_scale"QUERY_MAX_NAME = "q_max"+def is_attention_calibration_tensor_name(name: str) -> bool:+ """Return True when a serialized tensor name is the transient attention q_max."""+ return name.rsplit(".", 1)[-1] == QUERY_MAX_NAME++class QuantizedAttentionImpl(torch.nn.Module):"""QuantizedAttentionImpl module which wraps the functionality of the originaldiff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex baa1394f9..e23d2afbd 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -318,6 +318,7 @@ class TestAutoRound:assert weight_name in f.keys()assert f.get_tensor(weight_name).shape == torch.Size([1])assert f.get_tensor(weight_name).dtype == torch.float32 or f.get_tensor(weight_name).dtype == torch.bfloat16+ assert "model.decoder.layers.8.self_attn.q_max" not in f.keys()def test_awq_lmhead_export(self, dataloader):bits, sym, group_size = 4, False, 128diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex 8a6e6d95f..0a3c3282a 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -151,6 +151,8 @@ class TestLLMC:assert kv_cache_scheme["symmetric"] is Truedef test_mxfp8_llmcompressor_attention_config(self, tiny_opt_model_path, tmp_path):+ from safetensors import safe_open+ar = AutoRound(model=tiny_opt_model_path,iters=0,@@ -185,6 +187,11 @@ class TestLLMC:assert quantization_config["kv_cache_scheme"] is not Noneassert getattr(compressed_model.model.decoder.layers[0].self_attn, "q_scale", None) is not None+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as checkpoint:+ keys = list(checkpoint.keys())+ assert any(key.endswith(".self_attn.q_scale") for key in keys), "q_scale not found in checkpoint"+ assert not any(key.endswith(".q_max") for key in keys), "q_max should not be exported"+def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):scheme = AutoScheme(avg_bits=7,