Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: enhance attention module with query observation and legacy support (#2466)

Xin He376fdc0fd5f2
diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.py
index 546e2329d..d11684745 100644
--- a/auto_round/experimental/attention.py
+++ b/auto_round/experimental/attention.py
@@ -98,6 +98,19 @@ class QuantizedAttentionImpl(torch.nn.Module):
*args,
**kwargs,
):
+ self.observe(module, query)
+ # original attention
+ return ALL_ATTENTION_FUNCTIONS[self._original_impl](
+ module,
+ query,
+ key,
+ value,
+ *args,
+ **kwargs,
+ )
+
+ def observe(self, module: Module, query: Tensor):
+ """Update q_max/q_scale from a query laid out as [batch, heads, seq, head_dim]."""
if self.granularity == "head":
cur_query_max = query.abs().amax(dim=(0, 2, 3))
else:
@@ -109,15 +122,68 @@ class QuantizedAttentionImpl(torch.nn.Module):
update_parameter_data(module, query_max, QUERY_MAX_NAME)
_, query_scale = fp8_qdq(query, tensor_max=query_max, granularity=self.granularity)
update_parameter_data(module, query_scale.reshape(-1).detach(), QUERY_SCALE_NAME)
- # original attention
- return ALL_ATTENTION_FUNCTIONS[self._original_impl](
- module,
- query,
- key,
- value,
- *args,
- **kwargs,
- )
+
+
+# ----- legacy (remote-code) attention support ----- #
+
+_LEGACY_PATCHED_ATTRS = ("forward", "_flash_attention_forward")
+
+
+def _uses_attention_interface(module: Module) -> bool:
+ forward = type(module).forward
+ if forward is Module.forward:
+ return True
+ try:
+ source = inspect.getsource(forward)
+ except (OSError, TypeError):
+ return True
+ return "ALL_ATTENTION_FUNCTIONS" in source or "attention_interface" in source
+
+
+class _CaptureFirstMatmul(torch.overrides.TorchFunctionMode):
+ """Observe the query of the first 4D matmul (Q @ K^T) inside an eager attention forward."""
+
+ def __init__(self, module: Module):
+ super().__init__()
+ self.module = module
+ self.done = False
+
+ def __torch_function__(self, func, types, args=(), kwargs=None):
+ kwargs = kwargs or {}
+ if not self.done and func in (torch.matmul, Tensor.matmul, Tensor.__matmul__):
+ query = args[0]
+ if isinstance(query, Tensor) and query.dim() == 4:
+ self.done = True
+ self.module.impl.observe(self.module, query)
+ return func(*args, **kwargs)
+
+
+def _patch_legacy_attention(module: Module):
+ """Hook attention modules whose forward bypasses transformers' AttentionInterface."""
+ flash_forward = getattr(module, "_flash_attention_forward", None)
+ if callable(flash_forward):
+
+ def _observed_flash_forward(query_states, *args, **kwargs):
+ # flash-attn layout is [batch, seq, heads, head_dim]
+ module.impl.observe(module, query_states.transpose(1, 2))
+ return flash_forward(query_states, *args, **kwargs)
+
+ module._flash_attention_forward = _observed_flash_forward
+ return
+
+ orig_forward = module.forward
+
+ def _observed_forward(*args, **kwargs):
+ with _CaptureFirstMatmul(module):
+ return orig_forward(*args, **kwargs)
+
+ module.forward = _observed_forward
+
+
+def _unpatch_legacy_attention(module: Module):
+ for name in _LEGACY_PATCHED_ATTRS:
+ if name in module.__dict__:
+ delattr(module, name)
# ----- initialize ----- #
@@ -150,6 +216,14 @@ def init_hooked_attention(module: Module, config, granularity: str = "tensor"):
:param module: attention module to initialize with
"""
config = _get_attention_config(module, config)
+ if not hasattr(module, ATTN_IMPL_ATTR_NAME) and not _uses_attention_interface(module):
+ logger.info_once(
+ f"{module.__class__.__name__} does not dispatch through AttentionInterface, "
+ "observing query at the attention kernel boundary instead."
+ )
+ module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))
+ _patch_legacy_attention(module)
+ return
if not hasattr(module, ATTN_IMPL_ATTR_NAME):
if config._attn_implementation != HOOKED_ATTENTION_NAME:
# assumes only one model at a time
@@ -172,6 +246,12 @@ def prep_attention_module_for_calibration(module: torch.nn.Module, config, granu
def clean_up_hooked_attention(module, model):
if is_attention_module(module):
+ _unpatch_legacy_attention(module)
+ query_max = getattr(module, QUERY_MAX_NAME, None)
+ if isinstance(query_max, Tensor) and torch.isinf(query_max).all():
+ logger.warning_once(
+ f"{module.__class__.__name__} was never observed during calibration; its q_scale stays 0."
+ )
clean_model_parameters_and_buffers_(module, (QUERY_MAX_NAME,))
config = _get_attention_config(module, model.config)
if hasattr(config, "_auto_round_original_attn_impl"):
diff --git a/test/integration/test_cpu/test_inc_static_attn.py b/test/integration/test_cpu/test_inc_static_attn.py
index 7ba9924eb..d1797672f 100644
--- a/test/integration/test_cpu/test_inc_static_attn.py
+++ b/test/integration/test_cpu/test_inc_static_attn.py
@@ -48,6 +48,75 @@ def test_attention_context_uses_module_config_and_restores_it():
assert not hasattr(attention, QUERY_MAX_NAME)
+NUM_HEADS, HEAD_DIM, HIDDEN = 2, 8, 16
+
+
+class LegacyEagerAttention(torch.nn.Module):
+ """Mimics remote-code attention (e.g. Kimi-K2.6) computing Q @ K^T inline."""
+
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.layer_idx = 0
+ self.q_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)
+ self.k_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)
+ self.v_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)
+
+ def _split(self, x):
+ b, s, _ = x.shape
+ return x.view(b, s, NUM_HEADS, HEAD_DIM).transpose(1, 2)
+
+ def _qkv(self, hidden_states, past_key_value=None):
+ q = self._split(self.q_proj(hidden_states))
+ k = self._split(self.k_proj(hidden_states))
+ v = self._split(self.v_proj(hidden_states))
+ if past_key_value is not None:
+ k, v = past_key_value.update(k, v, self.layer_idx)
+ return q, k, v
+
+ def forward(self, hidden_states, past_key_value=None, **kwargs):
+ q, k, v = self._qkv(hidden_states, past_key_value)
+ weights = torch.matmul(q, k.transpose(2, 3)).softmax(-1)
+ out = torch.matmul(weights, v)
+ return out.transpose(1, 2).reshape(hidden_states.shape)
+
+
+class LegacyFlashAttention2(LegacyEagerAttention):
+ """Mimics remote-code FlashAttention2 calling flash_attn_func via _flash_attention_forward."""
+
+ def forward(self, hidden_states, past_key_value=None, **kwargs):
+ q, k, v = (t.transpose(1, 2) for t in self._qkv(hidden_states, past_key_value))
+ out = self._flash_attention_forward(q, k, v)
+ return out.reshape(hidden_states.shape)
+
+ def _flash_attention_forward(self, query_states, key_states, value_states):
+ return value_states
+
+
[email protected]("attn_cls", [LegacyEagerAttention, LegacyFlashAttention2])
+def test_legacy_attention_q_scale_is_calibrated(attn_cls):
+ torch.manual_seed(0)
+ config = PretrainedConfig()
+ config._attn_implementation = "eager"
+ attn = attn_cls(config)
+ model = torch.nn.Module()
+ model.config = config
+ model.attention = attn
+ hidden_states = torch.randn(1, 4, HIDDEN)
+
+ with torch.no_grad(), attention_quant_ctx(model):
+ attn(hidden_states)
+
+ expected = attn.q_proj(hidden_states).abs().max() / torch.finfo(torch.float8_e4m3fn).max
+ assert (attn.q_scale > 0).all(), f"q_scale must be positive, got {attn.q_scale}"
+ assert torch.allclose(attn.q_scale.data, expected.detach().reshape(1), rtol=1e-3)
+ # patches and transient tensors are removed after calibration
+ assert "forward" not in attn.__dict__
+ assert "_flash_attention_forward" not in attn.__dict__
+ assert not hasattr(attn, QUERY_MAX_NAME)
+ assert config._attn_implementation == "eager"
+
+
def test_deepseek_v2(tiny_deepseek_v2_model_path):
model_name = tiny_deepseek_v2_model_path
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=False)