PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: enhance attention module with query observation and legacy support (#2466)
Xin He376fdc0fd5f2
diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex 546e2329d..d11684745 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -98,6 +98,19 @@ class QuantizedAttentionImpl(torch.nn.Module):*args,**kwargs,):+ self.observe(module, query)+ # original attention+ return ALL_ATTENTION_FUNCTIONS[self._original_impl](+ module,+ query,+ key,+ value,+ *args,+ **kwargs,+ )++ def observe(self, module: Module, query: Tensor):+ """Update q_max/q_scale from a query laid out as [batch, heads, seq, head_dim]."""if self.granularity == "head":cur_query_max = query.abs().amax(dim=(0, 2, 3))else:@@ -109,15 +122,68 @@ class QuantizedAttentionImpl(torch.nn.Module):update_parameter_data(module, query_max, QUERY_MAX_NAME)_, query_scale = fp8_qdq(query, tensor_max=query_max, granularity=self.granularity)update_parameter_data(module, query_scale.reshape(-1).detach(), QUERY_SCALE_NAME)- # original attention- return ALL_ATTENTION_FUNCTIONS[self._original_impl](- module,- query,- key,- value,- *args,- **kwargs,- )+++# ----- legacy (remote-code) attention support ----- #++_LEGACY_PATCHED_ATTRS = ("forward", "_flash_attention_forward")+++def _uses_attention_interface(module: Module) -> bool:+ forward = type(module).forward+ if forward is Module.forward:+ return True+ try:+ source = inspect.getsource(forward)+ except (OSError, TypeError):+ return True+ return "ALL_ATTENTION_FUNCTIONS" in source or "attention_interface" in source+++class _CaptureFirstMatmul(torch.overrides.TorchFunctionMode):+ """Observe the query of the first 4D matmul (Q @ K^T) inside an eager attention forward."""++ def __init__(self, module: Module):+ super().__init__()+ self.module = module+ self.done = False++ def __torch_function__(self, func, types, args=(), kwargs=None):+ kwargs = kwargs or {}+ if not self.done and func in (torch.matmul, Tensor.matmul, Tensor.__matmul__):+ query = args[0]+ if isinstance(query, Tensor) and query.dim() == 4:+ self.done = True+ self.module.impl.observe(self.module, query)+ return func(*args, **kwargs)+++def _patch_legacy_attention(module: Module):+ """Hook attention modules whose forward bypasses transformers' AttentionInterface."""+ flash_forward = getattr(module, "_flash_attention_forward", None)+ if callable(flash_forward):++ def _observed_flash_forward(query_states, *args, **kwargs):+ # flash-attn layout is [batch, seq, heads, head_dim]+ module.impl.observe(module, query_states.transpose(1, 2))+ return flash_forward(query_states, *args, **kwargs)++ module._flash_attention_forward = _observed_flash_forward+ return++ orig_forward = module.forward++ def _observed_forward(*args, **kwargs):+ with _CaptureFirstMatmul(module):+ return orig_forward(*args, **kwargs)++ module.forward = _observed_forward+++def _unpatch_legacy_attention(module: Module):+ for name in _LEGACY_PATCHED_ATTRS:+ if name in module.__dict__:+ delattr(module, name)# ----- initialize ----- #@@ -150,6 +216,14 @@ def init_hooked_attention(module: Module, config, granularity: str = "tensor")::param module: attention module to initialize with"""config = _get_attention_config(module, config)+ if not hasattr(module, ATTN_IMPL_ATTR_NAME) and not _uses_attention_interface(module):+ logger.info_once(+ f"{module.__class__.__name__} does not dispatch through AttentionInterface, "+ "observing query at the attention kernel boundary instead."+ )+ module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))+ _patch_legacy_attention(module)+ returnif not hasattr(module, ATTN_IMPL_ATTR_NAME):if config._attn_implementation != HOOKED_ATTENTION_NAME:# assumes only one model at a time@@ -172,6 +246,12 @@ def prep_attention_module_for_calibration(module: torch.nn.Module, config, granudef clean_up_hooked_attention(module, model):if is_attention_module(module):+ _unpatch_legacy_attention(module)+ query_max = getattr(module, QUERY_MAX_NAME, None)+ if isinstance(query_max, Tensor) and torch.isinf(query_max).all():+ logger.warning_once(+ f"{module.__class__.__name__} was never observed during calibration; its q_scale stays 0."+ )clean_model_parameters_and_buffers_(module, (QUERY_MAX_NAME,))config = _get_attention_config(module, model.config)if hasattr(config, "_auto_round_original_attn_impl"):diff --git a/test/integration/test_cpu/test_inc_static_attn.py b/test/integration/test_cpu/test_inc_static_attn.pyindex 7ba9924eb..d1797672f 100644--- a/test/integration/test_cpu/test_inc_static_attn.py+++ b/test/integration/test_cpu/test_inc_static_attn.py@@ -48,6 +48,75 @@ def test_attention_context_uses_module_config_and_restores_it():assert not hasattr(attention, QUERY_MAX_NAME)+NUM_HEADS, HEAD_DIM, HIDDEN = 2, 8, 16+++class LegacyEagerAttention(torch.nn.Module):+ """Mimics remote-code attention (e.g. Kimi-K2.6) computing Q @ K^T inline."""++ def __init__(self, config):+ super().__init__()+ self.config = config+ self.layer_idx = 0+ self.q_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)+ self.k_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)+ self.v_proj = torch.nn.Linear(HIDDEN, HIDDEN, bias=False)++ def _split(self, x):+ b, s, _ = x.shape+ return x.view(b, s, NUM_HEADS, HEAD_DIM).transpose(1, 2)++ def _qkv(self, hidden_states, past_key_value=None):+ q = self._split(self.q_proj(hidden_states))+ k = self._split(self.k_proj(hidden_states))+ v = self._split(self.v_proj(hidden_states))+ if past_key_value is not None:+ k, v = past_key_value.update(k, v, self.layer_idx)+ return q, k, v++ def forward(self, hidden_states, past_key_value=None, **kwargs):+ q, k, v = self._qkv(hidden_states, past_key_value)+ weights = torch.matmul(q, k.transpose(2, 3)).softmax(-1)+ out = torch.matmul(weights, v)+ return out.transpose(1, 2).reshape(hidden_states.shape)+++class LegacyFlashAttention2(LegacyEagerAttention):+ """Mimics remote-code FlashAttention2 calling flash_attn_func via _flash_attention_forward."""++ def forward(self, hidden_states, past_key_value=None, **kwargs):+ q, k, v = (t.transpose(1, 2) for t in self._qkv(hidden_states, past_key_value))+ out = self._flash_attention_forward(q, k, v)+ return out.reshape(hidden_states.shape)++ def _flash_attention_forward(self, query_states, key_states, value_states):+ return value_states++[email protected]("attn_cls", [LegacyEagerAttention, LegacyFlashAttention2])+def test_legacy_attention_q_scale_is_calibrated(attn_cls):+ torch.manual_seed(0)+ config = PretrainedConfig()+ config._attn_implementation = "eager"+ attn = attn_cls(config)+ model = torch.nn.Module()+ model.config = config+ model.attention = attn+ hidden_states = torch.randn(1, 4, HIDDEN)++ with torch.no_grad(), attention_quant_ctx(model):+ attn(hidden_states)++ expected = attn.q_proj(hidden_states).abs().max() / torch.finfo(torch.float8_e4m3fn).max+ assert (attn.q_scale > 0).all(), f"q_scale must be positive, got {attn.q_scale}"+ assert torch.allclose(attn.q_scale.data, expected.detach().reshape(1), rtol=1e-3)+ # patches and transient tensors are removed after calibration+ assert "forward" not in attn.__dict__+ assert "_flash_attention_forward" not in attn.__dict__+ assert not hasattr(attn, QUERY_MAX_NAME)+ assert config._attn_implementation == "eager"++def test_deepseek_v2(tiny_deepseek_v2_model_path):model_name = tiny_deepseek_v2_model_pathmodel = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=False)