Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix llama4 kv quant (#1205)

Wang, Mengnicdd96a88f0b0
diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.py
index 999ca9f53..e90f9c0d5 100644
--- a/auto_round/experimental/utils.py
+++ b/auto_round/experimental/utils.py
@@ -70,12 +70,15 @@ def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype
def is_attention_module(module: torch.nn.Module):
# FIXME: Handle this better.
- return "attention" in module.__class__.__name__.lower() and (
- hasattr(module, "k_proj")
- or hasattr(module, "v_proj")
- or hasattr(module, "qkv_proj")
- or hasattr(module, "kv_b_proj") # for DeepSpeed
+ return (
+ "attention" in module.__class__.__name__.lower()
and module.__class__.__name__ != "Llama4VisionAttention" # llama4 vision attention doesn't have cache
+ and (
+ hasattr(module, "k_proj")
+ or hasattr(module, "v_proj")
+ or hasattr(module, "qkv_proj")
+ or hasattr(module, "kv_b_proj") # for DeepSpeed
+ )
)