Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix FP8 MLA (#1343)

Yi Liudb77f3b7e3d9
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.py
index 8ef85069a..d6fcb0e69 100644
--- a/auto_round_extension/vllm_ext/auto_round_ext.py
+++ b/auto_round_extension/vllm_ext/auto_round_ext.py
@@ -33,9 +33,9 @@ class AutoRoundExtensionConfig(_BaseAutoRoundConfig):
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
# FIXME: (yi) make it compatible with `AutoRoundConfig`
- from vllm.attention.layer import Attention
+ from vllm.attention.layer import Attention, MLAAttention
- if isinstance(layer, Attention):
+ if isinstance(layer, (Attention, MLAAttention)):
from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod
return AutoRoundKVCacheMethod(self)