PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix FP8 MLA (#1343)
Yi Liudb77f3b7e3d9
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.pyindex 8ef85069a..d6fcb0e69 100644--- a/auto_round_extension/vllm_ext/auto_round_ext.py+++ b/auto_round_extension/vllm_ext/auto_round_ext.py@@ -33,9 +33,9 @@ class AutoRoundExtensionConfig(_BaseAutoRoundConfig):def get_quant_method(self, layer: torch.nn.Module, prefix: str):# FIXME: (yi) make it compatible with `AutoRoundConfig`- from vllm.attention.layer import Attention+ from vllm.attention.layer import Attention, MLAAttention- if isinstance(layer, Attention):+ if isinstance(layer, (Attention, MLAAttention)):from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethodreturn AutoRoundKVCacheMethod(self)