Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

align scale for MLA kv (#1395)

Yi Liua267625ea775
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.py
index d6fcb0e69..636247841 100644
--- a/auto_round_extension/vllm_ext/auto_round_ext.py
+++ b/auto_round_extension/vllm_ext/auto_round_ext.py
@@ -36,7 +36,11 @@ class AutoRoundExtensionConfig(_BaseAutoRoundConfig):
from vllm.attention.layer import Attention, MLAAttention
if isinstance(layer, (Attention, MLAAttention)):
- from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod
+
+ from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod, AutoRoundKVCacheMethodForMLA
+
+ if isinstance(layer, MLAAttention):
+ return AutoRoundKVCacheMethodForMLA(self)
return AutoRoundKVCacheMethod(self)
if isinstance(layer, FusedMoE):
diff --git a/auto_round_extension/vllm_ext/kv_cache.py b/auto_round_extension/vllm_ext/kv_cache.py
index ec2b7e179..9b8572b06 100644
--- a/auto_round_extension/vllm_ext/kv_cache.py
+++ b/auto_round_extension/vllm_ext/kv_cache.py
@@ -36,3 +36,16 @@ class AutoRoundKVCacheMethod(BaseKVCacheMethod):
def validate_kv_cache_scheme(quant_config):
# FIXME: parse from quant_config
return True
+
+
+class AutoRoundKVCacheMethodForMLA(BaseKVCacheMethod):
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ super().process_weights_after_loading(layer)
+ k_scale = layer._k_scale
+ v_scale = layer._v_scale
+ aligned_kv_scale = max(k_scale, v_scale)
+ layer._k_scale.data.fill_(aligned_kv_scale)
+ layer._v_scale.data.fill_(aligned_kv_scale)
+ layer._k_scale_float = aligned_kv_scale.item()
+ layer._v_scale_float = aligned_kv_scale.item()
+ logger.warning_once("Aligned MLA k/v scales for MLA")