PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
align scale for MLA kv (#1395)
Yi Liua267625ea775
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.pyindex d6fcb0e69..636247841 100644--- a/auto_round_extension/vllm_ext/auto_round_ext.py+++ b/auto_round_extension/vllm_ext/auto_round_ext.py@@ -36,7 +36,11 @@ class AutoRoundExtensionConfig(_BaseAutoRoundConfig):from vllm.attention.layer import Attention, MLAAttentionif isinstance(layer, (Attention, MLAAttention)):- from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod++ from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod, AutoRoundKVCacheMethodForMLA++ if isinstance(layer, MLAAttention):+ return AutoRoundKVCacheMethodForMLA(self)return AutoRoundKVCacheMethod(self)if isinstance(layer, FusedMoE):diff --git a/auto_round_extension/vllm_ext/kv_cache.py b/auto_round_extension/vllm_ext/kv_cache.pyindex ec2b7e179..9b8572b06 100644--- a/auto_round_extension/vllm_ext/kv_cache.py+++ b/auto_round_extension/vllm_ext/kv_cache.py@@ -36,3 +36,16 @@ class AutoRoundKVCacheMethod(BaseKVCacheMethod):def validate_kv_cache_scheme(quant_config):# FIXME: parse from quant_configreturn True+++class AutoRoundKVCacheMethodForMLA(BaseKVCacheMethod):+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:+ super().process_weights_after_loading(layer)+ k_scale = layer._k_scale+ v_scale = layer._v_scale+ aligned_kv_scale = max(k_scale, v_scale)+ layer._k_scale.data.fill_(aligned_kv_scale)+ layer._v_scale.data.fill_(aligned_kv_scale)+ layer._k_scale_float = aligned_kv_scale.item()+ layer._v_scale_float = aligned_kv_scale.item()+ logger.warning_once("Aligned MLA k/v scales for MLA")