PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: share NVFP4 global scales for Wan QKV projections (#2379)
Wang, Chang36079c49c873
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 50b383ca2..d5b3acc52 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -311,6 +311,7 @@ def update_fused_layer_global_scales(For attention layers:- q/k/v projections share a single global scale.+ - Diffusers self-attention uses to_q/to_k/to_v (e.g. Wan).For MLP layers:- gate_proj and up_proj share a single global scale.@@ -366,6 +367,14 @@ def update_fused_layer_global_scales(_update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])return+ # Diffusers self-attention (e.g. Wan) uses to_q/to_k/to_v. Cross-attention+ # Q consumes different inputs from K/V and must not join their scale group.+ if not getattr(submodule, "is_cross_attention", False) and all(+ hasattr(submodule, projection) for projection in ("to_q", "to_k", "to_v")+ ):+ _update_global_scales([submodule.to_q, submodule.to_k, submodule.to_v])+ return+# ---------------- MLP ----------------if _is_mlp_module(submodule):_update_global_scales([submodule.gate_proj, submodule.up_proj])diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.pyindex 5cce40ef2..ac9576eae 100644--- a/test/unit/common/data_type/test_nvfp.py+++ b/test/unit/common/data_type/test_nvfp.py@@ -191,12 +191,12 @@ class TestFusedLayerGlobalScales:projection.weight_global_scale = torch.tensor([scale])return projection- def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch):+ @pytest.mark.parametrize("projection_names", [("q_proj", "k_proj", "v_proj"), ("to_q", "to_k", "to_v")])+ def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch, projection_names):monkeypatch.delenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", raising=False)attention = nn.Module()- attention.q_proj = self._projection(3.0)- attention.k_proj = self._projection(1.0)- attention.v_proj = self._projection(2.0)+ for name, scale in zip(projection_names, [3.0, 1.0, 2.0]):+ setattr(attention, name, self._projection(scale))mlp = nn.Module()mlp.gate_proj = self._projection(4.0)mlp.up_proj = self._projection(0.5)@@ -204,21 +204,23 @@ class TestFusedLayerGlobalScales:update_fused_layer_global_scales(attention)update_fused_layer_global_scales(mlp)- assert all(- proj.weight_global_scale.item() == 1.0 for proj in (attention.q_proj, attention.k_proj, attention.v_proj)- )+ assert all(getattr(attention, name).weight_global_scale.item() == 1.0 for name in projection_names)assert all(proj.weight_global_scale.item() == 0.5 for proj in (mlp.gate_proj, mlp.up_proj))- def test_fused_projection_scale_update_can_be_disabled(self, monkeypatch):- monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "0")+ @pytest.mark.parametrize(+ "projection_names,is_cross_attention",+ [(("q_proj", "k_proj", "v_proj"), False), (("to_q", "to_k", "to_v"), False), (("to_q", "to_k", "to_v"), True)],+ )+ def test_fused_projection_scales_are_preserved(self, monkeypatch, projection_names, is_cross_attention):+ monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1" if is_cross_attention else "0")attention = nn.Module()- attention.q_proj = self._projection(3.0)- attention.k_proj = self._projection(1.0)- attention.v_proj = self._projection(2.0)+ attention.is_cross_attention = is_cross_attention+ for name, scale in zip(projection_names, [3.0, 1.0, 2.0]):+ setattr(attention, name, self._projection(scale))update_fused_layer_global_scales(attention)- assert [proj.weight_global_scale.item() for proj in (attention.q_proj, attention.k_proj, attention.v_proj)] == [+ assert [getattr(attention, name).weight_global_scale.item() for name in projection_names] == [3.0,1.0,2.0,