Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: share NVFP4 global scales for Wan QKV projections (#2379)

Wang, Chang36079c49c873
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 50b383ca2..d5b3acc52 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -311,6 +311,7 @@ def update_fused_layer_global_scales(
For attention layers:
- q/k/v projections share a single global scale.
+ - Diffusers self-attention uses to_q/to_k/to_v (e.g. Wan).
For MLP layers:
- gate_proj and up_proj share a single global scale.
@@ -366,6 +367,14 @@ def update_fused_layer_global_scales(
_update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])
return
+ # Diffusers self-attention (e.g. Wan) uses to_q/to_k/to_v. Cross-attention
+ # Q consumes different inputs from K/V and must not join their scale group.
+ if not getattr(submodule, "is_cross_attention", False) and all(
+ hasattr(submodule, projection) for projection in ("to_q", "to_k", "to_v")
+ ):
+ _update_global_scales([submodule.to_q, submodule.to_k, submodule.to_v])
+ return
+
# ---------------- MLP ----------------
if _is_mlp_module(submodule):
_update_global_scales([submodule.gate_proj, submodule.up_proj])
diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.py
index 5cce40ef2..ac9576eae 100644
--- a/test/unit/common/data_type/test_nvfp.py
+++ b/test/unit/common/data_type/test_nvfp.py
@@ -191,12 +191,12 @@ class TestFusedLayerGlobalScales:
projection.weight_global_scale = torch.tensor([scale])
return projection
- def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch):
+ @pytest.mark.parametrize("projection_names", [("q_proj", "k_proj", "v_proj"), ("to_q", "to_k", "to_v")])
+ def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch, projection_names):
monkeypatch.delenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", raising=False)
attention = nn.Module()
- attention.q_proj = self._projection(3.0)
- attention.k_proj = self._projection(1.0)
- attention.v_proj = self._projection(2.0)
+ for name, scale in zip(projection_names, [3.0, 1.0, 2.0]):
+ setattr(attention, name, self._projection(scale))
mlp = nn.Module()
mlp.gate_proj = self._projection(4.0)
mlp.up_proj = self._projection(0.5)
@@ -204,21 +204,23 @@ class TestFusedLayerGlobalScales:
update_fused_layer_global_scales(attention)
update_fused_layer_global_scales(mlp)
- assert all(
- proj.weight_global_scale.item() == 1.0 for proj in (attention.q_proj, attention.k_proj, attention.v_proj)
- )
+ assert all(getattr(attention, name).weight_global_scale.item() == 1.0 for name in projection_names)
assert all(proj.weight_global_scale.item() == 0.5 for proj in (mlp.gate_proj, mlp.up_proj))
- def test_fused_projection_scale_update_can_be_disabled(self, monkeypatch):
- monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "0")
+ @pytest.mark.parametrize(
+ "projection_names,is_cross_attention",
+ [(("q_proj", "k_proj", "v_proj"), False), (("to_q", "to_k", "to_v"), False), (("to_q", "to_k", "to_v"), True)],
+ )
+ def test_fused_projection_scales_are_preserved(self, monkeypatch, projection_names, is_cross_attention):
+ monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1" if is_cross_attention else "0")
attention = nn.Module()
- attention.q_proj = self._projection(3.0)
- attention.k_proj = self._projection(1.0)
- attention.v_proj = self._projection(2.0)
+ attention.is_cross_attention = is_cross_attention
+ for name, scale in zip(projection_names, [3.0, 1.0, 2.0]):
+ setattr(attention, name, self._projection(scale))
update_fused_layer_global_scales(attention)
- assert [proj.weight_global_scale.item() for proj in (attention.q_proj, attention.k_proj, attention.v_proj)] == [
+ assert [getattr(attention, name).weight_global_scale.item() for name in projection_names] == [
3.0,
1.0,
2.0,