Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: skip QKV fusion when projections are kept packed (#17132)

Daniel Handa0df6a9e3f9
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 4ed08924c40..15d9f218c34 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -1004,9 +1004,8 @@ def fused_qkv_added_bytes(pipe: Any) -> int:
return 0
try:
import torch
- from .diffusion_native_quant import is_native_quantised
- if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):
+ if _quantised_projections(pipe):
return 0
if unfused_projections_will_drop(unet):
return 0 # the fused weights replace the projections they were built from
@@ -2050,15 +2049,29 @@ def _enable_fp16_accumulation(
return False
-def _fuse_qkv(pipe: Any, logger: Any) -> bool:
- from .diffusion_native_quant import is_native_quantised
+def _quantised_projections(pipe: Any) -> bool:
+ """Whether a denoiser keeps native int8 / fp8 or packed nvfp4 / mxfp8 Linears, whose ``.weight`` is a dense
+ widen: diffusers' fuse would concatenate those into a persistent dense ``to_qkv`` beside the compact buffers."""
+ from .diffusion_native_quant import is_native_linear
+ from .diffusion_packed_block import is_packed_block_linear
- # Fusing reads each projection's .weight, which a native layer dequantises: a dense to_qkv beside the int8 buffers.
- if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):
+ # the pipe-level fuse reaches any U-Net, not only the whole-compile ones
+ for model in (*_denoiser_dits(pipe), getattr(pipe, "unet", None)):
+ if model is None or not callable(getattr(model, "modules", None)):
+ continue
+ try:
+ if any(is_native_linear(m) or is_packed_block_linear(m) for m in model.modules()):
+ return True
+ except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes
+ continue
+ return False
+
+
+def _fuse_qkv(pipe: Any, logger: Any) -> bool:
+ # Fusing reads each projection's .weight, which a quantised layer dequantises: a dense to_qkv beside its buffers.
+ if _quantised_projections(pipe):
if logger is not None:
- logger.info(
- "diffusion.speed: fuse_qkv skipped (native quantised projections stay unfused)"
- )
+ logger.info("diffusion.speed: fuse_qkv skipped (quantised projections stay unfused)")
return False
from . import diffusion_lora_quant_merge as lora_quant_merge
diff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.py
index 3abb3fea75b..6eb9426f233 100644
--- a/studio/backend/tests/test_diffusion_packed_block.py
+++ b/studio/backend/tests/test_diffusion_packed_block.py
@@ -309,3 +309,70 @@ def test_the_triton_widen_on_the_gpu_matches_the_load_time_widen(dtype):
got = got.cpu()
assert torch.equal(torch.isnan(got.float()), nan)
assert torch.equal(_bits(got)[~nan], _bits(want)[~nan])
+
+
+def _packed_projections(module, names = ("to_q", "to_k", "to_v")):
+ cls = pb.packed_block_linear_class()
+ for name, mod in list(module.named_modules()):
+ if isinstance(mod, torch.nn.Linear) and name.rsplit(".", 1)[-1] in names:
+ rows, cols = mod.weight.shape
+ layer = cls(
+ "mxfp8",
+ cols,
+ rows,
+ codes = torch.ones(rows, cols).to(torch.float8_e4m3fn),
+ scale = torch.full((rows, cols // 32), 127, dtype = torch.uint8),
+ bias = None if mod.bias is None else mod.bias.detach(),
+ compute_dtype = torch.bfloat16,
+ )
+ parent, _, leaf = name.rpartition(".")
+ setattr(module.get_submodule(parent) if parent else module, leaf, layer)
+ return module
+
+
+def test_qkv_fusion_skips_packed_projections():
+ import types
+
+ from core.inference import diffusion_speed
+
+ calls = []
+
+ class _Dit(torch.nn.Module):
+ def __init__(self, packed):
+ super().__init__()
+ self.to_q = torch.nn.Linear(64, 64).to(torch.bfloat16)
+ if packed:
+ _packed_projections(self)
+
+ def fuse_qkv_projections(self):
+ calls.append(self)
+
+ packed = types.SimpleNamespace(transformer = _Dit(packed = True))
+ assert diffusion_speed._fuse_qkv(packed, None) is False and not calls
+ dense = types.SimpleNamespace(transformer = _Dit(packed = False))
+ assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1
+
+
+def test_qkv_fusion_of_a_packed_flux_builds_no_dense_to_qkv():
+ import types
+
+ diffusers = pytest.importorskip("diffusers")
+ from core.inference import diffusion_speed
+
+ dit = diffusers.FluxTransformer2DModel(
+ in_channels = 8,
+ num_layers = 1,
+ num_single_layers = 0,
+ attention_head_dim = 16,
+ num_attention_heads = 2,
+ joint_attention_dim = 24,
+ pooled_projection_dim = 16,
+ axes_dims_rope = (4, 6, 6),
+ ).to(torch.bfloat16)
+ _packed_projections(dit)
+ before = sum(t.numel() * t.element_size() for t in (*dit.parameters(), *dit.buffers()))
+ pipe = types.SimpleNamespace(transformer = dit, unet = None, vae = None)
+ assert diffusion_speed._fuse_qkv(pipe, None) is False
+ after = sum(t.numel() * t.element_size() for t in (*dit.parameters(), *dit.buffers()))
+ assert after == before
+ assert not [n for n, _m in dit.named_modules() if n.endswith(("to_qkv", "to_added_qkv"))]