Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: skip QKV fusion when projections are kept packed (#17132)
Daniel Handa0df6a9e3f9
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 4ed08924c40..15d9f218c34 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -1004,9 +1004,8 @@ def fused_qkv_added_bytes(pipe: Any) -> int:return 0try:import torch- from .diffusion_native_quant import is_native_quantised- if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):+ if _quantised_projections(pipe):return 0if unfused_projections_will_drop(unet):return 0 # the fused weights replace the projections they were built from@@ -2050,15 +2049,29 @@ def _enable_fp16_accumulation(return False-def _fuse_qkv(pipe: Any, logger: Any) -> bool:- from .diffusion_native_quant import is_native_quantised+def _quantised_projections(pipe: Any) -> bool:+ """Whether a denoiser keeps native int8 / fp8 or packed nvfp4 / mxfp8 Linears, whose ``.weight`` is a dense+ widen: diffusers' fuse would concatenate those into a persistent dense ``to_qkv`` beside the compact buffers."""+ from .diffusion_native_quant import is_native_linear+ from .diffusion_packed_block import is_packed_block_linear- # Fusing reads each projection's .weight, which a native layer dequantises: a dense to_qkv beside the int8 buffers.- if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):+ # the pipe-level fuse reaches any U-Net, not only the whole-compile ones+ for model in (*_denoiser_dits(pipe), getattr(pipe, "unet", None)):+ if model is None or not callable(getattr(model, "modules", None)):+ continue+ try:+ if any(is_native_linear(m) or is_packed_block_linear(m) for m in model.modules()):+ return True+ except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes+ continue+ return False+++def _fuse_qkv(pipe: Any, logger: Any) -> bool:+ # Fusing reads each projection's .weight, which a quantised layer dequantises: a dense to_qkv beside its buffers.+ if _quantised_projections(pipe):if logger is not None:- logger.info(- "diffusion.speed: fuse_qkv skipped (native quantised projections stay unfused)"- )+ logger.info("diffusion.speed: fuse_qkv skipped (quantised projections stay unfused)")return Falsefrom . import diffusion_lora_quant_merge as lora_quant_mergediff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.pyindex 3abb3fea75b..6eb9426f233 100644--- a/studio/backend/tests/test_diffusion_packed_block.py+++ b/studio/backend/tests/test_diffusion_packed_block.py@@ -309,3 +309,70 @@ def test_the_triton_widen_on_the_gpu_matches_the_load_time_widen(dtype):got = got.cpu()assert torch.equal(torch.isnan(got.float()), nan)assert torch.equal(_bits(got)[~nan], _bits(want)[~nan])+++def _packed_projections(module, names = ("to_q", "to_k", "to_v")):+ cls = pb.packed_block_linear_class()+ for name, mod in list(module.named_modules()):+ if isinstance(mod, torch.nn.Linear) and name.rsplit(".", 1)[-1] in names:+ rows, cols = mod.weight.shape+ layer = cls(+ "mxfp8",+ cols,+ rows,+ codes = torch.ones(rows, cols).to(torch.float8_e4m3fn),+ scale = torch.full((rows, cols // 32), 127, dtype = torch.uint8),+ bias = None if mod.bias is None else mod.bias.detach(),+ compute_dtype = torch.bfloat16,+ )+ parent, _, leaf = name.rpartition(".")+ setattr(module.get_submodule(parent) if parent else module, leaf, layer)+ return module+++def test_qkv_fusion_skips_packed_projections():+ import types++ from core.inference import diffusion_speed++ calls = []++ class _Dit(torch.nn.Module):+ def __init__(self, packed):+ super().__init__()+ self.to_q = torch.nn.Linear(64, 64).to(torch.bfloat16)+ if packed:+ _packed_projections(self)++ def fuse_qkv_projections(self):+ calls.append(self)++ packed = types.SimpleNamespace(transformer = _Dit(packed = True))+ assert diffusion_speed._fuse_qkv(packed, None) is False and not calls+ dense = types.SimpleNamespace(transformer = _Dit(packed = False))+ assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1+++def test_qkv_fusion_of_a_packed_flux_builds_no_dense_to_qkv():+ import types++ diffusers = pytest.importorskip("diffusers")+ from core.inference import diffusion_speed++ dit = diffusers.FluxTransformer2DModel(+ in_channels = 8,+ num_layers = 1,+ num_single_layers = 0,+ attention_head_dim = 16,+ num_attention_heads = 2,+ joint_attention_dim = 24,+ pooled_projection_dim = 16,+ axes_dims_rope = (4, 6, 6),+ ).to(torch.bfloat16)+ _packed_projections(dit)+ before = sum(t.numel() * t.element_size() for t in (*dit.parameters(), *dit.buffers()))+ pipe = types.SimpleNamespace(transformer = dit, unet = None, vae = None)+ assert diffusion_speed._fuse_qkv(pipe, None) is False+ after = sum(t.numel() * t.element_size() for t in (*dit.parameters(), *dit.buffers()))+ assert after == before+ assert not [n for n, _m in dit.named_modules() if n.endswith(("to_qkv", "to_added_qkv"))]