Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: pair Qwen-Image-2.1's guidance branches per streamed block (#17111)

Daniel Hanc953832da2a8
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index f92607536fb..dc743d428ca 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -1131,7 +1131,7 @@ class CfgBatchRouter:
return mine
def _pair(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
- from .diffusion_cfg_pair import NotPairable, root_forward
+ from .diffusion_cfg_pair import NotPairable, SkipStep, root_forward
torch = _torch()
module = self.module_ref()
@@ -1140,7 +1140,7 @@ class CfgBatchRouter:
self.disable("the denoiser's own forward is no longer reachable below its hooks")
return None
try:
- second = self._predict(latent)
+ second = self.paired.align((args, kwargs), self._predict(latent))
snapshots = _snapshot(second)
except _NotBatchable as exc:
self._shape_off(shape, f"not pairable: {exc}")
@@ -1150,6 +1150,8 @@ class CfgBatchRouter:
mine, other = self.paired.run(
module, self.orig_forward, root_fn, (args, kwargs), second
)
+ except SkipStep:
+ return None
except NotPairable as exc:
self._shape_off(shape, f"not pairable: {exc}")
return None
diff --git a/studio/backend/core/inference/diffusion_cfg_pair.py b/studio/backend/core/inference/diffusion_cfg_pair.py
index 7ea75ec230e..998018cb83e 100644
--- a/studio/backend/core/inference/diffusion_cfg_pair.py
+++ b/studio/backend/core/inference/diffusion_cfg_pair.py
@@ -3,7 +3,8 @@
"""Streamed denoiser: each block runs both guidance branches while its weights are on the card, so a step uploads
them once instead of once per branch. Every block call keeps its plain inputs and shapes, so outputs are
-bit-identical. UNSLOTH_DIFFUSION_CFG_PAIR=0 streams each branch on its own."""
+bit-identical. UNSLOTH_DIFFUSION_CFG_PAIR=0 streams each branch on its own; UNSLOTH_DIFFUSION_CFG_PAIR_Q21=0 only
+Qwen-Image-2.1's."""
from __future__ import annotations
@@ -11,6 +12,7 @@ import os
from typing import Any, Optional
CFG_PAIR_ENV = "UNSLOTH_DIFFUSION_CFG_PAIR"
+CFG_PAIR_Q21_ENV = "UNSLOTH_DIFFUSION_CFG_PAIR_Q21"
# class -> (block list attribute, the block output's names in the next block's keyword arguments)
PAIRED_CLASSES = {
@@ -18,6 +20,19 @@ PAIRED_CLASSES = {
"transformer_blocks",
("encoder_hidden_states", "hidden_states"),
),
+ "QwenImage21Transformer2DModel": ("transformer_blocks", ("hidden_states",)),
+}
+
+# (cache kwarg, block entry kwarg, mode kwarg, pairing mode): decode steps only read their branch's prefix K/V; the
+# prefill writes it and walks every block's processor, so it stays plain.
+BRANCH_CACHES = {
+ "QwenImage21Transformer2DModel": ("kv_cache", "layer_cache", "kv_cache_mode", "cached"),
+}
+
+# Prologue modules replayed from the first branch when both pass the same input objects: block-level offload hooks each
+# child of a Sequential, so calling it again would upload it and step its prefetch chain. Modulation reads only those.
+SHARED_PROLOGUE = {
+ "QwenImage21Transformer2DModel": (("modulation",), ("timestep", "hidden_states")),
}
_OFF = ("0", "off", "false", "no", "disable", "disabled")
@@ -31,6 +46,10 @@ class NotPairable(Exception):
pass
+class SkipStep(NotPairable):
+ """This step stays plain; later steps of the shape may still pair."""
+
+
class _Captured(Exception):
pass
@@ -75,6 +94,11 @@ def _spec(module: Any) -> Optional[tuple]:
cls = type(module)
if not cls.__module__.startswith("diffusers."):
return None
+ if (
+ cls.__name__ in BRANCH_CACHES
+ and (os.environ.get(CFG_PAIR_Q21_ENV) or "").strip().lower() in _OFF
+ ):
+ return None
return PAIRED_CLASSES.get(cls.__name__)
@@ -104,6 +128,10 @@ def pair_decline(module: Any, fallback_forward: Any = None) -> Optional[str]:
if any(below_hook_ref(b) is None for b in blocks):
return "a block runs without an offload hook to pair below"
inside = {id(m) for b in blocks for m in b.modules()}
+ for name in SHARED_PROLOGUE.get(type(module).__name__, ((), ()))[0]:
+ shared = getattr(module, "_modules", {}).get(name)
+ if isinstance(shared, torch.nn.Module):
+ inside.update(id(m) for m in shared.modules())
for name, sub in module.named_modules():
if sub is module or id(sub) in inside:
continue
@@ -138,6 +166,13 @@ def _modules() -> tuple:
return _Probe, _Inject
+def _replay(out: Any) -> Any:
+ _, inject_cls = _modules()
+ stand_in = inject_cls()
+ stand_in.out = out
+ return stand_in
+
+
class PairedForward:
"""Second branch rides the first's block calls; its prologue/epilogue run the stock forward on a one-block list."""
@@ -149,9 +184,22 @@ class PairedForward:
self.probe_list = torch.nn.ModuleList([self.probe])
self.inject_list = torch.nn.ModuleList([self.inject])
self.blocks_attr, self.carry = _spec(module)
+ self.branch_cache = BRANCH_CACHES.get(type(module).__name__)
+
+ def align(self, first: tuple, second: tuple) -> tuple:
+ """The template was recorded on the prefill step; both branches share this step's cache mode."""
+ if self.branch_cache is None:
+ return second
+ mode_kw = self.branch_cache[2]
+ args2, kwargs2 = second
+ if mode_kw not in kwargs2 or mode_kw not in first[1]:
+ return second
+ return args2, {**kwargs2, mode_kw: first[1][mode_kw]}
def run(self, module: Any, outer: Any, root_fn: Any, first: tuple, second: tuple) -> tuple:
"""(first output, second output). ``outer`` is the hooked forward a plain first-branch call goes through."""
+ import torch
+
(args1, kwargs1), (args2, kwargs2) = first, second
if args1 or args2:
raise NotPairable("positional arguments")
@@ -161,6 +209,17 @@ class PairedForward:
att = kw.get("attention_kwargs") or {}
if isinstance(att, dict) and att.get("scale", 1.0) != 1.0:
raise NotPairable("a LoRA scale in attention_kwargs")
+ shared_names, same_inputs = SHARED_PROLOGUE.get(type(module).__name__, ((), ()))
+ if shared_names and any(kwargs2.get(k) is not kwargs1.get(k) for k in same_inputs):
+ raise SkipStep("the branches do not share the prologue inputs")
+ second_cache = None
+ if self.branch_cache is not None:
+ cache_kw, _entry_kw, mode_kw, mode = self.branch_cache
+ if kwargs1.get(mode_kw) != mode or kwargs2.get(mode_kw) != mode:
+ raise SkipStep(f"{mode_kw} is not {mode!r}")
+ second_cache = kwargs2.get(cache_kw)
+ if second_cache is None or second_cache is kwargs1.get(cache_kw):
+ raise NotPairable("the branches do not own separate caches")
mods = module._modules
blocks = mods[self.blocks_attr]
refs = [below_hook_ref(b) for b in blocks]
@@ -168,14 +227,21 @@ class PairedForward:
raise NotPairable("a block lost its offload hook")
last = len(blocks) - 1
state: dict = {}
+ shared_out: dict = {}
def nested(stand_in: Any) -> Any:
saved = mods[self.blocks_attr]
+ saved_shared = {name: mods[name] for name in shared_names}
+ if set(shared_out) != set(shared_names):
+ raise NotPairable("the first branch skipped a shared prologue module")
mods[self.blocks_attr] = stand_in
+ for name in shared_names:
+ mods[name] = _replay(shared_out[name])
try:
return root_fn(**kwargs2)
finally:
mods[self.blocks_attr] = saved
+ mods.update(saved_shared)
def paired(index: int, compute: Any) -> Any:
def call(*a: Any, **k: Any) -> Any:
@@ -190,8 +256,13 @@ class PairedForward:
if p_args:
raise NotPairable("positional block arguments")
state["kwargs"] = dict(p_kwargs)
+ if second_cache is not None and self.branch_cache[1] not in p_kwargs:
+ raise NotPairable("a block call carries no cache entry")
+ if second_cache is not None:
+ state["kwargs"][self.branch_cache[1]] = second_cache.get_layer(index)
second_out = compute(**state["kwargs"])
- state["kwargs"].update(zip(self.carry, second_out))
+ carried = (second_out,) if isinstance(second_out, torch.Tensor) else second_out
+ state["kwargs"].update(zip(self.carry, carried))
if index == last:
self.inject.out = second_out
try:
@@ -203,6 +274,12 @@ class PairedForward:
return call
saved = [r.forward for r in refs]
+ handles = [
+ mods[name].register_forward_hook(
+ lambda _m, _a, out, _name = name: shared_out.__setitem__(_name, out)
+ )
+ for name in shared_names
+ ]
try:
for i, r in enumerate(refs):
r.forward = paired(i, saved[i])
@@ -210,6 +287,8 @@ class PairedForward:
finally:
for r, fn in zip(refs, saved):
r.forward = fn
+ for handle in handles:
+ handle.remove()
if "out" not in state:
raise NotPairable("the forward never reached its last block")
return mine, state["out"]
diff --git a/studio/backend/tests/test_diffusion_cfg_pair.py b/studio/backend/tests/test_diffusion_cfg_pair.py
index ced2159f0fd..019d851ab44 100644
--- a/studio/backend/tests/test_diffusion_cfg_pair.py
+++ b/studio/backend/tests/test_diffusion_cfg_pair.py
@@ -165,3 +165,152 @@ def test_a_lora_scale_or_controlnet_call_stays_plain():
pf.run(model, model.forward, None, ((), {"attention_kwargs": {"scale": 0.5}}), ((), {}))
with pytest.raises(cp.NotPairable):
pf.run(model, model.forward, None, ((), {"controlnet_block_samples": [1]}), ((), {}))
+
+
+# ---------------------------------------------------------------------------------------------- Qwen-Image-2.1
+
+
+def _q21_mod():
+ return pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
+
+
+def _q21(layers = 3):
+ qmod = _q21_mod()
+ torch.manual_seed(0)
+ m = qmod.QwenImage21Transformer2DModel(
+ num_layers = layers,
+ attention_head_dim = 16,
+ num_attention_heads = 2,
+ context_in_dim = 24,
+ in_channels = 8,
+ out_channels = 8,
+ axes_dims_rope = (4, 6, 6),
+ ).eval()
+ with torch.no_grad():
+ for p in m.parameters():
+ p.normal_(0, 0.2)
+ return m
+
+
+def _q21_loop(
+ model,
+ steps = 5,
+ *,
+ cond = True,
+ cache = True,
+ swap_neg_at = None,
+ scale = 4.0,
+):
+ """The diffusers Qwen-Image-2.1 loop: per-branch prompts, masks and prefix KV caches; step 1 prefills."""
+ qmod = _q21_mod()
+ g = torch.Generator().manual_seed(3)
+ th, tw, ch, cw = 4, 6, 4, 4
+ target = th * tw
+
+ def prompt(text):
+ vlm = torch.zeros(text, dtype = torch.bool)
+ if cond:
+ vlm = torch.cat(
+ [vlm[: text // 2], torch.ones(ch * cw // 4, dtype = torch.bool), vlm[text // 2 :]]
+ )
+ mask = torch.cat([vlm, torch.ones(target // 4, dtype = torch.bool)])[None]
+ return torch.randn(1, vlm.numel(), 24, generator = g), mask
+
+ pos, pos_mask = prompt(9)
+ neg, neg_mask = prompt(5)
+ shapes = [[(1, ch, cw), (1, th, tw)] if cond else [(1, th, tw)]]
+ image = torch.randn(1, ch * cw, 8, generator = g) if cond else None
+ latents = torch.randn(1, target, 8, generator = g)
+ n = len(model.transformer_blocks)
+ caches = (qmod.QwenImage21KVCache(n), qmod.QwenImage21KVCache(n)) if cache else (None, None)
+ outs = []
+ with torch.inference_mode():
+ for i in range(steps):
+ if i == swap_neg_at:
+ neg = torch.randn_like(neg)
+ mode = None if not cache else ("extract" if i == 0 else "cached")
+ hs = latents if image is None else torch.cat([image, latents], 1)
+ common = dict(
+ hidden_states = hs,
+ timestep = torch.full((1,), 1.0 - i / steps),
+ img_shapes = shapes,
+ kv_cache_mode = mode,
+ return_dict = False,
+ )
+ c = model(encoder_hidden_states = pos, img_mask = pos_mask, kv_cache = caches[0], **common)[0]
+ u = model(encoder_hidden_states = neg, img_mask = neg_mask, kv_cache = caches[1], **common)[0]
+ c, u = c[:, -target:], u[:, -target:]
+ pred = u + scale * (c - u)
+ latents = latents - 0.1 * pred
+ outs.append((c.clone(), u.clone()))
+ return latents, outs
+
+
+def _q21_fast(request, monkeypatch):
+ from core.inference import diffusion_qwenimage21 as q21
+
+ q21.uninstall()
+ if request.param:
+ assert q21.install()
+ yield request.param
+ q21.uninstall()
+
+
[email protected]("_q21_fast", [False, True], indirect = True, ids = ["stock", "fast_step"])
[email protected]("cond", [True, False])
+def test_q21_decode_steps_pair_bit_identical_one_upload_per_block(_q21_fast, cond):
+ ref, ref_outs = _q21_loop(_offload(_q21()), cond = cond)
+ model = _offload(_q21())
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None
+ router = cb.router_for(pipe)
+ assert router.paired is not None and router.paired.branch_cache is not None
+ counts = _block_calls(model)
+ shared = []
+ model.modulation[1].register_forward_pre_hook(lambda *_a: shared.append(1))
+ got, outs = _q21_loop(model, cond = cond)
+ assert torch.equal(got, ref)
+ for (c, u), (rc, ru) in zip(outs, ref_outs):
+ assert torch.equal(c, rc) and torch.equal(u, ru)
+ # step 1 prefills both caches plain (2 calls per block); steps 2-5 pair (1 call per block)
+ assert counts == [2 + 4] * len(counts)
+ assert len(shared) == 2 + 4
+ assert router.stats["paired_steps"] == 4 and router.stats["hits"] == 4
+ assert router.stats["misses"] == 0
+ cb.end_cfg_batch_render(pipe)
+ got2, _ = _q21_loop(model, cond = cond)
+ assert torch.equal(got2, ref)
+ assert router.stats["paired_steps"] == 8
+
+
+def test_q21_a_changed_negative_prompt_reruns_plain():
+ ref, _ = _q21_loop(_offload(_q21()), steps = 6, swap_neg_at = 3)
+ model = _offload(_q21())
+ pipe = _Pipe(model)
+ cb.install_cfg_batch(pipe, offload_policy = "group")
+ router = cb.router_for(pipe)
+ got, _ = _q21_loop(model, steps = 6, swap_neg_at = 3)
+ assert torch.equal(got, ref)
+ assert router.stats["misses"] == 1 and router.stats["hits"] == 4
+
+
+def test_q21_without_a_prefix_cache_stays_plain_and_exact():
+ ref, _ = _q21_loop(_offload(_q21()), cache = False)
+ model = _offload(_q21())
+ pipe = _Pipe(model)
+ cb.install_cfg_batch(pipe, offload_policy = "group")
+ router = cb.router_for(pipe)
+ counts = _block_calls(model)
+ got, _ = _q21_loop(model, cache = False)
+ assert torch.equal(got, ref)
+ assert router.stats["paired_steps"] == 0 and counts == [2 * 5] * len(counts)
+
+
+def test_q21_kill_switch(monkeypatch):
+ monkeypatch.setenv(cp.CFG_PAIR_Q21_ENV, "0")
+ model = _offload(_q21())
+ assert cb.cfg_pair_decline(_Pipe(model), offload_policy = "group") == (
+ "QwenImage21Transformer2DModel has no paired block loop"
+ )
+ assert cb.cfg_pair_decline(_Pipe(_offload(_tiny())), offload_policy = "group") is None