Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: pair Qwen-Image-2.1's guidance branches per streamed block (#17111)
Daniel Hanc953832da2a8
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.pyindex f92607536fb..dc743d428ca 100644--- a/studio/backend/core/inference/diffusion_cfg_batch.py+++ b/studio/backend/core/inference/diffusion_cfg_batch.py@@ -1131,7 +1131,7 @@ class CfgBatchRouter:return minedef _pair(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:- from .diffusion_cfg_pair import NotPairable, root_forward+ from .diffusion_cfg_pair import NotPairable, SkipStep, root_forwardtorch = _torch()module = self.module_ref()@@ -1140,7 +1140,7 @@ class CfgBatchRouter:self.disable("the denoiser's own forward is no longer reachable below its hooks")return Nonetry:- second = self._predict(latent)+ second = self.paired.align((args, kwargs), self._predict(latent))snapshots = _snapshot(second)except _NotBatchable as exc:self._shape_off(shape, f"not pairable: {exc}")@@ -1150,6 +1150,8 @@ class CfgBatchRouter:mine, other = self.paired.run(module, self.orig_forward, root_fn, (args, kwargs), second)+ except SkipStep:+ return Noneexcept NotPairable as exc:self._shape_off(shape, f"not pairable: {exc}")return Nonediff --git a/studio/backend/core/inference/diffusion_cfg_pair.py b/studio/backend/core/inference/diffusion_cfg_pair.pyindex 7ea75ec230e..998018cb83e 100644--- a/studio/backend/core/inference/diffusion_cfg_pair.py+++ b/studio/backend/core/inference/diffusion_cfg_pair.py@@ -3,7 +3,8 @@"""Streamed denoiser: each block runs both guidance branches while its weights are on the card, so a step uploadsthem once instead of once per branch. Every block call keeps its plain inputs and shapes, so outputs are-bit-identical. UNSLOTH_DIFFUSION_CFG_PAIR=0 streams each branch on its own."""+bit-identical. UNSLOTH_DIFFUSION_CFG_PAIR=0 streams each branch on its own; UNSLOTH_DIFFUSION_CFG_PAIR_Q21=0 only+Qwen-Image-2.1's."""from __future__ import annotations@@ -11,6 +12,7 @@ import osfrom typing import Any, OptionalCFG_PAIR_ENV = "UNSLOTH_DIFFUSION_CFG_PAIR"+CFG_PAIR_Q21_ENV = "UNSLOTH_DIFFUSION_CFG_PAIR_Q21"# class -> (block list attribute, the block output's names in the next block's keyword arguments)PAIRED_CLASSES = {@@ -18,6 +20,19 @@ PAIRED_CLASSES = {"transformer_blocks",("encoder_hidden_states", "hidden_states"),),+ "QwenImage21Transformer2DModel": ("transformer_blocks", ("hidden_states",)),+}++# (cache kwarg, block entry kwarg, mode kwarg, pairing mode): decode steps only read their branch's prefix K/V; the+# prefill writes it and walks every block's processor, so it stays plain.+BRANCH_CACHES = {+ "QwenImage21Transformer2DModel": ("kv_cache", "layer_cache", "kv_cache_mode", "cached"),+}++# Prologue modules replayed from the first branch when both pass the same input objects: block-level offload hooks each+# child of a Sequential, so calling it again would upload it and step its prefetch chain. Modulation reads only those.+SHARED_PROLOGUE = {+ "QwenImage21Transformer2DModel": (("modulation",), ("timestep", "hidden_states")),}_OFF = ("0", "off", "false", "no", "disable", "disabled")@@ -31,6 +46,10 @@ class NotPairable(Exception):pass+class SkipStep(NotPairable):+ """This step stays plain; later steps of the shape may still pair."""++class _Captured(Exception):pass@@ -75,6 +94,11 @@ def _spec(module: Any) -> Optional[tuple]:cls = type(module)if not cls.__module__.startswith("diffusers."):return None+ if (+ cls.__name__ in BRANCH_CACHES+ and (os.environ.get(CFG_PAIR_Q21_ENV) or "").strip().lower() in _OFF+ ):+ return Nonereturn PAIRED_CLASSES.get(cls.__name__)@@ -104,6 +128,10 @@ def pair_decline(module: Any, fallback_forward: Any = None) -> Optional[str]:if any(below_hook_ref(b) is None for b in blocks):return "a block runs without an offload hook to pair below"inside = {id(m) for b in blocks for m in b.modules()}+ for name in SHARED_PROLOGUE.get(type(module).__name__, ((), ()))[0]:+ shared = getattr(module, "_modules", {}).get(name)+ if isinstance(shared, torch.nn.Module):+ inside.update(id(m) for m in shared.modules())for name, sub in module.named_modules():if sub is module or id(sub) in inside:continue@@ -138,6 +166,13 @@ def _modules() -> tuple:return _Probe, _Inject+def _replay(out: Any) -> Any:+ _, inject_cls = _modules()+ stand_in = inject_cls()+ stand_in.out = out+ return stand_in++class PairedForward:"""Second branch rides the first's block calls; its prologue/epilogue run the stock forward on a one-block list."""@@ -149,9 +184,22 @@ class PairedForward:self.probe_list = torch.nn.ModuleList([self.probe])self.inject_list = torch.nn.ModuleList([self.inject])self.blocks_attr, self.carry = _spec(module)+ self.branch_cache = BRANCH_CACHES.get(type(module).__name__)++ def align(self, first: tuple, second: tuple) -> tuple:+ """The template was recorded on the prefill step; both branches share this step's cache mode."""+ if self.branch_cache is None:+ return second+ mode_kw = self.branch_cache[2]+ args2, kwargs2 = second+ if mode_kw not in kwargs2 or mode_kw not in first[1]:+ return second+ return args2, {**kwargs2, mode_kw: first[1][mode_kw]}def run(self, module: Any, outer: Any, root_fn: Any, first: tuple, second: tuple) -> tuple:"""(first output, second output). ``outer`` is the hooked forward a plain first-branch call goes through."""+ import torch+(args1, kwargs1), (args2, kwargs2) = first, secondif args1 or args2:raise NotPairable("positional arguments")@@ -161,6 +209,17 @@ class PairedForward:att = kw.get("attention_kwargs") or {}if isinstance(att, dict) and att.get("scale", 1.0) != 1.0:raise NotPairable("a LoRA scale in attention_kwargs")+ shared_names, same_inputs = SHARED_PROLOGUE.get(type(module).__name__, ((), ()))+ if shared_names and any(kwargs2.get(k) is not kwargs1.get(k) for k in same_inputs):+ raise SkipStep("the branches do not share the prologue inputs")+ second_cache = None+ if self.branch_cache is not None:+ cache_kw, _entry_kw, mode_kw, mode = self.branch_cache+ if kwargs1.get(mode_kw) != mode or kwargs2.get(mode_kw) != mode:+ raise SkipStep(f"{mode_kw} is not {mode!r}")+ second_cache = kwargs2.get(cache_kw)+ if second_cache is None or second_cache is kwargs1.get(cache_kw):+ raise NotPairable("the branches do not own separate caches")mods = module._modulesblocks = mods[self.blocks_attr]refs = [below_hook_ref(b) for b in blocks]@@ -168,14 +227,21 @@ class PairedForward:raise NotPairable("a block lost its offload hook")last = len(blocks) - 1state: dict = {}+ shared_out: dict = {}def nested(stand_in: Any) -> Any:saved = mods[self.blocks_attr]+ saved_shared = {name: mods[name] for name in shared_names}+ if set(shared_out) != set(shared_names):+ raise NotPairable("the first branch skipped a shared prologue module")mods[self.blocks_attr] = stand_in+ for name in shared_names:+ mods[name] = _replay(shared_out[name])try:return root_fn(**kwargs2)finally:mods[self.blocks_attr] = saved+ mods.update(saved_shared)def paired(index: int, compute: Any) -> Any:def call(*a: Any, **k: Any) -> Any:@@ -190,8 +256,13 @@ class PairedForward:if p_args:raise NotPairable("positional block arguments")state["kwargs"] = dict(p_kwargs)+ if second_cache is not None and self.branch_cache[1] not in p_kwargs:+ raise NotPairable("a block call carries no cache entry")+ if second_cache is not None:+ state["kwargs"][self.branch_cache[1]] = second_cache.get_layer(index)second_out = compute(**state["kwargs"])- state["kwargs"].update(zip(self.carry, second_out))+ carried = (second_out,) if isinstance(second_out, torch.Tensor) else second_out+ state["kwargs"].update(zip(self.carry, carried))if index == last:self.inject.out = second_outtry:@@ -203,6 +274,12 @@ class PairedForward:return callsaved = [r.forward for r in refs]+ handles = [+ mods[name].register_forward_hook(+ lambda _m, _a, out, _name = name: shared_out.__setitem__(_name, out)+ )+ for name in shared_names+ ]try:for i, r in enumerate(refs):r.forward = paired(i, saved[i])@@ -210,6 +287,8 @@ class PairedForward:finally:for r, fn in zip(refs, saved):r.forward = fn+ for handle in handles:+ handle.remove()if "out" not in state:raise NotPairable("the forward never reached its last block")return mine, state["out"]diff --git a/studio/backend/tests/test_diffusion_cfg_pair.py b/studio/backend/tests/test_diffusion_cfg_pair.pyindex ced2159f0fd..019d851ab44 100644--- a/studio/backend/tests/test_diffusion_cfg_pair.py+++ b/studio/backend/tests/test_diffusion_cfg_pair.py@@ -165,3 +165,152 @@ def test_a_lora_scale_or_controlnet_call_stays_plain():pf.run(model, model.forward, None, ((), {"attention_kwargs": {"scale": 0.5}}), ((), {}))with pytest.raises(cp.NotPairable):pf.run(model, model.forward, None, ((), {"controlnet_block_samples": [1]}), ((), {}))+++# ---------------------------------------------------------------------------------------------- Qwen-Image-2.1+++def _q21_mod():+ return pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")+++def _q21(layers = 3):+ qmod = _q21_mod()+ torch.manual_seed(0)+ m = qmod.QwenImage21Transformer2DModel(+ num_layers = layers,+ attention_head_dim = 16,+ num_attention_heads = 2,+ context_in_dim = 24,+ in_channels = 8,+ out_channels = 8,+ axes_dims_rope = (4, 6, 6),+ ).eval()+ with torch.no_grad():+ for p in m.parameters():+ p.normal_(0, 0.2)+ return m+++def _q21_loop(+ model,+ steps = 5,+ *,+ cond = True,+ cache = True,+ swap_neg_at = None,+ scale = 4.0,+):+ """The diffusers Qwen-Image-2.1 loop: per-branch prompts, masks and prefix KV caches; step 1 prefills."""+ qmod = _q21_mod()+ g = torch.Generator().manual_seed(3)+ th, tw, ch, cw = 4, 6, 4, 4+ target = th * tw++ def prompt(text):+ vlm = torch.zeros(text, dtype = torch.bool)+ if cond:+ vlm = torch.cat(+ [vlm[: text // 2], torch.ones(ch * cw // 4, dtype = torch.bool), vlm[text // 2 :]]+ )+ mask = torch.cat([vlm, torch.ones(target // 4, dtype = torch.bool)])[None]+ return torch.randn(1, vlm.numel(), 24, generator = g), mask++ pos, pos_mask = prompt(9)+ neg, neg_mask = prompt(5)+ shapes = [[(1, ch, cw), (1, th, tw)] if cond else [(1, th, tw)]]+ image = torch.randn(1, ch * cw, 8, generator = g) if cond else None+ latents = torch.randn(1, target, 8, generator = g)+ n = len(model.transformer_blocks)+ caches = (qmod.QwenImage21KVCache(n), qmod.QwenImage21KVCache(n)) if cache else (None, None)+ outs = []+ with torch.inference_mode():+ for i in range(steps):+ if i == swap_neg_at:+ neg = torch.randn_like(neg)+ mode = None if not cache else ("extract" if i == 0 else "cached")+ hs = latents if image is None else torch.cat([image, latents], 1)+ common = dict(+ hidden_states = hs,+ timestep = torch.full((1,), 1.0 - i / steps),+ img_shapes = shapes,+ kv_cache_mode = mode,+ return_dict = False,+ )+ c = model(encoder_hidden_states = pos, img_mask = pos_mask, kv_cache = caches[0], **common)[0]+ u = model(encoder_hidden_states = neg, img_mask = neg_mask, kv_cache = caches[1], **common)[0]+ c, u = c[:, -target:], u[:, -target:]+ pred = u + scale * (c - u)+ latents = latents - 0.1 * pred+ outs.append((c.clone(), u.clone()))+ return latents, outs+++def _q21_fast(request, monkeypatch):+ from core.inference import diffusion_qwenimage21 as q21++ q21.uninstall()+ if request.param:+ assert q21.install()+ yield request.param+ q21.uninstall()++[email protected]("_q21_fast", [False, True], indirect = True, ids = ["stock", "fast_step"])[email protected]("cond", [True, False])+def test_q21_decode_steps_pair_bit_identical_one_upload_per_block(_q21_fast, cond):+ ref, ref_outs = _q21_loop(_offload(_q21()), cond = cond)+ model = _offload(_q21())+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None+ router = cb.router_for(pipe)+ assert router.paired is not None and router.paired.branch_cache is not None+ counts = _block_calls(model)+ shared = []+ model.modulation[1].register_forward_pre_hook(lambda *_a: shared.append(1))+ got, outs = _q21_loop(model, cond = cond)+ assert torch.equal(got, ref)+ for (c, u), (rc, ru) in zip(outs, ref_outs):+ assert torch.equal(c, rc) and torch.equal(u, ru)+ # step 1 prefills both caches plain (2 calls per block); steps 2-5 pair (1 call per block)+ assert counts == [2 + 4] * len(counts)+ assert len(shared) == 2 + 4+ assert router.stats["paired_steps"] == 4 and router.stats["hits"] == 4+ assert router.stats["misses"] == 0+ cb.end_cfg_batch_render(pipe)+ got2, _ = _q21_loop(model, cond = cond)+ assert torch.equal(got2, ref)+ assert router.stats["paired_steps"] == 8+++def test_q21_a_changed_negative_prompt_reruns_plain():+ ref, _ = _q21_loop(_offload(_q21()), steps = 6, swap_neg_at = 3)+ model = _offload(_q21())+ pipe = _Pipe(model)+ cb.install_cfg_batch(pipe, offload_policy = "group")+ router = cb.router_for(pipe)+ got, _ = _q21_loop(model, steps = 6, swap_neg_at = 3)+ assert torch.equal(got, ref)+ assert router.stats["misses"] == 1 and router.stats["hits"] == 4+++def test_q21_without_a_prefix_cache_stays_plain_and_exact():+ ref, _ = _q21_loop(_offload(_q21()), cache = False)+ model = _offload(_q21())+ pipe = _Pipe(model)+ cb.install_cfg_batch(pipe, offload_policy = "group")+ router = cb.router_for(pipe)+ counts = _block_calls(model)+ got, _ = _q21_loop(model, cache = False)+ assert torch.equal(got, ref)+ assert router.stats["paired_steps"] == 0 and counts == [2 * 5] * len(counts)+++def test_q21_kill_switch(monkeypatch):+ monkeypatch.setenv(cp.CFG_PAIR_Q21_ENV, "0")+ model = _offload(_q21())+ assert cb.cfg_pair_decline(_Pipe(model), offload_policy = "group") == (+ "QwenImage21Transformer2DModel has no paired block loop"+ )+ assert cb.cfg_pair_decline(_Pipe(_offload(_tiny())), offload_policy = "group") is None