Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: CFG-parallel for U-Net families (SDXL) by batch split (#16679)
Daniel Haneb805d3b19a2
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex 4ab7e593fd2..f97a2cabdf6 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -52,6 +52,9 @@ def multigpu_mode() -> str:return "auto"+UNET_CFG_PARALLEL_ENV = "UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET"++def cfg_parallel_mode() -> str:"""``off`` / ``auto`` (default) / ``force`` (also across different card models)."""if multigpu_mode() == "off":@@ -3013,10 +3016,145 @@ def _replica_call(replica: Any, args: tuple, kwargs: dict) -> Callable[[], Any]:return lambda: replica(*args, **kwargs)+# U-Net keyword arguments a batch split may carry; any other tensor-bearing keyword keeps the call whole on the source.+_UNET_SPLIT_KWARGS = frozenset(+ {+ "encoder_hidden_states",+ "timestep_cond",+ "added_cond_kwargs",+ "cross_attention_kwargs",+ "down_block_additional_residuals",+ "mid_block_additional_residual",+ "return_dict",+ }+)+++def _unet_cfg_parallel_disabled() -> bool:+ return (os.environ.get(UNET_CFG_PARALLEL_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ )+++def _split_batch(+ value: Any,+ n: int,+ half: int,+ depth: int = 0,+) -> Any:+ """``value`` with every tensor of leading size ``n`` cut to its ``half`` (0 = first, 1 = second) along dim 0;+ other tensors and plain values are shared. Raises TypeError on an object it cannot see into."""+ torch = _torch()+ if isinstance(value, torch.Tensor):+ if value.dim() >= 1 and value.shape[0] == n:+ b = n // 2+ return value[:b] if half == 0 else value[b:]+ return value+ if value is None or isinstance(value, _PLAIN_TYPES):+ return value+ if depth > 4:+ raise TypeError("nested too deep")+ if isinstance(value, dict):+ return {k: _split_batch(v, n, half, depth + 1) for k, v in value.items()}+ if isinstance(value, (list, tuple)):+ return type(value)(_split_batch(v, n, half, depth + 1) for v in value)+ raise TypeError(f"cannot split {type(value).__name__}")+++def _join_batch(a: Any, b: Any) -> Any:+ torch = _torch()+ if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):+ return torch.cat([a, b.to(a.device)], dim = 0)+ if isinstance(a, tuple) and isinstance(b, tuple) and len(a) == len(b) and a:+ return (_join_batch(a[0], b[0]),) + tuple(a[1:])+ sample_a, sample_b = getattr(a, "sample", None), getattr(b, "sample", None)+ if isinstance(sample_a, torch.Tensor) and isinstance(sample_b, torch.Tensor):+ return type(a)(sample = torch.cat([sample_a, sample_b.to(sample_a.device)], dim = 0))+ raise TypeError(f"unexpected U-Net output {type(a).__name__}")+++class UnetCfgParallelRouter(CfgParallelRouter):+ """CFG-parallel for a U-Net (SDXL / SD 1.x / 2.x): diffusers sends both guidance branches through ONE call at batch+ ``2 * B``, so there is no second call to route. Each such call is cut in half along the batch: the first half runs+ on the source card, the second on the replica at the same time (:meth:`CfgParallelRouter.run_pair`), and the two+ outputs are concatenated. A U-Net treats every sample on its own, so the math is the same; the bits differ only+ where the batch-B kernels round differently from batch 2B. Any even batch splits (two images without guidance+ too, which then builds the replica). Odd batches, unknown inputs, a compiled-whole module and an unavailable+ replica run the whole call on the source. ``UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET=0`` keeps U-Nets on one card."""++ def __call__(self, *args: Any, **kwargs: Any) -> Any:+ torch = _torch()+ module = self.module_ref()+ if (+ self._pairing+ or self.disabled_reason is not None+ or _background_compile_call()+ or torch.compiler.is_compiling()+ # compiled whole after the router was installed (a deferred speed profile): its graph owns the call+ or getattr(module, "_compiled_call_impl", None) is not None+ ):+ self.stats["primary_calls"] += 1+ return self._primary(*args, **kwargs)+ sample = kwargs.get("sample", args[0] if args else None)+ n = sample.shape[0] if isinstance(sample, torch.Tensor) and sample.dim() >= 1 else 0+ halves = None+ if (+ n >= 2+ and n % 2 == 0+ and set(k for k in kwargs if k not in ("sample", "timestep")) <= _UNET_SPLIT_KWARGS+ ):+ try:+ halves = [(_split_batch(args, n, h), _split_batch(kwargs, n, h)) for h in (0, 1)]+ except TypeError:+ halves = None+ if halves is None:+ self.stats["primary_calls"] += 1+ return self._primary(*args, **kwargs)+ (a0, k0), (a1, k1) = halves+ self.stats["primary_calls"] += 1+ # the primary half waits for a cold replica (a first call may compile) and marks its signature warm+ res = self.run_pair(lambda: self._primary_noting(None, a0, k0), a1, k1)+ if res is None:+ return self._primary(*args, **kwargs)+ first, second = res+ if second is None:+ second = self._primary(*a1, **k1)+ return _join_batch(first, second)++_CFG_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgParallelRouter]" = weakref.WeakKeyDictionary()_CFG_DECLINES: "weakref.WeakKeyDictionary[Any, str]" = weakref.WeakKeyDictionary()+def _cfg_denoiser(pipe: Any) -> Any:+ """The module CFG-parallel routes: ``pipe.transformer``, else a U-Net (batch split, :class:`UnetCfgParallelRouter`)+ unless ``UNSLOTH_DIFFUSION_CFG_PARALLEL_UNET=0``."""+ denoiser = getattr(pipe, "transformer", None)+ if denoiser is not None:+ return denoiser+ unet = getattr(pipe, "unet", None)+ if (+ unet is not None+ and type(unet).__name__ == "UNet2DConditionModel"+ and not _unet_cfg_parallel_disabled()+ ):+ if _batch_coupled_processors(unet):+ return None # PAG-style processors chunk the batch themselves: a dim-0 split would mix their thirds+ return unet+ return None+++def _batch_coupled_processors(unet: Any) -> bool:+ try:+ processors = getattr(unet, "attn_processors", None) or {}+ return any("PAG" in type(p).__name__ for p in processors.values())+ except Exception: # noqa: BLE001 - unreadable: treat as coupled+ return True++def cfg_parallel_decline(pipe: Any, *, offload_policy: str, selection: Optional[MultiGpuSelection]) -> Optional[str]:@@ -3038,9 +3176,14 @@ def cfg_parallel_decline(if bg_compile_enabled():# its replays call the denoiser, which would enter the router and race a render's branch statereturn "background compile replays through the denoiser (UNSLOTH_DIFFUSION_BG_COMPILE)"- denoiser = getattr(pipe, "transformer", None)+ denoiser = _cfg_denoiser(pipe)torch = _torch()if not isinstance(denoiser, torch.nn.Module):+ if (+ isinstance(getattr(pipe, "unet", None), torch.nn.Module)+ and _unet_cfg_parallel_disabled()+ ):+ return f"U-Net CFG-parallel off ({UNET_CFG_PARALLEL_ENV}=0)"return "no DiT denoiser (a U-Net batches its guidance branches)"from .diffusion_layer_split import is_layer_split@@ -3089,7 +3232,7 @@ def install_cfg_parallel(selection = current_selection()reason = cfg_parallel_decline(pipe, offload_policy = offload_policy, selection = selection)if reason is None:- denoiser = pipe.transformer+ denoiser = _cfg_denoiser(pipe)primary = _module_device(denoiser)probes = probe_devices((selection.primary, *selection.secondaries))same = [@@ -3133,7 +3276,12 @@ def install_cfg_parallel(graph = _resident_graph(denoiser)if graph is not None:del denoiser.__dict__["forward"]- router = CfgParallelRouter(+ router_cls = (+ UnetCfgParallelRouter+ if denoiser is not getattr(pipe, "transformer", None)+ else CfgParallelRouter+ )+ router = router_cls(denoiser, primary = primary, secondary = f"cuda:{best}", logger = logger)router.inner = graph@@ -3147,8 +3295,9 @@ def install_cfg_parallel(denoiser._unsloth_cfg_router = routerif logger is not None:logger.info(- "diffusion.multigpu: CFG-parallel armed (replica on cuda:%s when guidance runs two branches)",+ "diffusion.multigpu: CFG-parallel armed (replica on cuda:%s when guidance runs two branches%s)",best,+ "; U-Net batch split" if router_cls is UnetCfgParallelRouter else "",)return Noneif selection is not None:diff --git a/studio/backend/tests/test_diffusion_unet_cfg_parallel.py b/studio/backend/tests/test_diffusion_unet_cfg_parallel.pynew file mode 100644index 00000000000..199e1990b5c--- /dev/null+++ b/studio/backend/tests/test_diffusion_unet_cfg_parallel.py@@ -0,0 +1,226 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CFG-parallel for U-Nets (SDXL / SD): one batch-2B call cut into two batch-B halves, the second on the replica."""++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_multigpu as mg # noqa: E402+++class UNet2DConditionModel(torch.nn.Module):+ """Named like diffusers' class: the router keys on the class name."""++ def __init__(self):+ super().__init__()+ self.conv = torch.nn.Conv2d(4, 4, 3, padding = 1)+ self.text = torch.nn.Linear(8, 4)+ self.add = torch.nn.Linear(6, 4)+ self.calls = []++ def forward(+ self,+ sample,+ timestep,+ encoder_hidden_states,+ added_cond_kwargs = None,+ cross_attention_kwargs = None,+ return_dict = True,+ **extra,+ ):+ self.calls.append(int(sample.shape[0]))+ h = self.conv(sample) * timestep+ h = h + self.text(encoder_hidden_states).mean(1)[:, :, None, None]+ if added_cond_kwargs:+ h = h + self.add(added_cond_kwargs["time_ids"])[:, :, None, None]+ for key, value in extra.items():+ if isinstance(value, torch.Tensor):+ h = h + value.sum()+ return (h,) if not return_dict else type("Out", (), {"sample": h})()+++def _router(unet):+ replicas = []++ def factory():+ r = mg.build_replica(unet, torch.device("cpu"))+ replicas.append(r)+ return r++ router = mg.UnetCfgParallelRouter(unet, primary = "cpu", secondary = "cpu", replica_factory = factory)+ unet.forward = router+ return router, replicas+++def _sdxl_loop(+ unet,+ latents,+ steps,+ batch = 1,+ cfg = True,+ **extra,+):+ g = torch.Generator().manual_seed(3)+ cond = torch.randn(batch, 5, 8, generator = g)+ uncond = torch.randn(batch, 5, 8, generator = g)+ ids = torch.randn(batch, 6, generator = g)+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ if cfg:+ x = torch.cat([latents] * 2)+ emb = torch.cat([uncond, cond])+ tid = torch.cat([ids, ids])+ else:+ x, emb, tid = latents, cond, ids+ out = unet(+ x,+ t,+ encoder_hidden_states = emb,+ added_cond_kwargs = {"time_ids": tid},+ return_dict = False,+ **extra,+ )[0]+ if cfg:+ u, c = out.chunk(2)+ out = u + 5.0 * (c - u)+ latents = latents - 0.1 * out+ return latents+++def test_unet_cfg_parallel_matches_one_card_and_runs_halves_on_the_replica():+ torch.manual_seed(0)+ unet = UNet2DConditionModel()+ lat = torch.randn(1, 4, 8, 8)+ ref = _sdxl_loop(unet, lat.clone(), steps = 4)+ assert unet.calls == [2, 2, 2, 2]+ unet.calls.clear()+ router, replicas = _router(unet)+ out = _sdxl_loop(unet, lat.clone(), steps = 4)+ torch.testing.assert_close(out, ref, rtol = 1e-5, atol = 1e-6)+ assert len(replicas) == 1 and replicas[0] is not unet+ # the source runs one half per step; the replica the other+ assert unet.calls == [1, 1, 1, 1]+ assert replicas[0].calls == [1, 1, 1, 1]+ assert router.stats["pairs_parallel"] == 4 and router.stats["replica_calls"] == 4+ assert router.describe()["disabled_reason"] is None+++def test_unet_batch_two_per_branch_is_cut_in_half():+ unet = UNet2DConditionModel()+ lat = torch.randn(2, 4, 8, 8)+ ref = _sdxl_loop(unet, lat.clone(), steps = 2, batch = 2)+ unet.calls.clear()+ router, replicas = _router(unet)+ out = _sdxl_loop(unet, lat.clone(), steps = 2, batch = 2)+ torch.testing.assert_close(out, ref, rtol = 1e-5, atol = 1e-6)+ assert unet.calls == [2, 2] and replicas[0].calls == [2, 2]+++def test_unet_odd_batch_stays_whole_on_the_source():+ unet = UNet2DConditionModel()+ router, replicas = _router(unet)+ _sdxl_loop(unet, torch.randn(1, 4, 8, 8), steps = 3, cfg = False)+ assert unet.calls == [1, 1, 1] and replicas == [] and router.replica is None+ assert router.stats["replica_calls"] == 0+++def test_unet_unknown_tensor_keyword_stays_whole_on_the_source():+ unet = UNet2DConditionModel()+ router, replicas = _router(unet)+ _sdxl_loop(unet, torch.randn(1, 4, 8, 8), steps = 2, mystery = torch.ones(2, 3))+ assert unet.calls == [2, 2] and replicas == []+++def test_unet_replica_failure_recomputes_on_the_source():+ unet = UNet2DConditionModel()+ lat = torch.randn(1, 4, 8, 8)+ ref = _sdxl_loop(unet, lat.clone(), steps = 3)++ def broken():+ raise RuntimeError("no room")++ router = mg.UnetCfgParallelRouter(unet, primary = "cpu", secondary = "cpu", replica_factory = broken)+ unet.forward = router+ out = _sdxl_loop(unet, lat.clone(), steps = 3)+ torch.testing.assert_close(out, ref, rtol = 1e-5, atol = 1e-6)+ assert "replica unavailable" in (router.disabled_reason or "")+++def test_unet_is_a_cfg_parallel_candidate(monkeypatch):+ class Pipe:+ transformer = None+ unet = UNet2DConditionModel()++ sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ # resident on the CPU here: past the U-Net gate, refused only for not being on a CUDA card+ assert (+ mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = sel)+ == "denoiser is not on a CUDA card"+ )+ assert mg._cfg_denoiser(Pipe()) is Pipe.unet+ monkeypatch.setenv(mg.UNET_CFG_PARALLEL_ENV, "0")+ assert mg._cfg_denoiser(Pipe()) is None+ assert mg.UNET_CFG_PARALLEL_ENV in mg.cfg_parallel_decline(+ Pipe(), offload_policy = "none", selection = sel+ )+++def test_real_diffusers_unet_split_matches_batched():+ diffusers = pytest.importorskip("diffusers")+ torch.manual_seed(0)+ unet = diffusers.UNet2DConditionModel(+ sample_size = 8,+ in_channels = 4,+ out_channels = 4,+ layers_per_block = 1,+ block_out_channels = (8, 16),+ down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"),+ up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),+ cross_attention_dim = 8,+ attention_head_dim = 2,+ norm_num_groups = 4,+ addition_embed_type = "text_time",+ addition_time_embed_dim = 4,+ projection_class_embeddings_input_dim = 8 + 6 * 4,+ ).eval()+ x = torch.randn(2, 4, 8, 8)+ emb = torch.randn(2, 5, 8)+ added = {"text_embeds": torch.randn(2, 8), "time_ids": torch.randn(2, 6)}+ t = torch.tensor(500.0)+ with torch.no_grad():+ ref = unet(x, t, encoder_hidden_states = emb, added_cond_kwargs = added, return_dict = False)[0]+ router = mg.UnetCfgParallelRouter(+ unet,+ primary = "cpu",+ secondary = "cpu",+ replica_factory = lambda: mg.build_replica(unet, torch.device("cpu")),+ )+ unet.forward = router+ out = unet(x, t, encoder_hidden_states = emb, added_cond_kwargs = added, return_dict = False)[0]+ out2 = unet(x, t, encoder_hidden_states = emb, added_cond_kwargs = added).sample+ torch.testing.assert_close(out, ref, rtol = 1e-4, atol = 1e-5)+ torch.testing.assert_close(out2, ref, rtol = 1e-4, atol = 1e-5)+ assert router.stats["pairs_parallel"] == 2+++def test_unet_with_batch_chunking_processors_is_not_a_candidate():+ class PAGCFGIdentitySelfAttnProcessor2_0:+ pass++ class Pipe:+ transformer = None+ unet = UNet2DConditionModel()++ Pipe.unet.attn_processors = {"down.attn1.processor": PAGCFGIdentitySelfAttnProcessor2_0()}+ assert mg._cfg_denoiser(Pipe()) is None+++def test_unet_compiled_whole_after_install_runs_whole_on_the_source():+ unet = UNet2DConditionModel()+ router, replicas = _router(unet)+ # a deferred whole-module compile landed after the router: nn.Module.__call__ now goes through it+ unet._compiled_call_impl = lambda *a, **k: unet.forward(*a, **k)+ _sdxl_loop(unet, torch.randn(1, 4, 8, 8), steps = 2)+ assert unet.calls == [2, 2] and replicas == []