Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: single-card CFG batching for a DiT streamed from host memory (#13787)
Daniel Hand47cdddc5f7d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 99094681eed..1befc53a69f 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -108,6 +108,7 @@ from .diffusion_device import (resolve_selected_cuda_ordinal,resolve_selected_cuda_ordinals,)+from .diffusion_cfg_batch import cfg_batch_status, end_cfg_batch_render, install_cfg_batchfrom .diffusion_multigpu import (AUTOMATIC_ORDINALS,end_cfg_render,@@ -9605,6 +9606,8 @@ class DiffusionBackend:)# After every forward wrapper of the speed / graph layers, so the replica sees the final module.install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)+ # both branches through one offloaded forward; declines when CFG-parallel took the slot+ install_cfg_batch(pipe, offload_policy = effective_policy, logger = logger)te_releaser = te_release.maybe_install(pipe, plan, logger = logger)state = _LoadState(pipe = pipe,@@ -12698,6 +12701,7 @@ class DiffusionBackend:if restore_vae is not None:restore_vae()end_cfg_render(getattr(state, "pipe", None))+ end_cfg_batch_render(getattr(state, "pipe", None))if restore_resident is not None:try:restore_resident()@@ -12983,6 +12987,7 @@ class DiffusionBackend:"transformer_cache": state.transformer_cache,"transformer_cache_stats": static_skip_stats(state.pipe),"multi_gpu": multigpu_placement_status(state.pipe),+ "cfg_batch": cfg_batch_status(state.pipe),"resolved": resolved,"generation_defaults": _generation_defaults_for(state.repo_id,diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.pynew file mode 100644index 00000000000..186acedefed--- /dev/null+++ b/studio/backend/core/inference/diffusion_cfg_batch.py@@ -0,0 +1,747 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CFG batching: both guidance branches of a step through ONE forward of an offloaded denoiser.++A denoiser streamed from host memory (group / sequential offload) uploads its weights once per call, and a guided step+calls it twice (conditional, then unconditional), so it pays the upload twice. ComfyUI batches the two branches into+one call (``comfy/samplers.py`` ``calc_cond_batch``); diffusers pipelines call the transformer once per branch. This+router batches them for the classes whose forward keeps no state between calls:++- the first branch of a step builds the second branch's arguments from the previous step's second call (same latent+ object, the first branch's per-step tensors, the second branch's constants), runs both as one call of twice the+ batch, returns the first half and keeps the second;+- the second call is verified against what was batched (every argument bitwise) and gets the kept half; anything+ else (a changed argument, no second call, a third call) reruns that branch plain and counts as a miss.++Priced by measurement, per request shape: a plain pair is timed on the second step of a shape (the first warms up),+the batched pair (concatenate, forward, split, verify) on the next two, and batching stays on only when the faster+batched pair beats the plain pair by ``_MEASURED_FRACTION``. Memory: the doubled call is attempted only when the card+has room for twice the activations a plain call allocated; an out-of-memory reruns plain and turns the shape off.++Not bit-exact: a doubled batch can pick other GEMM / attention tiles (FLUX.2-klein-base, offloaded: LPIPS ~0.01 vs two+calls). ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact one-branch-per-call path; ``force`` also batches a resident+denoiser and an unlisted class. Declined for per-call activation scaling (NVFP4 without frozen global scales), whose+scale would change with the doubled batch, and for denoisers already wrapped by another forward layer.+"""++from __future__ import annotations++import os+import time+import weakref+from typing import Any, Optional++CFG_BATCH_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH"++# Forward keeps no state between calls, both branches share every per-step input but the text conditioning. Z-Image is+# absent on purpose: its pipeline already batches the branches (a list of latents, one call per step).+STATELESS_CLASSES = frozenset(+ {+ "Flux2Transformer2DModel",+ "FluxTransformer2DModel",+ "QwenImageTransformer2DModel",+ "WanTransformer3DModel",+ }+)++# Arguments that are one entry per sample (a doubled call needs both branches' entries); compared, not concatenated.+_PER_SAMPLE_LISTS = ("img_shapes",)+# (embeddings, mask) pairs a branch pads to its own prompt length: padded to the longer one with masked zeros.+_PADDED_TEXT = (("encoder_hidden_states", "encoder_hidden_states_mask"),)++_MEASURED_FRACTION = 0.92+_BATCHED_SAMPLES = 2+_MISS_LIMIT = 3+_OFF_TRANSIENT = "transient"+++def cfg_batch_mode() -> str:+ """``off`` / ``auto`` (default: an offloaded denoiser of a listed class) / ``force``. Independent of the multi-GPU+ switches: a single card is the usual case."""+ raw = (os.environ.get(CFG_BATCH_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no", "disable", "disabled"):+ return "off"+ if raw == "force":+ return "force"+ return "auto"+++def _torch():+ import torch+ return torch+++def _mg():+ from . import diffusion_multigpu as mg+ return mg+++class _NotBatchable(Exception):+ pass+++def _streamed(module: Any) -> bool:+ """Offload machinery anywhere in the denoiser: its weights live off the card between calls."""+ mg = _mg()+ for sub in module.modules():+ if mg._registry_kind(sub) == "offload" or getattr(sub, "_hf_hook", None) is not None:+ return True+ override = sub.__dict__.get("forward")+ if override is not None and mg._is_offload_object(override):+ return True+ return False+++def _foreign_hook(module: Any) -> Optional[str]:+ """A non-offload diffusers hook on the denoiser or a block (a step cache)."""+ mg = _mg()+ for name, sub in module.named_modules():+ if mg._registry_kind(sub) == "other":+ return f"{name or '<root>'} carries {', '.join(sorted(sub._diffusers_hook.hooks))}"+ return None+++def _foreign_layer(module: Any) -> Optional[str]:+ """A non-offload diffusers hook (a step cache) or a root forward override that is not offload machinery."""+ mg = _mg()+ hooked = _foreign_hook(module)+ if hooked is not None:+ return hooked+ override = module.__dict__.get("forward")+ if override is not None and not mg._is_offload_object(override):+ return f"forward already wrapped by {type(override).__name__}"+ return None+++def per_call_scaled(module: Any) -> Optional[str]:+ """A layer whose activation global scale comes from each call's amax: a doubled batch shares one scale across both+ branches, so it is not the same computation. ComfyUI's dynamic NVFP4 linear and torchao's NVFP4 tensor do this;+ Studio's own NVFP4 linear freezes its activation scale (``a_gsf``), and per-row scaling (Studio's fp8 / int8) is+ per token, so both are unaffected."""+ for name, sub in module.named_modules():+ cls = type(sub).__name__+ upper = cls.upper()+ if "NVFP4" in upper and "DYNAMIC" in upper:+ return f"{name}: {cls}"+ if "NVFP4" in upper and "a_gsf" not in sub._buffers and "input_scale" not in sub._buffers:+ return f"{name}: {cls} (no frozen activation scale)"+ for pname, p in sub.named_parameters(recurse = False):+ if type(p).__name__ == "NVFP4Tensor":+ return f"{name}.{pname}: NVFP4Tensor"+ return None+++def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:+ """Why both guidance branches cannot share one forward on this load, else None."""+ mode = cfg_batch_mode()+ if mode == "off":+ return f"{CFG_BATCH_ENV}=0"+ torch = _torch()+ denoiser = getattr(pipe, "transformer", None)+ if not isinstance(denoiser, torch.nn.Module):+ return "no DiT denoiser"+ if getattr(pipe, "transformer_2", None) is not None:+ return "two-expert denoiser"+ if getattr(denoiser, "_compiled_call_impl", None) is not None:+ return "denoiser is compiled whole"+ if getattr(pipe, "_unsloth_cuda_graphs", None):+ return "the denoiser replays CUDA graphs"+ foreign = _foreign_layer(denoiser)+ if foreign is not None:+ return foreign+ scaled = per_call_scaled(denoiser)+ if scaled is not None and mode != "force":+ return f"per-call activation scaling ({scaled})"+ if mode == "force":+ return None+ if type(denoiser).__name__ not in STATELESS_CLASSES or not type(denoiser).__module__.startswith(+ "diffusers."+ ):+ return f"{type(denoiser).__name__} is not on the stateless-forward list ({CFG_BATCH_ENV}=force opts it in)"+ if offload_policy not in ("group", "streaming", "sequential") or not _streamed(denoiser):+ return "denoiser is resident (two calls upload nothing)"+ return None+++def _usable_free_mib(device: Any) -> Optional[int]:+ """Driver free plus this process's reserved-but-unallocated cache on ``device``."""+ torch = _torch()+ try:+ free, _total = torch.cuda.mem_get_info(device)+ idle = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)+ return int(free + max(0, idle)) >> 20+ except Exception: # noqa: BLE001 - unreadable: the out-of-memory backstop decides+ return None+++class _Shape:+ """Per request shape: warm-up, timing, verdict and the run of misses."""++ def __init__(self) -> None:+ self.steps = 0+ self.plain_pair_s: Optional[float] = None+ self.batched_s: list[float] = []+ self.activation_mib: Optional[int] = None+ self.off: Optional[str] = None+ self.off_kind: Optional[str] = None+ self.misses = 0+++class CfgBatchRouter:+ """Offloaded denoiser ``forward`` running both guidance branches of a step through one forward."""++ def __init__(+ self,+ module: Any,+ *,+ reason: str,+ logger: Any = None,+ ) -> None:+ self.module_ref = weakref.ref(module)+ self.orig_forward = module.forward+ self.had_override = "forward" in module.__dict__+ self.prev_override = module.__dict__.get("forward")+ self.reason = reason+ self.logger = logger+ self.disabled: Optional[str] = None+ self.shapes: dict = {}+ self.stats = {"batched_steps": 0, "hits": 0, "misses": 0, "plain_calls": 0}+ self._reset_step()+ self.template: Optional[tuple] = None+ self.prev_latent: Any = None+ self.prev_calls = 0+ self.last_shape: Optional[_Shape] = None+ self.render_off: Optional[str] = None+ self.render_checked = False++ def _reset_step(self) -> None:+ self.branch = 0+ self.b0: Optional[tuple] = None+ self.kept: Optional[dict] = None+ self.shape: Optional[_Shape] = None+ self.pending_t0: Optional[float] = None+ self.pending_plain: Optional[float] = None++ def describe(self) -> dict[str, Any]:+ shape = self.shape or self.last_shape+ return {+ "armed_for": self.reason,+ "disabled_reason": self.disabled,+ "shape_off": None if shape is None else shape.off,+ "plain_pair_s": None+ if shape is None or shape.plain_pair_s is None+ else round(shape.plain_pair_s, 4),+ "batched_pair_s": None+ if shape is None or not shape.batched_s+ else round(min(shape.batched_s), 4),+ "displaced": self._displaced(),+ **self.stats,+ }++ def end_render(self) -> None:+ """Drop every tensor held between calls (the kept half, the template's constants, the latent)."""+ self.kept = None+ self.template = None+ self.prev_latent = None+ self.prev_calls = 0+ self.render_off = None+ self.render_checked = False+ self._reset_step()+ for shape in self.shapes.values():+ if (+ shape.off_kind == _OFF_TRANSIENT+ ): # memory pressure is per render: retry on the next one+ shape.off = shape.off_kind = None++ def disable(self, reason: str) -> None:+ if self.disabled is None:+ self.disabled = reason+ if self.logger is not None:+ self.logger.warning("diffusion.cfg_batch: off for this load (%s)", reason)+ self.kept = None+ self.template = None++ def _shape_off(+ self,+ shape: _Shape,+ reason: str,+ kind: Optional[str] = None,+ ) -> None:+ if shape.off is None:+ shape.off, shape.off_kind = reason, kind+ if self.logger is not None:+ self.logger.info("diffusion.cfg_batch: off for this request shape (%s)", reason)+ self.kept = None++ def _displaced(self) -> bool:+ module = self.module_ref()+ return module is None or module.__dict__.get("forward") is not self++ def __call__(self, *args: Any, **kwargs: Any) -> Any:+ mg = _mg()+ latent = mg._latent_arg(args, kwargs)+ if self.disabled is not None or latent is None or self._displaced():+ return self._plain(args, kwargs)+ if not self.render_checked:+ # first call of a render: a request-time step cache may have hooked the blocks since install+ module = self.module_ref()+ self.render_off = _foreign_hook(module) if module is not None else None+ self.render_checked = True+ if self.render_off is not None:+ return self._plain(args, kwargs)+ if self.prev_latent is not None and latent is self.prev_latent:+ self.branch += 1+ else:+ self._new_step(args, kwargs, latent)+ shape = self.shape+ if self.branch == 0:+ if self.template is not None and shape.off is None and shape.plain_pair_s is not None:+ out = self._batched(args, kwargs, latent, shape)+ if out is not None:+ return out+ if shape.off is None and shape.plain_pair_s is None and shape.steps >= 2:+ return self._timed_first(args, kwargs, latent, shape)+ return self._plain(args, kwargs)+ if self.branch == 1:+ if self.kept is not None:+ out = self._take(args, kwargs, shape)+ if out is not None:+ return out+ self._record_template(args, kwargs, latent)+ if self.pending_t0 is not None:+ return self._timed_second(args, kwargs, latent, shape)+ return self._plain(args, kwargs)+ self.disable("more than two denoiser calls per step")+ return self._plain(args, kwargs)++ def _new_step(self, args: tuple, kwargs: dict, latent: Any) -> None:+ if self.kept is not None:+ self._miss(self.shape, "the second branch never came")+ if self.prev_latent is not None and self.branch == 0:+ self.template = (+ None # the last step made one call (guidance ended or off): nothing to predict from+ )+ self._reset_step()+ self.prev_latent = latent+ self.b0 = (args, kwargs)+ mg = _mg()+ key = tuple(tuple(t.shape) for t in mg._iter_tensors((args, kwargs)))+ shape = self.shapes.get(key)+ if shape is None:+ shape = self.shapes[key] = _Shape()+ if shape is not self.last_shape:+ self.template = (+ None # a template predicts its own shape's second call: one plain pair first+ )+ shape.steps += 1+ self.shape = self.last_shape = shape++ def _plain(self, args: tuple, kwargs: dict) -> Any:+ self.stats["plain_calls"] += 1+ return self.orig_forward(*args, **kwargs)++ def _sync(self, latent: Any) -> bool:+ if getattr(latent.device, "type", None) != "cuda":+ return False+ _torch().cuda.synchronize(latent.device)+ return True++ def _timed_first(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:+ torch = _torch()+ if not self._sync(latent):+ shape.plain_pair_s = 0.0 # no device timer: batch on the memory gate alone+ return self._plain(args, kwargs)+ # never reset the peak counters (other measurements own them); a call that stayed under an+ # earlier peak leaves its activations unknown+ base = torch.cuda.memory_allocated(latent.device)+ before = torch.cuda.max_memory_allocated(latent.device)+ t0 = time.perf_counter()+ out = self._plain(args, kwargs)+ self._sync(latent)+ self.pending_t0 = time.perf_counter() - t0+ peak = torch.cuda.max_memory_allocated(latent.device)+ shape.activation_mib = (int(peak - base) >> 20) if peak > before else None+ return out++ def _timed_second(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:+ self._sync(latent)+ t0 = time.perf_counter()+ out = self._plain(args, kwargs)+ self._sync(latent)+ shape.plain_pair_s = self.pending_t0 + (time.perf_counter() - t0)+ self.pending_t0 = None+ return out++ def _judge(self, shape: _Shape) -> None:+ if len(shape.batched_s) < _BATCHED_SAMPLES or shape.plain_pair_s in (None, 0.0):+ return+ batched = min(shape.batched_s)+ if batched > _MEASURED_FRACTION * shape.plain_pair_s:+ self._shape_off(+ shape,+ f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",+ )++ def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:+ """How the second call's arguments relate to the first's: the latent, a first-branch tensor (same object, or a+ small per-call scalar recomputed with equal bytes), or a constant of the second branch."""+ mg = _mg()+ torch = _torch()+ b0_leaves = dict(mg._leaves(self.b0)) if self.b0 is not None else {}+ b0_by_id = {id(v): p for p, v in b0_leaves.items() if isinstance(v, torch.Tensor)}+ kinds: dict = {}+ for path, leaf in mg._leaves((args, kwargs)):+ if isinstance(leaf, torch.Tensor):+ if leaf is latent:+ kinds[path] = ("latent",)+ elif id(leaf) in b0_by_id:+ kinds[path] = ("b0", b0_by_id[id(leaf)])+ elif (+ isinstance(b0_leaves.get(path), torch.Tensor)+ and leaf.numel() <= mg._SMALL_TENSOR_NUMEL+ and mg._bitwise_equal(leaf, b0_leaves[path])+ ):+ kinds[path] = ("b0", path)+ else:+ kinds[path] = ("const", leaf)+ else:+ kinds[path] = ("value", leaf)+ self.template = ((args, kwargs), kinds)++ def _predict(self, latent: Any) -> tuple:+ mg = _mg()+ structure, kinds = self.template+ b0_leaves = dict(mg._leaves(self.b0))+ values: dict = {}+ for path, kind in kinds.items():+ if kind[0] == "latent":+ values[path] = latent+ elif kind[0] == "b0":+ if kind[1] not in b0_leaves:+ raise _NotBatchable("the first branch changed structure")+ values[path] = b0_leaves[kind[1]]+ else:+ values[path] = kind[1]+ try:+ return mg._rebuild(structure, values)+ except KeyError:+ raise _NotBatchable("the first branch changed structure") from None++ def _memory_ok(self, latent: Any, shape: _Shape) -> bool:+ if shape.activation_mib is None or getattr(latent.device, "type", None) != "cuda":+ return True+ free = _usable_free_mib(latent.device)+ return free is None or free >= 2 * shape.activation_mib++ def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:+ torch = _torch()+ timing = len(shape.batched_s) < _BATCHED_SAMPLES and shape.plain_pair_s not in (None, 0.0)+ if not self._memory_ok(latent, shape):+ return None+ if timing:+ self._sync(latent)+ t0 = time.perf_counter()+ try:+ second = self._predict(latent)+ b_args, b_kwargs = merge_branches((args, kwargs), second, int(latent.shape[0]))+ snapshots = _snapshot(second)+ except _NotBatchable as exc:+ self._shape_off(shape, f"not batchable: {exc}")+ return None+ except Exception as exc: # noqa: BLE001 - the merge and snapshots allocate too+ if not _is_oom(exc):+ raise+ try:+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ self._shape_off(shape, "out of memory preparing the doubled call", _OFF_TRANSIENT)+ return None+ try:+ out = self.orig_forward(*b_args, **b_kwargs)+ mine, other = split_output(out, int(latent.shape[0]))+ except Exception as exc: # noqa: BLE001 - the branch reruns plain+ del b_args, b_kwargs+ if _is_oom(exc):+ try:+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ self._shape_off(shape, "out of memory on the doubled call", _OFF_TRANSIENT)+ return None+ if isinstance(exc, _NotBatchable):+ self._shape_off(shape, f"output cannot be split per branch ({exc})")+ return None+ self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")+ return None+ self.kept = {+ "out": other,+ "snapshots": snapshots,+ "t0": t0 if timing else None,+ "latent": latent,+ }+ self.stats["batched_steps"] += 1+ return mine++ def _take(self, args: tuple, kwargs: dict, shape: _Shape) -> Any:+ kept, self.kept = self.kept, None+ if not _matches(kept["snapshots"], (args, kwargs)):+ self._miss(shape, "the second branch differed from what was batched")+ return None+ if kept["t0"] is not None:+ self._sync(kept["latent"])+ shape.batched_s.append(time.perf_counter() - kept["t0"])+ self._judge(shape)+ self.stats["hits"] += 1+ shape.misses = 0+ return kept["out"]++ def _miss(self, shape: Optional[_Shape], why: str) -> None:+ self.kept = None+ self.stats["misses"] += 1+ if shape is None:+ return+ shape.misses += 1+ if shape.misses >= _MISS_LIMIT:+ self._shape_off(shape, f"{shape.misses} consecutive misses ({why})")+++def _is_oom(exc: BaseException) -> bool:+ try:+ from .diffusion_batched import is_oom_error+ return bool(is_oom_error(exc))+ except Exception: # noqa: BLE001+ return "out of memory" in str(exc).lower()+++def _snapshot(call: tuple) -> list:+ """The second branch's predicted arguments, tensors cloned: an in-place write between the two calls is caught."""+ mg = _mg()+ torch = _torch()+ return [+ (p, v.detach().clone() if isinstance(v, torch.Tensor) else v) for p, v in mg._leaves(call)+ ]+++def _matches(snapshots: list, call: tuple) -> bool:+ mg = _mg()+ torch = _torch()+ actual = mg._leaves(call)+ if len(actual) != len(snapshots):+ return False+ for (pa, va), (ps, vs) in zip(actual, snapshots):+ if pa != ps:+ return False+ if isinstance(vs, torch.Tensor):+ if not mg._bitwise_equal(va, vs):+ return False+ elif not mg._same_value(va, vs):+ return False+ return True+++def _pad_text(first: dict, second: dict) -> None:+ """Pad the shorter prompt's embeddings (and its mask) with masked zeros so both branches stack (Qwen-Image pads+ each prompt to its own length; the masked keys leave every image token's attention unchanged)."""+ torch = _torch()+ for emb_key, mask_key in _PADDED_TEXT:+ a, b = first.get(emb_key), second.get(emb_key)+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)) or a.dim() < 3:+ continue+ if a.shape[1] == b.shape[1]:+ continue+ if mask_key not in first and mask_key not in second:+ raise _NotBatchable(f"{emb_key}: prompt lengths differ and the model takes no mask")+ length = max(a.shape[1], b.shape[1])+ given = [+ m for m in (first.get(mask_key), second.get(mask_key)) if isinstance(m, torch.Tensor)+ ]+ dtype = given[0].dtype if given else torch.long+ for kw, emb in ((first, a), (second, b)):+ mask = kw.get(mask_key)+ if mask is None:+ mask = torch.ones(emb.shape[:2], dtype = dtype, device = emb.device)+ mask = mask.to(dtype)+ pad = length - emb.shape[1]+ if pad:+ emb = torch.cat((emb, emb.new_zeros((emb.shape[0], pad, *emb.shape[2:]))), dim = 1)+ mask = torch.cat((mask, mask.new_zeros((mask.shape[0], pad))), dim = 1)+ kw[emb_key] = emb+ kw[mask_key] = mask+++def merge_branches(first_call: tuple, second_call: tuple, batch: int) -> tuple[tuple, dict]:+ """One call carrying both branches: per-sample tensors concatenated on dim 0, equal shared values passed once,+ per-sample lists joined, text padded to a common length when the model takes a mask."""+ mg = _mg()+ torch = _torch()+ args_a, kwargs_a = first_call+ args_b, kwargs_b = second_call+ kwargs_a, kwargs_b = dict(kwargs_a), dict(kwargs_b)+ lists = {}+ for key in _PER_SAMPLE_LISTS:+ if key in kwargs_a or key in kwargs_b:+ la, lb = kwargs_a.pop(key, None), kwargs_b.pop(key, None)+ if la is None and lb is None:+ lists[key] = None+ continue+ if not (isinstance(la, list) and isinstance(lb, list)) or la != lb:+ raise _NotBatchable(f"{key} differs per branch")+ lists[key] = list(la) + list(lb) if len(la) == batch else la+ _pad_text(kwargs_a, kwargs_b)+ first, second = mg._leaves((args_a, kwargs_a)), mg._leaves((args_b, kwargs_b))+ if len(first) != len(second):+ raise _NotBatchable("the branches pass different arguments")+ values: dict = {}+ for (path_a, a), (path_b, b) in zip(first, second):+ if path_a != path_b:+ raise _NotBatchable("the branches pass different arguments")+ if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor):+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):+ raise _NotBatchable(f"{path_a}: tensor in one branch only")+ if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:+ raise _NotBatchable(f"{path_a}: {tuple(a.shape)} vs {tuple(b.shape)}")+ if a.dim() >= 1 and a.shape[0] == batch:+ values[path_a] = torch.cat((a, b), dim = 0)+ elif a is b or mg._bitwise_equal(a, b):+ values[path_a] = a+ else:+ raise _NotBatchable(f"{path_a}: differs per branch without a batch dim")+ elif mg._same_value(a, b):+ values[path_a] = a+ else:+ raise _NotBatchable(f"{path_a}: differs per branch")+ b_args, b_kwargs = mg._rebuild((args_a, kwargs_a), values)+ b_kwargs = dict(b_kwargs)+ b_kwargs.update(lists)+ return b_args, b_kwargs+++def split_output(+ out: Any,+ batch: int,+ depth: int = 0,+) -> tuple[Any, Any]:+ """(first, second) branch of a doubled output, split on a leading dim of ``2 * batch``."""+ torch = _torch()+ if isinstance(out, torch.Tensor):+ if out.dim() >= 1 and out.shape[0] == 2 * batch:+ return out[:batch], out[batch:]+ raise _NotBatchable(f"output of shape {tuple(out.shape)} has no doubled batch")+ if out is None or isinstance(out, (bool, int, float, str)):+ return out, out+ if depth > 4:+ raise _NotBatchable("output nests too deep")+ if isinstance(out, tuple) and not hasattr(out, "_fields"):+ pairs = [split_output(v, batch, depth + 1) for v in out]+ return tuple(p[0] for p in pairs), tuple(p[1] for p in pairs)+ if isinstance(out, list):+ pairs = [split_output(v, batch, depth + 1) for v in out]+ return [p[0] for p in pairs], [p[1] for p in pairs]+ if isinstance(out, dict):+ pairs = {k: split_output(v, batch, depth + 1) for k, v in out.items()}+ return type(out)((k, v[0]) for k, v in pairs.items()), type(out)(+ (k, v[1]) for k, v in pairs.items()+ )+ fields = getattr(out, "__dataclass_fields__", None)+ if fields is not None:+ import dataclasses+ pairs = {name: split_output(getattr(out, name), batch, depth + 1) for name in fields}+ return (+ dataclasses.replace(out, **{k: v[0] for k, v in pairs.items()}),+ dataclasses.replace(out, **{k: v[1] for k, v in pairs.items()}),+ )+ raise _NotBatchable(f"output type {type(out).__name__}")+++_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgBatchRouter]" = weakref.WeakKeyDictionary()+++def install_cfg_batch(+ pipe: Any,+ *,+ offload_policy: str,+ logger: Any = None,+) -> Optional[str]:+ """Arm :class:`CfgBatchRouter` on ``pipe.transformer``; the decline reason, or None when armed. An optional lever:+ it never fails a load."""+ try:+ return _install(pipe, offload_policy = offload_policy, logger = logger)+ except Exception as exc: # noqa: BLE001+ if logger is not None:+ logger.info("diffusion.cfg_batch: not armed (%s: %s)", type(exc).__name__, exc)+ return f"{type(exc).__name__}: {exc}"+++def _install(+ pipe: Any,+ *,+ offload_policy: str,+ logger: Any = None,+) -> Optional[str]:+ reason = cfg_batch_decline(pipe, offload_policy = offload_policy)+ if reason is not None:+ if logger is not None:+ logger.info("diffusion.cfg_batch: not armed (%s)", reason)+ return reason+ denoiser = pipe.transformer+ router = CfgBatchRouter(+ denoiser,+ reason = "forced"+ if cfg_batch_mode() == "force"+ else f"offloaded denoiser ({offload_policy})",+ logger = logger,+ )+ try:+ _ROUTERS[pipe] = router+ except TypeError:+ return "pipeline cannot be tracked (not weak-referenceable)"+ denoiser.__dict__["forward"] = router+ if logger is not None:+ logger.info("diffusion.cfg_batch: armed (%s)", router.reason)+ return None+++def router_for(pipe: Any) -> Optional[CfgBatchRouter]:+ if pipe is None:+ return None+ try:+ return _ROUTERS.get(pipe)+ except TypeError:+ return None+++def end_cfg_batch_render(pipe: Any) -> None:+ """Per render: drop what the router holds between calls. Nothing to do when no router is armed."""+ router = router_for(pipe)+ if router is not None:+ try:+ router.end_render()+ except Exception: # noqa: BLE001 - cleanup never masks the render's own outcome+ pass+++def cfg_batch_status(pipe: Any) -> Optional[dict]:+ router = router_for(pipe)+ try:+ return None if router is None else router.describe()+ except Exception: # noqa: BLE001+ return None+++def uninstall_cfg_batch(pipe: Any) -> None:+ router = router_for(pipe)+ if router is None:+ return+ try:+ _ROUTERS.pop(pipe, None)+ except TypeError:+ pass+ router.end_render()+ module = router.module_ref()+ if module is not None and module.__dict__.get("forward") is router:+ if router.had_override:+ module.__dict__["forward"] = router.prev_override+ else:+ del module.__dict__["forward"]diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 93b9bc25b22..d387dc251ef 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -106,6 +106,7 @@ from .diffusion_device import (resolve_selected_cuda_ordinal,resolve_selected_cuda_ordinals,)+from .diffusion_cfg_batch import cfg_batch_status, end_cfg_batch_render, install_cfg_batchfrom .diffusion_multigpu import (AUTOMATIC_ORDINALS,end_cfg_render,@@ -9283,6 +9284,8 @@ class VideoBackend:logger = logger,)install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)+ # both branches through one offloaded forward; declines when CFG-parallel took the slot+ install_cfg_batch(pipe, offload_policy = offload_policy, logger = logger)# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.from . import diffusion_bg_compile as bg_compile@@ -12022,6 +12025,7 @@ class VideoBackend:raisefinally:end_cfg_render(getattr(state, "pipe", None))+ end_cfg_batch_render(getattr(state, "pipe", None))try:from . import video_stream_residencyvideo_stream_residency.discard_request(getattr(state, "pipe", None))@@ -12981,6 +12985,7 @@ class VideoBackend:and state.engine != "sd_cpp",},"multi_gpu": multigpu_placement_status(getattr(state, "pipe", None)),+ "cfg_batch": cfg_batch_status(getattr(state, "pipe", None)),"resolved": resolved,}diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.pynew file mode 100644index 00000000000..3761f2cfd8f--- /dev/null+++ b/studio/backend/tests/test_diffusion_cfg_batch.py@@ -0,0 +1,390 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CFG batching (diffusion_cfg_batch): both guidance branches through one offloaded forward, CPU fakes."""++from __future__ import annotations++import types++import pytest+import torch++from core.inference import diffusion_cfg_batch as cb+++class Flux2Transformer2DModel(torch.nn.Module):+ """Stateless per-sample forward in the FLUX.2-klein call shape (hidden, timestep, text, ids)."""++ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.linspace(0.5, 1.5, 8))+ self.calls = []++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ txt_ids,+ img_ids,+ guidance = None,+ joint_attention_kwargs = None,+ return_dict = False,+ ):+ self.calls.append(int(hidden_states.shape[0]))+ text = encoder_hidden_states.mean(dim = 1, keepdim = True)+ out = (+ hidden_states * self.w+ + text+ + timestep.view(-1, 1, 1)+ + txt_ids.float().mean()+ + img_ids.float().mean()+ )+ return (out,)+++class QwenImageTransformer2DModel(torch.nn.Module):+ """Qwen-Image call shape: a per-prompt-length text with a mask, and a per-sample ``img_shapes`` list (as the+ zero_cond_t path reads it)."""++ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.full((8,), 0.75))++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ encoder_hidden_states_mask = None,+ img_shapes = None,+ guidance = None,+ attention_kwargs = None,+ return_dict = False,+ ):+ assert len(img_shapes) == hidden_states.shape[0]+ if encoder_hidden_states_mask is None:+ encoder_hidden_states_mask = torch.ones(encoder_hidden_states.shape[:2])+ m = encoder_hidden_states_mask.float().unsqueeze(-1)+ text = (encoder_hidden_states * m).sum(1, keepdim = True) / m.sum(1, keepdim = True)+ return (hidden_states * self.w + text + timestep.view(-1, 1, 1),)+++for cls in (Flux2Transformer2DModel, QwenImageTransformer2DModel):+ cls.__module__ = "diffusers.models.fake"+++def _offloaded(module):+ # what the decline reads as offload machinery: a diffusers group-offloading registry on a submodule+ sub = torch.nn.Linear(1, 1)+ module.add_module("offloaded_group", sub)+ sub._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})+ return module+++class _Pipe:+ def __init__(self, transformer):+ self.transformer = transformer+++def _flux_loop(+ model,+ steps = 6,+ *,+ neg = None,+ guided = None,+ between = None,+):+ """The diffusers FLUX.2-klein loop: per step one latent object, cond then uncond call, CFG combine."""+ torch.manual_seed(0)+ latents = torch.randn(1, 4, 8)+ pos = torch.randn(1, 5, 8)+ neg = torch.randn(1, 5, 8) if neg is None else neg+ txt_ids = torch.zeros(1, 5, 4)+ img_ids = torch.ones(1, 4, 4)+ outs = []+ for i in range(steps):+ t = torch.full((1,), 1.0 - i / steps)+ latent_model_input = latents.clone()+ cond = model(+ hidden_states = latent_model_input,+ timestep = t / 1000,+ encoder_hidden_states = pos,+ txt_ids = txt_ids,+ img_ids = img_ids,+ return_dict = False,+ )[0]+ pred = cond+ if between is not None:+ between(i, neg)+ if guided is None or guided(i):+ uncond = model(+ hidden_states = latent_model_input,+ timestep = t / 1000,+ encoder_hidden_states = neg,+ txt_ids = txt_ids,+ img_ids = img_ids,+ return_dict = False,+ )[0]+ pred = uncond + 4.0 * (cond - uncond)+ latents = latents - 0.1 * pred+ outs.append(pred)+ return latents, outs+++def _armed(+ model,+ monkeypatch,+ mode = None,+):+ if mode is not None:+ monkeypatch.setenv(cb.CFG_BATCH_ENV, mode)+ pipe = _Pipe(_offloaded(model))+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None+ return pipe, cb.router_for(pipe)+++def test_batched_steps_match_two_calls_and_engage(monkeypatch):+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ got, _ = _flux_loop(model)+ torch.testing.assert_close(got, ref)+ assert router.stats["batched_steps"] == 4 and router.stats["hits"] == 4+ assert model.calls[-2:] == [2, 2]+ assert cb.cfg_batch_status(pipe)["batched_steps"] == 4+ cb.end_cfg_batch_render(pipe)+ assert cb.cfg_batch_status(pipe)["plain_pair_s"] is not None or not torch.cuda.is_available()+ assert cb.cfg_batch_status(pipe)["shape_off"] is None+++def test_qwen_unequal_prompt_lengths_and_img_shapes_batch(monkeypatch):+ def loop(model):+ torch.manual_seed(1)+ latents = torch.randn(1, 4, 8)+ pos, neg = torch.randn(1, 7, 8), torch.randn(1, 3, 8)+ pos_mask, neg_mask = torch.ones(1, 7, dtype = torch.long), None+ img_shapes = [[(1, 2, 2)]]+ for i in range(6):+ t = torch.full((1,), 1.0 - i / 6)+ c = model(+ hidden_states = latents,+ timestep = t / 1000,+ encoder_hidden_states = pos,+ encoder_hidden_states_mask = pos_mask,+ img_shapes = img_shapes,+ return_dict = False,+ )[0]+ u = model(+ hidden_states = latents,+ timestep = t / 1000,+ encoder_hidden_states = neg,+ encoder_hidden_states_mask = neg_mask,+ img_shapes = img_shapes,+ return_dict = False,+ )[0]+ latents = latents - 0.1 * (u + 4.0 * (c - u))+ return latents++ ref = loop(_offloaded(QwenImageTransformer2DModel()))+ model = QwenImageTransformer2DModel()+ _pipe, router = _armed(model, monkeypatch)+ torch.testing.assert_close(loop(model), ref)+ assert router.stats["batched_steps"] == 4, router.describe()+++def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monkeypatch):+ # uncond prompt rewritten in place: the doubled call must miss, rerun plain, and match the two-call result+ def between(i, neg):+ if i >= 2:+ neg.add_(0.5)++ ref, _ = _flux_loop(+ _offloaded(Flux2Transformer2DModel()), steps = 8, neg = torch.zeros(1, 5, 8), between = between+ )+ model = Flux2Transformer2DModel()+ _pipe, router = _armed(model, monkeypatch)+ got, _ = _flux_loop(model, steps = 8, neg = torch.zeros(1, 5, 8), between = between)+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)+ assert router.stats["misses"] == 3 and router.stats["hits"] == 0+ assert router.shape.off is not None and "3 consecutive misses" in router.shape.off+++def test_guidance_ending_wastes_at_most_one_doubled_call(monkeypatch):+ model = Flux2Transformer2DModel()+ _pipe, router = _armed(model, monkeypatch)+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()), steps = 8, guided = lambda i: i < 5)+ got, _ = _flux_loop(model, steps = 8, guided = lambda i: i < 5)+ torch.testing.assert_close(got, ref)+ assert router.stats["misses"] == 1 # step 6's doubled call found no second branch+ assert model.calls[-2:] == [1, 1]+++def test_a_third_call_per_step_turns_batching_off(monkeypatch):+ model = Flux2Transformer2DModel()+ _pipe, router = _armed(model, monkeypatch)+ x = torch.randn(1, 4, 8)+ kw = dict(+ timestep = torch.ones(1),+ encoder_hidden_states = torch.randn(1, 5, 8),+ txt_ids = torch.zeros(1, 5, 4),+ img_ids = torch.ones(1, 4, 4),+ )+ for _ in range(3):+ model(hidden_states = x, **kw)+ assert router.disabled == "more than two denoiser calls per step"+++def test_a_displaced_router_passes_calls_through(monkeypatch):+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ model.__dict__["forward"] = lambda *a, **k: router(*a, **k) # an outer layer took the slot+ _flux_loop(model)+ assert router.stats["batched_steps"] == 0 and router.describe()["displaced"]+++def test_install_uninstall_restores_the_forward_slot(monkeypatch):+ model = _offloaded(Flux2Transformer2DModel())+ wrapper = model.forward+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None+ assert isinstance(model.__dict__["forward"], cb.CfgBatchRouter)+ cb.uninstall_cfg_batch(pipe)+ assert "forward" not in model.__dict__ and model.forward == wrapper+ cb.end_cfg_batch_render(pipe) # no router: nothing to do+ assert cb.cfg_batch_status(pipe) is None+++def test_switches_are_independent_of_multi_gpu(monkeypatch):+ pipe = _Pipe(_offloaded(Flux2Transformer2DModel()))+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU", "0")+ assert cb.cfg_batch_decline(pipe, offload_policy = "group") is None+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "0")+ assert cb.cfg_batch_decline(pipe, offload_policy = "group") == f"{cb.CFG_BATCH_ENV}=0"+++def test_declines(monkeypatch):+ resident = _Pipe(Flux2Transformer2DModel())+ assert "resident" in cb.cfg_batch_decline(resident, offload_policy = "none")++ class ZImageTransformer2DModel(Flux2Transformer2DModel):+ pass++ ZImageTransformer2DModel.__module__ = "diffusers.models.fake"+ assert "not on the stateless-forward list" in cb.cfg_batch_decline(+ _Pipe(_offloaded(ZImageTransformer2DModel())), offload_policy = "group"+ )++ class ComfyNVFP4DynamicLinear(torch.nn.Linear):+ pass++ dyn = _offloaded(Flux2Transformer2DModel())+ dyn.add_module("proj", ComfyNVFP4DynamicLinear(2, 2))+ assert "per-call activation scaling" in cb.cfg_batch_decline(_Pipe(dyn), offload_policy = "group")++ class NVFP4FlashInferLinear(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer("a_gsf", torch.ones(()))++ frozen = _offloaded(Flux2Transformer2DModel())+ frozen.add_module("proj", NVFP4FlashInferLinear())+ assert cb.cfg_batch_decline(_Pipe(frozen), offload_policy = "group") is None++ two = _Pipe(_offloaded(Flux2Transformer2DModel()))+ two.transformer_2 = torch.nn.Linear(1, 1)+ assert cb.cfg_batch_decline(two, offload_policy = "group") == "two-expert denoiser"+ graphs = _Pipe(_offloaded(Flux2Transformer2DModel()))+ graphs._unsloth_cuda_graphs = (object(),)+ assert "CUDA graphs" in cb.cfg_batch_decline(graphs, offload_policy = "group")+ cached = _offloaded(Flux2Transformer2DModel())+ cached._diffusers_hook = types.SimpleNamespace(hooks = {"first_block_cache": object()})+ assert "first_block_cache" in cb.cfg_batch_decline(_Pipe(cached), offload_policy = "group")+++def test_sync_waits_on_the_latents_card(monkeypatch):+ seen = []+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: seen.append(device))+ router = cb.CfgBatchRouter(Flux2Transformer2DModel(), reason = "test")+ latent = types.SimpleNamespace(device = torch.device("cuda", 3))+ assert router._sync(latent)+ assert seen == [torch.device("cuda", 3)]+++def test_merge_pads_text_and_joins_per_sample_lists():+ a = (+ (),+ {+ "hidden_states": torch.zeros(1, 2),+ "encoder_hidden_states": torch.ones(1, 3, 2),+ "encoder_hidden_states_mask": torch.ones(1, 3),+ "img_shapes": [[(1, 2, 2)]],+ "scale": 1.0,+ },+ )+ b = (+ (),+ {+ "hidden_states": torch.zeros(1, 2),+ "encoder_hidden_states": torch.ones(1, 5, 2),+ "encoder_hidden_states_mask": None,+ "img_shapes": [[(1, 2, 2)]],+ "scale": 1.0,+ },+ )+ _args, kw = cb.merge_branches(a, b, 1)+ assert kw["encoder_hidden_states"].shape == (2, 5, 2)+ assert kw["encoder_hidden_states_mask"].tolist() == [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1]]+ assert kw["img_shapes"] == [[(1, 2, 2)], [(1, 2, 2)]]+ b[1]["scale"] = 2.0+ with pytest.raises(cb._NotBatchable):+ cb.merge_branches(a, b, 1)+++def test_an_oom_preparing_the_doubled_call_reruns_plain_and_clears_next_render(monkeypatch):+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ real = cb.merge_branches++ def oom(*a, **k):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(cb, "merge_branches", oom)+ got, _ = _flux_loop(model)+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0) # the render ran plain, it did not abort+ assert router.stats["batched_steps"] == 0 and "out of memory" in router.shape.off+ cb.end_cfg_batch_render(pipe)+ monkeypatch.setattr(cb, "merge_branches", real)+ got, _ = _flux_loop(model)+ torch.testing.assert_close(got, ref)+ assert router.stats["batched_steps"] > 0 # the transient verdict did not outlive its render+++def test_a_step_cache_hooked_at_request_time_keeps_the_render_plain(monkeypatch):+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))+ model = Flux2Transformer2DModel()+ pipe, router = _armed(model, monkeypatch)+ cache = torch.nn.Linear(1, 1)+ model.add_module("fbcache_block", cache)+ cache._diffusers_hook = types.SimpleNamespace(hooks = {"fbc_leader_block_hook": object()})+ got, _ = _flux_loop(model)+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)+ assert router.stats["batched_steps"] == 0 and "fbc_leader_block_hook" in router.render_off+ del model._modules["fbcache_block"]+ cb.end_cfg_batch_render(pipe)+ _flux_loop(model)+ assert router.stats["batched_steps"] > 0+++def test_a_new_request_shape_starts_without_the_previous_shapes_template(monkeypatch):+ model = Flux2Transformer2DModel()+ _pipe, router = _armed(model, monkeypatch)+ _flux_loop(model)+ first = router.last_shape+ assert router.template is not None+ router.prev_latent = None # a new chunk: a different latent object+ big = torch.randn(2, 4, 8)+ router._new_step((), {"hidden_states": big}, big)+ assert router.last_shape is not first and router.template is None