Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: single-card CFG batching for a DiT streamed from host memory (#13787)

Daniel Hand47cdddc5f7d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 99094681eed..1befc53a69f 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -108,6 +108,7 @@ from .diffusion_device import (
resolve_selected_cuda_ordinal,
resolve_selected_cuda_ordinals,
)
+from .diffusion_cfg_batch import cfg_batch_status, end_cfg_batch_render, install_cfg_batch
from .diffusion_multigpu import (
AUTOMATIC_ORDINALS,
end_cfg_render,
@@ -9605,6 +9606,8 @@ class DiffusionBackend:
)
# After every forward wrapper of the speed / graph layers, so the replica sees the final module.
install_cfg_parallel(pipe, offload_policy = effective_policy, logger = logger)
+ # both branches through one offloaded forward; declines when CFG-parallel took the slot
+ install_cfg_batch(pipe, offload_policy = effective_policy, logger = logger)
te_releaser = te_release.maybe_install(pipe, plan, logger = logger)
state = _LoadState(
pipe = pipe,
@@ -12698,6 +12701,7 @@ class DiffusionBackend:
if restore_vae is not None:
restore_vae()
end_cfg_render(getattr(state, "pipe", None))
+ end_cfg_batch_render(getattr(state, "pipe", None))
if restore_resident is not None:
try:
restore_resident()
@@ -12983,6 +12987,7 @@ class DiffusionBackend:
"transformer_cache": state.transformer_cache,
"transformer_cache_stats": static_skip_stats(state.pipe),
"multi_gpu": multigpu_placement_status(state.pipe),
+ "cfg_batch": cfg_batch_status(state.pipe),
"resolved": resolved,
"generation_defaults": _generation_defaults_for(
state.repo_id,
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
new file mode 100644
index 00000000000..186acedefed
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -0,0 +1,747 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CFG batching: both guidance branches of a step through ONE forward of an offloaded denoiser.
+
+A denoiser streamed from host memory (group / sequential offload) uploads its weights once per call, and a guided step
+calls it twice (conditional, then unconditional), so it pays the upload twice. ComfyUI batches the two branches into
+one call (``comfy/samplers.py`` ``calc_cond_batch``); diffusers pipelines call the transformer once per branch. This
+router batches them for the classes whose forward keeps no state between calls:
+
+- the first branch of a step builds the second branch's arguments from the previous step's second call (same latent
+ object, the first branch's per-step tensors, the second branch's constants), runs both as one call of twice the
+ batch, returns the first half and keeps the second;
+- the second call is verified against what was batched (every argument bitwise) and gets the kept half; anything
+ else (a changed argument, no second call, a third call) reruns that branch plain and counts as a miss.
+
+Priced by measurement, per request shape: a plain pair is timed on the second step of a shape (the first warms up),
+the batched pair (concatenate, forward, split, verify) on the next two, and batching stays on only when the faster
+batched pair beats the plain pair by ``_MEASURED_FRACTION``. Memory: the doubled call is attempted only when the card
+has room for twice the activations a plain call allocated; an out-of-memory reruns plain and turns the shape off.
+
+Not bit-exact: a doubled batch can pick other GEMM / attention tiles (FLUX.2-klein-base, offloaded: LPIPS ~0.01 vs two
+calls). ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact one-branch-per-call path; ``force`` also batches a resident
+denoiser and an unlisted class. Declined for per-call activation scaling (NVFP4 without frozen global scales), whose
+scale would change with the doubled batch, and for denoisers already wrapped by another forward layer.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+import weakref
+from typing import Any, Optional
+
+CFG_BATCH_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH"
+
+# Forward keeps no state between calls, both branches share every per-step input but the text conditioning. Z-Image is
+# absent on purpose: its pipeline already batches the branches (a list of latents, one call per step).
+STATELESS_CLASSES = frozenset(
+ {
+ "Flux2Transformer2DModel",
+ "FluxTransformer2DModel",
+ "QwenImageTransformer2DModel",
+ "WanTransformer3DModel",
+ }
+)
+
+# Arguments that are one entry per sample (a doubled call needs both branches' entries); compared, not concatenated.
+_PER_SAMPLE_LISTS = ("img_shapes",)
+# (embeddings, mask) pairs a branch pads to its own prompt length: padded to the longer one with masked zeros.
+_PADDED_TEXT = (("encoder_hidden_states", "encoder_hidden_states_mask"),)
+
+_MEASURED_FRACTION = 0.92
+_BATCHED_SAMPLES = 2
+_MISS_LIMIT = 3
+_OFF_TRANSIENT = "transient"
+
+
+def cfg_batch_mode() -> str:
+ """``off`` / ``auto`` (default: an offloaded denoiser of a listed class) / ``force``. Independent of the multi-GPU
+ switches: a single card is the usual case."""
+ raw = (os.environ.get(CFG_BATCH_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "disable", "disabled"):
+ return "off"
+ if raw == "force":
+ return "force"
+ return "auto"
+
+
+def _torch():
+ import torch
+ return torch
+
+
+def _mg():
+ from . import diffusion_multigpu as mg
+ return mg
+
+
+class _NotBatchable(Exception):
+ pass
+
+
+def _streamed(module: Any) -> bool:
+ """Offload machinery anywhere in the denoiser: its weights live off the card between calls."""
+ mg = _mg()
+ for sub in module.modules():
+ if mg._registry_kind(sub) == "offload" or getattr(sub, "_hf_hook", None) is not None:
+ return True
+ override = sub.__dict__.get("forward")
+ if override is not None and mg._is_offload_object(override):
+ return True
+ return False
+
+
+def _foreign_hook(module: Any) -> Optional[str]:
+ """A non-offload diffusers hook on the denoiser or a block (a step cache)."""
+ mg = _mg()
+ for name, sub in module.named_modules():
+ if mg._registry_kind(sub) == "other":
+ return f"{name or '<root>'} carries {', '.join(sorted(sub._diffusers_hook.hooks))}"
+ return None
+
+
+def _foreign_layer(module: Any) -> Optional[str]:
+ """A non-offload diffusers hook (a step cache) or a root forward override that is not offload machinery."""
+ mg = _mg()
+ hooked = _foreign_hook(module)
+ if hooked is not None:
+ return hooked
+ override = module.__dict__.get("forward")
+ if override is not None and not mg._is_offload_object(override):
+ return f"forward already wrapped by {type(override).__name__}"
+ return None
+
+
+def per_call_scaled(module: Any) -> Optional[str]:
+ """A layer whose activation global scale comes from each call's amax: a doubled batch shares one scale across both
+ branches, so it is not the same computation. ComfyUI's dynamic NVFP4 linear and torchao's NVFP4 tensor do this;
+ Studio's own NVFP4 linear freezes its activation scale (``a_gsf``), and per-row scaling (Studio's fp8 / int8) is
+ per token, so both are unaffected."""
+ for name, sub in module.named_modules():
+ cls = type(sub).__name__
+ upper = cls.upper()
+ if "NVFP4" in upper and "DYNAMIC" in upper:
+ return f"{name}: {cls}"
+ if "NVFP4" in upper and "a_gsf" not in sub._buffers and "input_scale" not in sub._buffers:
+ return f"{name}: {cls} (no frozen activation scale)"
+ for pname, p in sub.named_parameters(recurse = False):
+ if type(p).__name__ == "NVFP4Tensor":
+ return f"{name}.{pname}: NVFP4Tensor"
+ return None
+
+
+def cfg_batch_decline(pipe: Any, *, offload_policy: str) -> Optional[str]:
+ """Why both guidance branches cannot share one forward on this load, else None."""
+ mode = cfg_batch_mode()
+ if mode == "off":
+ return f"{CFG_BATCH_ENV}=0"
+ torch = _torch()
+ denoiser = getattr(pipe, "transformer", None)
+ if not isinstance(denoiser, torch.nn.Module):
+ return "no DiT denoiser"
+ if getattr(pipe, "transformer_2", None) is not None:
+ return "two-expert denoiser"
+ if getattr(denoiser, "_compiled_call_impl", None) is not None:
+ return "denoiser is compiled whole"
+ if getattr(pipe, "_unsloth_cuda_graphs", None):
+ return "the denoiser replays CUDA graphs"
+ foreign = _foreign_layer(denoiser)
+ if foreign is not None:
+ return foreign
+ scaled = per_call_scaled(denoiser)
+ if scaled is not None and mode != "force":
+ return f"per-call activation scaling ({scaled})"
+ if mode == "force":
+ return None
+ if type(denoiser).__name__ not in STATELESS_CLASSES or not type(denoiser).__module__.startswith(
+ "diffusers."
+ ):
+ return f"{type(denoiser).__name__} is not on the stateless-forward list ({CFG_BATCH_ENV}=force opts it in)"
+ if offload_policy not in ("group", "streaming", "sequential") or not _streamed(denoiser):
+ return "denoiser is resident (two calls upload nothing)"
+ return None
+
+
+def _usable_free_mib(device: Any) -> Optional[int]:
+ """Driver free plus this process's reserved-but-unallocated cache on ``device``."""
+ torch = _torch()
+ try:
+ free, _total = torch.cuda.mem_get_info(device)
+ idle = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
+ return int(free + max(0, idle)) >> 20
+ except Exception: # noqa: BLE001 - unreadable: the out-of-memory backstop decides
+ return None
+
+
+class _Shape:
+ """Per request shape: warm-up, timing, verdict and the run of misses."""
+
+ def __init__(self) -> None:
+ self.steps = 0
+ self.plain_pair_s: Optional[float] = None
+ self.batched_s: list[float] = []
+ self.activation_mib: Optional[int] = None
+ self.off: Optional[str] = None
+ self.off_kind: Optional[str] = None
+ self.misses = 0
+
+
+class CfgBatchRouter:
+ """Offloaded denoiser ``forward`` running both guidance branches of a step through one forward."""
+
+ def __init__(
+ self,
+ module: Any,
+ *,
+ reason: str,
+ logger: Any = None,
+ ) -> None:
+ self.module_ref = weakref.ref(module)
+ self.orig_forward = module.forward
+ self.had_override = "forward" in module.__dict__
+ self.prev_override = module.__dict__.get("forward")
+ self.reason = reason
+ self.logger = logger
+ self.disabled: Optional[str] = None
+ self.shapes: dict = {}
+ self.stats = {"batched_steps": 0, "hits": 0, "misses": 0, "plain_calls": 0}
+ self._reset_step()
+ self.template: Optional[tuple] = None
+ self.prev_latent: Any = None
+ self.prev_calls = 0
+ self.last_shape: Optional[_Shape] = None
+ self.render_off: Optional[str] = None
+ self.render_checked = False
+
+ def _reset_step(self) -> None:
+ self.branch = 0
+ self.b0: Optional[tuple] = None
+ self.kept: Optional[dict] = None
+ self.shape: Optional[_Shape] = None
+ self.pending_t0: Optional[float] = None
+ self.pending_plain: Optional[float] = None
+
+ def describe(self) -> dict[str, Any]:
+ shape = self.shape or self.last_shape
+ return {
+ "armed_for": self.reason,
+ "disabled_reason": self.disabled,
+ "shape_off": None if shape is None else shape.off,
+ "plain_pair_s": None
+ if shape is None or shape.plain_pair_s is None
+ else round(shape.plain_pair_s, 4),
+ "batched_pair_s": None
+ if shape is None or not shape.batched_s
+ else round(min(shape.batched_s), 4),
+ "displaced": self._displaced(),
+ **self.stats,
+ }
+
+ def end_render(self) -> None:
+ """Drop every tensor held between calls (the kept half, the template's constants, the latent)."""
+ self.kept = None
+ self.template = None
+ self.prev_latent = None
+ self.prev_calls = 0
+ self.render_off = None
+ self.render_checked = False
+ self._reset_step()
+ for shape in self.shapes.values():
+ if (
+ shape.off_kind == _OFF_TRANSIENT
+ ): # memory pressure is per render: retry on the next one
+ shape.off = shape.off_kind = None
+
+ def disable(self, reason: str) -> None:
+ if self.disabled is None:
+ self.disabled = reason
+ if self.logger is not None:
+ self.logger.warning("diffusion.cfg_batch: off for this load (%s)", reason)
+ self.kept = None
+ self.template = None
+
+ def _shape_off(
+ self,
+ shape: _Shape,
+ reason: str,
+ kind: Optional[str] = None,
+ ) -> None:
+ if shape.off is None:
+ shape.off, shape.off_kind = reason, kind
+ if self.logger is not None:
+ self.logger.info("diffusion.cfg_batch: off for this request shape (%s)", reason)
+ self.kept = None
+
+ def _displaced(self) -> bool:
+ module = self.module_ref()
+ return module is None or module.__dict__.get("forward") is not self
+
+ def __call__(self, *args: Any, **kwargs: Any) -> Any:
+ mg = _mg()
+ latent = mg._latent_arg(args, kwargs)
+ if self.disabled is not None or latent is None or self._displaced():
+ return self._plain(args, kwargs)
+ if not self.render_checked:
+ # first call of a render: a request-time step cache may have hooked the blocks since install
+ module = self.module_ref()
+ self.render_off = _foreign_hook(module) if module is not None else None
+ self.render_checked = True
+ if self.render_off is not None:
+ return self._plain(args, kwargs)
+ if self.prev_latent is not None and latent is self.prev_latent:
+ self.branch += 1
+ else:
+ self._new_step(args, kwargs, latent)
+ shape = self.shape
+ if self.branch == 0:
+ if self.template is not None and shape.off is None and shape.plain_pair_s is not None:
+ out = self._batched(args, kwargs, latent, shape)
+ if out is not None:
+ return out
+ if shape.off is None and shape.plain_pair_s is None and shape.steps >= 2:
+ return self._timed_first(args, kwargs, latent, shape)
+ return self._plain(args, kwargs)
+ if self.branch == 1:
+ if self.kept is not None:
+ out = self._take(args, kwargs, shape)
+ if out is not None:
+ return out
+ self._record_template(args, kwargs, latent)
+ if self.pending_t0 is not None:
+ return self._timed_second(args, kwargs, latent, shape)
+ return self._plain(args, kwargs)
+ self.disable("more than two denoiser calls per step")
+ return self._plain(args, kwargs)
+
+ def _new_step(self, args: tuple, kwargs: dict, latent: Any) -> None:
+ if self.kept is not None:
+ self._miss(self.shape, "the second branch never came")
+ if self.prev_latent is not None and self.branch == 0:
+ self.template = (
+ None # the last step made one call (guidance ended or off): nothing to predict from
+ )
+ self._reset_step()
+ self.prev_latent = latent
+ self.b0 = (args, kwargs)
+ mg = _mg()
+ key = tuple(tuple(t.shape) for t in mg._iter_tensors((args, kwargs)))
+ shape = self.shapes.get(key)
+ if shape is None:
+ shape = self.shapes[key] = _Shape()
+ if shape is not self.last_shape:
+ self.template = (
+ None # a template predicts its own shape's second call: one plain pair first
+ )
+ shape.steps += 1
+ self.shape = self.last_shape = shape
+
+ def _plain(self, args: tuple, kwargs: dict) -> Any:
+ self.stats["plain_calls"] += 1
+ return self.orig_forward(*args, **kwargs)
+
+ def _sync(self, latent: Any) -> bool:
+ if getattr(latent.device, "type", None) != "cuda":
+ return False
+ _torch().cuda.synchronize(latent.device)
+ return True
+
+ def _timed_first(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
+ torch = _torch()
+ if not self._sync(latent):
+ shape.plain_pair_s = 0.0 # no device timer: batch on the memory gate alone
+ return self._plain(args, kwargs)
+ # never reset the peak counters (other measurements own them); a call that stayed under an
+ # earlier peak leaves its activations unknown
+ base = torch.cuda.memory_allocated(latent.device)
+ before = torch.cuda.max_memory_allocated(latent.device)
+ t0 = time.perf_counter()
+ out = self._plain(args, kwargs)
+ self._sync(latent)
+ self.pending_t0 = time.perf_counter() - t0
+ peak = torch.cuda.max_memory_allocated(latent.device)
+ shape.activation_mib = (int(peak - base) >> 20) if peak > before else None
+ return out
+
+ def _timed_second(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
+ self._sync(latent)
+ t0 = time.perf_counter()
+ out = self._plain(args, kwargs)
+ self._sync(latent)
+ shape.plain_pair_s = self.pending_t0 + (time.perf_counter() - t0)
+ self.pending_t0 = None
+ return out
+
+ def _judge(self, shape: _Shape) -> None:
+ if len(shape.batched_s) < _BATCHED_SAMPLES or shape.plain_pair_s in (None, 0.0):
+ return
+ batched = min(shape.batched_s)
+ if batched > _MEASURED_FRACTION * shape.plain_pair_s:
+ self._shape_off(
+ shape,
+ f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",
+ )
+
+ def _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:
+ """How the second call's arguments relate to the first's: the latent, a first-branch tensor (same object, or a
+ small per-call scalar recomputed with equal bytes), or a constant of the second branch."""
+ mg = _mg()
+ torch = _torch()
+ b0_leaves = dict(mg._leaves(self.b0)) if self.b0 is not None else {}
+ b0_by_id = {id(v): p for p, v in b0_leaves.items() if isinstance(v, torch.Tensor)}
+ kinds: dict = {}
+ for path, leaf in mg._leaves((args, kwargs)):
+ if isinstance(leaf, torch.Tensor):
+ if leaf is latent:
+ kinds[path] = ("latent",)
+ elif id(leaf) in b0_by_id:
+ kinds[path] = ("b0", b0_by_id[id(leaf)])
+ elif (
+ isinstance(b0_leaves.get(path), torch.Tensor)
+ and leaf.numel() <= mg._SMALL_TENSOR_NUMEL
+ and mg._bitwise_equal(leaf, b0_leaves[path])
+ ):
+ kinds[path] = ("b0", path)
+ else:
+ kinds[path] = ("const", leaf)
+ else:
+ kinds[path] = ("value", leaf)
+ self.template = ((args, kwargs), kinds)
+
+ def _predict(self, latent: Any) -> tuple:
+ mg = _mg()
+ structure, kinds = self.template
+ b0_leaves = dict(mg._leaves(self.b0))
+ values: dict = {}
+ for path, kind in kinds.items():
+ if kind[0] == "latent":
+ values[path] = latent
+ elif kind[0] == "b0":
+ if kind[1] not in b0_leaves:
+ raise _NotBatchable("the first branch changed structure")
+ values[path] = b0_leaves[kind[1]]
+ else:
+ values[path] = kind[1]
+ try:
+ return mg._rebuild(structure, values)
+ except KeyError:
+ raise _NotBatchable("the first branch changed structure") from None
+
+ def _memory_ok(self, latent: Any, shape: _Shape) -> bool:
+ if shape.activation_mib is None or getattr(latent.device, "type", None) != "cuda":
+ return True
+ free = _usable_free_mib(latent.device)
+ return free is None or free >= 2 * shape.activation_mib
+
+ def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
+ torch = _torch()
+ timing = len(shape.batched_s) < _BATCHED_SAMPLES and shape.plain_pair_s not in (None, 0.0)
+ if not self._memory_ok(latent, shape):
+ return None
+ if timing:
+ self._sync(latent)
+ t0 = time.perf_counter()
+ try:
+ second = self._predict(latent)
+ b_args, b_kwargs = merge_branches((args, kwargs), second, int(latent.shape[0]))
+ snapshots = _snapshot(second)
+ except _NotBatchable as exc:
+ self._shape_off(shape, f"not batchable: {exc}")
+ return None
+ except Exception as exc: # noqa: BLE001 - the merge and snapshots allocate too
+ if not _is_oom(exc):
+ raise
+ try:
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ self._shape_off(shape, "out of memory preparing the doubled call", _OFF_TRANSIENT)
+ return None
+ try:
+ out = self.orig_forward(*b_args, **b_kwargs)
+ mine, other = split_output(out, int(latent.shape[0]))
+ except Exception as exc: # noqa: BLE001 - the branch reruns plain
+ del b_args, b_kwargs
+ if _is_oom(exc):
+ try:
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ self._shape_off(shape, "out of memory on the doubled call", _OFF_TRANSIENT)
+ return None
+ if isinstance(exc, _NotBatchable):
+ self._shape_off(shape, f"output cannot be split per branch ({exc})")
+ return None
+ self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")
+ return None
+ self.kept = {
+ "out": other,
+ "snapshots": snapshots,
+ "t0": t0 if timing else None,
+ "latent": latent,
+ }
+ self.stats["batched_steps"] += 1
+ return mine
+
+ def _take(self, args: tuple, kwargs: dict, shape: _Shape) -> Any:
+ kept, self.kept = self.kept, None
+ if not _matches(kept["snapshots"], (args, kwargs)):
+ self._miss(shape, "the second branch differed from what was batched")
+ return None
+ if kept["t0"] is not None:
+ self._sync(kept["latent"])
+ shape.batched_s.append(time.perf_counter() - kept["t0"])
+ self._judge(shape)
+ self.stats["hits"] += 1
+ shape.misses = 0
+ return kept["out"]
+
+ def _miss(self, shape: Optional[_Shape], why: str) -> None:
+ self.kept = None
+ self.stats["misses"] += 1
+ if shape is None:
+ return
+ shape.misses += 1
+ if shape.misses >= _MISS_LIMIT:
+ self._shape_off(shape, f"{shape.misses} consecutive misses ({why})")
+
+
+def _is_oom(exc: BaseException) -> bool:
+ try:
+ from .diffusion_batched import is_oom_error
+ return bool(is_oom_error(exc))
+ except Exception: # noqa: BLE001
+ return "out of memory" in str(exc).lower()
+
+
+def _snapshot(call: tuple) -> list:
+ """The second branch's predicted arguments, tensors cloned: an in-place write between the two calls is caught."""
+ mg = _mg()
+ torch = _torch()
+ return [
+ (p, v.detach().clone() if isinstance(v, torch.Tensor) else v) for p, v in mg._leaves(call)
+ ]
+
+
+def _matches(snapshots: list, call: tuple) -> bool:
+ mg = _mg()
+ torch = _torch()
+ actual = mg._leaves(call)
+ if len(actual) != len(snapshots):
+ return False
+ for (pa, va), (ps, vs) in zip(actual, snapshots):
+ if pa != ps:
+ return False
+ if isinstance(vs, torch.Tensor):
+ if not mg._bitwise_equal(va, vs):
+ return False
+ elif not mg._same_value(va, vs):
+ return False
+ return True
+
+
+def _pad_text(first: dict, second: dict) -> None:
+ """Pad the shorter prompt's embeddings (and its mask) with masked zeros so both branches stack (Qwen-Image pads
+ each prompt to its own length; the masked keys leave every image token's attention unchanged)."""
+ torch = _torch()
+ for emb_key, mask_key in _PADDED_TEXT:
+ a, b = first.get(emb_key), second.get(emb_key)
+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)) or a.dim() < 3:
+ continue
+ if a.shape[1] == b.shape[1]:
+ continue
+ if mask_key not in first and mask_key not in second:
+ raise _NotBatchable(f"{emb_key}: prompt lengths differ and the model takes no mask")
+ length = max(a.shape[1], b.shape[1])
+ given = [
+ m for m in (first.get(mask_key), second.get(mask_key)) if isinstance(m, torch.Tensor)
+ ]
+ dtype = given[0].dtype if given else torch.long
+ for kw, emb in ((first, a), (second, b)):
+ mask = kw.get(mask_key)
+ if mask is None:
+ mask = torch.ones(emb.shape[:2], dtype = dtype, device = emb.device)
+ mask = mask.to(dtype)
+ pad = length - emb.shape[1]
+ if pad:
+ emb = torch.cat((emb, emb.new_zeros((emb.shape[0], pad, *emb.shape[2:]))), dim = 1)
+ mask = torch.cat((mask, mask.new_zeros((mask.shape[0], pad))), dim = 1)
+ kw[emb_key] = emb
+ kw[mask_key] = mask
+
+
+def merge_branches(first_call: tuple, second_call: tuple, batch: int) -> tuple[tuple, dict]:
+ """One call carrying both branches: per-sample tensors concatenated on dim 0, equal shared values passed once,
+ per-sample lists joined, text padded to a common length when the model takes a mask."""
+ mg = _mg()
+ torch = _torch()
+ args_a, kwargs_a = first_call
+ args_b, kwargs_b = second_call
+ kwargs_a, kwargs_b = dict(kwargs_a), dict(kwargs_b)
+ lists = {}
+ for key in _PER_SAMPLE_LISTS:
+ if key in kwargs_a or key in kwargs_b:
+ la, lb = kwargs_a.pop(key, None), kwargs_b.pop(key, None)
+ if la is None and lb is None:
+ lists[key] = None
+ continue
+ if not (isinstance(la, list) and isinstance(lb, list)) or la != lb:
+ raise _NotBatchable(f"{key} differs per branch")
+ lists[key] = list(la) + list(lb) if len(la) == batch else la
+ _pad_text(kwargs_a, kwargs_b)
+ first, second = mg._leaves((args_a, kwargs_a)), mg._leaves((args_b, kwargs_b))
+ if len(first) != len(second):
+ raise _NotBatchable("the branches pass different arguments")
+ values: dict = {}
+ for (path_a, a), (path_b, b) in zip(first, second):
+ if path_a != path_b:
+ raise _NotBatchable("the branches pass different arguments")
+ if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor):
+ if not (isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor)):
+ raise _NotBatchable(f"{path_a}: tensor in one branch only")
+ if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device:
+ raise _NotBatchable(f"{path_a}: {tuple(a.shape)} vs {tuple(b.shape)}")
+ if a.dim() >= 1 and a.shape[0] == batch:
+ values[path_a] = torch.cat((a, b), dim = 0)
+ elif a is b or mg._bitwise_equal(a, b):
+ values[path_a] = a
+ else:
+ raise _NotBatchable(f"{path_a}: differs per branch without a batch dim")
+ elif mg._same_value(a, b):
+ values[path_a] = a
+ else:
+ raise _NotBatchable(f"{path_a}: differs per branch")
+ b_args, b_kwargs = mg._rebuild((args_a, kwargs_a), values)
+ b_kwargs = dict(b_kwargs)
+ b_kwargs.update(lists)
+ return b_args, b_kwargs
+
+
+def split_output(
+ out: Any,
+ batch: int,
+ depth: int = 0,
+) -> tuple[Any, Any]:
+ """(first, second) branch of a doubled output, split on a leading dim of ``2 * batch``."""
+ torch = _torch()
+ if isinstance(out, torch.Tensor):
+ if out.dim() >= 1 and out.shape[0] == 2 * batch:
+ return out[:batch], out[batch:]
+ raise _NotBatchable(f"output of shape {tuple(out.shape)} has no doubled batch")
+ if out is None or isinstance(out, (bool, int, float, str)):
+ return out, out
+ if depth > 4:
+ raise _NotBatchable("output nests too deep")
+ if isinstance(out, tuple) and not hasattr(out, "_fields"):
+ pairs = [split_output(v, batch, depth + 1) for v in out]
+ return tuple(p[0] for p in pairs), tuple(p[1] for p in pairs)
+ if isinstance(out, list):
+ pairs = [split_output(v, batch, depth + 1) for v in out]
+ return [p[0] for p in pairs], [p[1] for p in pairs]
+ if isinstance(out, dict):
+ pairs = {k: split_output(v, batch, depth + 1) for k, v in out.items()}
+ return type(out)((k, v[0]) for k, v in pairs.items()), type(out)(
+ (k, v[1]) for k, v in pairs.items()
+ )
+ fields = getattr(out, "__dataclass_fields__", None)
+ if fields is not None:
+ import dataclasses
+ pairs = {name: split_output(getattr(out, name), batch, depth + 1) for name in fields}
+ return (
+ dataclasses.replace(out, **{k: v[0] for k, v in pairs.items()}),
+ dataclasses.replace(out, **{k: v[1] for k, v in pairs.items()}),
+ )
+ raise _NotBatchable(f"output type {type(out).__name__}")
+
+
+_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgBatchRouter]" = weakref.WeakKeyDictionary()
+
+
+def install_cfg_batch(
+ pipe: Any,
+ *,
+ offload_policy: str,
+ logger: Any = None,
+) -> Optional[str]:
+ """Arm :class:`CfgBatchRouter` on ``pipe.transformer``; the decline reason, or None when armed. An optional lever:
+ it never fails a load."""
+ try:
+ return _install(pipe, offload_policy = offload_policy, logger = logger)
+ except Exception as exc: # noqa: BLE001
+ if logger is not None:
+ logger.info("diffusion.cfg_batch: not armed (%s: %s)", type(exc).__name__, exc)
+ return f"{type(exc).__name__}: {exc}"
+
+
+def _install(
+ pipe: Any,
+ *,
+ offload_policy: str,
+ logger: Any = None,
+) -> Optional[str]:
+ reason = cfg_batch_decline(pipe, offload_policy = offload_policy)
+ if reason is not None:
+ if logger is not None:
+ logger.info("diffusion.cfg_batch: not armed (%s)", reason)
+ return reason
+ denoiser = pipe.transformer
+ router = CfgBatchRouter(
+ denoiser,
+ reason = "forced"
+ if cfg_batch_mode() == "force"
+ else f"offloaded denoiser ({offload_policy})",
+ logger = logger,
+ )
+ try:
+ _ROUTERS[pipe] = router
+ except TypeError:
+ return "pipeline cannot be tracked (not weak-referenceable)"
+ denoiser.__dict__["forward"] = router
+ if logger is not None:
+ logger.info("diffusion.cfg_batch: armed (%s)", router.reason)
+ return None
+
+
+def router_for(pipe: Any) -> Optional[CfgBatchRouter]:
+ if pipe is None:
+ return None
+ try:
+ return _ROUTERS.get(pipe)
+ except TypeError:
+ return None
+
+
+def end_cfg_batch_render(pipe: Any) -> None:
+ """Per render: drop what the router holds between calls. Nothing to do when no router is armed."""
+ router = router_for(pipe)
+ if router is not None:
+ try:
+ router.end_render()
+ except Exception: # noqa: BLE001 - cleanup never masks the render's own outcome
+ pass
+
+
+def cfg_batch_status(pipe: Any) -> Optional[dict]:
+ router = router_for(pipe)
+ try:
+ return None if router is None else router.describe()
+ except Exception: # noqa: BLE001
+ return None
+
+
+def uninstall_cfg_batch(pipe: Any) -> None:
+ router = router_for(pipe)
+ if router is None:
+ return
+ try:
+ _ROUTERS.pop(pipe, None)
+ except TypeError:
+ pass
+ router.end_render()
+ module = router.module_ref()
+ if module is not None and module.__dict__.get("forward") is router:
+ if router.had_override:
+ module.__dict__["forward"] = router.prev_override
+ else:
+ del module.__dict__["forward"]
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 93b9bc25b22..d387dc251ef 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -106,6 +106,7 @@ from .diffusion_device import (
resolve_selected_cuda_ordinal,
resolve_selected_cuda_ordinals,
)
+from .diffusion_cfg_batch import cfg_batch_status, end_cfg_batch_render, install_cfg_batch
from .diffusion_multigpu import (
AUTOMATIC_ORDINALS,
end_cfg_render,
@@ -9283,6 +9284,8 @@ class VideoBackend:
logger = logger,
)
install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)
+ # both branches through one offloaded forward; declines when CFG-parallel took the slot
+ install_cfg_batch(pipe, offload_policy = offload_policy, logger = logger)
# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.
from . import diffusion_bg_compile as bg_compile
@@ -12022,6 +12025,7 @@ class VideoBackend:
raise
finally:
end_cfg_render(getattr(state, "pipe", None))
+ end_cfg_batch_render(getattr(state, "pipe", None))
try:
from . import video_stream_residency
video_stream_residency.discard_request(getattr(state, "pipe", None))
@@ -12981,6 +12985,7 @@ class VideoBackend:
and state.engine != "sd_cpp",
},
"multi_gpu": multigpu_placement_status(getattr(state, "pipe", None)),
+ "cfg_batch": cfg_batch_status(getattr(state, "pipe", None)),
"resolved": resolved,
}
diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.py
new file mode 100644
index 00000000000..3761f2cfd8f
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_cfg_batch.py
@@ -0,0 +1,390 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CFG batching (diffusion_cfg_batch): both guidance branches through one offloaded forward, CPU fakes."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+
+from core.inference import diffusion_cfg_batch as cb
+
+
+class Flux2Transformer2DModel(torch.nn.Module):
+ """Stateless per-sample forward in the FLUX.2-klein call shape (hidden, timestep, text, ids)."""
+
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.linspace(0.5, 1.5, 8))
+ self.calls = []
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ txt_ids,
+ img_ids,
+ guidance = None,
+ joint_attention_kwargs = None,
+ return_dict = False,
+ ):
+ self.calls.append(int(hidden_states.shape[0]))
+ text = encoder_hidden_states.mean(dim = 1, keepdim = True)
+ out = (
+ hidden_states * self.w
+ + text
+ + timestep.view(-1, 1, 1)
+ + txt_ids.float().mean()
+ + img_ids.float().mean()
+ )
+ return (out,)
+
+
+class QwenImageTransformer2DModel(torch.nn.Module):
+ """Qwen-Image call shape: a per-prompt-length text with a mask, and a per-sample ``img_shapes`` list (as the
+ zero_cond_t path reads it)."""
+
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.full((8,), 0.75))
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ encoder_hidden_states_mask = None,
+ img_shapes = None,
+ guidance = None,
+ attention_kwargs = None,
+ return_dict = False,
+ ):
+ assert len(img_shapes) == hidden_states.shape[0]
+ if encoder_hidden_states_mask is None:
+ encoder_hidden_states_mask = torch.ones(encoder_hidden_states.shape[:2])
+ m = encoder_hidden_states_mask.float().unsqueeze(-1)
+ text = (encoder_hidden_states * m).sum(1, keepdim = True) / m.sum(1, keepdim = True)
+ return (hidden_states * self.w + text + timestep.view(-1, 1, 1),)
+
+
+for cls in (Flux2Transformer2DModel, QwenImageTransformer2DModel):
+ cls.__module__ = "diffusers.models.fake"
+
+
+def _offloaded(module):
+ # what the decline reads as offload machinery: a diffusers group-offloading registry on a submodule
+ sub = torch.nn.Linear(1, 1)
+ module.add_module("offloaded_group", sub)
+ sub._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})
+ return module
+
+
+class _Pipe:
+ def __init__(self, transformer):
+ self.transformer = transformer
+
+
+def _flux_loop(
+ model,
+ steps = 6,
+ *,
+ neg = None,
+ guided = None,
+ between = None,
+):
+ """The diffusers FLUX.2-klein loop: per step one latent object, cond then uncond call, CFG combine."""
+ torch.manual_seed(0)
+ latents = torch.randn(1, 4, 8)
+ pos = torch.randn(1, 5, 8)
+ neg = torch.randn(1, 5, 8) if neg is None else neg
+ txt_ids = torch.zeros(1, 5, 4)
+ img_ids = torch.ones(1, 4, 4)
+ outs = []
+ for i in range(steps):
+ t = torch.full((1,), 1.0 - i / steps)
+ latent_model_input = latents.clone()
+ cond = model(
+ hidden_states = latent_model_input,
+ timestep = t / 1000,
+ encoder_hidden_states = pos,
+ txt_ids = txt_ids,
+ img_ids = img_ids,
+ return_dict = False,
+ )[0]
+ pred = cond
+ if between is not None:
+ between(i, neg)
+ if guided is None or guided(i):
+ uncond = model(
+ hidden_states = latent_model_input,
+ timestep = t / 1000,
+ encoder_hidden_states = neg,
+ txt_ids = txt_ids,
+ img_ids = img_ids,
+ return_dict = False,
+ )[0]
+ pred = uncond + 4.0 * (cond - uncond)
+ latents = latents - 0.1 * pred
+ outs.append(pred)
+ return latents, outs
+
+
+def _armed(
+ model,
+ monkeypatch,
+ mode = None,
+):
+ if mode is not None:
+ monkeypatch.setenv(cb.CFG_BATCH_ENV, mode)
+ pipe = _Pipe(_offloaded(model))
+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None
+ return pipe, cb.router_for(pipe)
+
+
+def test_batched_steps_match_two_calls_and_engage(monkeypatch):
+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ got, _ = _flux_loop(model)
+ torch.testing.assert_close(got, ref)
+ assert router.stats["batched_steps"] == 4 and router.stats["hits"] == 4
+ assert model.calls[-2:] == [2, 2]
+ assert cb.cfg_batch_status(pipe)["batched_steps"] == 4
+ cb.end_cfg_batch_render(pipe)
+ assert cb.cfg_batch_status(pipe)["plain_pair_s"] is not None or not torch.cuda.is_available()
+ assert cb.cfg_batch_status(pipe)["shape_off"] is None
+
+
+def test_qwen_unequal_prompt_lengths_and_img_shapes_batch(monkeypatch):
+ def loop(model):
+ torch.manual_seed(1)
+ latents = torch.randn(1, 4, 8)
+ pos, neg = torch.randn(1, 7, 8), torch.randn(1, 3, 8)
+ pos_mask, neg_mask = torch.ones(1, 7, dtype = torch.long), None
+ img_shapes = [[(1, 2, 2)]]
+ for i in range(6):
+ t = torch.full((1,), 1.0 - i / 6)
+ c = model(
+ hidden_states = latents,
+ timestep = t / 1000,
+ encoder_hidden_states = pos,
+ encoder_hidden_states_mask = pos_mask,
+ img_shapes = img_shapes,
+ return_dict = False,
+ )[0]
+ u = model(
+ hidden_states = latents,
+ timestep = t / 1000,
+ encoder_hidden_states = neg,
+ encoder_hidden_states_mask = neg_mask,
+ img_shapes = img_shapes,
+ return_dict = False,
+ )[0]
+ latents = latents - 0.1 * (u + 4.0 * (c - u))
+ return latents
+
+ ref = loop(_offloaded(QwenImageTransformer2DModel()))
+ model = QwenImageTransformer2DModel()
+ _pipe, router = _armed(model, monkeypatch)
+ torch.testing.assert_close(loop(model), ref)
+ assert router.stats["batched_steps"] == 4, router.describe()
+
+
+def test_a_changed_second_branch_reruns_plain_and_stops_after_three_misses(monkeypatch):
+ # uncond prompt rewritten in place: the doubled call must miss, rerun plain, and match the two-call result
+ def between(i, neg):
+ if i >= 2:
+ neg.add_(0.5)
+
+ ref, _ = _flux_loop(
+ _offloaded(Flux2Transformer2DModel()), steps = 8, neg = torch.zeros(1, 5, 8), between = between
+ )
+ model = Flux2Transformer2DModel()
+ _pipe, router = _armed(model, monkeypatch)
+ got, _ = _flux_loop(model, steps = 8, neg = torch.zeros(1, 5, 8), between = between)
+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)
+ assert router.stats["misses"] == 3 and router.stats["hits"] == 0
+ assert router.shape.off is not None and "3 consecutive misses" in router.shape.off
+
+
+def test_guidance_ending_wastes_at_most_one_doubled_call(monkeypatch):
+ model = Flux2Transformer2DModel()
+ _pipe, router = _armed(model, monkeypatch)
+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()), steps = 8, guided = lambda i: i < 5)
+ got, _ = _flux_loop(model, steps = 8, guided = lambda i: i < 5)
+ torch.testing.assert_close(got, ref)
+ assert router.stats["misses"] == 1 # step 6's doubled call found no second branch
+ assert model.calls[-2:] == [1, 1]
+
+
+def test_a_third_call_per_step_turns_batching_off(monkeypatch):
+ model = Flux2Transformer2DModel()
+ _pipe, router = _armed(model, monkeypatch)
+ x = torch.randn(1, 4, 8)
+ kw = dict(
+ timestep = torch.ones(1),
+ encoder_hidden_states = torch.randn(1, 5, 8),
+ txt_ids = torch.zeros(1, 5, 4),
+ img_ids = torch.ones(1, 4, 4),
+ )
+ for _ in range(3):
+ model(hidden_states = x, **kw)
+ assert router.disabled == "more than two denoiser calls per step"
+
+
+def test_a_displaced_router_passes_calls_through(monkeypatch):
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ model.__dict__["forward"] = lambda *a, **k: router(*a, **k) # an outer layer took the slot
+ _flux_loop(model)
+ assert router.stats["batched_steps"] == 0 and router.describe()["displaced"]
+
+
+def test_install_uninstall_restores_the_forward_slot(monkeypatch):
+ model = _offloaded(Flux2Transformer2DModel())
+ wrapper = model.forward
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "group") is None
+ assert isinstance(model.__dict__["forward"], cb.CfgBatchRouter)
+ cb.uninstall_cfg_batch(pipe)
+ assert "forward" not in model.__dict__ and model.forward == wrapper
+ cb.end_cfg_batch_render(pipe) # no router: nothing to do
+ assert cb.cfg_batch_status(pipe) is None
+
+
+def test_switches_are_independent_of_multi_gpu(monkeypatch):
+ pipe = _Pipe(_offloaded(Flux2Transformer2DModel()))
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU", "0")
+ assert cb.cfg_batch_decline(pipe, offload_policy = "group") is None
+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "0")
+ assert cb.cfg_batch_decline(pipe, offload_policy = "group") == f"{cb.CFG_BATCH_ENV}=0"
+
+
+def test_declines(monkeypatch):
+ resident = _Pipe(Flux2Transformer2DModel())
+ assert "resident" in cb.cfg_batch_decline(resident, offload_policy = "none")
+
+ class ZImageTransformer2DModel(Flux2Transformer2DModel):
+ pass
+
+ ZImageTransformer2DModel.__module__ = "diffusers.models.fake"
+ assert "not on the stateless-forward list" in cb.cfg_batch_decline(
+ _Pipe(_offloaded(ZImageTransformer2DModel())), offload_policy = "group"
+ )
+
+ class ComfyNVFP4DynamicLinear(torch.nn.Linear):
+ pass
+
+ dyn = _offloaded(Flux2Transformer2DModel())
+ dyn.add_module("proj", ComfyNVFP4DynamicLinear(2, 2))
+ assert "per-call activation scaling" in cb.cfg_batch_decline(_Pipe(dyn), offload_policy = "group")
+
+ class NVFP4FlashInferLinear(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.register_buffer("a_gsf", torch.ones(()))
+
+ frozen = _offloaded(Flux2Transformer2DModel())
+ frozen.add_module("proj", NVFP4FlashInferLinear())
+ assert cb.cfg_batch_decline(_Pipe(frozen), offload_policy = "group") is None
+
+ two = _Pipe(_offloaded(Flux2Transformer2DModel()))
+ two.transformer_2 = torch.nn.Linear(1, 1)
+ assert cb.cfg_batch_decline(two, offload_policy = "group") == "two-expert denoiser"
+ graphs = _Pipe(_offloaded(Flux2Transformer2DModel()))
+ graphs._unsloth_cuda_graphs = (object(),)
+ assert "CUDA graphs" in cb.cfg_batch_decline(graphs, offload_policy = "group")
+ cached = _offloaded(Flux2Transformer2DModel())
+ cached._diffusers_hook = types.SimpleNamespace(hooks = {"first_block_cache": object()})
+ assert "first_block_cache" in cb.cfg_batch_decline(_Pipe(cached), offload_policy = "group")
+
+
+def test_sync_waits_on_the_latents_card(monkeypatch):
+ seen = []
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: seen.append(device))
+ router = cb.CfgBatchRouter(Flux2Transformer2DModel(), reason = "test")
+ latent = types.SimpleNamespace(device = torch.device("cuda", 3))
+ assert router._sync(latent)
+ assert seen == [torch.device("cuda", 3)]
+
+
+def test_merge_pads_text_and_joins_per_sample_lists():
+ a = (
+ (),
+ {
+ "hidden_states": torch.zeros(1, 2),
+ "encoder_hidden_states": torch.ones(1, 3, 2),
+ "encoder_hidden_states_mask": torch.ones(1, 3),
+ "img_shapes": [[(1, 2, 2)]],
+ "scale": 1.0,
+ },
+ )
+ b = (
+ (),
+ {
+ "hidden_states": torch.zeros(1, 2),
+ "encoder_hidden_states": torch.ones(1, 5, 2),
+ "encoder_hidden_states_mask": None,
+ "img_shapes": [[(1, 2, 2)]],
+ "scale": 1.0,
+ },
+ )
+ _args, kw = cb.merge_branches(a, b, 1)
+ assert kw["encoder_hidden_states"].shape == (2, 5, 2)
+ assert kw["encoder_hidden_states_mask"].tolist() == [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1]]
+ assert kw["img_shapes"] == [[(1, 2, 2)], [(1, 2, 2)]]
+ b[1]["scale"] = 2.0
+ with pytest.raises(cb._NotBatchable):
+ cb.merge_branches(a, b, 1)
+
+
+def test_an_oom_preparing_the_doubled_call_reruns_plain_and_clears_next_render(monkeypatch):
+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ real = cb.merge_branches
+
+ def oom(*a, **k):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(cb, "merge_branches", oom)
+ got, _ = _flux_loop(model)
+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0) # the render ran plain, it did not abort
+ assert router.stats["batched_steps"] == 0 and "out of memory" in router.shape.off
+ cb.end_cfg_batch_render(pipe)
+ monkeypatch.setattr(cb, "merge_branches", real)
+ got, _ = _flux_loop(model)
+ torch.testing.assert_close(got, ref)
+ assert router.stats["batched_steps"] > 0 # the transient verdict did not outlive its render
+
+
+def test_a_step_cache_hooked_at_request_time_keeps_the_render_plain(monkeypatch):
+ ref, _ = _flux_loop(_offloaded(Flux2Transformer2DModel()))
+ model = Flux2Transformer2DModel()
+ pipe, router = _armed(model, monkeypatch)
+ cache = torch.nn.Linear(1, 1)
+ model.add_module("fbcache_block", cache)
+ cache._diffusers_hook = types.SimpleNamespace(hooks = {"fbc_leader_block_hook": object()})
+ got, _ = _flux_loop(model)
+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)
+ assert router.stats["batched_steps"] == 0 and "fbc_leader_block_hook" in router.render_off
+ del model._modules["fbcache_block"]
+ cb.end_cfg_batch_render(pipe)
+ _flux_loop(model)
+ assert router.stats["batched_steps"] > 0
+
+
+def test_a_new_request_shape_starts_without_the_previous_shapes_template(monkeypatch):
+ model = Flux2Transformer2DModel()
+ _pipe, router = _armed(model, monkeypatch)
+ _flux_loop(model)
+ first = router.last_shape
+ assert router.template is not None
+ router.prev_latent = None # a new chunk: a different latent object
+ big = torch.randn(2, 4, 8)
+ router._new_step((), {"hidden_states": big}, big)
+ assert router.last_shape is not first and router.template is None