Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: whole-step CUDA graphs under offload on top of per-block graphs (L4 FLUX.1 16 GB 10% faster, HunyuanVideo-1.5 1 GiB lighter) (#12707)

Daniel Han8de2cde9b7f9
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.json
index b3c5b387808..b5f77d707e3 100644
--- a/scripts/exec_literals_baseline.json
+++ b/scripts/exec_literals_baseline.json
@@ -12,9 +12,9 @@
{
"file": "studio/backend/core/inference/diffusion_capture_safe.py",
"sink": "compile",
- "digest": "05c1d218ea0c8e6d",
+ "digest": "1f39880672dc563b",
"count": 1,
- "reason": "Rebuilds the installed diffusers HunyuanImage forward from its own inspect.getsource with one fixed block swapped; the source is the imported library's code, not user or network input."
+ "reason": "Rebuilds the installed diffusers HunyuanImage / HunyuanVideo 1.5 forward from its own inspect.getsource with fixed blocks swapped (compiled with dont_inherit so this module's future import does not leak in); the source is the imported library's code, not user or network input."
},
{
"file": "studio/backend/core/inference/diffusion_capture_safe.py",
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index b8db11b6af8..3fd22f13b3d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -180,6 +180,7 @@ from .diffusion_speed import (
fp16_unet_offloaded,
fresh_compile_count,
int8_gemm_live,
+ arm_graphs_after_placement,
normalize_speed_mode,
resolve_speed_mode,
restore_backend_flags,
@@ -7336,6 +7337,8 @@ class DiffusionBackend:
if not speed_deferred:
if _denoiser_hooked(pipe):
compile_pipe_below_offload_hooks(pipe, logger)
+ # the whole-step graph, with the offload copies recorded in it, where placement allows one
+ arm_graphs_after_placement(pipe, speed_applied, logger)
cuda_graph.arm_block_graphs(
pipe,
speed_applied,
@@ -9303,6 +9306,7 @@ class DiffusionBackend:
engage_pinned_denoisers(state.pipe, speed_applied, logger)
if _denoiser_hooked(state.pipe):
compile_pipe_below_offload_hooks(state.pipe, logger)
+ arm_graphs_after_placement(state.pipe, speed_applied, logger)
cuda_graph.arm_block_graphs(
state.pipe,
speed_applied,
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 0eb59028928..7fdeb507f33 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -31,6 +31,7 @@ Best-effort: an unavailable backend falls back to the diffusers default. torch/d
from __future__ import annotations
+import functools
import os
import re
import threading
@@ -1952,6 +1953,15 @@ def _set_hunyuan_null_mask(module: Any, enabled: bool) -> None:
setattr(attn, _NULL_ATTN_FLAG, enabled)
+def _hunyuan_null_mask_state(module: Any) -> bool:
+ """The null-mask flag this call's pre-hook set (read by the graph layer as part of its key)."""
+ for blk in getattr(module, "transformer_blocks", []):
+ attn = getattr(blk, "attn", None)
+ if attn is not None:
+ return bool(getattr(attn, _NULL_ATTN_FLAG, False))
+ return False
+
+
def _null_mask_processor_cls():
"""Build (once, lazily) a HunyuanVideo15AttnProcessor2_0 subclass whose ``__call__`` runs
attn_mask=None when the DiT is flagged (padding already removed by the pre-hook); otherwise it
@@ -2074,6 +2084,61 @@ def _trim_stream(states, mask):
return states, mask, all_valid
+_TRIM_MEMO_ATTR = "_unsloth_trim_memo"
+
+
+def _trim_plan(module: Any, kwargs: dict) -> dict:
+ """The trim decisions for this call's image / mask tensors: the host reads (``_trim_stream``'s) made once per
+ set of inputs. A pipeline hands the SAME prompt tensors to every step, so a step after the first reuses the plan
+ and makes no host wait (a CUDA-graph replay of the step then runs without one). Keyed on the tensors themselves,
+ held here, and their version counters, so new or edited inputs plan afresh."""
+ import torch
+
+ names = ("image_embeds", "encoder_attention_mask", "encoder_attention_mask_2")
+ srcs = tuple(kwargs.get(n) for n in names)
+ # An inference tensor (renders run under torch.inference_mode) has no version counter and reading one raises;
+ # it is keyed on identity alone. It cannot be written outside inference mode, and the pipeline hands every step
+ # the encoder's own outputs, which nothing edits in place.
+ versions = tuple(
+ ("inference" if t.is_inference() else t._version) if torch.is_tensor(t) else None
+ for t in srcs
+ )
+ memo = module.__dict__.setdefault(_TRIM_MEMO_ATTR, [])
+ for held, held_versions, plan in memo:
+ if held_versions == versions and all(a is b for a, b in zip(held, srcs)):
+ return plan
+ image = srcs[0]
+ plan = {
+ "t2v": bool(image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()))
+ }
+ for name, mask in zip(names[1:], srcs[1:]):
+ if mask is None or not torch.is_tensor(mask) or mask.dim() != 2:
+ plan[name] = (None, True)
+ continue
+ mb = mask.bool()
+ keep = mb.any(dim = 0) # column valid for at least one batch element
+ index = None
+ if not bool(keep.all()):
+ index = keep.nonzero().squeeze(1)
+ mb = mb.index_select(1, index)
+ # vacuously True for a 0-length stream, fine for an unused secondary stream (byt5 in t2v)
+ plan[name] = (index, bool(mb.all().item()))
+ memo.insert(0, (srcs, versions, plan))
+ del memo[4:] # the CFG branches of one render
+ return plan
+
+
+def _apply_trim(states: Any, mask: Any, decided: tuple) -> tuple:
+ """``_trim_stream`` with its decisions already made: the same columns, gathered on the device."""
+ if states is None or mask is None or mask.dim() != 2:
+ return states, mask, True
+ index, all_valid = decided
+ if index is not None:
+ states = states.index_select(1, index)
+ mask = mask.index_select(1, index)
+ return states, mask, all_valid
+
+
def _hunyuan_trim_pre_hook(module, args, kwargs):
"""Eager forward pre-hook: strip padded text tokens so the joint attention runs fused.
@@ -2103,8 +2168,9 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):
try:
null_ok = True
+ plan = _trim_plan(module, kwargs)
image = kwargs.get("image_embeds")
- if image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()):
+ if plan["t2v"]:
# All-zero image == "no image" (t2v). Emptying the token axis removes the 729 padded image tokens; is_t2v
# stays True in forward (all() of empty is vacuously True).
kwargs["image_embeds"] = image[:, :0]
@@ -2119,7 +2185,7 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):
if skey not in kwargs:
null_ok = null_ok and not required
continue
- states, mask, all_valid = _trim_stream(kwargs.get(skey), kwargs.get(mkey))
+ states, mask, all_valid = _apply_trim(kwargs.get(skey), kwargs.get(mkey), plan[mkey])
kwargs[skey] = states
kwargs[mkey] = mask
null_ok = null_ok and all_valid
@@ -2210,6 +2276,8 @@ def install_hunyuan_attention_trim(
continue
# installation and every idle period start in the conservative state
_set_hunyuan_null_mask(dit, False)
+ # The flag picks the attention branch inside the forward, so a CUDA graph keys on it (diffusion_cuda_graph).
+ dit._unsloth_graph_key_extra = functools.partial(_hunyuan_null_mask_state, dit)
if getattr(dit, "_unsloth_trim_hook", None) is None:
pre_handle = None
try:
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index 6e8777aa606..371961f3e23 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -20,6 +20,15 @@ from typing import Any, Callable, Optional
from .diffusion_bg_compile import capture_suppressed as _bg_capture_suppressed
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
+
+def _step_recording() -> bool:
+ try:
+ from .diffusion_cuda_graph import step_recording
+ except Exception: # noqa: BLE001
+ return False
+ return step_recording()
+
+
BLOCK_GRAPHS_ENV = "UNSLOTH_DIFFUSION_BLOCK_GRAPHS"
# Recordings kept per block (input layouts x weight placements); the least recently replayed goes first.
@@ -438,7 +447,8 @@ class BlockGraph:
return self.compute(*args, **kwargs)
def __call__(self, *args: Any, **kwargs: Any) -> Any:
- if _bg_capture_suppressed():
+ if _bg_capture_suppressed() or _step_recording():
+ # a whole-step graph above records this block's kernels (diffusion_cuda_graph), not a nested replay
return self.compute(*args, **kwargs)
if not self.enabled or self.bypassed or self.poisoned or self.churned or _bg_eager_forced():
return self._eager(args, kwargs)
diff --git a/studio/backend/core/inference/diffusion_capture_safe.py b/studio/backend/core/inference/diffusion_capture_safe.py
index 56e8f373cdc..8e9f2507efa 100644
--- a/studio/backend/core/inference/diffusion_capture_safe.py
+++ b/studio/backend/core/inference/diffusion_capture_safe.py
@@ -39,6 +39,12 @@ The rewrite takes the INSTALLED forward's own source and swaps just that block,
unpatchify) stays whatever the installed diffusers ships. The block is matched line for line; if it
changed, nothing is rewritten and the graph layer declines the load with the reason.
+HunyuanVideo-1.5 has the same merge over three streams ([image ; byt5 ; mllm], padded byt5/mllm rows
+zeroed) plus ``is_t2v = torch.all(image_embeds == 0)`` read by ``if`` on the host. Both blocks are
+swapped: the branch becomes ``torch.where`` on the device bool (keeping ``* 0.0`` exactly), the merge
+the same stable-sort permutation with ``torch.where`` zeroing. Identical in diffusers 0.36.0 through
+0.40.0.
+
Kill switch: ``UNSLOTH_DIFFUSION_CAPTURE_SAFE=0`` (the graph is then declined for these classes).
"""
@@ -129,6 +135,148 @@ def merge_text_streams(
return states, mask
+_HUNYUANVIDEO15_CLS = "HunyuanVideo15Transformer3DModel"
+
+# HunyuanVideo-1.5 (diffusers 0.36.0 through 0.40.0, unchanged): ``is_t2v`` is a 0-d device bool that
+# ``if`` reads on the host, then the per-batch merge of [image ; byt5 ; mllm] uses boolean-mask
+# indexing like HunyuanImage's, with the padded byt5/mllm rows replaced by zeros.
+_HV15_T2V_BLOCK: tuple[str, ...] = (
+ "is_t2v = torch.all(image_embeds == 0)",
+ "if is_t2v:",
+ "encoder_hidden_states_3 = encoder_hidden_states_3 * 0.0",
+ "encoder_attention_mask_3 = torch.zeros(",
+ "(batch_size, encoder_hidden_states_3.shape[1]),",
+ "dtype=encoder_attention_mask.dtype,",
+ "device=encoder_attention_mask.device,",
+ ")",
+ "else:",
+ "encoder_attention_mask_3 = torch.ones(",
+ "(batch_size, encoder_hidden_states_3.shape[1]),",
+ "dtype=encoder_attention_mask.dtype,",
+ "device=encoder_attention_mask.device,",
+ ")",
+)
+
+_HV15_MERGE_BLOCK: tuple[str, ...] = (
+ "new_encoder_hidden_states = []",
+ "new_encoder_attention_mask = []",
+ "for text, text_mask, text_2, text_mask_2, image, image_mask in zip(",
+ "encoder_hidden_states,",
+ "encoder_attention_mask,",
+ "encoder_hidden_states_2,",
+ "encoder_attention_mask_2,",
+ "encoder_hidden_states_3,",
+ "encoder_attention_mask_3,",
+ "):",
+ "# Concatenate: [valid_image, valid_byt5, valid_mllm, invalid_image, invalid_byt5, invalid_mllm]",
+ "new_encoder_hidden_states.append(",
+ "torch.cat(",
+ "[",
+ "image[image_mask], # valid image",
+ "text_2[text_mask_2], # valid byt5",
+ "text[text_mask], # valid mllm",
+ "image[~image_mask], # invalid image",
+ "torch.zeros_like(text_2[~text_mask_2]), # invalid byt5 (zeroed)",
+ "torch.zeros_like(text[~text_mask]), # invalid mllm (zeroed)",
+ "],",
+ "dim=0,",
+ ")",
+ ")",
+ "# Apply same reordering to attention masks",
+ "new_encoder_attention_mask.append(",
+ "torch.cat(",
+ "[",
+ "image_mask[image_mask],",
+ "text_mask_2[text_mask_2],",
+ "text_mask[text_mask],",
+ "image_mask[~image_mask],",
+ "text_mask_2[~text_mask_2],",
+ "text_mask[~text_mask],",
+ "],",
+ "dim=0,",
+ ")",
+ ")",
+ "encoder_hidden_states = torch.stack(new_encoder_hidden_states)",
+ "encoder_attention_mask = torch.stack(new_encoder_attention_mask)",
+)
+
+_HV15_T2V_HELPER_NAME = "_unsloth_hv15_image_stream"
+_HV15_T2V_CALL = (
+ f"encoder_hidden_states_3, encoder_attention_mask_3 = {_HV15_T2V_HELPER_NAME}("
+ "image_embeds, encoder_hidden_states_3, encoder_attention_mask, batch_size)"
+)
+_HV15_MERGE_HELPER_NAME = "_unsloth_hv15_merge_streams"
+_HV15_MERGE_CALL = (
+ f"encoder_hidden_states, encoder_attention_mask = {_HV15_MERGE_HELPER_NAME}("
+ "encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2, "
+ "encoder_hidden_states_3, encoder_attention_mask_3)"
+)
+
+
+def hv15_image_stream(
+ image_embeds: Any, encoder_hidden_states_3: Any, encoder_attention_mask: Any, batch_size: int
+) -> tuple[Any, Any]:
+ """HunyuanVideo-1.5's ``if is_t2v`` branch as arithmetic; bit-identical to both stock arms.
+
+ ``torch.where`` picks each element from ``states * 0.0`` (the stock t2v arm, NaN/-0.0 semantics
+ included) or ``states`` (the i2v arm) by the 0-d device bool, so the host never reads it. The
+ mask is all zeros (t2v) or all ones (i2v) in the primary mask's dtype and device, as stock.
+ """
+ import torch
+
+ is_t2v = torch.all(image_embeds == 0)
+ states = torch.where(is_t2v, encoder_hidden_states_3 * 0.0, encoder_hidden_states_3)
+ mask = (
+ torch.logical_not(is_t2v)
+ .to(dtype = encoder_attention_mask.dtype, device = encoder_attention_mask.device)
+ .expand(batch_size, encoder_hidden_states_3.shape[1])
+ .contiguous()
+ )
+ return states, mask
+
+
+def hv15_merge_streams(
+ encoder_hidden_states: Any,
+ encoder_attention_mask: Any,
+ encoder_hidden_states_2: Any,
+ encoder_attention_mask_2: Any,
+ encoder_hidden_states_3: Any,
+ encoder_attention_mask_3: Any,
+) -> tuple[Any, Any]:
+ """HunyuanVideo-1.5's three-stream merge without a host sync; bit-identical to the stock loop.
+
+ Streams: mllm ``[B, S1, D]``, byt5 ``[B, S2, D]``, image ``[B, S3, D]``, each with a bool mask.
+ Stock order per batch element is [valid image, valid byt5, valid mllm, padded image, padded
+ byt5 zeroed, padded mllm zeroed]: a stable sort of ``[image ; byt5 ; mllm]`` on "is padding".
+ Padded byt5/mllm rows become exact zeros (``torch.where`` with 0, as ``zeros_like``), padded
+ image rows keep their values. ``torch.cat`` promotes dtypes as the stock per-row cat does.
+ """
+ import torch
+
+ states = torch.cat(
+ [encoder_hidden_states_3, encoder_hidden_states_2, encoder_hidden_states], dim = 1
+ )
+ mask = torch.cat(
+ [encoder_attention_mask_3, encoder_attention_mask_2, encoder_attention_mask], dim = 1
+ )
+ padded = ~mask
+ zero_rows = torch.cat(
+ [
+ torch.zeros_like(encoder_attention_mask_3),
+ ~encoder_attention_mask_2,
+ ~encoder_attention_mask,
+ ],
+ dim = 1,
+ )
+ states = torch.where(
+ zero_rows.unsqueeze(-1), torch.zeros((), dtype = states.dtype, device = states.device), states
+ )
+ order = torch.argsort(padded.to(torch.uint8), dim = 1, stable = True)
+ states = torch.gather(states, 1, order.unsqueeze(-1).expand(-1, -1, states.shape[-1]))
+ mask = torch.gather(mask, 1, order)
+ return states, mask
+
+
def _find_block(lines: list[str], block: tuple[str, ...]) -> Optional[tuple[int, int]]:
"""``(start, end)`` line indices (end exclusive) of ``block`` in ``lines``, blank lines ignored.
@@ -158,10 +306,15 @@ def _signature_key(fn: Callable) -> Optional[tuple]:
return tuple((p.name, p.kind, repr(p.default)) for p in params)
-def _rewrite_hunyuanimage(forward: Callable) -> Callable:
- """Rebuild ``forward`` from its own source with the merge block swapped for the helper call.
+def _rebuild_forward(
+ forward: Callable,
+ swaps: tuple[tuple[tuple[str, ...], str, str], ...],
+ helpers: dict[str, Callable],
+) -> Callable:
+ """Rebuild ``forward`` from its own source with each ``(block, replacement, what)`` swapped.
- Raises ``RuntimeError`` naming what did not match, so the reason reaches the status payload."""
+ Every block must occur exactly once. Raises ``RuntimeError`` naming what did not match, so the
+ reason reaches the status payload."""
target = inspect.unwrap(forward)
module = sys.modules.get(getattr(target, "__module__", "") or "")
if module is None:
@@ -171,20 +324,30 @@ def _rewrite_hunyuanimage(forward: Callable) -> Callable:
except (OSError, TypeError) as exc:
raise RuntimeError(f"its source is unreadable ({exc})") from None
lines = source.splitlines()
- span = _find_block(lines, _HUNYUANIMAGE_MERGE_BLOCK)
- if span is None:
- raise RuntimeError("the boolean-mask text merge block changed in this diffusers version")
- start, end = span
- indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]
- new_lines = lines[:start] + [indent + _MERGE_CALL] + lines[end:]
+ spans = []
+ for block, replacement, what in swaps:
+ span = _find_block(lines, block)
+ if span is None:
+ raise RuntimeError(f"the {what} block changed in this diffusers version")
+ spans.append((span, replacement))
+ spans.sort(key = lambda item: item[0][0])
+ for (prev, _), (nxt, _) in zip(spans, spans[1:]):
+ if nxt[0] < prev[1]:
+ raise RuntimeError("the rewritten blocks overlap")
+ # Splice from the bottom so earlier spans keep their line indices.
+ for (start, end), replacement in reversed(spans):
+ indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]
+ lines = lines[:start] + [indent + replacement] + lines[end:]
# A source-level decorator (``@apply_lora_scale`` from 0.37) re-applies when the def executes,
# so the rebuilt function carries the installed version's LoRA handling unchanged.
- new_source = "\n".join(new_lines) + "\n"
+ new_source = "\n".join(lines) + "\n"
filename = f"<unsloth capture-safe {getattr(target, '__qualname__', 'forward')}>"
namespace = dict(vars(module))
- namespace[_MERGE_HELPER_NAME] = merge_text_streams
- code = compile(new_source, filename, "exec")
- exec(code, namespace) # noqa: S102 - the installed diffusers' own source with one block replaced
+ namespace.update(helpers)
+ # dont_inherit: this module's ``from __future__ import annotations`` must not stringify the
+ # rebuilt forward's annotations.
+ code = compile(new_source, filename, "exec", dont_inherit = True)
+ exec(code, namespace) # noqa: S102 - the installed diffusers' own source with known blocks replaced
rebuilt = namespace.get(target.__name__)
if not callable(rebuilt):
raise RuntimeError("the rebuilt source did not define the forward")
@@ -196,18 +359,48 @@ def _rewrite_hunyuanimage(forward: Callable) -> Callable:
return rebuilt
+def _rewrite_hunyuanimage(forward: Callable) -> Callable:
+ """HunyuanImage-2.1: the boolean-mask text merge becomes ``merge_text_streams``."""
+ return _rebuild_forward(
+ forward,
+ ((_HUNYUANIMAGE_MERGE_BLOCK, _MERGE_CALL, "boolean-mask text merge"),),
+ {_MERGE_HELPER_NAME: merge_text_streams},
+ )
+
+
+def _rewrite_hunyuanvideo15(forward: Callable) -> Callable:
+ """HunyuanVideo-1.5: the ``if is_t2v`` branch and the three-stream merge, both made device-only."""
+ return _rebuild_forward(
+ forward,
+ (
+ (_HV15_T2V_BLOCK, _HV15_T2V_CALL, "t2v image-stream branch"),
+ (_HV15_MERGE_BLOCK, _HV15_MERGE_CALL, "boolean-mask three-stream merge"),
+ ),
+ {
+ _HV15_T2V_HELPER_NAME: hv15_image_stream,
+ _HV15_MERGE_HELPER_NAME: hv15_merge_streams,
+ },
+ )
+
+
# class name -> (why the stock forward cannot be captured, rewriter)
_REWRITES: dict[str, tuple[str, Callable[[Callable], Callable]]] = {
_HUNYUANIMAGE_CLS: (
"boolean-mask indexing in its text-stream merge syncs the host",
_rewrite_hunyuanimage,
),
+ _HUNYUANVIDEO15_CLS: (
+ "its t2v branch reads a device bool on the host and boolean-mask indexing in its "
+ "three-stream merge syncs the host",
+ _rewrite_hunyuanvideo15,
+ ),
}
-# Declined at load so status says "off" instead of arming a wrapper that refuses every step.
+# Declined at load so status says "off" instead of arming a wrapper that refuses every step. Qwen-Image-2.1 only
+# lands here when its fast step (diffusion_qwenimage21, which plans tensors-only decode calls) is not installed.
_UNCAPTURABLE: dict[str, str] = {
"QwenImage21Transformer2DModel": "its pipeline passes the prefix KV cache as a Python object "
- "on every step and its forward syncs the host",
+ "on every step; the Studio fast step that replays decode steps from tensors is not installed",
}
_CACHE: dict = {}
@@ -229,6 +422,16 @@ def resolve(cls: type) -> tuple[Optional[Callable], Optional[str]]:
Never raises. Cheap for an unknown class: a name lookup, no torch import."""
owner = _defining_class(cls)
+ planned = vars(owner).get("forward") if owner is not None else None
+ if getattr(planned, "__unsloth_graph_plan__", None) is not None:
+ # A Studio forward that plans its own capture-safe calls (Qwen-Image-2.1's fast step).
+ if capture_safe_disabled():
+ return (
+ None,
+ f"{owner.__name__} forward is not capture-safe ({_UNCAPTURABLE.get(owner.__name__, 'planned')}); "
+ f"rewrite disabled by {CAPTURE_SAFE_ENV}",
+ )
+ return planned, None
if owner is not None and owner.__name__ in _UNCAPTURABLE and owner.__name__ not in _REWRITES:
return (
None,
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 71152e0471e..c71733fc17e 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -17,6 +17,7 @@ import os
import threading
import traceback
import weakref
+import types
from functools import update_wrapper
from typing import Any, Optional
@@ -26,6 +27,10 @@ from .diffusion_bg_compile import eager_forced as _bg_eager_forced
CUDA_GRAPH_DISABLE_ENV = "UNSLOTH_DISABLE_CUDA_GRAPH"
# Master switch for every diffusion CUDA graph (whole-forward and per-block): "0" turns them all off.
CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"
+# =0: an offloaded denoiser never records its whole step with the copies inside (per-block graphs still may).
+OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"
+# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.
+STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"
# Per module. Legitimate second keys exist, but each costs a pool-sized slice of VRAM.
MAX_GRAPHS_PER_MODULE = 4
@@ -42,6 +47,29 @@ def _torch():
return torch
+# Comma list of family names (or "all") armed whatever the family declares: for measuring a family before it opts in.
+FORCE_FAMILIES_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPH_FAMILIES"
+# A module attribute: zero-arg callable returning hashable per-call state that the forward branches on.
+GRAPH_KEY_EXTRA_ATTR = "_unsloth_graph_key_extra"
+
+
+def _family_forced(family: Any) -> bool:
+ raw = (os.environ.get(FORCE_FAMILIES_ENV) or "").strip().lower()
+ if not raw:
+ return False
+ names = {n.strip() for n in raw.split(",") if n.strip()}
+ return "all" in names or str(getattr(family, "name", "")).lower() in names
+
+
+def offload_graphs_enabled() -> bool:
+ return (os.environ.get(OFFLOAD_CUDA_GRAPH_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
def cuda_graph_disabled() -> bool:
"""Whether an env kill switch is set. Read before any torch or pipe inspection."""
if os.environ.get(CUDA_GRAPHS_ENV, "").strip().lower() in ("0", "off", "false", "no"):
@@ -109,6 +137,23 @@ def hold_off_capture():
_CAPTURE_LOCK.release()
+# Raised while a whole step warms up or records: per-block graphs then run their compute, not nested replays.
+_STEP_RECORDING = [0]
+
+
+def _recording_step():
+ _STEP_RECORDING[0] += 1
+ try:
+ yield
+ finally:
+ _STEP_RECORDING[0] -= 1
+
+
+def step_recording() -> bool:
+ return _STEP_RECORDING[0] > 0
+
+
def capture_in_progress() -> bool:
"""Whether a capture is recording, as a snapshot for reporting.
@@ -119,6 +164,26 @@ def capture_in_progress() -> bool:
return _CAPTURE_DEPTH > 0
+def _graph_without_flush(graph: Any, pool: Any = None):
+ """``torch.cuda.graph`` minus its ``empty_cache`` / host ``emptyCache``: same capture stream discipline, same
+ global capture mode, same pool."""
+ torch = _torch()
+ torch.cuda.synchronize()
+ # torch.cuda.graph's own capture stream: pool blocks are reused only from the stream that recorded them.
+ stream = getattr(torch.cuda.graph, "default_capture_stream", None)
+ if stream is None or getattr(stream, "device", None) != torch.device(
+ "cuda", torch.cuda.current_device()
+ ):
+ stream = torch.cuda.Stream()
+ with torch.cuda.stream(stream):
+ graph.capture_begin(pool = pool, capture_error_mode = "global")
+ try:
+ yield
+ finally:
+ graph.capture_end()
+
+
# Not ``torch.utils._pytree``: it makes an unregistered object a LEAF, which must stay visible.
@@ -169,6 +234,12 @@ def graph_key(obj: Any) -> tuple:
return ("o", type(obj).__name__, id(obj))
+def _copy_mode(entry: Any) -> Any:
+ if getattr(entry, "inference_copy", False):
+ return _torch().inference_mode()
+ return contextlib.nullcontext()
+
+
def _uncapturable(key: tuple) -> bool:
kind = key[0]
if kind == "o":
@@ -200,10 +271,17 @@ _LIVE_WRAPPERS: "weakref.WeakSet" = weakref.WeakSet()
def _drop_pool_if_unused() -> None:
+ """Forget the shared pool id once no live graph records into it: capturing into a pool whose graphs are all gone
+ dies on the allocator's use_count assert. Checked per graph, not per wrapper: a wrapper still holding graphs in an
+ EARLIER pool (one dropped before) does not keep the current id alive."""
try:
+ pool = _POOL_BOX[0]
+ if pool is None:
+ return
for wrapper in tuple(_LIVE_WRAPPERS):
- if wrapper.cache:
- return
+ for entry in tuple(wrapper.cache.values()):
+ if getattr(entry, "pool_token", pool) == pool: # handles are fresh tuples per call
+ return
_POOL_BOX[0] = None
except Exception: # noqa: BLE001
pass
@@ -243,52 +321,6 @@ def refuse_if_exhausted(logger: Any = None) -> None:
_EXHAUSTED_LOGGED = False
-def _abandon_capture_pool(pool: Any) -> bool:
- """After a capture that raised: take the device allocator off the capture's pool if ``capture_end`` never did.
-
- ``CUDAGraph::capture_end`` checks ``cudaStreamEndCapture`` before ``endAllocateToPool``, so an invalidated capture
- leaves the pool in the allocator's ``captures_underway``: on torch 2.6 every later ``empty_cache`` then trips
- ``INTERNAL ASSERT captures_underway.empty()``; later torch skips the global release instead, so ``empty_cache``
- frees nothing. That failed graph never releases the reference its ``capture_begin`` took (its reset releases only
- once ``capture_end`` got past the pool), so it is released here, and ONLY when this call ended the recording: had
- ``capture_end`` got that far the graph owns the reference and releases it itself. True when this call ended it."""
- torch = _torch()
- end = getattr(torch._C, "_cuda_endAllocateToPool", None) or getattr(
- torch._C, "_cuda_endAllocateCurrentStreamToPool", None
- )
- if end is None or pool is None:
- return False
- try:
- device = torch.cuda.current_device()
- end(device, pool)
- except Exception: # noqa: BLE001 - capture_end already took it off, or capture_begin never put it on
- return False
- try:
- torch._C._cuda_releasePool(device, pool)
- except Exception: # noqa: BLE001
- pass
- return True
-
-
-def _heal_generators() -> None:
- """Take every CUDA default generator out of graph-capture mode after a FAILED capture.
-
- ``CUDAGraph.capture_end`` ends the generators' capture only after ``cudaStreamEndCapture`` succeeds. When the
- capture was invalidated (a host sync inside it, a kernel that may not be recorded) it raises first, and every later
- eager draw from the generator (``torch.randn(device = "cuda")``, a pipeline's noise) fails with "Offset increment
- outside graph capture encountered unexpectedly" for the rest of the process. A clone of the state keeps the seed
- and the eager offset, so the eager sequence continues as if the capture had never run."""
- try:
- torch = _torch()
- for gen in getattr(torch.cuda, "default_generators", ()) or ():
- clone = getattr(gen, "clone_state", None)
- restore = getattr(gen, "graphsafe_set_state", None)
- if callable(clone) and callable(restore):
- restore(clone())
- except Exception: # noqa: BLE001
- pass
-
-
def retire_failed_capture(
graph: Any,
pool: Any,
@@ -368,6 +400,149 @@ def _prewarm_token_counts(live: list) -> tuple:
return tuple(sorted(counts)[:8])
+def _pool_bytes(graph: Any) -> int:
+ """Device bytes the graph's private pool holds (shared by every graph of the load), 0 if unreadable."""
+ try:
+ pool = tuple(graph.pool())
+ total = 0
+ for seg in _torch().cuda.memory_snapshot():
+ if tuple(seg.get("segment_pool_id") or ()) == pool:
+ total += int(seg.get("total_size", 0))
+ return total
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def _current_stream() -> Any:
+ try:
+ return _torch().cuda.current_stream()
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _restore_stream(stream: Any) -> None:
+ if stream is None:
+ return
+ try:
+ torch = _torch()
+ if torch.cuda.current_stream() != stream:
+ torch.cuda.set_stream(stream)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _abandon_capture_pool(pool: Any) -> bool:
+ """After a capture that raised: take the allocator off the capture's pool if ``capture_end`` never did.
+
+ ``CUDAGraph::capture_end`` checks ``cudaStreamEndCapture`` before ``endAllocateToPool``, so an invalidated capture
+ can leave the pool in the allocator's ``captures_underway``: on torch 2.6 every later ``empty_cache`` then trips
+ ``INTERNAL ASSERT captures_underway.empty()`` and no later capture in the process records; later torch skips the
+ global release instead, so ``empty_cache`` frees nothing. That failed graph never releases the reference its
+ ``capture_begin`` took (its reset releases only once ``capture_end`` got past the pool), so it is released here,
+ and ONLY when this call ended the capture: had ``capture_end`` got that far (an error raised in the forward that
+ did not invalidate the stream, a failed instantiate) the graph owns the reference and releases it itself, and a
+ second release here would drop a shared pool under live graphs or abort the process. True when this call ended it."""
+ torch = _torch()
+ end = getattr(torch._C, "_cuda_endAllocateToPool", None) or getattr(
+ torch._C, "_cuda_endAllocateCurrentStreamToPool", None
+ )
+ if end is None or pool is None:
+ return False
+ try:
+ device = torch.cuda.current_device()
+ end(device, pool)
+ except Exception: # noqa: BLE001 - capture_end already took it off, or capture_begin never put it on
+ return False
+ try:
+ torch._C._cuda_releasePool(device, pool)
+ except Exception: # noqa: BLE001
+ pass
+ return True
+
+
+def _heal_generators() -> None:
+ """Take every CUDA default generator out of graph-capture mode after a FAILED capture.
+
+ ``CUDAGraph.capture_end`` ends the generators' capture only after ``cudaStreamEndCapture`` succeeds. When the
+ capture was invalidated (a host sync inside it, a kernel that may not be recorded) it raises first, and every later
+ eager draw from the generator (``torch.randn(device = "cuda")``, a pipeline's noise) fails with "Offset increment
+ outside graph capture encountered unexpectedly" for the rest of the process. A clone of the state keeps the seed
+ and the eager offset, so the eager sequence continues as if the capture had never run."""
+ try:
+ torch = _torch()
+ for gen in getattr(torch.cuda, "default_generators", ()) or ():
+ clone = getattr(gen, "clone_state", None)
+ restore = getattr(gen, "graphsafe_set_state", None)
+ if callable(clone) and callable(restore):
+ restore(clone())
+ except Exception: # noqa: BLE001
+ pass
+
+
+# Offloaded: each key's first SPEED_SAMPLES replays are judged against its own eager steps (_judge_keys).
+SPEED_SAMPLES = 3
+# Eager steps timed before a key's capture (after one untimed warm-up); never the capture's own warm-ups, whose
+# copies follow the recorded schedule and so measure the replay.
+SPEED_EAGER_SAMPLES = 2
+# A key's replay this much slower than its eager step (and by at least SPEED_MIN_MS) drops that key's graph.
+SPEED_MARGIN = 0.03
+SPEED_MIN_MS = 2.0
+# Streamed: the recording holds a ring and a pool the eager step does not, so it must be at least this much faster.
+SPEED_GAIN = 0.01
+SPEED_CHECK_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_GRAPH_SPEED_CHECK"
+
+
+def speed_check_enabled() -> bool:
+ return (os.environ.get(SPEED_CHECK_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _timing_events() -> tuple:
+ torch = _torch()
+ return torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+
+
+def live_pool_bytes() -> int:
+ """Bytes the shared graph pool holds while any graph lives (the status number)."""
+ try:
+ return max(
+ [int(w.stats.get("pool_bytes", 0)) for w in tuple(_LIVE_WRAPPERS) if w.cache] or [0]
+ )
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def live_pool_free_bytes() -> int:
+ """Bytes of the live shared pool that are reserved but NOT allocated: what ``memory_reserved() -
+ memory_allocated()`` would wrongly credit as reusable. The pool's allocated part (the graphs' static outputs) is
+ already outside that difference. 0 when no graph lives or the snapshot is unreadable."""
+ # Every pool a live graph recorded into: a failed capture retires the box's pool while earlier graphs keep it.
+ pools = {
+ tuple(entry.pool_token)
+ for w in tuple(_LIVE_WRAPPERS)
+ for entry in tuple(w.cache.values())
+ if getattr(entry, "pool_token", None) is not None
+ }
+ if not pools:
+ return 0
+ try:
+ free = 0
+ cuda = _torch().cuda
+ device = cuda.current_device() # the snapshot spans every card
+ for seg in cuda.memory_snapshot():
+ if seg.get("device", device) != device:
+ continue
+ if tuple(seg.get("segment_pool_id") or ()) in pools:
+ free += int(seg.get("total_size", 0)) - int(seg.get("allocated_size", 0))
+ return max(0, free)
+ except Exception: # noqa: BLE001 - over-counting only makes the guard stricter
+ return live_pool_bytes()
+
+
def _warn(logger: Any, what: str, exc: Any) -> None:
if logger is not None:
logger.warning("diffusion.cuda_graph: %s failed: %s", what, exc)
@@ -385,7 +560,17 @@ def _outer_layer(module: Any) -> Any:
class _Entry:
"""One captured graph plus the static buffers it replays into."""
- __slots__ = ("graph", "static", "in_spec", "out_spec", "out_tensors")
+ __slots__ = (
+ "graph",
+ "static",
+ "in_spec",
+ "out_spec",
+ "out_tensors",
+ "sticky",
+ "last",
+ "inference_copy",
+ "pool_token",
+ )
class GraphedForward:
@@ -402,25 +587,44 @@ class GraphedForward:
max_graphs: int = MAX_GRAPHS_PER_MODULE,
logger: Any = None,
call: Any = None,
+ placement: Optional["OffloadPlacement"] = None,
) -> None:
self.module = module
+ # How the weights reach the GPU (None = resident); see OffloadPlacement.
+ self.placement = placement
# The CLASS forward: the eager callable, whatever is already in the instance slot. A class
# whose stock forward syncs the host gets its capture-safe rewrite (bit-identical) instead.
# ``call`` overrides it (GraphedCompiledCall: the module's whole-module compiled callable).
safe = None
- if call is None:
+ if call is None or placement is not None:
try:
from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
safe, _ = _capture_safe(type(module))
except Exception: # noqa: BLE001 - the stock forward is still a correct eager callable
safe = None
self.capture_safe = safe is not None
+ # A forward whose call carries a non-tensor object (Qwen-Image-2.1's KV cache) names a planner that turns a
+ # call into a tensors-only one, or None for "run this call eager"; ``sticky`` kwargs only change between
+ # renders, so a replay copies them when the caller passes a different tensor object.
+ self.plan = getattr(safe, "__unsloth_graph_plan__", None) if safe is not None else None
+ self.sticky = frozenset(getattr(safe, "__unsloth_graph_sticky__", ()) or ())
+ # How a planned step enters through the offload hooks (offload_placement refuses a placement without it).
+ self.placed = getattr(safe, "__unsloth_graph_placed__", None) if safe is not None else None
+ if placement is not None:
+ # Model offload calls the slot with the weights already on the GPU, so a rewrite can sit there; a block
+ # offload slot holds the hook chain, which then reaches the planned step through ``_placed_step``.
+ rewrite = safe is not None and self.plan is None and placement.mode == "model"
+ call = safe.__get__(module) if rewrite else placement.eager
if call is not None:
self.orig = call
else:
self.orig = (safe if safe is not None else type(module).forward).__get__(module)
try:
- update_wrapper(self, self.orig)
+ update_wrapper(self, self.orig, updated = ())
+ # The wrapped callable's attributes, but never over this wrapper's own: under block offload ``orig`` can
+ # be the static step skip, whose ``plan`` / ``stats`` would otherwise replace the graph's.
+ for name, value in dict(getattr(self.orig, "__dict__", None) or {}).items():
+ self.__dict__.setdefault(name, value)
except Exception: # noqa: BLE001
pass
try:
@@ -438,6 +642,17 @@ class GraphedForward:
self.capture_error: Optional[dict] = None
self.cache: dict = {}
self.cap_hit = False
+ self._warmed: dict = {}
+ # Keys recorded before: a placement change re-records them with no warm-up (same shapes, new addresses).
+ self._recorded: dict = {}
+ # Offloaded only, per graph key: {"seen": eager calls, "eager": [(start, end)], "graph": [(start, end)],
+ # "verdict": None until judged, then "" (kept) or the reason it was dropped}.
+ self._judge: dict = {}
+ self._dropped: set = set() # keys that run eager for the load
+ self._slower: Optional[str] = None
+ # Per-block fallback for the calls this graph leaves eager: its arming callable and the handle it returned.
+ self.fallback: Any = None
+ self.fallback_handle: Any = None
self.protect_keyed: Optional[bool] = None
self.protect_ctl: Any = None
self.stats = {
@@ -449,11 +664,21 @@ class GraphedForward:
"refused_host_tensor": 0,
"refused_object": 0,
"cap_skips": 0,
+ "invalidations": 0,
+ "pool_bytes": 0,
+ "planned_eager": 0,
+ "shape_warmups": 0,
+ "evictions": 0,
+ "speed_eager": 0,
}
+ self.valid_token: Any = None
_LIVE_WRAPPERS.add(self)
def install(self) -> "GraphedForward":
"""Set ``forward`` via ``__dict__`` (``__setattr__`` would inspect it), under any outer step-skip layer."""
+ if self.placement is not None:
+ self.placement.install(self)
+ return self
outer = _outer_layer(self.module)
if outer is not None:
outer.inner = self
@@ -462,12 +687,22 @@ class GraphedForward:
return self
def uninstall(self) -> "GraphedForward":
+ if self.placement is not None:
+ self.placement.uninstall(self)
+ return self
outer = _outer_layer(self.module)
if outer is not None:
if outer.inner is self:
outer.inner = None
return self
- self.module.__dict__.pop("forward", None)
+ if self.module.__dict__.get("forward") is self:
+ self.module.__dict__.pop("forward", None)
+ return self
+ # Offload hooks installed after this wrapper wrap it: unwrap it inside their chain, keeping the hooks.
+ registry = getattr(self.module, "_diffusers_hook", None)
+ for ref in list(getattr(registry, "_fn_refs", None) or ()):
+ if getattr(ref, "forward", None) is self:
+ ref.forward = type(self.module).forward.__get__(self.module)
return self
def enable(self) -> "GraphedForward":
@@ -484,13 +719,21 @@ class GraphedForward:
def set_bypass(self, on: bool) -> "GraphedForward":
"""Run eager without dropping the captured graphs (a chunk under a step cache)."""
self.bypassed = bool(on)
+ if getattr(self, "fallback_handle", None) is not None:
+ self.fallback_handle.set_bypass(on)
return self
def _release(self) -> None:
"""Return the dropped graphs' pool segments to the device; ``gc.collect`` alone leaves them in the private pool."""
try:
gc.collect()
- cuda = getattr(_torch(), "cuda", None)
+ torch = _torch()
+ cuda = getattr(torch, "cuda", None)
+ if getattr(self, "placement", None) is not None:
+ # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them
+ clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)
+ if callable(clear):
+ clear()
if cuda is not None and hasattr(cuda, "empty_cache"):
cuda.empty_cache()
except Exception: # noqa: BLE001 - best effort
@@ -499,6 +742,12 @@ class GraphedForward:
def reset(self) -> "GraphedForward":
"""Drop every captured graph (weights changed under us: LoRA load, unload, adapter switch)."""
self.cache.clear()
+ self._recorded.clear()
+ if getattr(self, "fallback_handle", None) is not None:
+ try:
+ self.fallback_handle.reset()
+ except Exception: # noqa: BLE001
+ pass
self._release()
return self
@@ -521,7 +770,16 @@ class GraphedForward:
def free(self) -> "GraphedForward":
"""Drop the graphs and the slot."""
self.reset()
+ self.fallback = None
+ if getattr(self, "fallback_handle", None) is not None:
+ try:
+ self.fallback_handle.free()
+ except Exception: # noqa: BLE001
+ pass
+ self.fallback_handle = None
self.uninstall()
+ if self.placement is not None:
+ self.placement.release_slots()
self.enabled = False
_LIVE_WRAPPERS.discard(self)
return self
@@ -538,7 +796,11 @@ class GraphedForward:
"warmup": int(self.warmup),
"max_graphs": int(self.max_graphs),
"cap_hit": bool(self.cap_hit),
+ "placement": None if self.placement is None else self.placement.mode,
"stats": dict(self.stats),
+ "fallback": None
+ if getattr(self, "fallback_handle", None) is None
+ else self.fallback_handle.describe(),
"capture_error": None
if not error
else {
@@ -547,14 +809,149 @@ class GraphedForward:
},
}
+ def _judge_keys(self) -> None:
+ """Drop the graph of every key whose replays measured slower than its own eager steps.
+
+ A replay records the onload copies too, and how fast a recorded copy schedule runs depends on the machine and
+ on how it was recorded (an A100 VM: a Qwen-Image-2.1 prompt length recorded after the first replayed 16%
+ slower than eager while the first matched it; an L4 ran replayed copies 8% faster). Each key's first replays
+ are timed against its own eager steps with CUDA events, read only once they have completed (no host wait)."""
+ for key, state in list(self._judge.items()):
+ if state["verdict"] is not None or len(state["graph"]) < SPEED_SAMPLES:
+ continue
+ # Event queries and the drop's syncs / frees are prohibited while another thread records: judge later.
+ with hold_off_capture() as safe:
+ if not safe or not all(end.query() for _, end in state["graph"]):
+ continue
+ try:
+ # Best of each, so a stall on a shared card in either window cannot tip the verdict alone.
+ eager = min(start.elapsed_time(end) for start, end in state["eager"])
+ graph = min(start.elapsed_time(end) for start, end in state["graph"])
+ except Exception: # noqa: BLE001
+ state["verdict"] = ""
+ continue
+ self.stats["eager_ms"] = round(eager, 3)
+ self.stats["replay_ms"] = round(graph, 3)
+ placement = getattr(self, "placement", None)
+ streams = placement is not None and placement.streams()
+ if streams:
+ kept = graph < eager * (1.0 - SPEED_GAIN)
+ else:
+ kept = graph <= eager * (1.0 + SPEED_MARGIN) or graph - eager < SPEED_MIN_MS
+ if kept:
+ state["verdict"] = ""
+ continue
+ if graph > eager:
+ reason = f"replay measured slower than the eager step on this device ({graph:.1f} vs {eager:.1f} ms)"
+ else:
+ reason = (
+ f"replay measured within {SPEED_GAIN:.0%} of the eager step on this device ({graph:.1f} vs "
+ f"{eager:.1f} ms), not worth the memory a streamed recording holds"
+ )
+ state["verdict"] = reason
+ ring = False
+ if streams:
+ # One placement, one verdict: its other keys stream the same copies.
+ ring = bool(placement.decline(reason))
+ for other in list(self.cache):
+ if other != key:
+ self.cache.pop(other, None)
+ self._dropped.add(other)
+ self._slower = reason
+ self._dropped.add(key)
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.cuda_graph: %s drops a graph: %s",
+ type(self.module).__name__,
+ reason,
+ )
+ if self.cache.pop(key, None) is not None:
+ try:
+ _torch().cuda.current_stream().synchronize() # no replay still running on what is freed
+ except Exception: # noqa: BLE001
+ pass
+ _drop_pool_if_unused()
+ if not self.cache:
+ self.capture_error = {"type": "Refused", "msg": reason}
+ if not ring:
+ # (with a ring, its drop at the end of this forward flushes once: a mid-forward flush fragments the cache)
+ self._release()
+
+ def _timed_eager(self, call: Any, args: tuple, kwargs: dict, key: Any) -> Any:
+ """One of the caller's eager steps before ``key`` records (its eager reference).
+
+ ``call`` is the step the graph would record (the planned step under a plan), on fresh copies of the caller's
+ tensors. The first call of a key may still compile or autotune and is not timed (for a planned module it is
+ also the new prompt length's warm-up); the next SPEED_EAGER_SAMPLES are."""
+ self.stats["eager_calls"] += 1
+ self.stats["speed_eager"] += 1
+ # On copies laid out like the capture's statics, as the shape warm-up does: compiled code guards on strides and
+ # storage offsets, so steps on the caller's views compile a variant the capture then compiles again.
+ live: list = []
+ spec = _flatten((args, kwargs), live)
+ try:
+ statics = self._statics(live)
+ except _torch().cuda.OutOfMemoryError:
+ # No room for the probe's copies: decline the key and run the caller's own step.
+ self._dropped.add(key)
+ self._slower = "no memory for the speed probe's input copies"
+ return call(*args, **kwargs)
+ args, kwargs = _rebuild(spec, statics)
+ state = self._judge_state(key)
+ state["seen"] += 1
+ if state["seen"] == 1:
+ self._warmed[key] = None
+ return call(*args, **kwargs)
+ # Event records are prohibited while another thread records a graph: such a step goes untimed.
+ start = end = None
+ with hold_off_capture() as safe:
+ if safe:
+ start, end = _timing_events()
+ start.record()
+ out = call(*args, **kwargs)
+ if start is not None:
+ with hold_off_capture() as safe:
+ if safe:
+ end.record()
+ state["eager"].append((start, end))
+ return out
+
+ def _judge_state(self, key: Any) -> dict:
+ state = self._judge.get(key)
+ if state is None:
+ state = self._judge[key] = {"seen": 0, "eager": [], "graph": [], "verdict": None}
+ while len(self._judge) > 4 * self.max_graphs:
+ self._judge.pop(next(iter(self._judge)))
+ return state
+
+ def _placed_step(self, **step: Any) -> Any:
+ return self.placed(self.orig, step)
+
def _eager(self, args: tuple, kwargs: dict) -> Any:
self.stats["eager_calls"] += 1
return self.orig(*args, **kwargs)
+ def _engage_fallback(self) -> None:
+ """Arm the per-block graphs once this whole-step graph dropped a key or failed: the calls it now leaves eager
+ run through them. Between forwards' blocks (the step has not started), so the slot ring can change."""
+ fallback, self.fallback = self.fallback, None
+ try:
+ self.fallback_handle = fallback()
+ except Exception as exc: # noqa: BLE001
+ _warn(self.logger, "per-block fallback", exc)
+ self.fallback_handle = None
+ if self.fallback_handle is not None and self.logger is not None:
+ self.logger.info(
+ "diffusion.cuda_graph: %s records per block for the steps its whole-step graph leaves eager",
+ type(self.module).__name__,
+ )
+
def __call__(self, *args, **kwargs):
if _bg_capture_suppressed():
# Background compile thread: never capture off the render thread.
return self.orig(*args, **kwargs)
+ if getattr(self, "fallback", None) is not None and (self.poisoned or self._dropped):
+ self._engage_fallback()
if (
not self.enabled
or _bg_eager_forced()
@@ -569,6 +966,61 @@ class GraphedForward:
if kwargs.get("return_dict", True) is not False:
return self._eager(args, kwargs)
+ if self.placement is not None:
+ # The recorded graphs replay the weight addresses of the placement they saw; a new placement records anew.
+ token = self.placement.token()
+ if token != self.valid_token:
+ if self.cache:
+ self.stats["invalidations"] += 1
+ # Not reset(): its empty_cache on every model-offload onload makes the next onload pay the
+ # cudaMallocs again. The dropped pool's blocks stay with the allocator.
+ self.cache.clear()
+ _drop_pool_if_unused() # the next capture must not record into a pool no graph holds
+ self._warmed.clear()
+ self.valid_token = token
+ refusal = self.placement.refusal(self.stats)
+ if self._judge and speed_check_enabled():
+ self._judge_keys()
+ if self._dropped and getattr(self, "fallback", None) is not None:
+ self._engage_fallback()
+ refusal = self.placement.refusal(self.stats)
+ skip = getattr(self.placement, "skip", None)
+ if refusal is None and skip is not None and _skips_planned(skip):
+ # This render skips steps: the skip layer must see every call, so they all run eager (per-block
+ # graphs, where armed, still record below the hooks).
+ self.stats["skip_eager"] = self.stats.get("skip_eager", 0) + 1
+ if getattr(self, "fallback", None) is not None:
+ self._engage_fallback()
+ return self._eager(args, kwargs)
+ if refusal is not None:
+ if self.logger is not None and getattr(self, "_refusal_logged", None) != refusal:
+ self._refusal_logged = refusal
+ self.logger.info(
+ "diffusion.cuda_graph: %s runs eager: %s",
+ type(self.module).__name__,
+ refusal,
+ )
+ self.capture_error = {"type": "Refused", "msg": refusal}
+ return self._eager(args, kwargs)
+ if (self.capture_error or {}).get("type") == "Refused" and (
+ self.cache or not self._slower
+ ):
+ self.capture_error = None
+
+ call = self.orig
+ # Every eager fallback below runs the caller's own call, never the planned one.
+ eager_args, eager_kwargs = args, kwargs
+ if self.plan is not None:
+ planned = self.plan(self.module, args, kwargs)
+ if planned is None:
+ self.stats["planned_eager"] += 1
+ return self._eager(args, kwargs)
+ call, kwargs = planned
+ args = ()
+ if self.placement is not None:
+ # The offload hooks onload the top-level weights: the step must enter through the hooked forward.
+ call = self._placed_step
+
try:
key = graph_key((args, kwargs))
if self.protect_keyed is None:
@@ -588,18 +1040,51 @@ class GraphedForward:
if self.protect_keyed:
# One graph per branch: a W4A4 replay at a W4A16 step means the lever never fired.
key = key + protect_graph_key(self.protect_ctl)
+ extra = getattr(self.module, GRAPH_KEY_EXTRA_ATTR, None)
+ if callable(extra):
+ # Python state a pre-hook set for THIS call that picks a branch inside the forward (HunyuanVideo-1.5's
+ # null-mask flag): a graph recorded on one branch must never replay for the other.
+ key = key + (("x", extra()),)
entry = self.cache.get(key)
except Exception: # noqa: BLE001 - an unhashable tree is simply not capturable
self.stats["refused_object"] += 1
- return self._eager(args, kwargs)
+ return self._eager(eager_args, eager_kwargs)
if entry is None:
if _uncapturable(key):
self.stats["refused_object"] += 1
- return self._eager(args, kwargs)
+ return self._eager(eager_args, eager_kwargs)
if _has_float(key):
self.stats["refused_float"] += 1
- return self._eager(args, kwargs)
+ return self._eager(eager_args, eager_kwargs)
+ if self.placement is not None and speed_check_enabled():
+ if key in self._dropped:
+ return self._eager(eager_args, eager_kwargs)
+ state = self._judge.get(key)
+ if state is None or (
+ state["verdict"] is None and len(state["eager"]) < SPEED_EAGER_SAMPLES
+ ):
+ # The eager reference for _judge_keys, timed with CUDA events and never waited on.
+ return self._timed_eager(call, args, kwargs, key)
+ if self.plan is not None and self.stats["captures"] and key not in self._warmed:
+ # A new prompt length: this real step is the new shape's warm-up; the next one records.
+ self._warmed[key] = None
+ while len(self._warmed) > 4 * self.max_graphs:
+ self._warmed.pop(next(iter(self._warmed)))
+ self.stats["shape_warmups"] += 1
+ # On copies laid out like the capture's statics (fresh, offset 0): compiled code guards on strides and
+ # storage offsets, so a step on the caller's views would warm a variant the capture then misses and
+ # compiles inside the capture, which fails it.
+ live: list = []
+ spec = _flatten((args, kwargs), live)
+ warm_args, warm_kwargs = _rebuild(spec, self._statics(live))
+ return call(*warm_args, **warm_kwargs)
+ if len(self.cache) >= self.max_graphs and self.plan is not None:
+ # A planned call keys on the prompt length, so evict the least recently replayed graph.
+ self.cache.pop(next(iter(self.cache)))
+ self.stats["evictions"] += 1
+ # The last graph on the shared pool takes the pool with it; never hand its id to the next capture.
+ _drop_pool_if_unused()
if len(self.cache) >= self.max_graphs:
# The recorded graphs keep replaying: one rare new key is no reason to drop them.
self.stats["cap_skips"] += 1
@@ -612,11 +1097,22 @@ class GraphedForward:
self.max_graphs,
type(self.module).__name__,
)
- return self._eager(args, kwargs)
+ return self._eager(eager_args, eager_kwargs)
captured = None
+ stream_before = _current_stream()
try:
- captured = self._capture(args, kwargs)
+ captured = self._capture(args, kwargs, call, key)
except Exception as exc: # noqa: BLE001 - a failed capture must never fail the render
+ # torch.cuda.graph's __exit__ raises in capture_end before it leaves its capture stream, so the thread
+ # would run everything after (the eager retry, the rest of the process) on the capture stream.
+ _restore_stream(stream_before)
+ if "already recording" not in str(
+ exc
+ ): # a collision: the generators are the live capture's
+ _heal_generators() # before anything eager draws from the CUDA RNG
+ if self.placement is not None:
+ self.placement.recover() # the eager retry must not wait on the dead capture's copies
+ self.placement.release_slots() # nothing will replay into the ring for this load
self.poison(exc)
self.stats["fallbacks"] += 1
if self.logger is not None:
@@ -631,7 +1127,7 @@ class GraphedForward:
exc.__traceback__ = None
if captured is None:
self._release()
- return self._eager(args, kwargs)
+ return self._eager(eager_args, eager_kwargs)
entry = captured
self.cache[key] = entry
if self.logger is not None:
@@ -643,20 +1139,98 @@ class GraphedForward:
len(entry.static),
)
+ elif self.plan is not None:
+ self.cache[key] = self.cache.pop(key)
+
live: list = []
_flatten((args, kwargs), live)
- for dst, src in zip(entry.static, live):
- dst.copy_(src)
+ last = entry.last
+ state = self._judge.get(key) if self.placement is not None else None
+ timing = (
+ state is not None and state["verdict"] is None and len(state["graph"]) < SPEED_SAMPLES
+ )
+ start = None
+ if timing:
+ # Through the input copies and output clones: the eager step pays neither. Not while another thread
+ # records, whose capture prohibits event records.
+ with hold_off_capture() as safe:
+ if safe:
+ start, end = _timing_events()
+ start.record()
+ with _copy_mode(entry):
+ for index, (dst, src) in enumerate(zip(entry.static, live)):
+ if last is not None and entry.sticky[index]:
+ ref = last[index]
+ if ref is not None and ref() is src:
+ continue
+ dst.copy_(src)
+ last[index] = weakref.ref(src)
+ continue
+ dst.copy_(src)
entry.graph.replay()
- self.stats["replays"] += 1
# Cloned: every replay writes the SAME buffers, which the pipeline holds across steps.
- return _rebuild(entry.out_spec, [t.clone() for t in entry.out_tensors])
+ outs = [t.clone() for t in entry.out_tensors]
+ if start is not None:
+ with hold_off_capture() as safe:
+ if safe:
+ end.record()
+ state["graph"].append((start, end))
+ self.stats["replays"] += 1
+ skip = getattr(self.placement, "skip", None) if self.placement is not None else None
+ if skip is not None and getattr(skip, "counting", False):
+ # The replay passed the (pass-through) skip layer by: count the computed step as it would have.
+ skip.stats["calls"] += 1
+ skip.stats["computed"] += 1
+ return _rebuild(entry.out_spec, outs)
+
+ def _statics(self, live: list) -> list:
+ """Fresh copies of ``live``, as a capture records from (and the shape warm-up step runs on)."""
+ torch = _torch()
+ if self.plan is not None:
+ # Inference tensors in, inference tensors as statics: compiled code guards on the dispatch keys, so a
+ # mismatch makes the graph record its own (separately autotuned, so not bit-equal) compiled variant.
+ static = []
+ for t in live:
+ with torch.inference_mode(bool(t.is_inference())):
+ static.append(torch.empty_like(t))
+ else:
+ # A tensor made under ``torch.inference_mode()``, which renders run in, refuses ``copy_``.
+ with torch.inference_mode(False):
+ static = [torch.empty_like(t) for t in live]
+ # Writing into an inference tensor is only allowed inside inference mode.
+ copying = types.SimpleNamespace(
+ inference_copy = self.plan is not None and any(t.is_inference() for t in static)
+ )
+ with _copy_mode(copying):
+ for dst, src in zip(static, live):
+ dst.copy_(src)
+ return static
- def _capture(self, args: tuple, kwargs: dict) -> _Entry:
+ def _capture(
+ self,
+ args: tuple,
+ kwargs: dict,
+ call: Any = None,
+ key: Any = None,
+ ) -> _Entry:
torch = _torch()
+ call = self.orig if call is None else call
entry = _Entry()
live: list = []
entry.in_spec = _flatten((args, kwargs), live)
+ entry.sticky, entry.last = None, None
+ if self.sticky:
+ # Flat positions of the sticky kwargs, in ``_flatten``'s order (args, then kwargs by insertion).
+ marks: list = []
+ _flatten(args, marks)
+ sticky = [False] * len(marks)
+ for name, value in kwargs.items():
+ part: list = []
+ _flatten(value, part)
+ sticky.extend([name in self.sticky] * len(part))
+ if any(sticky):
+ entry.sticky = tuple(sticky)
+ entry.last = [weakref.ref(t) if s else None for t, s in zip(live, sticky)]
bad = [(i, str(t.device)) for i, t in enumerate(live) if t.device.type != "cuda"]
if bad:
@@ -675,11 +1249,9 @@ class GraphedForward:
f"graph at whatever value this capture saw, so this load runs eager"
)
- # A tensor made under ``torch.inference_mode()``, which renders run in, refuses ``copy_``.
- with torch.inference_mode(False):
- entry.static = [torch.empty_like(t) for t in live]
- for dst, src in zip(entry.static, live):
- dst.copy_(src)
+ entry.static = self._statics(live)
+ # Writing into an inference tensor is only allowed inside inference mode.
+ entry.inference_copy = self.plan is not None and any(t.is_inference() for t in entry.static)
static_args, static_kwargs = _rebuild(entry.in_spec, entry.static)
if nvfp4_layers:
@@ -687,27 +1259,70 @@ class GraphedForward:
from .diffusion_nvfp4_linear import nvfp4_prewarm
nvfp4_prewarm(self.module, _prewarm_token_counts(live), logger = self.logger)
+ refuse_if_exhausted(
+ self.logger
+ ) # before the warm-ups, and before the step skip's counters are saved
+ # Under an offload placement the capture also records the copy-stream onloads (OffloadPlacement.record).
+ record = call if self.placement is None else self.placement.recorder(call)
# Side stream: a workspace first created DURING capture is only valid while recording.
- side = torch.cuda.Stream()
- side.wait_stream(torch.cuda.current_stream())
- with torch.cuda.stream(side):
- for _ in range(self.warmup):
- self.orig(*static_args, **static_kwargs)
- torch.cuda.current_stream().wait_stream(side)
- torch.cuda.synchronize()
-
- refuse_if_exhausted(self.logger)
- graph = torch.cuda.CUDAGraph()
- # An explicit pool id, never None: a capture that fails must hand its pool back (retire_failed_capture).
- pool = _POOL_BOX[0]
- if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):
- pool = torch.cuda.graph_pool_handle()
- try:
- with _capturing(), torch.cuda.graph(graph, pool = pool):
- out = self.orig(*static_args, **static_kwargs)
- except BaseException as exc:
- retire_failed_capture(graph, pool, exc)
- raise
+ # A planned module's later captures were warmed by a real step (``__call__``): no warm-up, no cache flushes.
+ recapture = self.plan is not None and self.stats["captures"] > 0
+ # An offloaded key re-recorded after its weights moved is already warm: it records like a planned re-capture.
+ recapture = recapture or (
+ self.placement is not None and key is not None and key in self._recorded
+ )
+ if self.placement is not None:
+ self.placement.before_capture() # anything the recorded copies land in exists before the capture
+ self.valid_token = (
+ self.placement.token()
+ ) # the placement this capture records (the ring included)
+ # A pass-through step skip under the hooks counts every call; the warm-ups and the recording are not steps.
+ skip = getattr(self.placement, "skip", None) if self.placement is not None else None
+ skip_stats = (
+ dict(skip.stats)
+ if skip is not None and isinstance(getattr(skip, "stats", None), dict)
+ else None
+ )
+ # Per-block graphs under the hooks run their compute while the whole step warms up and records.
+ # A key warmed by its timed eager steps records straight away: a side-stream warm-up would cache a second set
+ # of activations (the cold render's peak).
+ warm = (
+ self.placement is not None
+ and key is not None
+ and int((getattr(self, "_judge", {}).get(key) or {}).get("seen", 0)) >= 1
+ )
+ with _recording_step():
+ if not recapture and not warm:
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ # A re-record under a new offload placement only needs the forward to have run once on it.
+ again = self.placement is not None and self.stats["captures"] > 0
+ for _ in range(min(self.warmup, 1) if again else self.warmup):
+ record(*static_args, **static_kwargs)
+ torch.cuda.current_stream().wait_stream(side)
+ torch.cuda.synchronize()
+
+ graph = torch.cuda.CUDAGraph()
+ # An explicit pool id, never None: a capture that fails must hand its pool back (retire_failed_capture).
+ pool = _POOL_BOX[0]
+ if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):
+ pool = torch.cuda.graph_pool_handle()
+ recorder = _graph_without_flush if recapture else torch.cuda.graph
+ try:
+ with _capturing(), recorder(graph, pool = pool):
+ out = record(*static_args, **static_kwargs)
+ except BaseException as exc:
+ retire_failed_capture(graph, pool, exc)
+ raise
+ finally:
+ if skip_stats is not None:
+ skip.stats = skip_stats
+ self.stats["pool_bytes"] = _pool_bytes(graph)
+ if key is not None:
+ self._recorded[key] = None
+ while len(self._recorded) > 4 * self.max_graphs:
+ self._recorded.pop(next(iter(self._recorded)))
if _POOL_BOX[0] is None:
try:
_POOL_BOX[0] = graph.pool()
@@ -715,6 +1330,8 @@ class GraphedForward:
pass
entry.graph = graph
+ # The pool this graph recorded into (see _drop_pool_if_unused), not whatever the box holds by now.
+ entry.pool_token = pool if pool is not None else _POOL_BOX[0]
out_tensors: list = []
entry.out_spec = _flatten(out, out_tensors)
entry.out_tensors = out_tensors
@@ -743,6 +1360,479 @@ class GraphedCompiledCall(GraphedForward):
return self
+# Offloaded denoisers.
+
+
+def _leaf_ptrs(t: Any, out: list) -> None:
+ """Data pointers of ``t``'s storage, through tensor subclasses (torchao int8 / fp8 weights) to their inner tensors."""
+ flatten = getattr(type(t), "__tensor_flatten__", None)
+ if flatten is not None and type(t).__name__ not in ("Tensor", "Parameter"):
+ try:
+ names, _ctx = t.__tensor_flatten__()
+ for name in names:
+ _leaf_ptrs(getattr(t, name), out)
+ return
+ except Exception: # noqa: BLE001 - fall through to the plain pointer
+ pass
+ try:
+ out.append((str(t.device), int(t.data_ptr())))
+ except Exception: # noqa: BLE001
+ out.append(("?", id(t)))
+
+
+def _module_tensors(module: Any) -> list:
+ seen: set = set()
+ out: list = []
+ for t in list(module.parameters()) + list(module.buffers()):
+ if id(t) not in seen:
+ seen.add(id(t))
+ out.append(t)
+ return out
+
+
+class OffloadPlacement:
+ """Where an offloaded denoiser's replay sits and when its recorded addresses stop holding.
+
+ ``group``: diffusers block offload driven by the event-fenced prefetch (diffusion_offload_prefetch). The replay
+ replaces the module's hooked forward, so ONE graph records the whole step: the copy-stream copies of every
+ streamed group from its pinned host tensor into the graph pool, the compute stream waiting on each copy's event,
+ and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays the
+ prefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.
+
+ ``model``: whole-module CPU offload (accelerate). The hook moves the module on the eager path first and the replay
+ sits under it, in ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after
+ ``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""
+
+ MODEL_MAX_INVALIDATIONS = 2
+ # Past MODEL_MAX_INVALIDATIONS, re-recording continues while each recording serves at least this many replays.
+ MODEL_MIN_REPLAYS_PER_RECORD = 4
+
+ def __init__(self, module: Any, mode: str):
+ self.module = module
+ self.mode = mode
+ self.prev: Any = None
+ self._epoch: Any = None
+ self._fp: Any = None
+ self._checked: Any = object()
+ self._why: Optional[str] = None
+ self._tensors: Optional[list] = None
+ # Set when a key's replay did not pay on a streaming placement: nothing records again for this load.
+ self._declined: Optional[str] = None
+ self._slots = False
+ if mode == "group":
+ self.prev = module.__dict__.get("forward")
+ else:
+ self.prev = getattr(module, "_old_forward", None)
+ if self.prev is None:
+ raise RuntimeError(f"no {mode} offload forward to wrap")
+ self.eager = self.prev
+ # A static step skip (diffusion_step_skip) the replay would bypass: see GraphedForward.__call__.
+ self.skip: Any = self.prev if getattr(self.prev, "_unsloth_outer_forward", False) else None
+ # A capture-safe rewrite (diffusion_capture_safe) under block offload: the hook chain ends in the bound class
+ # forward, which the rewrite replaces while the graph is installed (bit-identical, so eager calls may take it).
+ self._inner: Any = None
+ self._inner_stock: Any = None
+ self._inner_safe: Any = None
+ if mode == "group":
+ safe = _plain_rewrite(module)
+ if safe is not None:
+ ref = _innermost_class_forward(module)
+ if ref is None:
+ raise RuntimeError(
+ "the block-offload hook chain does not end in the class forward"
+ )
+ self._inner, self._inner_stock = ref, ref.forward
+ self._inner_safe = safe.__get__(module)
+
+ def install(self, handle: Any) -> None:
+ if self.mode == "group":
+ if self._inner is not None:
+ self._inner.forward = self._inner_safe
+ self.module.__dict__["forward"] = handle
+ self._enable_slots()
+ else:
+ self.module._old_forward = handle
+
+ def _prefetcher(self) -> Any:
+ from .diffusion_offload_prefetch import module_prefetcher
+ return module_prefetcher(self.module)
+
+ def _enable_slots(self) -> None:
+ """Streamed groups copy into the prefetcher's slot ring (diffusion_offload_prefetch), so the recorded copies
+ land at fixed addresses outside the graph pool: the pool then holds one step's activations, not a second
+ prefetch window beside the eager one. Off with UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS=0."""
+ if (os.environ.get(STEP_SLOTS_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return
+ if not self.streams():
+ return
+ try:
+ pf = self._prefetcher()
+ if pf is not None and callable(getattr(pf, "enable_slots", None)):
+ pf.enable_slots()
+ self._slots = bool(getattr(pf, "slot_of", None))
+ except Exception: # noqa: BLE001
+ self._slots = False
+
+ def streams(self) -> bool:
+ """Whether this placement moves weights every step (model offload, or a block group not pinned resident):
+ the recording then holds memory beside the eager step's."""
+ if self.mode != "group":
+ return True
+ try:
+ from .diffusion_memory import _offload_groups
+ pf = self._prefetcher()
+ for group in _offload_groups(self.module):
+ if getattr(group, "_unsloth_resident", False) or getattr(
+ group, "_unsloth_pinned_top", False
+ ):
+ continue
+ if pf is None or pf.owns(group):
+ return True
+ except Exception: # noqa: BLE001 - unknown layout: treat as streaming (the stricter speed check)
+ return True
+ return False
+
+ def before_capture(self) -> None:
+ """Allocate the slot ring before a capture, so no slot is carved out of the graph pool."""
+ if not self._slots:
+ return
+ try:
+ pf = self._prefetcher()
+ if pf is not None and callable(getattr(pf, "materialize_slots", None)):
+ pf.materialize_slots()
+ except Exception: # noqa: BLE001
+ pass
+
+ def release_slots(self) -> None:
+ """Hand the slot ring back once no graph of this placement can replay into it."""
+ if not self._slots:
+ return
+ self._slots = False
+ try:
+ pf = self._prefetcher()
+ if pf is not None and callable(getattr(pf, "drop_slots_after_forward", None)):
+ pf.drop_slots_after_forward()
+ except Exception: # noqa: BLE001
+ pass
+
+ def decline(self, reason: str) -> bool:
+ """No more recording on this placement for the load; hand the slot ring back after the current forward.
+ True when that drop is scheduled (it then also releases the dropped graphs' pool)."""
+ self._declined = reason
+ if not self._slots:
+ return False
+ self._slots = False
+ try:
+ pf = self._prefetcher()
+ if pf is not None and callable(getattr(pf, "drop_slots_after_forward", None)):
+ pf.drop_slots_after_forward()
+ return True
+ except Exception: # noqa: BLE001
+ pass
+ return False
+
+ def uninstall(self, handle: Any) -> None:
+ if self._inner is not None and self._inner.forward is self._inner_safe:
+ self._inner.forward = self._inner_stock
+ if self.mode == "group":
+ if self.module.__dict__.get("forward") is handle:
+ self.module.__dict__["forward"] = self.prev
+ return
+ if getattr(self.module, "_old_forward", None) is handle:
+ self.module._old_forward = self.prev
+ if (
+ self.module.__dict__.get("forward") is handle
+ ): # hooks removed since (diffusers re-enables them per call)
+ self.module.__dict__.pop("forward", None)
+
+ def recorder(self, call: Any) -> Any:
+ """What a capture records for ``call``: under block offload the prefetcher's copies join the capture."""
+ if self.mode != "group":
+ return call
+
+ def record(*args: Any, **kwargs: Any) -> Any:
+ from .diffusion_offload_prefetch import module_prefetcher
+
+ pf = module_prefetcher(self.module)
+ pf.begin() # inside the capture: forks the copy stream in, and end() joins it back
+ try:
+ return call(*args, **kwargs)
+ finally:
+ pf.end()
+
+ return record
+
+ def record(self, *args: Any, **kwargs: Any) -> Any:
+ return self.recorder(self.eager)(*args, **kwargs)
+
+ def recover(self) -> None:
+ """After a failed capture: drop what the prefetcher queued inside it (see GroupPrefetcher.abandon)."""
+ if self.mode != "group":
+ return
+ try:
+ from .diffusion_offload_prefetch import module_prefetcher
+ pf = module_prefetcher(self.module)
+ if pf is not None:
+ pf.abandon()
+ except Exception: # noqa: BLE001
+ pass
+
+ def _fingerprint(self) -> tuple:
+ out: list = []
+ if self.mode == "group":
+ from .diffusion_memory import _offload_groups
+ from .diffusion_offload_prefetch import _group_tensors, module_prefetcher
+
+ pf = module_prefetcher(self.module)
+ for group in _offload_groups(self.module):
+ resident = bool(getattr(group, "_unsloth_resident", False))
+ out.append((id(group), resident, bool(pf is not None and pf.owns(group))))
+ tensors = _group_tensors(group)
+ if resident:
+ for t in tensors:
+ _leaf_ptrs(t, out)
+ else:
+ cpu = getattr(group, "cpu_param_dict", None) or {}
+ for t in tensors:
+ _leaf_ptrs(cpu.get(t, t), out)
+ # Streamed copies land in the slot ring when it is on: a new ring is a new placement.
+ raws = getattr(pf, "slot_raw", None) or {}
+ out.append(("slots", tuple((i, int(raws[i].data_ptr())) for i in sorted(raws))))
+ return tuple(out)
+ # Read from the module's registry on every call: Module.to() replaces buffers with new tensor objects on each
+ # offload / onload, so a list taken once would keep fingerprinting the obsolete ones.
+ for t in _module_tensors(self.module):
+ _leaf_ptrs(t, out)
+ return tuple(out)
+
+ def token(self) -> Any:
+ if self.mode == "group":
+ from .diffusion_offload_prefetch import placement_epoch
+
+ epoch = placement_epoch()
+ if epoch != self._epoch:
+ self._epoch = epoch
+ self._fp = self._fingerprint()
+ return self._fp
+ self._fp = self._fingerprint()
+ return self._fp
+
+ def refusal(self, stats: dict) -> Optional[str]:
+ if getattr(self, "_declined", None) is not None:
+ return self._declined
+ if self.mode == "model":
+ # Judged once per placement, when the weights land (the fingerprint is rebuilt every call).
+ if self._checked != self._fp:
+ self._checked = self._fp
+ moves = int(stats.get("invalidations", 0))
+ records = max(1, int(stats.get("captures", 0)))
+ replays = int(stats.get("replays", 0))
+ self._why = None
+ if (
+ moves >= self.MODEL_MAX_INVALIDATIONS
+ and replays < self.MODEL_MIN_REPLAYS_PER_RECORD * records
+ ):
+ self._why = (
+ f"model offload placed the weights at new addresses on {moves} onloads, {replays} replays "
+ f"over {records} recordings; each re-record costs a step"
+ )
+ return self._why
+ if self._checked is not self._fp:
+ from .diffusion_offload_prefetch import capture_refusal
+ self._checked = self._fp
+ self._why = capture_refusal(self.module)
+ return self._why
+
+
+def _skips_planned(skip: Any) -> bool:
+ plan = getattr(skip, "plan", None) or ()
+ return bool(plan) and not all(plan)
+
+
+def offload_placement(module: Any) -> tuple:
+ """``(placement, refusal)`` for ``module`` as it is placed now: ``(None, None)`` when nothing moves it."""
+ try:
+ subs = [m for m in module.modules() if m is not module]
+ except Exception: # noqa: BLE001
+ subs = []
+ if any(getattr(m, "_hf_hook", None) is not None for m in subs):
+ return None, (
+ "leaf-level (sequential) offload uploads every layer synchronously from host memory inside the forward"
+ )
+ hf_hook = getattr(module, "_hf_hook", None)
+ compiled = getattr(module, "_compiled_call_impl", None) is not None
+ if hf_hook is not None:
+ if compiled:
+ return (
+ None,
+ "whole-module compiled denoiser: the CPU-offload hook runs inside its compiled call",
+ )
+ if "CpuOffload" not in type(hf_hook).__name__:
+ return (
+ None,
+ f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",
+ )
+ why = _forward_refusal(module, "model")
+ if why is not None:
+ return None, why
+ try:
+ return OffloadPlacement(module, "model"), None
+ except Exception as exc: # noqa: BLE001
+ return None, f"model offload hook unreadable ({exc})"
+ registry = getattr(module, "_diffusers_hook", None)
+ hooks = getattr(registry, "hooks", None) or {}
+ if not any("offload" in str(key) for key in hooks):
+ return None, None
+ if compiled:
+ return (
+ None,
+ "whole-module compiled denoiser: the offload hooks run inside its compiled call",
+ )
+ try:
+ from diffusers.hooks import group_offloading as go
+ name = getattr(go, "_GROUP_OFFLOADING", "group_offloading")
+ except Exception: # noqa: BLE001
+ name = "group_offloading"
+ if name not in hooks:
+ return None, "an offload hook other than block offloading moves the weights"
+ skip = None
+ for fn_ref in getattr(registry, "_fn_refs", None) or ():
+ if getattr(getattr(fn_ref, "forward", None), "_unsloth_outer_forward", False):
+ # The static step skip under the offload hook: a render that plans skips runs every call through it
+ # (GraphedForward.__call__); one that plans none passes straight through it, so the graph may replay.
+ skip = fn_ref.forward
+ why = _forward_refusal(module, "group")
+ if why is not None:
+ return None, why
+ from .diffusion_offload_prefetch import capture_refusal
+
+ why = capture_refusal(module)
+ if why is not None and "background" not in why:
+ return None, why
+ try:
+ placement = OffloadPlacement(module, "group")
+ except Exception as exc: # noqa: BLE001
+ return None, f"block offload hook unreadable ({exc})"
+ placement.skip = skip
+ return placement, None
+
+
+def _forward_refusal(module: Any, mode: str) -> Optional[str]:
+ """Why ``module``'s own forward cannot record under ``mode`` offload, else None (see GraphedForward.__init__)."""
+ try:
+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
+ safe, why = _capture_safe(type(module))
+ except Exception: # noqa: BLE001 - the stock forward is then what records
+ return None
+ if why is not None:
+ return why
+ if safe is None:
+ return None
+ if getattr(safe, "__unsloth_graph_plan__", None) is not None:
+ if getattr(safe, "__unsloth_graph_placed__", None) is None:
+ return f"{type(module).__name__} plans its graph steps but has no entry through the offload hooks"
+ return None
+ if mode == "group" and _innermost_class_forward(module) is None:
+ return (
+ f"{type(module).__name__} records only through its capture-safe forward, and the block-offload hook "
+ "chain does not end in the class forward it would replace"
+ )
+ return None
+
+
+def _plain_rewrite(module: Any) -> Any:
+ """``module``'s capture-safe class-forward rewrite when it is a plain one (not a planned step), else None."""
+ try:
+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
+ safe, _ = _capture_safe(type(module))
+ except Exception: # noqa: BLE001
+ return None
+ if safe is None or getattr(safe, "__unsloth_graph_plan__", None) is not None:
+ return None
+ return safe
+
+
+def _innermost_class_forward(module: Any) -> Any:
+ """The diffusers hook-chain reference whose ``forward`` is ``module``'s bound class forward, else None."""
+ registry = getattr(module, "_diffusers_hook", None)
+ refs = getattr(registry, "_fn_refs", None) or ()
+ if not refs:
+ return None
+ ref = refs[0]
+ if getattr(ref, "original_forward", None) is not None:
+ return None # a hook with its own new_forward owns the innermost call
+ current = getattr(ref, "forward", None)
+ owner = next((k for k in type(module).__mro__ if "forward" in vars(k)), None)
+ if owner is None or getattr(current, "__self__", None) is not module:
+ return None
+ if getattr(current, "__func__", None) is not vars(owner)["forward"]:
+ return None
+ return ref
+
+
+def arm_after_placement(
+ pipe: Any,
+ *,
+ logger: Any = None,
+ max_graphs: int = MAX_GRAPHS_PER_MODULE,
+) -> tuple:
+ """Arm the graphs once placement is final, in the slot each denoiser's offload leaves free. Returns
+ ``(handles, reason)``: the reason names what kept a denoiser eager, else what engaged."""
+ uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or (), logger = logger)
+ handles: list = []
+ refusals: list = []
+ modes: list = []
+ for module in _denoiser_modules(pipe):
+ placement, why = offload_placement(module)
+ if why is not None:
+ refusals.append(f"{type(module).__name__}: {why}")
+ continue
+ if placement is None and getattr(pipe, "_unsloth_cuda_graph_offload_only", False):
+ refusals.append(
+ f"{type(module).__name__}: the family records offloaded steps only, and it stays resident"
+ )
+ continue
+ try:
+ if placement is None:
+ kind = (
+ GraphedCompiledCall
+ if getattr(module, "_compiled_call_impl", None) is not None
+ else GraphedForward
+ )
+ handles.append(kind(module, max_graphs = max_graphs, logger = logger).enable())
+ modes.append("resident")
+ else:
+ handles.append(
+ GraphedForward(
+ module, max_graphs = max_graphs, logger = logger, placement = placement
+ ).enable()
+ )
+ modes.append(placement.mode)
+ except Exception as exc: # noqa: BLE001
+ refusals.append(f"{type(module).__name__}: {exc}")
+ installed = tuple(handles)
+ try:
+ pipe._unsloth_cuda_graphs = installed
+ except Exception as exc: # noqa: BLE001
+ _warn(logger, "handle stash", exc)
+ if refusals:
+ reason = "; ".join(refusals)
+ else:
+ labels = {
+ "resident": "resident",
+ "group": "block-streamed (copies recorded in the graph)",
+ "model": "model offload",
+ }
+ reason = "captured per input shape: " + ", ".join(sorted({labels.get(m, m) for m in modes}))
+ if logger is not None:
+ logger.info(
+ "diffusion.cuda_graph: armed on %d denoiser module(s) after placement (%s)",
+ len(installed),
+ reason,
+ )
+ return installed, reason
+
+
def _denoiser_modules(pipe: Any) -> list:
"""What the graph layer arms: every denoiser DiT, else a whole-compile-list U-Net."""
from .diffusion_speed import _denoiser_dits, _denoiser_unet
@@ -764,8 +1854,10 @@ def graph_eligible(
speed_mode: str,
family_default: bool = True,
logger: Any = None,
+ offloaded: bool = False,
) -> tuple[bool, str]:
- """Whether this load may be graphed, and a short reason when it may not. Cheapest first."""
+ """Whether this load may be graphed, and a short reason when it may not. Cheapest first. ``offloaded``: the
+ denoiser moves (graphs armed after placement), which a family can opt into on its own (``offload_cuda_graph``)."""
if cuda_graph_disabled():
return False, _disabled_reason()
@@ -811,8 +1903,10 @@ def graph_eligible(
_warn(logger, "availability probe", exc)
return False, "torch.cuda unavailable"
- if not bool(getattr(family, "supports_cuda_graph", family_default)):
- return False, "family opts out"
+ wanted = bool(getattr(family, "supports_cuda_graph", family_default))
+ wanted = wanted or (offloaded and bool(getattr(family, "offload_cuda_graph", False)))
+ if not wanted and not _family_forced(family):
+ return False, str(getattr(family, "cuda_graph_decline", None) or "family opts out")
# A denoiser whose forward syncs the host would only invalidate its capture and run eager; decline
# up front with the reason when its capture-safe rewrite did not apply.
@@ -904,9 +1998,19 @@ def stats(handles: Any) -> dict:
"eager_calls": 0,
"fallbacks": 0,
"cap_skips": 0,
+ "invalidations": 0,
+ "pool_bytes": 0,
+ "planned_eager": 0,
+ "shape_warmups": 0,
+ "evictions": 0,
+ "speed_eager": 0,
**{field: 0 for field in _REFUSALS},
+ "placements": [],
"poisoned": False,
"capture_error": None,
+ # Offloaded only: the measured eager step and replay the speed check judged (ms), once judged.
+ "eager_ms": None,
+ "replay_ms": None,
}
for handle in handles or ():
try:
@@ -917,9 +2021,20 @@ def stats(handles: Any) -> dict:
"eager_calls",
"fallbacks",
"cap_skips",
+ "invalidations",
+ "planned_eager",
+ "shape_warmups",
+ "evictions",
+ "speed_eager",
*_REFUSALS,
):
out[field] += int(handle.stats.get(field, 0))
+ out["pool_bytes"] = max(out["pool_bytes"], int(handle.stats.get("pool_bytes", 0)))
+ if out["eager_ms"] is None and handle.stats.get("eager_ms") is not None:
+ out["eager_ms"] = float(handle.stats["eager_ms"])
+ out["replay_ms"] = float(handle.stats["replay_ms"])
+ placement = getattr(handle, "placement", None)
+ out["placements"].append("resident" if placement is None else placement.mode)
if handle.poisoned:
out["poisoned"] = True
error = handle.capture_error
@@ -941,11 +2056,24 @@ def never_engaged(handles: Any) -> Optional[str]:
if reasons and len(reasons) == len(handles):
return None if any(r is None for r in reasons) else "; ".join(reasons)
s = stats(handles)
+ refused = [
+ str((h.capture_error or {}).get("msg"))
+ for h in handles
+ if (getattr(h, "capture_error", None) or {}).get("type") == "Refused"
+ ]
+ if refused and not s["captures"] and not s["replays"]:
+ return f"armed, but every denoiser call so far ran eager: {refused[0]}"
+ if refused and not s["graphs"]:
+ # Recorded, then dropped for the load: the steps run eager from here on.
+ return f"captured, then dropped: {refused[0]}; denoiser steps now run eager"
if all(getattr(h, "poisoned", False) for h in handles):
error = s["capture_error"] or {}
return f"capture failed ({error.get('type') or 'error'}); every denoiser step runs eager"
if s["captures"] or s["replays"] or not s["eager_calls"]:
return None
+ if s["eager_calls"] == s["speed_eager"]:
+ # Every eager call so far was an offloaded step timed for the speed check, before its first capture.
+ return None
parts = [
f"{s[field]} {label}"
for field, label in (
@@ -1031,6 +2159,14 @@ def arm_block_graphs(
handles = tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
whole = [h for h in handles if isinstance(h, GraphedForward)]
prior = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")
+ step = [h for h in whole if h.placement is not None or h.plan is not None]
+ if whole and len(step) == len(whole):
+ # The whole step stays primary; where per-block graphs are the default they arm for the calls it leaves eager.
+ pipe._unsloth_cuda_graph_mode = "step"
+ why = _block_basics(target, family, cache_engaged, speed_mode, family_default)
+ if why is None:
+ _attach_block_fallback(pipe, step, target, hooked, pinned, logger)
+ return handles
if not hooked and (whole or "not capture-safe" not in prior):
# Nothing moves the denoiser: the speed layer's decision stands, except a forward that is not capture-safe,
# whose blocks still record.
@@ -1115,6 +2251,41 @@ def arm_block_graphs(
return tuple(armed)
+def _attach_block_fallback(
+ pipe: Any, step: list, target: Any, hooked: bool, pinned: bool, logger: Any
+) -> None:
+ try:
+ from .diffusion_block_graph import (
+ block_graphs_disabled,
+ block_graphs_requested,
+ install_block_graphs,
+ )
+ except Exception as exc: # noqa: BLE001
+ _warn(logger, "block graph import", exc)
+ return
+ if block_graphs_disabled():
+ return
+ for handle in step:
+ model_offload = getattr(handle.placement, "mode", None) == "model"
+ stays = (not hooked or bool(pinned)) and not model_offload
+ if not (stays or block_graphs_requested()):
+ continue
+
+ def arm(module: Any = handle.module) -> Any:
+ armed, _why = install_block_graphs(
+ module, device = getattr(target, "torch_device", None), logger = logger
+ )
+ if armed is not None:
+ _set_reason(
+ pipe,
+ str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")
+ + "; the steps it leaves eager record per block",
+ )
+ return armed
+
+ handle.fallback = arm
+
+
def _set_reason(pipe: Any, reason: str) -> None:
try:
pipe._unsloth_cuda_graph_reason = reason
@@ -1125,9 +2296,15 @@ def _set_reason(pipe: Any, reason: str) -> None:
def status_reason(pipe: Any, on: bool) -> str:
"""The cuda_graph badge text: the block layer's own sentence when it armed, else why graphs are off."""
if on:
- if getattr(pipe, "_unsloth_cuda_graph_mode", None) == "blocks":
- return str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or WHOLE_REASON)
- return WHOLE_REASON
+ mode = getattr(pipe, "_unsloth_cuda_graph_mode", None)
+ reason = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")
+ if mode == "blocks":
+ return reason or WHOLE_REASON
+ head, _, tail = reason.partition("; ")
+ if mode == "step" and head.startswith("captured") and "resident" not in head:
+ # "denoiser step captured per input shape: block-streamed (copies recorded in the graph), replayed ..."
+ return f"denoiser step {head}, replayed bit-identically" + (f"; {tail}" if tail else "")
+ return WHOLE_REASON + (f"; {tail}" if mode == "step" and tail else "")
return str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "speed tier does not capture")
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 5c5543fde4c..19503fe20d3 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -678,9 +678,10 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
lives outside it and is not counted here, which is correct: it is not device memory this
generation can allocate into.
- A captured CUDA graph's pool (``diffusion_cuda_graph``) is reserved but not allocated, so it is
- credited here, yet ordinary allocations cannot reuse it while a graph holds it, so on a graphed
- load the guard reads about one step of activations high.
+ A captured CUDA graph's pool (``diffusion_cuda_graph``) is mostly reserved but not allocated, yet ordinary
+ allocations cannot reuse it while a graph holds it (on a block-streamed denoiser it also holds the
+ prefetch window), so its unallocated bytes are not credited (its allocated part is already outside the
+ difference).
Falls back to the plain snapshot on any failure or non-cuda device."""
if getattr(target, "device", "cpu") != "cuda":
@@ -703,6 +704,12 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
reclaimable -= _block_graph_pool_bytes(device)
except Exception: # noqa: BLE001
pass
+ try:
+ # The whole-step graphs' shared pool: only its unallocated part sits inside the difference above.
+ from .diffusion_cuda_graph import live_pool_free_bytes
+ reclaimable -= live_pool_free_bytes()
+ except Exception: # noqa: BLE001 -- no graph layer: nothing held
+ pass
if reclaimable <= 0:
return snapshot
free = int(snapshot.free_mib) + reclaimable // (1024 * 1024)
@@ -2665,6 +2672,8 @@ def _keep_groups_resident(
left -= need
kept += need
state["streamed"] = _streamed_group_count(ordered)
+ if kept:
+ _bump_placement_epoch()
if kept and onload.type == "cuda":
torch.cuda.synchronize(onload)
if logger is not None and kept:
@@ -2682,6 +2691,14 @@ def _keep_groups_resident(
return 0
+def _bump_placement_epoch() -> None:
+ try:
+ from .diffusion_offload_prefetch import bump_placement_epoch
+ bump_placement_epoch()
+ except Exception: # noqa: BLE001 - no graph layer to invalidate
+ pass
+
+
def _streamed_group_count(groups: list) -> int:
return sum(
1
@@ -2699,6 +2716,7 @@ def _release_group(group: Any) -> None:
group.__dict__.pop(name, None)
group._unsloth_resident = False
group.offload_()
+ _bump_placement_epoch()
def release_resident_groups(
@@ -3486,7 +3504,9 @@ class _GroupPinner:
if tensor.device.type == "cpu" and tensor.data_ptr() == src.data_ptr():
tensor.data = pinned
self.pinned += src.nbytes
+ # Done before the epoch: a capture check that sees the new epoch must also see this group ready.
self._done[id(group)].set()
+ _bump_placement_epoch()
except Exception as exc: # noqa: BLE001 - diffusers pins what is left on each onload
failed = exc
finally:
@@ -3494,6 +3514,8 @@ class _GroupPinner:
pool.shutdown(wait = True, cancel_futures = True)
for event in self._done.values():
event.set()
+ if failed is not None:
+ _bump_placement_epoch()
if self.logger is not None:
try:
if failed is not None:
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index a71189accf2..69ead084a71 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -28,6 +28,20 @@ PREFETCHER_ATTR = "_unsloth_group_prefetcher"
_BG_PIN_ATTR = "_unsloth_background_pin"
+# Bumped whenever a denoiser's device placement or a streamed group's host copy changes (residency pinned or released,
+# prefetch installed, a background pin swapping host tensors). A CUDA graph recorded over a placement replays its
+# addresses, so diffusion_cuda_graph drops graphs recorded under an older epoch.
+_PLACEMENT_EPOCH = [0]
+
+
+def bump_placement_epoch() -> None:
+ _PLACEMENT_EPOCH[0] += 1
+
+
+def placement_epoch() -> int:
+ return _PLACEMENT_EPOCH[0]
+
+
def async_prefetch_enabled() -> bool:
return (os.environ.get(ASYNC_PREFETCH_ENV) or "").strip().lower() not in (
"0",
@@ -316,6 +330,9 @@ class GroupPrefetcher:
streams = [g.stream for g in self.groups if getattr(g, "stream", None) is not None]
self.stream = streams[0] if streams else None
self.fence_first = False
+ # True while a CUDA-graph capture records this forward (diffusion_cuda_graph): the copy stream is joined into
+ # the capture at begin() and back at end(), so the copies become graph nodes refilling the same buffers.
+ self.capturing = False
self.stats = {"forwards": 0, "copies": 0, "prefetched": 0, "missed": 0, "dropped": 0}
# Slot ring (``enable_slots``): group id -> slot index; a streamed group's copy lands in place in that slot,
# always at the same offsets, so the blocks a CUDA graph recorded read the same addresses every step.
@@ -370,6 +387,8 @@ class GroupPrefetcher:
return False
pinner = getattr(group, _BG_PIN_ATTR, None)
if pinner is not None:
+ if getattr(self, "capturing", False) and not _pinned_done(pinner, group):
+ raise RuntimeError("a background pin is still swapping this group's host copies")
pinner.wait(group)
go = _go()
is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)
@@ -384,6 +403,9 @@ class GroupPrefetcher:
for i, t in enumerate(_group_tensors(group)):
src = cpu[t]
if not src.is_pinned():
+ if getattr(self, "capturing", False):
+ # a graph would replay this copy from a temporary pinned buffer reused after the call
+ raise RuntimeError("a streamed group's host copy is not pinned")
src = (
src.pin_memory()
) # diffusers' low_cpu_mem_usage path: the host allocator fences reuse
@@ -561,14 +583,64 @@ class GroupPrefetcher:
raw = self.slot_raw.get(where)
if raw is None:
- raw = torch.empty(self.slot_size, dtype = torch.uint8, device = self.device)
- self.slot_raw[where] = raw
- self.slot_bytes += self.slot_size
+ raw = self._alloc_slot(where)
cpu = group.cpu_param_dict
views = _slot_views(raw, [cpu[t] for t in _group_tensors(group)], self.device)
self.slot_buffers[gid] = views
return views
+ def _alloc_slot(self, where: int) -> Any:
+ import torch
+
+ raw = torch.empty(self.slot_size, dtype = torch.uint8, device = self.device)
+ self.slot_raw[where] = raw
+ self.slot_bytes += self.slot_size
+ bump_placement_epoch() # a whole-step graph keys its recorded copies on the ring's addresses
+ return raw
+
+ def materialize_slots(self) -> None:
+ """Allocate every slot of the ring now (a whole-step capture must not carve one out of its graph pool)."""
+ if not self.slot_of or not self.slot_size:
+ return
+ # Only the slots a streamed group fills: a resident group that is released later allocates its own on use.
+ streamed = {
+ self.slot_of[id(g)]
+ for g in self.groups
+ if id(g) in self.slot_of and not getattr(g, "_unsloth_resident", False)
+ }
+ for where in sorted(streamed):
+ if where not in self.slot_raw:
+ self._alloc_slot(where)
+
+ def drop_slots_after_forward(self) -> None:
+ """Drop the ring once the current forward ends (``end``), or now between forwards."""
+ if self.active:
+ self._drop_slots_at_end = True
+ return
+ self._drop_ring()
+
+ def _drop_ring(self) -> None:
+ import torch
+
+ from .diffusion_cuda_graph import hold_off_capture
+ with hold_off_capture() as safe:
+ if not safe:
+ # Another thread is recording: its capture prohibits these syncs and frees. Retry after a later forward.
+ self._drop_slots_at_end = True
+ return
+ self._drop_slots_at_end = False
+ self.disable_slots()
+ try:
+ import gc
+
+ gc.collect() # a declined step graph's recordings, whose pool is flushed below with the ring
+ clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)
+ if callable(clear):
+ clear() # and the workspace cuBLAS made for their capture stream
+ torch.cuda.empty_cache() # the ring's segments go back to the device, not just to the cache
+ except Exception: # noqa: BLE001
+ pass
+
def disable_slots(self) -> None:
"""Drop the ring. Only between forwards: every streamed group is back on the host by then."""
import torch
@@ -577,6 +649,7 @@ class GroupPrefetcher:
return
if self.slot_raw:
torch.cuda.synchronize(self.device)
+ bump_placement_epoch()
self.slot_of = {}
self.slot_buffers = {}
self.slot_raw = {}
@@ -594,6 +667,13 @@ class GroupPrefetcher:
self.active = True
self.on_order = bool(self.order)
self.pending = True
+ self.capturing = False
+ if self.stream is not None:
+ import torch
+ if torch.cuda.is_current_stream_capturing():
+ # fork the copy stream into the capture before anything is queued on it
+ self.capturing = True
+ self.stream.wait_stream(self._compute())
def kick(self) -> None:
"""First block onload of a forward (streamed or resident): queue the first ``depth`` streamed groups."""
@@ -620,8 +700,34 @@ class GroupPrefetcher:
self._release(group, self.ready.pop(gid))
self.ready.clear()
self.inflight_bytes = 0
+ if getattr(self, "capturing", False):
+ self.capturing = False
+ self._compute().wait_stream(
+ self.stream
+ ) # join the copy stream back before the capture ends
if self.seen:
self.order = list(self.seen)
+ if getattr(self, "_drop_slots_at_end", False):
+ self._drop_ring()
+
+ def abandon(self) -> None:
+ """A capture failed mid-forward: the events and device copies it queued died with it. Forget them (never wait
+ on a captured event outside its graph) and put every streamed group back on its host copy. Outside capture."""
+ import torch
+
+ self.active = self.on_order = self.pending = self.capturing = False
+ self.ready.clear()
+ self.inflight_bytes = 0
+ try:
+ torch.cuda.synchronize(self.device)
+ except Exception: # noqa: BLE001 - the copies below are what matters
+ pass
+ for group in self.groups:
+ if self.owns(group):
+ try:
+ type(group).offload_(group)
+ except Exception: # noqa: BLE001 - a group the capture never reached is already on the host
+ pass
def _adopt_top_group(
@@ -819,6 +925,7 @@ def install_group_prefetch(
module.register_forward_pre_hook(_eager(lambda *_a, **_k: pf.begin()))
module.register_forward_hook(_eager(lambda *_a, **_k: pf.end()), always_call = True)
setattr(module, PREFETCHER_ATTR, pf)
+ bump_placement_epoch()
state = getattr(module, "_unsloth_stream_state", None)
if not isinstance(state, dict):
state = {"streamed": 1}
@@ -848,3 +955,41 @@ def install_group_prefetch(
def module_prefetcher(module: Any) -> Optional[GroupPrefetcher]:
return getattr(module, PREFETCHER_ATTR, None)
+
+
+def _pinned_done(pinner: Any, group: Any) -> bool:
+ event = getattr(pinner, "_done", {}).get(id(group))
+ return event is None or event.is_set()
+
+
+def capture_refusal(module: Any) -> Optional[str]:
+ """Why a CUDA graph cannot record ``module``'s block-streamed forward with its copies inside, else None.
+
+ A replay repeats every copy from the host address it recorded into the device buffer it recorded, so every
+ streamed group must copy from a pinned host tensor no background pin is still replacing, and every group must be
+ driven by the prefetcher, pinned resident, or the dense top-level group's pinned upload."""
+ pf = module_prefetcher(module)
+ if pf is None:
+ return "block streaming without the event-fenced prefetch (its stream waits are host synchronizations)"
+ try:
+ from .diffusion_memory import _offload_groups
+ for group in _offload_groups(module):
+ if getattr(group, "offload_to_disk_path", None):
+ return "a group streams from disk"
+ if getattr(group, "_unsloth_resident", False) or getattr(
+ group, "_unsloth_pinned_top", False
+ ):
+ continue
+ if not pf.owns(group):
+ return "a streamed group is not driven by the prefetcher"
+ pinner = getattr(group, _BG_PIN_ATTR, None)
+ if pinner is not None and not _pinned_done(pinner, group):
+ return "host copies are still being pinned in the background"
+ cpu = getattr(group, "cpu_param_dict", None) or {}
+ for t in _group_tensors(group):
+ src = cpu.get(t)
+ if src is None or not src.is_pinned():
+ return "a streamed group's host copy is not pinned"
+ except Exception as exc: # noqa: BLE001 - unknown layout: never capture it
+ return f"offload layout unreadable ({type(exc).__name__})"
+ return None
diff --git a/studio/backend/core/inference/diffusion_qwenimage21.py b/studio/backend/core/inference/diffusion_qwenimage21.py
index c9b7b690cc4..d255b6f4774 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21.py
@@ -115,6 +115,7 @@ class _Layout:
"vlm_row",
"_segments",
"_vlm_text",
+ "_tails",
"__weakref__",
)
@@ -136,6 +137,7 @@ def _build_layout(model: Any, mod: Any, img_mask: Any, shapes: list, device: Any
lay.vlm_row = img_mask[0].clone() # the caller may reuse and rewrite its mask
lay._segments = None
lay._vlm_text = {}
+ lay._tails = None
return lay
@@ -199,26 +201,38 @@ def _layout_for(
return lay
-def _cached_step(
+def _tails(lay: _Layout) -> tuple:
+ """The decode rows of the RoPE table and modulation mask; one view per layout, so a graph replay can tell by
+ identity that its copy is current."""
+ if lay._tails is None:
+ lay._tails = (lay.rotary_emb[lay.prefix_len :], lay.target_token_mask[lay.prefix_len :])
+ return lay._tails
+
+
+def _cached_core(
model: Any,
- lay: _Layout,
text_dtype: Any,
+ total: int,
+ prefix_len: int,
hidden_states: Any,
timestep: Any,
encoder_hidden_states_mask: Any,
- kv_cache: Any,
+ text_positions: Any,
+ vlm_text: Any,
+ rotary_emb: Any,
+ modulation_mask: Any,
+ layer_cache: Any,
) -> Any:
+ """One decode step from the prefix KV cache. Reads no host value off the device, so it records as a CUDA graph;
+ the eager step and the graph run this same code. ``layer_cache(i)`` returns block ``i``'s cache entry."""
import torch
batch_size = hidden_states.shape[0]
device = hidden_states.device
- prefix_len = lay.prefix_len
hidden_states = model.img_in(hidden_states)
# Keep the stock (batch, total, dim) layout so the blocks see the same strides.
- full = torch.empty(
- (batch_size, lay.total, hidden_states.shape[2]), dtype = text_dtype, device = device
- )
- full[:, prefix_len:] = hidden_states[:, hidden_states.shape[1] - (lay.total - prefix_len) :]
+ full = torch.empty((batch_size, total, hidden_states.shape[2]), dtype = text_dtype, device = device)
+ full[:, prefix_len:] = hidden_states[:, hidden_states.shape[1] - (total - prefix_len) :]
timestep = timestep.to(hidden_states.dtype)
timestep = torch.cat([timestep, timestep.new_zeros(1)], dim = 0)
@@ -227,15 +241,11 @@ def _cached_step(
attention_mask = None
if encoder_hidden_states_mask is not None:
- joint_key_valid = torch.ones(batch_size, lay.total, dtype = torch.bool, device = device)
- joint_key_valid[:, lay.text_positions] = encoder_hidden_states_mask.bool()[
- :, _vlm_text(lay, encoder_hidden_states_mask.shape[1])
- ]
+ joint_key_valid = torch.ones(batch_size, total, dtype = torch.bool, device = device)
+ joint_key_valid[:, text_positions] = encoder_hidden_states_mask.bool()[:, vlm_text]
attention_mask = joint_key_valid[:, None, None, :]
joint_hidden_states = full[:, prefix_len:]
- rotary_emb = lay.rotary_emb[prefix_len:]
- modulation_mask = lay.target_token_mask[prefix_len:]
for index_block, block in enumerate(model.transformer_blocks):
joint_hidden_states = block(
hidden_states = joint_hidden_states,
@@ -243,7 +253,7 @@ def _cached_step(
rotary_emb = rotary_emb,
attention_mask = attention_mask,
target_token_mask = modulation_mask,
- layer_cache = kv_cache.get_layer(index_block),
+ layer_cache = layer_cache(index_block),
kv_cache_mode = "cached",
cache_write_slice = None,
segments = None,
@@ -253,6 +263,202 @@ def _cached_step(
return model.proj_out(joint_hidden_states)
+def _cached_step(
+ model: Any,
+ lay: _Layout,
+ text_dtype: Any,
+ hidden_states: Any,
+ timestep: Any,
+ encoder_hidden_states_mask: Any,
+ kv_cache: Any,
+) -> Any:
+ rotary_emb, modulation_mask = _tails(lay)
+ return _cached_core(
+ model,
+ text_dtype,
+ lay.total,
+ lay.prefix_len,
+ hidden_states,
+ timestep,
+ encoder_hidden_states_mask,
+ lay.text_positions,
+ None
+ if encoder_hidden_states_mask is None
+ else _vlm_text(lay, encoder_hidden_states_mask.shape[1]),
+ rotary_emb,
+ modulation_mask,
+ kv_cache.get_layer,
+ )
+
+
+# ---------------------------------------------------------------------------------------------- CUDA graph step
+#
+# ``diffusion_cuda_graph.GraphedForward`` records one call into static buffers keyed by every input's shape. The
+# stock call cannot be recorded: the pipeline hands the prefix K/V over as a ``QwenImage21KVCache`` object (an opaque
+# leaf to the graph layer) and step 0 fills that cache from Python. ``graph_plan`` turns a decode step into a call
+# whose every input is a tensor or a constant: the per-layer K/V, the layout's index and RoPE tensors, latents,
+# timestep and text mask. Step 0 (the prefill) stays eager.
+#
+# The K/V, RoPE and index inputs only change when a new render (or a new layout) starts, so they are "sticky": the
+# replay copies one into its static buffer only when the caller passes a different tensor object than last time.
+
+GRAPH_STICKY = ("text_positions", "vlm_text", "rotary_emb", "modulation_mask", "kv")
+
+# A graph keeps its own copy of the prefix K/V (one per graph, up to the per-module cap). Text-only prefixes stay far
+# below this; a 1 MP condition image is ~2.2 GiB at 32 blocks, and those steps run eager rather than pin that twice.
+GRAPH_KV_MAX_BYTES = 1 << 30
+
+
+def _graph_step(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states_mask,
+ text_positions,
+ vlm_text,
+ rotary_emb,
+ modulation_mask,
+ kv,
+ total,
+ prefix_len,
+ text_dtype,
+):
+ import torch
+
+ mod = _module()
+ caches = []
+ for key, value in kv:
+ entry = mod.QwenImage21KVLayerCache()
+ entry.store(key, value)
+ caches.append(entry)
+ output = _cached_core(
+ self,
+ getattr(torch, text_dtype),
+ total,
+ prefix_len,
+ hidden_states,
+ timestep,
+ encoder_hidden_states_mask,
+ text_positions,
+ vlm_text,
+ rotary_emb,
+ modulation_mask,
+ caches.__getitem__,
+ )
+ return (output,)
+
+
+# Under block offload the graph wrapper sits in the forward slot under the offload hooks, which onload the top-level
+# weights: a planned step enters through the hooked forward as ``kv_cache_mode = GRAPH_STEP_MODE`` with the plan as
+# ``kv_cache`` (the forward's signature stays the pipeline's).
+GRAPH_STEP_MODE = "unsloth_graph_step"
+
+
+def placed_step(forward: Any, step: dict) -> Any:
+ return forward(
+ step["hidden_states"],
+ None,
+ step["timestep"],
+ None,
+ None,
+ kv_cache = step,
+ kv_cache_mode = GRAPH_STEP_MODE,
+ return_dict = False,
+ )
+
+
+def _module() -> Any:
+ import importlib
+ return importlib.import_module(_MODULE)
+
+
+def graph_plan(module: Any, args: tuple, kwargs: dict) -> Optional[tuple]:
+ """``(callable, kwargs)`` replaying this decode step from tensors only, or None to run the call eager.
+
+ Never syncs the host on a decode step of a render whose prefill went through the fast forward (the layout is
+ then found by identity). Never raises."""
+ try:
+ return _graph_plan(module, args, kwargs)
+ except Exception: # noqa: BLE001 - an unplannable call simply runs eager
+ return None
+
+
+def _graph_plan(module: Any, args: tuple, kwargs: dict) -> Optional[tuple]:
+ import torch
+
+ if (
+ args
+ or kwargs.get("kv_cache_mode") != "cached"
+ or kwargs.get("return_dict", True) is not False
+ ):
+ return None
+ kv_cache = kwargs.get("kv_cache")
+ hidden_states = kwargs.get("hidden_states")
+ encoder_hidden_states = kwargs.get("encoder_hidden_states")
+ timestep = kwargs.get("timestep")
+ img_mask = kwargs.get("img_mask")
+ img_shapes = kwargs.get("img_shapes")
+ mask = kwargs.get("encoder_hidden_states_mask")
+ known = {
+ "hidden_states",
+ "encoder_hidden_states",
+ "timestep",
+ "img_shapes",
+ "img_mask",
+ "encoder_hidden_states_mask",
+ "attention_kwargs",
+ "kv_cache",
+ "kv_cache_mode",
+ "return_dict",
+ }
+ # LoRA scale rides in attention_kwargs through the forward's decorator, which the graph step bypasses.
+ if set(kwargs) - known or kwargs.get("attention_kwargs"):
+ return None
+ if kv_cache is None or not torch.is_tensor(hidden_states) or not torch.is_tensor(timestep):
+ return None
+ if (
+ not torch.is_tensor(encoder_hidden_states)
+ or not torch.is_tensor(img_mask)
+ or img_shapes is None
+ ):
+ return None
+ if torch.is_grad_enabled() or fast_step_disabled() or not module.config.causal_condition:
+ return None
+ if not getattr(vars(type(module)).get("forward"), "__unsloth_q21_fast_step__", False):
+ return None
+ mod = _module()
+ lay = _layout_for(module, mod, img_mask, img_shapes, hidden_states.device, reuse_identity = True)
+ _, text_dtype = _text_dtype(module, encoder_hidden_states)
+ if text_dtype is None or not lay.tail_is_image:
+ return None
+ kv = []
+ kv_bytes = 0
+ for index in range(len(module.transformer_blocks)):
+ entry = kv_cache.get_layer(index)
+ key, value = getattr(entry, "k", None), getattr(entry, "v", None)
+ if not torch.is_tensor(key) or not torch.is_tensor(value):
+ return None
+ kv_bytes += key.numel() * key.element_size() + value.numel() * value.element_size()
+ kv.append((key, value))
+ if kv_bytes > GRAPH_KV_MAX_BYTES:
+ return None
+ rotary_emb, modulation_mask = _tails(lay)
+ plan = {
+ "hidden_states": hidden_states,
+ "timestep": timestep,
+ "encoder_hidden_states_mask": mask,
+ "text_positions": lay.text_positions if mask is not None else None,
+ "vlm_text": _vlm_text(lay, mask.shape[1]) if mask is not None else None,
+ "rotary_emb": rotary_emb,
+ "modulation_mask": modulation_mask,
+ "kv": tuple(kv),
+ "total": int(lay.total),
+ "prefix_len": int(lay.prefix_len),
+ "text_dtype": str(text_dtype).rsplit(".", 1)[-1],
+ }
+ return _graph_step.__get__(module), plan
+
+
def _autocast_state(device_type: str) -> tuple:
"""Autocast decides the text projection's output dtype as much as the weights do."""
import torch
@@ -294,6 +500,8 @@ def _make_forward(mod: Any, stock: Any) -> Any:
kv_cache_mode = None,
return_dict = True,
):
+ if kv_cache_mode == GRAPH_STEP_MODE:
+ return _graph_step(self, **kv_cache)
if torch.is_grad_enabled() or torch.compiler.is_compiling() or fast_step_disabled():
return stock_inner(
self,
@@ -430,6 +638,10 @@ def _make_forward(mod: Any, stock: Any) -> Any:
wrapped = lora_scale("attention_kwargs")(forward) if callable(lora_scale) else forward
wrapped.__unsloth_q21_fast_step__ = True
wrapped.__unsloth_stock_forward__ = stock
+ # Read by diffusion_capture_safe: decode steps record as CUDA graphs through graph_plan.
+ wrapped.__unsloth_graph_plan__ = graph_plan
+ wrapped.__unsloth_graph_sticky__ = GRAPH_STICKY
+ wrapped.__unsloth_graph_placed__ = placed_step
return wrapped
@@ -458,8 +670,7 @@ def install(logger: Any = None) -> bool:
if fast_step_disabled():
return False
try:
- import importlib
- mod = importlib.import_module(_MODULE)
+ mod = _module()
except Exception: # noqa: BLE001 - diffusers without Qwen-Image 2.1
return False
cls = getattr(mod, _CLASS, None)
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index f5230978fe2..1f0e25fa407 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -587,20 +587,33 @@ def apply_speed_optims(
if mode in (SPEED_DEFAULT, SPEED_MAX):
cuda_graph = None
ok, reason = False, "cuda graph layer unavailable"
+ moved = bool(offload_active if denoiser_offloaded is None else denoiser_offloaded)
try:
from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle
+
+ after_placement = moved and cuda_graph.offload_graphs_enabled()
ok, reason = cuda_graph.graph_eligible(
target,
family = family,
pipe = pipe,
- offload_active = offload_active
- if denoiser_offloaded is None
- else bool(denoiser_offloaded),
+ offload_active = moved and not after_placement,
cache_active = cache_active if cache_engaged is None else bool(cache_engaged),
speed_mode = mode,
family_default = cuda_graph_default,
logger = logger,
+ offloaded = after_placement,
)
+ if ok and after_placement:
+ # The slot and the recorded copies depend on the hooks placement installs: arm_graphs_after_placement.
+ ok, reason = False, "armed after placement"
+ try:
+ pipe._unsloth_cuda_graph_after_placement = True
+ # A family that opts in only offloaded must not get a resident graph if the denoiser stays put.
+ pipe._unsloth_cuda_graph_offload_only = not bool(
+ getattr(family, "supports_cuda_graph", cuda_graph_default)
+ )
+ except Exception: # noqa: BLE001
+ pass
except Exception as exc: # noqa: BLE001 - an unimportable graph layer means eager, never a failed load
_warn(logger, "cuda graph eligibility", exc)
# Stashed either way: status reports WHY graphs are off, not just that they are.
@@ -625,6 +638,51 @@ def _onload_device(target: Any) -> Optional[str]:
return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"
+def arm_graphs_after_placement(
+ pipe: Any,
+ applied: dict,
+ logger: Any = None,
+) -> dict:
+ """Arm the denoiser graphs for the placement that actually happened: replaces a resident-planned graph a hook now
+ wraps, and arms the one apply_speed_optims deferred for an offloaded denoiser (the copies are recorded in it)."""
+ pending = bool(getattr(pipe, "_unsloth_cuda_graph_after_placement", False))
+ hooked = bool(applied.get("cuda_graph")) and _denoiser_moves(pipe)
+ if not (pending or hooked):
+ return applied
+ try:
+ from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle
+
+ if not cuda_graph.offload_graphs_enabled():
+ cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
+ pipe._unsloth_cuda_graphs = ()
+ pipe._unsloth_cuda_graph_reason = "offload active"
+ applied["cuda_graph"] = False
+ return applied
+ handles, reason = cuda_graph.arm_after_placement(pipe, logger = logger)
+ pipe._unsloth_cuda_graph_after_placement = False
+ pipe._unsloth_cuda_graph_reason = reason
+ applied["cuda_graph"] = bool(handles)
+ except Exception as exc: # noqa: BLE001 - the load proceeds eager
+ _warn(logger, "cuda graph arm after placement", exc)
+ applied["cuda_graph"] = False
+ return applied
+
+
+def _denoiser_moves(pipe: Any) -> bool:
+ for module in _denoiser_dits(pipe) or [m for m in [_denoiser_unet(pipe)] if m is not None]:
+ if getattr(module, "_hf_hook", None) is not None:
+ return True
+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
+ if any("offload" in str(key) for key in hooks):
+ return True
+ try:
+ if any(getattr(m, "_hf_hook", None) is not None for m in module.modules()):
+ return True
+ except Exception: # noqa: BLE001
+ return True
+ return False
+
+
def engage_pinned_denoisers(
pipe: Any,
applied: dict,
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 444c35d582b..120b379d935 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -135,6 +135,7 @@ from .diffusion_speed import (
SPEED_MAX,
SPEED_OFF,
apply_speed_optims,
+ arm_graphs_after_placement,
resolve_speed_mode,
restore_backend_flags,
settle_compile_fallback,
@@ -6649,6 +6650,14 @@ class VideoBackend:
speed_optims += ("vae_untiled_when_fits",)
# Wan's decode also grows within a single tile, which tiling alone cannot bound.
install_decoder_sync(pipe, target, logger = logger)
+ # A family that opts in to graphs arms them against the placement that landed (offload hooks included).
+ graph_applied = arm_graphs_after_placement(
+ pipe, {"cuda_graph": "cuda_graph" in speed_optims}, logger
+ )
+ if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:
+ speed_optims += ("cuda_graph",)
+ elif not graph_applied.get("cuda_graph"):
+ speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")
# Last, so the bf16 entry wraps whatever decode path the steps above installed. Not on SPEED_OFF.
if getattr(fam, "vae_force_fp32", False) and effective_speed != SPEED_OFF:
vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)
@@ -6679,6 +6688,14 @@ class VideoBackend:
cache_reason,
RESOLVED_UNSUPPORTED if static_decline else None,
),
+ "cuda_graph": (
+ None,
+ "on" if "cuda_graph" in speed_optims else "off",
+ str(
+ getattr(pipe, "_unsloth_cuda_graph_reason", None)
+ or "speed tier does not capture"
+ ),
+ ),
"transformer_quant": (
transformer_quant_requested,
transformer_quant_engaged or "off",
@@ -7647,6 +7664,23 @@ class VideoBackend:
)
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)
+ # H3 declines graphs (cuda_graph_decline), so this only acts when a family override forces them: the graph was
+ # deferred to the placement above, so arm it against the hooks that landed, or record why it stays eager,
+ # rather than leaving the pending note in the status.
+ graph_applied = arm_graphs_after_placement(
+ speed_view, {"cuda_graph": "cuda_graph" in speed_optims}, logger
+ )
+ if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:
+ speed_optims += ("cuda_graph",)
+ elif not graph_applied.get("cuda_graph"):
+ speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")
+ if speed_view is not pipe:
+ for attr in ("_unsloth_cuda_graph_reason", "_unsloth_cuda_graphs"):
+ if hasattr(speed_view, attr):
+ try:
+ setattr(pipe, attr, getattr(speed_view, attr))
+ except Exception: # noqa: BLE001
+ pass
# nothing here compiles, so it follows the REQUESTED tier, not the denoiser's eager downgrade above
try:
from .video_minimax_h3_vae import apply_h3_vae_speedups
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 02d5a3f69e5..714712b24c3 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -83,6 +83,10 @@ class VideoFamily:
# Opt-in to the CUDA-graph capture of the denoiser forward (diffusion_cuda_graph.py): only for a family that runs
# ONE forward per step with no CFG and no step cache, so a capture has one stable input tree to replay.
supports_cuda_graph: bool = False
+ # Opt-in to the capture when the denoiser is offloaded: the replay then records the onloads too (set from an A/B).
+ offload_cuda_graph: bool = False
+ # Status text when the family declines the capture (the measured reason), instead of "family opts out".
+ cuda_graph_decline: Optional[str] = None
# Video DiTs are bf16-native, so fp16 promotes to float32; defaults True.
fp16_incompatible: bool = True
# "native" = measured accurate in plain float16 on fp16-only cards; None = promote (unmeasured).
@@ -207,6 +211,12 @@ _FAMILIES: tuple[VideoFamily, ...] = (
# capture peak, on a family already needing 87.5 GB, and 31.1 s on the first render (break-even ~530 videos).
# This is the CHEAPEST grid H3 ships, so the larger presets can only be more GPU-bound.
supports_cuda_graph = False,
+ # Streamed, H3's prequant groups onload through diffusers' stream path, which waits on the host per group: a
+ # capture cannot hold them (arm_after_placement refuses it by name when the family is forced).
+ cuda_graph_decline = (
+ "measured GPU-bound: 1.0018x resident for 3.93 GB held; streamed, its group onloads wait on the host "
+ "(27 per step at 16 GB), which a graph cannot record"
+ ),
gguf_repo = "unsloth/MiniMax-H3-GGUF",
# Hosted pre-quantized FL2VA denoisers. The modular workflow builds each component through its own
# from_pretrained, so there is no dense module to quantise in place: these are the ONLY way to run the 66.3 GB
@@ -301,6 +311,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
bf16_components_gb = (10.0, 11.4, 2.8),
vae_force_fp32 = True,
cudnn_benchmark = False,
+ # Streamed: faster than eager with bit-identical latents; two replays per step (CFG), one graph per shape.
+ offload_cuda_graph = True,
# UMT5 keeps its overflowing `wo` in fp32 itself; the VAE stays fp32 (vae_force_fp32).
fp16_guard = "native",
# Byte-identical mirror of QuantStack/Wan2.2-TI2V-5B-GGUF (13 quants + companion VAE).
@@ -377,6 +389,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
bf16_components_gb = (16.6, 14.8, 2.4),
fp16_guard = "native",
cudnn_benchmark = False,
+ # Needs the capture-safe forward (diffusion_capture_safe) and the memoized trim (diffusion_attention).
+ offload_cuda_graph = True,
),
# The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its
# own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.
diff --git a/studio/backend/tests/test_diffusion_attention_trim.py b/studio/backend/tests/test_diffusion_attention_trim.py
index 0f816ae11f1..904916e088d 100644
--- a/studio/backend/tests/test_diffusion_attention_trim.py
+++ b/studio/backend/tests/test_diffusion_attention_trim.py
@@ -250,3 +250,85 @@ def test_post_hook_always_clears_flag_after_forward_and_on_exception():
with pytest.raises(RuntimeError):
dit()
assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is False for b in dit.transformer_blocks)
+
+
+def _t2v_kwargs(device = "cpu"):
+ return {
+ "image_embeds": torch.zeros(1, 5, 3, device = device),
+ "encoder_hidden_states": torch.arange(8.0, device = device).reshape(1, 4, 2),
+ "encoder_attention_mask": torch.tensor([[1, 1, 0, 0]], device = device),
+ "encoder_hidden_states_2": torch.arange(3.0, device = device).reshape(1, 3, 1),
+ "encoder_attention_mask_2": torch.tensor([[1, 0, 0]], device = device),
+ }
+
+
+def test_trim_pre_hook_plans_once_per_input_tensors_and_matches_the_stock_trim():
+ # A pipeline hands the same prompt tensors to every step: the decisions (host reads) are made once and reused.
+ dit = _fake_dit()
+ base = _t2v_kwargs()
+ first = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ second = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ for key in base:
+ assert torch.equal(first[key], second[key])
+ want_s, want_m, _ = att._trim_stream(
+ base["encoder_hidden_states"], base["encoder_attention_mask"]
+ )
+ assert torch.equal(second["encoder_hidden_states"], want_s) and torch.equal(
+ second["encoder_attention_mask"], want_m
+ )
+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 1
+ # an edited mask (version bump) or a new tensor plans afresh
+ base["encoder_attention_mask"][0, 2] = 1
+ third = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ assert third["encoder_hidden_states"].shape == (1, 3, 2)
+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 2
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_trim_pre_hook_makes_no_host_wait_after_the_first_step():
+ import warnings
+
+ dit = _fake_dit()
+ base = _t2v_kwargs("cuda")
+ att._hunyuan_trim_pre_hook(dit, (), dict(base)) # step 0 reads the host
+ torch.cuda.synchronize()
+ prev = torch.cuda.get_sync_debug_mode()
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ out = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ finally:
+ torch.cuda.set_sync_debug_mode(prev)
+ assert not [w for w in caught if "synchroniz" in str(w.message).lower()]
+ assert out["encoder_hidden_states"].shape == (1, 2, 2)
+ assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is True for b in dit.transformer_blocks)
+
+
+def test_the_null_mask_flag_keys_the_cuda_graph():
+ # The flag picks the attention branch inside the forward: the graph layer keys on it (GRAPH_KEY_EXTRA_ATTR).
+ import core.inference.diffusion_cuda_graph as cg
+
+ dit = _fake_dit()
+ dit.transformer_blocks[0].attn.processor = None
+ assert att._hunyuan_null_mask_state(dit) is False
+ att._set_hunyuan_null_mask(dit, True)
+ assert att._hunyuan_null_mask_state(dit) is True
+ assert cg.GRAPH_KEY_EXTRA_ATTR == "_unsloth_graph_key_extra"
+
+
+def test_trim_pre_hook_trims_and_plans_once_for_inference_tensors():
+ # Renders run under torch.inference_mode: an inference tensor has no version counter (reading one raises), and
+ # the hook must still trim (not fall back to the untrimmed inputs) and reuse its plan.
+ dit = _fake_dit()
+ with torch.inference_mode():
+ base = _t2v_kwargs()
+ out = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ again = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]
+ assert out["image_embeds"].shape == (1, 0, 3)
+ assert out["encoder_hidden_states"].shape == (1, 2, 2) and out[
+ "encoder_hidden_states_2"
+ ].shape == (1, 1, 1)
+ assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is True for b in dit.transformer_blocks)
+ assert torch.equal(out["encoder_hidden_states"], again["encoder_hidden_states"])
+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 1
diff --git a/studio/backend/tests/test_diffusion_capture_safe.py b/studio/backend/tests/test_diffusion_capture_safe.py
index cf6f8fe656a..46bbfe109be 100644
--- a/studio/backend/tests/test_diffusion_capture_safe.py
+++ b/studio/backend/tests/test_diffusion_capture_safe.py
@@ -404,7 +404,8 @@ def test_qwen_image_21_is_declined_with_the_reason(fresh_cache):
assert forward is None
assert why == (
"QwenImage21Transformer2DModel forward is not capture-safe (its pipeline passes the prefix KV "
- "cache as a Python object on every step and its forward syncs the host)"
+ "cache as a Python object on every step; the Studio fast step that replays decode steps from "
+ "tensors is not installed)"
)
Sub = type("Sub", (QwenImage21Transformer2DModel,), {"forward": lambda self, x: x})
assert cs.resolve(Sub) == (None, None)
diff --git a/studio/backend/tests/test_diffusion_capture_safe_hv15.py b/studio/backend/tests/test_diffusion_capture_safe_hv15.py
new file mode 100644
index 00000000000..6f49452efac
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_capture_safe_hv15.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""HunyuanVideo-1.5's capture-safe forward (``diffusion_capture_safe``).
+
+CPU: a tiny random ``HunyuanVideo15Transformer3DModel`` run through the stock and the rewritten
+forward must agree bit for bit over t2v (all-zero and emptied image stream) and i2v, batch 1 and 2,
+partially padded masks, and the inputs Studio's eager trim pre-hook hands the forward. The rewritten
+forward must also dispatch no op that reads a device value on the host.
+
+CUDA: the rewritten forward records into a ``torch.cuda.CUDAGraph`` and replays bit-identical to
+eager; the stock forward does not record.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_capture_safe as cs
+from core.inference import diffusion_cuda_graph as cg
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+
+_CLS = getattr(diffusers, "HunyuanVideo15Transformer3DModel", None)
+pytestmark = pytest.mark.skipif(
+ _CLS is None, reason = "diffusers has no HunyuanVideo15Transformer3DModel"
+)
+
+_TEXT_DIM, _BYT5_DIM, _IMAGE_DIM = 32, 24, 16
+
+
+def fresh_cache(monkeypatch):
+ monkeypatch.setattr(cs, "_CACHE", {})
+ monkeypatch.delenv(cs.CAPTURE_SAFE_ENV, raising = False)
+
+
+def _tiny_model(device = "cpu"):
+ torch.manual_seed(0)
+ model = _CLS(
+ in_channels = 4,
+ out_channels = 4,
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ num_layers = 1,
+ num_refiner_layers = 1,
+ text_embed_dim = _TEXT_DIM,
+ text_embed_2_dim = _BYT5_DIM,
+ image_embed_dim = _IMAGE_DIM,
+ rope_axes_dim = (4, 6, 6),
+ )
+ # Random (non-zero-init) weights everywhere, so no path is trivially zero.
+ with torch.no_grad():
+ for p in model.parameters():
+ p.copy_(torch.randn_like(p) * 0.2)
+ return model.to(device).eval()
+
+
+# (mllm mask rows, byt5 mask rows): right-padded, interleaved, fully valid, byt5 fully padded.
+_MASKS = {
+ "b1_right_padded": ([[1, 1, 1, 0, 0, 0, 0]], [[1, 1, 0, 0, 0]]),
+ "b2_interleaved": (
+ [[1, 0, 1, 1, 0, 0, 1], [1, 1, 0, 0, 0, 0, 0]],
+ [[0, 1, 0, 1, 1], [1, 1, 1, 0, 0]],
+ ),
+ "b2_valid_and_empty_byt5": ([[1] * 7, [1, 1, 1, 1, 0, 0, 0]], [[1] * 5, [0] * 5]),
+}
+_IMAGES = ("t2v_zero", "t2v_empty", "i2v")
+
+
+def _inputs(
+ masks,
+ image,
+ device = "cpu",
+ seed = 1,
+):
+ gen = torch.Generator().manual_seed(seed)
+ m1 = torch.tensor(masks[0])
+ m2 = torch.tensor(masks[1])
+ batch = m1.shape[0]
+ if image == "t2v_zero":
+ img = torch.zeros(batch, 3, _IMAGE_DIM)
+ elif image == "t2v_empty":
+ img = torch.zeros(batch, 0, _IMAGE_DIM)
+ else:
+ img = torch.randn(batch, 3, _IMAGE_DIM, generator = gen)
+ kw = dict(
+ hidden_states = torch.randn(batch, 4, 2, 4, 4, generator = gen),
+ timestep = torch.full((batch,), 500.0),
+ encoder_hidden_states = torch.randn(batch, m1.shape[1], _TEXT_DIM, generator = gen),
+ encoder_attention_mask = m1,
+ encoder_hidden_states_2 = torch.randn(batch, m2.shape[1], _BYT5_DIM, generator = gen),
+ encoder_attention_mask_2 = m2,
+ image_embeds = img,
+ return_dict = False,
+ )
+ return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in kw.items()}
+
+
+def _trimmed(model, kw):
+ """The kwargs Studio's eager trim pre-hook hands the forward (empty t2v image, trimmed text)."""
+ from core.inference import diffusion_attention as da
+
+ _, out = da._hunyuan_trim_pre_hook(model, (), dict(kw))
+ da._set_hunyuan_null_mask(model, False)
+ return out
+
+
+def test_hv15_rewrite_is_registered_and_resolves(fresh_cache):
+ safe, why = cs.resolve(_CLS)
+ assert why is None, why
+ assert getattr(safe, "__unsloth_capture_safe__", False) is True
+ import inspect
+
+ assert inspect.signature(safe) == inspect.signature(_CLS.forward)
+ src = inspect.getsource(safe)
+ assert cs._HV15_MERGE_HELPER_NAME in src and cs._HV15_T2V_HELPER_NAME in src
+ assert "if is_t2v:" not in src and "image[image_mask]" not in src
+
+
[email protected]("trim", [False, True], ids = ["untrimmed", "trimmed"])
[email protected]("image", _IMAGES)
[email protected]("masks", list(_MASKS.values()), ids = list(_MASKS))
+def test_hv15_rewritten_forward_is_bit_identical(fresh_cache, masks, image, trim):
+ safe, why = cs.resolve(_CLS)
+ assert why is None, why
+ model = _tiny_model()
+ kw = _inputs(masks, image)
+ if trim:
+ kw = _trimmed(model, kw)
+ with torch.no_grad():
+ want = _CLS.forward(model, **kw)[0]
+ got = safe(model, **kw)[0]
+ assert torch.isfinite(want).all()
+ assert torch.equal(want, got)
+
+
+def _stock_image_stream(image_embeds, states, primary_mask, batch_size):
+ is_t2v = torch.all(image_embeds == 0)
+ if is_t2v:
+ states = states * 0.0
+ mask = torch.zeros((batch_size, states.shape[1]), dtype = primary_mask.dtype)
+ else:
+ mask = torch.ones((batch_size, states.shape[1]), dtype = primary_mask.dtype)
+ return states, mask
+
+
+def _stock_merge(e1, m1, e2, m2, e3, m3):
+ states, masks = [], []
+ for t, tm, t2, tm2, im, imm in zip(e1, m1, e2, m2, e3, m3):
+ states.append(
+ torch.cat(
+ [
+ im[imm],
+ t2[tm2],
+ t[tm],
+ im[~imm],
+ torch.zeros_like(t2[~tm2]),
+ torch.zeros_like(t[~tm]),
+ ],
+ dim = 0,
+ )
+ )
+ masks.append(torch.cat([imm[imm], tm2[tm2], tm[tm], imm[~imm], tm2[~tm2], tm[~tm]], dim = 0))
+ return torch.stack(states), torch.stack(masks)
+
+
[email protected]("t2v", [True, False])
[email protected]("mask_dtype", [torch.int64, torch.bool, torch.float32])
+def test_hv15_image_stream_matches_both_stock_arms_bitwise(t2v, mask_dtype):
+ # inf and NaN make ``states * 0.0`` differ from zeros: the t2v arm must keep stock's NaNs and -0.0.
+ states = torch.tensor([[[1.0, -2.0, float("inf")], [float("nan"), -0.0, 3.0]]]).repeat(2, 1, 1)
+ image = torch.zeros(2, 2, 5) if t2v else torch.randn(2, 2, 5)
+ primary = torch.ones(2, 4, dtype = mask_dtype)
+ want = _stock_image_stream(image, states, primary, 2)
+ got = cs.hv15_image_stream(image, states, primary, 2)
+ assert torch.equal(want[0].view(torch.int32), got[0].view(torch.int32))
+ assert got[1].dtype == want[1].dtype and torch.equal(got[1], want[1])
+
+
[email protected]("image_len", [0, 3])
[email protected]("image_valid", [True, False])
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_hv15_merge_is_the_stock_permutation_bitwise(image_len, image_valid, dtype):
+ gen = torch.Generator().manual_seed(0)
+ m1 = torch.tensor([[1, 0, 1, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1]]).bool()
+ m2 = torch.tensor([[0, 1, 0, 1, 1], [0, 0, 0, 0, 0]]).bool()
+ m3 = torch.full((2, image_len), image_valid)
+ e1 = torch.randn(2, 7, 6, generator = gen).to(dtype)
+ e2 = torch.randn(2, 5, 6, generator = gen).to(dtype)
+ e3 = torch.randn(2, image_len, 6, generator = gen).to(dtype)
+ e2[0, 0, 0] = -0.0 # a padded byt5 row holding -0.0 / NaN must come out +0.0 like zeros_like
+ e1[0, 1, 1] = float("nan")
+ want = _stock_merge(e1, m1, e2, m2, e3, m3)
+ got = cs.hv15_merge_streams(e1, m1, e2, m2, e3, m3)
+ assert got[0].dtype == want[0].dtype and got[1].dtype == torch.bool
+ assert torch.equal(
+ want[0].view(torch.int16 if dtype == torch.bfloat16 else torch.int32),
+ got[0].view(torch.int16 if dtype == torch.bfloat16 else torch.int32),
+ )
+ assert torch.equal(want[1], got[1])
+
+
+# Ops that read a device value on the host (a sync, illegal during capture) or size an output by one.
+_HOST_READS = (
+ "aten.index.Tensor",
+ "aten.nonzero",
+ "aten.masked_select",
+ "aten.unique",
+ "aten.item",
+ "aten._local_scalar_dense",
+ "aten.is_nonzero",
+)
+
+
+def _aten_ops(fn):
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ class _Record(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.ops: set = set()
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ self.ops.add(str(func))
+ return func(*args, **(kwargs or {}))
+
+ with _Record() as rec:
+ fn()
+ return rec.ops
+
+
[email protected]("image", ["t2v_zero", "i2v"])
+def test_hv15_rewritten_forward_reads_nothing_on_the_host(fresh_cache, image):
+ safe, why = cs.resolve(_CLS)
+ assert why is None, why
+ model = _tiny_model()
+ kw = _inputs(_MASKS["b2_interleaved"], image)
+ with torch.no_grad():
+ stock = _aten_ops(lambda: _CLS.forward(model, **kw))
+ ours = _aten_ops(lambda: safe(model, **kw))
+ assert any(op.startswith(_HOST_READS) for op in stock), sorted(stock)
+ assert not any(op.startswith(_HOST_READS) for op in ours), sorted(
+ o for o in ours if o.startswith(_HOST_READS)
+ )
+
+
+def test_hv15_drifted_block_declines_with_the_reason(fresh_cache, monkeypatch):
+ drifted = tuple(
+ "image[~image_mask], # padded image"
+ if line == "image[~image_mask], # invalid image"
+ else line
+ for line in cs._HV15_MERGE_BLOCK
+ )
+ monkeypatch.setattr(cs, "_HV15_MERGE_BLOCK", drifted)
+ safe, why = cs.resolve(_CLS)
+ assert safe is None
+ assert "HunyuanVideo15Transformer3DModel forward is not capture-safe" in why
+ assert "three-stream merge block changed" in why
+
+
+def test_hv15_kill_switch_declines(fresh_cache, monkeypatch):
+ monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")
+ safe, why = cs.resolve(_CLS)
+ assert safe is None and cs.CAPTURE_SAFE_ENV in why
+
+
+_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+
+def _capture(
+ fn,
+ kw,
+ pool = None,
+):
+ static = {k: (v.clone() if torch.is_tensor(v) else v) for k, v in kw.items()}
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side), torch.no_grad():
+ for _ in range(2):
+ fn(**static)
+ torch.cuda.current_stream().wait_stream(side)
+ graph = torch.cuda.CUDAGraph()
+ with torch.no_grad(), torch.cuda.graph(graph, pool = pool):
+ out = fn(**static)[0]
+ return graph, static, out
+
+
+@_cuda
[email protected]("image", ["t2v_zero", "t2v_empty", "i2v"])
+def test_hv15_rewritten_forward_captures_and_replays_bit_identical(fresh_cache, image):
+ safe, why = cs.resolve(_CLS)
+ assert why is None, why
+ model = _tiny_model("cuda")
+ kw = _inputs(_MASKS["b2_interleaved"], image, device = "cuda")
+ graph, static, out = _capture(lambda **k: safe(model, **k), kw)
+ for seed in (2, 3):
+ # New values in the SAME buffers (t2v stays all-zero / empty, i2v gets new embeds).
+ fresh = _inputs(_MASKS["b2_interleaved"], image, device = "cuda", seed = seed)
+ for k, v in fresh.items():
+ if torch.is_tensor(v):
+ static[k].copy_(v)
+ graph.replay()
+ torch.cuda.synchronize()
+ with torch.no_grad():
+ eager = safe(model, **fresh)[0]
+ stock = _CLS.forward(model, **fresh)[0]
+ assert torch.equal(out, eager)
+ assert torch.equal(eager, stock)
+ del graph
+
+
+@_cuda
+def test_hv15_stock_forward_does_not_capture(fresh_cache):
+ # Last in the file: a failed capture can leave the CUDA context unusable for later captures.
+ model = _tiny_model("cuda")
+ kw = _inputs(_MASKS["b2_interleaved"], "t2v_zero", device = "cuda")
+ stream = torch.cuda.current_stream()
+ pool = torch.cuda.graph_pool_handle()
+ with pytest.raises(RuntimeError):
+ _capture(lambda **k: _CLS.forward(model, **k), kw, pool = pool)
+ # torch.cuda.graph's __exit__ raises in capture_end before it leaves its capture stream: without this every later
+ # test in the process ran on that stream (a prefetch test then saw its copies on a "compute" stream)
+ torch.cuda.set_stream(stream)
+ # torch leaves the CUDA generators in capture mode after an invalidated capture; later tests draw from them
+ cg._heal_generators()
+ # and the allocator still allocating to the dead capture's pool: every later empty_cache then freed nothing (a
+ # later test's reserved-memory check failed only when it ran after this file)
+ cg._abandon_capture_pool(pool)
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index b7165271e21..42b752d0679 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -576,6 +576,19 @@ def test_pool_survives_while_another_wrapper_still_holds_a_graph(stub_torch):
assert cg._POOL_BOX[0] == pool
+def test_a_graph_in_an_earlier_pool_does_not_keep_the_current_token(stub_torch):
+ """Liveness is per graph: a wrapper still holding a graph in a pool dropped earlier (a placement move re-recorded
+ another wrapper into a fresh pool) must not keep the current token alive after its own last graph is gone."""
+ first, second = _armed(), _armed()
+ first(_t(), timestep = _t((1,)), return_dict = False)
+ cg._POOL_BOX[0] = None # what a drop of the first pool's token leaves
+ second(_t(), timestep = _t((1,)), return_dict = False)
+ current = cg._POOL_BOX[0]
+ assert current is not None and first.cache
+ cg.reset_all([second])
+ assert cg._POOL_BOX[0] is None
+
+
def test_reset_all_forgets_the_pool_token_with_the_last_graph(stub_torch):
"""A reset that destroys the last graph in the shared pool must forget its token, or the next
capture dies on the allocator's "use_count > 0 INTERNAL ASSERT FAILED"."""
@@ -787,6 +800,12 @@ def test_graph_eligible_family_opt_in_on_the_video_backend(stub_torch, monkeypat
assert _eligible(monkeypatch, family = opted_in, family_default = False)[0] is True
bare = types.SimpleNamespace()
assert _eligible(monkeypatch, family = bare, family_default = False)[0] is False
+ # a family can opt in only for an offloaded denoiser
+ offload_only = types.SimpleNamespace(offload_cuda_graph = True)
+ assert _eligible(monkeypatch, family = offload_only, family_default = False)[0] is False
+ assert (
+ _eligible(monkeypatch, family = offload_only, family_default = False, offloaded = True)[0] is True
+ )
def test_stats_and_describe_are_json_safe(stub_torch):
@@ -804,11 +823,20 @@ def test_stats_and_describe_are_json_safe(stub_torch):
"eager_calls": 1,
"fallbacks": 0,
"cap_skips": 0,
+ "invalidations": 0,
+ "pool_bytes": aggregate["pool_bytes"],
+ "planned_eager": 0,
+ "shape_warmups": 0,
+ "evictions": 0,
+ "speed_eager": 0,
"refused_float": 0,
"refused_host_tensor": 0,
"refused_object": 0,
+ "placements": ["resident"],
"poisoned": False,
"capture_error": None,
+ "eager_ms": None,
+ "replay_ms": None,
}
assert json.loads(json.dumps(aggregate)) == aggregate
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 98a21982437..125929e1f58 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -1168,6 +1168,28 @@ def test_reclaimable_snapshot_credits_cached_blocks_without_flushing(monkeypatch
assert snap.free_mib == 2_000 + 4 * 1024
+def test_reclaimable_snapshot_does_not_credit_a_live_cuda_graph_pool(monkeypatch):
+ # A captured graph's pool is reserved but not allocated, and nothing else can allocate into it while the
+ # graph lives (under block offload it also holds the prefetch window): the guard must see it as used.
+ from core.inference import diffusion_cuda_graph as cg
+ from core.inference import diffusion_memory as dm
+
+ monkeypatch.setattr(
+ dm,
+ "snapshot_device_memory",
+ lambda target: DeviceMemory(
+ "cuda", "cuda", "discrete_vram", free_mib = 2_000, total_mib = 16_302
+ ),
+ )
+ _fake_torch_allocator(monkeypatch, reserved = 6 * 1024**3, allocated = 2 * 1024**3)
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 3 * 1024**3)
+ assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000 + 1024
+ monkeypatch.setattr(
+ cg, "live_pool_free_bytes", lambda: 5 * 1024**3
+ ) # the pool is all of the cached bytes
+ assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000
+
+
def test_reclaimable_snapshot_never_claims_more_than_the_card(monkeypatch):
# The credit is arithmetic, so a bogus allocator reading must not invent memory the card
# does not have and talk the guard out of a real refusal.
diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
new file mode 100644
index 00000000000..aa92798480a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
@@ -0,0 +1,759 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CUDA graphs over an offloaded denoiser (``diffusion_cuda_graph.arm_after_placement``).
+
+Block streaming: Studio's ``_apply_group_offload`` with the event-fenced prefetch; one graph records the whole forward
+with the copy-stream copies inside it. Partial residency: some groups pinned, the rest streamed; a release / restore
+moves the resident weights and the graph records again. Model offload: accelerate's CPU-offload hook moves the module
+eagerly, the replay sits under it and records again when the weights land elsewhere. Every case checks the replay is
+bit-identical to the eager forward of the same placement and that a replay makes no host synchronization."""
+
+from __future__ import annotations
+
+import copy
+import types
+import warnings
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_cuda_graph as cg # noqa: E402
+import core.inference.diffusion_memory as dm # noqa: E402
+import core.inference.diffusion_offload_prefetch as op # noqa: E402
+import core.inference.diffusion_speed as ds # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ op.ASYNC_PREFETCH_ENV,
+ op.PREFETCH_DEPTH_ENV,
+ cg.OFFLOAD_CUDA_GRAPH_ENV,
+ cg.CUDA_GRAPH_DISABLE_ENV,
+ cg.SPEED_CHECK_ENV,
+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
+ "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
+ "UNSLOTH_DIFFUSION_PIN_TOP_GROUP",
+ ):
+ monkeypatch.delenv(name, raising = False)
+ # timing a tiny forward on a shared card is noise: these tests check engagement and exactness
+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0")
+ yield
+ cg._POOL_BOX[0] = None
+
+
+class _Net(torch.nn.Module):
+ def __init__(
+ self,
+ width = 512,
+ blocks = 8,
+ ):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, width)
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(width, width) for _ in range(blocks))
+ self.proj_out = torch.nn.Linear(width, 64)
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ out = self.proj_out(x)
+ return (out,) if not return_dict else {"sample": out}
+
+
+def _cuda():
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ pytest.skip("needs NVIDIA CUDA")
+ pytest.importorskip("diffusers.hooks")
+
+
+def _pipe(net):
+ return types.SimpleNamespace(transformer = net, components = {"transformer": net})
+
+
+def _count_syncs(fn):
+ prev = torch.cuda.get_sync_debug_mode()
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ out = fn()
+ n = sum(1 for w in caught if "synchroniz" in str(w.message).lower())
+ finally:
+ torch.cuda.set_sync_debug_mode(prev)
+ return out, n
+
+
+def _inputs(seed):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ return torch.randn(16, 64, generator = g).cuda(), torch.rand(1, generator = g).cuda()
+
+
+def _run(net, seeds):
+ outs = []
+ with torch.no_grad():
+ for seed in seeds:
+ x, t = _inputs(seed)
+ outs.append(net(x, t, return_dict = False)[0].clone())
+ torch.cuda.synchronize()
+ return outs
+
+
+def _streamed(net, resident_mib = None):
+ kwargs = {"resident_transformer_mib": resident_mib} if resident_mib else {}
+ assert dm._apply_group_offload(_pipe(net), "cuda", None, **kwargs)
+ return net
+
+
+def _eager_reference(resident_mib, seeds):
+ torch.manual_seed(0)
+ net = _streamed(_Net(), resident_mib)
+ return _run(net, seeds)
+
+
[email protected]("resident_mib", [None, 3])
+def test_block_streamed_forward_replays_bit_identically_with_no_host_sync(resident_mib):
+ _cuda()
+ seeds = list(range(6))
+ want = _eager_reference(resident_mib, seeds)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), resident_mib)
+ pipe = _pipe(net)
+ handles, reason = cg.arm_after_placement(pipe)
+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason
+ assert reason.startswith("captured")
+ got = _run(net, seeds[:2]) # first call records, the second replays
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 2
+ with torch.no_grad():
+ for seed in seeds[2:]:
+ x, t = _inputs(seed)
+ out, syncs = _count_syncs(lambda: net(x, t, return_dict = False)[0].clone())
+ assert syncs == 0
+ got.append(out)
+ torch.cuda.synchronize()
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["eager_calls"] == 0
+ # weights are back on their host copies between calls; the streamed ones were copied inside the replay
+ assert net.blocks[-1].weight.device.type == "cpu"
+ pf = op.module_prefetcher(net)
+ assert pf.stats["missed"] == 0
+ # the pool (the step's activations plus the streamed copies) is what the generate-time guard leaves uncredited
+ assert cg.live_pool_bytes() >= handles[0].stats["pool_bytes"] > 0
+ # the guard leaves out only the pool's unallocated part; its allocated outputs are already not reclaimable
+ assert 0 < cg.live_pool_free_bytes() <= cg.live_pool_bytes()
+ cg.uninstall_all(handles)
+ assert net.__dict__["forward"] is handles[0].placement.prev # the offload hook chain is back
+
+
+def test_release_and_restore_records_again_and_stays_exact():
+ _cuda()
+ seeds = [0, 1, 2, 3]
+ want = _eager_reference(3, seeds)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ pipe = _pipe(net)
+ handles, _ = cg.arm_after_placement(pipe)
+ got = _run(net, seeds[:2])
+ restore = dm.release_resident_groups(pipe, 10_000, None)
+ assert restore is not None
+ got += _run(net, seeds[2:3]) # everything streams now: a new placement, recorded again
+ restore()
+ got += _run(net, seeds[3:])
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handles[0].stats["invalidations"] >= 1 and handles[0].stats["captures"] >= 2
+
+
+def test_unpinned_host_copies_keep_the_forward_eager_with_the_reason(monkeypatch):
+ _cuda()
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
+ torch.manual_seed(0)
+ net = _Net()
+ pipe = _pipe(net)
+ assert dm._apply_group_offload(pipe, "cuda", None, stream_text_encoders = False)
+ placement, why = cg.offload_placement(net)
+ if why is None:
+ pytest.skip("this diffusers pins the stream groups regardless")
+ assert placement is None and ("pinned" in why or "prefetch" in why)
+ handles, reason = cg.arm_after_placement(pipe)
+ assert handles == () and "Net" in reason
+
+
+def test_kill_switch_keeps_the_old_refusal(monkeypatch):
+ monkeypatch.setenv(cg.OFFLOAD_CUDA_GRAPH_ENV, "0")
+ assert not cg.offload_graphs_enabled()
+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_after_placement = True, _unsloth_cuda_graphs = ())
+ monkeypatch.setattr(ds, "_denoiser_dits", lambda p: [])
+ applied = ds.arm_graphs_after_placement(pipe, {"cuda_graph": False})
+ assert applied["cuda_graph"] is False and pipe._unsloth_cuda_graph_reason == "offload active"
+
+
+def test_sequential_offload_is_refused_with_the_reason():
+ net = _Net()
+ net.blocks[0]._hf_hook = object()
+ placement, why = cg.offload_placement(net)
+ assert placement is None and "sequential" in why
+
+
+def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move():
+ _cuda()
+ accelerate = pytest.importorskip("accelerate")
+ from accelerate import cpu_offload_with_hook
+
+ seeds = [0, 1, 2, 3, 4]
+ torch.manual_seed(0)
+ ref = _Net().cuda()
+ want = _run(ref, seeds)
+ torch.manual_seed(0)
+ net = _Net()
+ other = torch.nn.Linear(4, 4)
+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")
+ _other, other_hook = cpu_offload_with_hook(
+ other, execution_device = "cuda", prev_module_hook = hook
+ )
+ pipe = _pipe(net)
+ handles, reason = cg.arm_after_placement(pipe)
+ assert len(handles) == 1 and handles[0].placement.mode == "model", reason
+ got = _run(net, seeds[:3])
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 3
+ hook.offload() # what the next pipeline stage does: the weights go back to the host
+ assert net.proj_in.weight.device.type == "cpu"
+ got += _run(
+ net, seeds[3:]
+ ) # onloaded again by the hook; recorded again only if the addresses moved
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handles[0].stats["captures"] == 1 + handles[0].stats["invalidations"]
+ del accelerate, other_hook
+
+
+def test_model_offload_re_records_a_moved_module_without_a_warm_up():
+ """Every onload of a model-offloaded denoiser may land at new addresses: the same key records again with no
+ warm-up step (its kernels exist), and replays stay exact across many moves."""
+ _cuda()
+ pytest.importorskip("accelerate")
+ from accelerate import cpu_offload_with_hook
+
+ torch.manual_seed(0)
+ ref = _Net().cuda()
+ torch.manual_seed(0)
+ net, hook = cpu_offload_with_hook(_Net(), execution_device = "cuda")
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1, reason
+ handle = handles[0]
+ calls = []
+ stock = handle.placement.eager
+
+ def counting(*a, **k):
+ calls.append(1)
+ return stock(*a, **k)
+
+ handle.orig = counting # what the warm-up and the capture call
+ flushes = []
+ handle._release = lambda: flushes.append(1) # gc + empty_cache: a move must not pay them
+ for render in range(4):
+ seeds = [10 * render + i for i in range(5)]
+ want = _run(ref, seeds)
+ got = _run(net, seeds)
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ hook.offload()
+ torch.cuda.empty_cache()
+ # a block of the size the denoiser freed, so the next onload is likely to land elsewhere
+ _hold = torch.empty(4 << 20, device = "cuda")
+ moves = handle.stats["invalidations"]
+ assert moves >= 1 # the onloads did land elsewhere, so the re-record path ran
+ assert handle.stats["captures"] == 1 + moves, (handle.stats, handle.capture_error)
+ assert handle.placement.refusal(handle.stats) is None
+ # first recording: WARMUP_ITERS warm-ups + the capture call; each re-record: the capture call only
+ assert len(calls) == cg.WARMUP_ITERS + 1 + moves
+ assert flushes == []
+ del _hold
+
+
+def test_model_offload_stops_recording_only_when_moves_outpace_replays():
+ placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)
+ placement.mode = "model"
+ placement._checked = object()
+ many = cg.OffloadPlacement.MODEL_MAX_INVALIDATIONS
+
+ def judge(stats, where):
+ placement._fp = where # a new placement: judged afresh
+ return placement.refusal(stats)
+
+ assert judge({"invalidations": 1}, 1) is None
+ assert "new addresses" in judge({"invalidations": many, "captures": many, "replays": many}, 2)
+ # one move per render, a render's worth of replays per recording: keep recording
+ assert judge({"invalidations": 5, "captures": 5, "replays": 5 * 8}, 3) is None
+ # judged once per placement: the recording just made does not count against it until the next move
+ assert placement.refusal({"invalidations": 5, "captures": 6, "replays": 5 * 8 + 1}) is None
+
+
+class _FailsInCapture(_Net):
+ """A forward that makes a host-syncing call after its first blocks, but only while a graph records it."""
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ for i, block in enumerate(self.blocks):
+ x = torch.nn.functional.gelu(block(x))
+ if i == 3 and torch.cuda.is_current_stream_capturing():
+ x = (
+ x + float(x.sum().item()) * 0
+ ) # host read: invalidates the capture with copies already queued
+ out = self.proj_out(x)
+ return (out,) if not return_dict else {"sample": out}
+
+
+def test_a_failed_streamed_capture_falls_back_eager_without_stale_copies():
+ _cuda()
+ seeds = [0, 1, 2]
+ torch.manual_seed(0)
+ want = _run(_streamed(_FailsInCapture()), seeds)
+ torch.manual_seed(0)
+ net = _streamed(_FailsInCapture())
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1, reason
+ stream = torch.cuda.current_stream()
+ got = _run(
+ net, seeds
+ ) # the first call's capture fails: it and every later call run eager, correctly
+ assert torch.cuda.current_stream() == stream # not left on the failed capture's stream
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handles[0].poisoned and handles[0].stats["captures"] == 0
+ assert op.module_prefetcher(net).ready == {} or all(
+ e is None for e in op.module_prefetcher(net).ready.values()
+ )
+ assert net.blocks[-1].weight.device.type == "cpu"
+ cg.uninstall_all(handles)
+
+
+class _SyncsOnStock(_Net):
+ """Stock forward reads a value on the host (as HunyuanImage-2.1's text merge does); its rewrite does not."""
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ gate = x.new_tensor(1.0) if bool((x.abs().sum() >= 0).item()) else x.new_tensor(0.0)
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ out = self.proj_out(x) * gate
+ return (out,) if not return_dict else {"sample": out}
+
+
+def _capture_safe_forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+):
+ x = self.proj_in(x) * t
+ total = x.abs().sum()
+ gate = torch.where(
+ total >= 0, torch.ones_like(total), torch.zeros_like(total)
+ ) # on the device: no host read
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ out = self.proj_out(x) * gate
+ return (out,) if not return_dict else {"sample": out}
+
+
+def test_a_capture_safe_rewrite_records_under_block_offload(monkeypatch):
+ """The block-offload hook chain ends in the bound class forward; a capture-safe rewrite takes that slot while the
+ graph is installed, so a family whose stock forward syncs the host still records when streamed."""
+ _cuda()
+ import core.inference.diffusion_capture_safe as cs
+
+ monkeypatch.setattr(
+ cs,
+ "resolve",
+ lambda cls: (_capture_safe_forward, None) if cls is _SyncsOnStock else (None, None),
+ )
+ seeds = [0, 1, 2, 3]
+ torch.manual_seed(0)
+ want = _run(_streamed(_SyncsOnStock()), seeds)
+ torch.manual_seed(0)
+ net = _streamed(_SyncsOnStock())
+ inner = cg._innermost_class_forward(net)
+ assert inner is not None and inner.forward.__func__ is _SyncsOnStock.forward
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason
+ got = _run(net, seeds[:2])
+ with torch.no_grad():
+ for seed in seeds[2:]:
+ x, t = _inputs(seed)
+ out, syncs = _count_syncs(lambda: net(x, t, return_dict = False)[0].clone())
+ assert syncs == 0
+ got.append(out)
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert not handles[0].poisoned, handles[0].capture_error
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 4
+ cg.uninstall_all(handles)
+ assert (
+ inner.forward.__func__ is _SyncsOnStock.forward
+ ) # the stock forward is back with the hook chain
+
+
+class _Branchy(_Net):
+ """Branches on Python state a pre-hook sets per call (HunyuanVideo-1.5's null-mask flag)."""
+
+ flag = False
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ if self.flag:
+ x = x * 2
+ out = self.proj_out(x)
+ return (out,) if not return_dict else {"sample": out}
+
+
+def test_per_call_python_state_keys_the_graph():
+ _cuda()
+ torch.manual_seed(0)
+ net = _Branchy().cuda()
+ net._unsloth_graph_key_extra = lambda: net.flag
+ handle = cg.GraphedForward(net).enable()
+ try:
+ outs = {}
+ with torch.no_grad():
+ for flag in (False, True, False, True):
+ net.flag = flag
+ x, t = _inputs(1)
+ outs.setdefault(flag, []).append(net(x, t, return_dict = False)[0].clone())
+ handle.set_bypass(True)
+ want = net(x, t, return_dict = False)[0]
+ handle.set_bypass(False)
+ assert torch.equal(outs[flag][-1], want)
+ finally:
+ cg.uninstall_all([handle])
+ assert handle.stats["captures"] == 2 and handle.stats["replays"] == 4
+
+
+def test_a_failed_capture_leaves_the_cuda_rng_usable_and_on_its_sequence():
+ """An invalidated capture raises in cudaStreamEndCapture before torch ends the generators' capture; every later
+ eager CUDA draw (a pipeline's noise) then failed for the rest of the process."""
+ _cuda()
+ torch.cuda.manual_seed(123)
+ want = torch.randn(8, device = "cuda")
+ torch.manual_seed(0)
+ net = _FailsInCapture().cuda()
+ handle = cg.GraphedForward(net).enable()
+ try:
+ torch.cuda.manual_seed(123)
+ _run(net, [0])
+ assert handle.poisoned
+ got = torch.randn(8, device = "cuda")
+ finally:
+ cg.uninstall_all([handle])
+ assert torch.equal(got, want)
+
+
+def test_a_failed_capture_takes_the_allocator_off_its_pool(monkeypatch):
+ """capture_end raises in cudaStreamEndCapture before endAllocateToPool, so an invalidated capture left its pool in
+ the allocator's captures_underway: on torch 2.6 the next empty_cache tripped INTERNAL ASSERT captures_underway.empty()
+ and no later capture recorded (seen on an A100); later torch skips the global release instead, so empty_cache freed
+ nothing. Reproduced here on any torch by putting the pool back under capture before the raise."""
+ _cuda()
+ real_end = torch.cuda.CUDAGraph.capture_end
+ begin = torch._C._cuda_beginAllocateCurrentStreamToPool
+ stuck = []
+
+ def end_like_torch_2_6(self):
+ real_end(self)
+ stuck.append(self.pool())
+ begin(torch.cuda.current_device(), stuck[-1])
+ raise RuntimeError("CUDA error: operation failed due to a previous error during capture")
+
+ torch.manual_seed(0)
+ net = _streamed(_Net())
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ try:
+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_end", end_like_torch_2_6)
+ got = _run(net, [0])
+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_end", real_end)
+ assert stuck and handles[0].poisoned and handles[0].stats["fallbacks"] == 1
+ # While the pool stayed under capture, empty_cache raised INTERNAL ASSERT (torch 2.6) or silently freed nothing
+ # outside the private pools (later torch skips the global release while captures_underway is not empty).
+ block = torch.empty(256 << 20, dtype = torch.uint8, device = "cuda")
+ held = torch.cuda.memory_reserved()
+ del block
+ torch.cuda.empty_cache()
+ assert torch.cuda.memory_reserved() <= held - (256 << 20)
+ assert torch.equal(got[0], _eager_reference(None, [0])[0])
+ torch.manual_seed(0)
+ other = _streamed(_Net())
+ again, _ = cg.arm_after_placement(_pipe(other))
+ _run(other, [0, 1])
+ assert (
+ again[0].stats["captures"] == 1 and again[0].stats["replays"] == 2
+ ) # later captures still record
+ cg.uninstall_all(again)
+ finally:
+ end = (
+ getattr(torch._C, "_cuda_endAllocateToPool", None)
+ or torch._C._cuda_endAllocateCurrentStreamToPool
+ )
+ for pool in stuck: # never leave the process broken for the tests after this one
+ try:
+ end(torch.cuda.current_device(), pool)
+ except Exception: # noqa: BLE001 - the fix already took it off
+ pass
+ cg.uninstall_all(handles)
+
+
+def test_model_offload_fingerprint_follows_replaced_buffers():
+ """Module.to() puts NEW tensor objects in _buffers on every move: the fingerprint must read the registry each call,
+ not a list taken once (that list keeps describing the obsolete buffers)."""
+ torch.manual_seed(0)
+ net = _Net()
+ net.register_buffer("scale", torch.ones(4))
+ net._old_forward = net.forward # what accelerate's CpuOffload leaves for the replay slot
+ placement = cg.OffloadPlacement(net, "model")
+ before = placement.token()
+ net.scale = torch.full((4,), 9.0) # a new buffer object, as Module.to() makes on a move
+ assert placement.token() != before
+
+
+class _Ev:
+ def __init__(
+ self,
+ ms,
+ done = True,
+ ):
+ self.ms, self.done = ms, done
+
+ def query(self):
+ return self.done
+
+ def elapsed_time(self, end):
+ return end.ms - self.ms
+
+
+def _judged(
+ eager_ms,
+ graph_ms,
+ done = True,
+ others = (),
+):
+ """A handle with key "k" timed (and ``others`` already kept), judged once; returns (handle, k's verdict)."""
+ import os
+
+ os.environ.pop(cg.SPEED_CHECK_ENV, None)
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.stats, handle.logger, handle._slower, handle.capture_error = {}, None, None, None
+ handle.cache = {"k": object(), **{o: object() for o in others}}
+ handle._dropped, handle._release = set(), lambda: None
+ handle._judge = {
+ "k": {
+ "seen": 3,
+ "eager": [(_Ev(0), _Ev(ms)) for ms in eager_ms],
+ "graph": [(_Ev(0), _Ev(ms, done)) for ms in graph_ms],
+ "verdict": None,
+ },
+ **{o: {"seen": 3, "eager": [], "graph": [], "verdict": ""} for o in others},
+ }
+ handle._judge_keys()
+ return handle, handle._judge["k"]["verdict"]
+
+
+def test_an_offloaded_replay_slower_than_eager_drops_the_graphs():
+ """The copy engine runs a replay's onload copies at a different rate per machine (an A100 VM: 16% slower than the
+ stream's own copies); a key's first replays are timed against its own eager steps and its graph goes when slower."""
+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0])
+ assert why and "slower" in why and handle.cache == {} and handle._dropped == {"k"}
+ assert handle.capture_error == {
+ "type": "Refused",
+ "msg": why,
+ } # no graph left: the status names it
+ handle._judge_keys()
+ assert handle._judge["k"]["verdict"] == why # the verdict holds for the load
+ handle.placement, handle.poisoned = None, False
+ assert (
+ cg.stats([handle])["eager_ms"] == 550.0 and cg.stats([handle])["replay_ms"] == 654.0
+ ) # in the status
+ handle, why = _judged(
+ [550.0, 552.0], [560.0, 700.0, 710.0]
+ ) # one clean replay: a stall is not the verdict
+ assert why == "" and handle.cache
+ handle, why = _judged([550.0, 552.0], [540.0, 541.0, 539.0])
+ assert why == "" and handle.cache and not handle._dropped
+ handle, why = _judged([550.0], [560.0, 561.0, 559.0]) # inside the margin: kept
+ assert why == "" and handle.cache
+ handle, why = _judged(
+ [550.0], [655.0, 657.0, 654.0], done = False
+ ) # not finished yet: never waits on them
+ assert why is None and handle._slower is None and handle.cache
+ handle, why = _judged([550.0], [655.0, 657.0]) # too few replays yet
+ assert why is None and handle._slower is None
+
+
+def test_a_slower_key_drops_only_its_own_graph():
+ """On an A100 VM Qwen-Image-2.1's first prompt length replayed at eager speed and the next one 16% slower: only the
+ slower key goes eager, the other keeps replaying, and the status stays on."""
+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0], others = ("first",))
+ assert why and set(handle.cache) == {"first"} and handle._dropped == {"k"}
+ assert handle.capture_error is None
+
+
+def test_the_eager_reference_is_the_callers_own_steps_not_the_recorded_warm_ups(monkeypatch):
+ """The capture's warm-ups run under the recorder, whose copies follow the captured schedule: on an A100 VM that
+ schedule was the slow part (a replay 17% slower than eager went unjudged because its warm-ups were as slow). The
+ reference is the caller's eager steps on the stock prefetch path, timed before the first capture."""
+ _cuda()
+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)
+ monkeypatch.setattr(
+ cg, "SPEED_MARGIN", 1e9
+ ) # this checks where the reference comes from, not a tiny net's speed
+ monkeypatch.setattr(
+ cg, "SPEED_GAIN", -1e9
+ ) # nor whether a streamed key gains enough to keep its memory
+ seeds = list(range(8))
+ want = _eager_reference(None, seeds)
+ torch.manual_seed(0)
+ net = _streamed(_Net())
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ handle = handles[0]
+ recorded_steps = []
+ recorder = handle.placement.recorder
+ monkeypatch.setattr(
+ handle.placement, "recorder", lambda call: recorded_steps.append(1) or recorder(call)
+ )
+ got = _run(
+ net, seeds[:3]
+ ) # one untimed (may compile), then SPEED_EAGER_SAMPLES timed, all eager
+ assert (
+ handle.stats["captures"] == 0 and handle.stats["eager_calls"] == 1 + cg.SPEED_EAGER_SAMPLES
+ )
+ (state,) = handle._judge.values()
+ assert len(state["eager"]) == cg.SPEED_EAGER_SAMPLES
+ eager_events = list(state["eager"])
+ got += _run(net, seeds[3:]) # records, then replays
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == len(seeds) - 3
+ assert state["eager"] == eager_events # the recorder's warm-ups added nothing
+ assert (
+ handle.stats["eager_ms"] > 0 and handle.stats["replay_ms"] > 0 and state["verdict"] == ""
+ ) # judged, kept
+ assert recorded_steps # and the capture did go through the recorder
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ cg.uninstall_all(handles)
+
+
+def test_the_timed_eager_steps_run_on_capture_like_copies(monkeypatch):
+ """Compiled code guards on strides and storage offsets: eager steps on the caller's views compiled a variant the
+ capture then compiled again (a cold Qwen-Image-2.1 render on an RTX PRO 6000 went from 16.8 to 23.1 s)."""
+ _cuda()
+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)
+ monkeypatch.setattr(cg, "SPEED_MARGIN", 1e9)
+ monkeypatch.setattr(
+ cg, "SPEED_GAIN", -1e9
+ ) # nor whether a streamed key gains enough to keep its memory
+ torch.manual_seed(0)
+ net = _streamed(_Net())
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ offsets = []
+ hook = net.proj_in.register_forward_pre_hook(
+ lambda _m, a: offsets.append(a[0].storage_offset())
+ )
+ try:
+ with torch.no_grad():
+ for seed in range(3):
+ x, t = _inputs(seed)
+ padded = torch.cat([torch.zeros(1, 64, device = "cuda"), x])[
+ 1:
+ ] # a view at offset 64, like a slice
+ assert padded.storage_offset() == 64
+ want = _eager_reference(None, [seed])[0]
+ assert torch.equal(net(padded, t, return_dict = False)[0], want)
+ assert handles[0].stats["captures"] == 0 and offsets == [0, 0, 0]
+ finally:
+ hook.remove()
+ cg.uninstall_all(handles)
+
+
+def test_every_key_times_its_own_eager_reference_before_it_records(monkeypatch):
+ """A new input shape (a new prompt length) after the first capture is judged on its own eager steps too."""
+ _cuda()
+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)
+ monkeypatch.setattr(cg, "SPEED_MARGIN", 1e9)
+ monkeypatch.setattr(
+ cg, "SPEED_GAIN", -1e9
+ ) # nor whether a streamed key gains enough to keep its memory
+ torch.manual_seed(0)
+ net = _streamed(_Net())
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ handle = handles[0]
+ try:
+ with torch.no_grad():
+ for rows in (16, 16, 16, 16, 24, 24, 24, 24, 24):
+ x = torch.randn(rows, 64, device = "cuda")
+ t = torch.rand(1, device = "cuda")
+ net(x, t, return_dict = False)
+ torch.cuda.synchronize()
+ assert handle.stats["captures"] == 2 and handle.stats["speed_eager"] == 2 * (
+ 1 + cg.SPEED_EAGER_SAMPLES
+ )
+ assert len(handle._judge) == 2 and all(
+ len(s["eager"]) == cg.SPEED_EAGER_SAMPLES for s in handle._judge.values()
+ )
+ finally:
+ cg.uninstall_all(handles)
+
+
+def test_status_stays_on_while_the_eager_reference_is_timed():
+ """The timed eager steps before the first capture are not a refusal: no "ran eager (bypassed)" status meanwhile."""
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.stats = {"captures": 0, "replays": 0, "eager_calls": 3, "speed_eager": 3}
+ handle.cache, handle.capture_error, handle.poisoned, handle.placement = {}, None, False, None
+ assert cg.never_engaged([handle]) is None
+ handle.stats.update(eager_calls = 4) # one eager call for another reason: reported
+ assert "ran eager" in cg.never_engaged([handle])
+
+
+def test_status_names_graphs_dropped_after_replaying():
+ """A graph dropped mid-load (slower than eager) must not leave the status saying the step replays."""
+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0])
+ handle.stats.update(captures = 1, replays = 5, eager_calls = 40)
+ handle.capture_error, handle.poisoned, handle.placement = (
+ {"type": "Refused", "msg": why},
+ False,
+ None,
+ )
+ resolved = {
+ "cuda_graph": {"value": "on", "status": "applied", "reason": "captured per input shape"}
+ }
+ live, optims = cg.live_status(resolved, ["cuda_graph", "int8"], [handle])
+ assert (
+ live["cuda_graph"]["value"] == "off"
+ and "dropped" in live["cuda_graph"]["reason"]
+ and why in live["cuda_graph"]["reason"]
+ )
+ assert optims == ["int8"] and resolved["cuda_graph"]["value"] == "on"
+ handle.cache = {
+ "k": object()
+ } # a live graph again (refusal lifted, recorded anew): status stays as recorded
+ handle.capture_error = None
+ assert cg.live_status(resolved, ["cuda_graph"], [handle])[0]["cuda_graph"]["value"] == "on"
diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.py
index a8a1fdbeeae..bc55b7328e8 100644
--- a/studio/backend/tests/test_diffusion_offload_precision.py
+++ b/studio/backend/tests/test_diffusion_offload_precision.py
@@ -613,14 +613,25 @@ def test_quantised_blocks_survive_group_offload_under_no_grad(
offloaded(x)
[email protected]("offload_graphs", [True, False])
@pytest.mark.parametrize(
"offload_active, denoiser_offloaded, expected",
[(False, None, False), (True, None, True), (True, False, False), (True, True, True)],
)
-def test_graph_gate_follows_the_denoiser(monkeypatch, offload_active, denoiser_offloaded, expected):
+def test_graph_gate_follows_the_denoiser(
+ monkeypatch, offload_active, denoiser_offloaded, expected, offload_graphs
+):
+ """With offload graphs on, a moved denoiser is judged on everything else and armed after placement; with the
+ kill switch, a moved denoiser is refused as before."""
from core.inference import diffusion_cuda_graph as dcg
from core.inference import diffusion_speed
+ if offload_graphs:
+ monkeypatch.delenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, raising = False)
+ expected = False
+ else:
+ monkeypatch.setenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, "0")
+
seen: list = []
def _eligible(target, **kwargs):
diff --git a/studio/backend/tests/test_diffusion_offload_step_graphs.py b/studio/backend/tests/test_diffusion_offload_step_graphs.py
new file mode 100644
index 00000000000..52b28ef28e3
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_step_graphs.py
@@ -0,0 +1,816 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The whole-step graph of an offloaded denoiser on top of the per-block graphs (diffusion_block_graph).
+
+The step graph records a forward whose blocks compile below their offload hooks; the per-block graphs run their compute
+while it records, and arm as its fallback where they are the default. A streamed placement records its copies into the
+prefetcher's slot ring and keeps a key only when the replay pays, else hands the ring back."""
+
+from __future__ import annotations
+
+import copy
+import types
+import warnings
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_block_graph as bg # noqa: E402
+import core.inference.diffusion_cuda_graph as cg # noqa: E402
+import core.inference.diffusion_memory as dm # noqa: E402
+import core.inference.diffusion_offload_prefetch as op # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ bg.BLOCK_GRAPHS_ENV,
+ bg.COMPILE_BELOW_HOOKS_ENV,
+ cg.CUDA_GRAPHS_ENV,
+ cg.CUDA_GRAPH_DISABLE_ENV,
+ cg.OFFLOAD_CUDA_GRAPH_ENV,
+ cg.STEP_SLOTS_ENV,
+ op.ASYNC_PREFETCH_ENV,
+ op.PREFETCH_DEPTH_ENV,
+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
+ "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
+ "UNSLOTH_DIFFUSION_PIN_TOP_GROUP",
+ ):
+ monkeypatch.delenv(name, raising = False)
+ # engagement and exactness here; the speed check is driven explicitly where a test needs it
+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0")
+ yield
+ cg._POOL_BOX[0] = None
+
+
+class Block(torch.nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ self.lin = torch.nn.Linear(width, width)
+
+ def forward(self, x):
+ return torch.nn.functional.gelu(self.lin(x))
+
+
+class Net(torch.nn.Module):
+ _repeated_blocks = ["Block"]
+
+ def __init__(
+ self,
+ width = 256,
+ blocks = 8,
+ ):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(16, width)
+ self.blocks = torch.nn.ModuleList(Block(width) for _ in range(blocks))
+ self.proj_out = torch.nn.Linear(width, 16)
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ for block in self.blocks:
+ x = block(x)
+ out = self.proj_out(x)
+ return (out,) if not return_dict else {"sample": out}
+
+
+def _pipe(net):
+ return types.SimpleNamespace(transformer = net, components = {"transformer": net})
+
+
+def _target():
+ return types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda")
+
+
+# --- CPU -------------------------------------------------------------------------------------------------------
+
+
+def test_a_block_graph_runs_its_compute_while_a_whole_step_records():
+ calls = []
+ shared = bg._Shared(None, None)
+ graph = bg.BlockGraph(Block(4), lambda *a, **k: calls.append(1) or "computed", shared)
+ with cg._recording_step():
+ assert cg.step_recording()
+ assert graph(torch.zeros(2, 4)) == "computed"
+ assert not cg.step_recording()
+ assert calls == [1]
+ # no first sighting, no recording, no eager count: the step above records the kernels themselves
+ assert graph.stats["eager_calls"] == 0 and not graph.seen and not graph.cache
+
+
+def _step_handle(
+ net,
+ mode = "group",
+ plan = None,
+):
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.module = net
+ handle.placement = types.SimpleNamespace(mode = mode) if mode else None
+ handle.plan = plan
+ handle.fallback = None
+ handle.fallback_handle = None
+ return handle
+
+
+def _arm(pipe, monkeypatch, *, hooked, pinned):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ applied = {"cuda_graph": True}
+ handles = cg.arm_block_graphs(
+ pipe,
+ applied,
+ target = _target(),
+ family = types.SimpleNamespace(),
+ hooked = hooked,
+ pinned = pinned,
+ )
+ return handles, applied
+
+
+def test_an_armed_step_graph_stays_primary_with_per_block_graphs_as_its_fallback(monkeypatch):
+ net = Net(blocks = 3)
+ pipe = _pipe(net)
+ step = _step_handle(net)
+ pipe._unsloth_cuda_graphs = (step,)
+ pipe._unsloth_cuda_graph_reason = (
+ "captured per input shape: block-streamed (copies recorded in the graph)"
+ )
+ handles, applied = _arm(pipe, monkeypatch, hooked = True, pinned = True)
+ assert handles == (step,) and applied["cuda_graph"]
+ assert pipe._unsloth_cuda_graph_mode == "step" and callable(step.fallback)
+ assert cg.status_reason(pipe, True) == (
+ "denoiser step captured per input shape: block-streamed (copies recorded in the graph), replayed bit-identically"
+ )
+ # the fallback arms the per-block layer below the module's blocks, once
+ armed = step.fallback()
+ assert isinstance(armed, bg.BlockGraphSet) and len(armed.graphs) == 3
+ assert cg.status_reason(pipe, True).endswith("; the steps it leaves eager record per block")
+ armed.free()
+
+
+def test_a_streamed_step_graph_gets_a_per_block_fallback_only_when_asked(monkeypatch):
+ net = Net(blocks = 3)
+ pipe = _pipe(net)
+ step = _step_handle(net)
+ pipe._unsloth_cuda_graphs = (step,)
+ _arm(pipe, monkeypatch, hooked = True, pinned = False)
+ assert step.fallback is None # streamed: per-block graphs are opt-in, as without the step graph
+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "1")
+ _arm(pipe, monkeypatch, hooked = True, pinned = False)
+ assert callable(step.fallback)
+ model = _step_handle(net, mode = "model")
+ pipe._unsloth_cuda_graphs = (model,)
+ monkeypatch.delenv(bg.BLOCK_GRAPHS_ENV)
+ _arm(pipe, monkeypatch, hooked = True, pinned = True)
+ assert model.fallback is None
+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "0")
+ pinned = _step_handle(net)
+ pipe._unsloth_cuda_graphs = (pinned,)
+ _arm(pipe, monkeypatch, hooked = True, pinned = True)
+ assert pinned.fallback is None
+
+
+def test_a_planned_resident_step_keeps_its_graph_and_the_blocks_wait_as_fallback(monkeypatch):
+ net = Net(blocks = 2)
+ pipe = _pipe(net)
+ step = _step_handle(net, mode = None, plan = lambda *a: None)
+ pipe._unsloth_cuda_graphs = (step,)
+ pipe._unsloth_cuda_graph_reason = "captured per input shape: resident"
+ handles, _ = _arm(pipe, monkeypatch, hooked = False, pinned = False)
+ assert handles == (step,) and callable(step.fallback)
+ assert cg.status_reason(pipe, True) == cg.WHOLE_REASON
+
+
+class _Ev:
+ def __init__(self, ms):
+ self.ms = ms
+
+ def query(self):
+ return True
+
+ def elapsed_time(self, end):
+ return end.ms - self.ms
+
+
+class _Placement:
+ def __init__(
+ self,
+ streams,
+ ring = False,
+ ):
+ self._streams = streams
+ self.ring = ring
+ self.declined = None
+ self.mode = "group"
+
+ def streams(self):
+ return self._streams
+
+ def decline(self, reason):
+ self.declined = reason
+ return self.ring
+
+
+def _judged(
+ placement,
+ eager_ms,
+ graph_ms,
+ others = (),
+):
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.stats, handle.logger, handle._slower, handle.capture_error = {}, None, None, None
+ handle.placement = placement
+ handle.cache = {"k": object(), **{o: object() for o in others}}
+ handle._dropped, handle.released = set(), []
+ handle._release = lambda: handle.released.append(1)
+ handle._judge = {
+ "k": {
+ "seen": 3,
+ "eager": [(_Ev(0), _Ev(ms)) for ms in eager_ms],
+ "graph": [(_Ev(0), _Ev(ms)) for ms in graph_ms],
+ "verdict": None,
+ },
+ **{o: {"seen": 3, "eager": [], "graph": [], "verdict": ""} for o in others},
+ }
+ handle._judge_keys()
+ return handle, handle._judge["k"]["verdict"]
+
+
+def test_a_streamed_key_is_kept_only_when_its_replay_pays():
+ # a tie on a streaming placement: the ring and the pool are not worth it, and no other key records this load
+ placement = _Placement(streams = True)
+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7], others = ("j",))
+ assert why and "within 1%" in why and placement.declined == why
+ assert handle.cache == {} and handle._dropped == {"k", "j"} and handle.released == [1]
+ # with a slot ring the flush waits for the ring's drop at the end of the forward: one flush, not one mid-forward
+ placement = _Placement(streams = True, ring = True)
+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])
+ assert placement.declined == why and handle.cache == {} and handle.released == []
+ # a real gain keeps it
+ placement = _Placement(streams = True)
+ handle, why = _judged(placement, [100.0, 101.0], [97.0, 97.5, 98.0])
+ assert why == "" and placement.declined is None and "k" in handle.cache
+ # pinned (nothing streams): a tie keeps the graph, as before
+ placement = _Placement(streams = False)
+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])
+ assert why == "" and placement.declined is None and "k" in handle.cache
+
+
+def test_no_key_is_judged_while_another_graph_records(monkeypatch):
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)
+ placement = _Placement(streams = True)
+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])
+ assert why is None and placement.declined is None and "k" in handle.cache
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ handle._judge_keys()
+ assert handle._judge["k"]["verdict"]
+
+
+class _Wrapper:
+ def __init__(self, cache):
+ self.cache = cache
+
+
+def test_every_live_graph_pool_stays_out_of_the_reclaimable_memory(monkeypatch):
+ # an earlier graph's pool after a later capture failed and retired the shared id
+ monkeypatch.setattr(cg, "_POOL_BOX", [None])
+ live = _Wrapper({"k": types.SimpleNamespace(pool_token = (0, 7))})
+ monkeypatch.setattr(cg, "_LIVE_WRAPPERS", {live})
+ segments = [
+ {"device": 0, "segment_pool_id": (0, 7), "total_size": 100, "allocated_size": 40},
+ {"device": 0, "segment_pool_id": (0, 0), "total_size": 500, "allocated_size": 0},
+ {
+ "device": 1,
+ "segment_pool_id": (0, 7),
+ "total_size": 900,
+ "allocated_size": 0,
+ }, # another card
+ ]
+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ assert cg.live_pool_free_bytes() == 60
+
+
+def test_only_the_streamed_groups_slots_are_allocated_before_a_capture(monkeypatch):
+ pf = op.GroupPrefetcher.__new__(op.GroupPrefetcher)
+ streamed, resident = types.SimpleNamespace(), types.SimpleNamespace(_unsloth_resident = True)
+ others = [types.SimpleNamespace(_unsloth_resident = True) for _ in range(2)]
+ pf.groups = [resident, others[0], streamed, others[1]]
+ pf.slot_of = {id(g): i % 3 for i, g in enumerate(pf.groups)}
+ pf.slot_size, pf.slot_raw = 16, {}
+ made = []
+ monkeypatch.setattr(pf, "_alloc_slot", lambda where: made.append(where), raising = False)
+ pf.materialize_slots()
+ assert made == [2]
+
+
+def test_a_failed_streamed_capture_hands_the_ring_back(monkeypatch):
+ _cuda()
+ net, handle = _armed_streamed(monkeypatch)
+ pf = op.module_prefetcher(net)
+
+ def broken(*args, **kwargs):
+ handle.placement.before_capture()
+ raise RuntimeError("capture failed")
+
+ monkeypatch.setattr(handle, "_capture", broken)
+ ref = _net().cuda()
+ _call(net, 0)
+ assert torch.equal(_call(net, 1), _call(ref, 1))
+ assert handle.poisoned and not pf.slot_raw
+ handle.free()
+
+
+def test_a_declined_placement_refuses_every_later_call():
+ placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)
+ placement.mode = "group"
+ placement._declined = None
+ placement._slots = False
+ placement.decline("replay measured within 1% of the eager step")
+ assert placement.refusal({}) == "replay measured within 1% of the eager step"
+
+
+def test_both_graph_pools_are_kept_out_of_the_reclaimable_memory(monkeypatch):
+ monkeypatch.setattr(
+ dm,
+ "snapshot_device_memory",
+ lambda target: dm.DeviceMemory(
+ "cuda", "cuda", "discrete_vram", free_mib = 1000, total_mib = 8000
+ ),
+ )
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a: 900 << 20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a: 100 << 20)
+ monkeypatch.setattr(bg, "pool_bytes", lambda device = None: 300 << 20)
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 200 << 20)
+ mem = dm.reclaimable_snapshot_device_memory(types.SimpleNamespace(device = "cuda"))
+ assert mem.free_mib == 1000 + 800 - 300 - 200
+
+
+# --- CUDA ------------------------------------------------------------------------------------------------------
+
+
+def _cuda():
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ pytest.skip("needs NVIDIA CUDA")
+ pytest.importorskip("diffusers.hooks")
+
+
+def _streamed(net, resident_mib = None):
+ kwargs = {"resident_transformer_mib": resident_mib} if resident_mib else {}
+ assert dm._apply_group_offload(_pipe(net), "cuda", None, **kwargs)
+ # the background pin swaps host copies (a new placement): let it finish so the counts below are exact
+ for group in dm._offload_groups(net):
+ pinner = getattr(group, op._BG_PIN_ATTR, None)
+ if pinner is not None:
+ pinner.wait(group)
+ return net
+
+
+def _inputs(seed, rows = 8):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ return torch.randn(rows, 16, generator = g).cuda(), torch.rand(1, generator = g).cuda()
+
+
+def _call(
+ net,
+ seed,
+ rows = 8,
+):
+ x, t = _inputs(seed, rows)
+ with torch.no_grad():
+ out = net(x, t, return_dict = False)[0].clone()
+ torch.cuda.synchronize()
+ return out
+
+
+def _syncs(fn):
+ prev = torch.cuda.get_sync_debug_mode()
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ out = fn()
+ finally:
+ torch.cuda.set_sync_debug_mode(prev)
+ return out, sum(1 for w in caught if "synchroniz" in str(w.message).lower())
+
+
+def _net():
+ torch.manual_seed(0)
+ return Net()
+
+
+def _armed_streamed(monkeypatch, slots = True):
+ if not slots:
+ monkeypatch.setenv(cg.STEP_SLOTS_ENV, "0")
+ net = _streamed(_net())
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason
+ return net, handles[0]
+
+
+def test_streamed_step_copies_land_in_the_slot_ring_not_the_graph_pool(monkeypatch):
+ _cuda()
+ ref = _net().cuda()
+ want = [_call(ref, s) for s in range(4)]
+ real_to_device = op._to_device
+ fresh = []
+
+ def to_device(src, device):
+ if torch.cuda.is_current_stream_capturing():
+ fresh.append(tuple(src.shape)) # a copy the graph records into a pool allocation
+ return real_to_device(src, device)
+
+ monkeypatch.setattr(op, "_to_device", to_device)
+ counts = {}
+ for slots in (False, True):
+ cg._POOL_BOX[0] = None
+ fresh.clear()
+ net, handle = _armed_streamed(monkeypatch, slots)
+ pf = op.module_prefetcher(net)
+ got = [_call(net, 0), _call(net, 1)] # records, then replays
+ got += [_call(net, 2), _call(net, 3)]
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 4
+ counts[slots] = len(fresh)
+ if slots:
+ assert handle.placement._slots and pf.slot_raw and pf.stats["slot_fills"] > 0
+ assert pf.stats.get("slot_fallbacks", 0) == 0
+ else:
+ assert not pf.slot_raw
+ handle.free()
+ monkeypatch.delenv(cg.STEP_SLOTS_ENV, raising = False)
+ # without the ring every streamed block's weight and bias is a fresh pool allocation inside the recording; with
+ # it only the dense top-level group's are (proj_in / proj_out: 4 tensors)
+ assert counts[False] >= 2 * 7 and counts[True] <= 4
+
+
+def test_a_declined_streamed_placement_hands_the_ring_back_and_runs_eager(monkeypatch):
+ _cuda()
+ ref = _net().cuda()
+ net, handle = _armed_streamed(monkeypatch)
+ pf = op.module_prefetcher(net)
+ _call(net, 0)
+ _call(net, 1)
+ assert pf.slot_raw and handle.stats["replays"] == 2 # the recording call replays too
+ handle.placement.decline("replay measured within 1% of the eager step")
+ handle.cache.clear()
+ for seed in (2, 3):
+ assert torch.equal(_call(net, seed), _call(ref, seed))
+ assert (
+ not pf.slot_raw and not pf.slot_of
+ ) # dropped at the end of the first forward after the verdict
+ assert handle.stats["captures"] == 1 and handle.capture_error["type"] == "Refused"
+
+
+def test_a_new_slot_ring_is_a_new_placement(monkeypatch):
+ _cuda()
+ net, handle = _armed_streamed(monkeypatch)
+ pf = op.module_prefetcher(net)
+ _call(net, 0)
+ _call(net, 1)
+ assert handle.stats["captures"] == 1
+ pf.disable_slots()
+ pf.enable_slots()
+ ref = _net().cuda()
+ assert torch.equal(_call(net, 2), _call(ref, 2))
+ assert handle.stats["invalidations"] == 1 and handle.stats["captures"] == 2
+
+
+def test_a_failed_pinned_step_graph_hands_its_calls_to_per_block_graphs(monkeypatch):
+ _cuda()
+ ref = _net().cuda()
+ net = _streamed(_net(), resident_mib = 64) # every group pinned resident under its hooks
+ pipe = _pipe(net)
+ cg.arm_after_placement(pipe)
+ handles = cg.arm_block_graphs(
+ pipe,
+ {"cuda_graph": True},
+ target = _target(),
+ family = types.SimpleNamespace(),
+ hooked = True,
+ pinned = True,
+ )
+ step = handles[0]
+ assert step.placement is not None and callable(step.fallback)
+ assert not step.placement.streams() and not step.placement._slots
+ _call(net, 0)
+ assert step.stats["captures"] == 1 and step.fallback_handle is None
+ step.poisoned = True # as a failed capture leaves it
+ for seed in (1, 2, 3):
+ assert torch.equal(_call(net, seed), _call(ref, seed))
+ blocks = step.fallback_handle
+ assert isinstance(blocks, bg.BlockGraphSet) and step.fallback is None
+ # first call of each block runs it, the second records and replays, the third replays
+ assert blocks.stats["captures"] == 8 and blocks.stats["replays"] == 16
+ step.free()
+ assert step.fallback_handle is None
+ assert not any(
+ isinstance(r.forward, bg.BlockGraph) for b in net.blocks for r in b._diffusers_hook._fn_refs
+ )
+
+
+def test_per_block_graphs_under_a_recording_step_record_nothing(monkeypatch):
+ _cuda()
+ ref = _net().cuda()
+ net = _streamed(_net(), resident_mib = 64)
+ pipe = _pipe(net)
+ handles, _ = cg.arm_after_placement(pipe)
+ blocks, reason = bg.install_block_graphs(net, device = "cuda")
+ assert reason == "armed"
+ want = [_call(ref, s) for s in range(3)]
+ got = [_call(net, s) for s in range(3)] # the step records on its first call and replays after
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 3
+ assert blocks.stats["captures"] == 0 and blocks.stats["eager_calls"] == 0
+ blocks.free()
+ handles[0].free()
+
+
+def test_a_compiled_streamed_step_records_with_no_graph_break_and_a_new_shape_adds_none(
+ monkeypatch,
+):
+ _cuda()
+ import torch._dynamo.utils as du
+
+ torch._dynamo.reset()
+ ref = _net().cuda()
+ net = _net()
+ kwargs = {"fullgraph": False, "dynamic": None}
+ for b in net.blocks:
+ b.compile(**kwargs)
+ net._unsloth_regional_compile_kwargs = kwargs
+ _streamed(net)
+ assert bg.compile_below_offload_hooks(net) == 8
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1, reason
+ breaks = sum(du.counters["graph_break"].values())
+ for rows in (8, 8, 8, 24, 24, 24):
+ torch.testing.assert_close(
+ _call(net, rows, rows), _call(ref, rows, rows), rtol = 1e-4, atol = 1e-4
+ )
+ assert (
+ sum(du.counters["graph_break"].values()) == breaks
+ ) # the hooks stay outside every compiled region
+ s = handles[0].stats
+ assert s["captures"] == 2 and s["replays"] == 6 and s["fallbacks"] == 0
+ first = _call(net, 5, 24)
+ assert torch.equal(_call(net, 5, 24), first)
+ handles[0].free()
+
+
+def test_a_static_step_skip_under_the_hooks_is_replayed_past_only_while_it_plans_no_skip():
+ _cuda()
+ from core.inference import diffusion_step_skip as sk
+
+ ref = _net().cuda()
+ net = _net()
+ pipe = _pipe(net)
+ assert sk.install_static_step_skip(pipe, settings = {"min_steps": 12}) == sk.TC_STATIC
+ _streamed(net, resident_mib = 64)
+ cg.arm_after_placement(pipe)
+ step = cg.arm_block_graphs(
+ pipe,
+ {"cuda_graph": True},
+ target = _target(),
+ family = types.SimpleNamespace(),
+ hooked = True,
+ pinned = True,
+ )[0]
+ skip = step.placement.skip
+ assert isinstance(skip, sk.StaticStepSkip)
+ # a render below min_steps plans no skip: the step records and replays straight past the skip layer
+ skip.reset(8)
+ for seed in range(3):
+ assert torch.equal(_call(net, seed), _call(ref, seed))
+ assert step.stats["captures"] == 1 and step.stats["replays"] == 3
+ assert (
+ skip.stats["calls"] == 3 and skip.stats["computed"] == 3
+ ) # the replays are counted as computed steps
+ # a render that plans skips: every call goes through the skip layer, the per-block graphs take the compute
+ skip.reset(16)
+ assert not all(skip.plan)
+ replays = step.stats["replays"]
+ for i in range(16):
+ _call(net, 0)
+ skip.step_end()
+ assert step.stats["replays"] == replays and step.stats["skip_eager"] == 16
+ assert skip.stats["skipped"] > 0
+ assert (
+ isinstance(step.fallback_handle, bg.BlockGraphSet)
+ and step.fallback_handle.stats["replays"] > 0
+ )
+ step.free()
+ sk.uninstall_static_step_skip(pipe)
+
+
[email protected]("timed", [True, False])
+def test_a_key_warmed_by_its_timed_eager_steps_records_without_another_warm_up(monkeypatch, timed):
+ _cuda()
+ if timed:
+ monkeypatch.delenv(cg.SPEED_CHECK_ENV)
+ ref = _net().cuda()
+ net, handle = _armed_streamed(monkeypatch)
+ recorded = []
+ real = handle.placement.recorder
+
+ def recorder(call):
+ inner = real(call)
+
+ def record(*a, **k):
+ recorded.append(torch.cuda.current_stream())
+ return inner(*a, **k)
+
+ return record
+
+ handle.placement.recorder = recorder
+ for seed in range(5):
+ assert torch.equal(_call(net, seed), _call(ref, seed))
+ assert handle.stats["captures"] == 1
+ # timed: 3 eager steps ran the key on capture-like inputs, so only the recording itself calls the step;
+ # untimed: the side-stream warm-ups run first, as for any first capture
+ assert len(recorded) == (1 if timed else 1 + cg.WARMUP_ITERS)
+ assert handle.stats["speed_eager"] == (3 if timed else 0)
+ handle.free()
+
+
+class _SyncsWhileRecorded(Block):
+ def forward(self, x):
+ y = super().forward(x)
+ if torch.cuda.is_current_stream_capturing():
+ y = y + float(y.sum().item()) * 0 # a host read: invalidates the recording
+ return y
+
+
+def test_a_failed_block_recording_takes_the_allocator_off_its_pool():
+ """As for the whole step (test_diffusion_offload_cuda_graph): capture_end raises before endAllocateToPool, so a
+ failed per-block recording left the allocator on the block pool and every later empty_cache freed nothing."""
+ _cuda()
+ torch.manual_seed(0)
+ net = Net(blocks = 2).cuda()
+ net.blocks[1] = _SyncsWhileRecorded(256).cuda()
+ Net._repeated_blocks = ["Block", "_SyncsWhileRecorded"]
+ try:
+ handle, reason = bg.install_block_graphs(net, device = "cuda")
+ assert reason == "armed"
+ x, t = _inputs(0)
+ with torch.no_grad():
+ for _ in range(3):
+ net(x, t, return_dict = False)
+ torch.cuda.synchronize()
+ assert handle.graphs[1].poisoned and handle.stats["fallbacks"] == 1
+ torch.cuda.empty_cache()
+ baseline = torch.cuda.memory_reserved()
+ block = torch.empty(256 << 20, dtype = torch.uint8, device = "cuda")
+ del block
+ torch.cuda.empty_cache()
+ # stuck on the dead pool, empty_cache released nothing and the 256 MiB segment stayed
+ assert torch.cuda.memory_reserved() < baseline + (128 << 20)
+ torch.randn(4, device = "cuda") # the CUDA generator left capture mode
+ handle.free()
+ # a later recording in the process still works
+ other = Net(blocks = 2).cuda()
+ again, _ = bg.install_block_graphs(other, device = "cuda")
+ with torch.no_grad():
+ for _ in range(3):
+ other(x, t, return_dict = False)
+ assert again.stats["captures"] == 2 and again.stats["replays"] == 4
+ again.free()
+ finally:
+ Net._repeated_blocks = ["Block"]
+
+
+def test_releasing_an_offloaded_step_graph_frees_its_capture_stream_workspace(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ torch._C, "_cuda_clearCublasWorkspaces", lambda: calls.append(1), raising = False
+ )
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: calls.append(2))
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.placement = _Placement(streams = True)
+ handle._release()
+ assert calls == [1, 2] # the recordings' own cuBLAS workspace first, then the segments
+ handle.placement = None
+ calls.clear()
+ handle._release()
+ assert calls == [2] # a resident graph's reset keeps the compute stream's workspace
+
+
+def test_freeing_a_streamed_step_graph_hands_the_slot_ring_back(monkeypatch):
+ _cuda()
+ net, handle = _armed_streamed(monkeypatch)
+ pf = op.module_prefetcher(net)
+ _call(net, 0)
+ _call(net, 1)
+ assert pf.slot_raw
+ handle.free()
+ assert not pf.slot_raw and not pf.slot_of
+ ref = _net().cuda()
+ assert torch.equal(_call(net, 2), _call(ref, 2))
+
+
+def test_the_first_capture_keeps_its_pool_while_its_graph_lives(monkeypatch):
+ _cuda()
+ monkeypatch.setattr(cg, "_POOL_BOX", [None])
+ net = _net().cuda()
+ handle = cg.GraphedForward(net, logger = None).enable()
+ _call(net, 0)
+ _call(net, 1)
+ assert handle.stats["captures"] == 1 and cg._POOL_BOX[0] is not None
+ cg._drop_pool_if_unused()
+ assert cg._POOL_BOX[0] is not None
+ handle.free()
+
+
+def test_freeing_a_resident_graph_keeps_offload_hooks_added_after_it():
+ _cuda()
+ net = _net()
+ handle = cg.GraphedForward(net, logger = None).enable()
+ _streamed(net)
+ hooked = net.__dict__.get("forward")
+ assert hooked is not None and hooked is not handle
+ handle.free()
+ assert net.__dict__.get("forward") is hooked
+ ref = _net().cuda()
+ assert torch.equal(_call(net, 0), _call(ref, 0))
+
+
+def _probe_handle():
+ handle = cg.GraphedForward.__new__(cg.GraphedForward)
+ handle.stats = {"eager_calls": 0, "speed_eager": 0}
+ handle._dropped, handle._slower, handle._judge, handle._warmed = set(), None, {}, {}
+ handle.max_graphs = 4
+ handle.plan = None
+ return handle
+
+
+def test_a_speed_probe_without_memory_for_its_copies_runs_the_callers_step(monkeypatch):
+ handle = _probe_handle()
+
+ def oom(live):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(handle, "_statics", oom)
+ x = torch.ones(2)
+ assert torch.equal(handle._timed_eager(lambda t: t * 2, (x,), {}, "k"), x * 2)
+ assert "k" in handle._dropped
+
+
+def test_no_probe_step_is_timed_while_another_graph_records(monkeypatch):
+ handle = _probe_handle()
+ monkeypatch.setattr(handle, "_statics", lambda live: list(live))
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)
+ x = torch.ones(2)
+ for _ in range(3):
+ handle._timed_eager(lambda t: t + 1, (x,), {}, "k")
+ assert handle._judge["k"]["seen"] == 3 and handle._judge["k"]["eager"] == []
+
+
+def test_a_ring_drop_waits_while_another_graph_records(monkeypatch):
+ pf = op.GroupPrefetcher.__new__(op.GroupPrefetcher)
+ pf._drop_slots_at_end = False
+ dropped = []
+ monkeypatch.setattr(pf, "disable_slots", lambda: dropped.append(1), raising = False)
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)
+ pf._drop_ring()
+ assert dropped == [] and pf._drop_slots_at_end
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ pf._drop_ring()
+ assert dropped == [1] and not pf._drop_slots_at_end
+
+
+def test_a_capture_collision_leaves_the_cuda_generators_alone(monkeypatch):
+ _cuda()
+ net = _net().cuda()
+ handle = cg.GraphedForward(net, logger = None).enable()
+ healed = []
+ monkeypatch.setattr(cg, "_heal_generators", lambda: healed.append(1))
+
+ def collided(*args, **kwargs):
+ raise RuntimeError("beginAllocateToPool: already recording to mempool_id")
+
+ monkeypatch.setattr(handle, "_capture", collided)
+ ref = _net().cuda()
+ for seed in (0, 1):
+ assert torch.equal(_call(net, seed), _call(ref, seed))
+ assert handle.poisoned and healed == []
+ handle.free()
+
+
+def test_an_offload_only_family_gets_no_resident_graph():
+ net = _net()
+ pipe = _pipe(net)
+ pipe._unsloth_cuda_graph_offload_only = True
+ handles, reason = cg.arm_after_placement(pipe)
+ assert handles == () and "offloaded steps only" in reason
+ pipe._unsloth_cuda_graph_offload_only = False
+ handles, _ = cg.arm_after_placement(pipe)
+ assert len(handles) == 1 and handles[0].placement is None
+ cg.uninstall_all(handles)
diff --git a/studio/backend/tests/test_diffusion_qwenimage21_graph.py b/studio/backend/tests/test_diffusion_qwenimage21_graph.py
new file mode 100644
index 00000000000..7d7a303e852
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_qwenimage21_graph.py
@@ -0,0 +1,471 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image-2.1 decode steps as CUDA graphs: ``diffusion_qwenimage21.graph_plan`` + ``GraphedForward``.
+
+CPU tests check the plan (what runs eager, that the planned call is tensors-only and bit-identical to the eager
+fast step). CUDA tests record real graphs on a tiny random transformer and compare whole renders against the
+stock diffusers forward, bit for bit."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+from core.inference import diffusion_capture_safe as cs
+from core.inference import diffusion_cuda_graph as cg
+from core.inference import diffusion_qwenimage21 as q
+
+torch = pytest.importorskip("torch")
+qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
+
+# Sibling import: pytest puts the rootdir on sys.path, not this directory.
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_qwenimage21 import CASES, _inputs, _model # noqa: E402
+
+_CUDA = torch.cuda.is_available()
+
+
[email protected](autouse = True)
+def _stock_after(monkeypatch):
+ for env in (q.FAST_STEP_ENV, cs.CAPTURE_SAFE_ENV, cg.CUDA_GRAPH_DISABLE_ENV):
+ monkeypatch.delenv(env, raising = False)
+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0") # tiny forwards on a shared card: timing is noise
+ q.uninstall()
+ yield
+ q.uninstall()
+
+
+def _call_kwargs(inp, hs, timestep, kv, mode):
+ return dict(
+ hidden_states = hs,
+ timestep = timestep,
+ encoder_hidden_states = inp["encoder_hidden_states"],
+ encoder_hidden_states_mask = inp["encoder_hidden_states_mask"],
+ img_shapes = inp["img_shapes"],
+ img_mask = inp["img_mask"],
+ attention_kwargs = None,
+ kv_cache = kv,
+ kv_cache_mode = mode,
+ return_dict = False,
+ )
+
+
+def _render(
+ m,
+ inp,
+ steps = 5,
+ call = None,
+):
+ """The pipeline's loop: step 0 prefills the cache, later steps decode from it."""
+ call = call or m
+ outs = []
+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
+ lat = inp["latents"]
+ with torch.inference_mode():
+ for i in range(steps):
+ hs = lat if inp["cond"] is None else torch.cat([inp["cond"], lat], 1)
+ timestep = torch.full((hs.shape[0],), 1.0 - i / steps, device = hs.device)
+ out = call(**_call_kwargs(inp, hs, timestep, kv, "extract" if i == 0 else "cached"))[0]
+ outs.append(out.clone())
+ lat = lat - 0.1 * out[:, -lat.shape[1] :]
+ return outs
+
+
+def _prefilled(m, inp):
+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
+ hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)
+ t = torch.full((hs.shape[0],), 0.9, device = hs.device)
+ with torch.inference_mode():
+ m(**_call_kwargs(inp, hs, t, kv, "extract"))
+ return kv, hs
+
+
[email protected]("case", CASES)
+def test_planned_decode_step_is_bit_identical_to_the_eager_fast_step(case):
+ m = _model()
+ inp = _inputs(**case)
+ assert q.install()
+ kv, hs = _prefilled(m, inp)
+ t = torch.full((hs.shape[0],), 0.5)
+ with torch.inference_mode():
+ kwargs = _call_kwargs(inp, hs, t, kv, "cached")
+ want = m(**kwargs)[0]
+ plan = q.graph_plan(m, (), kwargs)
+ assert plan is not None
+ call, planned = plan
+ got = call(**planned)[0]
+ assert torch.equal(want, got)
+ # Everything the graph layer has to copy is a tensor or a constant: no opaque object, no float.
+ key = cg.graph_key(((), planned))
+ assert not cg._uncapturable(key)
+ assert not cg._has_float(key)
+ assert len(planned["kv"]) == len(m.transformer_blocks)
+
+
+def test_what_the_plan_leaves_eager(monkeypatch):
+ m = _model()
+ inp = _inputs(batch = 2, pad = 2)
+ assert q.install()
+ kv, hs = _prefilled(m, inp)
+ t = torch.full((2,), 0.5)
+ with torch.inference_mode():
+ cached = _call_kwargs(inp, hs, t, kv, "cached")
+ assert q.graph_plan(m, (), cached) is not None
+ # The prefill fills the cache from Python; it stays eager.
+ assert q.graph_plan(m, (), {**cached, "kv_cache_mode": "extract"}) is None
+ assert q.graph_plan(m, (), {**cached, "kv_cache": None, "kv_cache_mode": None}) is None
+ assert q.graph_plan(m, (), {**cached, "return_dict": True}) is None
+ # A LoRA scale rides in attention_kwargs through the forward's decorator.
+ assert q.graph_plan(m, (), {**cached, "attention_kwargs": {"scale": 0.5}}) is None
+ assert (
+ q.graph_plan(m, (hs,), {k: v for k, v in cached.items() if k != "hidden_states"})
+ is None
+ )
+ assert q.graph_plan(m, (), {**cached, "unknown": 1}) is None
+ # An empty cache (no prefill yet) is not plannable, and never raises.
+ empty = qmod.QwenImage21KVCache(len(m.transformer_blocks))
+ assert q.graph_plan(m, (), {**cached, "kv_cache": empty}) is None
+ assert q.graph_plan(m, (), {**cached, "img_mask": "not a tensor"}) is None
+ with torch.enable_grad():
+ assert q.graph_plan(m, (), cached) is None
+ # A prefix K/V too large to keep a second copy of per graph stays eager.
+ monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 16)
+ with torch.inference_mode():
+ assert q.graph_plan(m, (), cached) is None
+ monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 1 << 30)
+ monkeypatch.setenv(q.FAST_STEP_ENV, "0")
+ with torch.inference_mode():
+ assert q.graph_plan(m, (), cached) is None
+
+
+def test_resolve_follows_the_installed_fast_step(monkeypatch):
+ cls = qmod.QwenImage21Transformer2DModel
+ forward, why = cs.resolve(cls)
+ assert forward is None and "is not installed" in why
+ assert q.install()
+ forward, why = cs.resolve(cls)
+ assert why is None
+ assert forward is vars(cls)["forward"]
+ assert forward.__unsloth_graph_plan__ is q.graph_plan
+ monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")
+ forward, why = cs.resolve(cls)
+ assert forward is None and cs.CAPTURE_SAFE_ENV in why
+ monkeypatch.delenv(cs.CAPTURE_SAFE_ENV)
+ q.uninstall()
+ assert cs.resolve(cls)[0] is None
+
+
+def test_graph_eligible_accepts_qwen_image_21_once_the_fast_step_is_installed(monkeypatch):
+ import types
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+
+ def eligible():
+ return cg.graph_eligible(
+ types.SimpleNamespace(device = "cuda", backend = "cuda"),
+ family = types.SimpleNamespace(),
+ pipe = types.SimpleNamespace(transformer = _model()),
+ offload_active = False,
+ cache_active = False,
+ speed_mode = "default",
+ )
+
+ ok, why = eligible()
+ assert ok is False and why.startswith(
+ "QwenImage21Transformer2DModel forward is not capture-safe"
+ )
+ assert q.install()
+ assert eligible() == (True, "eligible")
+
+
+def test_a_failed_capture_falls_back_to_the_callers_own_call():
+ """A planned call whose capture fails (here: host tensors) must run the stock call, not the planned kwargs."""
+ m = _model()
+ inp = _inputs(batch = 2, pad = 3)
+ ref = _render(m, inp)
+ assert q.install()
+ handle = cg.GraphedForward(m).enable()
+ try:
+ got = _render(m, _inputs(batch = 2, pad = 3))
+ finally:
+ cg.uninstall_all([handle])
+ for a, b in zip(ref, got):
+ assert torch.equal(a, b)
+ assert handle.poisoned and handle.stats["refused_host_tensor"] == 1
+ assert cg._POOL_BOX[0] is None
+
+
+# ------------------------------------------------------------------------------------------------- CUDA
+
+
+def _cuda_inputs(**case):
+ return {k: (v.to("cuda") if torch.is_tensor(v) else v) for k, v in _inputs(**case).items()}
+
+
+def _graphed(m, **kw):
+ handle = cg.GraphedForward(m, **kw).enable()
+ assert handle.capture_safe and handle.plan is q.graph_plan
+ return handle
+
+
[email protected](not _CUDA, reason = "needs CUDA")
[email protected]("case", CASES)
+def test_graphed_render_is_bit_identical_to_the_stock_forward(case):
+ m = _model().to("cuda")
+ inp = _cuda_inputs(**case)
+ ref = _render(m, inp)
+ assert q.install()
+ handle = _graphed(m)
+ try:
+ got = _render(m, inp)
+ again = _render(m, _cuda_inputs(**case))
+ finally:
+ cg.uninstall_all([handle])
+ for a, b, c in zip(ref, got, again):
+ assert torch.equal(a, b)
+ assert torch.equal(a, c)
+ assert not handle.poisoned, handle.capture_error
+ # Two renders: two prefills eager, one capture, every decode step replayed (the capturing one included).
+ assert handle.stats["planned_eager"] == 2
+ assert handle.stats["captures"] == 1
+ assert handle.stats["replays"] == 2 * (5 - 1)
+ assert not handle.poisoned, handle.capture_error
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_evicting_the_only_graph_does_not_reuse_its_dead_pool():
+ m = _model().to("cuda")
+ lengths = (9, 11, 13)
+ refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]
+ assert q.install()
+ handle = _graphed(m, max_graphs = 1)
+ try:
+ for n, ref in zip(lengths, refs):
+ for a, b in zip(ref, _render(m, _cuda_inputs(text = n, seed = n))):
+ assert torch.equal(a, b)
+ finally:
+ cg.uninstall_all([handle])
+ assert not handle.poisoned, handle.capture_error
+ assert handle.stats["captures"] == 3 and handle.stats["evictions"] == 2
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_a_new_prompt_length_recaptures_and_the_oldest_graph_is_evicted():
+ m = _model().to("cuda")
+ lengths = (9, 11, 9, 13)
+ refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]
+ assert q.install()
+ handle = _graphed(m, max_graphs = 2)
+ try:
+ for n, ref in zip(lengths, refs):
+ got = _render(m, _cuda_inputs(text = n, seed = n))
+ for a, b in zip(ref, got):
+ assert torch.equal(a, b)
+ finally:
+ cg.uninstall_all([handle])
+ # 9 and 11 capture, 9 replays, 13 evicts the least recently replayed (11). After the first capture a new length
+ # runs its first decode step for real (that is its warm-up) and records on the next one.
+ assert handle.stats["captures"] == 3
+ assert handle.stats["shape_warmups"] == 2
+ assert handle.stats["evictions"] == 1
+ assert handle.stats["cap_skips"] == 0
+ assert handle.stats["replays"] == 4 * 4 - 2
+ assert not handle.poisoned
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_a_new_prompt_length_warms_on_tensors_laid_out_like_the_capture(monkeypatch):
+ """Compiled blocks guard on strides and storage offsets. The warm-up step of a new length must run on fresh copies
+ (as the capture's statics are), else it compiles a variant for the caller's views and the capture that follows
+ compiles inside the recording, which fails it (seen on the real model under block offload)."""
+ m = _model().to("cuda")
+ assert q.install()
+ handle = _graphed(m)
+ stock = q._graph_step
+ seen: list = []
+
+ def spy(self, **kw):
+ if handle.stats["shape_warmups"] and not seen:
+ leaves: list = []
+ cg._flatten(kw, leaves)
+ seen.append(leaves)
+ return stock(self, **kw)
+
+ monkeypatch.setattr(q, "_graph_step", spy)
+ try:
+ for n in (9, 11):
+ _render(m, _cuda_inputs(text = n, seed = n))
+ finally:
+ cg.uninstall_all([handle])
+ assert handle.stats["shape_warmups"] == 1 and handle.stats["captures"] == 2
+ assert seen and all(t.storage_offset() == 0 for t in seen[0])
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_alternating_caches_of_one_shape_replay_their_own_prefix():
+ """True CFG: cond and uncond share a graph key but not a KV cache; the sticky copy must follow identity."""
+ m = _model().to("cuda")
+ cond, uncond = _cuda_inputs(seed = 1), _cuda_inputs(seed = 2)
+ assert q.install()
+
+ def two_stream_render():
+ out = []
+ kvs = [qmod.QwenImage21KVCache(len(m.transformer_blocks)) for _ in range(2)]
+ lat = cond["latents"]
+ with torch.inference_mode():
+ for i in range(4):
+ t = torch.full((1,), 1.0 - i / 4, device = "cuda")
+ mode = "extract" if i == 0 else "cached"
+ a = m(**_call_kwargs(cond, lat, t, kvs[0], mode))[0][:, -lat.shape[1] :]
+ b = m(**_call_kwargs(uncond, lat, t, kvs[1], mode))[0][:, -lat.shape[1] :]
+ out += [a.clone(), b.clone()]
+ lat = lat - 0.1 * (a + b)
+ return out
+
+ ref = two_stream_render()
+ handle = _graphed(m)
+ try:
+ got = two_stream_render()
+ finally:
+ cg.uninstall_all([handle])
+ for a, b in zip(ref, got):
+ assert torch.equal(a, b)
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 6
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_graph_statics_match_the_eager_inputs_strides_and_inference_mode():
+ """A static that is not an inference tensor where eager passes one selects another compiled variant."""
+ m = _model().to("cuda")
+ inp = _cuda_inputs()
+ assert q.install()
+ handle = _graphed(m)
+ try:
+ _render(m, inp)
+ (entry,) = handle.cache.values()
+ kv, hs = _prefilled(m, inp)
+ with torch.inference_mode():
+ # Decode-step latents come out of the previous step, made in inference mode like everything else here.
+ hs = hs.clone()
+ t = torch.full((1,), 0.5, device = "cuda")
+ _, planned = q.graph_plan(m, (), _call_kwargs(inp, hs, t, kv, "cached"))
+ live: list = []
+ cg._flatten(((), planned), live)
+ assert len(live) == len(entry.static)
+ for src, dst in zip(live, entry.static):
+ assert dst.stride() == src.stride()
+ assert dst.is_inference() == src.is_inference()
+ finally:
+ cg.uninstall_all([handle])
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_planned_decode_steps_do_not_sync_the_host():
+ m = _model().to("cuda")
+ inp = _cuda_inputs(batch = 2, pad = 2)
+ assert q.install()
+ handle = _graphed(m)
+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
+ try:
+ with torch.inference_mode():
+ for i in range(5):
+ t = torch.full((2,), 1.0 - i / 5, device = "cuda")
+ # Step 1 captures (its warm-up and capture synchronize by design); later replays must not.
+ if i >= 2:
+ torch.cuda.set_sync_debug_mode("error")
+ try:
+ m(**_call_kwargs(inp, inp["latents"], t, kv, "extract" if i == 0 else "cached"))
+ finally:
+ torch.cuda.set_sync_debug_mode("default")
+ finally:
+ cg.uninstall_all([handle])
+ assert handle.stats["replays"] == 4
+
+
+# ------------------------------------------------------------------------------------------- offloaded
+
+
+def _block_streamed(m):
+ import types
+
+ from core.inference import diffusion_memory as dm
+
+ pipe = types.SimpleNamespace(transformer = m, components = {"transformer": m})
+ assert dm._apply_group_offload(pipe, "cuda", None)
+ return pipe
+
+
[email protected](not _CUDA, reason = "needs CUDA")
[email protected]("case", [CASES[1], CASES[3]])
+def test_block_streamed_decode_steps_replay_bit_identically(case):
+ """Under block offload the planned step enters through the offload hooks (which onload the top-level weights)
+ and the copy-stream onloads are recorded with it: renders match the eager streamed and the resident forward."""
+ pytest.importorskip("diffusers.hooks")
+ from core.inference import diffusion_offload_prefetch as op
+
+ resident = _render(_model().to("cuda"), _cuda_inputs(**case))
+ assert q.install() # before the hooks, as Studio's load does
+ m = _model()
+ pipe = _block_streamed(m)
+ eager = _render(m, _cuda_inputs(**case))
+ handles, reason = cg.arm_after_placement(pipe)
+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason
+ handle = handles[0]
+ assert handle.plan is q.graph_plan and handle.placed is q.placed_step
+ try:
+ got = _render(m, _cuda_inputs(**case))
+ again = _render(m, _cuda_inputs(**case))
+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
+ inp = _cuda_inputs(**case)
+ hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)
+ with torch.inference_mode():
+ for i in range(3):
+ t = torch.full((hs.shape[0],), 1.0 - i / 5, device = "cuda")
+ if i:
+ torch.cuda.set_sync_debug_mode(
+ "error"
+ ) # a replay of the streamed step never waits on the host
+ try:
+ m(**_call_kwargs(inp, hs, t, kv, "extract" if i == 0 else "cached"))
+ finally:
+ torch.cuda.set_sync_debug_mode("default")
+ finally:
+ cg.uninstall_all(handles)
+ assert not handle.poisoned, handle.capture_error
+ for r, e, a, b in zip(resident, eager, got, again):
+ assert torch.equal(e, a) and torch.equal(e, b)
+ assert torch.equal(r, a)
+ assert handle.stats["captures"] == 1
+ assert handle.stats["replays"] == 2 * (5 - 1) + 2
+ assert next(m.transformer_blocks[-1].parameters()).device.type == "cpu"
+ assert op.module_prefetcher(m).stats["missed"] == 0
+
+
+def test_offload_placement_names_a_forward_that_cannot_record_under_the_hooks(monkeypatch):
+ m = _model()
+
+ def planned(*_a, **_k):
+ return None
+
+ planned.__unsloth_graph_plan__ = q.graph_plan
+ monkeypatch.setattr(cs, "resolve", lambda cls: (planned, None))
+ assert "no entry through the offload hooks" in cg._forward_refusal(m, "group")
+ planned.__unsloth_graph_placed__ = q.placed_step
+ assert cg._forward_refusal(m, "group") is None
+
+ def rewrite(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(cs, "resolve", lambda cls: (rewrite, None))
+ assert "capture-safe forward" in cg._forward_refusal(m, "group")
+ assert (
+ cg._forward_refusal(m, "model") is None
+ ) # model offload calls the slot with the weights already onloaded
+ monkeypatch.setattr(cs, "resolve", lambda cls: (None, "X forward is not capture-safe (why)"))
+ assert cg._forward_refusal(m, "model") == "X forward is not capture-safe (why)"
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 7cafb332bb0..52510253521 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -9599,6 +9599,41 @@ def test_cuda_graph_is_a_per_family_opt_in():
opted_in = [f.name for f in _FAMILIES if f.supports_cuda_graph]
assert opted_in == [], f"{opted_in} opts into CUDA graphs with no measurement on record"
+ # Offloaded denoisers are a different trade: the replay records the onloads and the step makes no host wait.
+ # Each opt-in carries a capped-card measurement in its family comment.
+ offloaded = sorted(f.name for f in _FAMILIES if f.offload_cuda_graph)
+ assert offloaded == sorted(OFFLOAD_GRAPH_FAMILIES), offloaded
+ assert (
+ h3.offload_cuda_graph is False
+ ) # streamed H3 onloads with host waits; a graph cannot record them
+ assert (
+ "GPU-bound" in h3.cuda_graph_decline and "wait on the host" in h3.cuda_graph_decline
+ ) # the measured reasons
+
+
+OFFLOAD_GRAPH_FAMILIES = ("wan2.2-ti2v-5b", "hunyuanvideo-1.5")
+
+
+def test_h3_modular_load_arms_a_deferred_graph_after_placement():
+ """A forced H3 graph is deferred to placement by apply_speed_optims; the modular load must arm it (or record why
+ it stays eager). Without the call the status read "off: armed after placement" for the whole load."""
+ import ast
+ import textwrap
+
+ src = textwrap.dedent(inspect.getsource(video_module.VideoBackend._load_h3_modular_pipeline))
+ calls = [
+ node
+ for node in ast.walk(ast.parse(src))
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "arm_graphs_after_placement"
+ ]
+ assert calls and isinstance(calls[0].args[0], ast.Name) and calls[0].args[0].id == "speed_view"
+ speed = src.index("applied = apply_speed_optims(")
+ arm = src.index("arm_graphs_after_placement(")
+ status = src.index('"cuda_graph": (')
+ assert (
+ speed < arm < status
+ ) # after the deferring speed pass, before the resolved record reads the reason
def test_every_rebuilt_speed_target_carries_the_backend():