Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: whole-step CUDA graphs under offload on top of per-block graphs (L4 FLUX.1 16 GB 10% faster, HunyuanVideo-1.5 1 GiB lighter) (#12707)
Daniel Han8de2cde9b7f9
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.jsonindex b3c5b387808..b5f77d707e3 100644--- a/scripts/exec_literals_baseline.json+++ b/scripts/exec_literals_baseline.json@@ -12,9 +12,9 @@{"file": "studio/backend/core/inference/diffusion_capture_safe.py","sink": "compile",- "digest": "05c1d218ea0c8e6d",+ "digest": "1f39880672dc563b","count": 1,- "reason": "Rebuilds the installed diffusers HunyuanImage forward from its own inspect.getsource with one fixed block swapped; the source is the imported library's code, not user or network input."+ "reason": "Rebuilds the installed diffusers HunyuanImage / HunyuanVideo 1.5 forward from its own inspect.getsource with fixed blocks swapped (compiled with dont_inherit so this module's future import does not leak in); the source is the imported library's code, not user or network input."},{"file": "studio/backend/core/inference/diffusion_capture_safe.py",diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex b8db11b6af8..3fd22f13b3d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -180,6 +180,7 @@ from .diffusion_speed import (fp16_unet_offloaded,fresh_compile_count,int8_gemm_live,+ arm_graphs_after_placement,normalize_speed_mode,resolve_speed_mode,restore_backend_flags,@@ -7336,6 +7337,8 @@ class DiffusionBackend:if not speed_deferred:if _denoiser_hooked(pipe):compile_pipe_below_offload_hooks(pipe, logger)+ # the whole-step graph, with the offload copies recorded in it, where placement allows one+ arm_graphs_after_placement(pipe, speed_applied, logger)cuda_graph.arm_block_graphs(pipe,speed_applied,@@ -9303,6 +9306,7 @@ class DiffusionBackend:engage_pinned_denoisers(state.pipe, speed_applied, logger)if _denoiser_hooked(state.pipe):compile_pipe_below_offload_hooks(state.pipe, logger)+ arm_graphs_after_placement(state.pipe, speed_applied, logger)cuda_graph.arm_block_graphs(state.pipe,speed_applied,diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 0eb59028928..7fdeb507f33 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -31,6 +31,7 @@ Best-effort: an unavailable backend falls back to the diffusers default. torch/dfrom __future__ import annotations+import functoolsimport osimport reimport threading@@ -1952,6 +1953,15 @@ def _set_hunyuan_null_mask(module: Any, enabled: bool) -> None:setattr(attn, _NULL_ATTN_FLAG, enabled)+def _hunyuan_null_mask_state(module: Any) -> bool:+ """The null-mask flag this call's pre-hook set (read by the graph layer as part of its key)."""+ for blk in getattr(module, "transformer_blocks", []):+ attn = getattr(blk, "attn", None)+ if attn is not None:+ return bool(getattr(attn, _NULL_ATTN_FLAG, False))+ return False++def _null_mask_processor_cls():"""Build (once, lazily) a HunyuanVideo15AttnProcessor2_0 subclass whose ``__call__`` runsattn_mask=None when the DiT is flagged (padding already removed by the pre-hook); otherwise it@@ -2074,6 +2084,61 @@ def _trim_stream(states, mask):return states, mask, all_valid+_TRIM_MEMO_ATTR = "_unsloth_trim_memo"+++def _trim_plan(module: Any, kwargs: dict) -> dict:+ """The trim decisions for this call's image / mask tensors: the host reads (``_trim_stream``'s) made once per+ set of inputs. A pipeline hands the SAME prompt tensors to every step, so a step after the first reuses the plan+ and makes no host wait (a CUDA-graph replay of the step then runs without one). Keyed on the tensors themselves,+ held here, and their version counters, so new or edited inputs plan afresh."""+ import torch++ names = ("image_embeds", "encoder_attention_mask", "encoder_attention_mask_2")+ srcs = tuple(kwargs.get(n) for n in names)+ # An inference tensor (renders run under torch.inference_mode) has no version counter and reading one raises;+ # it is keyed on identity alone. It cannot be written outside inference mode, and the pipeline hands every step+ # the encoder's own outputs, which nothing edits in place.+ versions = tuple(+ ("inference" if t.is_inference() else t._version) if torch.is_tensor(t) else None+ for t in srcs+ )+ memo = module.__dict__.setdefault(_TRIM_MEMO_ATTR, [])+ for held, held_versions, plan in memo:+ if held_versions == versions and all(a is b for a, b in zip(held, srcs)):+ return plan+ image = srcs[0]+ plan = {+ "t2v": bool(image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()))+ }+ for name, mask in zip(names[1:], srcs[1:]):+ if mask is None or not torch.is_tensor(mask) or mask.dim() != 2:+ plan[name] = (None, True)+ continue+ mb = mask.bool()+ keep = mb.any(dim = 0) # column valid for at least one batch element+ index = None+ if not bool(keep.all()):+ index = keep.nonzero().squeeze(1)+ mb = mb.index_select(1, index)+ # vacuously True for a 0-length stream, fine for an unused secondary stream (byt5 in t2v)+ plan[name] = (index, bool(mb.all().item()))+ memo.insert(0, (srcs, versions, plan))+ del memo[4:] # the CFG branches of one render+ return plan+++def _apply_trim(states: Any, mask: Any, decided: tuple) -> tuple:+ """``_trim_stream`` with its decisions already made: the same columns, gathered on the device."""+ if states is None or mask is None or mask.dim() != 2:+ return states, mask, True+ index, all_valid = decided+ if index is not None:+ states = states.index_select(1, index)+ mask = mask.index_select(1, index)+ return states, mask, all_valid++def _hunyuan_trim_pre_hook(module, args, kwargs):"""Eager forward pre-hook: strip padded text tokens so the joint attention runs fused.@@ -2103,8 +2168,9 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):try:null_ok = True+ plan = _trim_plan(module, kwargs)image = kwargs.get("image_embeds")- if image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()):+ if plan["t2v"]:# All-zero image == "no image" (t2v). Emptying the token axis removes the 729 padded image tokens; is_t2v# stays True in forward (all() of empty is vacuously True).kwargs["image_embeds"] = image[:, :0]@@ -2119,7 +2185,7 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):if skey not in kwargs:null_ok = null_ok and not requiredcontinue- states, mask, all_valid = _trim_stream(kwargs.get(skey), kwargs.get(mkey))+ states, mask, all_valid = _apply_trim(kwargs.get(skey), kwargs.get(mkey), plan[mkey])kwargs[skey] = stateskwargs[mkey] = masknull_ok = null_ok and all_valid@@ -2210,6 +2276,8 @@ def install_hunyuan_attention_trim(continue# installation and every idle period start in the conservative state_set_hunyuan_null_mask(dit, False)+ # The flag picks the attention branch inside the forward, so a CUDA graph keys on it (diffusion_cuda_graph).+ dit._unsloth_graph_key_extra = functools.partial(_hunyuan_null_mask_state, dit)if getattr(dit, "_unsloth_trim_hook", None) is None:pre_handle = Nonetry:diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex 6e8777aa606..371961f3e23 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -20,6 +20,15 @@ from typing import Any, Callable, Optionalfrom .diffusion_bg_compile import capture_suppressed as _bg_capture_suppressedfrom .diffusion_bg_compile import eager_forced as _bg_eager_forced++def _step_recording() -> bool:+ try:+ from .diffusion_cuda_graph import step_recording+ except Exception: # noqa: BLE001+ return False+ return step_recording()++BLOCK_GRAPHS_ENV = "UNSLOTH_DIFFUSION_BLOCK_GRAPHS"# Recordings kept per block (input layouts x weight placements); the least recently replayed goes first.@@ -438,7 +447,8 @@ class BlockGraph:return self.compute(*args, **kwargs)def __call__(self, *args: Any, **kwargs: Any) -> Any:- if _bg_capture_suppressed():+ if _bg_capture_suppressed() or _step_recording():+ # a whole-step graph above records this block's kernels (diffusion_cuda_graph), not a nested replayreturn self.compute(*args, **kwargs)if not self.enabled or self.bypassed or self.poisoned or self.churned or _bg_eager_forced():return self._eager(args, kwargs)diff --git a/studio/backend/core/inference/diffusion_capture_safe.py b/studio/backend/core/inference/diffusion_capture_safe.pyindex 56e8f373cdc..8e9f2507efa 100644--- a/studio/backend/core/inference/diffusion_capture_safe.py+++ b/studio/backend/core/inference/diffusion_capture_safe.py@@ -39,6 +39,12 @@ The rewrite takes the INSTALLED forward's own source and swaps just that block,unpatchify) stays whatever the installed diffusers ships. The block is matched line for line; if itchanged, nothing is rewritten and the graph layer declines the load with the reason.+HunyuanVideo-1.5 has the same merge over three streams ([image ; byt5 ; mllm], padded byt5/mllm rows+zeroed) plus ``is_t2v = torch.all(image_embeds == 0)`` read by ``if`` on the host. Both blocks are+swapped: the branch becomes ``torch.where`` on the device bool (keeping ``* 0.0`` exactly), the merge+the same stable-sort permutation with ``torch.where`` zeroing. Identical in diffusers 0.36.0 through+0.40.0.+Kill switch: ``UNSLOTH_DIFFUSION_CAPTURE_SAFE=0`` (the graph is then declined for these classes)."""@@ -129,6 +135,148 @@ def merge_text_streams(return states, mask+_HUNYUANVIDEO15_CLS = "HunyuanVideo15Transformer3DModel"++# HunyuanVideo-1.5 (diffusers 0.36.0 through 0.40.0, unchanged): ``is_t2v`` is a 0-d device bool that+# ``if`` reads on the host, then the per-batch merge of [image ; byt5 ; mllm] uses boolean-mask+# indexing like HunyuanImage's, with the padded byt5/mllm rows replaced by zeros.+_HV15_T2V_BLOCK: tuple[str, ...] = (+ "is_t2v = torch.all(image_embeds == 0)",+ "if is_t2v:",+ "encoder_hidden_states_3 = encoder_hidden_states_3 * 0.0",+ "encoder_attention_mask_3 = torch.zeros(",+ "(batch_size, encoder_hidden_states_3.shape[1]),",+ "dtype=encoder_attention_mask.dtype,",+ "device=encoder_attention_mask.device,",+ ")",+ "else:",+ "encoder_attention_mask_3 = torch.ones(",+ "(batch_size, encoder_hidden_states_3.shape[1]),",+ "dtype=encoder_attention_mask.dtype,",+ "device=encoder_attention_mask.device,",+ ")",+)++_HV15_MERGE_BLOCK: tuple[str, ...] = (+ "new_encoder_hidden_states = []",+ "new_encoder_attention_mask = []",+ "for text, text_mask, text_2, text_mask_2, image, image_mask in zip(",+ "encoder_hidden_states,",+ "encoder_attention_mask,",+ "encoder_hidden_states_2,",+ "encoder_attention_mask_2,",+ "encoder_hidden_states_3,",+ "encoder_attention_mask_3,",+ "):",+ "# Concatenate: [valid_image, valid_byt5, valid_mllm, invalid_image, invalid_byt5, invalid_mllm]",+ "new_encoder_hidden_states.append(",+ "torch.cat(",+ "[",+ "image[image_mask], # valid image",+ "text_2[text_mask_2], # valid byt5",+ "text[text_mask], # valid mllm",+ "image[~image_mask], # invalid image",+ "torch.zeros_like(text_2[~text_mask_2]), # invalid byt5 (zeroed)",+ "torch.zeros_like(text[~text_mask]), # invalid mllm (zeroed)",+ "],",+ "dim=0,",+ ")",+ ")",+ "# Apply same reordering to attention masks",+ "new_encoder_attention_mask.append(",+ "torch.cat(",+ "[",+ "image_mask[image_mask],",+ "text_mask_2[text_mask_2],",+ "text_mask[text_mask],",+ "image_mask[~image_mask],",+ "text_mask_2[~text_mask_2],",+ "text_mask[~text_mask],",+ "],",+ "dim=0,",+ ")",+ ")",+ "encoder_hidden_states = torch.stack(new_encoder_hidden_states)",+ "encoder_attention_mask = torch.stack(new_encoder_attention_mask)",+)++_HV15_T2V_HELPER_NAME = "_unsloth_hv15_image_stream"+_HV15_T2V_CALL = (+ f"encoder_hidden_states_3, encoder_attention_mask_3 = {_HV15_T2V_HELPER_NAME}("+ "image_embeds, encoder_hidden_states_3, encoder_attention_mask, batch_size)"+)+_HV15_MERGE_HELPER_NAME = "_unsloth_hv15_merge_streams"+_HV15_MERGE_CALL = (+ f"encoder_hidden_states, encoder_attention_mask = {_HV15_MERGE_HELPER_NAME}("+ "encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2, "+ "encoder_hidden_states_3, encoder_attention_mask_3)"+)+++def hv15_image_stream(+ image_embeds: Any, encoder_hidden_states_3: Any, encoder_attention_mask: Any, batch_size: int+) -> tuple[Any, Any]:+ """HunyuanVideo-1.5's ``if is_t2v`` branch as arithmetic; bit-identical to both stock arms.++ ``torch.where`` picks each element from ``states * 0.0`` (the stock t2v arm, NaN/-0.0 semantics+ included) or ``states`` (the i2v arm) by the 0-d device bool, so the host never reads it. The+ mask is all zeros (t2v) or all ones (i2v) in the primary mask's dtype and device, as stock.+ """+ import torch++ is_t2v = torch.all(image_embeds == 0)+ states = torch.where(is_t2v, encoder_hidden_states_3 * 0.0, encoder_hidden_states_3)+ mask = (+ torch.logical_not(is_t2v)+ .to(dtype = encoder_attention_mask.dtype, device = encoder_attention_mask.device)+ .expand(batch_size, encoder_hidden_states_3.shape[1])+ .contiguous()+ )+ return states, mask+++def hv15_merge_streams(+ encoder_hidden_states: Any,+ encoder_attention_mask: Any,+ encoder_hidden_states_2: Any,+ encoder_attention_mask_2: Any,+ encoder_hidden_states_3: Any,+ encoder_attention_mask_3: Any,+) -> tuple[Any, Any]:+ """HunyuanVideo-1.5's three-stream merge without a host sync; bit-identical to the stock loop.++ Streams: mllm ``[B, S1, D]``, byt5 ``[B, S2, D]``, image ``[B, S3, D]``, each with a bool mask.+ Stock order per batch element is [valid image, valid byt5, valid mllm, padded image, padded+ byt5 zeroed, padded mllm zeroed]: a stable sort of ``[image ; byt5 ; mllm]`` on "is padding".+ Padded byt5/mllm rows become exact zeros (``torch.where`` with 0, as ``zeros_like``), padded+ image rows keep their values. ``torch.cat`` promotes dtypes as the stock per-row cat does.+ """+ import torch++ states = torch.cat(+ [encoder_hidden_states_3, encoder_hidden_states_2, encoder_hidden_states], dim = 1+ )+ mask = torch.cat(+ [encoder_attention_mask_3, encoder_attention_mask_2, encoder_attention_mask], dim = 1+ )+ padded = ~mask+ zero_rows = torch.cat(+ [+ torch.zeros_like(encoder_attention_mask_3),+ ~encoder_attention_mask_2,+ ~encoder_attention_mask,+ ],+ dim = 1,+ )+ states = torch.where(+ zero_rows.unsqueeze(-1), torch.zeros((), dtype = states.dtype, device = states.device), states+ )+ order = torch.argsort(padded.to(torch.uint8), dim = 1, stable = True)+ states = torch.gather(states, 1, order.unsqueeze(-1).expand(-1, -1, states.shape[-1]))+ mask = torch.gather(mask, 1, order)+ return states, mask++def _find_block(lines: list[str], block: tuple[str, ...]) -> Optional[tuple[int, int]]:"""``(start, end)`` line indices (end exclusive) of ``block`` in ``lines``, blank lines ignored.@@ -158,10 +306,15 @@ def _signature_key(fn: Callable) -> Optional[tuple]:return tuple((p.name, p.kind, repr(p.default)) for p in params)-def _rewrite_hunyuanimage(forward: Callable) -> Callable:- """Rebuild ``forward`` from its own source with the merge block swapped for the helper call.+def _rebuild_forward(+ forward: Callable,+ swaps: tuple[tuple[tuple[str, ...], str, str], ...],+ helpers: dict[str, Callable],+) -> Callable:+ """Rebuild ``forward`` from its own source with each ``(block, replacement, what)`` swapped.- Raises ``RuntimeError`` naming what did not match, so the reason reaches the status payload."""+ Every block must occur exactly once. Raises ``RuntimeError`` naming what did not match, so the+ reason reaches the status payload."""target = inspect.unwrap(forward)module = sys.modules.get(getattr(target, "__module__", "") or "")if module is None:@@ -171,20 +324,30 @@ def _rewrite_hunyuanimage(forward: Callable) -> Callable:except (OSError, TypeError) as exc:raise RuntimeError(f"its source is unreadable ({exc})") from Nonelines = source.splitlines()- span = _find_block(lines, _HUNYUANIMAGE_MERGE_BLOCK)- if span is None:- raise RuntimeError("the boolean-mask text merge block changed in this diffusers version")- start, end = span- indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]- new_lines = lines[:start] + [indent + _MERGE_CALL] + lines[end:]+ spans = []+ for block, replacement, what in swaps:+ span = _find_block(lines, block)+ if span is None:+ raise RuntimeError(f"the {what} block changed in this diffusers version")+ spans.append((span, replacement))+ spans.sort(key = lambda item: item[0][0])+ for (prev, _), (nxt, _) in zip(spans, spans[1:]):+ if nxt[0] < prev[1]:+ raise RuntimeError("the rewritten blocks overlap")+ # Splice from the bottom so earlier spans keep their line indices.+ for (start, end), replacement in reversed(spans):+ indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]+ lines = lines[:start] + [indent + replacement] + lines[end:]# A source-level decorator (``@apply_lora_scale`` from 0.37) re-applies when the def executes,# so the rebuilt function carries the installed version's LoRA handling unchanged.- new_source = "\n".join(new_lines) + "\n"+ new_source = "\n".join(lines) + "\n"filename = f"<unsloth capture-safe {getattr(target, '__qualname__', 'forward')}>"namespace = dict(vars(module))- namespace[_MERGE_HELPER_NAME] = merge_text_streams- code = compile(new_source, filename, "exec")- exec(code, namespace) # noqa: S102 - the installed diffusers' own source with one block replaced+ namespace.update(helpers)+ # dont_inherit: this module's ``from __future__ import annotations`` must not stringify the+ # rebuilt forward's annotations.+ code = compile(new_source, filename, "exec", dont_inherit = True)+ exec(code, namespace) # noqa: S102 - the installed diffusers' own source with known blocks replacedrebuilt = namespace.get(target.__name__)if not callable(rebuilt):raise RuntimeError("the rebuilt source did not define the forward")@@ -196,18 +359,48 @@ def _rewrite_hunyuanimage(forward: Callable) -> Callable:return rebuilt+def _rewrite_hunyuanimage(forward: Callable) -> Callable:+ """HunyuanImage-2.1: the boolean-mask text merge becomes ``merge_text_streams``."""+ return _rebuild_forward(+ forward,+ ((_HUNYUANIMAGE_MERGE_BLOCK, _MERGE_CALL, "boolean-mask text merge"),),+ {_MERGE_HELPER_NAME: merge_text_streams},+ )+++def _rewrite_hunyuanvideo15(forward: Callable) -> Callable:+ """HunyuanVideo-1.5: the ``if is_t2v`` branch and the three-stream merge, both made device-only."""+ return _rebuild_forward(+ forward,+ (+ (_HV15_T2V_BLOCK, _HV15_T2V_CALL, "t2v image-stream branch"),+ (_HV15_MERGE_BLOCK, _HV15_MERGE_CALL, "boolean-mask three-stream merge"),+ ),+ {+ _HV15_T2V_HELPER_NAME: hv15_image_stream,+ _HV15_MERGE_HELPER_NAME: hv15_merge_streams,+ },+ )++# class name -> (why the stock forward cannot be captured, rewriter)_REWRITES: dict[str, tuple[str, Callable[[Callable], Callable]]] = {_HUNYUANIMAGE_CLS: ("boolean-mask indexing in its text-stream merge syncs the host",_rewrite_hunyuanimage,),+ _HUNYUANVIDEO15_CLS: (+ "its t2v branch reads a device bool on the host and boolean-mask indexing in its "+ "three-stream merge syncs the host",+ _rewrite_hunyuanvideo15,+ ),}-# Declined at load so status says "off" instead of arming a wrapper that refuses every step.+# Declined at load so status says "off" instead of arming a wrapper that refuses every step. Qwen-Image-2.1 only+# lands here when its fast step (diffusion_qwenimage21, which plans tensors-only decode calls) is not installed._UNCAPTURABLE: dict[str, str] = {"QwenImage21Transformer2DModel": "its pipeline passes the prefix KV cache as a Python object "- "on every step and its forward syncs the host",+ "on every step; the Studio fast step that replays decode steps from tensors is not installed",}_CACHE: dict = {}@@ -229,6 +422,16 @@ def resolve(cls: type) -> tuple[Optional[Callable], Optional[str]]:Never raises. Cheap for an unknown class: a name lookup, no torch import."""owner = _defining_class(cls)+ planned = vars(owner).get("forward") if owner is not None else None+ if getattr(planned, "__unsloth_graph_plan__", None) is not None:+ # A Studio forward that plans its own capture-safe calls (Qwen-Image-2.1's fast step).+ if capture_safe_disabled():+ return (+ None,+ f"{owner.__name__} forward is not capture-safe ({_UNCAPTURABLE.get(owner.__name__, 'planned')}); "+ f"rewrite disabled by {CAPTURE_SAFE_ENV}",+ )+ return planned, Noneif owner is not None and owner.__name__ in _UNCAPTURABLE and owner.__name__ not in _REWRITES:return (None,diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 71152e0471e..c71733fc17e 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -17,6 +17,7 @@ import osimport threadingimport tracebackimport weakref+import typesfrom functools import update_wrapperfrom typing import Any, Optional@@ -26,6 +27,10 @@ from .diffusion_bg_compile import eager_forced as _bg_eager_forcedCUDA_GRAPH_DISABLE_ENV = "UNSLOTH_DISABLE_CUDA_GRAPH"# Master switch for every diffusion CUDA graph (whole-forward and per-block): "0" turns them all off.CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"+# =0: an offloaded denoiser never records its whole step with the copies inside (per-block graphs still may).+OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"+# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.+STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"# Per module. Legitimate second keys exist, but each costs a pool-sized slice of VRAM.MAX_GRAPHS_PER_MODULE = 4@@ -42,6 +47,29 @@ def _torch():return torch+# Comma list of family names (or "all") armed whatever the family declares: for measuring a family before it opts in.+FORCE_FAMILIES_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPH_FAMILIES"+# A module attribute: zero-arg callable returning hashable per-call state that the forward branches on.+GRAPH_KEY_EXTRA_ATTR = "_unsloth_graph_key_extra"+++def _family_forced(family: Any) -> bool:+ raw = (os.environ.get(FORCE_FAMILIES_ENV) or "").strip().lower()+ if not raw:+ return False+ names = {n.strip() for n in raw.split(",") if n.strip()}+ return "all" in names or str(getattr(family, "name", "")).lower() in names+++def offload_graphs_enabled() -> bool:+ return (os.environ.get(OFFLOAD_CUDA_GRAPH_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )++def cuda_graph_disabled() -> bool:"""Whether an env kill switch is set. Read before any torch or pipe inspection."""if os.environ.get(CUDA_GRAPHS_ENV, "").strip().lower() in ("0", "off", "false", "no"):@@ -109,6 +137,23 @@ def hold_off_capture():_CAPTURE_LOCK.release()+# Raised while a whole step warms up or records: per-block graphs then run their compute, not nested replays.+_STEP_RECORDING = [0]+++def _recording_step():+ _STEP_RECORDING[0] += 1+ try:+ yield+ finally:+ _STEP_RECORDING[0] -= 1+++def step_recording() -> bool:+ return _STEP_RECORDING[0] > 0++def capture_in_progress() -> bool:"""Whether a capture is recording, as a snapshot for reporting.@@ -119,6 +164,26 @@ def capture_in_progress() -> bool:return _CAPTURE_DEPTH > 0+def _graph_without_flush(graph: Any, pool: Any = None):+ """``torch.cuda.graph`` minus its ``empty_cache`` / host ``emptyCache``: same capture stream discipline, same+ global capture mode, same pool."""+ torch = _torch()+ torch.cuda.synchronize()+ # torch.cuda.graph's own capture stream: pool blocks are reused only from the stream that recorded them.+ stream = getattr(torch.cuda.graph, "default_capture_stream", None)+ if stream is None or getattr(stream, "device", None) != torch.device(+ "cuda", torch.cuda.current_device()+ ):+ stream = torch.cuda.Stream()+ with torch.cuda.stream(stream):+ graph.capture_begin(pool = pool, capture_error_mode = "global")+ try:+ yield+ finally:+ graph.capture_end()++# Not ``torch.utils._pytree``: it makes an unregistered object a LEAF, which must stay visible.@@ -169,6 +234,12 @@ def graph_key(obj: Any) -> tuple:return ("o", type(obj).__name__, id(obj))+def _copy_mode(entry: Any) -> Any:+ if getattr(entry, "inference_copy", False):+ return _torch().inference_mode()+ return contextlib.nullcontext()++def _uncapturable(key: tuple) -> bool:kind = key[0]if kind == "o":@@ -200,10 +271,17 @@ _LIVE_WRAPPERS: "weakref.WeakSet" = weakref.WeakSet()def _drop_pool_if_unused() -> None:+ """Forget the shared pool id once no live graph records into it: capturing into a pool whose graphs are all gone+ dies on the allocator's use_count assert. Checked per graph, not per wrapper: a wrapper still holding graphs in an+ EARLIER pool (one dropped before) does not keep the current id alive."""try:+ pool = _POOL_BOX[0]+ if pool is None:+ returnfor wrapper in tuple(_LIVE_WRAPPERS):- if wrapper.cache:- return+ for entry in tuple(wrapper.cache.values()):+ if getattr(entry, "pool_token", pool) == pool: # handles are fresh tuples per call+ return_POOL_BOX[0] = Noneexcept Exception: # noqa: BLE001pass@@ -243,52 +321,6 @@ def refuse_if_exhausted(logger: Any = None) -> None:_EXHAUSTED_LOGGED = False-def _abandon_capture_pool(pool: Any) -> bool:- """After a capture that raised: take the device allocator off the capture's pool if ``capture_end`` never did.-- ``CUDAGraph::capture_end`` checks ``cudaStreamEndCapture`` before ``endAllocateToPool``, so an invalidated capture- leaves the pool in the allocator's ``captures_underway``: on torch 2.6 every later ``empty_cache`` then trips- ``INTERNAL ASSERT captures_underway.empty()``; later torch skips the global release instead, so ``empty_cache``- frees nothing. That failed graph never releases the reference its ``capture_begin`` took (its reset releases only- once ``capture_end`` got past the pool), so it is released here, and ONLY when this call ended the recording: had- ``capture_end`` got that far the graph owns the reference and releases it itself. True when this call ended it."""- torch = _torch()- end = getattr(torch._C, "_cuda_endAllocateToPool", None) or getattr(- torch._C, "_cuda_endAllocateCurrentStreamToPool", None- )- if end is None or pool is None:- return False- try:- device = torch.cuda.current_device()- end(device, pool)- except Exception: # noqa: BLE001 - capture_end already took it off, or capture_begin never put it on- return False- try:- torch._C._cuda_releasePool(device, pool)- except Exception: # noqa: BLE001- pass- return True---def _heal_generators() -> None:- """Take every CUDA default generator out of graph-capture mode after a FAILED capture.-- ``CUDAGraph.capture_end`` ends the generators' capture only after ``cudaStreamEndCapture`` succeeds. When the- capture was invalidated (a host sync inside it, a kernel that may not be recorded) it raises first, and every later- eager draw from the generator (``torch.randn(device = "cuda")``, a pipeline's noise) fails with "Offset increment- outside graph capture encountered unexpectedly" for the rest of the process. A clone of the state keeps the seed- and the eager offset, so the eager sequence continues as if the capture had never run."""- try:- torch = _torch()- for gen in getattr(torch.cuda, "default_generators", ()) or ():- clone = getattr(gen, "clone_state", None)- restore = getattr(gen, "graphsafe_set_state", None)- if callable(clone) and callable(restore):- restore(clone())- except Exception: # noqa: BLE001- pass--def retire_failed_capture(graph: Any,pool: Any,@@ -368,6 +400,149 @@ def _prewarm_token_counts(live: list) -> tuple:return tuple(sorted(counts)[:8])+def _pool_bytes(graph: Any) -> int:+ """Device bytes the graph's private pool holds (shared by every graph of the load), 0 if unreadable."""+ try:+ pool = tuple(graph.pool())+ total = 0+ for seg in _torch().cuda.memory_snapshot():+ if tuple(seg.get("segment_pool_id") or ()) == pool:+ total += int(seg.get("total_size", 0))+ return total+ except Exception: # noqa: BLE001+ return 0+++def _current_stream() -> Any:+ try:+ return _torch().cuda.current_stream()+ except Exception: # noqa: BLE001+ return None+++def _restore_stream(stream: Any) -> None:+ if stream is None:+ return+ try:+ torch = _torch()+ if torch.cuda.current_stream() != stream:+ torch.cuda.set_stream(stream)+ except Exception: # noqa: BLE001+ pass+++def _abandon_capture_pool(pool: Any) -> bool:+ """After a capture that raised: take the allocator off the capture's pool if ``capture_end`` never did.++ ``CUDAGraph::capture_end`` checks ``cudaStreamEndCapture`` before ``endAllocateToPool``, so an invalidated capture+ can leave the pool in the allocator's ``captures_underway``: on torch 2.6 every later ``empty_cache`` then trips+ ``INTERNAL ASSERT captures_underway.empty()`` and no later capture in the process records; later torch skips the+ global release instead, so ``empty_cache`` frees nothing. That failed graph never releases the reference its+ ``capture_begin`` took (its reset releases only once ``capture_end`` got past the pool), so it is released here,+ and ONLY when this call ended the capture: had ``capture_end`` got that far (an error raised in the forward that+ did not invalidate the stream, a failed instantiate) the graph owns the reference and releases it itself, and a+ second release here would drop a shared pool under live graphs or abort the process. True when this call ended it."""+ torch = _torch()+ end = getattr(torch._C, "_cuda_endAllocateToPool", None) or getattr(+ torch._C, "_cuda_endAllocateCurrentStreamToPool", None+ )+ if end is None or pool is None:+ return False+ try:+ device = torch.cuda.current_device()+ end(device, pool)+ except Exception: # noqa: BLE001 - capture_end already took it off, or capture_begin never put it on+ return False+ try:+ torch._C._cuda_releasePool(device, pool)+ except Exception: # noqa: BLE001+ pass+ return True+++def _heal_generators() -> None:+ """Take every CUDA default generator out of graph-capture mode after a FAILED capture.++ ``CUDAGraph.capture_end`` ends the generators' capture only after ``cudaStreamEndCapture`` succeeds. When the+ capture was invalidated (a host sync inside it, a kernel that may not be recorded) it raises first, and every later+ eager draw from the generator (``torch.randn(device = "cuda")``, a pipeline's noise) fails with "Offset increment+ outside graph capture encountered unexpectedly" for the rest of the process. A clone of the state keeps the seed+ and the eager offset, so the eager sequence continues as if the capture had never run."""+ try:+ torch = _torch()+ for gen in getattr(torch.cuda, "default_generators", ()) or ():+ clone = getattr(gen, "clone_state", None)+ restore = getattr(gen, "graphsafe_set_state", None)+ if callable(clone) and callable(restore):+ restore(clone())+ except Exception: # noqa: BLE001+ pass+++# Offloaded: each key's first SPEED_SAMPLES replays are judged against its own eager steps (_judge_keys).+SPEED_SAMPLES = 3+# Eager steps timed before a key's capture (after one untimed warm-up); never the capture's own warm-ups, whose+# copies follow the recorded schedule and so measure the replay.+SPEED_EAGER_SAMPLES = 2+# A key's replay this much slower than its eager step (and by at least SPEED_MIN_MS) drops that key's graph.+SPEED_MARGIN = 0.03+SPEED_MIN_MS = 2.0+# Streamed: the recording holds a ring and a pool the eager step does not, so it must be at least this much faster.+SPEED_GAIN = 0.01+SPEED_CHECK_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_GRAPH_SPEED_CHECK"+++def speed_check_enabled() -> bool:+ return (os.environ.get(SPEED_CHECK_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _timing_events() -> tuple:+ torch = _torch()+ return torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)+++def live_pool_bytes() -> int:+ """Bytes the shared graph pool holds while any graph lives (the status number)."""+ try:+ return max(+ [int(w.stats.get("pool_bytes", 0)) for w in tuple(_LIVE_WRAPPERS) if w.cache] or [0]+ )+ except Exception: # noqa: BLE001+ return 0+++def live_pool_free_bytes() -> int:+ """Bytes of the live shared pool that are reserved but NOT allocated: what ``memory_reserved() -+ memory_allocated()`` would wrongly credit as reusable. The pool's allocated part (the graphs' static outputs) is+ already outside that difference. 0 when no graph lives or the snapshot is unreadable."""+ # Every pool a live graph recorded into: a failed capture retires the box's pool while earlier graphs keep it.+ pools = {+ tuple(entry.pool_token)+ for w in tuple(_LIVE_WRAPPERS)+ for entry in tuple(w.cache.values())+ if getattr(entry, "pool_token", None) is not None+ }+ if not pools:+ return 0+ try:+ free = 0+ cuda = _torch().cuda+ device = cuda.current_device() # the snapshot spans every card+ for seg in cuda.memory_snapshot():+ if seg.get("device", device) != device:+ continue+ if tuple(seg.get("segment_pool_id") or ()) in pools:+ free += int(seg.get("total_size", 0)) - int(seg.get("allocated_size", 0))+ return max(0, free)+ except Exception: # noqa: BLE001 - over-counting only makes the guard stricter+ return live_pool_bytes()++def _warn(logger: Any, what: str, exc: Any) -> None:if logger is not None:logger.warning("diffusion.cuda_graph: %s failed: %s", what, exc)@@ -385,7 +560,17 @@ def _outer_layer(module: Any) -> Any:class _Entry:"""One captured graph plus the static buffers it replays into."""- __slots__ = ("graph", "static", "in_spec", "out_spec", "out_tensors")+ __slots__ = (+ "graph",+ "static",+ "in_spec",+ "out_spec",+ "out_tensors",+ "sticky",+ "last",+ "inference_copy",+ "pool_token",+ )class GraphedForward:@@ -402,25 +587,44 @@ class GraphedForward:max_graphs: int = MAX_GRAPHS_PER_MODULE,logger: Any = None,call: Any = None,+ placement: Optional["OffloadPlacement"] = None,) -> None:self.module = module+ # How the weights reach the GPU (None = resident); see OffloadPlacement.+ self.placement = placement# The CLASS forward: the eager callable, whatever is already in the instance slot. A class# whose stock forward syncs the host gets its capture-safe rewrite (bit-identical) instead.# ``call`` overrides it (GraphedCompiledCall: the module's whole-module compiled callable).safe = None- if call is None:+ if call is None or placement is not None:try:from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415safe, _ = _capture_safe(type(module))except Exception: # noqa: BLE001 - the stock forward is still a correct eager callablesafe = Noneself.capture_safe = safe is not None+ # A forward whose call carries a non-tensor object (Qwen-Image-2.1's KV cache) names a planner that turns a+ # call into a tensors-only one, or None for "run this call eager"; ``sticky`` kwargs only change between+ # renders, so a replay copies them when the caller passes a different tensor object.+ self.plan = getattr(safe, "__unsloth_graph_plan__", None) if safe is not None else None+ self.sticky = frozenset(getattr(safe, "__unsloth_graph_sticky__", ()) or ())+ # How a planned step enters through the offload hooks (offload_placement refuses a placement without it).+ self.placed = getattr(safe, "__unsloth_graph_placed__", None) if safe is not None else None+ if placement is not None:+ # Model offload calls the slot with the weights already on the GPU, so a rewrite can sit there; a block+ # offload slot holds the hook chain, which then reaches the planned step through ``_placed_step``.+ rewrite = safe is not None and self.plan is None and placement.mode == "model"+ call = safe.__get__(module) if rewrite else placement.eagerif call is not None:self.orig = callelse:self.orig = (safe if safe is not None else type(module).forward).__get__(module)try:- update_wrapper(self, self.orig)+ update_wrapper(self, self.orig, updated = ())+ # The wrapped callable's attributes, but never over this wrapper's own: under block offload ``orig`` can+ # be the static step skip, whose ``plan`` / ``stats`` would otherwise replace the graph's.+ for name, value in dict(getattr(self.orig, "__dict__", None) or {}).items():+ self.__dict__.setdefault(name, value)except Exception: # noqa: BLE001passtry:@@ -438,6 +642,17 @@ class GraphedForward:self.capture_error: Optional[dict] = Noneself.cache: dict = {}self.cap_hit = False+ self._warmed: dict = {}+ # Keys recorded before: a placement change re-records them with no warm-up (same shapes, new addresses).+ self._recorded: dict = {}+ # Offloaded only, per graph key: {"seen": eager calls, "eager": [(start, end)], "graph": [(start, end)],+ # "verdict": None until judged, then "" (kept) or the reason it was dropped}.+ self._judge: dict = {}+ self._dropped: set = set() # keys that run eager for the load+ self._slower: Optional[str] = None+ # Per-block fallback for the calls this graph leaves eager: its arming callable and the handle it returned.+ self.fallback: Any = None+ self.fallback_handle: Any = Noneself.protect_keyed: Optional[bool] = Noneself.protect_ctl: Any = Noneself.stats = {@@ -449,11 +664,21 @@ class GraphedForward:"refused_host_tensor": 0,"refused_object": 0,"cap_skips": 0,+ "invalidations": 0,+ "pool_bytes": 0,+ "planned_eager": 0,+ "shape_warmups": 0,+ "evictions": 0,+ "speed_eager": 0,}+ self.valid_token: Any = None_LIVE_WRAPPERS.add(self)def install(self) -> "GraphedForward":"""Set ``forward`` via ``__dict__`` (``__setattr__`` would inspect it), under any outer step-skip layer."""+ if self.placement is not None:+ self.placement.install(self)+ return selfouter = _outer_layer(self.module)if outer is not None:outer.inner = self@@ -462,12 +687,22 @@ class GraphedForward:return selfdef uninstall(self) -> "GraphedForward":+ if self.placement is not None:+ self.placement.uninstall(self)+ return selfouter = _outer_layer(self.module)if outer is not None:if outer.inner is self:outer.inner = Nonereturn self- self.module.__dict__.pop("forward", None)+ if self.module.__dict__.get("forward") is self:+ self.module.__dict__.pop("forward", None)+ return self+ # Offload hooks installed after this wrapper wrap it: unwrap it inside their chain, keeping the hooks.+ registry = getattr(self.module, "_diffusers_hook", None)+ for ref in list(getattr(registry, "_fn_refs", None) or ()):+ if getattr(ref, "forward", None) is self:+ ref.forward = type(self.module).forward.__get__(self.module)return selfdef enable(self) -> "GraphedForward":@@ -484,13 +719,21 @@ class GraphedForward:def set_bypass(self, on: bool) -> "GraphedForward":"""Run eager without dropping the captured graphs (a chunk under a step cache)."""self.bypassed = bool(on)+ if getattr(self, "fallback_handle", None) is not None:+ self.fallback_handle.set_bypass(on)return selfdef _release(self) -> None:"""Return the dropped graphs' pool segments to the device; ``gc.collect`` alone leaves them in the private pool."""try:gc.collect()- cuda = getattr(_torch(), "cuda", None)+ torch = _torch()+ cuda = getattr(torch, "cuda", None)+ if getattr(self, "placement", None) is not None:+ # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them+ clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)+ if callable(clear):+ clear()if cuda is not None and hasattr(cuda, "empty_cache"):cuda.empty_cache()except Exception: # noqa: BLE001 - best effort@@ -499,6 +742,12 @@ class GraphedForward:def reset(self) -> "GraphedForward":"""Drop every captured graph (weights changed under us: LoRA load, unload, adapter switch)."""self.cache.clear()+ self._recorded.clear()+ if getattr(self, "fallback_handle", None) is not None:+ try:+ self.fallback_handle.reset()+ except Exception: # noqa: BLE001+ passself._release()return self@@ -521,7 +770,16 @@ class GraphedForward:def free(self) -> "GraphedForward":"""Drop the graphs and the slot."""self.reset()+ self.fallback = None+ if getattr(self, "fallback_handle", None) is not None:+ try:+ self.fallback_handle.free()+ except Exception: # noqa: BLE001+ pass+ self.fallback_handle = Noneself.uninstall()+ if self.placement is not None:+ self.placement.release_slots()self.enabled = False_LIVE_WRAPPERS.discard(self)return self@@ -538,7 +796,11 @@ class GraphedForward:"warmup": int(self.warmup),"max_graphs": int(self.max_graphs),"cap_hit": bool(self.cap_hit),+ "placement": None if self.placement is None else self.placement.mode,"stats": dict(self.stats),+ "fallback": None+ if getattr(self, "fallback_handle", None) is None+ else self.fallback_handle.describe(),"capture_error": Noneif not errorelse {@@ -547,14 +809,149 @@ class GraphedForward:},}+ def _judge_keys(self) -> None:+ """Drop the graph of every key whose replays measured slower than its own eager steps.++ A replay records the onload copies too, and how fast a recorded copy schedule runs depends on the machine and+ on how it was recorded (an A100 VM: a Qwen-Image-2.1 prompt length recorded after the first replayed 16%+ slower than eager while the first matched it; an L4 ran replayed copies 8% faster). Each key's first replays+ are timed against its own eager steps with CUDA events, read only once they have completed (no host wait)."""+ for key, state in list(self._judge.items()):+ if state["verdict"] is not None or len(state["graph"]) < SPEED_SAMPLES:+ continue+ # Event queries and the drop's syncs / frees are prohibited while another thread records: judge later.+ with hold_off_capture() as safe:+ if not safe or not all(end.query() for _, end in state["graph"]):+ continue+ try:+ # Best of each, so a stall on a shared card in either window cannot tip the verdict alone.+ eager = min(start.elapsed_time(end) for start, end in state["eager"])+ graph = min(start.elapsed_time(end) for start, end in state["graph"])+ except Exception: # noqa: BLE001+ state["verdict"] = ""+ continue+ self.stats["eager_ms"] = round(eager, 3)+ self.stats["replay_ms"] = round(graph, 3)+ placement = getattr(self, "placement", None)+ streams = placement is not None and placement.streams()+ if streams:+ kept = graph < eager * (1.0 - SPEED_GAIN)+ else:+ kept = graph <= eager * (1.0 + SPEED_MARGIN) or graph - eager < SPEED_MIN_MS+ if kept:+ state["verdict"] = ""+ continue+ if graph > eager:+ reason = f"replay measured slower than the eager step on this device ({graph:.1f} vs {eager:.1f} ms)"+ else:+ reason = (+ f"replay measured within {SPEED_GAIN:.0%} of the eager step on this device ({graph:.1f} vs "+ f"{eager:.1f} ms), not worth the memory a streamed recording holds"+ )+ state["verdict"] = reason+ ring = False+ if streams:+ # One placement, one verdict: its other keys stream the same copies.+ ring = bool(placement.decline(reason))+ for other in list(self.cache):+ if other != key:+ self.cache.pop(other, None)+ self._dropped.add(other)+ self._slower = reason+ self._dropped.add(key)+ if self.logger is not None:+ self.logger.info(+ "diffusion.cuda_graph: %s drops a graph: %s",+ type(self.module).__name__,+ reason,+ )+ if self.cache.pop(key, None) is not None:+ try:+ _torch().cuda.current_stream().synchronize() # no replay still running on what is freed+ except Exception: # noqa: BLE001+ pass+ _drop_pool_if_unused()+ if not self.cache:+ self.capture_error = {"type": "Refused", "msg": reason}+ if not ring:+ # (with a ring, its drop at the end of this forward flushes once: a mid-forward flush fragments the cache)+ self._release()++ def _timed_eager(self, call: Any, args: tuple, kwargs: dict, key: Any) -> Any:+ """One of the caller's eager steps before ``key`` records (its eager reference).++ ``call`` is the step the graph would record (the planned step under a plan), on fresh copies of the caller's+ tensors. The first call of a key may still compile or autotune and is not timed (for a planned module it is+ also the new prompt length's warm-up); the next SPEED_EAGER_SAMPLES are."""+ self.stats["eager_calls"] += 1+ self.stats["speed_eager"] += 1+ # On copies laid out like the capture's statics, as the shape warm-up does: compiled code guards on strides and+ # storage offsets, so steps on the caller's views compile a variant the capture then compiles again.+ live: list = []+ spec = _flatten((args, kwargs), live)+ try:+ statics = self._statics(live)+ except _torch().cuda.OutOfMemoryError:+ # No room for the probe's copies: decline the key and run the caller's own step.+ self._dropped.add(key)+ self._slower = "no memory for the speed probe's input copies"+ return call(*args, **kwargs)+ args, kwargs = _rebuild(spec, statics)+ state = self._judge_state(key)+ state["seen"] += 1+ if state["seen"] == 1:+ self._warmed[key] = None+ return call(*args, **kwargs)+ # Event records are prohibited while another thread records a graph: such a step goes untimed.+ start = end = None+ with hold_off_capture() as safe:+ if safe:+ start, end = _timing_events()+ start.record()+ out = call(*args, **kwargs)+ if start is not None:+ with hold_off_capture() as safe:+ if safe:+ end.record()+ state["eager"].append((start, end))+ return out++ def _judge_state(self, key: Any) -> dict:+ state = self._judge.get(key)+ if state is None:+ state = self._judge[key] = {"seen": 0, "eager": [], "graph": [], "verdict": None}+ while len(self._judge) > 4 * self.max_graphs:+ self._judge.pop(next(iter(self._judge)))+ return state++ def _placed_step(self, **step: Any) -> Any:+ return self.placed(self.orig, step)+def _eager(self, args: tuple, kwargs: dict) -> Any:self.stats["eager_calls"] += 1return self.orig(*args, **kwargs)+ def _engage_fallback(self) -> None:+ """Arm the per-block graphs once this whole-step graph dropped a key or failed: the calls it now leaves eager+ run through them. Between forwards' blocks (the step has not started), so the slot ring can change."""+ fallback, self.fallback = self.fallback, None+ try:+ self.fallback_handle = fallback()+ except Exception as exc: # noqa: BLE001+ _warn(self.logger, "per-block fallback", exc)+ self.fallback_handle = None+ if self.fallback_handle is not None and self.logger is not None:+ self.logger.info(+ "diffusion.cuda_graph: %s records per block for the steps its whole-step graph leaves eager",+ type(self.module).__name__,+ )+def __call__(self, *args, **kwargs):if _bg_capture_suppressed():# Background compile thread: never capture off the render thread.return self.orig(*args, **kwargs)+ if getattr(self, "fallback", None) is not None and (self.poisoned or self._dropped):+ self._engage_fallback()if (not self.enabledor _bg_eager_forced()@@ -569,6 +966,61 @@ class GraphedForward:if kwargs.get("return_dict", True) is not False:return self._eager(args, kwargs)+ if self.placement is not None:+ # The recorded graphs replay the weight addresses of the placement they saw; a new placement records anew.+ token = self.placement.token()+ if token != self.valid_token:+ if self.cache:+ self.stats["invalidations"] += 1+ # Not reset(): its empty_cache on every model-offload onload makes the next onload pay the+ # cudaMallocs again. The dropped pool's blocks stay with the allocator.+ self.cache.clear()+ _drop_pool_if_unused() # the next capture must not record into a pool no graph holds+ self._warmed.clear()+ self.valid_token = token+ refusal = self.placement.refusal(self.stats)+ if self._judge and speed_check_enabled():+ self._judge_keys()+ if self._dropped and getattr(self, "fallback", None) is not None:+ self._engage_fallback()+ refusal = self.placement.refusal(self.stats)+ skip = getattr(self.placement, "skip", None)+ if refusal is None and skip is not None and _skips_planned(skip):+ # This render skips steps: the skip layer must see every call, so they all run eager (per-block+ # graphs, where armed, still record below the hooks).+ self.stats["skip_eager"] = self.stats.get("skip_eager", 0) + 1+ if getattr(self, "fallback", None) is not None:+ self._engage_fallback()+ return self._eager(args, kwargs)+ if refusal is not None:+ if self.logger is not None and getattr(self, "_refusal_logged", None) != refusal:+ self._refusal_logged = refusal+ self.logger.info(+ "diffusion.cuda_graph: %s runs eager: %s",+ type(self.module).__name__,+ refusal,+ )+ self.capture_error = {"type": "Refused", "msg": refusal}+ return self._eager(args, kwargs)+ if (self.capture_error or {}).get("type") == "Refused" and (+ self.cache or not self._slower+ ):+ self.capture_error = None++ call = self.orig+ # Every eager fallback below runs the caller's own call, never the planned one.+ eager_args, eager_kwargs = args, kwargs+ if self.plan is not None:+ planned = self.plan(self.module, args, kwargs)+ if planned is None:+ self.stats["planned_eager"] += 1+ return self._eager(args, kwargs)+ call, kwargs = planned+ args = ()+ if self.placement is not None:+ # The offload hooks onload the top-level weights: the step must enter through the hooked forward.+ call = self._placed_step+try:key = graph_key((args, kwargs))if self.protect_keyed is None:@@ -588,18 +1040,51 @@ class GraphedForward:if self.protect_keyed:# One graph per branch: a W4A4 replay at a W4A16 step means the lever never fired.key = key + protect_graph_key(self.protect_ctl)+ extra = getattr(self.module, GRAPH_KEY_EXTRA_ATTR, None)+ if callable(extra):+ # Python state a pre-hook set for THIS call that picks a branch inside the forward (HunyuanVideo-1.5's+ # null-mask flag): a graph recorded on one branch must never replay for the other.+ key = key + (("x", extra()),)entry = self.cache.get(key)except Exception: # noqa: BLE001 - an unhashable tree is simply not capturableself.stats["refused_object"] += 1- return self._eager(args, kwargs)+ return self._eager(eager_args, eager_kwargs)if entry is None:if _uncapturable(key):self.stats["refused_object"] += 1- return self._eager(args, kwargs)+ return self._eager(eager_args, eager_kwargs)if _has_float(key):self.stats["refused_float"] += 1- return self._eager(args, kwargs)+ return self._eager(eager_args, eager_kwargs)+ if self.placement is not None and speed_check_enabled():+ if key in self._dropped:+ return self._eager(eager_args, eager_kwargs)+ state = self._judge.get(key)+ if state is None or (+ state["verdict"] is None and len(state["eager"]) < SPEED_EAGER_SAMPLES+ ):+ # The eager reference for _judge_keys, timed with CUDA events and never waited on.+ return self._timed_eager(call, args, kwargs, key)+ if self.plan is not None and self.stats["captures"] and key not in self._warmed:+ # A new prompt length: this real step is the new shape's warm-up; the next one records.+ self._warmed[key] = None+ while len(self._warmed) > 4 * self.max_graphs:+ self._warmed.pop(next(iter(self._warmed)))+ self.stats["shape_warmups"] += 1+ # On copies laid out like the capture's statics (fresh, offset 0): compiled code guards on strides and+ # storage offsets, so a step on the caller's views would warm a variant the capture then misses and+ # compiles inside the capture, which fails it.+ live: list = []+ spec = _flatten((args, kwargs), live)+ warm_args, warm_kwargs = _rebuild(spec, self._statics(live))+ return call(*warm_args, **warm_kwargs)+ if len(self.cache) >= self.max_graphs and self.plan is not None:+ # A planned call keys on the prompt length, so evict the least recently replayed graph.+ self.cache.pop(next(iter(self.cache)))+ self.stats["evictions"] += 1+ # The last graph on the shared pool takes the pool with it; never hand its id to the next capture.+ _drop_pool_if_unused()if len(self.cache) >= self.max_graphs:# The recorded graphs keep replaying: one rare new key is no reason to drop them.self.stats["cap_skips"] += 1@@ -612,11 +1097,22 @@ class GraphedForward:self.max_graphs,type(self.module).__name__,)- return self._eager(args, kwargs)+ return self._eager(eager_args, eager_kwargs)captured = None+ stream_before = _current_stream()try:- captured = self._capture(args, kwargs)+ captured = self._capture(args, kwargs, call, key)except Exception as exc: # noqa: BLE001 - a failed capture must never fail the render+ # torch.cuda.graph's __exit__ raises in capture_end before it leaves its capture stream, so the thread+ # would run everything after (the eager retry, the rest of the process) on the capture stream.+ _restore_stream(stream_before)+ if "already recording" not in str(+ exc+ ): # a collision: the generators are the live capture's+ _heal_generators() # before anything eager draws from the CUDA RNG+ if self.placement is not None:+ self.placement.recover() # the eager retry must not wait on the dead capture's copies+ self.placement.release_slots() # nothing will replay into the ring for this loadself.poison(exc)self.stats["fallbacks"] += 1if self.logger is not None:@@ -631,7 +1127,7 @@ class GraphedForward:exc.__traceback__ = Noneif captured is None:self._release()- return self._eager(args, kwargs)+ return self._eager(eager_args, eager_kwargs)entry = capturedself.cache[key] = entryif self.logger is not None:@@ -643,20 +1139,98 @@ class GraphedForward:len(entry.static),)+ elif self.plan is not None:+ self.cache[key] = self.cache.pop(key)+live: list = []_flatten((args, kwargs), live)- for dst, src in zip(entry.static, live):- dst.copy_(src)+ last = entry.last+ state = self._judge.get(key) if self.placement is not None else None+ timing = (+ state is not None and state["verdict"] is None and len(state["graph"]) < SPEED_SAMPLES+ )+ start = None+ if timing:+ # Through the input copies and output clones: the eager step pays neither. Not while another thread+ # records, whose capture prohibits event records.+ with hold_off_capture() as safe:+ if safe:+ start, end = _timing_events()+ start.record()+ with _copy_mode(entry):+ for index, (dst, src) in enumerate(zip(entry.static, live)):+ if last is not None and entry.sticky[index]:+ ref = last[index]+ if ref is not None and ref() is src:+ continue+ dst.copy_(src)+ last[index] = weakref.ref(src)+ continue+ dst.copy_(src)entry.graph.replay()- self.stats["replays"] += 1# Cloned: every replay writes the SAME buffers, which the pipeline holds across steps.- return _rebuild(entry.out_spec, [t.clone() for t in entry.out_tensors])+ outs = [t.clone() for t in entry.out_tensors]+ if start is not None:+ with hold_off_capture() as safe:+ if safe:+ end.record()+ state["graph"].append((start, end))+ self.stats["replays"] += 1+ skip = getattr(self.placement, "skip", None) if self.placement is not None else None+ if skip is not None and getattr(skip, "counting", False):+ # The replay passed the (pass-through) skip layer by: count the computed step as it would have.+ skip.stats["calls"] += 1+ skip.stats["computed"] += 1+ return _rebuild(entry.out_spec, outs)++ def _statics(self, live: list) -> list:+ """Fresh copies of ``live``, as a capture records from (and the shape warm-up step runs on)."""+ torch = _torch()+ if self.plan is not None:+ # Inference tensors in, inference tensors as statics: compiled code guards on the dispatch keys, so a+ # mismatch makes the graph record its own (separately autotuned, so not bit-equal) compiled variant.+ static = []+ for t in live:+ with torch.inference_mode(bool(t.is_inference())):+ static.append(torch.empty_like(t))+ else:+ # A tensor made under ``torch.inference_mode()``, which renders run in, refuses ``copy_``.+ with torch.inference_mode(False):+ static = [torch.empty_like(t) for t in live]+ # Writing into an inference tensor is only allowed inside inference mode.+ copying = types.SimpleNamespace(+ inference_copy = self.plan is not None and any(t.is_inference() for t in static)+ )+ with _copy_mode(copying):+ for dst, src in zip(static, live):+ dst.copy_(src)+ return static- def _capture(self, args: tuple, kwargs: dict) -> _Entry:+ def _capture(+ self,+ args: tuple,+ kwargs: dict,+ call: Any = None,+ key: Any = None,+ ) -> _Entry:torch = _torch()+ call = self.orig if call is None else callentry = _Entry()live: list = []entry.in_spec = _flatten((args, kwargs), live)+ entry.sticky, entry.last = None, None+ if self.sticky:+ # Flat positions of the sticky kwargs, in ``_flatten``'s order (args, then kwargs by insertion).+ marks: list = []+ _flatten(args, marks)+ sticky = [False] * len(marks)+ for name, value in kwargs.items():+ part: list = []+ _flatten(value, part)+ sticky.extend([name in self.sticky] * len(part))+ if any(sticky):+ entry.sticky = tuple(sticky)+ entry.last = [weakref.ref(t) if s else None for t, s in zip(live, sticky)]bad = [(i, str(t.device)) for i, t in enumerate(live) if t.device.type != "cuda"]if bad:@@ -675,11 +1249,9 @@ class GraphedForward:f"graph at whatever value this capture saw, so this load runs eager")- # A tensor made under ``torch.inference_mode()``, which renders run in, refuses ``copy_``.- with torch.inference_mode(False):- entry.static = [torch.empty_like(t) for t in live]- for dst, src in zip(entry.static, live):- dst.copy_(src)+ entry.static = self._statics(live)+ # Writing into an inference tensor is only allowed inside inference mode.+ entry.inference_copy = self.plan is not None and any(t.is_inference() for t in entry.static)static_args, static_kwargs = _rebuild(entry.in_spec, entry.static)if nvfp4_layers:@@ -687,27 +1259,70 @@ class GraphedForward:from .diffusion_nvfp4_linear import nvfp4_prewarmnvfp4_prewarm(self.module, _prewarm_token_counts(live), logger = self.logger)+ refuse_if_exhausted(+ self.logger+ ) # before the warm-ups, and before the step skip's counters are saved+ # Under an offload placement the capture also records the copy-stream onloads (OffloadPlacement.record).+ record = call if self.placement is None else self.placement.recorder(call)# Side stream: a workspace first created DURING capture is only valid while recording.- side = torch.cuda.Stream()- side.wait_stream(torch.cuda.current_stream())- with torch.cuda.stream(side):- for _ in range(self.warmup):- self.orig(*static_args, **static_kwargs)- torch.cuda.current_stream().wait_stream(side)- torch.cuda.synchronize()-- refuse_if_exhausted(self.logger)- graph = torch.cuda.CUDAGraph()- # An explicit pool id, never None: a capture that fails must hand its pool back (retire_failed_capture).- pool = _POOL_BOX[0]- if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):- pool = torch.cuda.graph_pool_handle()- try:- with _capturing(), torch.cuda.graph(graph, pool = pool):- out = self.orig(*static_args, **static_kwargs)- except BaseException as exc:- retire_failed_capture(graph, pool, exc)- raise+ # A planned module's later captures were warmed by a real step (``__call__``): no warm-up, no cache flushes.+ recapture = self.plan is not None and self.stats["captures"] > 0+ # An offloaded key re-recorded after its weights moved is already warm: it records like a planned re-capture.+ recapture = recapture or (+ self.placement is not None and key is not None and key in self._recorded+ )+ if self.placement is not None:+ self.placement.before_capture() # anything the recorded copies land in exists before the capture+ self.valid_token = (+ self.placement.token()+ ) # the placement this capture records (the ring included)+ # A pass-through step skip under the hooks counts every call; the warm-ups and the recording are not steps.+ skip = getattr(self.placement, "skip", None) if self.placement is not None else None+ skip_stats = (+ dict(skip.stats)+ if skip is not None and isinstance(getattr(skip, "stats", None), dict)+ else None+ )+ # Per-block graphs under the hooks run their compute while the whole step warms up and records.+ # A key warmed by its timed eager steps records straight away: a side-stream warm-up would cache a second set+ # of activations (the cold render's peak).+ warm = (+ self.placement is not None+ and key is not None+ and int((getattr(self, "_judge", {}).get(key) or {}).get("seen", 0)) >= 1+ )+ with _recording_step():+ if not recapture and not warm:+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ # A re-record under a new offload placement only needs the forward to have run once on it.+ again = self.placement is not None and self.stats["captures"] > 0+ for _ in range(min(self.warmup, 1) if again else self.warmup):+ record(*static_args, **static_kwargs)+ torch.cuda.current_stream().wait_stream(side)+ torch.cuda.synchronize()++ graph = torch.cuda.CUDAGraph()+ # An explicit pool id, never None: a capture that fails must hand its pool back (retire_failed_capture).+ pool = _POOL_BOX[0]+ if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):+ pool = torch.cuda.graph_pool_handle()+ recorder = _graph_without_flush if recapture else torch.cuda.graph+ try:+ with _capturing(), recorder(graph, pool = pool):+ out = record(*static_args, **static_kwargs)+ except BaseException as exc:+ retire_failed_capture(graph, pool, exc)+ raise+ finally:+ if skip_stats is not None:+ skip.stats = skip_stats+ self.stats["pool_bytes"] = _pool_bytes(graph)+ if key is not None:+ self._recorded[key] = None+ while len(self._recorded) > 4 * self.max_graphs:+ self._recorded.pop(next(iter(self._recorded)))if _POOL_BOX[0] is None:try:_POOL_BOX[0] = graph.pool()@@ -715,6 +1330,8 @@ class GraphedForward:passentry.graph = graph+ # The pool this graph recorded into (see _drop_pool_if_unused), not whatever the box holds by now.+ entry.pool_token = pool if pool is not None else _POOL_BOX[0]out_tensors: list = []entry.out_spec = _flatten(out, out_tensors)entry.out_tensors = out_tensors@@ -743,6 +1360,479 @@ class GraphedCompiledCall(GraphedForward):return self+# Offloaded denoisers.+++def _leaf_ptrs(t: Any, out: list) -> None:+ """Data pointers of ``t``'s storage, through tensor subclasses (torchao int8 / fp8 weights) to their inner tensors."""+ flatten = getattr(type(t), "__tensor_flatten__", None)+ if flatten is not None and type(t).__name__ not in ("Tensor", "Parameter"):+ try:+ names, _ctx = t.__tensor_flatten__()+ for name in names:+ _leaf_ptrs(getattr(t, name), out)+ return+ except Exception: # noqa: BLE001 - fall through to the plain pointer+ pass+ try:+ out.append((str(t.device), int(t.data_ptr())))+ except Exception: # noqa: BLE001+ out.append(("?", id(t)))+++def _module_tensors(module: Any) -> list:+ seen: set = set()+ out: list = []+ for t in list(module.parameters()) + list(module.buffers()):+ if id(t) not in seen:+ seen.add(id(t))+ out.append(t)+ return out+++class OffloadPlacement:+ """Where an offloaded denoiser's replay sits and when its recorded addresses stop holding.++ ``group``: diffusers block offload driven by the event-fenced prefetch (diffusion_offload_prefetch). The replay+ replaces the module's hooked forward, so ONE graph records the whole step: the copy-stream copies of every+ streamed group from its pinned host tensor into the graph pool, the compute stream waiting on each copy's event,+ and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays the+ prefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.++ ``model``: whole-module CPU offload (accelerate). The hook moves the module on the eager path first and the replay+ sits under it, in ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after+ ``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""++ MODEL_MAX_INVALIDATIONS = 2+ # Past MODEL_MAX_INVALIDATIONS, re-recording continues while each recording serves at least this many replays.+ MODEL_MIN_REPLAYS_PER_RECORD = 4++ def __init__(self, module: Any, mode: str):+ self.module = module+ self.mode = mode+ self.prev: Any = None+ self._epoch: Any = None+ self._fp: Any = None+ self._checked: Any = object()+ self._why: Optional[str] = None+ self._tensors: Optional[list] = None+ # Set when a key's replay did not pay on a streaming placement: nothing records again for this load.+ self._declined: Optional[str] = None+ self._slots = False+ if mode == "group":+ self.prev = module.__dict__.get("forward")+ else:+ self.prev = getattr(module, "_old_forward", None)+ if self.prev is None:+ raise RuntimeError(f"no {mode} offload forward to wrap")+ self.eager = self.prev+ # A static step skip (diffusion_step_skip) the replay would bypass: see GraphedForward.__call__.+ self.skip: Any = self.prev if getattr(self.prev, "_unsloth_outer_forward", False) else None+ # A capture-safe rewrite (diffusion_capture_safe) under block offload: the hook chain ends in the bound class+ # forward, which the rewrite replaces while the graph is installed (bit-identical, so eager calls may take it).+ self._inner: Any = None+ self._inner_stock: Any = None+ self._inner_safe: Any = None+ if mode == "group":+ safe = _plain_rewrite(module)+ if safe is not None:+ ref = _innermost_class_forward(module)+ if ref is None:+ raise RuntimeError(+ "the block-offload hook chain does not end in the class forward"+ )+ self._inner, self._inner_stock = ref, ref.forward+ self._inner_safe = safe.__get__(module)++ def install(self, handle: Any) -> None:+ if self.mode == "group":+ if self._inner is not None:+ self._inner.forward = self._inner_safe+ self.module.__dict__["forward"] = handle+ self._enable_slots()+ else:+ self.module._old_forward = handle++ def _prefetcher(self) -> Any:+ from .diffusion_offload_prefetch import module_prefetcher+ return module_prefetcher(self.module)++ def _enable_slots(self) -> None:+ """Streamed groups copy into the prefetcher's slot ring (diffusion_offload_prefetch), so the recorded copies+ land at fixed addresses outside the graph pool: the pool then holds one step's activations, not a second+ prefetch window beside the eager one. Off with UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS=0."""+ if (os.environ.get(STEP_SLOTS_ENV) or "").strip().lower() in ("0", "off", "false", "no"):+ return+ if not self.streams():+ return+ try:+ pf = self._prefetcher()+ if pf is not None and callable(getattr(pf, "enable_slots", None)):+ pf.enable_slots()+ self._slots = bool(getattr(pf, "slot_of", None))+ except Exception: # noqa: BLE001+ self._slots = False++ def streams(self) -> bool:+ """Whether this placement moves weights every step (model offload, or a block group not pinned resident):+ the recording then holds memory beside the eager step's."""+ if self.mode != "group":+ return True+ try:+ from .diffusion_memory import _offload_groups+ pf = self._prefetcher()+ for group in _offload_groups(self.module):+ if getattr(group, "_unsloth_resident", False) or getattr(+ group, "_unsloth_pinned_top", False+ ):+ continue+ if pf is None or pf.owns(group):+ return True+ except Exception: # noqa: BLE001 - unknown layout: treat as streaming (the stricter speed check)+ return True+ return False++ def before_capture(self) -> None:+ """Allocate the slot ring before a capture, so no slot is carved out of the graph pool."""+ if not self._slots:+ return+ try:+ pf = self._prefetcher()+ if pf is not None and callable(getattr(pf, "materialize_slots", None)):+ pf.materialize_slots()+ except Exception: # noqa: BLE001+ pass++ def release_slots(self) -> None:+ """Hand the slot ring back once no graph of this placement can replay into it."""+ if not self._slots:+ return+ self._slots = False+ try:+ pf = self._prefetcher()+ if pf is not None and callable(getattr(pf, "drop_slots_after_forward", None)):+ pf.drop_slots_after_forward()+ except Exception: # noqa: BLE001+ pass++ def decline(self, reason: str) -> bool:+ """No more recording on this placement for the load; hand the slot ring back after the current forward.+ True when that drop is scheduled (it then also releases the dropped graphs' pool)."""+ self._declined = reason+ if not self._slots:+ return False+ self._slots = False+ try:+ pf = self._prefetcher()+ if pf is not None and callable(getattr(pf, "drop_slots_after_forward", None)):+ pf.drop_slots_after_forward()+ return True+ except Exception: # noqa: BLE001+ pass+ return False++ def uninstall(self, handle: Any) -> None:+ if self._inner is not None and self._inner.forward is self._inner_safe:+ self._inner.forward = self._inner_stock+ if self.mode == "group":+ if self.module.__dict__.get("forward") is handle:+ self.module.__dict__["forward"] = self.prev+ return+ if getattr(self.module, "_old_forward", None) is handle:+ self.module._old_forward = self.prev+ if (+ self.module.__dict__.get("forward") is handle+ ): # hooks removed since (diffusers re-enables them per call)+ self.module.__dict__.pop("forward", None)++ def recorder(self, call: Any) -> Any:+ """What a capture records for ``call``: under block offload the prefetcher's copies join the capture."""+ if self.mode != "group":+ return call++ def record(*args: Any, **kwargs: Any) -> Any:+ from .diffusion_offload_prefetch import module_prefetcher++ pf = module_prefetcher(self.module)+ pf.begin() # inside the capture: forks the copy stream in, and end() joins it back+ try:+ return call(*args, **kwargs)+ finally:+ pf.end()++ return record++ def record(self, *args: Any, **kwargs: Any) -> Any:+ return self.recorder(self.eager)(*args, **kwargs)++ def recover(self) -> None:+ """After a failed capture: drop what the prefetcher queued inside it (see GroupPrefetcher.abandon)."""+ if self.mode != "group":+ return+ try:+ from .diffusion_offload_prefetch import module_prefetcher+ pf = module_prefetcher(self.module)+ if pf is not None:+ pf.abandon()+ except Exception: # noqa: BLE001+ pass++ def _fingerprint(self) -> tuple:+ out: list = []+ if self.mode == "group":+ from .diffusion_memory import _offload_groups+ from .diffusion_offload_prefetch import _group_tensors, module_prefetcher++ pf = module_prefetcher(self.module)+ for group in _offload_groups(self.module):+ resident = bool(getattr(group, "_unsloth_resident", False))+ out.append((id(group), resident, bool(pf is not None and pf.owns(group))))+ tensors = _group_tensors(group)+ if resident:+ for t in tensors:+ _leaf_ptrs(t, out)+ else:+ cpu = getattr(group, "cpu_param_dict", None) or {}+ for t in tensors:+ _leaf_ptrs(cpu.get(t, t), out)+ # Streamed copies land in the slot ring when it is on: a new ring is a new placement.+ raws = getattr(pf, "slot_raw", None) or {}+ out.append(("slots", tuple((i, int(raws[i].data_ptr())) for i in sorted(raws))))+ return tuple(out)+ # Read from the module's registry on every call: Module.to() replaces buffers with new tensor objects on each+ # offload / onload, so a list taken once would keep fingerprinting the obsolete ones.+ for t in _module_tensors(self.module):+ _leaf_ptrs(t, out)+ return tuple(out)++ def token(self) -> Any:+ if self.mode == "group":+ from .diffusion_offload_prefetch import placement_epoch++ epoch = placement_epoch()+ if epoch != self._epoch:+ self._epoch = epoch+ self._fp = self._fingerprint()+ return self._fp+ self._fp = self._fingerprint()+ return self._fp++ def refusal(self, stats: dict) -> Optional[str]:+ if getattr(self, "_declined", None) is not None:+ return self._declined+ if self.mode == "model":+ # Judged once per placement, when the weights land (the fingerprint is rebuilt every call).+ if self._checked != self._fp:+ self._checked = self._fp+ moves = int(stats.get("invalidations", 0))+ records = max(1, int(stats.get("captures", 0)))+ replays = int(stats.get("replays", 0))+ self._why = None+ if (+ moves >= self.MODEL_MAX_INVALIDATIONS+ and replays < self.MODEL_MIN_REPLAYS_PER_RECORD * records+ ):+ self._why = (+ f"model offload placed the weights at new addresses on {moves} onloads, {replays} replays "+ f"over {records} recordings; each re-record costs a step"+ )+ return self._why+ if self._checked is not self._fp:+ from .diffusion_offload_prefetch import capture_refusal+ self._checked = self._fp+ self._why = capture_refusal(self.module)+ return self._why+++def _skips_planned(skip: Any) -> bool:+ plan = getattr(skip, "plan", None) or ()+ return bool(plan) and not all(plan)+++def offload_placement(module: Any) -> tuple:+ """``(placement, refusal)`` for ``module`` as it is placed now: ``(None, None)`` when nothing moves it."""+ try:+ subs = [m for m in module.modules() if m is not module]+ except Exception: # noqa: BLE001+ subs = []+ if any(getattr(m, "_hf_hook", None) is not None for m in subs):+ return None, (+ "leaf-level (sequential) offload uploads every layer synchronously from host memory inside the forward"+ )+ hf_hook = getattr(module, "_hf_hook", None)+ compiled = getattr(module, "_compiled_call_impl", None) is not None+ if hf_hook is not None:+ if compiled:+ return (+ None,+ "whole-module compiled denoiser: the CPU-offload hook runs inside its compiled call",+ )+ if "CpuOffload" not in type(hf_hook).__name__:+ return (+ None,+ f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",+ )+ why = _forward_refusal(module, "model")+ if why is not None:+ return None, why+ try:+ return OffloadPlacement(module, "model"), None+ except Exception as exc: # noqa: BLE001+ return None, f"model offload hook unreadable ({exc})"+ registry = getattr(module, "_diffusers_hook", None)+ hooks = getattr(registry, "hooks", None) or {}+ if not any("offload" in str(key) for key in hooks):+ return None, None+ if compiled:+ return (+ None,+ "whole-module compiled denoiser: the offload hooks run inside its compiled call",+ )+ try:+ from diffusers.hooks import group_offloading as go+ name = getattr(go, "_GROUP_OFFLOADING", "group_offloading")+ except Exception: # noqa: BLE001+ name = "group_offloading"+ if name not in hooks:+ return None, "an offload hook other than block offloading moves the weights"+ skip = None+ for fn_ref in getattr(registry, "_fn_refs", None) or ():+ if getattr(getattr(fn_ref, "forward", None), "_unsloth_outer_forward", False):+ # The static step skip under the offload hook: a render that plans skips runs every call through it+ # (GraphedForward.__call__); one that plans none passes straight through it, so the graph may replay.+ skip = fn_ref.forward+ why = _forward_refusal(module, "group")+ if why is not None:+ return None, why+ from .diffusion_offload_prefetch import capture_refusal++ why = capture_refusal(module)+ if why is not None and "background" not in why:+ return None, why+ try:+ placement = OffloadPlacement(module, "group")+ except Exception as exc: # noqa: BLE001+ return None, f"block offload hook unreadable ({exc})"+ placement.skip = skip+ return placement, None+++def _forward_refusal(module: Any, mode: str) -> Optional[str]:+ """Why ``module``'s own forward cannot record under ``mode`` offload, else None (see GraphedForward.__init__)."""+ try:+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415+ safe, why = _capture_safe(type(module))+ except Exception: # noqa: BLE001 - the stock forward is then what records+ return None+ if why is not None:+ return why+ if safe is None:+ return None+ if getattr(safe, "__unsloth_graph_plan__", None) is not None:+ if getattr(safe, "__unsloth_graph_placed__", None) is None:+ return f"{type(module).__name__} plans its graph steps but has no entry through the offload hooks"+ return None+ if mode == "group" and _innermost_class_forward(module) is None:+ return (+ f"{type(module).__name__} records only through its capture-safe forward, and the block-offload hook "+ "chain does not end in the class forward it would replace"+ )+ return None+++def _plain_rewrite(module: Any) -> Any:+ """``module``'s capture-safe class-forward rewrite when it is a plain one (not a planned step), else None."""+ try:+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415+ safe, _ = _capture_safe(type(module))+ except Exception: # noqa: BLE001+ return None+ if safe is None or getattr(safe, "__unsloth_graph_plan__", None) is not None:+ return None+ return safe+++def _innermost_class_forward(module: Any) -> Any:+ """The diffusers hook-chain reference whose ``forward`` is ``module``'s bound class forward, else None."""+ registry = getattr(module, "_diffusers_hook", None)+ refs = getattr(registry, "_fn_refs", None) or ()+ if not refs:+ return None+ ref = refs[0]+ if getattr(ref, "original_forward", None) is not None:+ return None # a hook with its own new_forward owns the innermost call+ current = getattr(ref, "forward", None)+ owner = next((k for k in type(module).__mro__ if "forward" in vars(k)), None)+ if owner is None or getattr(current, "__self__", None) is not module:+ return None+ if getattr(current, "__func__", None) is not vars(owner)["forward"]:+ return None+ return ref+++def arm_after_placement(+ pipe: Any,+ *,+ logger: Any = None,+ max_graphs: int = MAX_GRAPHS_PER_MODULE,+) -> tuple:+ """Arm the graphs once placement is final, in the slot each denoiser's offload leaves free. Returns+ ``(handles, reason)``: the reason names what kept a denoiser eager, else what engaged."""+ uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or (), logger = logger)+ handles: list = []+ refusals: list = []+ modes: list = []+ for module in _denoiser_modules(pipe):+ placement, why = offload_placement(module)+ if why is not None:+ refusals.append(f"{type(module).__name__}: {why}")+ continue+ if placement is None and getattr(pipe, "_unsloth_cuda_graph_offload_only", False):+ refusals.append(+ f"{type(module).__name__}: the family records offloaded steps only, and it stays resident"+ )+ continue+ try:+ if placement is None:+ kind = (+ GraphedCompiledCall+ if getattr(module, "_compiled_call_impl", None) is not None+ else GraphedForward+ )+ handles.append(kind(module, max_graphs = max_graphs, logger = logger).enable())+ modes.append("resident")+ else:+ handles.append(+ GraphedForward(+ module, max_graphs = max_graphs, logger = logger, placement = placement+ ).enable()+ )+ modes.append(placement.mode)+ except Exception as exc: # noqa: BLE001+ refusals.append(f"{type(module).__name__}: {exc}")+ installed = tuple(handles)+ try:+ pipe._unsloth_cuda_graphs = installed+ except Exception as exc: # noqa: BLE001+ _warn(logger, "handle stash", exc)+ if refusals:+ reason = "; ".join(refusals)+ else:+ labels = {+ "resident": "resident",+ "group": "block-streamed (copies recorded in the graph)",+ "model": "model offload",+ }+ reason = "captured per input shape: " + ", ".join(sorted({labels.get(m, m) for m in modes}))+ if logger is not None:+ logger.info(+ "diffusion.cuda_graph: armed on %d denoiser module(s) after placement (%s)",+ len(installed),+ reason,+ )+ return installed, reason++def _denoiser_modules(pipe: Any) -> list:"""What the graph layer arms: every denoiser DiT, else a whole-compile-list U-Net."""from .diffusion_speed import _denoiser_dits, _denoiser_unet@@ -764,8 +1854,10 @@ def graph_eligible(speed_mode: str,family_default: bool = True,logger: Any = None,+ offloaded: bool = False,) -> tuple[bool, str]:- """Whether this load may be graphed, and a short reason when it may not. Cheapest first."""+ """Whether this load may be graphed, and a short reason when it may not. Cheapest first. ``offloaded``: the+ denoiser moves (graphs armed after placement), which a family can opt into on its own (``offload_cuda_graph``)."""if cuda_graph_disabled():return False, _disabled_reason()@@ -811,8 +1903,10 @@ def graph_eligible(_warn(logger, "availability probe", exc)return False, "torch.cuda unavailable"- if not bool(getattr(family, "supports_cuda_graph", family_default)):- return False, "family opts out"+ wanted = bool(getattr(family, "supports_cuda_graph", family_default))+ wanted = wanted or (offloaded and bool(getattr(family, "offload_cuda_graph", False)))+ if not wanted and not _family_forced(family):+ return False, str(getattr(family, "cuda_graph_decline", None) or "family opts out")# A denoiser whose forward syncs the host would only invalidate its capture and run eager; decline# up front with the reason when its capture-safe rewrite did not apply.@@ -904,9 +1998,19 @@ def stats(handles: Any) -> dict:"eager_calls": 0,"fallbacks": 0,"cap_skips": 0,+ "invalidations": 0,+ "pool_bytes": 0,+ "planned_eager": 0,+ "shape_warmups": 0,+ "evictions": 0,+ "speed_eager": 0,**{field: 0 for field in _REFUSALS},+ "placements": [],"poisoned": False,"capture_error": None,+ # Offloaded only: the measured eager step and replay the speed check judged (ms), once judged.+ "eager_ms": None,+ "replay_ms": None,}for handle in handles or ():try:@@ -917,9 +2021,20 @@ def stats(handles: Any) -> dict:"eager_calls","fallbacks","cap_skips",+ "invalidations",+ "planned_eager",+ "shape_warmups",+ "evictions",+ "speed_eager",*_REFUSALS,):out[field] += int(handle.stats.get(field, 0))+ out["pool_bytes"] = max(out["pool_bytes"], int(handle.stats.get("pool_bytes", 0)))+ if out["eager_ms"] is None and handle.stats.get("eager_ms") is not None:+ out["eager_ms"] = float(handle.stats["eager_ms"])+ out["replay_ms"] = float(handle.stats["replay_ms"])+ placement = getattr(handle, "placement", None)+ out["placements"].append("resident" if placement is None else placement.mode)if handle.poisoned:out["poisoned"] = Trueerror = handle.capture_error@@ -941,11 +2056,24 @@ def never_engaged(handles: Any) -> Optional[str]:if reasons and len(reasons) == len(handles):return None if any(r is None for r in reasons) else "; ".join(reasons)s = stats(handles)+ refused = [+ str((h.capture_error or {}).get("msg"))+ for h in handles+ if (getattr(h, "capture_error", None) or {}).get("type") == "Refused"+ ]+ if refused and not s["captures"] and not s["replays"]:+ return f"armed, but every denoiser call so far ran eager: {refused[0]}"+ if refused and not s["graphs"]:+ # Recorded, then dropped for the load: the steps run eager from here on.+ return f"captured, then dropped: {refused[0]}; denoiser steps now run eager"if all(getattr(h, "poisoned", False) for h in handles):error = s["capture_error"] or {}return f"capture failed ({error.get('type') or 'error'}); every denoiser step runs eager"if s["captures"] or s["replays"] or not s["eager_calls"]:return None+ if s["eager_calls"] == s["speed_eager"]:+ # Every eager call so far was an offloaded step timed for the speed check, before its first capture.+ return Noneparts = [f"{s[field]} {label}"for field, label in (@@ -1031,6 +2159,14 @@ def arm_block_graphs(handles = tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())whole = [h for h in handles if isinstance(h, GraphedForward)]prior = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")+ step = [h for h in whole if h.placement is not None or h.plan is not None]+ if whole and len(step) == len(whole):+ # The whole step stays primary; where per-block graphs are the default they arm for the calls it leaves eager.+ pipe._unsloth_cuda_graph_mode = "step"+ why = _block_basics(target, family, cache_engaged, speed_mode, family_default)+ if why is None:+ _attach_block_fallback(pipe, step, target, hooked, pinned, logger)+ return handlesif not hooked and (whole or "not capture-safe" not in prior):# Nothing moves the denoiser: the speed layer's decision stands, except a forward that is not capture-safe,# whose blocks still record.@@ -1115,6 +2251,41 @@ def arm_block_graphs(return tuple(armed)+def _attach_block_fallback(+ pipe: Any, step: list, target: Any, hooked: bool, pinned: bool, logger: Any+) -> None:+ try:+ from .diffusion_block_graph import (+ block_graphs_disabled,+ block_graphs_requested,+ install_block_graphs,+ )+ except Exception as exc: # noqa: BLE001+ _warn(logger, "block graph import", exc)+ return+ if block_graphs_disabled():+ return+ for handle in step:+ model_offload = getattr(handle.placement, "mode", None) == "model"+ stays = (not hooked or bool(pinned)) and not model_offload+ if not (stays or block_graphs_requested()):+ continue++ def arm(module: Any = handle.module) -> Any:+ armed, _why = install_block_graphs(+ module, device = getattr(target, "torch_device", None), logger = logger+ )+ if armed is not None:+ _set_reason(+ pipe,+ str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")+ + "; the steps it leaves eager record per block",+ )+ return armed++ handle.fallback = arm++def _set_reason(pipe: Any, reason: str) -> None:try:pipe._unsloth_cuda_graph_reason = reason@@ -1125,9 +2296,15 @@ def _set_reason(pipe: Any, reason: str) -> None:def status_reason(pipe: Any, on: bool) -> str:"""The cuda_graph badge text: the block layer's own sentence when it armed, else why graphs are off."""if on:- if getattr(pipe, "_unsloth_cuda_graph_mode", None) == "blocks":- return str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or WHOLE_REASON)- return WHOLE_REASON+ mode = getattr(pipe, "_unsloth_cuda_graph_mode", None)+ reason = str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "")+ if mode == "blocks":+ return reason or WHOLE_REASON+ head, _, tail = reason.partition("; ")+ if mode == "step" and head.startswith("captured") and "resident" not in head:+ # "denoiser step captured per input shape: block-streamed (copies recorded in the graph), replayed ..."+ return f"denoiser step {head}, replayed bit-identically" + (f"; {tail}" if tail else "")+ return WHOLE_REASON + (f"; {tail}" if mode == "step" and tail else "")return str(getattr(pipe, "_unsloth_cuda_graph_reason", None) or "speed tier does not capture")diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 5c5543fde4c..19503fe20d3 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -678,9 +678,10 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:lives outside it and is not counted here, which is correct: it is not device memory thisgeneration can allocate into.- A captured CUDA graph's pool (``diffusion_cuda_graph``) is reserved but not allocated, so it is- credited here, yet ordinary allocations cannot reuse it while a graph holds it, so on a graphed- load the guard reads about one step of activations high.+ A captured CUDA graph's pool (``diffusion_cuda_graph``) is mostly reserved but not allocated, yet ordinary+ allocations cannot reuse it while a graph holds it (on a block-streamed denoiser it also holds the+ prefetch window), so its unallocated bytes are not credited (its allocated part is already outside the+ difference).Falls back to the plain snapshot on any failure or non-cuda device."""if getattr(target, "device", "cpu") != "cuda":@@ -703,6 +704,12 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:reclaimable -= _block_graph_pool_bytes(device)except Exception: # noqa: BLE001pass+ try:+ # The whole-step graphs' shared pool: only its unallocated part sits inside the difference above.+ from .diffusion_cuda_graph import live_pool_free_bytes+ reclaimable -= live_pool_free_bytes()+ except Exception: # noqa: BLE001 -- no graph layer: nothing held+ passif reclaimable <= 0:return snapshotfree = int(snapshot.free_mib) + reclaimable // (1024 * 1024)@@ -2665,6 +2672,8 @@ def _keep_groups_resident(left -= needkept += needstate["streamed"] = _streamed_group_count(ordered)+ if kept:+ _bump_placement_epoch()if kept and onload.type == "cuda":torch.cuda.synchronize(onload)if logger is not None and kept:@@ -2682,6 +2691,14 @@ def _keep_groups_resident(return 0+def _bump_placement_epoch() -> None:+ try:+ from .diffusion_offload_prefetch import bump_placement_epoch+ bump_placement_epoch()+ except Exception: # noqa: BLE001 - no graph layer to invalidate+ pass++def _streamed_group_count(groups: list) -> int:return sum(1@@ -2699,6 +2716,7 @@ def _release_group(group: Any) -> None:group.__dict__.pop(name, None)group._unsloth_resident = Falsegroup.offload_()+ _bump_placement_epoch()def release_resident_groups(@@ -3486,7 +3504,9 @@ class _GroupPinner:if tensor.device.type == "cpu" and tensor.data_ptr() == src.data_ptr():tensor.data = pinnedself.pinned += src.nbytes+ # Done before the epoch: a capture check that sees the new epoch must also see this group ready.self._done[id(group)].set()+ _bump_placement_epoch()except Exception as exc: # noqa: BLE001 - diffusers pins what is left on each onloadfailed = excfinally:@@ -3494,6 +3514,8 @@ class _GroupPinner:pool.shutdown(wait = True, cancel_futures = True)for event in self._done.values():event.set()+ if failed is not None:+ _bump_placement_epoch()if self.logger is not None:try:if failed is not None:diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex a71189accf2..69ead084a71 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -28,6 +28,20 @@ PREFETCHER_ATTR = "_unsloth_group_prefetcher"_BG_PIN_ATTR = "_unsloth_background_pin"+# Bumped whenever a denoiser's device placement or a streamed group's host copy changes (residency pinned or released,+# prefetch installed, a background pin swapping host tensors). A CUDA graph recorded over a placement replays its+# addresses, so diffusion_cuda_graph drops graphs recorded under an older epoch.+_PLACEMENT_EPOCH = [0]+++def bump_placement_epoch() -> None:+ _PLACEMENT_EPOCH[0] += 1+++def placement_epoch() -> int:+ return _PLACEMENT_EPOCH[0]++def async_prefetch_enabled() -> bool:return (os.environ.get(ASYNC_PREFETCH_ENV) or "").strip().lower() not in ("0",@@ -316,6 +330,9 @@ class GroupPrefetcher:streams = [g.stream for g in self.groups if getattr(g, "stream", None) is not None]self.stream = streams[0] if streams else Noneself.fence_first = False+ # True while a CUDA-graph capture records this forward (diffusion_cuda_graph): the copy stream is joined into+ # the capture at begin() and back at end(), so the copies become graph nodes refilling the same buffers.+ self.capturing = Falseself.stats = {"forwards": 0, "copies": 0, "prefetched": 0, "missed": 0, "dropped": 0}# Slot ring (``enable_slots``): group id -> slot index; a streamed group's copy lands in place in that slot,# always at the same offsets, so the blocks a CUDA graph recorded read the same addresses every step.@@ -370,6 +387,8 @@ class GroupPrefetcher:return Falsepinner = getattr(group, _BG_PIN_ATTR, None)if pinner is not None:+ if getattr(self, "capturing", False) and not _pinned_done(pinner, group):+ raise RuntimeError("a background pin is still swapping this group's host copies")pinner.wait(group)go = _go()is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)@@ -384,6 +403,9 @@ class GroupPrefetcher:for i, t in enumerate(_group_tensors(group)):src = cpu[t]if not src.is_pinned():+ if getattr(self, "capturing", False):+ # a graph would replay this copy from a temporary pinned buffer reused after the call+ raise RuntimeError("a streamed group's host copy is not pinned")src = (src.pin_memory()) # diffusers' low_cpu_mem_usage path: the host allocator fences reuse@@ -561,14 +583,64 @@ class GroupPrefetcher:raw = self.slot_raw.get(where)if raw is None:- raw = torch.empty(self.slot_size, dtype = torch.uint8, device = self.device)- self.slot_raw[where] = raw- self.slot_bytes += self.slot_size+ raw = self._alloc_slot(where)cpu = group.cpu_param_dictviews = _slot_views(raw, [cpu[t] for t in _group_tensors(group)], self.device)self.slot_buffers[gid] = viewsreturn views+ def _alloc_slot(self, where: int) -> Any:+ import torch++ raw = torch.empty(self.slot_size, dtype = torch.uint8, device = self.device)+ self.slot_raw[where] = raw+ self.slot_bytes += self.slot_size+ bump_placement_epoch() # a whole-step graph keys its recorded copies on the ring's addresses+ return raw++ def materialize_slots(self) -> None:+ """Allocate every slot of the ring now (a whole-step capture must not carve one out of its graph pool)."""+ if not self.slot_of or not self.slot_size:+ return+ # Only the slots a streamed group fills: a resident group that is released later allocates its own on use.+ streamed = {+ self.slot_of[id(g)]+ for g in self.groups+ if id(g) in self.slot_of and not getattr(g, "_unsloth_resident", False)+ }+ for where in sorted(streamed):+ if where not in self.slot_raw:+ self._alloc_slot(where)++ def drop_slots_after_forward(self) -> None:+ """Drop the ring once the current forward ends (``end``), or now between forwards."""+ if self.active:+ self._drop_slots_at_end = True+ return+ self._drop_ring()++ def _drop_ring(self) -> None:+ import torch++ from .diffusion_cuda_graph import hold_off_capture+ with hold_off_capture() as safe:+ if not safe:+ # Another thread is recording: its capture prohibits these syncs and frees. Retry after a later forward.+ self._drop_slots_at_end = True+ return+ self._drop_slots_at_end = False+ self.disable_slots()+ try:+ import gc++ gc.collect() # a declined step graph's recordings, whose pool is flushed below with the ring+ clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)+ if callable(clear):+ clear() # and the workspace cuBLAS made for their capture stream+ torch.cuda.empty_cache() # the ring's segments go back to the device, not just to the cache+ except Exception: # noqa: BLE001+ pass+def disable_slots(self) -> None:"""Drop the ring. Only between forwards: every streamed group is back on the host by then."""import torch@@ -577,6 +649,7 @@ class GroupPrefetcher:returnif self.slot_raw:torch.cuda.synchronize(self.device)+ bump_placement_epoch()self.slot_of = {}self.slot_buffers = {}self.slot_raw = {}@@ -594,6 +667,13 @@ class GroupPrefetcher:self.active = Trueself.on_order = bool(self.order)self.pending = True+ self.capturing = False+ if self.stream is not None:+ import torch+ if torch.cuda.is_current_stream_capturing():+ # fork the copy stream into the capture before anything is queued on it+ self.capturing = True+ self.stream.wait_stream(self._compute())def kick(self) -> None:"""First block onload of a forward (streamed or resident): queue the first ``depth`` streamed groups."""@@ -620,8 +700,34 @@ class GroupPrefetcher:self._release(group, self.ready.pop(gid))self.ready.clear()self.inflight_bytes = 0+ if getattr(self, "capturing", False):+ self.capturing = False+ self._compute().wait_stream(+ self.stream+ ) # join the copy stream back before the capture endsif self.seen:self.order = list(self.seen)+ if getattr(self, "_drop_slots_at_end", False):+ self._drop_ring()++ def abandon(self) -> None:+ """A capture failed mid-forward: the events and device copies it queued died with it. Forget them (never wait+ on a captured event outside its graph) and put every streamed group back on its host copy. Outside capture."""+ import torch++ self.active = self.on_order = self.pending = self.capturing = False+ self.ready.clear()+ self.inflight_bytes = 0+ try:+ torch.cuda.synchronize(self.device)+ except Exception: # noqa: BLE001 - the copies below are what matters+ pass+ for group in self.groups:+ if self.owns(group):+ try:+ type(group).offload_(group)+ except Exception: # noqa: BLE001 - a group the capture never reached is already on the host+ passdef _adopt_top_group(@@ -819,6 +925,7 @@ def install_group_prefetch(module.register_forward_pre_hook(_eager(lambda *_a, **_k: pf.begin()))module.register_forward_hook(_eager(lambda *_a, **_k: pf.end()), always_call = True)setattr(module, PREFETCHER_ATTR, pf)+ bump_placement_epoch()state = getattr(module, "_unsloth_stream_state", None)if not isinstance(state, dict):state = {"streamed": 1}@@ -848,3 +955,41 @@ def install_group_prefetch(def module_prefetcher(module: Any) -> Optional[GroupPrefetcher]:return getattr(module, PREFETCHER_ATTR, None)+++def _pinned_done(pinner: Any, group: Any) -> bool:+ event = getattr(pinner, "_done", {}).get(id(group))+ return event is None or event.is_set()+++def capture_refusal(module: Any) -> Optional[str]:+ """Why a CUDA graph cannot record ``module``'s block-streamed forward with its copies inside, else None.++ A replay repeats every copy from the host address it recorded into the device buffer it recorded, so every+ streamed group must copy from a pinned host tensor no background pin is still replacing, and every group must be+ driven by the prefetcher, pinned resident, or the dense top-level group's pinned upload."""+ pf = module_prefetcher(module)+ if pf is None:+ return "block streaming without the event-fenced prefetch (its stream waits are host synchronizations)"+ try:+ from .diffusion_memory import _offload_groups+ for group in _offload_groups(module):+ if getattr(group, "offload_to_disk_path", None):+ return "a group streams from disk"+ if getattr(group, "_unsloth_resident", False) or getattr(+ group, "_unsloth_pinned_top", False+ ):+ continue+ if not pf.owns(group):+ return "a streamed group is not driven by the prefetcher"+ pinner = getattr(group, _BG_PIN_ATTR, None)+ if pinner is not None and not _pinned_done(pinner, group):+ return "host copies are still being pinned in the background"+ cpu = getattr(group, "cpu_param_dict", None) or {}+ for t in _group_tensors(group):+ src = cpu.get(t)+ if src is None or not src.is_pinned():+ return "a streamed group's host copy is not pinned"+ except Exception as exc: # noqa: BLE001 - unknown layout: never capture it+ return f"offload layout unreadable ({type(exc).__name__})"+ return Nonediff --git a/studio/backend/core/inference/diffusion_qwenimage21.py b/studio/backend/core/inference/diffusion_qwenimage21.pyindex c9b7b690cc4..d255b6f4774 100644--- a/studio/backend/core/inference/diffusion_qwenimage21.py+++ b/studio/backend/core/inference/diffusion_qwenimage21.py@@ -115,6 +115,7 @@ class _Layout:"vlm_row","_segments","_vlm_text",+ "_tails","__weakref__",)@@ -136,6 +137,7 @@ def _build_layout(model: Any, mod: Any, img_mask: Any, shapes: list, device: Anylay.vlm_row = img_mask[0].clone() # the caller may reuse and rewrite its masklay._segments = Nonelay._vlm_text = {}+ lay._tails = Nonereturn lay@@ -199,26 +201,38 @@ def _layout_for(return lay-def _cached_step(+def _tails(lay: _Layout) -> tuple:+ """The decode rows of the RoPE table and modulation mask; one view per layout, so a graph replay can tell by+ identity that its copy is current."""+ if lay._tails is None:+ lay._tails = (lay.rotary_emb[lay.prefix_len :], lay.target_token_mask[lay.prefix_len :])+ return lay._tails+++def _cached_core(model: Any,- lay: _Layout,text_dtype: Any,+ total: int,+ prefix_len: int,hidden_states: Any,timestep: Any,encoder_hidden_states_mask: Any,- kv_cache: Any,+ text_positions: Any,+ vlm_text: Any,+ rotary_emb: Any,+ modulation_mask: Any,+ layer_cache: Any,) -> Any:+ """One decode step from the prefix KV cache. Reads no host value off the device, so it records as a CUDA graph;+ the eager step and the graph run this same code. ``layer_cache(i)`` returns block ``i``'s cache entry."""import torchbatch_size = hidden_states.shape[0]device = hidden_states.device- prefix_len = lay.prefix_lenhidden_states = model.img_in(hidden_states)# Keep the stock (batch, total, dim) layout so the blocks see the same strides.- full = torch.empty(- (batch_size, lay.total, hidden_states.shape[2]), dtype = text_dtype, device = device- )- full[:, prefix_len:] = hidden_states[:, hidden_states.shape[1] - (lay.total - prefix_len) :]+ full = torch.empty((batch_size, total, hidden_states.shape[2]), dtype = text_dtype, device = device)+ full[:, prefix_len:] = hidden_states[:, hidden_states.shape[1] - (total - prefix_len) :]timestep = timestep.to(hidden_states.dtype)timestep = torch.cat([timestep, timestep.new_zeros(1)], dim = 0)@@ -227,15 +241,11 @@ def _cached_step(attention_mask = Noneif encoder_hidden_states_mask is not None:- joint_key_valid = torch.ones(batch_size, lay.total, dtype = torch.bool, device = device)- joint_key_valid[:, lay.text_positions] = encoder_hidden_states_mask.bool()[- :, _vlm_text(lay, encoder_hidden_states_mask.shape[1])- ]+ joint_key_valid = torch.ones(batch_size, total, dtype = torch.bool, device = device)+ joint_key_valid[:, text_positions] = encoder_hidden_states_mask.bool()[:, vlm_text]attention_mask = joint_key_valid[:, None, None, :]joint_hidden_states = full[:, prefix_len:]- rotary_emb = lay.rotary_emb[prefix_len:]- modulation_mask = lay.target_token_mask[prefix_len:]for index_block, block in enumerate(model.transformer_blocks):joint_hidden_states = block(hidden_states = joint_hidden_states,@@ -243,7 +253,7 @@ def _cached_step(rotary_emb = rotary_emb,attention_mask = attention_mask,target_token_mask = modulation_mask,- layer_cache = kv_cache.get_layer(index_block),+ layer_cache = layer_cache(index_block),kv_cache_mode = "cached",cache_write_slice = None,segments = None,@@ -253,6 +263,202 @@ def _cached_step(return model.proj_out(joint_hidden_states)+def _cached_step(+ model: Any,+ lay: _Layout,+ text_dtype: Any,+ hidden_states: Any,+ timestep: Any,+ encoder_hidden_states_mask: Any,+ kv_cache: Any,+) -> Any:+ rotary_emb, modulation_mask = _tails(lay)+ return _cached_core(+ model,+ text_dtype,+ lay.total,+ lay.prefix_len,+ hidden_states,+ timestep,+ encoder_hidden_states_mask,+ lay.text_positions,+ None+ if encoder_hidden_states_mask is None+ else _vlm_text(lay, encoder_hidden_states_mask.shape[1]),+ rotary_emb,+ modulation_mask,+ kv_cache.get_layer,+ )+++# ---------------------------------------------------------------------------------------------- CUDA graph step+#+# ``diffusion_cuda_graph.GraphedForward`` records one call into static buffers keyed by every input's shape. The+# stock call cannot be recorded: the pipeline hands the prefix K/V over as a ``QwenImage21KVCache`` object (an opaque+# leaf to the graph layer) and step 0 fills that cache from Python. ``graph_plan`` turns a decode step into a call+# whose every input is a tensor or a constant: the per-layer K/V, the layout's index and RoPE tensors, latents,+# timestep and text mask. Step 0 (the prefill) stays eager.+#+# The K/V, RoPE and index inputs only change when a new render (or a new layout) starts, so they are "sticky": the+# replay copies one into its static buffer only when the caller passes a different tensor object than last time.++GRAPH_STICKY = ("text_positions", "vlm_text", "rotary_emb", "modulation_mask", "kv")++# A graph keeps its own copy of the prefix K/V (one per graph, up to the per-module cap). Text-only prefixes stay far+# below this; a 1 MP condition image is ~2.2 GiB at 32 blocks, and those steps run eager rather than pin that twice.+GRAPH_KV_MAX_BYTES = 1 << 30+++def _graph_step(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states_mask,+ text_positions,+ vlm_text,+ rotary_emb,+ modulation_mask,+ kv,+ total,+ prefix_len,+ text_dtype,+):+ import torch++ mod = _module()+ caches = []+ for key, value in kv:+ entry = mod.QwenImage21KVLayerCache()+ entry.store(key, value)+ caches.append(entry)+ output = _cached_core(+ self,+ getattr(torch, text_dtype),+ total,+ prefix_len,+ hidden_states,+ timestep,+ encoder_hidden_states_mask,+ text_positions,+ vlm_text,+ rotary_emb,+ modulation_mask,+ caches.__getitem__,+ )+ return (output,)+++# Under block offload the graph wrapper sits in the forward slot under the offload hooks, which onload the top-level+# weights: a planned step enters through the hooked forward as ``kv_cache_mode = GRAPH_STEP_MODE`` with the plan as+# ``kv_cache`` (the forward's signature stays the pipeline's).+GRAPH_STEP_MODE = "unsloth_graph_step"+++def placed_step(forward: Any, step: dict) -> Any:+ return forward(+ step["hidden_states"],+ None,+ step["timestep"],+ None,+ None,+ kv_cache = step,+ kv_cache_mode = GRAPH_STEP_MODE,+ return_dict = False,+ )+++def _module() -> Any:+ import importlib+ return importlib.import_module(_MODULE)+++def graph_plan(module: Any, args: tuple, kwargs: dict) -> Optional[tuple]:+ """``(callable, kwargs)`` replaying this decode step from tensors only, or None to run the call eager.++ Never syncs the host on a decode step of a render whose prefill went through the fast forward (the layout is+ then found by identity). Never raises."""+ try:+ return _graph_plan(module, args, kwargs)+ except Exception: # noqa: BLE001 - an unplannable call simply runs eager+ return None+++def _graph_plan(module: Any, args: tuple, kwargs: dict) -> Optional[tuple]:+ import torch++ if (+ args+ or kwargs.get("kv_cache_mode") != "cached"+ or kwargs.get("return_dict", True) is not False+ ):+ return None+ kv_cache = kwargs.get("kv_cache")+ hidden_states = kwargs.get("hidden_states")+ encoder_hidden_states = kwargs.get("encoder_hidden_states")+ timestep = kwargs.get("timestep")+ img_mask = kwargs.get("img_mask")+ img_shapes = kwargs.get("img_shapes")+ mask = kwargs.get("encoder_hidden_states_mask")+ known = {+ "hidden_states",+ "encoder_hidden_states",+ "timestep",+ "img_shapes",+ "img_mask",+ "encoder_hidden_states_mask",+ "attention_kwargs",+ "kv_cache",+ "kv_cache_mode",+ "return_dict",+ }+ # LoRA scale rides in attention_kwargs through the forward's decorator, which the graph step bypasses.+ if set(kwargs) - known or kwargs.get("attention_kwargs"):+ return None+ if kv_cache is None or not torch.is_tensor(hidden_states) or not torch.is_tensor(timestep):+ return None+ if (+ not torch.is_tensor(encoder_hidden_states)+ or not torch.is_tensor(img_mask)+ or img_shapes is None+ ):+ return None+ if torch.is_grad_enabled() or fast_step_disabled() or not module.config.causal_condition:+ return None+ if not getattr(vars(type(module)).get("forward"), "__unsloth_q21_fast_step__", False):+ return None+ mod = _module()+ lay = _layout_for(module, mod, img_mask, img_shapes, hidden_states.device, reuse_identity = True)+ _, text_dtype = _text_dtype(module, encoder_hidden_states)+ if text_dtype is None or not lay.tail_is_image:+ return None+ kv = []+ kv_bytes = 0+ for index in range(len(module.transformer_blocks)):+ entry = kv_cache.get_layer(index)+ key, value = getattr(entry, "k", None), getattr(entry, "v", None)+ if not torch.is_tensor(key) or not torch.is_tensor(value):+ return None+ kv_bytes += key.numel() * key.element_size() + value.numel() * value.element_size()+ kv.append((key, value))+ if kv_bytes > GRAPH_KV_MAX_BYTES:+ return None+ rotary_emb, modulation_mask = _tails(lay)+ plan = {+ "hidden_states": hidden_states,+ "timestep": timestep,+ "encoder_hidden_states_mask": mask,+ "text_positions": lay.text_positions if mask is not None else None,+ "vlm_text": _vlm_text(lay, mask.shape[1]) if mask is not None else None,+ "rotary_emb": rotary_emb,+ "modulation_mask": modulation_mask,+ "kv": tuple(kv),+ "total": int(lay.total),+ "prefix_len": int(lay.prefix_len),+ "text_dtype": str(text_dtype).rsplit(".", 1)[-1],+ }+ return _graph_step.__get__(module), plan++def _autocast_state(device_type: str) -> tuple:"""Autocast decides the text projection's output dtype as much as the weights do."""import torch@@ -294,6 +500,8 @@ def _make_forward(mod: Any, stock: Any) -> Any:kv_cache_mode = None,return_dict = True,):+ if kv_cache_mode == GRAPH_STEP_MODE:+ return _graph_step(self, **kv_cache)if torch.is_grad_enabled() or torch.compiler.is_compiling() or fast_step_disabled():return stock_inner(self,@@ -430,6 +638,10 @@ def _make_forward(mod: Any, stock: Any) -> Any:wrapped = lora_scale("attention_kwargs")(forward) if callable(lora_scale) else forwardwrapped.__unsloth_q21_fast_step__ = Truewrapped.__unsloth_stock_forward__ = stock+ # Read by diffusion_capture_safe: decode steps record as CUDA graphs through graph_plan.+ wrapped.__unsloth_graph_plan__ = graph_plan+ wrapped.__unsloth_graph_sticky__ = GRAPH_STICKY+ wrapped.__unsloth_graph_placed__ = placed_stepreturn wrapped@@ -458,8 +670,7 @@ def install(logger: Any = None) -> bool:if fast_step_disabled():return Falsetry:- import importlib- mod = importlib.import_module(_MODULE)+ mod = _module()except Exception: # noqa: BLE001 - diffusers without Qwen-Image 2.1return Falsecls = getattr(mod, _CLASS, None)diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex f5230978fe2..1f0e25fa407 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -587,20 +587,33 @@ def apply_speed_optims(if mode in (SPEED_DEFAULT, SPEED_MAX):cuda_graph = Noneok, reason = False, "cuda graph layer unavailable"+ moved = bool(offload_active if denoiser_offloaded is None else denoiser_offloaded)try:from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle++ after_placement = moved and cuda_graph.offload_graphs_enabled()ok, reason = cuda_graph.graph_eligible(target,family = family,pipe = pipe,- offload_active = offload_active- if denoiser_offloaded is None- else bool(denoiser_offloaded),+ offload_active = moved and not after_placement,cache_active = cache_active if cache_engaged is None else bool(cache_engaged),speed_mode = mode,family_default = cuda_graph_default,logger = logger,+ offloaded = after_placement,)+ if ok and after_placement:+ # The slot and the recorded copies depend on the hooks placement installs: arm_graphs_after_placement.+ ok, reason = False, "armed after placement"+ try:+ pipe._unsloth_cuda_graph_after_placement = True+ # A family that opts in only offloaded must not get a resident graph if the denoiser stays put.+ pipe._unsloth_cuda_graph_offload_only = not bool(+ getattr(family, "supports_cuda_graph", cuda_graph_default)+ )+ except Exception: # noqa: BLE001+ passexcept Exception as exc: # noqa: BLE001 - an unimportable graph layer means eager, never a failed load_warn(logger, "cuda graph eligibility", exc)# Stashed either way: status reports WHY graphs are off, not just that they are.@@ -625,6 +638,51 @@ def _onload_device(target: Any) -> Optional[str]:return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"+def arm_graphs_after_placement(+ pipe: Any,+ applied: dict,+ logger: Any = None,+) -> dict:+ """Arm the denoiser graphs for the placement that actually happened: replaces a resident-planned graph a hook now+ wraps, and arms the one apply_speed_optims deferred for an offloaded denoiser (the copies are recorded in it)."""+ pending = bool(getattr(pipe, "_unsloth_cuda_graph_after_placement", False))+ hooked = bool(applied.get("cuda_graph")) and _denoiser_moves(pipe)+ if not (pending or hooked):+ return applied+ try:+ from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle++ if not cuda_graph.offload_graphs_enabled():+ cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())+ pipe._unsloth_cuda_graphs = ()+ pipe._unsloth_cuda_graph_reason = "offload active"+ applied["cuda_graph"] = False+ return applied+ handles, reason = cuda_graph.arm_after_placement(pipe, logger = logger)+ pipe._unsloth_cuda_graph_after_placement = False+ pipe._unsloth_cuda_graph_reason = reason+ applied["cuda_graph"] = bool(handles)+ except Exception as exc: # noqa: BLE001 - the load proceeds eager+ _warn(logger, "cuda graph arm after placement", exc)+ applied["cuda_graph"] = False+ return applied+++def _denoiser_moves(pipe: Any) -> bool:+ for module in _denoiser_dits(pipe) or [m for m in [_denoiser_unet(pipe)] if m is not None]:+ if getattr(module, "_hf_hook", None) is not None:+ return True+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}+ if any("offload" in str(key) for key in hooks):+ return True+ try:+ if any(getattr(m, "_hf_hook", None) is not None for m in module.modules()):+ return True+ except Exception: # noqa: BLE001+ return True+ return False++def engage_pinned_denoisers(pipe: Any,applied: dict,diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 444c35d582b..120b379d935 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -135,6 +135,7 @@ from .diffusion_speed import (SPEED_MAX,SPEED_OFF,apply_speed_optims,+ arm_graphs_after_placement,resolve_speed_mode,restore_backend_flags,settle_compile_fallback,@@ -6649,6 +6650,14 @@ class VideoBackend:speed_optims += ("vae_untiled_when_fits",)# Wan's decode also grows within a single tile, which tiling alone cannot bound.install_decoder_sync(pipe, target, logger = logger)+ # A family that opts in to graphs arms them against the placement that landed (offload hooks included).+ graph_applied = arm_graphs_after_placement(+ pipe, {"cuda_graph": "cuda_graph" in speed_optims}, logger+ )+ if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:+ speed_optims += ("cuda_graph",)+ elif not graph_applied.get("cuda_graph"):+ speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")# Last, so the bf16 entry wraps whatever decode path the steps above installed. Not on SPEED_OFF.if getattr(fam, "vae_force_fp32", False) and effective_speed != SPEED_OFF:vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)@@ -6679,6 +6688,14 @@ class VideoBackend:cache_reason,RESOLVED_UNSUPPORTED if static_decline else None,),+ "cuda_graph": (+ None,+ "on" if "cuda_graph" in speed_optims else "off",+ str(+ getattr(pipe, "_unsloth_cuda_graph_reason", None)+ or "speed tier does not capture"+ ),+ ),"transformer_quant": (transformer_quant_requested,transformer_quant_engaged or "off",@@ -7647,6 +7664,23 @@ class VideoBackend:)except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a loadlogger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)+ # H3 declines graphs (cuda_graph_decline), so this only acts when a family override forces them: the graph was+ # deferred to the placement above, so arm it against the hooks that landed, or record why it stays eager,+ # rather than leaving the pending note in the status.+ graph_applied = arm_graphs_after_placement(+ speed_view, {"cuda_graph": "cuda_graph" in speed_optims}, logger+ )+ if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:+ speed_optims += ("cuda_graph",)+ elif not graph_applied.get("cuda_graph"):+ speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")+ if speed_view is not pipe:+ for attr in ("_unsloth_cuda_graph_reason", "_unsloth_cuda_graphs"):+ if hasattr(speed_view, attr):+ try:+ setattr(pipe, attr, getattr(speed_view, attr))+ except Exception: # noqa: BLE001+ pass# nothing here compiles, so it follows the REQUESTED tier, not the denoiser's eager downgrade abovetry:from .video_minimax_h3_vae import apply_h3_vae_speedupsdiff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 02d5a3f69e5..714712b24c3 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -83,6 +83,10 @@ class VideoFamily:# Opt-in to the CUDA-graph capture of the denoiser forward (diffusion_cuda_graph.py): only for a family that runs# ONE forward per step with no CFG and no step cache, so a capture has one stable input tree to replay.supports_cuda_graph: bool = False+ # Opt-in to the capture when the denoiser is offloaded: the replay then records the onloads too (set from an A/B).+ offload_cuda_graph: bool = False+ # Status text when the family declines the capture (the measured reason), instead of "family opts out".+ cuda_graph_decline: Optional[str] = None# Video DiTs are bf16-native, so fp16 promotes to float32; defaults True.fp16_incompatible: bool = True# "native" = measured accurate in plain float16 on fp16-only cards; None = promote (unmeasured).@@ -207,6 +211,12 @@ _FAMILIES: tuple[VideoFamily, ...] = (# capture peak, on a family already needing 87.5 GB, and 31.1 s on the first render (break-even ~530 videos).# This is the CHEAPEST grid H3 ships, so the larger presets can only be more GPU-bound.supports_cuda_graph = False,+ # Streamed, H3's prequant groups onload through diffusers' stream path, which waits on the host per group: a+ # capture cannot hold them (arm_after_placement refuses it by name when the family is forced).+ cuda_graph_decline = (+ "measured GPU-bound: 1.0018x resident for 3.93 GB held; streamed, its group onloads wait on the host "+ "(27 per step at 16 GB), which a graph cannot record"+ ),gguf_repo = "unsloth/MiniMax-H3-GGUF",# Hosted pre-quantized FL2VA denoisers. The modular workflow builds each component through its own# from_pretrained, so there is no dense module to quantise in place: these are the ONLY way to run the 66.3 GB@@ -301,6 +311,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (bf16_components_gb = (10.0, 11.4, 2.8),vae_force_fp32 = True,cudnn_benchmark = False,+ # Streamed: faster than eager with bit-identical latents; two replays per step (CFG), one graph per shape.+ offload_cuda_graph = True,# UMT5 keeps its overflowing `wo` in fp32 itself; the VAE stays fp32 (vae_force_fp32).fp16_guard = "native",# Byte-identical mirror of QuantStack/Wan2.2-TI2V-5B-GGUF (13 quants + companion VAE).@@ -377,6 +389,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (bf16_components_gb = (16.6, 14.8, 2.4),fp16_guard = "native",cudnn_benchmark = False,+ # Needs the capture-safe forward (diffusion_capture_safe) and the memoized trim (diffusion_attention).+ offload_cuda_graph = True,),# The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its# own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.diff --git a/studio/backend/tests/test_diffusion_attention_trim.py b/studio/backend/tests/test_diffusion_attention_trim.pyindex 0f816ae11f1..904916e088d 100644--- a/studio/backend/tests/test_diffusion_attention_trim.py+++ b/studio/backend/tests/test_diffusion_attention_trim.py@@ -250,3 +250,85 @@ def test_post_hook_always_clears_flag_after_forward_and_on_exception():with pytest.raises(RuntimeError):dit()assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is False for b in dit.transformer_blocks)+++def _t2v_kwargs(device = "cpu"):+ return {+ "image_embeds": torch.zeros(1, 5, 3, device = device),+ "encoder_hidden_states": torch.arange(8.0, device = device).reshape(1, 4, 2),+ "encoder_attention_mask": torch.tensor([[1, 1, 0, 0]], device = device),+ "encoder_hidden_states_2": torch.arange(3.0, device = device).reshape(1, 3, 1),+ "encoder_attention_mask_2": torch.tensor([[1, 0, 0]], device = device),+ }+++def test_trim_pre_hook_plans_once_per_input_tensors_and_matches_the_stock_trim():+ # A pipeline hands the same prompt tensors to every step: the decisions (host reads) are made once and reused.+ dit = _fake_dit()+ base = _t2v_kwargs()+ first = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ second = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ for key in base:+ assert torch.equal(first[key], second[key])+ want_s, want_m, _ = att._trim_stream(+ base["encoder_hidden_states"], base["encoder_attention_mask"]+ )+ assert torch.equal(second["encoder_hidden_states"], want_s) and torch.equal(+ second["encoder_attention_mask"], want_m+ )+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 1+ # an edited mask (version bump) or a new tensor plans afresh+ base["encoder_attention_mask"][0, 2] = 1+ third = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ assert third["encoder_hidden_states"].shape == (1, 3, 2)+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 2++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_trim_pre_hook_makes_no_host_wait_after_the_first_step():+ import warnings++ dit = _fake_dit()+ base = _t2v_kwargs("cuda")+ att._hunyuan_trim_pre_hook(dit, (), dict(base)) # step 0 reads the host+ torch.cuda.synchronize()+ prev = torch.cuda.get_sync_debug_mode()+ torch.cuda.set_sync_debug_mode("warn")+ try:+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ out = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ finally:+ torch.cuda.set_sync_debug_mode(prev)+ assert not [w for w in caught if "synchroniz" in str(w.message).lower()]+ assert out["encoder_hidden_states"].shape == (1, 2, 2)+ assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is True for b in dit.transformer_blocks)+++def test_the_null_mask_flag_keys_the_cuda_graph():+ # The flag picks the attention branch inside the forward: the graph layer keys on it (GRAPH_KEY_EXTRA_ATTR).+ import core.inference.diffusion_cuda_graph as cg++ dit = _fake_dit()+ dit.transformer_blocks[0].attn.processor = None+ assert att._hunyuan_null_mask_state(dit) is False+ att._set_hunyuan_null_mask(dit, True)+ assert att._hunyuan_null_mask_state(dit) is True+ assert cg.GRAPH_KEY_EXTRA_ATTR == "_unsloth_graph_key_extra"+++def test_trim_pre_hook_trims_and_plans_once_for_inference_tensors():+ # Renders run under torch.inference_mode: an inference tensor has no version counter (reading one raises), and+ # the hook must still trim (not fall back to the untrimmed inputs) and reuse its plan.+ dit = _fake_dit()+ with torch.inference_mode():+ base = _t2v_kwargs()+ out = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ again = att._hunyuan_trim_pre_hook(dit, (), dict(base))[1]+ assert out["image_embeds"].shape == (1, 0, 3)+ assert out["encoder_hidden_states"].shape == (1, 2, 2) and out[+ "encoder_hidden_states_2"+ ].shape == (1, 1, 1)+ assert all(getattr(b.attn, att._NULL_ATTN_FLAG) is True for b in dit.transformer_blocks)+ assert torch.equal(out["encoder_hidden_states"], again["encoder_hidden_states"])+ assert len(dit.__dict__[att._TRIM_MEMO_ATTR]) == 1diff --git a/studio/backend/tests/test_diffusion_capture_safe.py b/studio/backend/tests/test_diffusion_capture_safe.pyindex cf6f8fe656a..46bbfe109be 100644--- a/studio/backend/tests/test_diffusion_capture_safe.py+++ b/studio/backend/tests/test_diffusion_capture_safe.py@@ -404,7 +404,8 @@ def test_qwen_image_21_is_declined_with_the_reason(fresh_cache):assert forward is Noneassert why == ("QwenImage21Transformer2DModel forward is not capture-safe (its pipeline passes the prefix KV "- "cache as a Python object on every step and its forward syncs the host)"+ "cache as a Python object on every step; the Studio fast step that replays decode steps from "+ "tensors is not installed)")Sub = type("Sub", (QwenImage21Transformer2DModel,), {"forward": lambda self, x: x})assert cs.resolve(Sub) == (None, None)diff --git a/studio/backend/tests/test_diffusion_capture_safe_hv15.py b/studio/backend/tests/test_diffusion_capture_safe_hv15.pynew file mode 100644index 00000000000..6f49452efac--- /dev/null+++ b/studio/backend/tests/test_diffusion_capture_safe_hv15.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""HunyuanVideo-1.5's capture-safe forward (``diffusion_capture_safe``).++CPU: a tiny random ``HunyuanVideo15Transformer3DModel`` run through the stock and the rewritten+forward must agree bit for bit over t2v (all-zero and emptied image stream) and i2v, batch 1 and 2,+partially padded masks, and the inputs Studio's eager trim pre-hook hands the forward. The rewritten+forward must also dispatch no op that reads a device value on the host.++CUDA: the rewritten forward records into a ``torch.cuda.CUDAGraph`` and replays bit-identical to+eager; the stock forward does not record.+"""++from __future__ import annotations++import pytest++from core.inference import diffusion_capture_safe as cs+from core.inference import diffusion_cuda_graph as cg++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")++_CLS = getattr(diffusers, "HunyuanVideo15Transformer3DModel", None)+pytestmark = pytest.mark.skipif(+ _CLS is None, reason = "diffusers has no HunyuanVideo15Transformer3DModel"+)++_TEXT_DIM, _BYT5_DIM, _IMAGE_DIM = 32, 24, 16+++def fresh_cache(monkeypatch):+ monkeypatch.setattr(cs, "_CACHE", {})+ monkeypatch.delenv(cs.CAPTURE_SAFE_ENV, raising = False)+++def _tiny_model(device = "cpu"):+ torch.manual_seed(0)+ model = _CLS(+ in_channels = 4,+ out_channels = 4,+ num_attention_heads = 2,+ attention_head_dim = 16,+ num_layers = 1,+ num_refiner_layers = 1,+ text_embed_dim = _TEXT_DIM,+ text_embed_2_dim = _BYT5_DIM,+ image_embed_dim = _IMAGE_DIM,+ rope_axes_dim = (4, 6, 6),+ )+ # Random (non-zero-init) weights everywhere, so no path is trivially zero.+ with torch.no_grad():+ for p in model.parameters():+ p.copy_(torch.randn_like(p) * 0.2)+ return model.to(device).eval()+++# (mllm mask rows, byt5 mask rows): right-padded, interleaved, fully valid, byt5 fully padded.+_MASKS = {+ "b1_right_padded": ([[1, 1, 1, 0, 0, 0, 0]], [[1, 1, 0, 0, 0]]),+ "b2_interleaved": (+ [[1, 0, 1, 1, 0, 0, 1], [1, 1, 0, 0, 0, 0, 0]],+ [[0, 1, 0, 1, 1], [1, 1, 1, 0, 0]],+ ),+ "b2_valid_and_empty_byt5": ([[1] * 7, [1, 1, 1, 1, 0, 0, 0]], [[1] * 5, [0] * 5]),+}+_IMAGES = ("t2v_zero", "t2v_empty", "i2v")+++def _inputs(+ masks,+ image,+ device = "cpu",+ seed = 1,+):+ gen = torch.Generator().manual_seed(seed)+ m1 = torch.tensor(masks[0])+ m2 = torch.tensor(masks[1])+ batch = m1.shape[0]+ if image == "t2v_zero":+ img = torch.zeros(batch, 3, _IMAGE_DIM)+ elif image == "t2v_empty":+ img = torch.zeros(batch, 0, _IMAGE_DIM)+ else:+ img = torch.randn(batch, 3, _IMAGE_DIM, generator = gen)+ kw = dict(+ hidden_states = torch.randn(batch, 4, 2, 4, 4, generator = gen),+ timestep = torch.full((batch,), 500.0),+ encoder_hidden_states = torch.randn(batch, m1.shape[1], _TEXT_DIM, generator = gen),+ encoder_attention_mask = m1,+ encoder_hidden_states_2 = torch.randn(batch, m2.shape[1], _BYT5_DIM, generator = gen),+ encoder_attention_mask_2 = m2,+ image_embeds = img,+ return_dict = False,+ )+ return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in kw.items()}+++def _trimmed(model, kw):+ """The kwargs Studio's eager trim pre-hook hands the forward (empty t2v image, trimmed text)."""+ from core.inference import diffusion_attention as da++ _, out = da._hunyuan_trim_pre_hook(model, (), dict(kw))+ da._set_hunyuan_null_mask(model, False)+ return out+++def test_hv15_rewrite_is_registered_and_resolves(fresh_cache):+ safe, why = cs.resolve(_CLS)+ assert why is None, why+ assert getattr(safe, "__unsloth_capture_safe__", False) is True+ import inspect++ assert inspect.signature(safe) == inspect.signature(_CLS.forward)+ src = inspect.getsource(safe)+ assert cs._HV15_MERGE_HELPER_NAME in src and cs._HV15_T2V_HELPER_NAME in src+ assert "if is_t2v:" not in src and "image[image_mask]" not in src++[email protected]("trim", [False, True], ids = ["untrimmed", "trimmed"])[email protected]("image", _IMAGES)[email protected]("masks", list(_MASKS.values()), ids = list(_MASKS))+def test_hv15_rewritten_forward_is_bit_identical(fresh_cache, masks, image, trim):+ safe, why = cs.resolve(_CLS)+ assert why is None, why+ model = _tiny_model()+ kw = _inputs(masks, image)+ if trim:+ kw = _trimmed(model, kw)+ with torch.no_grad():+ want = _CLS.forward(model, **kw)[0]+ got = safe(model, **kw)[0]+ assert torch.isfinite(want).all()+ assert torch.equal(want, got)+++def _stock_image_stream(image_embeds, states, primary_mask, batch_size):+ is_t2v = torch.all(image_embeds == 0)+ if is_t2v:+ states = states * 0.0+ mask = torch.zeros((batch_size, states.shape[1]), dtype = primary_mask.dtype)+ else:+ mask = torch.ones((batch_size, states.shape[1]), dtype = primary_mask.dtype)+ return states, mask+++def _stock_merge(e1, m1, e2, m2, e3, m3):+ states, masks = [], []+ for t, tm, t2, tm2, im, imm in zip(e1, m1, e2, m2, e3, m3):+ states.append(+ torch.cat(+ [+ im[imm],+ t2[tm2],+ t[tm],+ im[~imm],+ torch.zeros_like(t2[~tm2]),+ torch.zeros_like(t[~tm]),+ ],+ dim = 0,+ )+ )+ masks.append(torch.cat([imm[imm], tm2[tm2], tm[tm], imm[~imm], tm2[~tm2], tm[~tm]], dim = 0))+ return torch.stack(states), torch.stack(masks)++[email protected]("t2v", [True, False])[email protected]("mask_dtype", [torch.int64, torch.bool, torch.float32])+def test_hv15_image_stream_matches_both_stock_arms_bitwise(t2v, mask_dtype):+ # inf and NaN make ``states * 0.0`` differ from zeros: the t2v arm must keep stock's NaNs and -0.0.+ states = torch.tensor([[[1.0, -2.0, float("inf")], [float("nan"), -0.0, 3.0]]]).repeat(2, 1, 1)+ image = torch.zeros(2, 2, 5) if t2v else torch.randn(2, 2, 5)+ primary = torch.ones(2, 4, dtype = mask_dtype)+ want = _stock_image_stream(image, states, primary, 2)+ got = cs.hv15_image_stream(image, states, primary, 2)+ assert torch.equal(want[0].view(torch.int32), got[0].view(torch.int32))+ assert got[1].dtype == want[1].dtype and torch.equal(got[1], want[1])++[email protected]("image_len", [0, 3])[email protected]("image_valid", [True, False])[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_hv15_merge_is_the_stock_permutation_bitwise(image_len, image_valid, dtype):+ gen = torch.Generator().manual_seed(0)+ m1 = torch.tensor([[1, 0, 1, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1]]).bool()+ m2 = torch.tensor([[0, 1, 0, 1, 1], [0, 0, 0, 0, 0]]).bool()+ m3 = torch.full((2, image_len), image_valid)+ e1 = torch.randn(2, 7, 6, generator = gen).to(dtype)+ e2 = torch.randn(2, 5, 6, generator = gen).to(dtype)+ e3 = torch.randn(2, image_len, 6, generator = gen).to(dtype)+ e2[0, 0, 0] = -0.0 # a padded byt5 row holding -0.0 / NaN must come out +0.0 like zeros_like+ e1[0, 1, 1] = float("nan")+ want = _stock_merge(e1, m1, e2, m2, e3, m3)+ got = cs.hv15_merge_streams(e1, m1, e2, m2, e3, m3)+ assert got[0].dtype == want[0].dtype and got[1].dtype == torch.bool+ assert torch.equal(+ want[0].view(torch.int16 if dtype == torch.bfloat16 else torch.int32),+ got[0].view(torch.int16 if dtype == torch.bfloat16 else torch.int32),+ )+ assert torch.equal(want[1], got[1])+++# Ops that read a device value on the host (a sync, illegal during capture) or size an output by one.+_HOST_READS = (+ "aten.index.Tensor",+ "aten.nonzero",+ "aten.masked_select",+ "aten.unique",+ "aten.item",+ "aten._local_scalar_dense",+ "aten.is_nonzero",+)+++def _aten_ops(fn):+ from torch.utils._python_dispatch import TorchDispatchMode++ class _Record(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.ops: set = set()++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ self.ops.add(str(func))+ return func(*args, **(kwargs or {}))++ with _Record() as rec:+ fn()+ return rec.ops++[email protected]("image", ["t2v_zero", "i2v"])+def test_hv15_rewritten_forward_reads_nothing_on_the_host(fresh_cache, image):+ safe, why = cs.resolve(_CLS)+ assert why is None, why+ model = _tiny_model()+ kw = _inputs(_MASKS["b2_interleaved"], image)+ with torch.no_grad():+ stock = _aten_ops(lambda: _CLS.forward(model, **kw))+ ours = _aten_ops(lambda: safe(model, **kw))+ assert any(op.startswith(_HOST_READS) for op in stock), sorted(stock)+ assert not any(op.startswith(_HOST_READS) for op in ours), sorted(+ o for o in ours if o.startswith(_HOST_READS)+ )+++def test_hv15_drifted_block_declines_with_the_reason(fresh_cache, monkeypatch):+ drifted = tuple(+ "image[~image_mask], # padded image"+ if line == "image[~image_mask], # invalid image"+ else line+ for line in cs._HV15_MERGE_BLOCK+ )+ monkeypatch.setattr(cs, "_HV15_MERGE_BLOCK", drifted)+ safe, why = cs.resolve(_CLS)+ assert safe is None+ assert "HunyuanVideo15Transformer3DModel forward is not capture-safe" in why+ assert "three-stream merge block changed" in why+++def test_hv15_kill_switch_declines(fresh_cache, monkeypatch):+ monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")+ safe, why = cs.resolve(_CLS)+ assert safe is None and cs.CAPTURE_SAFE_ENV in why+++_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+++def _capture(+ fn,+ kw,+ pool = None,+):+ static = {k: (v.clone() if torch.is_tensor(v) else v) for k, v in kw.items()}+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side), torch.no_grad():+ for _ in range(2):+ fn(**static)+ torch.cuda.current_stream().wait_stream(side)+ graph = torch.cuda.CUDAGraph()+ with torch.no_grad(), torch.cuda.graph(graph, pool = pool):+ out = fn(**static)[0]+ return graph, static, out+++@_cuda[email protected]("image", ["t2v_zero", "t2v_empty", "i2v"])+def test_hv15_rewritten_forward_captures_and_replays_bit_identical(fresh_cache, image):+ safe, why = cs.resolve(_CLS)+ assert why is None, why+ model = _tiny_model("cuda")+ kw = _inputs(_MASKS["b2_interleaved"], image, device = "cuda")+ graph, static, out = _capture(lambda **k: safe(model, **k), kw)+ for seed in (2, 3):+ # New values in the SAME buffers (t2v stays all-zero / empty, i2v gets new embeds).+ fresh = _inputs(_MASKS["b2_interleaved"], image, device = "cuda", seed = seed)+ for k, v in fresh.items():+ if torch.is_tensor(v):+ static[k].copy_(v)+ graph.replay()+ torch.cuda.synchronize()+ with torch.no_grad():+ eager = safe(model, **fresh)[0]+ stock = _CLS.forward(model, **fresh)[0]+ assert torch.equal(out, eager)+ assert torch.equal(eager, stock)+ del graph+++@_cuda+def test_hv15_stock_forward_does_not_capture(fresh_cache):+ # Last in the file: a failed capture can leave the CUDA context unusable for later captures.+ model = _tiny_model("cuda")+ kw = _inputs(_MASKS["b2_interleaved"], "t2v_zero", device = "cuda")+ stream = torch.cuda.current_stream()+ pool = torch.cuda.graph_pool_handle()+ with pytest.raises(RuntimeError):+ _capture(lambda **k: _CLS.forward(model, **k), kw, pool = pool)+ # torch.cuda.graph's __exit__ raises in capture_end before it leaves its capture stream: without this every later+ # test in the process ran on that stream (a prefetch test then saw its copies on a "compute" stream)+ torch.cuda.set_stream(stream)+ # torch leaves the CUDA generators in capture mode after an invalidated capture; later tests draw from them+ cg._heal_generators()+ # and the allocator still allocating to the dead capture's pool: every later empty_cache then freed nothing (a+ # later test's reserved-memory check failed only when it ran after this file)+ cg._abandon_capture_pool(pool)diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.pyindex b7165271e21..42b752d0679 100644--- a/studio/backend/tests/test_diffusion_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_cuda_graph.py@@ -576,6 +576,19 @@ def test_pool_survives_while_another_wrapper_still_holds_a_graph(stub_torch):assert cg._POOL_BOX[0] == pool+def test_a_graph_in_an_earlier_pool_does_not_keep_the_current_token(stub_torch):+ """Liveness is per graph: a wrapper still holding a graph in a pool dropped earlier (a placement move re-recorded+ another wrapper into a fresh pool) must not keep the current token alive after its own last graph is gone."""+ first, second = _armed(), _armed()+ first(_t(), timestep = _t((1,)), return_dict = False)+ cg._POOL_BOX[0] = None # what a drop of the first pool's token leaves+ second(_t(), timestep = _t((1,)), return_dict = False)+ current = cg._POOL_BOX[0]+ assert current is not None and first.cache+ cg.reset_all([second])+ assert cg._POOL_BOX[0] is None++def test_reset_all_forgets_the_pool_token_with_the_last_graph(stub_torch):"""A reset that destroys the last graph in the shared pool must forget its token, or the nextcapture dies on the allocator's "use_count > 0 INTERNAL ASSERT FAILED"."""@@ -787,6 +800,12 @@ def test_graph_eligible_family_opt_in_on_the_video_backend(stub_torch, monkeypatassert _eligible(monkeypatch, family = opted_in, family_default = False)[0] is Truebare = types.SimpleNamespace()assert _eligible(monkeypatch, family = bare, family_default = False)[0] is False+ # a family can opt in only for an offloaded denoiser+ offload_only = types.SimpleNamespace(offload_cuda_graph = True)+ assert _eligible(monkeypatch, family = offload_only, family_default = False)[0] is False+ assert (+ _eligible(monkeypatch, family = offload_only, family_default = False, offloaded = True)[0] is True+ )def test_stats_and_describe_are_json_safe(stub_torch):@@ -804,11 +823,20 @@ def test_stats_and_describe_are_json_safe(stub_torch):"eager_calls": 1,"fallbacks": 0,"cap_skips": 0,+ "invalidations": 0,+ "pool_bytes": aggregate["pool_bytes"],+ "planned_eager": 0,+ "shape_warmups": 0,+ "evictions": 0,+ "speed_eager": 0,"refused_float": 0,"refused_host_tensor": 0,"refused_object": 0,+ "placements": ["resident"],"poisoned": False,"capture_error": None,+ "eager_ms": None,+ "replay_ms": None,}assert json.loads(json.dumps(aggregate)) == aggregatediff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.pyindex 98a21982437..125929e1f58 100644--- a/studio/backend/tests/test_diffusion_memory.py+++ b/studio/backend/tests/test_diffusion_memory.py@@ -1168,6 +1168,28 @@ def test_reclaimable_snapshot_credits_cached_blocks_without_flushing(monkeypatchassert snap.free_mib == 2_000 + 4 * 1024+def test_reclaimable_snapshot_does_not_credit_a_live_cuda_graph_pool(monkeypatch):+ # A captured graph's pool is reserved but not allocated, and nothing else can allocate into it while the+ # graph lives (under block offload it also holds the prefetch window): the guard must see it as used.+ from core.inference import diffusion_cuda_graph as cg+ from core.inference import diffusion_memory as dm++ monkeypatch.setattr(+ dm,+ "snapshot_device_memory",+ lambda target: DeviceMemory(+ "cuda", "cuda", "discrete_vram", free_mib = 2_000, total_mib = 16_302+ ),+ )+ _fake_torch_allocator(monkeypatch, reserved = 6 * 1024**3, allocated = 2 * 1024**3)+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 3 * 1024**3)+ assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000 + 1024+ monkeypatch.setattr(+ cg, "live_pool_free_bytes", lambda: 5 * 1024**3+ ) # the pool is all of the cached bytes+ assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000++def test_reclaimable_snapshot_never_claims_more_than_the_card(monkeypatch):# The credit is arithmetic, so a bogus allocator reading must not invent memory the card# does not have and talk the guard out of a real refusal.diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.pynew file mode 100644index 00000000000..aa92798480a--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py@@ -0,0 +1,759 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CUDA graphs over an offloaded denoiser (``diffusion_cuda_graph.arm_after_placement``).++Block streaming: Studio's ``_apply_group_offload`` with the event-fenced prefetch; one graph records the whole forward+with the copy-stream copies inside it. Partial residency: some groups pinned, the rest streamed; a release / restore+moves the resident weights and the graph records again. Model offload: accelerate's CPU-offload hook moves the module+eagerly, the replay sits under it and records again when the weights land elsewhere. Every case checks the replay is+bit-identical to the eager forward of the same placement and that a replay makes no host synchronization."""++from __future__ import annotations++import copy+import types+import warnings++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_cuda_graph as cg # noqa: E402+import core.inference.diffusion_memory as dm # noqa: E402+import core.inference.diffusion_offload_prefetch as op # noqa: E402+import core.inference.diffusion_speed as ds # noqa: E402++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in (+ op.ASYNC_PREFETCH_ENV,+ op.PREFETCH_DEPTH_ENV,+ cg.OFFLOAD_CUDA_GRAPH_ENV,+ cg.CUDA_GRAPH_DISABLE_ENV,+ cg.SPEED_CHECK_ENV,+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",+ "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",+ "UNSLOTH_DIFFUSION_PIN_TOP_GROUP",+ ):+ monkeypatch.delenv(name, raising = False)+ # timing a tiny forward on a shared card is noise: these tests check engagement and exactness+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0")+ yield+ cg._POOL_BOX[0] = None+++class _Net(torch.nn.Module):+ def __init__(+ self,+ width = 512,+ blocks = 8,+ ):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, width)+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(width, width) for _ in range(blocks))+ self.proj_out = torch.nn.Linear(width, 64)++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ for block in self.blocks:+ x = torch.nn.functional.gelu(block(x))+ out = self.proj_out(x)+ return (out,) if not return_dict else {"sample": out}+++def _cuda():+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ pytest.skip("needs NVIDIA CUDA")+ pytest.importorskip("diffusers.hooks")+++def _pipe(net):+ return types.SimpleNamespace(transformer = net, components = {"transformer": net})+++def _count_syncs(fn):+ prev = torch.cuda.get_sync_debug_mode()+ torch.cuda.set_sync_debug_mode("warn")+ try:+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ out = fn()+ n = sum(1 for w in caught if "synchroniz" in str(w.message).lower())+ finally:+ torch.cuda.set_sync_debug_mode(prev)+ return out, n+++def _inputs(seed):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ return torch.randn(16, 64, generator = g).cuda(), torch.rand(1, generator = g).cuda()+++def _run(net, seeds):+ outs = []+ with torch.no_grad():+ for seed in seeds:+ x, t = _inputs(seed)+ outs.append(net(x, t, return_dict = False)[0].clone())+ torch.cuda.synchronize()+ return outs+++def _streamed(net, resident_mib = None):+ kwargs = {"resident_transformer_mib": resident_mib} if resident_mib else {}+ assert dm._apply_group_offload(_pipe(net), "cuda", None, **kwargs)+ return net+++def _eager_reference(resident_mib, seeds):+ torch.manual_seed(0)+ net = _streamed(_Net(), resident_mib)+ return _run(net, seeds)++[email protected]("resident_mib", [None, 3])+def test_block_streamed_forward_replays_bit_identically_with_no_host_sync(resident_mib):+ _cuda()+ seeds = list(range(6))+ want = _eager_reference(resident_mib, seeds)+ torch.manual_seed(0)+ net = _streamed(_Net(), resident_mib)+ pipe = _pipe(net)+ handles, reason = cg.arm_after_placement(pipe)+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason+ assert reason.startswith("captured")+ got = _run(net, seeds[:2]) # first call records, the second replays+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 2+ with torch.no_grad():+ for seed in seeds[2:]:+ x, t = _inputs(seed)+ out, syncs = _count_syncs(lambda: net(x, t, return_dict = False)[0].clone())+ assert syncs == 0+ got.append(out)+ torch.cuda.synchronize()+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handles[0].stats["captures"] == 1 and handles[0].stats["eager_calls"] == 0+ # weights are back on their host copies between calls; the streamed ones were copied inside the replay+ assert net.blocks[-1].weight.device.type == "cpu"+ pf = op.module_prefetcher(net)+ assert pf.stats["missed"] == 0+ # the pool (the step's activations plus the streamed copies) is what the generate-time guard leaves uncredited+ assert cg.live_pool_bytes() >= handles[0].stats["pool_bytes"] > 0+ # the guard leaves out only the pool's unallocated part; its allocated outputs are already not reclaimable+ assert 0 < cg.live_pool_free_bytes() <= cg.live_pool_bytes()+ cg.uninstall_all(handles)+ assert net.__dict__["forward"] is handles[0].placement.prev # the offload hook chain is back+++def test_release_and_restore_records_again_and_stays_exact():+ _cuda()+ seeds = [0, 1, 2, 3]+ want = _eager_reference(3, seeds)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ pipe = _pipe(net)+ handles, _ = cg.arm_after_placement(pipe)+ got = _run(net, seeds[:2])+ restore = dm.release_resident_groups(pipe, 10_000, None)+ assert restore is not None+ got += _run(net, seeds[2:3]) # everything streams now: a new placement, recorded again+ restore()+ got += _run(net, seeds[3:])+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handles[0].stats["invalidations"] >= 1 and handles[0].stats["captures"] >= 2+++def test_unpinned_host_copies_keep_the_forward_eager_with_the_reason(monkeypatch):+ _cuda()+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")+ torch.manual_seed(0)+ net = _Net()+ pipe = _pipe(net)+ assert dm._apply_group_offload(pipe, "cuda", None, stream_text_encoders = False)+ placement, why = cg.offload_placement(net)+ if why is None:+ pytest.skip("this diffusers pins the stream groups regardless")+ assert placement is None and ("pinned" in why or "prefetch" in why)+ handles, reason = cg.arm_after_placement(pipe)+ assert handles == () and "Net" in reason+++def test_kill_switch_keeps_the_old_refusal(monkeypatch):+ monkeypatch.setenv(cg.OFFLOAD_CUDA_GRAPH_ENV, "0")+ assert not cg.offload_graphs_enabled()+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_after_placement = True, _unsloth_cuda_graphs = ())+ monkeypatch.setattr(ds, "_denoiser_dits", lambda p: [])+ applied = ds.arm_graphs_after_placement(pipe, {"cuda_graph": False})+ assert applied["cuda_graph"] is False and pipe._unsloth_cuda_graph_reason == "offload active"+++def test_sequential_offload_is_refused_with_the_reason():+ net = _Net()+ net.blocks[0]._hf_hook = object()+ placement, why = cg.offload_placement(net)+ assert placement is None and "sequential" in why+++def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move():+ _cuda()+ accelerate = pytest.importorskip("accelerate")+ from accelerate import cpu_offload_with_hook++ seeds = [0, 1, 2, 3, 4]+ torch.manual_seed(0)+ ref = _Net().cuda()+ want = _run(ref, seeds)+ torch.manual_seed(0)+ net = _Net()+ other = torch.nn.Linear(4, 4)+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")+ _other, other_hook = cpu_offload_with_hook(+ other, execution_device = "cuda", prev_module_hook = hook+ )+ pipe = _pipe(net)+ handles, reason = cg.arm_after_placement(pipe)+ assert len(handles) == 1 and handles[0].placement.mode == "model", reason+ got = _run(net, seeds[:3])+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 3+ hook.offload() # what the next pipeline stage does: the weights go back to the host+ assert net.proj_in.weight.device.type == "cpu"+ got += _run(+ net, seeds[3:]+ ) # onloaded again by the hook; recorded again only if the addresses moved+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handles[0].stats["captures"] == 1 + handles[0].stats["invalidations"]+ del accelerate, other_hook+++def test_model_offload_re_records_a_moved_module_without_a_warm_up():+ """Every onload of a model-offloaded denoiser may land at new addresses: the same key records again with no+ warm-up step (its kernels exist), and replays stay exact across many moves."""+ _cuda()+ pytest.importorskip("accelerate")+ from accelerate import cpu_offload_with_hook++ torch.manual_seed(0)+ ref = _Net().cuda()+ torch.manual_seed(0)+ net, hook = cpu_offload_with_hook(_Net(), execution_device = "cuda")+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1, reason+ handle = handles[0]+ calls = []+ stock = handle.placement.eager++ def counting(*a, **k):+ calls.append(1)+ return stock(*a, **k)++ handle.orig = counting # what the warm-up and the capture call+ flushes = []+ handle._release = lambda: flushes.append(1) # gc + empty_cache: a move must not pay them+ for render in range(4):+ seeds = [10 * render + i for i in range(5)]+ want = _run(ref, seeds)+ got = _run(net, seeds)+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ hook.offload()+ torch.cuda.empty_cache()+ # a block of the size the denoiser freed, so the next onload is likely to land elsewhere+ _hold = torch.empty(4 << 20, device = "cuda")+ moves = handle.stats["invalidations"]+ assert moves >= 1 # the onloads did land elsewhere, so the re-record path ran+ assert handle.stats["captures"] == 1 + moves, (handle.stats, handle.capture_error)+ assert handle.placement.refusal(handle.stats) is None+ # first recording: WARMUP_ITERS warm-ups + the capture call; each re-record: the capture call only+ assert len(calls) == cg.WARMUP_ITERS + 1 + moves+ assert flushes == []+ del _hold+++def test_model_offload_stops_recording_only_when_moves_outpace_replays():+ placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)+ placement.mode = "model"+ placement._checked = object()+ many = cg.OffloadPlacement.MODEL_MAX_INVALIDATIONS++ def judge(stats, where):+ placement._fp = where # a new placement: judged afresh+ return placement.refusal(stats)++ assert judge({"invalidations": 1}, 1) is None+ assert "new addresses" in judge({"invalidations": many, "captures": many, "replays": many}, 2)+ # one move per render, a render's worth of replays per recording: keep recording+ assert judge({"invalidations": 5, "captures": 5, "replays": 5 * 8}, 3) is None+ # judged once per placement: the recording just made does not count against it until the next move+ assert placement.refusal({"invalidations": 5, "captures": 6, "replays": 5 * 8 + 1}) is None+++class _FailsInCapture(_Net):+ """A forward that makes a host-syncing call after its first blocks, but only while a graph records it."""++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ for i, block in enumerate(self.blocks):+ x = torch.nn.functional.gelu(block(x))+ if i == 3 and torch.cuda.is_current_stream_capturing():+ x = (+ x + float(x.sum().item()) * 0+ ) # host read: invalidates the capture with copies already queued+ out = self.proj_out(x)+ return (out,) if not return_dict else {"sample": out}+++def test_a_failed_streamed_capture_falls_back_eager_without_stale_copies():+ _cuda()+ seeds = [0, 1, 2]+ torch.manual_seed(0)+ want = _run(_streamed(_FailsInCapture()), seeds)+ torch.manual_seed(0)+ net = _streamed(_FailsInCapture())+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1, reason+ stream = torch.cuda.current_stream()+ got = _run(+ net, seeds+ ) # the first call's capture fails: it and every later call run eager, correctly+ assert torch.cuda.current_stream() == stream # not left on the failed capture's stream+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handles[0].poisoned and handles[0].stats["captures"] == 0+ assert op.module_prefetcher(net).ready == {} or all(+ e is None for e in op.module_prefetcher(net).ready.values()+ )+ assert net.blocks[-1].weight.device.type == "cpu"+ cg.uninstall_all(handles)+++class _SyncsOnStock(_Net):+ """Stock forward reads a value on the host (as HunyuanImage-2.1's text merge does); its rewrite does not."""++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ gate = x.new_tensor(1.0) if bool((x.abs().sum() >= 0).item()) else x.new_tensor(0.0)+ for block in self.blocks:+ x = torch.nn.functional.gelu(block(x))+ out = self.proj_out(x) * gate+ return (out,) if not return_dict else {"sample": out}+++def _capture_safe_forward(+ self,+ x,+ t,+ return_dict = True,+):+ x = self.proj_in(x) * t+ total = x.abs().sum()+ gate = torch.where(+ total >= 0, torch.ones_like(total), torch.zeros_like(total)+ ) # on the device: no host read+ for block in self.blocks:+ x = torch.nn.functional.gelu(block(x))+ out = self.proj_out(x) * gate+ return (out,) if not return_dict else {"sample": out}+++def test_a_capture_safe_rewrite_records_under_block_offload(monkeypatch):+ """The block-offload hook chain ends in the bound class forward; a capture-safe rewrite takes that slot while the+ graph is installed, so a family whose stock forward syncs the host still records when streamed."""+ _cuda()+ import core.inference.diffusion_capture_safe as cs++ monkeypatch.setattr(+ cs,+ "resolve",+ lambda cls: (_capture_safe_forward, None) if cls is _SyncsOnStock else (None, None),+ )+ seeds = [0, 1, 2, 3]+ torch.manual_seed(0)+ want = _run(_streamed(_SyncsOnStock()), seeds)+ torch.manual_seed(0)+ net = _streamed(_SyncsOnStock())+ inner = cg._innermost_class_forward(net)+ assert inner is not None and inner.forward.__func__ is _SyncsOnStock.forward+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason+ got = _run(net, seeds[:2])+ with torch.no_grad():+ for seed in seeds[2:]:+ x, t = _inputs(seed)+ out, syncs = _count_syncs(lambda: net(x, t, return_dict = False)[0].clone())+ assert syncs == 0+ got.append(out)+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert not handles[0].poisoned, handles[0].capture_error+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 4+ cg.uninstall_all(handles)+ assert (+ inner.forward.__func__ is _SyncsOnStock.forward+ ) # the stock forward is back with the hook chain+++class _Branchy(_Net):+ """Branches on Python state a pre-hook sets per call (HunyuanVideo-1.5's null-mask flag)."""++ flag = False++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ if self.flag:+ x = x * 2+ out = self.proj_out(x)+ return (out,) if not return_dict else {"sample": out}+++def test_per_call_python_state_keys_the_graph():+ _cuda()+ torch.manual_seed(0)+ net = _Branchy().cuda()+ net._unsloth_graph_key_extra = lambda: net.flag+ handle = cg.GraphedForward(net).enable()+ try:+ outs = {}+ with torch.no_grad():+ for flag in (False, True, False, True):+ net.flag = flag+ x, t = _inputs(1)+ outs.setdefault(flag, []).append(net(x, t, return_dict = False)[0].clone())+ handle.set_bypass(True)+ want = net(x, t, return_dict = False)[0]+ handle.set_bypass(False)+ assert torch.equal(outs[flag][-1], want)+ finally:+ cg.uninstall_all([handle])+ assert handle.stats["captures"] == 2 and handle.stats["replays"] == 4+++def test_a_failed_capture_leaves_the_cuda_rng_usable_and_on_its_sequence():+ """An invalidated capture raises in cudaStreamEndCapture before torch ends the generators' capture; every later+ eager CUDA draw (a pipeline's noise) then failed for the rest of the process."""+ _cuda()+ torch.cuda.manual_seed(123)+ want = torch.randn(8, device = "cuda")+ torch.manual_seed(0)+ net = _FailsInCapture().cuda()+ handle = cg.GraphedForward(net).enable()+ try:+ torch.cuda.manual_seed(123)+ _run(net, [0])+ assert handle.poisoned+ got = torch.randn(8, device = "cuda")+ finally:+ cg.uninstall_all([handle])+ assert torch.equal(got, want)+++def test_a_failed_capture_takes_the_allocator_off_its_pool(monkeypatch):+ """capture_end raises in cudaStreamEndCapture before endAllocateToPool, so an invalidated capture left its pool in+ the allocator's captures_underway: on torch 2.6 the next empty_cache tripped INTERNAL ASSERT captures_underway.empty()+ and no later capture recorded (seen on an A100); later torch skips the global release instead, so empty_cache freed+ nothing. Reproduced here on any torch by putting the pool back under capture before the raise."""+ _cuda()+ real_end = torch.cuda.CUDAGraph.capture_end+ begin = torch._C._cuda_beginAllocateCurrentStreamToPool+ stuck = []++ def end_like_torch_2_6(self):+ real_end(self)+ stuck.append(self.pool())+ begin(torch.cuda.current_device(), stuck[-1])+ raise RuntimeError("CUDA error: operation failed due to a previous error during capture")++ torch.manual_seed(0)+ net = _streamed(_Net())+ handles, _ = cg.arm_after_placement(_pipe(net))+ try:+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_end", end_like_torch_2_6)+ got = _run(net, [0])+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_end", real_end)+ assert stuck and handles[0].poisoned and handles[0].stats["fallbacks"] == 1+ # While the pool stayed under capture, empty_cache raised INTERNAL ASSERT (torch 2.6) or silently freed nothing+ # outside the private pools (later torch skips the global release while captures_underway is not empty).+ block = torch.empty(256 << 20, dtype = torch.uint8, device = "cuda")+ held = torch.cuda.memory_reserved()+ del block+ torch.cuda.empty_cache()+ assert torch.cuda.memory_reserved() <= held - (256 << 20)+ assert torch.equal(got[0], _eager_reference(None, [0])[0])+ torch.manual_seed(0)+ other = _streamed(_Net())+ again, _ = cg.arm_after_placement(_pipe(other))+ _run(other, [0, 1])+ assert (+ again[0].stats["captures"] == 1 and again[0].stats["replays"] == 2+ ) # later captures still record+ cg.uninstall_all(again)+ finally:+ end = (+ getattr(torch._C, "_cuda_endAllocateToPool", None)+ or torch._C._cuda_endAllocateCurrentStreamToPool+ )+ for pool in stuck: # never leave the process broken for the tests after this one+ try:+ end(torch.cuda.current_device(), pool)+ except Exception: # noqa: BLE001 - the fix already took it off+ pass+ cg.uninstall_all(handles)+++def test_model_offload_fingerprint_follows_replaced_buffers():+ """Module.to() puts NEW tensor objects in _buffers on every move: the fingerprint must read the registry each call,+ not a list taken once (that list keeps describing the obsolete buffers)."""+ torch.manual_seed(0)+ net = _Net()+ net.register_buffer("scale", torch.ones(4))+ net._old_forward = net.forward # what accelerate's CpuOffload leaves for the replay slot+ placement = cg.OffloadPlacement(net, "model")+ before = placement.token()+ net.scale = torch.full((4,), 9.0) # a new buffer object, as Module.to() makes on a move+ assert placement.token() != before+++class _Ev:+ def __init__(+ self,+ ms,+ done = True,+ ):+ self.ms, self.done = ms, done++ def query(self):+ return self.done++ def elapsed_time(self, end):+ return end.ms - self.ms+++def _judged(+ eager_ms,+ graph_ms,+ done = True,+ others = (),+):+ """A handle with key "k" timed (and ``others`` already kept), judged once; returns (handle, k's verdict)."""+ import os++ os.environ.pop(cg.SPEED_CHECK_ENV, None)+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.stats, handle.logger, handle._slower, handle.capture_error = {}, None, None, None+ handle.cache = {"k": object(), **{o: object() for o in others}}+ handle._dropped, handle._release = set(), lambda: None+ handle._judge = {+ "k": {+ "seen": 3,+ "eager": [(_Ev(0), _Ev(ms)) for ms in eager_ms],+ "graph": [(_Ev(0), _Ev(ms, done)) for ms in graph_ms],+ "verdict": None,+ },+ **{o: {"seen": 3, "eager": [], "graph": [], "verdict": ""} for o in others},+ }+ handle._judge_keys()+ return handle, handle._judge["k"]["verdict"]+++def test_an_offloaded_replay_slower_than_eager_drops_the_graphs():+ """The copy engine runs a replay's onload copies at a different rate per machine (an A100 VM: 16% slower than the+ stream's own copies); a key's first replays are timed against its own eager steps and its graph goes when slower."""+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0])+ assert why and "slower" in why and handle.cache == {} and handle._dropped == {"k"}+ assert handle.capture_error == {+ "type": "Refused",+ "msg": why,+ } # no graph left: the status names it+ handle._judge_keys()+ assert handle._judge["k"]["verdict"] == why # the verdict holds for the load+ handle.placement, handle.poisoned = None, False+ assert (+ cg.stats([handle])["eager_ms"] == 550.0 and cg.stats([handle])["replay_ms"] == 654.0+ ) # in the status+ handle, why = _judged(+ [550.0, 552.0], [560.0, 700.0, 710.0]+ ) # one clean replay: a stall is not the verdict+ assert why == "" and handle.cache+ handle, why = _judged([550.0, 552.0], [540.0, 541.0, 539.0])+ assert why == "" and handle.cache and not handle._dropped+ handle, why = _judged([550.0], [560.0, 561.0, 559.0]) # inside the margin: kept+ assert why == "" and handle.cache+ handle, why = _judged(+ [550.0], [655.0, 657.0, 654.0], done = False+ ) # not finished yet: never waits on them+ assert why is None and handle._slower is None and handle.cache+ handle, why = _judged([550.0], [655.0, 657.0]) # too few replays yet+ assert why is None and handle._slower is None+++def test_a_slower_key_drops_only_its_own_graph():+ """On an A100 VM Qwen-Image-2.1's first prompt length replayed at eager speed and the next one 16% slower: only the+ slower key goes eager, the other keeps replaying, and the status stays on."""+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0], others = ("first",))+ assert why and set(handle.cache) == {"first"} and handle._dropped == {"k"}+ assert handle.capture_error is None+++def test_the_eager_reference_is_the_callers_own_steps_not_the_recorded_warm_ups(monkeypatch):+ """The capture's warm-ups run under the recorder, whose copies follow the captured schedule: on an A100 VM that+ schedule was the slow part (a replay 17% slower than eager went unjudged because its warm-ups were as slow). The+ reference is the caller's eager steps on the stock prefetch path, timed before the first capture."""+ _cuda()+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)+ monkeypatch.setattr(+ cg, "SPEED_MARGIN", 1e9+ ) # this checks where the reference comes from, not a tiny net's speed+ monkeypatch.setattr(+ cg, "SPEED_GAIN", -1e9+ ) # nor whether a streamed key gains enough to keep its memory+ seeds = list(range(8))+ want = _eager_reference(None, seeds)+ torch.manual_seed(0)+ net = _streamed(_Net())+ handles, _ = cg.arm_after_placement(_pipe(net))+ handle = handles[0]+ recorded_steps = []+ recorder = handle.placement.recorder+ monkeypatch.setattr(+ handle.placement, "recorder", lambda call: recorded_steps.append(1) or recorder(call)+ )+ got = _run(+ net, seeds[:3]+ ) # one untimed (may compile), then SPEED_EAGER_SAMPLES timed, all eager+ assert (+ handle.stats["captures"] == 0 and handle.stats["eager_calls"] == 1 + cg.SPEED_EAGER_SAMPLES+ )+ (state,) = handle._judge.values()+ assert len(state["eager"]) == cg.SPEED_EAGER_SAMPLES+ eager_events = list(state["eager"])+ got += _run(net, seeds[3:]) # records, then replays+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == len(seeds) - 3+ assert state["eager"] == eager_events # the recorder's warm-ups added nothing+ assert (+ handle.stats["eager_ms"] > 0 and handle.stats["replay_ms"] > 0 and state["verdict"] == ""+ ) # judged, kept+ assert recorded_steps # and the capture did go through the recorder+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ cg.uninstall_all(handles)+++def test_the_timed_eager_steps_run_on_capture_like_copies(monkeypatch):+ """Compiled code guards on strides and storage offsets: eager steps on the caller's views compiled a variant the+ capture then compiled again (a cold Qwen-Image-2.1 render on an RTX PRO 6000 went from 16.8 to 23.1 s)."""+ _cuda()+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)+ monkeypatch.setattr(cg, "SPEED_MARGIN", 1e9)+ monkeypatch.setattr(+ cg, "SPEED_GAIN", -1e9+ ) # nor whether a streamed key gains enough to keep its memory+ torch.manual_seed(0)+ net = _streamed(_Net())+ handles, _ = cg.arm_after_placement(_pipe(net))+ offsets = []+ hook = net.proj_in.register_forward_pre_hook(+ lambda _m, a: offsets.append(a[0].storage_offset())+ )+ try:+ with torch.no_grad():+ for seed in range(3):+ x, t = _inputs(seed)+ padded = torch.cat([torch.zeros(1, 64, device = "cuda"), x])[+ 1:+ ] # a view at offset 64, like a slice+ assert padded.storage_offset() == 64+ want = _eager_reference(None, [seed])[0]+ assert torch.equal(net(padded, t, return_dict = False)[0], want)+ assert handles[0].stats["captures"] == 0 and offsets == [0, 0, 0]+ finally:+ hook.remove()+ cg.uninstall_all(handles)+++def test_every_key_times_its_own_eager_reference_before_it_records(monkeypatch):+ """A new input shape (a new prompt length) after the first capture is judged on its own eager steps too."""+ _cuda()+ monkeypatch.delenv(cg.SPEED_CHECK_ENV, raising = False)+ monkeypatch.setattr(cg, "SPEED_MARGIN", 1e9)+ monkeypatch.setattr(+ cg, "SPEED_GAIN", -1e9+ ) # nor whether a streamed key gains enough to keep its memory+ torch.manual_seed(0)+ net = _streamed(_Net())+ handles, _ = cg.arm_after_placement(_pipe(net))+ handle = handles[0]+ try:+ with torch.no_grad():+ for rows in (16, 16, 16, 16, 24, 24, 24, 24, 24):+ x = torch.randn(rows, 64, device = "cuda")+ t = torch.rand(1, device = "cuda")+ net(x, t, return_dict = False)+ torch.cuda.synchronize()+ assert handle.stats["captures"] == 2 and handle.stats["speed_eager"] == 2 * (+ 1 + cg.SPEED_EAGER_SAMPLES+ )+ assert len(handle._judge) == 2 and all(+ len(s["eager"]) == cg.SPEED_EAGER_SAMPLES for s in handle._judge.values()+ )+ finally:+ cg.uninstall_all(handles)+++def test_status_stays_on_while_the_eager_reference_is_timed():+ """The timed eager steps before the first capture are not a refusal: no "ran eager (bypassed)" status meanwhile."""+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.stats = {"captures": 0, "replays": 0, "eager_calls": 3, "speed_eager": 3}+ handle.cache, handle.capture_error, handle.poisoned, handle.placement = {}, None, False, None+ assert cg.never_engaged([handle]) is None+ handle.stats.update(eager_calls = 4) # one eager call for another reason: reported+ assert "ran eager" in cg.never_engaged([handle])+++def test_status_names_graphs_dropped_after_replaying():+ """A graph dropped mid-load (slower than eager) must not leave the status saying the step replays."""+ handle, why = _judged([550.0, 552.0], [655.0, 657.0, 654.0])+ handle.stats.update(captures = 1, replays = 5, eager_calls = 40)+ handle.capture_error, handle.poisoned, handle.placement = (+ {"type": "Refused", "msg": why},+ False,+ None,+ )+ resolved = {+ "cuda_graph": {"value": "on", "status": "applied", "reason": "captured per input shape"}+ }+ live, optims = cg.live_status(resolved, ["cuda_graph", "int8"], [handle])+ assert (+ live["cuda_graph"]["value"] == "off"+ and "dropped" in live["cuda_graph"]["reason"]+ and why in live["cuda_graph"]["reason"]+ )+ assert optims == ["int8"] and resolved["cuda_graph"]["value"] == "on"+ handle.cache = {+ "k": object()+ } # a live graph again (refusal lifted, recorded anew): status stays as recorded+ handle.capture_error = None+ assert cg.live_status(resolved, ["cuda_graph"], [handle])[0]["cuda_graph"]["value"] == "on"diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.pyindex a8a1fdbeeae..bc55b7328e8 100644--- a/studio/backend/tests/test_diffusion_offload_precision.py+++ b/studio/backend/tests/test_diffusion_offload_precision.py@@ -613,14 +613,25 @@ def test_quantised_blocks_survive_group_offload_under_no_grad(offloaded(x)[email protected]("offload_graphs", [True, False])@pytest.mark.parametrize("offload_active, denoiser_offloaded, expected",[(False, None, False), (True, None, True), (True, False, False), (True, True, True)],)-def test_graph_gate_follows_the_denoiser(monkeypatch, offload_active, denoiser_offloaded, expected):+def test_graph_gate_follows_the_denoiser(+ monkeypatch, offload_active, denoiser_offloaded, expected, offload_graphs+):+ """With offload graphs on, a moved denoiser is judged on everything else and armed after placement; with the+ kill switch, a moved denoiser is refused as before."""from core.inference import diffusion_cuda_graph as dcgfrom core.inference import diffusion_speed+ if offload_graphs:+ monkeypatch.delenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, raising = False)+ expected = False+ else:+ monkeypatch.setenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, "0")+seen: list = []def _eligible(target, **kwargs):diff --git a/studio/backend/tests/test_diffusion_offload_step_graphs.py b/studio/backend/tests/test_diffusion_offload_step_graphs.pynew file mode 100644index 00000000000..52b28ef28e3--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_step_graphs.py@@ -0,0 +1,816 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The whole-step graph of an offloaded denoiser on top of the per-block graphs (diffusion_block_graph).++The step graph records a forward whose blocks compile below their offload hooks; the per-block graphs run their compute+while it records, and arm as its fallback where they are the default. A streamed placement records its copies into the+prefetcher's slot ring and keeps a key only when the replay pays, else hands the ring back."""++from __future__ import annotations++import copy+import types+import warnings++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_block_graph as bg # noqa: E402+import core.inference.diffusion_cuda_graph as cg # noqa: E402+import core.inference.diffusion_memory as dm # noqa: E402+import core.inference.diffusion_offload_prefetch as op # noqa: E402++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in (+ bg.BLOCK_GRAPHS_ENV,+ bg.COMPILE_BELOW_HOOKS_ENV,+ cg.CUDA_GRAPHS_ENV,+ cg.CUDA_GRAPH_DISABLE_ENV,+ cg.OFFLOAD_CUDA_GRAPH_ENV,+ cg.STEP_SLOTS_ENV,+ op.ASYNC_PREFETCH_ENV,+ op.PREFETCH_DEPTH_ENV,+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",+ "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",+ "UNSLOTH_DIFFUSION_PIN_TOP_GROUP",+ ):+ monkeypatch.delenv(name, raising = False)+ # engagement and exactness here; the speed check is driven explicitly where a test needs it+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0")+ yield+ cg._POOL_BOX[0] = None+++class Block(torch.nn.Module):+ def __init__(self, width):+ super().__init__()+ self.lin = torch.nn.Linear(width, width)++ def forward(self, x):+ return torch.nn.functional.gelu(self.lin(x))+++class Net(torch.nn.Module):+ _repeated_blocks = ["Block"]++ def __init__(+ self,+ width = 256,+ blocks = 8,+ ):+ super().__init__()+ self.proj_in = torch.nn.Linear(16, width)+ self.blocks = torch.nn.ModuleList(Block(width) for _ in range(blocks))+ self.proj_out = torch.nn.Linear(width, 16)++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ for block in self.blocks:+ x = block(x)+ out = self.proj_out(x)+ return (out,) if not return_dict else {"sample": out}+++def _pipe(net):+ return types.SimpleNamespace(transformer = net, components = {"transformer": net})+++def _target():+ return types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda")+++# --- CPU -------------------------------------------------------------------------------------------------------+++def test_a_block_graph_runs_its_compute_while_a_whole_step_records():+ calls = []+ shared = bg._Shared(None, None)+ graph = bg.BlockGraph(Block(4), lambda *a, **k: calls.append(1) or "computed", shared)+ with cg._recording_step():+ assert cg.step_recording()+ assert graph(torch.zeros(2, 4)) == "computed"+ assert not cg.step_recording()+ assert calls == [1]+ # no first sighting, no recording, no eager count: the step above records the kernels themselves+ assert graph.stats["eager_calls"] == 0 and not graph.seen and not graph.cache+++def _step_handle(+ net,+ mode = "group",+ plan = None,+):+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.module = net+ handle.placement = types.SimpleNamespace(mode = mode) if mode else None+ handle.plan = plan+ handle.fallback = None+ handle.fallback_handle = None+ return handle+++def _arm(pipe, monkeypatch, *, hooked, pinned):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ applied = {"cuda_graph": True}+ handles = cg.arm_block_graphs(+ pipe,+ applied,+ target = _target(),+ family = types.SimpleNamespace(),+ hooked = hooked,+ pinned = pinned,+ )+ return handles, applied+++def test_an_armed_step_graph_stays_primary_with_per_block_graphs_as_its_fallback(monkeypatch):+ net = Net(blocks = 3)+ pipe = _pipe(net)+ step = _step_handle(net)+ pipe._unsloth_cuda_graphs = (step,)+ pipe._unsloth_cuda_graph_reason = (+ "captured per input shape: block-streamed (copies recorded in the graph)"+ )+ handles, applied = _arm(pipe, monkeypatch, hooked = True, pinned = True)+ assert handles == (step,) and applied["cuda_graph"]+ assert pipe._unsloth_cuda_graph_mode == "step" and callable(step.fallback)+ assert cg.status_reason(pipe, True) == (+ "denoiser step captured per input shape: block-streamed (copies recorded in the graph), replayed bit-identically"+ )+ # the fallback arms the per-block layer below the module's blocks, once+ armed = step.fallback()+ assert isinstance(armed, bg.BlockGraphSet) and len(armed.graphs) == 3+ assert cg.status_reason(pipe, True).endswith("; the steps it leaves eager record per block")+ armed.free()+++def test_a_streamed_step_graph_gets_a_per_block_fallback_only_when_asked(monkeypatch):+ net = Net(blocks = 3)+ pipe = _pipe(net)+ step = _step_handle(net)+ pipe._unsloth_cuda_graphs = (step,)+ _arm(pipe, monkeypatch, hooked = True, pinned = False)+ assert step.fallback is None # streamed: per-block graphs are opt-in, as without the step graph+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "1")+ _arm(pipe, monkeypatch, hooked = True, pinned = False)+ assert callable(step.fallback)+ model = _step_handle(net, mode = "model")+ pipe._unsloth_cuda_graphs = (model,)+ monkeypatch.delenv(bg.BLOCK_GRAPHS_ENV)+ _arm(pipe, monkeypatch, hooked = True, pinned = True)+ assert model.fallback is None+ monkeypatch.setenv(bg.BLOCK_GRAPHS_ENV, "0")+ pinned = _step_handle(net)+ pipe._unsloth_cuda_graphs = (pinned,)+ _arm(pipe, monkeypatch, hooked = True, pinned = True)+ assert pinned.fallback is None+++def test_a_planned_resident_step_keeps_its_graph_and_the_blocks_wait_as_fallback(monkeypatch):+ net = Net(blocks = 2)+ pipe = _pipe(net)+ step = _step_handle(net, mode = None, plan = lambda *a: None)+ pipe._unsloth_cuda_graphs = (step,)+ pipe._unsloth_cuda_graph_reason = "captured per input shape: resident"+ handles, _ = _arm(pipe, monkeypatch, hooked = False, pinned = False)+ assert handles == (step,) and callable(step.fallback)+ assert cg.status_reason(pipe, True) == cg.WHOLE_REASON+++class _Ev:+ def __init__(self, ms):+ self.ms = ms++ def query(self):+ return True++ def elapsed_time(self, end):+ return end.ms - self.ms+++class _Placement:+ def __init__(+ self,+ streams,+ ring = False,+ ):+ self._streams = streams+ self.ring = ring+ self.declined = None+ self.mode = "group"++ def streams(self):+ return self._streams++ def decline(self, reason):+ self.declined = reason+ return self.ring+++def _judged(+ placement,+ eager_ms,+ graph_ms,+ others = (),+):+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.stats, handle.logger, handle._slower, handle.capture_error = {}, None, None, None+ handle.placement = placement+ handle.cache = {"k": object(), **{o: object() for o in others}}+ handle._dropped, handle.released = set(), []+ handle._release = lambda: handle.released.append(1)+ handle._judge = {+ "k": {+ "seen": 3,+ "eager": [(_Ev(0), _Ev(ms)) for ms in eager_ms],+ "graph": [(_Ev(0), _Ev(ms)) for ms in graph_ms],+ "verdict": None,+ },+ **{o: {"seen": 3, "eager": [], "graph": [], "verdict": ""} for o in others},+ }+ handle._judge_keys()+ return handle, handle._judge["k"]["verdict"]+++def test_a_streamed_key_is_kept_only_when_its_replay_pays():+ # a tie on a streaming placement: the ring and the pool are not worth it, and no other key records this load+ placement = _Placement(streams = True)+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7], others = ("j",))+ assert why and "within 1%" in why and placement.declined == why+ assert handle.cache == {} and handle._dropped == {"k", "j"} and handle.released == [1]+ # with a slot ring the flush waits for the ring's drop at the end of the forward: one flush, not one mid-forward+ placement = _Placement(streams = True, ring = True)+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])+ assert placement.declined == why and handle.cache == {} and handle.released == []+ # a real gain keeps it+ placement = _Placement(streams = True)+ handle, why = _judged(placement, [100.0, 101.0], [97.0, 97.5, 98.0])+ assert why == "" and placement.declined is None and "k" in handle.cache+ # pinned (nothing streams): a tie keeps the graph, as before+ placement = _Placement(streams = False)+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])+ assert why == "" and placement.declined is None and "k" in handle.cache+++def test_no_key_is_judged_while_another_graph_records(monkeypatch):+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)+ placement = _Placement(streams = True)+ handle, why = _judged(placement, [100.0, 100.4], [99.6, 99.8, 99.7])+ assert why is None and placement.declined is None and "k" in handle.cache+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ handle._judge_keys()+ assert handle._judge["k"]["verdict"]+++class _Wrapper:+ def __init__(self, cache):+ self.cache = cache+++def test_every_live_graph_pool_stays_out_of_the_reclaimable_memory(monkeypatch):+ # an earlier graph's pool after a later capture failed and retired the shared id+ monkeypatch.setattr(cg, "_POOL_BOX", [None])+ live = _Wrapper({"k": types.SimpleNamespace(pool_token = (0, 7))})+ monkeypatch.setattr(cg, "_LIVE_WRAPPERS", {live})+ segments = [+ {"device": 0, "segment_pool_id": (0, 7), "total_size": 100, "allocated_size": 40},+ {"device": 0, "segment_pool_id": (0, 0), "total_size": 500, "allocated_size": 0},+ {+ "device": 1,+ "segment_pool_id": (0, 7),+ "total_size": 900,+ "allocated_size": 0,+ }, # another card+ ]+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ assert cg.live_pool_free_bytes() == 60+++def test_only_the_streamed_groups_slots_are_allocated_before_a_capture(monkeypatch):+ pf = op.GroupPrefetcher.__new__(op.GroupPrefetcher)+ streamed, resident = types.SimpleNamespace(), types.SimpleNamespace(_unsloth_resident = True)+ others = [types.SimpleNamespace(_unsloth_resident = True) for _ in range(2)]+ pf.groups = [resident, others[0], streamed, others[1]]+ pf.slot_of = {id(g): i % 3 for i, g in enumerate(pf.groups)}+ pf.slot_size, pf.slot_raw = 16, {}+ made = []+ monkeypatch.setattr(pf, "_alloc_slot", lambda where: made.append(where), raising = False)+ pf.materialize_slots()+ assert made == [2]+++def test_a_failed_streamed_capture_hands_the_ring_back(monkeypatch):+ _cuda()+ net, handle = _armed_streamed(monkeypatch)+ pf = op.module_prefetcher(net)++ def broken(*args, **kwargs):+ handle.placement.before_capture()+ raise RuntimeError("capture failed")++ monkeypatch.setattr(handle, "_capture", broken)+ ref = _net().cuda()+ _call(net, 0)+ assert torch.equal(_call(net, 1), _call(ref, 1))+ assert handle.poisoned and not pf.slot_raw+ handle.free()+++def test_a_declined_placement_refuses_every_later_call():+ placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)+ placement.mode = "group"+ placement._declined = None+ placement._slots = False+ placement.decline("replay measured within 1% of the eager step")+ assert placement.refusal({}) == "replay measured within 1% of the eager step"+++def test_both_graph_pools_are_kept_out_of_the_reclaimable_memory(monkeypatch):+ monkeypatch.setattr(+ dm,+ "snapshot_device_memory",+ lambda target: dm.DeviceMemory(+ "cuda", "cuda", "discrete_vram", free_mib = 1000, total_mib = 8000+ ),+ )+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a: 900 << 20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a: 100 << 20)+ monkeypatch.setattr(bg, "pool_bytes", lambda device = None: 300 << 20)+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 200 << 20)+ mem = dm.reclaimable_snapshot_device_memory(types.SimpleNamespace(device = "cuda"))+ assert mem.free_mib == 1000 + 800 - 300 - 200+++# --- CUDA ------------------------------------------------------------------------------------------------------+++def _cuda():+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ pytest.skip("needs NVIDIA CUDA")+ pytest.importorskip("diffusers.hooks")+++def _streamed(net, resident_mib = None):+ kwargs = {"resident_transformer_mib": resident_mib} if resident_mib else {}+ assert dm._apply_group_offload(_pipe(net), "cuda", None, **kwargs)+ # the background pin swaps host copies (a new placement): let it finish so the counts below are exact+ for group in dm._offload_groups(net):+ pinner = getattr(group, op._BG_PIN_ATTR, None)+ if pinner is not None:+ pinner.wait(group)+ return net+++def _inputs(seed, rows = 8):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ return torch.randn(rows, 16, generator = g).cuda(), torch.rand(1, generator = g).cuda()+++def _call(+ net,+ seed,+ rows = 8,+):+ x, t = _inputs(seed, rows)+ with torch.no_grad():+ out = net(x, t, return_dict = False)[0].clone()+ torch.cuda.synchronize()+ return out+++def _syncs(fn):+ prev = torch.cuda.get_sync_debug_mode()+ torch.cuda.set_sync_debug_mode("warn")+ try:+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ out = fn()+ finally:+ torch.cuda.set_sync_debug_mode(prev)+ return out, sum(1 for w in caught if "synchroniz" in str(w.message).lower())+++def _net():+ torch.manual_seed(0)+ return Net()+++def _armed_streamed(monkeypatch, slots = True):+ if not slots:+ monkeypatch.setenv(cg.STEP_SLOTS_ENV, "0")+ net = _streamed(_net())+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason+ return net, handles[0]+++def test_streamed_step_copies_land_in_the_slot_ring_not_the_graph_pool(monkeypatch):+ _cuda()+ ref = _net().cuda()+ want = [_call(ref, s) for s in range(4)]+ real_to_device = op._to_device+ fresh = []++ def to_device(src, device):+ if torch.cuda.is_current_stream_capturing():+ fresh.append(tuple(src.shape)) # a copy the graph records into a pool allocation+ return real_to_device(src, device)++ monkeypatch.setattr(op, "_to_device", to_device)+ counts = {}+ for slots in (False, True):+ cg._POOL_BOX[0] = None+ fresh.clear()+ net, handle = _armed_streamed(monkeypatch, slots)+ pf = op.module_prefetcher(net)+ got = [_call(net, 0), _call(net, 1)] # records, then replays+ got += [_call(net, 2), _call(net, 3)]+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 4+ counts[slots] = len(fresh)+ if slots:+ assert handle.placement._slots and pf.slot_raw and pf.stats["slot_fills"] > 0+ assert pf.stats.get("slot_fallbacks", 0) == 0+ else:+ assert not pf.slot_raw+ handle.free()+ monkeypatch.delenv(cg.STEP_SLOTS_ENV, raising = False)+ # without the ring every streamed block's weight and bias is a fresh pool allocation inside the recording; with+ # it only the dense top-level group's are (proj_in / proj_out: 4 tensors)+ assert counts[False] >= 2 * 7 and counts[True] <= 4+++def test_a_declined_streamed_placement_hands_the_ring_back_and_runs_eager(monkeypatch):+ _cuda()+ ref = _net().cuda()+ net, handle = _armed_streamed(monkeypatch)+ pf = op.module_prefetcher(net)+ _call(net, 0)+ _call(net, 1)+ assert pf.slot_raw and handle.stats["replays"] == 2 # the recording call replays too+ handle.placement.decline("replay measured within 1% of the eager step")+ handle.cache.clear()+ for seed in (2, 3):+ assert torch.equal(_call(net, seed), _call(ref, seed))+ assert (+ not pf.slot_raw and not pf.slot_of+ ) # dropped at the end of the first forward after the verdict+ assert handle.stats["captures"] == 1 and handle.capture_error["type"] == "Refused"+++def test_a_new_slot_ring_is_a_new_placement(monkeypatch):+ _cuda()+ net, handle = _armed_streamed(monkeypatch)+ pf = op.module_prefetcher(net)+ _call(net, 0)+ _call(net, 1)+ assert handle.stats["captures"] == 1+ pf.disable_slots()+ pf.enable_slots()+ ref = _net().cuda()+ assert torch.equal(_call(net, 2), _call(ref, 2))+ assert handle.stats["invalidations"] == 1 and handle.stats["captures"] == 2+++def test_a_failed_pinned_step_graph_hands_its_calls_to_per_block_graphs(monkeypatch):+ _cuda()+ ref = _net().cuda()+ net = _streamed(_net(), resident_mib = 64) # every group pinned resident under its hooks+ pipe = _pipe(net)+ cg.arm_after_placement(pipe)+ handles = cg.arm_block_graphs(+ pipe,+ {"cuda_graph": True},+ target = _target(),+ family = types.SimpleNamespace(),+ hooked = True,+ pinned = True,+ )+ step = handles[0]+ assert step.placement is not None and callable(step.fallback)+ assert not step.placement.streams() and not step.placement._slots+ _call(net, 0)+ assert step.stats["captures"] == 1 and step.fallback_handle is None+ step.poisoned = True # as a failed capture leaves it+ for seed in (1, 2, 3):+ assert torch.equal(_call(net, seed), _call(ref, seed))+ blocks = step.fallback_handle+ assert isinstance(blocks, bg.BlockGraphSet) and step.fallback is None+ # first call of each block runs it, the second records and replays, the third replays+ assert blocks.stats["captures"] == 8 and blocks.stats["replays"] == 16+ step.free()+ assert step.fallback_handle is None+ assert not any(+ isinstance(r.forward, bg.BlockGraph) for b in net.blocks for r in b._diffusers_hook._fn_refs+ )+++def test_per_block_graphs_under_a_recording_step_record_nothing(monkeypatch):+ _cuda()+ ref = _net().cuda()+ net = _streamed(_net(), resident_mib = 64)+ pipe = _pipe(net)+ handles, _ = cg.arm_after_placement(pipe)+ blocks, reason = bg.install_block_graphs(net, device = "cuda")+ assert reason == "armed"+ want = [_call(ref, s) for s in range(3)]+ got = [_call(net, s) for s in range(3)] # the step records on its first call and replays after+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 3+ assert blocks.stats["captures"] == 0 and blocks.stats["eager_calls"] == 0+ blocks.free()+ handles[0].free()+++def test_a_compiled_streamed_step_records_with_no_graph_break_and_a_new_shape_adds_none(+ monkeypatch,+):+ _cuda()+ import torch._dynamo.utils as du++ torch._dynamo.reset()+ ref = _net().cuda()+ net = _net()+ kwargs = {"fullgraph": False, "dynamic": None}+ for b in net.blocks:+ b.compile(**kwargs)+ net._unsloth_regional_compile_kwargs = kwargs+ _streamed(net)+ assert bg.compile_below_offload_hooks(net) == 8+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1, reason+ breaks = sum(du.counters["graph_break"].values())+ for rows in (8, 8, 8, 24, 24, 24):+ torch.testing.assert_close(+ _call(net, rows, rows), _call(ref, rows, rows), rtol = 1e-4, atol = 1e-4+ )+ assert (+ sum(du.counters["graph_break"].values()) == breaks+ ) # the hooks stay outside every compiled region+ s = handles[0].stats+ assert s["captures"] == 2 and s["replays"] == 6 and s["fallbacks"] == 0+ first = _call(net, 5, 24)+ assert torch.equal(_call(net, 5, 24), first)+ handles[0].free()+++def test_a_static_step_skip_under_the_hooks_is_replayed_past_only_while_it_plans_no_skip():+ _cuda()+ from core.inference import diffusion_step_skip as sk++ ref = _net().cuda()+ net = _net()+ pipe = _pipe(net)+ assert sk.install_static_step_skip(pipe, settings = {"min_steps": 12}) == sk.TC_STATIC+ _streamed(net, resident_mib = 64)+ cg.arm_after_placement(pipe)+ step = cg.arm_block_graphs(+ pipe,+ {"cuda_graph": True},+ target = _target(),+ family = types.SimpleNamespace(),+ hooked = True,+ pinned = True,+ )[0]+ skip = step.placement.skip+ assert isinstance(skip, sk.StaticStepSkip)+ # a render below min_steps plans no skip: the step records and replays straight past the skip layer+ skip.reset(8)+ for seed in range(3):+ assert torch.equal(_call(net, seed), _call(ref, seed))+ assert step.stats["captures"] == 1 and step.stats["replays"] == 3+ assert (+ skip.stats["calls"] == 3 and skip.stats["computed"] == 3+ ) # the replays are counted as computed steps+ # a render that plans skips: every call goes through the skip layer, the per-block graphs take the compute+ skip.reset(16)+ assert not all(skip.plan)+ replays = step.stats["replays"]+ for i in range(16):+ _call(net, 0)+ skip.step_end()+ assert step.stats["replays"] == replays and step.stats["skip_eager"] == 16+ assert skip.stats["skipped"] > 0+ assert (+ isinstance(step.fallback_handle, bg.BlockGraphSet)+ and step.fallback_handle.stats["replays"] > 0+ )+ step.free()+ sk.uninstall_static_step_skip(pipe)++[email protected]("timed", [True, False])+def test_a_key_warmed_by_its_timed_eager_steps_records_without_another_warm_up(monkeypatch, timed):+ _cuda()+ if timed:+ monkeypatch.delenv(cg.SPEED_CHECK_ENV)+ ref = _net().cuda()+ net, handle = _armed_streamed(monkeypatch)+ recorded = []+ real = handle.placement.recorder++ def recorder(call):+ inner = real(call)++ def record(*a, **k):+ recorded.append(torch.cuda.current_stream())+ return inner(*a, **k)++ return record++ handle.placement.recorder = recorder+ for seed in range(5):+ assert torch.equal(_call(net, seed), _call(ref, seed))+ assert handle.stats["captures"] == 1+ # timed: 3 eager steps ran the key on capture-like inputs, so only the recording itself calls the step;+ # untimed: the side-stream warm-ups run first, as for any first capture+ assert len(recorded) == (1 if timed else 1 + cg.WARMUP_ITERS)+ assert handle.stats["speed_eager"] == (3 if timed else 0)+ handle.free()+++class _SyncsWhileRecorded(Block):+ def forward(self, x):+ y = super().forward(x)+ if torch.cuda.is_current_stream_capturing():+ y = y + float(y.sum().item()) * 0 # a host read: invalidates the recording+ return y+++def test_a_failed_block_recording_takes_the_allocator_off_its_pool():+ """As for the whole step (test_diffusion_offload_cuda_graph): capture_end raises before endAllocateToPool, so a+ failed per-block recording left the allocator on the block pool and every later empty_cache freed nothing."""+ _cuda()+ torch.manual_seed(0)+ net = Net(blocks = 2).cuda()+ net.blocks[1] = _SyncsWhileRecorded(256).cuda()+ Net._repeated_blocks = ["Block", "_SyncsWhileRecorded"]+ try:+ handle, reason = bg.install_block_graphs(net, device = "cuda")+ assert reason == "armed"+ x, t = _inputs(0)+ with torch.no_grad():+ for _ in range(3):+ net(x, t, return_dict = False)+ torch.cuda.synchronize()+ assert handle.graphs[1].poisoned and handle.stats["fallbacks"] == 1+ torch.cuda.empty_cache()+ baseline = torch.cuda.memory_reserved()+ block = torch.empty(256 << 20, dtype = torch.uint8, device = "cuda")+ del block+ torch.cuda.empty_cache()+ # stuck on the dead pool, empty_cache released nothing and the 256 MiB segment stayed+ assert torch.cuda.memory_reserved() < baseline + (128 << 20)+ torch.randn(4, device = "cuda") # the CUDA generator left capture mode+ handle.free()+ # a later recording in the process still works+ other = Net(blocks = 2).cuda()+ again, _ = bg.install_block_graphs(other, device = "cuda")+ with torch.no_grad():+ for _ in range(3):+ other(x, t, return_dict = False)+ assert again.stats["captures"] == 2 and again.stats["replays"] == 4+ again.free()+ finally:+ Net._repeated_blocks = ["Block"]+++def test_releasing_an_offloaded_step_graph_frees_its_capture_stream_workspace(monkeypatch):+ calls = []+ monkeypatch.setattr(+ torch._C, "_cuda_clearCublasWorkspaces", lambda: calls.append(1), raising = False+ )+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: calls.append(2))+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.placement = _Placement(streams = True)+ handle._release()+ assert calls == [1, 2] # the recordings' own cuBLAS workspace first, then the segments+ handle.placement = None+ calls.clear()+ handle._release()+ assert calls == [2] # a resident graph's reset keeps the compute stream's workspace+++def test_freeing_a_streamed_step_graph_hands_the_slot_ring_back(monkeypatch):+ _cuda()+ net, handle = _armed_streamed(monkeypatch)+ pf = op.module_prefetcher(net)+ _call(net, 0)+ _call(net, 1)+ assert pf.slot_raw+ handle.free()+ assert not pf.slot_raw and not pf.slot_of+ ref = _net().cuda()+ assert torch.equal(_call(net, 2), _call(ref, 2))+++def test_the_first_capture_keeps_its_pool_while_its_graph_lives(monkeypatch):+ _cuda()+ monkeypatch.setattr(cg, "_POOL_BOX", [None])+ net = _net().cuda()+ handle = cg.GraphedForward(net, logger = None).enable()+ _call(net, 0)+ _call(net, 1)+ assert handle.stats["captures"] == 1 and cg._POOL_BOX[0] is not None+ cg._drop_pool_if_unused()+ assert cg._POOL_BOX[0] is not None+ handle.free()+++def test_freeing_a_resident_graph_keeps_offload_hooks_added_after_it():+ _cuda()+ net = _net()+ handle = cg.GraphedForward(net, logger = None).enable()+ _streamed(net)+ hooked = net.__dict__.get("forward")+ assert hooked is not None and hooked is not handle+ handle.free()+ assert net.__dict__.get("forward") is hooked+ ref = _net().cuda()+ assert torch.equal(_call(net, 0), _call(ref, 0))+++def _probe_handle():+ handle = cg.GraphedForward.__new__(cg.GraphedForward)+ handle.stats = {"eager_calls": 0, "speed_eager": 0}+ handle._dropped, handle._slower, handle._judge, handle._warmed = set(), None, {}, {}+ handle.max_graphs = 4+ handle.plan = None+ return handle+++def test_a_speed_probe_without_memory_for_its_copies_runs_the_callers_step(monkeypatch):+ handle = _probe_handle()++ def oom(live):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(handle, "_statics", oom)+ x = torch.ones(2)+ assert torch.equal(handle._timed_eager(lambda t: t * 2, (x,), {}, "k"), x * 2)+ assert "k" in handle._dropped+++def test_no_probe_step_is_timed_while_another_graph_records(monkeypatch):+ handle = _probe_handle()+ monkeypatch.setattr(handle, "_statics", lambda live: list(live))+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)+ x = torch.ones(2)+ for _ in range(3):+ handle._timed_eager(lambda t: t + 1, (x,), {}, "k")+ assert handle._judge["k"]["seen"] == 3 and handle._judge["k"]["eager"] == []+++def test_a_ring_drop_waits_while_another_graph_records(monkeypatch):+ pf = op.GroupPrefetcher.__new__(op.GroupPrefetcher)+ pf._drop_slots_at_end = False+ dropped = []+ monkeypatch.setattr(pf, "disable_slots", lambda: dropped.append(1), raising = False)+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1)+ pf._drop_ring()+ assert dropped == [] and pf._drop_slots_at_end+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ pf._drop_ring()+ assert dropped == [1] and not pf._drop_slots_at_end+++def test_a_capture_collision_leaves_the_cuda_generators_alone(monkeypatch):+ _cuda()+ net = _net().cuda()+ handle = cg.GraphedForward(net, logger = None).enable()+ healed = []+ monkeypatch.setattr(cg, "_heal_generators", lambda: healed.append(1))++ def collided(*args, **kwargs):+ raise RuntimeError("beginAllocateToPool: already recording to mempool_id")++ monkeypatch.setattr(handle, "_capture", collided)+ ref = _net().cuda()+ for seed in (0, 1):+ assert torch.equal(_call(net, seed), _call(ref, seed))+ assert handle.poisoned and healed == []+ handle.free()+++def test_an_offload_only_family_gets_no_resident_graph():+ net = _net()+ pipe = _pipe(net)+ pipe._unsloth_cuda_graph_offload_only = True+ handles, reason = cg.arm_after_placement(pipe)+ assert handles == () and "offloaded steps only" in reason+ pipe._unsloth_cuda_graph_offload_only = False+ handles, _ = cg.arm_after_placement(pipe)+ assert len(handles) == 1 and handles[0].placement is None+ cg.uninstall_all(handles)diff --git a/studio/backend/tests/test_diffusion_qwenimage21_graph.py b/studio/backend/tests/test_diffusion_qwenimage21_graph.pynew file mode 100644index 00000000000..7d7a303e852--- /dev/null+++ b/studio/backend/tests/test_diffusion_qwenimage21_graph.py@@ -0,0 +1,471 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Qwen-Image-2.1 decode steps as CUDA graphs: ``diffusion_qwenimage21.graph_plan`` + ``GraphedForward``.++CPU tests check the plan (what runs eager, that the planned call is tensors-only and bit-identical to the eager+fast step). CUDA tests record real graphs on a tiny random transformer and compare whole renders against the+stock diffusers forward, bit for bit."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++from core.inference import diffusion_capture_safe as cs+from core.inference import diffusion_cuda_graph as cg+from core.inference import diffusion_qwenimage21 as q++torch = pytest.importorskip("torch")+qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")++# Sibling import: pytest puts the rootdir on sys.path, not this directory.+_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_qwenimage21 import CASES, _inputs, _model # noqa: E402++_CUDA = torch.cuda.is_available()++[email protected](autouse = True)+def _stock_after(monkeypatch):+ for env in (q.FAST_STEP_ENV, cs.CAPTURE_SAFE_ENV, cg.CUDA_GRAPH_DISABLE_ENV):+ monkeypatch.delenv(env, raising = False)+ monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0") # tiny forwards on a shared card: timing is noise+ q.uninstall()+ yield+ q.uninstall()+++def _call_kwargs(inp, hs, timestep, kv, mode):+ return dict(+ hidden_states = hs,+ timestep = timestep,+ encoder_hidden_states = inp["encoder_hidden_states"],+ encoder_hidden_states_mask = inp["encoder_hidden_states_mask"],+ img_shapes = inp["img_shapes"],+ img_mask = inp["img_mask"],+ attention_kwargs = None,+ kv_cache = kv,+ kv_cache_mode = mode,+ return_dict = False,+ )+++def _render(+ m,+ inp,+ steps = 5,+ call = None,+):+ """The pipeline's loop: step 0 prefills the cache, later steps decode from it."""+ call = call or m+ outs = []+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))+ lat = inp["latents"]+ with torch.inference_mode():+ for i in range(steps):+ hs = lat if inp["cond"] is None else torch.cat([inp["cond"], lat], 1)+ timestep = torch.full((hs.shape[0],), 1.0 - i / steps, device = hs.device)+ out = call(**_call_kwargs(inp, hs, timestep, kv, "extract" if i == 0 else "cached"))[0]+ outs.append(out.clone())+ lat = lat - 0.1 * out[:, -lat.shape[1] :]+ return outs+++def _prefilled(m, inp):+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))+ hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)+ t = torch.full((hs.shape[0],), 0.9, device = hs.device)+ with torch.inference_mode():+ m(**_call_kwargs(inp, hs, t, kv, "extract"))+ return kv, hs++[email protected]("case", CASES)+def test_planned_decode_step_is_bit_identical_to_the_eager_fast_step(case):+ m = _model()+ inp = _inputs(**case)+ assert q.install()+ kv, hs = _prefilled(m, inp)+ t = torch.full((hs.shape[0],), 0.5)+ with torch.inference_mode():+ kwargs = _call_kwargs(inp, hs, t, kv, "cached")+ want = m(**kwargs)[0]+ plan = q.graph_plan(m, (), kwargs)+ assert plan is not None+ call, planned = plan+ got = call(**planned)[0]+ assert torch.equal(want, got)+ # Everything the graph layer has to copy is a tensor or a constant: no opaque object, no float.+ key = cg.graph_key(((), planned))+ assert not cg._uncapturable(key)+ assert not cg._has_float(key)+ assert len(planned["kv"]) == len(m.transformer_blocks)+++def test_what_the_plan_leaves_eager(monkeypatch):+ m = _model()+ inp = _inputs(batch = 2, pad = 2)+ assert q.install()+ kv, hs = _prefilled(m, inp)+ t = torch.full((2,), 0.5)+ with torch.inference_mode():+ cached = _call_kwargs(inp, hs, t, kv, "cached")+ assert q.graph_plan(m, (), cached) is not None+ # The prefill fills the cache from Python; it stays eager.+ assert q.graph_plan(m, (), {**cached, "kv_cache_mode": "extract"}) is None+ assert q.graph_plan(m, (), {**cached, "kv_cache": None, "kv_cache_mode": None}) is None+ assert q.graph_plan(m, (), {**cached, "return_dict": True}) is None+ # A LoRA scale rides in attention_kwargs through the forward's decorator.+ assert q.graph_plan(m, (), {**cached, "attention_kwargs": {"scale": 0.5}}) is None+ assert (+ q.graph_plan(m, (hs,), {k: v for k, v in cached.items() if k != "hidden_states"})+ is None+ )+ assert q.graph_plan(m, (), {**cached, "unknown": 1}) is None+ # An empty cache (no prefill yet) is not plannable, and never raises.+ empty = qmod.QwenImage21KVCache(len(m.transformer_blocks))+ assert q.graph_plan(m, (), {**cached, "kv_cache": empty}) is None+ assert q.graph_plan(m, (), {**cached, "img_mask": "not a tensor"}) is None+ with torch.enable_grad():+ assert q.graph_plan(m, (), cached) is None+ # A prefix K/V too large to keep a second copy of per graph stays eager.+ monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 16)+ with torch.inference_mode():+ assert q.graph_plan(m, (), cached) is None+ monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 1 << 30)+ monkeypatch.setenv(q.FAST_STEP_ENV, "0")+ with torch.inference_mode():+ assert q.graph_plan(m, (), cached) is None+++def test_resolve_follows_the_installed_fast_step(monkeypatch):+ cls = qmod.QwenImage21Transformer2DModel+ forward, why = cs.resolve(cls)+ assert forward is None and "is not installed" in why+ assert q.install()+ forward, why = cs.resolve(cls)+ assert why is None+ assert forward is vars(cls)["forward"]+ assert forward.__unsloth_graph_plan__ is q.graph_plan+ monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")+ forward, why = cs.resolve(cls)+ assert forward is None and cs.CAPTURE_SAFE_ENV in why+ monkeypatch.delenv(cs.CAPTURE_SAFE_ENV)+ q.uninstall()+ assert cs.resolve(cls)[0] is None+++def test_graph_eligible_accepts_qwen_image_21_once_the_fast_step_is_installed(monkeypatch):+ import types++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)++ def eligible():+ return cg.graph_eligible(+ types.SimpleNamespace(device = "cuda", backend = "cuda"),+ family = types.SimpleNamespace(),+ pipe = types.SimpleNamespace(transformer = _model()),+ offload_active = False,+ cache_active = False,+ speed_mode = "default",+ )++ ok, why = eligible()+ assert ok is False and why.startswith(+ "QwenImage21Transformer2DModel forward is not capture-safe"+ )+ assert q.install()+ assert eligible() == (True, "eligible")+++def test_a_failed_capture_falls_back_to_the_callers_own_call():+ """A planned call whose capture fails (here: host tensors) must run the stock call, not the planned kwargs."""+ m = _model()+ inp = _inputs(batch = 2, pad = 3)+ ref = _render(m, inp)+ assert q.install()+ handle = cg.GraphedForward(m).enable()+ try:+ got = _render(m, _inputs(batch = 2, pad = 3))+ finally:+ cg.uninstall_all([handle])+ for a, b in zip(ref, got):+ assert torch.equal(a, b)+ assert handle.poisoned and handle.stats["refused_host_tensor"] == 1+ assert cg._POOL_BOX[0] is None+++# ------------------------------------------------------------------------------------------------- CUDA+++def _cuda_inputs(**case):+ return {k: (v.to("cuda") if torch.is_tensor(v) else v) for k, v in _inputs(**case).items()}+++def _graphed(m, **kw):+ handle = cg.GraphedForward(m, **kw).enable()+ assert handle.capture_safe and handle.plan is q.graph_plan+ return handle++[email protected](not _CUDA, reason = "needs CUDA")[email protected]("case", CASES)+def test_graphed_render_is_bit_identical_to_the_stock_forward(case):+ m = _model().to("cuda")+ inp = _cuda_inputs(**case)+ ref = _render(m, inp)+ assert q.install()+ handle = _graphed(m)+ try:+ got = _render(m, inp)+ again = _render(m, _cuda_inputs(**case))+ finally:+ cg.uninstall_all([handle])+ for a, b, c in zip(ref, got, again):+ assert torch.equal(a, b)+ assert torch.equal(a, c)+ assert not handle.poisoned, handle.capture_error+ # Two renders: two prefills eager, one capture, every decode step replayed (the capturing one included).+ assert handle.stats["planned_eager"] == 2+ assert handle.stats["captures"] == 1+ assert handle.stats["replays"] == 2 * (5 - 1)+ assert not handle.poisoned, handle.capture_error++[email protected](not _CUDA, reason = "needs CUDA")+def test_evicting_the_only_graph_does_not_reuse_its_dead_pool():+ m = _model().to("cuda")+ lengths = (9, 11, 13)+ refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]+ assert q.install()+ handle = _graphed(m, max_graphs = 1)+ try:+ for n, ref in zip(lengths, refs):+ for a, b in zip(ref, _render(m, _cuda_inputs(text = n, seed = n))):+ assert torch.equal(a, b)+ finally:+ cg.uninstall_all([handle])+ assert not handle.poisoned, handle.capture_error+ assert handle.stats["captures"] == 3 and handle.stats["evictions"] == 2++[email protected](not _CUDA, reason = "needs CUDA")+def test_a_new_prompt_length_recaptures_and_the_oldest_graph_is_evicted():+ m = _model().to("cuda")+ lengths = (9, 11, 9, 13)+ refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]+ assert q.install()+ handle = _graphed(m, max_graphs = 2)+ try:+ for n, ref in zip(lengths, refs):+ got = _render(m, _cuda_inputs(text = n, seed = n))+ for a, b in zip(ref, got):+ assert torch.equal(a, b)+ finally:+ cg.uninstall_all([handle])+ # 9 and 11 capture, 9 replays, 13 evicts the least recently replayed (11). After the first capture a new length+ # runs its first decode step for real (that is its warm-up) and records on the next one.+ assert handle.stats["captures"] == 3+ assert handle.stats["shape_warmups"] == 2+ assert handle.stats["evictions"] == 1+ assert handle.stats["cap_skips"] == 0+ assert handle.stats["replays"] == 4 * 4 - 2+ assert not handle.poisoned++[email protected](not _CUDA, reason = "needs CUDA")+def test_a_new_prompt_length_warms_on_tensors_laid_out_like_the_capture(monkeypatch):+ """Compiled blocks guard on strides and storage offsets. The warm-up step of a new length must run on fresh copies+ (as the capture's statics are), else it compiles a variant for the caller's views and the capture that follows+ compiles inside the recording, which fails it (seen on the real model under block offload)."""+ m = _model().to("cuda")+ assert q.install()+ handle = _graphed(m)+ stock = q._graph_step+ seen: list = []++ def spy(self, **kw):+ if handle.stats["shape_warmups"] and not seen:+ leaves: list = []+ cg._flatten(kw, leaves)+ seen.append(leaves)+ return stock(self, **kw)++ monkeypatch.setattr(q, "_graph_step", spy)+ try:+ for n in (9, 11):+ _render(m, _cuda_inputs(text = n, seed = n))+ finally:+ cg.uninstall_all([handle])+ assert handle.stats["shape_warmups"] == 1 and handle.stats["captures"] == 2+ assert seen and all(t.storage_offset() == 0 for t in seen[0])++[email protected](not _CUDA, reason = "needs CUDA")+def test_alternating_caches_of_one_shape_replay_their_own_prefix():+ """True CFG: cond and uncond share a graph key but not a KV cache; the sticky copy must follow identity."""+ m = _model().to("cuda")+ cond, uncond = _cuda_inputs(seed = 1), _cuda_inputs(seed = 2)+ assert q.install()++ def two_stream_render():+ out = []+ kvs = [qmod.QwenImage21KVCache(len(m.transformer_blocks)) for _ in range(2)]+ lat = cond["latents"]+ with torch.inference_mode():+ for i in range(4):+ t = torch.full((1,), 1.0 - i / 4, device = "cuda")+ mode = "extract" if i == 0 else "cached"+ a = m(**_call_kwargs(cond, lat, t, kvs[0], mode))[0][:, -lat.shape[1] :]+ b = m(**_call_kwargs(uncond, lat, t, kvs[1], mode))[0][:, -lat.shape[1] :]+ out += [a.clone(), b.clone()]+ lat = lat - 0.1 * (a + b)+ return out++ ref = two_stream_render()+ handle = _graphed(m)+ try:+ got = two_stream_render()+ finally:+ cg.uninstall_all([handle])+ for a, b in zip(ref, got):+ assert torch.equal(a, b)+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 6++[email protected](not _CUDA, reason = "needs CUDA")+def test_graph_statics_match_the_eager_inputs_strides_and_inference_mode():+ """A static that is not an inference tensor where eager passes one selects another compiled variant."""+ m = _model().to("cuda")+ inp = _cuda_inputs()+ assert q.install()+ handle = _graphed(m)+ try:+ _render(m, inp)+ (entry,) = handle.cache.values()+ kv, hs = _prefilled(m, inp)+ with torch.inference_mode():+ # Decode-step latents come out of the previous step, made in inference mode like everything else here.+ hs = hs.clone()+ t = torch.full((1,), 0.5, device = "cuda")+ _, planned = q.graph_plan(m, (), _call_kwargs(inp, hs, t, kv, "cached"))+ live: list = []+ cg._flatten(((), planned), live)+ assert len(live) == len(entry.static)+ for src, dst in zip(live, entry.static):+ assert dst.stride() == src.stride()+ assert dst.is_inference() == src.is_inference()+ finally:+ cg.uninstall_all([handle])++[email protected](not _CUDA, reason = "needs CUDA")+def test_planned_decode_steps_do_not_sync_the_host():+ m = _model().to("cuda")+ inp = _cuda_inputs(batch = 2, pad = 2)+ assert q.install()+ handle = _graphed(m)+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))+ try:+ with torch.inference_mode():+ for i in range(5):+ t = torch.full((2,), 1.0 - i / 5, device = "cuda")+ # Step 1 captures (its warm-up and capture synchronize by design); later replays must not.+ if i >= 2:+ torch.cuda.set_sync_debug_mode("error")+ try:+ m(**_call_kwargs(inp, inp["latents"], t, kv, "extract" if i == 0 else "cached"))+ finally:+ torch.cuda.set_sync_debug_mode("default")+ finally:+ cg.uninstall_all([handle])+ assert handle.stats["replays"] == 4+++# ------------------------------------------------------------------------------------------- offloaded+++def _block_streamed(m):+ import types++ from core.inference import diffusion_memory as dm++ pipe = types.SimpleNamespace(transformer = m, components = {"transformer": m})+ assert dm._apply_group_offload(pipe, "cuda", None)+ return pipe++[email protected](not _CUDA, reason = "needs CUDA")[email protected]("case", [CASES[1], CASES[3]])+def test_block_streamed_decode_steps_replay_bit_identically(case):+ """Under block offload the planned step enters through the offload hooks (which onload the top-level weights)+ and the copy-stream onloads are recorded with it: renders match the eager streamed and the resident forward."""+ pytest.importorskip("diffusers.hooks")+ from core.inference import diffusion_offload_prefetch as op++ resident = _render(_model().to("cuda"), _cuda_inputs(**case))+ assert q.install() # before the hooks, as Studio's load does+ m = _model()+ pipe = _block_streamed(m)+ eager = _render(m, _cuda_inputs(**case))+ handles, reason = cg.arm_after_placement(pipe)+ assert len(handles) == 1 and handles[0].placement.mode == "group", reason+ handle = handles[0]+ assert handle.plan is q.graph_plan and handle.placed is q.placed_step+ try:+ got = _render(m, _cuda_inputs(**case))+ again = _render(m, _cuda_inputs(**case))+ kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))+ inp = _cuda_inputs(**case)+ hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)+ with torch.inference_mode():+ for i in range(3):+ t = torch.full((hs.shape[0],), 1.0 - i / 5, device = "cuda")+ if i:+ torch.cuda.set_sync_debug_mode(+ "error"+ ) # a replay of the streamed step never waits on the host+ try:+ m(**_call_kwargs(inp, hs, t, kv, "extract" if i == 0 else "cached"))+ finally:+ torch.cuda.set_sync_debug_mode("default")+ finally:+ cg.uninstall_all(handles)+ assert not handle.poisoned, handle.capture_error+ for r, e, a, b in zip(resident, eager, got, again):+ assert torch.equal(e, a) and torch.equal(e, b)+ assert torch.equal(r, a)+ assert handle.stats["captures"] == 1+ assert handle.stats["replays"] == 2 * (5 - 1) + 2+ assert next(m.transformer_blocks[-1].parameters()).device.type == "cpu"+ assert op.module_prefetcher(m).stats["missed"] == 0+++def test_offload_placement_names_a_forward_that_cannot_record_under_the_hooks(monkeypatch):+ m = _model()++ def planned(*_a, **_k):+ return None++ planned.__unsloth_graph_plan__ = q.graph_plan+ monkeypatch.setattr(cs, "resolve", lambda cls: (planned, None))+ assert "no entry through the offload hooks" in cg._forward_refusal(m, "group")+ planned.__unsloth_graph_placed__ = q.placed_step+ assert cg._forward_refusal(m, "group") is None++ def rewrite(*_a, **_k):+ return None++ monkeypatch.setattr(cs, "resolve", lambda cls: (rewrite, None))+ assert "capture-safe forward" in cg._forward_refusal(m, "group")+ assert (+ cg._forward_refusal(m, "model") is None+ ) # model offload calls the slot with the weights already onloaded+ monkeypatch.setattr(cs, "resolve", lambda cls: (None, "X forward is not capture-safe (why)"))+ assert cg._forward_refusal(m, "model") == "X forward is not capture-safe (why)"diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 7cafb332bb0..52510253521 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -9599,6 +9599,41 @@ def test_cuda_graph_is_a_per_family_opt_in():opted_in = [f.name for f in _FAMILIES if f.supports_cuda_graph]assert opted_in == [], f"{opted_in} opts into CUDA graphs with no measurement on record"+ # Offloaded denoisers are a different trade: the replay records the onloads and the step makes no host wait.+ # Each opt-in carries a capped-card measurement in its family comment.+ offloaded = sorted(f.name for f in _FAMILIES if f.offload_cuda_graph)+ assert offloaded == sorted(OFFLOAD_GRAPH_FAMILIES), offloaded+ assert (+ h3.offload_cuda_graph is False+ ) # streamed H3 onloads with host waits; a graph cannot record them+ assert (+ "GPU-bound" in h3.cuda_graph_decline and "wait on the host" in h3.cuda_graph_decline+ ) # the measured reasons+++OFFLOAD_GRAPH_FAMILIES = ("wan2.2-ti2v-5b", "hunyuanvideo-1.5")+++def test_h3_modular_load_arms_a_deferred_graph_after_placement():+ """A forced H3 graph is deferred to placement by apply_speed_optims; the modular load must arm it (or record why+ it stays eager). Without the call the status read "off: armed after placement" for the whole load."""+ import ast+ import textwrap++ src = textwrap.dedent(inspect.getsource(video_module.VideoBackend._load_h3_modular_pipeline))+ calls = [+ node+ for node in ast.walk(ast.parse(src))+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "arm_graphs_after_placement"+ ]+ assert calls and isinstance(calls[0].args[0], ast.Name) and calls[0].args[0].id == "speed_view"+ speed = src.index("applied = apply_speed_optims(")+ arm = src.index("arm_graphs_after_placement(")+ status = src.index('"cuda_graph": (')+ assert (+ speed < arm < status+ ) # after the deferring speed pass, before the resolved record reads the reasondef test_every_rebuilt_speed_target_carries_the_backend():