Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Capture-safe forwards for denoisers whose stock forward cannot be recorded into a CUDA graph.
``diffusion_cuda_graph.GraphedForward`` records ONE whole denoiser forward. A forward that makes the
host wait on the device cannot be recorded: CUDA refuses the call with
``cudaErrorStreamCaptureUnsupported`` and ``capture_end`` then reports
``cudaErrorStreamCaptureInvalidated``, so the graph layer poisons the wrapper and the load runs eager.
HunyuanImage-2.1 is such a forward. ``HunyuanImageTransformer2DModel.forward`` merges its two text
streams (Qwen2.5-VL mllm and ByT5 glyph) per batch element with boolean-mask indexing::
new_encoder_hidden_states.append(
torch.cat(
[
text_2[text_mask_2], # valid byt5
text[text_mask], # valid mllm
text_2[~text_mask_2], # invalid byt5
text[~text_mask], # invalid mllm
],
dim=0,
)
)
``tensor[bool_mask]`` runs ``nonzero`` and copies the count to the host to size its output: a
device-to-host sync on every one of the eight index calls, every forward. Identical in diffusers
0.36.0 through 0.41.
The merge is a PERMUTATION, not a data-dependent shape: the four pieces always add back up to the
two streams' full lengths. Concatenating ``[byt5 ; mllm]`` and stable-sorting on "is padding" puts
the valid byt5 tokens first, then the valid mllm tokens, then the padded byt5, then the padded mllm,
each in its original order, which is exactly the order above. ``argsort`` and ``gather`` stay on the
device, so the rewritten forward records, and because it only moves rows it is bit-identical to the
stock one. Only the graph wrapper uses it; an ungraphed load keeps the stock forward.
The rewrite takes the INSTALLED forward's own source and swaps just that block, so everything else
(LoRA scaling, which moved from inline code in 0.36 to a decorator in 0.37, checkpointing, the
unpatchify) stays whatever the installed diffusers ships. The block is matched line for line; if it
changed, nothing is rewritten and the graph layer declines the load with the reason.
HunyuanVideo-1.5 has the same merge over three streams ([image ; byt5 ; mllm], padded byt5/mllm rows
zeroed) plus ``is_t2v = torch.all(image_embeds == 0)`` read by ``if`` on the host. Both blocks are
swapped: the branch becomes ``torch.where`` on the device bool (keeping ``* 0.0`` exactly), the merge
the same stable-sort permutation with ``torch.where`` zeroing. Identical in diffusers 0.36.0 through
0.40.0.
Kill switch: ``UNSLOTH_DIFFUSION_CAPTURE_SAFE=0`` (the graph is then declined for these classes).
"""
from __future__ import annotations
import inspect
import linecache
import os
import sys
import textwrap
import threading
from typing import Any, Callable, Optional
CAPTURE_SAFE_ENV = "UNSLOTH_DIFFUSION_CAPTURE_SAFE"
_HUNYUANIMAGE_CLS = "HunyuanImageTransformer2DModel"
# The stock merge block, stripped line by line (blank lines dropped). Order of the four pieces is
# what the permutation reproduces, so the match is exact rather than a needle search.
_HUNYUANIMAGE_MERGE_BLOCK: tuple[str, ...] = (
"# reorder and combine text tokens: combine valid tokens first, then padding",
"new_encoder_hidden_states = []",
"new_encoder_attention_mask = []",
"for text, text_mask, text_2, text_mask_2 in zip(",
"encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2",
"):",
"# Concatenate: [valid_mllm, valid_byt5, invalid_mllm, invalid_byt5]",
"new_encoder_hidden_states.append(",
"torch.cat(",
"[",
"text_2[text_mask_2], # valid byt5",
"text[text_mask], # valid mllm",
"text_2[~text_mask_2], # invalid byt5",
"text[~text_mask], # invalid mllm",
"],",
"dim=0,",
")",
")",
"# Apply same reordering to attention masks",
"new_encoder_attention_mask.append(",
"torch.cat(",
"[",
"text_mask_2[text_mask_2],",
"text_mask[text_mask],",
"text_mask_2[~text_mask_2],",
"text_mask[~text_mask],",
"],",
"dim=0,",
")",
")",
"encoder_hidden_states = torch.stack(new_encoder_hidden_states)",
"encoder_attention_mask = torch.stack(new_encoder_attention_mask)",
)
_MERGE_HELPER_NAME = "_unsloth_merge_text_streams"
_MERGE_CALL = (
f"encoder_hidden_states, encoder_attention_mask = {_MERGE_HELPER_NAME}("
"encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2)"
)
def capture_safe_disabled() -> bool:
return (os.environ.get(CAPTURE_SAFE_ENV) or "").strip().lower() in ("0", "off", "false", "no")
def merge_text_streams(
encoder_hidden_states: Any,
encoder_attention_mask: Any,
encoder_hidden_states_2: Any,
encoder_attention_mask_2: Any,
) -> tuple[Any, Any]:
"""HunyuanImage text-stream merge without a host sync, bit-identical to the stock per-row cat."""
import torch
states = torch.cat([encoder_hidden_states_2, encoder_hidden_states], dim = 1)
mask = torch.cat([encoder_attention_mask_2, encoder_attention_mask], dim = 1)
# Stable sort keeps byt5 ahead of mllm and each stream's order.
order = torch.argsort((~mask).to(torch.uint8), dim = 1, stable = True)
states = torch.gather(states, 1, order.unsqueeze(-1).expand(-1, -1, states.shape[-1]))
mask = torch.gather(mask, 1, order)
return states, mask
_HUNYUANVIDEO15_CLS = "HunyuanVideo15Transformer3DModel"
# HunyuanVideo-1.5 (diffusers 0.36.0 to 0.40.0): is_t2v is a device bool an if reads on the host.
_HV15_T2V_BLOCK: tuple[str, ...] = (
"is_t2v = torch.all(image_embeds == 0)",
"if is_t2v:",
"encoder_hidden_states_3 = encoder_hidden_states_3 * 0.0",
"encoder_attention_mask_3 = torch.zeros(",
"(batch_size, encoder_hidden_states_3.shape[1]),",
"dtype=encoder_attention_mask.dtype,",
"device=encoder_attention_mask.device,",
")",
"else:",
"encoder_attention_mask_3 = torch.ones(",
"(batch_size, encoder_hidden_states_3.shape[1]),",
"dtype=encoder_attention_mask.dtype,",
"device=encoder_attention_mask.device,",
")",
)
_HV15_MERGE_BLOCK: tuple[str, ...] = (
"new_encoder_hidden_states = []",
"new_encoder_attention_mask = []",
"for text, text_mask, text_2, text_mask_2, image, image_mask in zip(",
"encoder_hidden_states,",
"encoder_attention_mask,",
"encoder_hidden_states_2,",
"encoder_attention_mask_2,",
"encoder_hidden_states_3,",
"encoder_attention_mask_3,",
"):",
"# Concatenate: [valid_image, valid_byt5, valid_mllm, invalid_image, invalid_byt5, invalid_mllm]",
"new_encoder_hidden_states.append(",
"torch.cat(",
"[",
"image[image_mask], # valid image",
"text_2[text_mask_2], # valid byt5",
"text[text_mask], # valid mllm",
"image[~image_mask], # invalid image",
"torch.zeros_like(text_2[~text_mask_2]), # invalid byt5 (zeroed)",
"torch.zeros_like(text[~text_mask]), # invalid mllm (zeroed)",
"],",
"dim=0,",
")",
")",
"# Apply same reordering to attention masks",
"new_encoder_attention_mask.append(",
"torch.cat(",
"[",
"image_mask[image_mask],",
"text_mask_2[text_mask_2],",
"text_mask[text_mask],",
"image_mask[~image_mask],",
"text_mask_2[~text_mask_2],",
"text_mask[~text_mask],",
"],",
"dim=0,",
")",
")",
"encoder_hidden_states = torch.stack(new_encoder_hidden_states)",
"encoder_attention_mask = torch.stack(new_encoder_attention_mask)",
)
_HV15_T2V_HELPER_NAME = "_unsloth_hv15_image_stream"
_HV15_T2V_CALL = (
f"encoder_hidden_states_3, encoder_attention_mask_3 = {_HV15_T2V_HELPER_NAME}("
"image_embeds, encoder_hidden_states_3, encoder_attention_mask, batch_size)"
)
_HV15_MERGE_HELPER_NAME = "_unsloth_hv15_merge_streams"
_HV15_MERGE_CALL = (
f"encoder_hidden_states, encoder_attention_mask = {_HV15_MERGE_HELPER_NAME}("
"encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2, "
"encoder_hidden_states_3, encoder_attention_mask_3)"
)
def hv15_image_stream(
image_embeds: Any, encoder_hidden_states_3: Any, encoder_attention_mask: Any, batch_size: int
) -> tuple[Any, Any]:
"""HunyuanVideo-1.5's is_t2v branch as torch.where arithmetic, so the host never reads the flag."""
import torch
is_t2v = torch.all(image_embeds == 0)
states = torch.where(is_t2v, encoder_hidden_states_3 * 0.0, encoder_hidden_states_3)
mask = (
torch.logical_not(is_t2v)
.to(dtype = encoder_attention_mask.dtype, device = encoder_attention_mask.device)
.expand(batch_size, encoder_hidden_states_3.shape[1])
.contiguous()
)
return states, mask
def hv15_merge_streams(
encoder_hidden_states: Any,
encoder_attention_mask: Any,
encoder_hidden_states_2: Any,
encoder_attention_mask_2: Any,
encoder_hidden_states_3: Any,
encoder_attention_mask_3: Any,
) -> tuple[Any, Any]:
"""HunyuanVideo-1.5 three-stream merge without a host sync; matches the stock stable sort on padding."""
import torch
states = torch.cat(
[encoder_hidden_states_3, encoder_hidden_states_2, encoder_hidden_states], dim = 1
)
mask = torch.cat(
[encoder_attention_mask_3, encoder_attention_mask_2, encoder_attention_mask], dim = 1
)
padded = ~mask
zero_rows = torch.cat(
[
torch.zeros_like(encoder_attention_mask_3),
~encoder_attention_mask_2,
~encoder_attention_mask,
],
dim = 1,
)
states = torch.where(
zero_rows.unsqueeze(-1), torch.zeros((), dtype = states.dtype, device = states.device), states
)
order = torch.argsort(padded.to(torch.uint8), dim = 1, stable = True)
states = torch.gather(states, 1, order.unsqueeze(-1).expand(-1, -1, states.shape[-1]))
mask = torch.gather(mask, 1, order)
return states, mask
def _find_block(lines: list[str], block: tuple[str, ...]) -> Optional[tuple[int, int]]:
"""``(start, end)`` line indices (end exclusive) of ``block`` in ``lines``, blank lines ignored.
None unless it occurs exactly once."""
starts = [i for i, line in enumerate(lines) if line.strip() == block[0]]
found: list[tuple[int, int]] = []
for start in starts:
want = 0
i = start
while i < len(lines) and want < len(block):
text = lines[i].strip()
if text:
if text != block[want]:
break
want += 1
i += 1
if want == len(block):
found.append((start, i))
return found[0] if len(found) == 1 else None
def _signature_key(fn: Callable) -> Optional[tuple]:
try:
params = inspect.signature(fn).parameters.values()
except (TypeError, ValueError):
return None
return tuple((p.name, p.kind, repr(p.default)) for p in params)
def _rebuild_forward(
forward: Callable,
swaps: tuple[tuple[tuple[str, ...], str, str], ...],
helpers: dict[str, Callable],
) -> Callable:
"""Rebuilds forward from source; each block must match exactly once, else RuntimeError names it."""
target = inspect.unwrap(forward)
module = sys.modules.get(getattr(target, "__module__", "") or "")
if module is None:
raise RuntimeError("its defining module is not imported")
try:
source = textwrap.dedent(inspect.getsource(target))
except (OSError, TypeError) as exc:
raise RuntimeError(f"its source is unreadable ({exc})") from None
lines = source.splitlines()
spans = []
for block, replacement, what in swaps:
span = _find_block(lines, block)
if span is None:
raise RuntimeError(f"the {what} block changed in this diffusers version")
spans.append((span, replacement))
spans.sort(key = lambda item: item[0][0])
for (prev, _), (nxt, _) in zip(spans, spans[1:]):
if nxt[0] < prev[1]:
raise RuntimeError("the rewritten blocks overlap")
# Splice from the bottom so earlier spans keep their line indices.
for (start, end), replacement in reversed(spans):
indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]
lines = lines[:start] + [indent + replacement] + lines[end:]
# A source-level decorator (``@apply_lora_scale`` from 0.37) re-applies when the def executes,
# so the rebuilt function carries the installed version's LoRA handling unchanged.
new_source = "\n".join(lines) + "\n"
filename = f"<unsloth capture-safe {getattr(target, '__qualname__', 'forward')}>"
namespace = dict(vars(module))
namespace.update(helpers)
# dont_inherit: this module's __future__ annotations must not stringify the rebuilt forward.
code = compile(new_source, filename, "exec", dont_inherit = True)
exec(code, namespace) # noqa: S102 - the installed diffusers' own source with known blocks replaced
rebuilt = namespace.get(target.__name__)
if not callable(rebuilt):
raise RuntimeError("the rebuilt source did not define the forward")
# Tracebacks and inspect.getsource resolve through linecache.
linecache.cache[filename] = (len(new_source), None, new_source.splitlines(True), filename)
if _signature_key(rebuilt) != _signature_key(forward):
raise RuntimeError("the rebuilt forward's signature differs from the stock one")
rebuilt.__unsloth_capture_safe__ = True
return rebuilt
def _rewrite_hunyuanimage(forward: Callable) -> Callable:
"""HunyuanImage-2.1: the boolean-mask text merge becomes ``merge_text_streams``."""
return _rebuild_forward(
forward,
((_HUNYUANIMAGE_MERGE_BLOCK, _MERGE_CALL, "boolean-mask text merge"),),
{_MERGE_HELPER_NAME: merge_text_streams},
)
def _rewrite_hunyuanvideo15(forward: Callable) -> Callable:
"""HunyuanVideo-1.5: the ``if is_t2v`` branch and the three-stream merge, both made device-only."""
return _rebuild_forward(
forward,
(
(_HV15_T2V_BLOCK, _HV15_T2V_CALL, "t2v image-stream branch"),
(_HV15_MERGE_BLOCK, _HV15_MERGE_CALL, "boolean-mask three-stream merge"),
),
{
_HV15_T2V_HELPER_NAME: hv15_image_stream,
_HV15_MERGE_HELPER_NAME: hv15_merge_streams,
},
)
# class name -> (why the stock forward cannot be captured, rewriter)
_REWRITES: dict[str, tuple[str, Callable[[Callable], Callable]]] = {
_HUNYUANIMAGE_CLS: (
"boolean-mask indexing in its text-stream merge syncs the host",
_rewrite_hunyuanimage,
),
_HUNYUANVIDEO15_CLS: (
"its t2v branch reads a device bool on the host and boolean-mask indexing in its "
"three-stream merge syncs the host",
_rewrite_hunyuanvideo15,
),
}
# One short read: a pick is blocked on this in the UI, so a slow Hub must not stall the picker, and a timeout is just another fail-open
_UNCAPTURABLE: dict[str, str] = {
"QwenImage21Transformer2DModel": "its pipeline passes the prefix KV cache as a Python object "
"on every step; the Unsloth fast step that replays decode steps from tensors is not installed",
}
_CACHE: dict = {}
_CACHE_LOCK = threading.Lock()
def _defining_class(cls: type) -> Optional[type]:
"""The class in ``cls``'s MRO that defines the ``forward`` instances of ``cls`` run."""
for klass in getattr(cls, "__mro__", ()):
if "forward" in vars(klass):
return klass
return None
def resolve(cls: type) -> tuple[Optional[Callable], Optional[str]]:
"""(forward, None): capture-safe rewrite. (None, reason): stock cannot capture. Never raises."""
owner = _defining_class(cls)
planned = vars(owner).get("forward") if owner is not None else None
if getattr(planned, "__unsloth_graph_plan__", None) is not None:
# A Studio forward that plans its own capture-safe calls (Qwen-Image-2.1's fast step).
if capture_safe_disabled():
return (
None,
f"{owner.__name__} forward is not capture-safe ({_UNCAPTURABLE.get(owner.__name__, 'planned')}); "
f"rewrite disabled by {CAPTURE_SAFE_ENV}",
)
return planned, None
if owner is not None and owner.__name__ in _UNCAPTURABLE and owner.__name__ not in _REWRITES:
return (
None,
f"{owner.__name__} forward is not capture-safe ({_UNCAPTURABLE[owner.__name__]})",
)
if owner is None or owner.__name__ not in _REWRITES:
return None, None
why, rewrite = _REWRITES[owner.__name__]
if capture_safe_disabled():
return (
None,
f"{owner.__name__} forward is not capture-safe ({why}); rewrite disabled by {CAPTURE_SAFE_ENV}",
)
forward = vars(owner)["forward"]
key = (owner, forward)
with _CACHE_LOCK:
cached = _CACHE.get(key)
if cached is not None:
return cached
try:
result: tuple[Optional[Callable], Optional[str]] = (rewrite(forward), None)
except Exception as exc: # noqa: BLE001 - an unrewritable forward declines the graph, never fails a load
result = (
None,
f"{owner.__name__} forward is not capture-safe ({why}) and was not rewritten: {exc}",
)
with _CACHE_LOCK:
_CACHE[key] = result
return result