Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make the HunyuanImage-2.1 denoiser step capturable as a CUDA graph (#11753)

Daniel Hanb6215797ab8a
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.json
index e521c5ee8e2..b3c5b387808 100644
--- a/scripts/exec_literals_baseline.json
+++ b/scripts/exec_literals_baseline.json
@@ -9,6 +9,20 @@
"unsloth-cli.py"
],
"entries": [
+ {
+ "file": "studio/backend/core/inference/diffusion_capture_safe.py",
+ "sink": "compile",
+ "digest": "05c1d218ea0c8e6d",
+ "count": 1,
+ "reason": "Rebuilds the installed diffusers HunyuanImage forward from its own inspect.getsource with one fixed block swapped; the source is the imported library's code, not user or network input."
+ },
+ {
+ "file": "studio/backend/core/inference/diffusion_capture_safe.py",
+ "sink": "exec",
+ "digest": "4b6b3a49b588c0a6",
+ "count": 1,
+ "reason": "Rebuilds the installed diffusers HunyuanImage forward from its own inspect.getsource with one fixed block swapped; the source is the imported library's code, not user or network input."
+ },
{
"file": "unsloth/models/_utils.py",
"sink": "eval",
diff --git a/studio/backend/core/inference/diffusion_capture_safe.py b/studio/backend/core/inference/diffusion_capture_safe.py
new file mode 100644
index 00000000000..fb3b57cdf94
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_capture_safe.py
@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Capture-safe forwards for denoisers whose stock forward cannot be recorded into a CUDA graph.
+
+``diffusion_cuda_graph.GraphedForward`` records ONE whole denoiser forward. A forward that makes the
+host wait on the device cannot be recorded: CUDA refuses the call with
+``cudaErrorStreamCaptureUnsupported`` and ``capture_end`` then reports
+``cudaErrorStreamCaptureInvalidated``, so the graph layer poisons the wrapper and the load runs eager.
+
+HunyuanImage-2.1 is such a forward. ``HunyuanImageTransformer2DModel.forward`` merges its two text
+streams (Qwen2.5-VL mllm and ByT5 glyph) per batch element with boolean-mask indexing::
+
+ new_encoder_hidden_states.append(
+ torch.cat(
+ [
+ text_2[text_mask_2], # valid byt5
+ text[text_mask], # valid mllm
+ text_2[~text_mask_2], # invalid byt5
+ text[~text_mask], # invalid mllm
+ ],
+ dim=0,
+ )
+ )
+
+``tensor[bool_mask]`` runs ``nonzero`` and copies the count to the host to size its output: a
+device-to-host sync on every one of the eight index calls, every forward. Identical in diffusers
+0.36.0 through 0.41.
+
+The merge is a PERMUTATION, not a data-dependent shape: the four pieces always add back up to the
+two streams' full lengths. Concatenating ``[byt5 ; mllm]`` and stable-sorting on "is padding" puts
+the valid byt5 tokens first, then the valid mllm tokens, then the padded byt5, then the padded mllm,
+each in its original order, which is exactly the order above. ``argsort`` and ``gather`` stay on the
+device, so the rewritten forward records, and because it only moves rows it is bit-identical to the
+stock one. Only the graph wrapper uses it; an ungraphed load keeps the stock forward.
+
+The rewrite takes the INSTALLED forward's own source and swaps just that block, so everything else
+(LoRA scaling, which moved from inline code in 0.36 to a decorator in 0.37, checkpointing, the
+unpatchify) stays whatever the installed diffusers ships. The block is matched line for line; if it
+changed, nothing is rewritten and the graph layer declines the load with the reason.
+
+Kill switch: ``UNSLOTH_DIFFUSION_CAPTURE_SAFE=0`` (the graph is then declined for these classes).
+"""
+
+from __future__ import annotations
+
+import inspect
+import linecache
+import os
+import sys
+import textwrap
+import threading
+from typing import Any, Callable, Optional
+
+CAPTURE_SAFE_ENV = "UNSLOTH_DIFFUSION_CAPTURE_SAFE"
+
+_HUNYUANIMAGE_CLS = "HunyuanImageTransformer2DModel"
+
+# The stock merge block, stripped line by line (blank lines dropped). Order of the four pieces is
+# what the permutation reproduces, so the match is exact rather than a needle search.
+_HUNYUANIMAGE_MERGE_BLOCK: tuple[str, ...] = (
+ "# reorder and combine text tokens: combine valid tokens first, then padding",
+ "new_encoder_hidden_states = []",
+ "new_encoder_attention_mask = []",
+ "for text, text_mask, text_2, text_mask_2 in zip(",
+ "encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2",
+ "):",
+ "# Concatenate: [valid_mllm, valid_byt5, invalid_mllm, invalid_byt5]",
+ "new_encoder_hidden_states.append(",
+ "torch.cat(",
+ "[",
+ "text_2[text_mask_2], # valid byt5",
+ "text[text_mask], # valid mllm",
+ "text_2[~text_mask_2], # invalid byt5",
+ "text[~text_mask], # invalid mllm",
+ "],",
+ "dim=0,",
+ ")",
+ ")",
+ "# Apply same reordering to attention masks",
+ "new_encoder_attention_mask.append(",
+ "torch.cat(",
+ "[",
+ "text_mask_2[text_mask_2],",
+ "text_mask[text_mask],",
+ "text_mask_2[~text_mask_2],",
+ "text_mask[~text_mask],",
+ "],",
+ "dim=0,",
+ ")",
+ ")",
+ "encoder_hidden_states = torch.stack(new_encoder_hidden_states)",
+ "encoder_attention_mask = torch.stack(new_encoder_attention_mask)",
+)
+
+_MERGE_HELPER_NAME = "_unsloth_merge_text_streams"
+_MERGE_CALL = (
+ f"encoder_hidden_states, encoder_attention_mask = {_MERGE_HELPER_NAME}("
+ "encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2)"
+)
+
+
+def capture_safe_disabled() -> bool:
+ return (os.environ.get(CAPTURE_SAFE_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def merge_text_streams(
+ encoder_hidden_states: Any,
+ encoder_attention_mask: Any,
+ encoder_hidden_states_2: Any,
+ encoder_attention_mask_2: Any,
+) -> tuple[Any, Any]:
+ """HunyuanImage's text-stream merge without a host sync; bit-identical to the stock loop.
+
+ Takes the embedded mllm stream ``[B, S1, D]`` with its bool mask ``[B, S1]`` and the embedded
+ byt5 stream ``[B, S2, D]`` with its bool mask ``[B, S2]``; returns ``[B, S2 + S1, D]`` ordered
+ [valid byt5, valid mllm, padded byt5, padded mllm] and the matching mask (all True, then all
+ False), per batch element. ``torch.cat`` promotes dtypes exactly as the stock per-row cat does.
+ """
+ import torch
+
+ states = torch.cat([encoder_hidden_states_2, encoder_hidden_states], dim = 1)
+ mask = torch.cat([encoder_attention_mask_2, encoder_attention_mask], dim = 1)
+ # Key 0 for a valid token, 1 for padding; a STABLE sort keeps byt5 ahead of mllm and each
+ # stream's own order within both groups.
+ order = torch.argsort((~mask).to(torch.uint8), dim = 1, stable = True)
+ states = torch.gather(states, 1, order.unsqueeze(-1).expand(-1, -1, states.shape[-1]))
+ mask = torch.gather(mask, 1, order)
+ return states, mask
+
+
+def _find_block(lines: list[str], block: tuple[str, ...]) -> Optional[tuple[int, int]]:
+ """``(start, end)`` line indices (end exclusive) of ``block`` in ``lines``, blank lines ignored.
+
+ None unless it occurs exactly once."""
+ starts = [i for i, line in enumerate(lines) if line.strip() == block[0]]
+ found: list[tuple[int, int]] = []
+ for start in starts:
+ want = 0
+ i = start
+ while i < len(lines) and want < len(block):
+ text = lines[i].strip()
+ if text:
+ if text != block[want]:
+ break
+ want += 1
+ i += 1
+ if want == len(block):
+ found.append((start, i))
+ return found[0] if len(found) == 1 else None
+
+
+def _signature_key(fn: Callable) -> Optional[tuple]:
+ try:
+ params = inspect.signature(fn).parameters.values()
+ except (TypeError, ValueError):
+ return None
+ return tuple((p.name, p.kind, repr(p.default)) for p in params)
+
+
+def _rewrite_hunyuanimage(forward: Callable) -> Callable:
+ """Rebuild ``forward`` from its own source with the merge block swapped for the helper call.
+
+ Raises ``RuntimeError`` naming what did not match, so the reason reaches the status payload."""
+ target = inspect.unwrap(forward)
+ module = sys.modules.get(getattr(target, "__module__", "") or "")
+ if module is None:
+ raise RuntimeError("its defining module is not imported")
+ try:
+ source = textwrap.dedent(inspect.getsource(target))
+ except (OSError, TypeError) as exc:
+ raise RuntimeError(f"its source is unreadable ({exc})") from None
+ lines = source.splitlines()
+ span = _find_block(lines, _HUNYUANIMAGE_MERGE_BLOCK)
+ if span is None:
+ raise RuntimeError("the boolean-mask text merge block changed in this diffusers version")
+ start, end = span
+ indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())]
+ new_lines = lines[:start] + [indent + _MERGE_CALL] + lines[end:]
+ # A source-level decorator (``@apply_lora_scale`` from 0.37) re-applies when the def executes,
+ # so the rebuilt function carries the installed version's LoRA handling unchanged.
+ new_source = "\n".join(new_lines) + "\n"
+ filename = f"<unsloth capture-safe {getattr(target, '__qualname__', 'forward')}>"
+ namespace = dict(vars(module))
+ namespace[_MERGE_HELPER_NAME] = merge_text_streams
+ code = compile(new_source, filename, "exec")
+ exec(code, namespace) # noqa: S102 - the installed diffusers' own source with one block replaced
+ rebuilt = namespace.get(target.__name__)
+ if not callable(rebuilt):
+ raise RuntimeError("the rebuilt source did not define the forward")
+ # Tracebacks and inspect.getsource resolve through linecache.
+ linecache.cache[filename] = (len(new_source), None, new_source.splitlines(True), filename)
+ if _signature_key(rebuilt) != _signature_key(forward):
+ raise RuntimeError("the rebuilt forward's signature differs from the stock one")
+ rebuilt.__unsloth_capture_safe__ = True
+ return rebuilt
+
+
+# class name -> (why the stock forward cannot be captured, rewriter)
+_REWRITES: dict[str, tuple[str, Callable[[Callable], Callable]]] = {
+ _HUNYUANIMAGE_CLS: (
+ "boolean-mask indexing in its text-stream merge syncs the host",
+ _rewrite_hunyuanimage,
+ ),
+}
+
+_CACHE: dict = {}
+_CACHE_LOCK = threading.Lock()
+
+
+def _defining_class(cls: type) -> Optional[type]:
+ """The class in ``cls``'s MRO that defines the ``forward`` instances of ``cls`` run."""
+ for klass in getattr(cls, "__mro__", ()):
+ if "forward" in vars(klass):
+ return klass
+ return None
+
+
+def resolve(cls: type) -> tuple[Optional[Callable], Optional[str]]:
+ """``(forward, None)``: an unbound capture-safe replacement for ``cls.forward``.
+ ``(None, None)``: nothing known to be unsafe, capture the stock forward.
+ ``(None, reason)``: the stock forward is known not to capture and no rewrite applied.
+
+ Never raises. Cheap for an unknown class: a name lookup, no torch import."""
+ owner = _defining_class(cls)
+ if owner is None or owner.__name__ not in _REWRITES:
+ return None, None
+ why, rewrite = _REWRITES[owner.__name__]
+ if capture_safe_disabled():
+ return (
+ None,
+ f"{owner.__name__} forward is not capture-safe ({why}); rewrite disabled by {CAPTURE_SAFE_ENV}",
+ )
+ forward = vars(owner)["forward"]
+ key = (owner, forward)
+ with _CACHE_LOCK:
+ cached = _CACHE.get(key)
+ if cached is not None:
+ return cached
+ try:
+ result: tuple[Optional[Callable], Optional[str]] = (rewrite(forward), None)
+ except Exception as exc: # noqa: BLE001 - an unrewritable forward declines the graph, never fails a load
+ result = (
+ None,
+ f"{owner.__name__} forward is not capture-safe ({why}) and was not rewritten: {exc}",
+ )
+ with _CACHE_LOCK:
+ _CACHE[key] = result
+ return result
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 3c7d85d79d8..c013d6a5eab 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -222,8 +222,16 @@ class GraphedForward:
logger: Any = None,
) -> None:
self.module = module
- # The CLASS forward: the eager callable, whatever is already in the instance slot.
- self.orig = type(module).forward.__get__(module)
+ # The CLASS forward: the eager callable, whatever is already in the instance slot. A class
+ # whose stock forward syncs the host gets its capture-safe rewrite (bit-identical) instead.
+ safe = None
+ try:
+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
+ safe, _ = _capture_safe(type(module))
+ except Exception: # noqa: BLE001 - the stock forward is still a correct eager callable
+ safe = None
+ self.capture_safe = safe is not None
+ self.orig = (safe if safe is not None else type(module).forward).__get__(module)
try:
update_wrapper(self, self.orig)
except Exception: # noqa: BLE001
@@ -529,6 +537,17 @@ def graph_eligible(
if not bool(getattr(family, "supports_cuda_graph", family_default)):
return False, "family opts out"
+ # A denoiser whose forward syncs the host would only invalidate its capture and run eager; decline
+ # up front with the reason when its capture-safe rewrite did not apply.
+ try:
+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
+ for module in _denoiser_dits(pipe):
+ _, why = _capture_safe(type(module))
+ if why:
+ return False, why
+ except Exception as exc: # noqa: BLE001 - the capture itself still poisons safely
+ _warn(logger, "capture-safe probe", exc)
+
return True, "eligible"
diff --git a/studio/backend/tests/test_diffusion_capture_safe.py b/studio/backend/tests/test_diffusion_capture_safe.py
new file mode 100644
index 00000000000..8e2e5454e19
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_capture_safe.py
@@ -0,0 +1,396 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Unit tests for ``diffusion_capture_safe.py`` (HunyuanImage-2.1's capture-safe forward).
+
+CPU only. The rewrite is checked against synthetic modules carrying the stock merge block in both
+LoRA styles diffusers has shipped (inline in 0.36, a decorator from 0.37), against a drifted block,
+and, when the installed diffusers has the class, against a tiny random HunyuanImage transformer.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import textwrap
+import types
+
+import pytest
+
+from core.inference import diffusion_capture_safe as cs
+from core.inference import diffusion_cuda_graph as cg
+
+torch = pytest.importorskip("torch")
+
+# The stock merge block as diffusers 0.36.0 through 0.41 ship it (8-space indent inside the if).
+_STOCK_BLOCK = """
+ # reorder and combine text tokens: combine valid tokens first, then padding
+ new_encoder_hidden_states = []
+ new_encoder_attention_mask = []
+
+ for text, text_mask, text_2, text_mask_2 in zip(
+ encoder_hidden_states, encoder_attention_mask, encoder_hidden_states_2, encoder_attention_mask_2
+ ):
+ # Concatenate: [valid_mllm, valid_byt5, invalid_mllm, invalid_byt5]
+ new_encoder_hidden_states.append(
+ torch.cat(
+ [
+ text_2[text_mask_2], # valid byt5
+ text[text_mask], # valid mllm
+ text_2[~text_mask_2], # invalid byt5
+ text[~text_mask], # invalid mllm
+ ],
+ dim=0,
+ )
+ )
+
+ # Apply same reordering to attention masks
+ new_encoder_attention_mask.append(
+ torch.cat(
+ [
+ text_mask_2[text_mask_2],
+ text_mask[text_mask],
+ text_mask_2[~text_mask_2],
+ text_mask[~text_mask],
+ ],
+ dim=0,
+ )
+ )
+
+ encoder_hidden_states = torch.stack(new_encoder_hidden_states)
+ encoder_attention_mask = torch.stack(new_encoder_attention_mask)
+"""
+
+_MODULE_TEMPLATE = """
+import functools
+
+import torch
+
+SCALES = []
+
+
+def apply_lora_scale(kwargs_name):
+ def decorator(fn):
+ @functools.wraps(fn)
+ def wrapper(self, *args, **kwargs):
+ SCALES.append((kwargs.get(kwargs_name) or {{}}).get("scale", 1.0))
+ return fn(self, *args, **kwargs)
+
+ return wrapper
+
+ return decorator
+
+
+class HunyuanImageTransformer2DModel:
+ {decorator}
+ def forward(
+ self,
+ encoder_hidden_states,
+ encoder_attention_mask,
+ encoder_hidden_states_2=None,
+ encoder_attention_mask_2=None,
+ attention_kwargs=None,
+ return_dict=True,
+ ):
+ {inline_lora}
+ encoder_attention_mask = encoder_attention_mask.bool()
+ if encoder_hidden_states_2 is not None:
+ encoder_attention_mask_2 = encoder_attention_mask_2.bool()
+{block}
+ # the blocks see (states * 2) and a float view of the mask
+ return (encoder_hidden_states * 2, encoder_attention_mask.float())
+"""
+
+
+def _load_module(
+ tmp_path,
+ name,
+ *,
+ decorator = True,
+ block = _STOCK_BLOCK,
+):
+ source = _MODULE_TEMPLATE.format(
+ decorator = '@apply_lora_scale("attention_kwargs")' if decorator else "",
+ inline_lora = "SCALES.append((attention_kwargs or {}).get('scale', 1.0))"
+ if not decorator
+ else "pass",
+ block = block,
+ )
+ path = tmp_path / f"{name}.py"
+ path.write_text(textwrap.dedent(source))
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def fresh_cache(monkeypatch):
+ monkeypatch.setattr(cs, "_CACHE", {})
+ monkeypatch.delenv(cs.CAPTURE_SAFE_ENV, raising = False)
+ yield
+ for name in [n for n in sys.modules if n.startswith("_hyimg_fake_")]:
+ sys.modules.pop(name, None)
+
+
+def _reference_merge(e1, m1, e2, m2):
+ states, masks = [], []
+ for text, tm, text2, tm2 in zip(e1, m1, e2, m2):
+ states.append(torch.cat([text2[tm2], text[tm], text2[~tm2], text[~tm]], dim = 0))
+ masks.append(torch.cat([tm2[tm2], tm[tm], tm2[~tm2], tm[~tm]], dim = 0))
+ return torch.stack(states), torch.stack(masks)
+
+
+_MASKS = [
+ # (mllm mask rows, byt5 mask rows): right-padded, interleaved, all padding, all valid
+ ([[1, 1, 1, 0, 0, 0, 0]], [[1, 1, 0, 0, 0]]),
+ ([[1, 0, 1, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1]], [[0, 1, 0, 1, 1], [0, 0, 0, 0, 0]]),
+ ([[0] * 7, [1] * 7], [[1] * 5, [0] * 5]),
+ ([[1] * 7], [[1] * 5]),
+]
+
+
[email protected]("m1, m2", _MASKS)
+ "dtypes", [(torch.float32, torch.float32), (torch.bfloat16, torch.float32)]
+)
+def test_merge_text_streams_is_the_stock_permutation_bit_for_bit(m1, m2, dtypes):
+ gen = torch.Generator().manual_seed(0)
+ m1 = torch.tensor(m1).bool()
+ m2 = torch.tensor(m2).bool()
+ batch = m1.shape[0]
+ e1 = torch.randn(batch, m1.shape[1], 6, generator = gen).to(dtypes[0])
+ e2 = torch.randn(batch, m2.shape[1], 6, generator = gen).to(dtypes[1])
+
+ want_states, want_mask = _reference_merge(e1, m1, e2, m2)
+ got_states, got_mask = cs.merge_text_streams(e1, m1, e2, m2)
+
+ assert got_states.dtype == want_states.dtype
+ assert got_mask.dtype == torch.bool
+ assert torch.equal(got_states, want_states)
+ assert torch.equal(got_mask, want_mask)
+
+
+def _aten_ops(fn):
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ class _Record(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.ops: set = set()
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ self.ops.add(str(func))
+ return func(*args, **(kwargs or {}))
+
+ with _Record() as rec:
+ fn()
+ return rec.ops
+
+
+# Ops whose output shape depends on tensor VALUES: each one needs the count on the host.
+_DATA_DEPENDENT = (
+ "aten.index.Tensor",
+ "aten.nonzero",
+ "aten.masked_select",
+ "aten.unique",
+ "aten.item",
+)
+
+
+def test_merge_text_streams_has_no_data_dependent_op_where_the_stock_loop_does():
+ """The point of the rewrite: nothing sized by a mask, so nothing reads a count on the host."""
+ m1 = torch.tensor([[1, 0, 1]]).bool()
+ m2 = torch.tensor([[0, 1]]).bool()
+ e1, e2 = torch.randn(1, 3, 2), torch.randn(1, 2, 2)
+
+ stock = _aten_ops(lambda: _reference_merge(e1, m1, e2, m2))
+ assert any(op.startswith(_DATA_DEPENDENT) for op in stock), stock
+
+ ours = _aten_ops(lambda: cs.merge_text_streams(e1, m1, e2, m2))
+ assert not any(op.startswith(_DATA_DEPENDENT) for op in ours), ours
+
+
[email protected]("decorator", [True, False], ids = ["decorator-lora", "inline-lora"])
+def test_rewrite_matches_the_stock_forward_and_keeps_lora_handling(
+ tmp_path, fresh_cache, decorator
+):
+ mod = _load_module(tmp_path, f"_hyimg_fake_{int(decorator)}", decorator = decorator)
+ cls = mod.HunyuanImageTransformer2DModel
+ safe, why = cs.resolve(cls)
+ assert why is None
+ assert getattr(safe, "__unsloth_capture_safe__", False) is True
+
+ gen = torch.Generator().manual_seed(1)
+ m1 = torch.tensor([[1, 0, 1, 1, 0, 0, 1], [1, 1, 0, 0, 0, 0, 0]])
+ m2 = torch.tensor([[0, 1, 0, 1, 1], [1, 1, 1, 0, 0]])
+ kw = dict(
+ encoder_hidden_states = torch.randn(2, 7, 4, generator = gen),
+ encoder_attention_mask = m1,
+ encoder_hidden_states_2 = torch.randn(2, 5, 4, generator = gen),
+ encoder_attention_mask_2 = m2,
+ attention_kwargs = {"scale": 0.5},
+ )
+ obj = cls()
+ want = cls.forward(obj, **kw)
+ got = safe(obj, **kw)
+ assert torch.equal(want[0], got[0])
+ assert torch.equal(want[1], got[1])
+ # LoRA scaling ran on both paths: the decorator re-applied, or the inline code kept.
+ assert mod.SCALES == [0.5, 0.5]
+ # The byt5-free call skips the block on both paths.
+ kw2 = dict(encoder_hidden_states = kw["encoder_hidden_states"], encoder_attention_mask = m1)
+ assert torch.equal(cls.forward(obj, **kw2)[0], safe(obj, **kw2)[0])
+
+
+def test_rewrite_keeps_the_signature_and_readable_source(tmp_path, fresh_cache):
+ import inspect
+
+ mod = _load_module(tmp_path, "_hyimg_fake_sig")
+ safe, _ = cs.resolve(mod.HunyuanImageTransformer2DModel)
+ assert inspect.signature(safe) == inspect.signature(mod.HunyuanImageTransformer2DModel.forward)
+ src = inspect.getsource(safe)
+ assert cs._MERGE_HELPER_NAME in src
+ assert "text_2[text_mask_2]" not in src
+
+
+def test_resolve_is_cached_per_forward(tmp_path, fresh_cache):
+ mod = _load_module(tmp_path, "_hyimg_fake_cache")
+ first = cs.resolve(mod.HunyuanImageTransformer2DModel)
+ assert cs.resolve(mod.HunyuanImageTransformer2DModel) is first
+
+
+def test_drifted_block_is_not_rewritten_and_names_the_reason(tmp_path, fresh_cache):
+ drifted = _STOCK_BLOCK.replace(
+ "text_2[~text_mask_2], # invalid byt5\n text[~text_mask], # invalid mllm",
+ "text[~text_mask], # invalid mllm\n text_2[~text_mask_2], # invalid byt5",
+ )
+ assert drifted != _STOCK_BLOCK
+ mod = _load_module(tmp_path, "_hyimg_fake_drift", block = drifted)
+ safe, why = cs.resolve(mod.HunyuanImageTransformer2DModel)
+ assert safe is None
+ assert "HunyuanImageTransformer2DModel forward is not capture-safe" in why
+ assert "merge block changed" in why
+
+
+def test_kill_switch_declines(tmp_path, fresh_cache, monkeypatch):
+ mod = _load_module(tmp_path, "_hyimg_fake_kill")
+ monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")
+ safe, why = cs.resolve(mod.HunyuanImageTransformer2DModel)
+ assert safe is None
+ assert cs.CAPTURE_SAFE_ENV in why
+
+
+def test_unknown_class_needs_nothing(fresh_cache):
+ class FluxTransformer2DModel:
+ def forward(self, x):
+ return x
+
+ assert cs.resolve(FluxTransformer2DModel) == (None, None)
+ assert cs.resolve(types.SimpleNamespace) == (None, None)
+
+
+def test_subclass_overriding_forward_is_left_alone(tmp_path, fresh_cache):
+ mod = _load_module(tmp_path, "_hyimg_fake_sub")
+
+ class Custom(mod.HunyuanImageTransformer2DModel):
+ def forward(self, *args, **kwargs):
+ return "custom"
+
+ assert cs.resolve(Custom) == (None, None)
+
+ class Inherits(mod.HunyuanImageTransformer2DModel):
+ pass
+
+ assert cs.resolve(Inherits)[0] is not None
+
+
+def test_graphed_forward_records_the_capture_safe_forward(tmp_path, fresh_cache):
+ mod = _load_module(tmp_path, "_hyimg_fake_gf")
+ handle = cg.GraphedForward(mod.HunyuanImageTransformer2DModel())
+ try:
+ assert handle.capture_safe is True
+ assert handle.orig.__func__ is cs.resolve(mod.HunyuanImageTransformer2DModel)[0]
+ finally:
+ handle.free()
+
+
+def test_graphed_forward_keeps_the_class_forward_otherwise(tmp_path, fresh_cache):
+ mod = _load_module(
+ tmp_path, "_hyimg_fake_gf_drift", block = _STOCK_BLOCK.replace("dim=0,", "dim = 0,", 1)
+ )
+ handle = cg.GraphedForward(mod.HunyuanImageTransformer2DModel())
+ try:
+ assert handle.capture_safe is False
+ assert handle.orig.__func__ is mod.HunyuanImageTransformer2DModel.forward
+ finally:
+ handle.free()
+
+
+def _eligible_for(transformer):
+ return cg.graph_eligible(
+ types.SimpleNamespace(device = "cuda", backend = "cuda"),
+ family = types.SimpleNamespace(),
+ pipe = types.SimpleNamespace(transformer = transformer),
+ offload_active = False,
+ cache_active = False,
+ speed_mode = "default",
+ )
+
+
+def test_graph_eligible_declines_an_unrewritable_hunyuanimage(tmp_path, fresh_cache, monkeypatch):
+ monkeypatch.delenv(cg.CUDA_GRAPH_DISABLE_ENV, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ mod = _load_module(
+ tmp_path, "_hyimg_fake_elig_drift", block = _STOCK_BLOCK.replace("dim=0,", "dim = 0,", 1)
+ )
+ ok, reason = _eligible_for(mod.HunyuanImageTransformer2DModel())
+ assert ok is False
+ assert "not capture-safe" in reason
+
+ good = _load_module(tmp_path, "_hyimg_fake_elig_ok")
+ assert _eligible_for(good.HunyuanImageTransformer2DModel()) == (True, "eligible")
+
+
+def test_real_diffusers_hunyuanimage_rewrite_is_bit_identical(fresh_cache):
+ diffusers = pytest.importorskip("diffusers")
+ cls = getattr(diffusers, "HunyuanImageTransformer2DModel", None)
+ if cls is None:
+ pytest.skip("diffusers has no HunyuanImageTransformer2DModel")
+ safe, why = cs.resolve(cls)
+ assert why is None, why
+
+ torch.manual_seed(0)
+ model = cls(
+ in_channels = 4,
+ out_channels = 4,
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ num_layers = 1,
+ num_single_layers = 1,
+ num_refiner_layers = 1,
+ patch_size = (1, 1),
+ text_embed_dim = 32,
+ text_embed_2_dim = 24,
+ rope_axes_dim = (8, 8),
+ ).eval()
+ kw = dict(
+ encoder_hidden_states = torch.randn(2, 7, 32),
+ encoder_attention_mask = torch.tensor([[1, 1, 1, 0, 0, 0, 0], [1, 0, 1, 1, 1, 0, 1]]),
+ encoder_hidden_states_2 = torch.randn(2, 5, 24),
+ encoder_attention_mask_2 = torch.tensor([[1, 1, 0, 0, 0], [0, 0, 0, 0, 0]]),
+ return_dict = False,
+ )
+ latents = torch.randn(2, 4, 4, 4)
+ timestep = torch.tensor([500.0, 500.0])
+ with torch.no_grad():
+ want = cls.forward(model, latents, timestep, **kw)[0]
+ got = safe(model, latents, timestep, **kw)[0]
+ assert torch.equal(want, got)