Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: compile FLUX.1 dynamic, CUDA-graph the SDXL U-Net, decode one-frame Qwen-Image latents in 2D (#12060)

Daniel Hanc6eb2e66c415
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 6a0e673fb63..59ab84f1135 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -282,18 +282,24 @@ class GraphedForward:
warmup: int = WARMUP_ITERS,
max_graphs: int = MAX_GRAPHS_PER_MODULE,
logger: Any = None,
+ call: Any = None,
) -> None:
self.module = module
# The CLASS forward: the eager callable, whatever is already in the instance slot. A class
# whose stock forward syncs the host gets its capture-safe rewrite (bit-identical) instead.
+ # ``call`` overrides it (GraphedCompiledCall: the module's whole-module compiled callable).
safe = None
- try:
- from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
- safe, _ = _capture_safe(type(module))
- except Exception: # noqa: BLE001 - the stock forward is still a correct eager callable
- safe = None
+ if call is None:
+ try:
+ from .diffusion_capture_safe import resolve as _capture_safe # noqa: PLC0415
+ safe, _ = _capture_safe(type(module))
+ except Exception: # noqa: BLE001 - the stock forward is still a correct eager callable
+ safe = None
self.capture_safe = safe is not None
- self.orig = (safe if safe is not None else type(module).forward).__get__(module)
+ if call is not None:
+ self.orig = call
+ else:
+ self.orig = (safe if safe is not None else type(module).forward).__get__(module)
try:
update_wrapper(self, self.orig)
except Exception: # noqa: BLE001
@@ -585,6 +591,38 @@ class GraphedForward:
return entry
+class GraphedCompiledCall(GraphedForward):
+ """``GraphedForward`` for a whole-module compiled denoiser (SDXL U-Net): ``Module.__call__`` serves it from
+ ``_compiled_call_impl`` and never reads ``forward``, so the replay sits there and captures the compiled callable."""
+
+ def __init__(self, module: Any, **kwargs: Any) -> None:
+ compiled = getattr(module, "_compiled_call_impl", None)
+ if compiled is None:
+ raise RuntimeError(f"{type(module).__name__} is not whole-module compiled")
+ self.compiled = compiled
+ super().__init__(module, call = compiled, **kwargs)
+
+ def install(self) -> "GraphedCompiledCall":
+ self.module._compiled_call_impl = self
+ return self
+
+ def uninstall(self) -> "GraphedCompiledCall":
+ if getattr(self.module, "_compiled_call_impl", None) is self:
+ self.module._compiled_call_impl = self.compiled
+ return self
+
+
+def _denoiser_modules(pipe: Any) -> list:
+ """What the graph layer arms: every denoiser DiT, else a whole-compile-list U-Net."""
+ from .diffusion_speed import _denoiser_dits, _denoiser_unet
+
+ dits = _denoiser_dits(pipe)
+ if dits:
+ return dits
+ unet = _denoiser_unet(pipe)
+ return [unet] if unet is not None else []
+
+
def graph_eligible(
target: Any,
*,
@@ -623,11 +661,8 @@ def graph_eligible(
# Lazy: ``diffusion_speed`` imports this module, so a module-level import here is a cycle.
from .diffusion_speed import _denoiser_dits, _denoiser_unet
- if _denoiser_unet(pipe) is not None:
- # A whole-compiled U-Net serves from ``_compiled_call_impl``, which a slot swap bypasses.
- return False, "denoiser is a U-Net"
-
- if not _denoiser_dits(pipe):
+ # A whole-compiled U-Net serves from ``_compiled_call_impl``: GraphedCompiledCall captures there.
+ if not _denoiser_dits(pipe) and _denoiser_unet(pipe) is None:
return False, "no denoiser transformer"
try:
@@ -662,12 +697,15 @@ def install_cuda_graphs(
max_graphs: int = MAX_GRAPHS_PER_MODULE,
) -> tuple:
"""Arm one ``GraphedForward`` per denoiser module; the first denoising step captures."""
- from .diffusion_speed import _denoiser_dits
-
handles: list = []
- for module in _denoiser_dits(pipe):
+ for module in _denoiser_modules(pipe):
try:
- handles.append(GraphedForward(module, max_graphs = max_graphs, logger = logger).enable())
+ kind = (
+ GraphedCompiledCall
+ if getattr(module, "_compiled_call_impl", None) is not None
+ else GraphedForward
+ )
+ handles.append(kind(module, max_graphs = max_graphs, logger = logger).enable())
except Exception as exc: # noqa: BLE001 - a second expert may fail without failing the load
_warn(logger, f"install on {type(module).__name__}", exc)
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index c77ade0266c..da8a6b81123 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -437,6 +437,7 @@ def apply_speed_optims(
applied = {
"channels_last": False,
"vae_fp16_decode": False,
+ "vae_single_frame": False,
"vae_fused": False,
"cudnn_benchmark": False,
"tf32": False,
@@ -456,6 +457,7 @@ def apply_speed_optims(
on_cuda = getattr(target, "device", None) == "cuda"
family_allows_compile = bool(getattr(family, "supports_torch_compile", True))
+ applied["vae_single_frame"] = _vae_single_frame(pipe, logger)
# Lossless: a channels-last VAE speeds up its convs with no numeric change.
applied["channels_last"] = _vae_channels_last(
pipe, logger, fused = on_cuda and _fused_vae_planned(pipe)
@@ -581,6 +583,15 @@ def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool
)
+def _vae_single_frame(pipe: Any, logger: Any) -> bool:
+ try:
+ from . import diffusion_vae_single_frame # noqa: PLC0415
+ return diffusion_vae_single_frame.install(getattr(pipe, "vae", None), logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "vae single-frame path", exc)
+ return False
+
+
# channels_last: slower on the stock path (72.7 vs 89 ms), faster once the fused norms install (104.7 vs 121.0 ms).
_VAE_CHANNELS_LAST_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})
@@ -747,7 +758,19 @@ def _class_merges_streams(cls: type, broad: bool = False) -> bool:
return bool(_STREAM_MERGE_SOURCE.search(source))
+def _divisibility_proof_available() -> bool:
+ """Whether inductor proves ``(k*a + k*b) % (a + b) == 0`` (torch 2.14+, or the backport; see diffusion_inductor_backports)."""
+ try:
+ from . import diffusion_inductor_backports # noqa: PLC0415 - imports torch
+ return diffusion_inductor_backports.proof_available()
+ except Exception: # noqa: BLE001 - unanswerable: keep the static fallback
+ return False
+
+
def _dits_merge_streams(dits: list) -> bool:
+ """Whether a stream-merging block must compile static: not once inductor can prove the CantSplit split."""
+ if _divisibility_proof_available():
+ return False
broad = os.environ.get(_STREAM_MERGE_DETECT_ENV) == "1"
seen: set[type] = set()
for transformer in dits:
@@ -1157,11 +1180,12 @@ COMPILE_VAE_ENV = "UNSLOTH_DIFFUSION_COMPILE_VAE"
_VAE_TRUE_TOKENS = ("1", "true", "yes", "on")
_VAE_FALSE_TOKENS = ("0", "false", "no", "off")
-# Not a correctness list: these decode correctly compiled but measured SLOWER than eager.
+# Correct compiled but not worth it (Qwen-Image VAE: ~54 s first compile for ~23 ms/decode). Keyed by class, never by
+# the single-frame marker, which is installed after the compile-cache fingerprint is taken.
_VAE_COMPILE_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage", "AutoencoderKLWan"})
# ``auto`` compiles only measured VAEs: video DiTs also pass through apply_speed_optims.
-_VAE_COMPILE_ALLOW: frozenset[str] = frozenset({"AutoencoderKL"})
+_VAE_COMPILE_ALLOW: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})
def vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
@@ -1337,6 +1361,7 @@ def _compile_vae_decode(
try:
import torch
+ # dynamic=True: automatic dynamic pays a second compile on the next resolution for no steady-state gain.
kwargs: dict[str, Any] = {"fullgraph": False, "dynamic": True}
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
diff --git a/studio/backend/core/inference/diffusion_vae_single_frame.py b/studio/backend/core/inference/diffusion_vae_single_frame.py
new file mode 100644
index 00000000000..8fc540e8ddb
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_vae_single_frame.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Single-frame fast path for the Qwen-Image causal 3D VAE: one frame with no cache only meets the last
+temporal kernel slice, so each conv is exactly ``conv2d(x[:, :, 0], weight[:, :, -1], bias)``, and the
+first chunk of the cached walk equals ``decoder(x, feat_cache=None)``. Gated to single-frame untiled
+calls (tiled / multi-frame keep 3D convs). PSNR ~60 dB vs 3D (different cuDNN algo), so non-``off``
+tiers only. Kill switch, read at install: ``UNSLOTH_DIFFUSION_VAE_SINGLE_FRAME=0``."""
+
+from __future__ import annotations
+
+import os
+import types
+from typing import Any
+
+SINGLE_FRAME_ENV = "UNSLOTH_DIFFUSION_VAE_SINGLE_FRAME"
+
+_VAE_CLASSES = frozenset({"AutoencoderKLQwenImage"})
+_CONV_CLASSES = frozenset({"QwenImageCausalConv3d"})
+
+
+class _Gate:
+ def __init__(self) -> None:
+ self.on = False
+
+
+def single_frame_disabled() -> bool:
+ return (os.environ.get(SINGLE_FRAME_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _conv_forward(
+ self: Any,
+ x: Any,
+ cache_x: Any = None,
+) -> Any:
+ """``QwenImageCausalConv3d.forward`` with the one-frame, no-cache case as a 2D conv."""
+ import torch.nn.functional as F
+
+ pad = self._padding
+ kt = self.kernel_size[0]
+ if (
+ self._unsloth_sf_gate.on
+ and cache_x is None
+ and x.dim() == 5
+ and x.shape[2] == 1
+ and self.stride[0] == 1
+ and self.dilation[0] == 1
+ and pad[4] == kt - 1
+ and pad[5] == 0
+ and self.groups == 1
+ ):
+ # The front zero frames meet weight[:, :, :-1]; only the last slice sees the frame.
+ x2 = x[:, :, 0]
+ if pad[0] or pad[1] or pad[2] or pad[3]:
+ x2 = F.pad(x2, pad[:4])
+ out = F.conv2d(
+ x2,
+ self.weight[:, :, -1],
+ self.bias,
+ stride = self.stride[1:],
+ padding = 0,
+ dilation = self.dilation[1:],
+ )
+ return out.unsqueeze(2)
+ return type(self).forward(self, x, cache_x)
+
+
+def _single_frame(vae: Any, z: Any) -> bool:
+ try:
+ return bool(z.dim() == 5 and z.shape[2] == 1)
+ except Exception: # noqa: BLE001 - not a tensor: stock path
+ return False
+
+
+def _tiles(vae: Any, z: Any, sample_space: bool) -> bool:
+ if not getattr(vae, "use_tiling", False):
+ return False
+ ratio = 1 if sample_space else getattr(vae, "spatial_compression_ratio", 8)
+ h = getattr(vae, "tile_sample_min_height", 1 << 30) // ratio
+ w = getattr(vae, "tile_sample_min_width", 1 << 30) // ratio
+ return z.shape[-1] > w or z.shape[-2] > h
+
+
+def _fast_decode(
+ self: Any,
+ z: Any,
+ return_dict: bool = True,
+) -> Any:
+ stock = self._unsloth_stock_decode
+ if not _single_frame(self, z) or _tiles(self, z, False):
+ return stock(z, return_dict = return_dict)
+ import torch
+
+ gate = self._unsloth_sf_gate
+ gate.on = True
+ try:
+ x = self.post_quant_conv(z)
+ out = torch.clamp(self.decoder(x), min = -1.0, max = 1.0)
+ finally:
+ gate.on = False
+ if not return_dict:
+ return (out,)
+ from diffusers.models.autoencoders.vae import DecoderOutput
+
+ return DecoderOutput(sample = out)
+
+
+def _fast_encode(self: Any, x: Any) -> Any:
+ stock = self._unsloth_stock_encode
+ if not _single_frame(self, x) or _tiles(self, x, True):
+ return stock(x)
+ gate = self._unsloth_sf_gate
+ gate.on = True
+ try:
+ return self.quant_conv(self.encoder(x))
+ finally:
+ gate.on = False
+
+
+def install(vae: Any, logger: Any = None) -> bool:
+ """Arm the single-frame path on one VAE instance. Idempotent; False when the class is not covered."""
+ if vae is None or single_frame_disabled() or type(vae).__name__ not in _VAE_CLASSES:
+ return False
+ if getattr(vae, "_unsloth_single_frame", False):
+ return True
+ try:
+ convs = [m for m in vae.modules() if type(m).__name__ in _CONV_CLASSES]
+ if not convs or not callable(getattr(vae, "_decode", None)):
+ return False
+ for m in convs:
+ _ = (m._padding[5], m.kernel_size[0], m.stride[0], m.dilation[0], m.groups)
+ gate = _Gate()
+ for m in convs:
+ m._unsloth_sf_gate = gate
+ m.forward = types.MethodType(_conv_forward, m)
+ vae._unsloth_sf_gate = gate
+ vae._unsloth_stock_decode = vae._decode
+ vae._decode = types.MethodType(_fast_decode, vae)
+ if (
+ callable(getattr(vae, "_encode", None))
+ and hasattr(vae, "encoder")
+ and hasattr(vae, "quant_conv")
+ ):
+ vae._unsloth_stock_encode = vae._encode
+ vae._encode = types.MethodType(_fast_encode, vae)
+ vae._unsloth_single_frame = True
+ except Exception as exc: # noqa: BLE001 - an optimisation: the stock VAE still decodes
+ if logger is not None:
+ logger.warning("diffusion.vae: single-frame path not armed: %s", exc)
+ return False
+ if logger is not None:
+ logger.info(
+ "diffusion.vae: single-frame 2D path armed on %s (%d causal convs)",
+ type(vae).__name__,
+ len(convs),
+ )
+ return True
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index f82c9bbe6f0..386fa8fb78a 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -673,10 +673,6 @@ def test_graph_eligible_happy_path(stub_torch, monkeypatch):
({"speed_mode": "eager"}, "speed tier eager"),
({"speed_mode": "off"}, "speed tier off"),
({"speed_mode": None}, "speed tier off"),
- (
- {"pipe": types.SimpleNamespace(unet = UNet2DConditionModel(), transformer = _FakeDiT())},
- "denoiser is a U-Net",
- ),
({"pipe": types.SimpleNamespace()}, "no denoiser transformer"),
(
{"family": types.SimpleNamespace(supports_cuda_graph = False)},
@@ -691,6 +687,79 @@ def test_graph_eligible_refusals(stub_torch, monkeypatch, overrides, reason):
assert got == reason
+def test_graph_eligible_accepts_a_unet_denoiser(stub_torch, monkeypatch):
+ """A whole-compiled SDXL U-Net is captured at ``_compiled_call_impl`` (GraphedCompiledCall)."""
+ assert (
+ _eligible(monkeypatch, pipe = types.SimpleNamespace(unet = UNet2DConditionModel()))[0] is True
+ )
+ other = type("SomeOtherUNet", (), {})()
+ assert (
+ _eligible(monkeypatch, pipe = types.SimpleNamespace(unet = other))[1]
+ == "no denoiser transformer"
+ )
+
+
+class _CompiledUNet:
+ """``Module.compile``d stand-in: ``module(...)`` is served by ``_compiled_call_impl``."""
+
+ def __init__(self):
+ self.calls = 0
+
+ def compiled(
+ sample,
+ timestep = None,
+ return_dict = True,
+ ):
+ self.calls += 1
+ return (_FakeTensor((1, 4), value = ("out", self.calls), tag = "out"),)
+
+ self._compiled_call_impl = compiled
+
+ def forward(self, *args, **kwargs): # pragma: no cover - a compiled module never reads the slot
+ raise AssertionError("the forward slot must not be used")
+
+
+_CompiledUNet.__name__ = "UNet2DConditionModel" # the class NAME is what _denoiser_unet gates on
+
+
+def test_compiled_unet_is_graphed_at_the_compiled_call(stub_torch):
+ unet = _CompiledUNet()
+ compiled = unet._compiled_call_impl
+ pipe = types.SimpleNamespace(unet = unet)
+ handles = cg.install_cuda_graphs(pipe)
+ assert len(handles) == 1 and isinstance(handles[0], cg.GraphedCompiledCall)
+ assert unet._compiled_call_impl is handles[0]
+ assert "forward" not in unet.__dict__
+
+ for _ in range(2):
+ unet._compiled_call_impl(_t(), timestep = _t((1,)), return_dict = False)
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 2
+ # Warm-ups plus the capture call, all through the COMPILED callable.
+ assert unet.calls == cg.WARMUP_ITERS + 1
+
+ unet._compiled_call_impl(_t(), timestep = _t((1,))) # return_dict absent: eager, still compiled
+ assert unet.calls == cg.WARMUP_ITERS + 2
+
+ cg.uninstall_all(handles)
+ assert unet._compiled_call_impl is compiled
+
+
+def test_uncompiled_unet_is_graphed_at_the_forward_slot(stub_torch):
+ class UNet2DConditionModel(_FakeDiT): # noqa: N801
+ pass
+
+ unet = UNet2DConditionModel()
+ handles = cg.install_cuda_graphs(types.SimpleNamespace(unet = unet))
+ assert len(handles) == 1 and type(handles[0]) is cg.GraphedForward
+ assert unet.__dict__["forward"] is handles[0]
+ cg.uninstall_all(handles)
+
+
+def test_graphed_compiled_call_needs_a_compiled_module(stub_torch):
+ with pytest.raises(RuntimeError):
+ cg.GraphedCompiledCall(_FakeDiT())
+
+
def test_graph_eligible_refuses_under_the_kill_switch(stub_torch, monkeypatch):
monkeypatch.setenv(cg.CUDA_GRAPH_DISABLE_ENV, "1")
ok, reason = cg.graph_eligible(
@@ -1019,3 +1088,49 @@ def test_live_status_tolerates_a_record_without_the_control(stub_torch):
handle(_t(), timestep = 0.5, return_dict = False)
assert cg.live_status(None, ("cuda_graph",), (handle,)) == (None, [])
assert cg.live_status({}, (), (handle,)) == ({}, [])
+
+
+def test_real_cuda_compiled_module_capture_matches_the_compiled_call():
+ torch = pytest.importorskip("torch")
+ if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ try:
+ import triton # noqa: F401
+ except Exception:
+ pytest.skip("needs triton for inductor")
+
+ import torch.nn as nn
+
+ class UNet2DConditionModel(nn.Module): # noqa: N801
+ def __init__(self, dim):
+ super().__init__()
+ self.a = nn.Linear(dim, dim)
+ self.b = nn.Linear(dim, dim)
+
+ def forward(
+ self,
+ sample,
+ timestep,
+ return_dict = True,
+ ):
+ out = self.b(torch.nn.functional.silu(self.a(sample))) * timestep
+ return types.SimpleNamespace(sample = out) if return_dict else (out,)
+
+ torch.manual_seed(0)
+ unet = UNet2DConditionModel(64).to("cuda", torch.bfloat16).eval()
+ unet.compile(fullgraph = True, dynamic = False)
+ compiled = unet._compiled_call_impl
+ handles = cg.install_cuda_graphs(types.SimpleNamespace(unet = unet))
+ try:
+ assert isinstance(handles[0], cg.GraphedCompiledCall)
+ for _ in range(3):
+ x = torch.randn(2, 8, 64, device = "cuda", dtype = torch.bfloat16)
+ t = torch.randn((), device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ want = compiled(x, t, return_dict = False)[0]
+ got = unet(x, t, return_dict = False)[0]
+ assert torch.equal(got, want)
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["replays"] == 3
+ finally:
+ cg.uninstall_all(handles)
+ assert unet._compiled_call_impl is compiled
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index c7ffd5e980b..42889de8a47 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -427,6 +427,7 @@ def test_speed_off_applies_nothing(monkeypatch):
assert applied == {
"channels_last": False,
"vae_fp16_decode": False,
+ "vae_single_frame": False,
"vae_fused": False,
"cudnn_benchmark": False,
"tf32": False,
@@ -866,6 +867,36 @@ def test_dit_vae_decode_compile_max_tier_autotunes(monkeypatch):
]
[email protected]("vae_name", ["AutoencoderKL", "AutoencoderKLFlux2"])
+def test_max_tier_vae_decode_compiles_dynamic_once_for_every_resolution(monkeypatch, vae_name):
+ """Automatic dynamic paid a generalising VAE recompile on the second resolution (minutes on max); dynamic=True
+ compiles once, and a VAE compile never marks the load automatic-dynamic."""
+ torch = _stub_torch(monkeypatch)
+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)
+ pipe = _Pipe(with_compile = True, vae_cls = type(vae_name, (AutoencoderKL,), {}))
+ applied = apply_speed_optims(
+ pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX
+ )
+ assert applied["compiled_vae_decode"] is True
+ assert [c["dynamic"] for c in torch.compile_calls] == [True]
+ assert not getattr(pipe.vae, "_unsloth_auto_dynamic", False)
+
+
+def test_qwen_image_vae_decode_stays_eager_with_the_single_frame_path(monkeypatch):
+ """Not worth a compile on max (see _VAE_COMPILE_DENY), and never keyed on the marker installed after the
+ compile-cache fingerprint: the loader and apply_speed_optims must give the same answer."""
+ _stub_torch(monkeypatch)
+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)
+ AutoencoderKLQwenImage = type("AutoencoderKLQwenImage", (), {})
+ pipe = types.SimpleNamespace(vae = AutoencoderKLQwenImage())
+ assert ds_mod._vae_decode_compile_allowed(pipe, SPEED_MAX) is False
+ pipe.vae._unsloth_single_frame = True
+ assert ds_mod._vae_decode_compile_allowed(pipe, SPEED_MAX) is False
+ assert ds_mod._vae_decode_compile_allowed(pipe, SPEED_DEFAULT) is False
+ monkeypatch.setenv(ds_mod.COMPILE_VAE_ENV, "1")
+ assert ds_mod._vae_decode_compile_allowed(pipe, SPEED_DEFAULT) is True
+
+
def test_unet_vae_decode_compile_ignores_the_env(monkeypatch):
_stub_torch(monkeypatch)
monkeypatch.setenv(ds_mod.COMPILE_VAE_ENV, "0")
@@ -1634,6 +1665,12 @@ class _DualStreamBlock(_StreamBlock):
return hidden_states + 1, encoder_hidden_states + 1
+def no_divisibility_proof(monkeypatch):
+ """The static fallback only exists for a torch whose inductor cannot prove the stream-merge split."""
+ monkeypatch.setattr(ds_mod, "_divisibility_proof_available", lambda: False)
+
+
def _dit(*block_classes):
blocks = [cls() for cls in block_classes]
dit = types.SimpleNamespace(
@@ -1668,6 +1705,7 @@ def test_class_merges_streams_without_source_falls_back_to_the_name_list(monkeyp
ds_mod._class_merges_streams.cache_clear()
[email protected]("no_divisibility_proof")
def test_dits_merge_streams_honours_the_opt_in_env(monkeypatch):
monkeypatch.delenv(ds_mod._STREAM_MERGE_DETECT_ENV, raising = False)
assert ds_mod._dits_merge_streams([_dit(_MergingByArgOrderA)]) is False
@@ -1677,6 +1715,7 @@ def test_dits_merge_streams_honours_the_opt_in_env(monkeypatch):
assert ds_mod._dits_merge_streams([_dit(FluxSingleTransformerBlock)]) is True
[email protected]("no_divisibility_proof")
def test_dits_merge_streams_scans_every_denoiser():
assert ds_mod._dits_merge_streams([]) is False
assert ds_mod._dits_merge_streams([types.SimpleNamespace()]) is False
@@ -1689,6 +1728,7 @@ def test_dits_merge_streams_scans_every_denoiser():
)
[email protected]("no_divisibility_proof")
def test_speed_default_compiles_stream_merging_dit_with_static_shapes(monkeypatch):
"""FLUX.1 regression: dynamic=True cannot be codegen'd for a stream-merging block."""
_stub_torch(monkeypatch)
@@ -1705,6 +1745,7 @@ def test_speed_default_compiles_stream_merging_dit_with_static_shapes(monkeypatc
assert pipe.compile_kwargs["dynamic"] is not None
[email protected]("no_divisibility_proof")
def test_compiled_shapes_are_static_reports_the_stream_merging_downgrade(monkeypatch):
_stub_torch(monkeypatch)
merging = types.SimpleNamespace(transformer = _dit(FluxSingleTransformerBlock))
@@ -1728,6 +1769,30 @@ def test_the_loader_keys_the_compile_bundle_on_the_vae_decode_decision():
assert ds_mod.vae_decode_compile_allowed is not None
+def test_stream_merging_dit_compiles_dynamic_once_inductor_proves_the_split(monkeypatch):
+ """With the divisibility proof (torch 2.14+ or the backport) FLUX.1 compiles dynamic like every other DiT."""
+ _stub_torch(monkeypatch)
+ _stub_gguf_accel(monkeypatch)
+ monkeypatch.setattr(ds_mod, "_divisibility_proof_available", lambda: True)
+ pipe = _Pipe(with_compile = True)
+ pipe.transformer._repeated_blocks = ["FluxSingleTransformerBlock"]
+ block = FluxSingleTransformerBlock()
+ pipe.transformer.named_modules = lambda: [("blocks.0", block)]
+ applied = apply_speed_optims(
+ pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT
+ )
+ assert applied["compiled"] is True
+ assert pipe.compile_kwargs == {"fullgraph": True, "dynamic": True}
+ assert ds_mod._dits_merge_streams([_dit(FluxSingleTransformerBlock)]) is False
+ assert ds_mod.compiled_shapes_are_static(pipe, SPEED_DEFAULT) is False
+
+
+def test_divisibility_proof_probe_never_raises(monkeypatch):
+ from core.inference import diffusion_inductor_backports as bp
+ monkeypatch.setattr(bp, "proof_available", lambda: (_ for _ in ()).throw(RuntimeError("probe")))
+ assert ds_mod._divisibility_proof_available() is False
+
+
def test_speed_max_keeps_automatic_dynamic_for_a_stream_merging_dit(monkeypatch):
"""The static exception is the default tier's alone: max compiles every DiT with automatic dynamic."""
_stub_torch(monkeypatch)
@@ -2174,6 +2239,7 @@ def test_automatic_dynamic_compile_arms_the_prompt_length_allowlist(monkeypatch)
assert armed == []
[email protected]("no_divisibility_proof")
def test_speed_max_compiles_a_quantised_stream_merging_dit_static(monkeypatch):
"""A torchao FLUX block under automatic dynamic hits CantSplit on the first new resolution and drops to eager;
max compiles it static instead, and reports its artifacts as per-shape."""
diff --git a/studio/backend/tests/test_diffusion_stream_merge_dynamic.py b/studio/backend/tests/test_diffusion_stream_merge_dynamic.py
new file mode 100644
index 00000000000..7a74da6d9d5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_stream_merge_dynamic.py
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The inductor divisibility backport against the exact FLUX.1 CantSplit expression (needs a real torch)."""
+
+import pytest
+
+sizevars = pytest.importorskip("torch._inductor.sizevars")
+sympy = pytest.importorskip("sympy")
+
+from core.inference import diffusion_inductor_backports as bp # noqa: E402
+
+
+def backport():
+ bp.uninstall()
+ yield bp
+ bp.uninstall()
+
+
+def test_flux_single_block_split_is_proven(backport):
+ s31, s87 = sympy.symbols("s31 s87", integer = True, positive = True)
+ backport.install()
+ assert backport.proof_available() is True
+ allocator = sizevars.SizeVarAllocator()
+ # FluxSingleTransformerBlock: cat([attn, mlp], dim=2) over the merged text+image sequence.
+ assert allocator.statically_known_multiple_of(15360 * s31 + 15360 * s87, s31 + s87)
+ # With the text length static (mark_static on the text dim): still exact.
+ assert allocator.statically_known_multiple_of(15360 * s87 + 7864320, s87 + 512)
+
+
+def test_backport_stays_sound(backport):
+ s31, s87 = sympy.symbols("s31 s87", integer = True, positive = True)
+ backport.install()
+ allocator = sizevars.SizeVarAllocator()
+ assert not allocator.statically_known_multiple_of(15360 * s31 + 15359 * s87, s31 + s87)
+ assert not allocator.statically_known_multiple_of(15360 * s87 + 7864321, s87 + 512)
+ assert not allocator.statically_known_multiple_of(s31 + 1, s31 + s87)
+
+
+def test_kill_switch_keeps_the_static_fallback(backport, monkeypatch):
+ monkeypatch.setenv(bp.BACKPORTS_ENV, "0")
+ if bp._stock_proves(sizevars.SizeVarAllocator):
+ pytest.skip("this torch proves the split natively")
+ assert bp.proof_available() is False
diff --git a/studio/backend/tests/test_diffusion_vae_compile_dynamic.py b/studio/backend/tests/test_diffusion_vae_compile_dynamic.py
new file mode 100644
index 00000000000..2f1234f46e5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_compile_dynamic.py
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A compiled DiT VAE decode must not recompile when the resolution changes (real Dynamo, eager backend)."""
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+
+from core.inference import diffusion_speed as ds # noqa: E402
+
+
+def test_compiled_vae_decode_serves_a_second_resolution_without_a_recompile(monkeypatch):
+ from torch._dynamo.utils import counters
+
+ real_compile = torch.compile
+ seen = []
+
+ def eager_backend_compile(fn, **kwargs):
+ seen.append(dict(kwargs))
+ kwargs.pop("mode", None)
+ return real_compile(fn, backend = "eager", **kwargs)
+
+ monkeypatch.setattr(torch, "compile", eager_backend_compile)
+ monkeypatch.setattr(ds, "_install_inductor_backports", lambda *a, **k: False)
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKL(
+ block_out_channels = (32,), norm_num_groups = 32, latent_channels = 4, layers_per_block = 1
+ ).eval()
+ pipe = types.SimpleNamespace(vae = vae, transformer = types.SimpleNamespace())
+ torch._dynamo.reset()
+ assert ds._compile_vae_decode(pipe, None, max_autotune = True) is True
+ assert seen and seen[0]["dynamic"] is True
+ graphs = []
+ with torch.no_grad():
+ for side in (16, 24, 20):
+ before = int(counters["stats"]["unique_graphs"])
+ out = vae.decode(torch.randn(1, 4, side, side)).sample
+ assert out.shape[-1] == side
+ graphs.append(int(counters["stats"]["unique_graphs"]) - before)
+ torch._dynamo.reset()
+ assert graphs[0] >= 1
+ assert graphs[1:] == [0, 0], graphs
diff --git a/studio/backend/tests/test_diffusion_vae_single_frame.py b/studio/backend/tests/test_diffusion_vae_single_frame.py
new file mode 100644
index 00000000000..78d9f29a23a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_single_frame.py
@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The Qwen-Image VAE single-frame 2D path: same image as the stock causal-3D walk, stock path for video/tiles."""
+
+import copy
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+autoencoders = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_qwenimage")
+
+from core.inference import diffusion_vae_single_frame as sf # noqa: E402
+
+
+def _tiny_vae():
+ torch.manual_seed(0)
+ vae = autoencoders.AutoencoderKLQwenImage(
+ base_dim = 16,
+ z_dim = 4,
+ dim_mult = [1, 2, 2],
+ num_res_blocks = 1,
+ temperal_downsample = [False, True],
+ attn_scales = [],
+ )
+ return vae.float().eval()
+
+
[email protected](autouse = True)
+def _no_kill_switch(monkeypatch):
+ monkeypatch.delenv(sf.SINGLE_FRAME_ENV, raising = False)
+
+
+def test_single_frame_decode_and_encode_match_the_stock_walk():
+ stock = _tiny_vae()
+ fast = copy.deepcopy(stock)
+ assert sf.install(fast) is True
+ z = torch.randn(1, 4, 1, 16, 16)
+ x = torch.rand(1, 3, 1, 64, 64) * 2 - 1
+ with torch.no_grad():
+ want = stock.decode(z).sample
+ got = fast.decode(z).sample
+ assert got.shape == want.shape
+ assert torch.allclose(got, want, atol = 1e-5, rtol = 1e-4)
+ e_want = stock.encode(x).latent_dist.mean
+ e_got = fast.encode(x).latent_dist.mean
+ assert torch.allclose(e_got, e_want, atol = 1e-5, rtol = 1e-4)
+
+
+def test_multi_frame_and_tiled_calls_take_the_stock_path(monkeypatch):
+ stock = _tiny_vae()
+ fast = copy.deepcopy(stock)
+ sf.install(fast)
+ z = torch.randn(1, 4, 3, 16, 16)
+ with torch.no_grad():
+ # Bit-identical: the first chunk of the walk also reaches every conv with no cache and one frame.
+ assert torch.equal(fast.decode(z).sample, stock.decode(z).sample)
+ x = torch.rand(1, 3, 5, 64, 64) * 2 - 1
+ assert torch.equal(fast.encode(x).latent_dist.mean, stock.encode(x).latent_dist.mean)
+ for vae in (stock, fast):
+ vae.enable_tiling(tile_sample_min_height = 32, tile_sample_min_width = 32)
+ z1 = torch.randn(1, 4, 1, 16, 16)
+ with torch.no_grad():
+ assert torch.equal(fast.decode(z1).sample, stock.decode(z1).sample)
+ calls = []
+ real = fast._unsloth_stock_decode
+ fast._unsloth_stock_decode = lambda *a, **k: calls.append(1) or real(*a, **k)
+ fast.enable_tiling(tile_sample_min_height = 32, tile_sample_min_width = 32)
+ with torch.no_grad():
+ fast.decode(torch.randn(1, 4, 1, 16, 16))
+ assert calls == [1]
+
+
+def test_kill_switch_and_foreign_classes(monkeypatch):
+ assert sf.install(None) is False
+ assert sf.install(types.SimpleNamespace()) is False
+ monkeypatch.setenv(sf.SINGLE_FRAME_ENV, "0")
+ assert sf.install(_tiny_vae()) is False
+
+
+def test_install_is_idempotent_and_marks_the_vae():
+ vae = _tiny_vae()
+ assert sf.install(vae) is True
+ first = vae._decode
+ assert sf.install(vae) is True
+ assert vae._decode is first and vae._unsloth_single_frame is True
+
+
[email protected]("single_frame", ["1", "0"])
[email protected]("compile_env", [None, "1"])
+def test_compile_cache_key_matches_the_later_vae_compile(monkeypatch, single_frame, compile_env):
+ """The loader keys the compile bundle on vae_decode_compile_allowed() BEFORE apply_speed_optims installs the
+ single-frame path; the key must still equal what that later pass compiles."""
+ from core.inference import diffusion_speed as ds
+
+ monkeypatch.setenv(sf.SINGLE_FRAME_ENV, single_frame)
+ if compile_env is None:
+ monkeypatch.delenv(ds.COMPILE_VAE_ENV, raising = False)
+ else:
+ monkeypatch.setenv(ds.COMPILE_VAE_ENV, compile_env)
+ monkeypatch.setattr(torch, "compile", lambda fn, **kw: fn)
+ monkeypatch.setattr(ds, "compile_eligible", lambda *a, **k: True)
+ monkeypatch.setattr(ds, "_compile_repeated_blocks", lambda *a, **k: True)
+ monkeypatch.setattr(ds, "_install_inductor_backports", lambda *a, **k: False)
+ target = types.SimpleNamespace(
+ device = "cpu", dtype = torch.bfloat16, supports_default_torch_compile = True
+ )
+ family = types.SimpleNamespace(supports_torch_compile = True)
+ for mode in (ds.SPEED_MAX, ds.SPEED_DEFAULT):
+ vae = _tiny_vae()
+ pipe = types.SimpleNamespace(vae = vae, transformer = types.SimpleNamespace())
+ key = ds.vae_decode_compile_allowed(pipe, mode)
+ applied = ds.apply_speed_optims(pipe, target, is_gguf = False, family = family, speed_mode = mode)
+ assert applied["compiled_vae_decode"] is key, (mode, single_frame, compile_env)
+ # A second load of the same weights asks again with the path already armed.
+ assert ds.vae_decode_compile_allowed(pipe, mode) is key