Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: hand compiled blocks the recorded tensor kind on graphless steps (no recompile on a new prompt length) (#14448)

Daniel Han52533bbb8ac0
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 5f96f9b226f..a5e3859fced 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -1051,6 +1051,34 @@ class GraphedForward:
self.stats["eager_calls"] += 1
return self.orig(*args, **kwargs)
+ def _eager_as_recorded(self, args: tuple, kwargs: dict) -> Any:
+ """Eager step with inference leaves copied to normal tensors, matching what unplanned captures compiled
+ blocks for (else dispatch-key guards recompile them, e.g. FLUX.3 cached conditioning states)."""
+ if self.plan is not None:
+ return self._eager(args, kwargs)
+ live: list = []
+ try:
+ spec = _flatten((args, kwargs), live)
+ except TypeError:
+ return self._eager(args, kwargs)
+ picks = [
+ i
+ for i, t in enumerate(live)
+ if callable(getattr(t, "is_inference", None)) and t.is_inference()
+ ]
+ if not picks:
+ return self._eager(args, kwargs)
+ try:
+ copies = self._statics([live[i] for i in picks])
+ except (
+ _torch().cuda.OutOfMemoryError
+ ): # no room for the copies: run as before, at the cost of a recompile
+ return self._eager(args, kwargs)
+ for i, fresh in zip(picks, copies):
+ live[i] = fresh
+ args, kwargs = _rebuild(spec, live)
+ return self._eager(args, kwargs)
+
def _engage_fallback(self) -> None:
"""Arm the per-block graphs once this whole-step graph dropped a key or failed: the calls it now leaves eager
run through them. Between forwards' blocks (the step has not started), so the slot ring can change."""
@@ -1190,7 +1218,7 @@ class GraphedForward:
# A prompt length first seen after the first render: no warm-up, no capture, no eviction on this
# render. (An offloaded key it recorded before re-records after its weights moved, as before.)
self.stats["length_eager"] += 1
- return self._eager(eager_args, eager_kwargs)
+ return self._eager_as_recorded(eager_args, eager_kwargs)
if self.placement is not None and speed_check_enabled():
if key in self._dropped:
return self._eager(eager_args, eager_kwargs)
@@ -1231,7 +1259,7 @@ class GraphedForward:
self.max_graphs,
type(self.module).__name__,
)
- return self._eager(eager_args, eager_kwargs)
+ return self._eager_as_recorded(eager_args, eager_kwargs)
captured = None
stream_before = _current_stream()
try:
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index 3010fbb5a21..2408ba0167f 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -1469,3 +1469,151 @@ def test_the_every_length_switch_keeps_block_graphs(stub_torch, monkeypatch):
assert cg.prompt_length_reason(pipe) is not None
monkeypatch.setenv(cg.EVERY_LENGTH_ENV, "1")
assert cg.prompt_length_reason(pipe) is None
+
+
+def test_real_cuda_new_length_eager_step_hands_blocks_the_recorded_tensor_kind():
+ """A new-length eager step must not recompile compiled blocks by passing them inference tensors."""
+ torch = pytest.importorskip("torch")
+ if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ import torch.nn as nn
+ from torch._dynamo.testing import CompileCounter
+
+ torch._dynamo.reset()
+ counter = CompileCounter()
+
+ class Block(nn.Module):
+ def forward(self, x, mod):
+ return x * (1 + mod) + mod
+
+ class LenDiT(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.proj = nn.Linear(dim, dim)
+ self.block = torch.compile(Block(), backend = counter, dynamic = False)
+
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states,
+ mod,
+ return_dict = True,
+ ):
+ x = self.proj(hidden_states) + encoder_hidden_states.mean()
+ out = self.block(x, mod)
+ return types.SimpleNamespace(sample = out) if return_dict else (out,)
+
+ dim = 32
+ torch.manual_seed(0)
+ module = type("Flux3Transformer2DModel", (LenDiT,), {})(dim).to("cuda").eval()
+ handle = cg.GraphedForward(module).enable()
+ assert handle.length_varies
+ hidden = torch.randn(1, 16, dim, device = "cuda")
+
+ def render(text_len):
+ outs = []
+ with torch.inference_mode():
+ txt = torch.randn(1, text_len, dim, device = "cuda")
+ mod = torch.randn(1, 16, dim, device = "cuda")
+ x = module.proj(hidden) + txt.mean()
+ want = x * (1 + mod) + mod
+ for _ in range(3):
+ outs.append((module(hidden, txt, mod, return_dict = False)[0], want))
+ cg.seal_lengths([handle])
+ return outs
+
+ try:
+ render(5)
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 3
+ frames = counter.frame_count
+ for got, want in render(9):
+ assert torch.allclose(got, want)
+ assert handle.stats["length_eager"] == 3 and handle.stats["captures"] == 1
+ # Before: the block recompiled for an inference ``mod``.
+ assert counter.frame_count == frames
+ finally:
+ cg.uninstall_all([handle])
+
+
+def test_new_length_eager_step_hands_blocks_normal_tensors_on_cpu(monkeypatch):
+ """CPU twin of the CUDA test above, through the real ``__call__``."""
+ torch = pytest.importorskip("torch")
+ if not hasattr(torch, "_dynamo"):
+ pytest.skip("needs torch._dynamo")
+ import torch.nn as nn
+ from torch._dynamo.testing import CompileCounter
+
+ monkeypatch.delenv(cg.EVERY_LENGTH_ENV, raising = False)
+ torch._dynamo.reset()
+ counter = CompileCounter()
+ seen: list = []
+
+ class Block(nn.Module):
+ def forward(self, x, mod):
+ return x * (1 + mod) + mod
+
+ block = torch.compile(Block(), backend = counter, dynamic = False)
+
+ class LenDiT(nn.Module):
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states,
+ mod,
+ return_dict = True,
+ ):
+ seen.append(mod.is_inference())
+ return (block(hidden_states + encoder_hidden_states.mean(), mod),)
+
+ module = type("Flux3Transformer2DModel", (LenDiT,), {})()
+ handle = _armed(module)
+ assert handle.length_varies and handle.plan is None
+ hidden = torch.randn(1, 16, 8)
+ try:
+ with torch.inference_mode():
+ txt, mod = torch.randn(1, 9, 8), torch.randn(1, 16, 8)
+ statics = handle._statics([hidden, txt, mod])
+ handle.orig(*statics, return_dict = False)
+ frames = counter.frame_count
+ handle._sealed_scopes.add(cg._image_scope((hidden,), {}))
+ got = module(hidden, txt, mod, return_dict = False)[0]
+ assert handle.stats["length_eager"] == 1 and handle.stats["captures"] == 0
+ # Before: the block recompiled for an inference ``mod``.
+ assert counter.frame_count == frames
+ assert seen == [False, False]
+ assert torch.equal(got, (hidden + txt.mean()) * (1 + mod) + mod)
+ finally:
+ cg.uninstall_all([handle])
+
+
+def test_eager_as_recorded_runs_the_original_call_when_the_copies_do_not_fit(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import torch.nn as nn
+
+ seen: list = []
+
+ class LenDiT(nn.Module):
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states,
+ return_dict = True,
+ ):
+ seen.append(encoder_hidden_states.is_inference())
+ return (hidden_states + encoder_hidden_states.mean(),)
+
+ module = type("Flux3Transformer2DModel", (LenDiT,), {})()
+ handle = _armed(module)
+
+ def no_room(tensors):
+ raise torch.cuda.OutOfMemoryError("no room")
+
+ monkeypatch.setattr(handle, "_statics", no_room)
+ try:
+ with torch.inference_mode():
+ hidden, txt = torch.randn(1, 4, 8), torch.randn(1, 3, 8)
+ got = handle._eager_as_recorded((hidden, txt), {"return_dict": False})[0]
+ assert seen == [True]
+ assert torch.equal(got, hidden + txt.mean())
+ finally:
+ cg.uninstall_all([handle])