Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: hand compiled blocks the recorded tensor kind on graphless steps (no recompile on a new prompt length) (#14448)
Daniel Han52533bbb8ac0
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 5f96f9b226f..a5e3859fced 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -1051,6 +1051,34 @@ class GraphedForward:self.stats["eager_calls"] += 1return self.orig(*args, **kwargs)+ def _eager_as_recorded(self, args: tuple, kwargs: dict) -> Any:+ """Eager step with inference leaves copied to normal tensors, matching what unplanned captures compiled+ blocks for (else dispatch-key guards recompile them, e.g. FLUX.3 cached conditioning states)."""+ if self.plan is not None:+ return self._eager(args, kwargs)+ live: list = []+ try:+ spec = _flatten((args, kwargs), live)+ except TypeError:+ return self._eager(args, kwargs)+ picks = [+ i+ for i, t in enumerate(live)+ if callable(getattr(t, "is_inference", None)) and t.is_inference()+ ]+ if not picks:+ return self._eager(args, kwargs)+ try:+ copies = self._statics([live[i] for i in picks])+ except (+ _torch().cuda.OutOfMemoryError+ ): # no room for the copies: run as before, at the cost of a recompile+ return self._eager(args, kwargs)+ for i, fresh in zip(picks, copies):+ live[i] = fresh+ args, kwargs = _rebuild(spec, live)+ return self._eager(args, kwargs)+def _engage_fallback(self) -> None:"""Arm the per-block graphs once this whole-step graph dropped a key or failed: the calls it now leaves eagerrun through them. Between forwards' blocks (the step has not started), so the slot ring can change."""@@ -1190,7 +1218,7 @@ class GraphedForward:# A prompt length first seen after the first render: no warm-up, no capture, no eviction on this# render. (An offloaded key it recorded before re-records after its weights moved, as before.)self.stats["length_eager"] += 1- return self._eager(eager_args, eager_kwargs)+ return self._eager_as_recorded(eager_args, eager_kwargs)if self.placement is not None and speed_check_enabled():if key in self._dropped:return self._eager(eager_args, eager_kwargs)@@ -1231,7 +1259,7 @@ class GraphedForward:self.max_graphs,type(self.module).__name__,)- return self._eager(eager_args, eager_kwargs)+ return self._eager_as_recorded(eager_args, eager_kwargs)captured = Nonestream_before = _current_stream()try:diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.pyindex 3010fbb5a21..2408ba0167f 100644--- a/studio/backend/tests/test_diffusion_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_cuda_graph.py@@ -1469,3 +1469,151 @@ def test_the_every_length_switch_keeps_block_graphs(stub_torch, monkeypatch):assert cg.prompt_length_reason(pipe) is not Nonemonkeypatch.setenv(cg.EVERY_LENGTH_ENV, "1")assert cg.prompt_length_reason(pipe) is None+++def test_real_cuda_new_length_eager_step_hands_blocks_the_recorded_tensor_kind():+ """A new-length eager step must not recompile compiled blocks by passing them inference tensors."""+ torch = pytest.importorskip("torch")+ if not getattr(torch, "cuda", None) or not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ import torch.nn as nn+ from torch._dynamo.testing import CompileCounter++ torch._dynamo.reset()+ counter = CompileCounter()++ class Block(nn.Module):+ def forward(self, x, mod):+ return x * (1 + mod) + mod++ class LenDiT(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.proj = nn.Linear(dim, dim)+ self.block = torch.compile(Block(), backend = counter, dynamic = False)++ def forward(+ self,+ hidden_states,+ encoder_hidden_states,+ mod,+ return_dict = True,+ ):+ x = self.proj(hidden_states) + encoder_hidden_states.mean()+ out = self.block(x, mod)+ return types.SimpleNamespace(sample = out) if return_dict else (out,)++ dim = 32+ torch.manual_seed(0)+ module = type("Flux3Transformer2DModel", (LenDiT,), {})(dim).to("cuda").eval()+ handle = cg.GraphedForward(module).enable()+ assert handle.length_varies+ hidden = torch.randn(1, 16, dim, device = "cuda")++ def render(text_len):+ outs = []+ with torch.inference_mode():+ txt = torch.randn(1, text_len, dim, device = "cuda")+ mod = torch.randn(1, 16, dim, device = "cuda")+ x = module.proj(hidden) + txt.mean()+ want = x * (1 + mod) + mod+ for _ in range(3):+ outs.append((module(hidden, txt, mod, return_dict = False)[0], want))+ cg.seal_lengths([handle])+ return outs++ try:+ render(5)+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 3+ frames = counter.frame_count+ for got, want in render(9):+ assert torch.allclose(got, want)+ assert handle.stats["length_eager"] == 3 and handle.stats["captures"] == 1+ # Before: the block recompiled for an inference ``mod``.+ assert counter.frame_count == frames+ finally:+ cg.uninstall_all([handle])+++def test_new_length_eager_step_hands_blocks_normal_tensors_on_cpu(monkeypatch):+ """CPU twin of the CUDA test above, through the real ``__call__``."""+ torch = pytest.importorskip("torch")+ if not hasattr(torch, "_dynamo"):+ pytest.skip("needs torch._dynamo")+ import torch.nn as nn+ from torch._dynamo.testing import CompileCounter++ monkeypatch.delenv(cg.EVERY_LENGTH_ENV, raising = False)+ torch._dynamo.reset()+ counter = CompileCounter()+ seen: list = []++ class Block(nn.Module):+ def forward(self, x, mod):+ return x * (1 + mod) + mod++ block = torch.compile(Block(), backend = counter, dynamic = False)++ class LenDiT(nn.Module):+ def forward(+ self,+ hidden_states,+ encoder_hidden_states,+ mod,+ return_dict = True,+ ):+ seen.append(mod.is_inference())+ return (block(hidden_states + encoder_hidden_states.mean(), mod),)++ module = type("Flux3Transformer2DModel", (LenDiT,), {})()+ handle = _armed(module)+ assert handle.length_varies and handle.plan is None+ hidden = torch.randn(1, 16, 8)+ try:+ with torch.inference_mode():+ txt, mod = torch.randn(1, 9, 8), torch.randn(1, 16, 8)+ statics = handle._statics([hidden, txt, mod])+ handle.orig(*statics, return_dict = False)+ frames = counter.frame_count+ handle._sealed_scopes.add(cg._image_scope((hidden,), {}))+ got = module(hidden, txt, mod, return_dict = False)[0]+ assert handle.stats["length_eager"] == 1 and handle.stats["captures"] == 0+ # Before: the block recompiled for an inference ``mod``.+ assert counter.frame_count == frames+ assert seen == [False, False]+ assert torch.equal(got, (hidden + txt.mean()) * (1 + mod) + mod)+ finally:+ cg.uninstall_all([handle])+++def test_eager_as_recorded_runs_the_original_call_when_the_copies_do_not_fit(monkeypatch):+ torch = pytest.importorskip("torch")+ import torch.nn as nn++ seen: list = []++ class LenDiT(nn.Module):+ def forward(+ self,+ hidden_states,+ encoder_hidden_states,+ return_dict = True,+ ):+ seen.append(encoder_hidden_states.is_inference())+ return (hidden_states + encoder_hidden_states.mean(),)++ module = type("Flux3Transformer2DModel", (LenDiT,), {})()+ handle = _armed(module)++ def no_room(tensors):+ raise torch.cuda.OutOfMemoryError("no room")++ monkeypatch.setattr(handle, "_statics", no_room)+ try:+ with torch.inference_mode():+ hidden, txt = torch.randn(1, 4, 8), torch.randn(1, 3, 8)+ got = handle._eager_as_recorded((hidden, txt), {"return_dict": False})[0]+ assert seen == [True]+ assert torch.equal(got, hidden + txt.mean())+ finally:+ cg.uninstall_all([handle])