Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: record each card's CUDA graphs on its own capture stream, so compiled renders on a second GPU are correct (#14915)
Daniel Hanabe4383f3164
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex b0da1a3bd89..8bd05a52197 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -480,18 +480,36 @@ def capture_in_progress() -> bool:return _CAPTURE_DEPTH > 0+_CAPTURE_STREAMS: dict = {}+++def _capture_stream(index: int) -> Any:+ """One capture stream per card: torch's default capture stream lives on the first card that captured, so a+ capture for another card recorded none of its kernels. Pool blocks are reused only from the recording stream."""+ torch = _torch()+ stream = _CAPTURE_STREAMS.get(index)+ if stream is None:+ default = getattr(torch.cuda.graph, "default_capture_stream", None)+ if default is not None and getattr(default, "device", None) == torch.device("cuda", index):+ stream = default+ else:+ stream = torch.cuda.Stream(device = index)+ stream = _CAPTURE_STREAMS.setdefault(index, stream)+ return stream++@contextlib.contextmanager-def _graph_without_flush(graph: Any, pool: Any = None):+def _graph_without_flush(+ graph: Any,+ pool: Any = None,+ stream: Any = None,+):"""``torch.cuda.graph`` minus its ``empty_cache`` / host ``emptyCache``: same capture stream discipline, sameglobal capture mode, same pool."""torch = _torch()torch.cuda.synchronize()- # torch.cuda.graph's own capture stream: pool blocks are reused only from the stream that recorded them.- stream = getattr(torch.cuda.graph, "default_capture_stream", None)- if stream is None or getattr(stream, "device", None) != torch.device(- "cuda", torch.cuda.current_device()- ):- stream = torch.cuda.Stream()+ if stream is None:+ stream = _capture_stream(torch.cuda.current_device())with torch.cuda.stream(stream):graph.capture_begin(pool = pool, capture_error_mode = "global")try:@@ -1803,9 +1821,12 @@ class GraphedForward:if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):pool = torch.cuda.graph_pool_handle()recorder = _graph_without_flush if recapture else torch.cuda.graph+ card = next((t.device.index for t in live if t.device.index is not None), None)+ if card is None:+ card = torch.cuda.current_device()try:# _capturing hands back any retired pool first, so the no-flush re-record never records beside one.- with _capturing(), recorder(graph, pool = pool):+ with _capturing(), recorder(graph, pool = pool, stream = _capture_stream(card)):out = record(*static_args, **static_kwargs)except BaseException as exc:# A refusal is raised before the recording begins: the graph and the pool were never touched.diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.pyindex d94460d4789..34cb8748862 100644--- a/studio/backend/tests/test_diffusion_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_cuda_graph.py@@ -88,7 +88,8 @@ class _FakeGraph:class _FakeStream:- def __init__(self):+ def __init__(self, device = None):+ self.device = deviceself.waited: list = []def wait_stream(self, other):@@ -105,6 +106,7 @@ def _build_stub_torch():"empty_cache": 0,"inference_mode": [],"graph_error": None,+ "capture_streams": [],}torch._records = records@@ -135,8 +137,13 @@ def _build_stub_torch():return _FakeGraph()@contextlib.contextmanager- def _graph(graph, pool = None):+ def _graph(+ graph,+ pool = None,+ stream = None,+ ):records["graphs"].append((graph, pool))+ records["capture_streams"].append(stream)yield@contextlib.contextmanager@@ -198,9 +205,11 @@ def _clean_module_globals():"""The shared pool and the live-wrapper set are process-wide, so no test may inherit them."""cg._POOL_BOX[0] = Nonecg._LIVE_WRAPPERS.clear()+ cg._CAPTURE_STREAMS.clear()yieldcg._POOL_BOX[0] = Nonecg._LIVE_WRAPPERS.clear()+ cg._CAPTURE_STREAMS.clear()def _t(shape = (1, 4), **kwargs):@@ -288,6 +297,21 @@ def test_capture_then_replay_uses_one_graph_and_copies_statics(stub_torch):assert stub_torch._records["inference_mode"] == [False]+def test_each_card_records_on_its_own_capture_stream(stub_torch):+ """torch's default capture stream lives on the first card that captured; each card needs its own."""+ first = _armed()+ first(_t(device_index = 0), timestep = _t((1,), device_index = 0), return_dict = False)+ second = _armed()+ second(_t(device_index = 1), timestep = _t((1,), device_index = 1), return_dict = False)+ third = _armed()+ third(_t(device_index = 1), timestep = _t((1,), device_index = 1), return_dict = False)+ streams = stub_torch._records["capture_streams"]+ assert [s.device for s in streams] == [0, 1, 1]+ assert (+ streams[1] is streams[2]+ ), "one stream per card: pool blocks are reused only from the recording stream"++def test_new_shape_adds_a_second_graph(stub_torch):handle = _armed()handle(_t((1, 4)), timestep = _t((1,)), return_dict = False)diff --git a/studio/backend/tests/test_diffusion_multigpu_cuda.py b/studio/backend/tests/test_diffusion_multigpu_cuda.pyindex 02b839acf27..0da7f902d0a 100644--- a/studio/backend/tests/test_diffusion_multigpu_cuda.py+++ b/studio/backend/tests/test_diffusion_multigpu_cuda.py@@ -271,3 +271,34 @@ def test_an_in_place_edit_of_inference_weights_rebuilds_the_replica():torch.cuda.synchronize()assert router.stats["replica_rebuilds"] == 1assert torch.equal(out, reference)+++def test_a_step_graph_on_the_second_card_records_its_forward():+ # Regression: cuda:1 captured on torch's cuda:0 default capture stream, so replays returned the recording's output.+ from core.inference import diffusion_cuda_graph as cg++ def graphed_steps(card):+ torch.manual_seed(0)+ dit = _Dit().cuda(card)+ handle = cg.GraphedForward(dit).enable()+ outs, refs = [], []+ with torch.cuda.device(card), torch.inference_mode():+ for value in (1.0, 2.0, 3.0):+ x = torch.full((2, 4), value, device = f"cuda:{card}")+ t = torch.tensor(value, device = f"cuda:{card}")+ e = torch.randn(3, 4, device = f"cuda:{card}")+ refs.append(dit.__class__.forward(dit, x, t, e)[0].clone())+ outs.append(+ handle(hidden_states = x, timestep = t, encoder_hidden_states = e, return_dict = False)[+ 0+ ]+ )+ torch.cuda.synchronize()+ return handle, outs, refs++ first, outs0, refs0 = graphed_steps(0)+ second, outs1, refs1 = graphed_steps(1)+ for handle in (first, second):+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 3, handle.stats+ for out, ref in zip(outs0 + outs1, refs0 + refs1):+ assert torch.equal(out.cpu(), ref.cpu())