Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: record each card's CUDA graphs on its own capture stream, so compiled renders on a second GPU are correct (#14915)

Daniel Hanabe4383f3164
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index b0da1a3bd89..8bd05a52197 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -480,18 +480,36 @@ def capture_in_progress() -> bool:
return _CAPTURE_DEPTH > 0
+_CAPTURE_STREAMS: dict = {}
+
+
+def _capture_stream(index: int) -> Any:
+ """One capture stream per card: torch's default capture stream lives on the first card that captured, so a
+ capture for another card recorded none of its kernels. Pool blocks are reused only from the recording stream."""
+ torch = _torch()
+ stream = _CAPTURE_STREAMS.get(index)
+ if stream is None:
+ default = getattr(torch.cuda.graph, "default_capture_stream", None)
+ if default is not None and getattr(default, "device", None) == torch.device("cuda", index):
+ stream = default
+ else:
+ stream = torch.cuda.Stream(device = index)
+ stream = _CAPTURE_STREAMS.setdefault(index, stream)
+ return stream
+
+
@contextlib.contextmanager
-def _graph_without_flush(graph: Any, pool: Any = None):
+def _graph_without_flush(
+ graph: Any,
+ pool: Any = None,
+ stream: Any = None,
+):
"""``torch.cuda.graph`` minus its ``empty_cache`` / host ``emptyCache``: same capture stream discipline, same
global capture mode, same pool."""
torch = _torch()
torch.cuda.synchronize()
- # torch.cuda.graph's own capture stream: pool blocks are reused only from the stream that recorded them.
- stream = getattr(torch.cuda.graph, "default_capture_stream", None)
- if stream is None or getattr(stream, "device", None) != torch.device(
- "cuda", torch.cuda.current_device()
- ):
- stream = torch.cuda.Stream()
+ if stream is None:
+ stream = _capture_stream(torch.cuda.current_device())
with torch.cuda.stream(stream):
graph.capture_begin(pool = pool, capture_error_mode = "global")
try:
@@ -1803,9 +1821,12 @@ class GraphedForward:
if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):
pool = torch.cuda.graph_pool_handle()
recorder = _graph_without_flush if recapture else torch.cuda.graph
+ card = next((t.device.index for t in live if t.device.index is not None), None)
+ if card is None:
+ card = torch.cuda.current_device()
try:
# _capturing hands back any retired pool first, so the no-flush re-record never records beside one.
- with _capturing(), recorder(graph, pool = pool):
+ with _capturing(), recorder(graph, pool = pool, stream = _capture_stream(card)):
out = record(*static_args, **static_kwargs)
except BaseException as exc:
# A refusal is raised before the recording begins: the graph and the pool were never touched.
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index d94460d4789..34cb8748862 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -88,7 +88,8 @@ class _FakeGraph:
class _FakeStream:
- def __init__(self):
+ def __init__(self, device = None):
+ self.device = device
self.waited: list = []
def wait_stream(self, other):
@@ -105,6 +106,7 @@ def _build_stub_torch():
"empty_cache": 0,
"inference_mode": [],
"graph_error": None,
+ "capture_streams": [],
}
torch._records = records
@@ -135,8 +137,13 @@ def _build_stub_torch():
return _FakeGraph()
@contextlib.contextmanager
- def _graph(graph, pool = None):
+ def _graph(
+ graph,
+ pool = None,
+ stream = None,
+ ):
records["graphs"].append((graph, pool))
+ records["capture_streams"].append(stream)
yield
@contextlib.contextmanager
@@ -198,9 +205,11 @@ def _clean_module_globals():
"""The shared pool and the live-wrapper set are process-wide, so no test may inherit them."""
cg._POOL_BOX[0] = None
cg._LIVE_WRAPPERS.clear()
+ cg._CAPTURE_STREAMS.clear()
yield
cg._POOL_BOX[0] = None
cg._LIVE_WRAPPERS.clear()
+ cg._CAPTURE_STREAMS.clear()
def _t(shape = (1, 4), **kwargs):
@@ -288,6 +297,21 @@ def test_capture_then_replay_uses_one_graph_and_copies_statics(stub_torch):
assert stub_torch._records["inference_mode"] == [False]
+def test_each_card_records_on_its_own_capture_stream(stub_torch):
+ """torch's default capture stream lives on the first card that captured; each card needs its own."""
+ first = _armed()
+ first(_t(device_index = 0), timestep = _t((1,), device_index = 0), return_dict = False)
+ second = _armed()
+ second(_t(device_index = 1), timestep = _t((1,), device_index = 1), return_dict = False)
+ third = _armed()
+ third(_t(device_index = 1), timestep = _t((1,), device_index = 1), return_dict = False)
+ streams = stub_torch._records["capture_streams"]
+ assert [s.device for s in streams] == [0, 1, 1]
+ assert (
+ streams[1] is streams[2]
+ ), "one stream per card: pool blocks are reused only from the recording stream"
+
+
def test_new_shape_adds_a_second_graph(stub_torch):
handle = _armed()
handle(_t((1, 4)), timestep = _t((1,)), return_dict = False)
diff --git a/studio/backend/tests/test_diffusion_multigpu_cuda.py b/studio/backend/tests/test_diffusion_multigpu_cuda.py
index 02b839acf27..0da7f902d0a 100644
--- a/studio/backend/tests/test_diffusion_multigpu_cuda.py
+++ b/studio/backend/tests/test_diffusion_multigpu_cuda.py
@@ -271,3 +271,34 @@ def test_an_in_place_edit_of_inference_weights_rebuilds_the_replica():
torch.cuda.synchronize()
assert router.stats["replica_rebuilds"] == 1
assert torch.equal(out, reference)
+
+
+def test_a_step_graph_on_the_second_card_records_its_forward():
+ # Regression: cuda:1 captured on torch's cuda:0 default capture stream, so replays returned the recording's output.
+ from core.inference import diffusion_cuda_graph as cg
+
+ def graphed_steps(card):
+ torch.manual_seed(0)
+ dit = _Dit().cuda(card)
+ handle = cg.GraphedForward(dit).enable()
+ outs, refs = [], []
+ with torch.cuda.device(card), torch.inference_mode():
+ for value in (1.0, 2.0, 3.0):
+ x = torch.full((2, 4), value, device = f"cuda:{card}")
+ t = torch.tensor(value, device = f"cuda:{card}")
+ e = torch.randn(3, 4, device = f"cuda:{card}")
+ refs.append(dit.__class__.forward(dit, x, t, e)[0].clone())
+ outs.append(
+ handle(hidden_states = x, timestep = t, encoder_hidden_states = e, return_dict = False)[
+ 0
+ ]
+ )
+ torch.cuda.synchronize()
+ return handle, outs, refs
+
+ first, outs0, refs0 = graphed_steps(0)
+ second, outs1, refs1 = graphed_steps(1)
+ for handle in (first, second):
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] == 3, handle.stats
+ for out, ref in zip(outs0 + outs1, refs0 + refs1):
+ assert torch.equal(out.cpu(), ref.cpu())