Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: record CUDA graphs thread-locally and keep other threads' syncs out of a recording (#14964)

Daniel Han9dfd0451b6c9
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index cce596d2bd6..e4fbd8def1c 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -813,13 +813,19 @@ def _time_rocm_flash_block(
for _ in range(1 if long else 2):
for fn in arms.values():
block(fn)
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(device)
for _ in range(3 if long else 5):
for name, fn in arms.items():
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(device)
t0 = time.perf_counter()
block(fn)
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(device)
times[name].append((time.perf_counter() - t0) * 1000.0)
med = {name: sorted(v)[len(v) // 2] for name, v in times.items()}
return {
diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.py
index bf0f06097f9..d1309ca6e42 100644
--- a/studio/backend/core/inference/diffusion_bg_compile.py
+++ b/studio/backend/core/inference/diffusion_bg_compile.py
@@ -302,7 +302,8 @@ class BackgroundCompile:
with mode:
self.module(*args, **kwargs)
if torch.cuda.is_available():
- torch.cuda.synchronize()
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture()
self.compile_s = time.perf_counter() - t0
self._finish("done")
if self.logger is not None:
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index bd467675906..14a4887f494 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -633,7 +633,9 @@ class CfgBatchRouter:
def _sync(self, latent: Any) -> bool:
if getattr(latent.device, "type", None) != "cuda":
return False
- _torch().cuda.synchronize(latent.device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(latent.device)
return True
def _measured_plain(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
diff --git a/studio/backend/core/inference/diffusion_comfy_block.py b/studio/backend/core/inference/diffusion_comfy_block.py
index ce1931b8492..c7c7f15d249 100644
--- a/studio/backend/core/inference/diffusion_comfy_block.py
+++ b/studio/backend/core/inference/diffusion_comfy_block.py
@@ -368,8 +368,11 @@ def mxfp8_runtime_reason(target: Any) -> Optional[str]:
if index not in _MX_PROBE:
try:
with torch.cuda.device(index):
- x = torch.randn(128, 64, device = "cuda", dtype = torch.bfloat16)
- w = torch.randn(32, 64, device = "cuda", dtype = torch.bfloat16)
+ # An explicit CPU generator: up to torch 2.11 a default-generator draw raises while another thread
+ # records, and a CPU draw moved to the device needs no CUDA generator (absent on a spoofed / CPU-only torch).
+ gen = torch.Generator().manual_seed(0)
+ x = torch.randn(128, 64, dtype = torch.bfloat16, generator = gen).to("cuda")
+ w = torch.randn(32, 64, dtype = torch.bfloat16, generator = gen).to("cuda")
wq, wsf = mx_quantize_activation(w)
linear = mxfp8_linear_class()(64, 32, codes = wq, tiled_scale = wsf)
y = linear(x).float()
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
index 30be47ac5d8..4c8f442468d 100644
--- a/studio/backend/core/inference/diffusion_convrot_quant.py
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -884,7 +884,9 @@ def _rotq_probe(index: int, cfg: tuple) -> bool:
rq, rs = rotquant_reference(x, 256, kind)
if not (torch.equal(q, rq) and torch.equal(s, rs)):
return False
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001
return False
@@ -946,7 +948,9 @@ def _h16_probe(index: int) -> Any:
break
if not ok:
break
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
if ok:
return mode
except Exception: # noqa: BLE001
@@ -971,7 +975,9 @@ def _swiglu_probe(index: int) -> bool:
q, s = _h16_launch(gate, kind, value)
if not codes_close(q, s, rq, rs):
return False
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001
return False
@@ -1062,9 +1068,15 @@ def _unlisted_latency(
given, else the GEMM alone (a lower bound)."""
import torch
from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+ from .diffusion_cuda_graph import CAPTURE_MODE, _capturing, synchronize_outside_capture
dev = torch.device("cuda", index)
- xs = [torch.randn(m, k, device = dev).to(torch.bfloat16) for m, k in _UNLISTED_LATENCY_SHAPES]
+ # An explicit CPU generator: a default-generator draw raises while another thread records (up to torch 2.11).
+ gen = torch.Generator().manual_seed(0)
+ xs = [
+ torch.randn(m, k, generator = gen).to(dev, torch.bfloat16)
+ for m, k in _UNLISTED_LATENCY_SHAPES
+ ]
h = build_convrot_hadamard(256, device = dev, dtype = torch.bfloat16)
def stock(x):
@@ -1082,17 +1094,18 @@ def _unlisted_latency(
for x in xs:
fn(x) # compile / autotune outside the capture
torch.cuda.current_stream(dev).wait_stream(side)
- torch.cuda.synchronize(dev)
+ synchronize_outside_capture(dev)
for name, fn in fns.items():
graph = torch.cuda.CUDAGraph()
- with torch.cuda.graph(graph):
+ # Serialised with Studio's captures and recorded thread_local, like every other capture.
+ with _capturing(), torch.cuda.graph(graph, capture_error_mode = CAPTURE_MODE):
for _ in range(reps):
for x in xs:
fn(x)
graphs[name] = graph
for graph in graphs.values():
graph.replay()
- torch.cuda.synchronize(dev)
+ synchronize_outside_capture(dev)
start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
best = {name: float("inf") for name in graphs}
for _ in range(7):
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index b8ea363a293..adc752f7759 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -10,6 +10,7 @@ Not a hook: hooks move ``nn.Module._call_impl`` off its fast path and offload ow
from __future__ import annotations
+import collections
import contextlib
import gc
import inspect
@@ -126,25 +127,26 @@ def _disabled_reason() -> str:
return f"disabled by {CUDA_GRAPH_DISABLE_ENV}"
-# ``torch.cuda.graph`` begins its capture in CUDA's default global mode, which prohibits the
-# "potentially unsafe" calls -- cudaEventQuery among them -- from EVERY thread for as long as it
-# records. The denoise progress poller queries a CUDA event ten times a second and the denoiser
-# captures its graphs during the first steps of a render, so the two really do overlap.
-#
-# A flag read before the query would be a TOCTOU: the reader can see "no capture", the capture can
-# begin, and the query then lands inside it and invalidates it. So the lock has to span the CUDA
-# call itself, not just the depth. Capture raises the depth UNDER the lock before it enters
-# ``torch.cuda.graph``, and a querier holds the same lock ACROSS its query, which leaves only two
-# orderings and both are safe: either the querier holds the lock and capture entry waits the few
-# microseconds a query takes, or capture got there first and the querier sees a non-zero depth and
-# skips. The depth is not a flag because nothing says two pipelines cannot be capturing at once,
-# and neither holds the lock while it records, so captures do not serialise against each other.
+# ``thread_local`` capture mode: global mode lets any thread's unsafe call break the recording. A device sync or a
+# second capture still breaks it, so captures are mutually exclusive and syncs use ``synchronize_outside_capture``.
+# ``hold_off_capture`` holds the lock ACROSS the call (a flag read first is a TOCTOU); depth is raised under the lock
+# and is a count because one thread's captures can nest.
_CAPTURE_LOCK = threading.Lock()
_CAPTURE_DEPTH = 0
# CFG-parallel replica work is counted from queueing until its inputs drop; a capture waits for the count to drain.
_CAPTURE_IDLE = threading.Condition(_CAPTURE_LOCK)
_OUTSIDE_JOBS = 0
-_OUTSIDE_LOCAL = threading.local()
+
+
+class _ThreadGateState(threading.local):
+ """Per-thread gate counters; class defaults keep hot-path reads cheap."""
+
+ depth = 0
+ recording = 0
+ lock_held = 0
+
+
+_OUTSIDE_LOCAL = _ThreadGateState()
# A count that never drains (dead worker) must not hang captures: after this long the capture is refused and the step
# runs eager. Symmetrically, replica work that waits this long on a recording is refused and its branch runs on the source.
OUTSIDE_WAIT_S = 120.0
@@ -181,7 +183,7 @@ def capture_refused_now() -> bool:
def _wait_outside_drained(own: int) -> None:
- """Under ``_CAPTURE_IDLE``: wait for other threads' held work to drain (a global capture forbids its CUDA calls), else raise ``CaptureRefused``."""
+ """Under ``_CAPTURE_IDLE``: wait for other threads' held work to drain, else raise ``CaptureRefused``."""
drained = _other_holds(own, _OUTSIDE_PARKED[0]) <= 0
if not drained and not _OUTSIDE_STUCK[0]:
if own:
@@ -223,12 +225,16 @@ def _capturing():
"""
global _CAPTURE_DEPTH
own = getattr(_OUTSIDE_LOCAL, "depth", 0)
+ mine = getattr(_OUTSIDE_LOCAL, "recording", 0)
seen = _retired_pools_seen()
with _CAPTURE_IDLE:
for _ in range(_ENTRY_SETTLE_TRIES):
+ _wait_recordings_ended(mine)
if seen is not None:
_settle_retired_pools(seen)
_wait_outside_drained(own)
+ if _CAPTURE_DEPTH - mine > 0:
+ continue # a capture entered while this one waited (the lock was released): wait it out again
# Retirement takes _RETIRE_LOCK, so one is either counted before this check (flush again, still under
# the capture lock) or lands once the depth is up (due for the next capture): never in between.
with _RETIRE_LOCK:
@@ -238,14 +244,121 @@ def _capturing():
seen = _RETIRED_POOLS[0]
else:
raise CaptureRefused(
- "graph pools kept retiring while this capture entered; capture deferred"
+ "graph pools kept retiring (or captures kept entering) while this capture entered; capture deferred"
)
+ _OUTSIDE_LOCAL.recording = mine + 1
try:
yield
finally:
+ _OUTSIDE_LOCAL.recording = mine
with _CAPTURE_IDLE:
_CAPTURE_DEPTH -= 1
_CAPTURE_IDLE.notify_all()
+ if _DEFERRED_RELEASES and not mine:
+ drain_deferred_releases()
+
+
+def _wait_recordings_ended(mine: int) -> None:
+ """Under ``_CAPTURE_IDLE``: wait for other threads' recordings, else raise ``CaptureRefused``."""
+ if _CAPTURE_DEPTH - mine <= 0:
+ return
+ if not _CAPTURE_IDLE.wait_for(lambda: _CAPTURE_DEPTH - mine <= 0, timeout = RETIRED_FLUSH_WAIT_S):
+ raise CaptureRefused(
+ f"another thread's CUDA graph capture still recording after {RETIRED_FLUSH_WAIT_S:g}s; capture deferred"
+ )
+
+
+def recording_here() -> bool:
+ """Whether THIS thread is inside ``_capturing``."""
+ return getattr(_OUTSIDE_LOCAL, "recording", 0) > 0
+
+
+def _sync_device(torch: Any, device: Any) -> None:
+ if device is None:
+ torch.cuda.synchronize()
+ else:
+ torch.cuda.synchronize(device)
+
+
+def synchronize_outside_capture(device: Any = None, *, refuse: bool = False) -> None:
+ """Device sync that waits out another thread's recording and holds captures off while it runs.
+
+ After ``RETIRED_FLUSH_WAIT_S`` it raises ``CaptureRefused`` if ``refuse`` (nothing recorded yet), else syncs anyway.
+ Under ``hold_off_capture`` or this thread's own recording it syncs at once, so it never self-deadlocks.
+ Not for finalizers: one can run while its thread holds the (non-reentrant) capture lock."""
+ torch = _torch()
+ if _OUTSIDE_LOCAL.recording or _OUTSIDE_LOCAL.lock_held:
+ _sync_device(torch, device)
+ return
+ global _OUTSIDE_JOBS
+ with _CAPTURE_IDLE:
+ # Check and count atomically, so no capture slips in between.
+ ended = _CAPTURE_DEPTH == 0 or _CAPTURE_IDLE.wait_for(
+ lambda: _CAPTURE_DEPTH == 0, timeout = RETIRED_FLUSH_WAIT_S
+ )
+ if not ended and refuse:
+ raise CaptureRefused(
+ f"another thread's CUDA graph capture still recording after {RETIRED_FLUSH_WAIT_S:g}s; capture deferred"
+ )
+ _OUTSIDE_JOBS += 1
+ try:
+ _sync_device(torch, device)
+ finally:
+ capture_release()
+ drain_deferred_releases()
+
+
+# Releases a finalizer could not run at once (the capture lock was taken, possibly by its own thread, or a capture was
+# recording). A deque: append / popleft run no Python code, so a finalizer firing mid-call cannot re-enter them.
+_DEFERRED_RELEASES: "collections.deque" = collections.deque()
+
+
+def run_if_capture_idle(fn: Any) -> bool:
+ """Run ``fn()`` now under the capture lock with no capture recording, else return False without waiting.
+
+ Never blocks on the capture gate, so it is safe from a weakref finalizer: one can run on a thread already holding
+ the (non-reentrant) lock, e.g. inside a ``Condition.wait_for`` setup. ``fn`` may sync and free device memory: no
+ capture can enter while it runs."""
+ if _OUTSIDE_LOCAL.recording:
+ return False
+ if _OUTSIDE_LOCAL.lock_held:
+ # Under this thread's own hold_off_capture: the lock is held here, so the depth cannot change under fn.
+ if _CAPTURE_DEPTH != 0:
+ return False
+ fn()
+ return True
+ if not _CAPTURE_LOCK.acquire(blocking = False):
+ return False
+ try:
+ if _CAPTURE_DEPTH != 0:
+ return False
+ fn()
+ return True
+ finally:
+ _CAPTURE_LOCK.release()
+
+
+def release_when_capture_idle(fn: Any) -> None:
+ """For finalizers: run ``fn`` as ``run_if_capture_idle`` does, else queue it for the next gated point (an explicit
+ ``synchronize_outside_capture`` or a capture's exit). Never blocks."""
+ if not run_if_capture_idle(fn):
+ _DEFERRED_RELEASES.append(fn)
+
+
+def drain_deferred_releases() -> None:
+ """Run the queued finalizer releases the gate allows now; the rest stay queued. Never blocks."""
+ for _ in range(len(_DEFERRED_RELEASES)):
+ try:
+ fn = _DEFERRED_RELEASES.popleft()
+ except IndexError:
+ return
+ try:
+ done = run_if_capture_idle(fn)
+ except Exception: # noqa: BLE001 - a failed release is dropped, as the finalizer would have
+ continue
+ if not done:
+ _DEFERRED_RELEASES.append(fn)
+ return
def capture_hold() -> None:
@@ -304,9 +417,14 @@ def hold_off_capture():
if not _CAPTURE_LOCK.acquire(blocking = False):
yield False
return
+ held = getattr(_OUTSIDE_LOCAL, "lock_held", 0)
+ _OUTSIDE_LOCAL.lock_held = (
+ held + 1
+ ) # synchronize_outside_capture must not re-take the lock under it
try:
yield _CAPTURE_DEPTH == 0
finally:
+ _OUTSIDE_LOCAL.lock_held = held
_CAPTURE_LOCK.release()
@@ -480,6 +598,17 @@ def capture_in_progress() -> bool:
return _CAPTURE_DEPTH > 0
+CAPTURE_MODE = "thread_local"
+
+
+def _graph_with_flush(
+ graph: Any,
+ pool: Any = None,
+ stream: Any = None,
+):
+ return _torch().cuda.graph(graph, pool = pool, stream = stream, capture_error_mode = CAPTURE_MODE)
+
+
_CAPTURE_STREAMS: dict = {}
@@ -505,13 +634,13 @@ def _graph_without_flush(
stream: Any = None,
):
"""``torch.cuda.graph`` minus its ``empty_cache`` / host ``emptyCache``: same capture stream discipline, same
- global capture mode, same pool."""
+ capture mode (``CAPTURE_MODE``), same pool."""
torch = _torch()
torch.cuda.synchronize()
if stream is None:
stream = _capture_stream(torch.cuda.current_device())
with torch.cuda.stream(stream):
- graph.capture_begin(pool = pool, capture_error_mode = "global")
+ graph.capture_begin(pool = pool, capture_error_mode = CAPTURE_MODE)
try:
yield
finally:
@@ -1834,14 +1963,21 @@ class GraphedForward:
for _ in range(min(self.warmup, 1) if again else self.warmup):
record(*static_args, **static_kwargs)
torch.cuda.current_stream().wait_stream(side)
- torch.cuda.synchronize()
+ try:
+ synchronize_outside_capture(
+ refuse = True
+ ) # a device sync: never inside another thread's recording
+ except CaptureRefused:
+ if skip_stats is not None:
+ skip.stats = skip_stats
+ raise
graph = torch.cuda.CUDAGraph()
# An explicit pool id, never None: a capture that fails must hand its pool back (retire_failed_capture).
pool = _POOL_BOX[0]
if pool is None and callable(getattr(torch.cuda, "graph_pool_handle", None)):
pool = torch.cuda.graph_pool_handle()
- recorder = _graph_without_flush if recapture else torch.cuda.graph
+ recorder = _graph_without_flush if recapture else _graph_with_flush
card = next((t.device.index for t in live if t.device.index is not None), None)
if card is None:
card = torch.cuda.current_device()
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index 0418636dd1d..19399147bfe 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -826,7 +826,9 @@ def _act_quant_device_ok(index: int) -> bool:
rq, rs = reference_act_quant(x)
if not (torch.equal(q, rq) and s.dtype == rs.dtype and torch.equal(s, rs)):
return False
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001 - any failure keeps torchao's quant
return False
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index a20df067592..d5bc5e6b526 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -452,7 +452,9 @@ def _probe(index: int, cfg: tuple) -> bool:
ws = torch.ones(w.shape[0], device = dev, dtype = torch.bfloat16)
if not torch.equal(_launch(a, w, xs, ws, None, cfg), reference(a, w, xs, ws, None)):
return False
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001 - out of shared memory, compile failure, ...
return False
diff --git a/studio/backend/core/inference/diffusion_measured_peaks.py b/studio/backend/core/inference/diffusion_measured_peaks.py
index 86008d6945d..4c48cee0536 100644
--- a/studio/backend/core/inference/diffusion_measured_peaks.py
+++ b/studio/backend/core/inference/diffusion_measured_peaks.py
@@ -562,7 +562,9 @@ class PeakProbe:
if dev.index is None:
dev = torch.device("cuda", torch.cuda.current_device())
self.window = open_window(dev)
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
torch.cuda.reset_peak_memory_stats(dev)
_reset_range_peak(dev)
self.base = int(torch.cuda.memory_allocated(dev)) + _range_bytes(dev)
@@ -594,7 +596,9 @@ class PeakProbe:
return None
try:
import torch
- torch.cuda.synchronize(self.device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(self.device)
# Stable-range bytes sit outside the allocator: count their high-water mark (not what is still mapped:
# a model-offload range is unmapped before decode ends) so the peak stays on the stock scale.
peak = (
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 04ff43c63fa..03b6bac45b4 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1180,7 +1180,9 @@ def settled_snapshot_device_memory(
return snapshot_device_memory(target)
try:
import torch
- torch.cuda.synchronize()
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture()
torch.cuda.empty_cache()
except Exception: # noqa: BLE001 - settle is best-effort; the snapshot below still runs
pass
@@ -4027,7 +4029,8 @@ def _keep_groups_resident(
if kept:
_bump_placement_epoch()
if kept and onload.type == "cuda":
- torch.cuda.synchronize(onload)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(onload)
if logger is not None and kept:
logger.info(
"diffusion.memory: %s keeps %d MiB resident (%d of %d offload groups); the rest streams",
@@ -4207,7 +4210,8 @@ def release_resident_groups(
state["streamed"] = _streamed_group_count(_offload_groups(module))
device = getattr(released[0], "_unsloth_resident_device", None)
if device is not None and torch.device(device).type == "cuda":
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(device)
if logger is not None:
logger.info(
"diffusion.memory: streaming %d MiB of resident groups for %s",
@@ -4496,7 +4500,8 @@ def _return_module_tensors(module: Any, onload: Any) -> None:
else:
t.data = t.data.to(onload)
if onload.type == "cuda":
- torch.cuda.synchronize(onload)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(onload)
def _unwind_group_offload(
diff --git a/studio/backend/core/inference/diffusion_moss_fused.py b/studio/backend/core/inference/diffusion_moss_fused.py
index d2ec270e64d..62a01c59322 100644
--- a/studio/backend/core/inference/diffusion_moss_fused.py
+++ b/studio/backend/core/inference/diffusion_moss_fused.py
@@ -393,7 +393,9 @@ def qk_rope_device_ok(index: int) -> bool:
ulp = (torch.cat((pair, pair), -1) * 2.0**-6).clamp_min(2.0**-14)
if bool((diff > ulp).any()) or float((diff > 0).float().mean()) > 5e-3:
return False
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return True
except Exception: # noqa: BLE001
return False
diff --git a/studio/backend/core/inference/diffusion_nvfp4_dispatch.py b/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
index bd4481e3671..4f9b59300a1 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
@@ -97,8 +97,11 @@ def _run_verify(device: Any) -> tuple:
try:
import flashinfer
with torch.cuda.device(device):
- x = torch.randn(256, 512, device = device, dtype = torch.bfloat16) * 0.05
- w = torch.randn(128, 512, device = device, dtype = torch.bfloat16) * 0.02
+ # An explicit CPU generator: up to torch 2.11 a default-generator draw raises while another thread
+ # records, and a CPU draw moved to the device needs no CUDA generator (absent on a spoofed / CPU-only torch).
+ gen = torch.Generator().manual_seed(0)
+ x = torch.randn(256, 512, dtype = torch.bfloat16, generator = gen).to(device) * 0.05
+ w = torch.randn(128, 512, dtype = torch.bfloat16, generator = gen).to(device) * 0.02
a_gsf = _global_scale(x)
w_gsf = _global_scale(w)
alpha = (1.0 / (a_gsf * w_gsf)).float()
@@ -142,7 +145,9 @@ def _run_verify(device: Any) -> tuple:
],
tactic = tactic,
)
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(device)
if not torch.equal(want, got):
return False, "the cached GEMM is not bit-identical to mm_fp4"
except Exception as exc: # noqa: BLE001 - a raise is a failed verify
diff --git a/studio/backend/core/inference/diffusion_nvfp4_linear.py b/studio/backend/core/inference/diffusion_nvfp4_linear.py
index a93ad64b104..2b1626c5166 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_linear.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_linear.py
@@ -328,7 +328,9 @@ def _prewarm_shapes(modules, shapes, *, logger, tuned_box) -> None:
x = torch.zeros(m, module.in_features, device = device, dtype = torch.bfloat16)
with flashinfer.autotune(True):
module(x)
- torch.cuda.synchronize(device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(device)
except Exception as exc: # noqa: BLE001 - a shape that will not tune still runs
_log(logger, "debug", f"[nvfp4] prewarm {key} failed ({type(exc).__name__}: {exc})")
continue
diff --git a/studio/backend/core/inference/diffusion_nvfp4_ops.py b/studio/backend/core/inference/diffusion_nvfp4_ops.py
index 91fdd11008c..02c1610bb22 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_ops.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_ops.py
@@ -394,8 +394,11 @@ def _preflight_probe(dev: Any) -> bool:
import flashinfer
import torch
with torch.cuda.device(dev):
- x = torch.randn(128, 256, device = dev, dtype = torch.bfloat16) * 0.05
- w = torch.randn(128, 256, device = dev, dtype = torch.bfloat16) * 0.02
+ # An explicit CPU generator: up to torch 2.11 a default-generator draw raises while another thread
+ # records, and a CPU draw moved to the device needs no CUDA generator (absent on a spoofed / CPU-only torch).
+ gen = torch.Generator().manual_seed(0)
+ x = torch.randn(128, 256, dtype = torch.bfloat16, generator = gen).to(dev) * 0.05
+ w = torch.randn(128, 256, dtype = torch.bfloat16, generator = gen).to(dev) * 0.02
a_gsf, w_gsf = global_scale(x), global_scale(w)
xq, x_sf = flashinfer.nvfp4_quantize(x, a_gsf, do_shuffle = False)
wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
@@ -410,7 +413,9 @@ def _preflight_probe(dev: Any) -> bool:
out = out,
backend = DEFAULT_MM_BACKEND,
)
- torch.cuda.synchronize(dev)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(dev)
return bool(torch.isfinite(y).all().item())
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index b2860e6c21a..3f2c408575c 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -672,7 +672,8 @@ class GroupPrefetcher:
if self.active:
return
if self.slot_raw:
- torch.cuda.synchronize(self.device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(self.device)
bump_placement_epoch()
self.slot_of = {}
self.slot_buffers = {}
@@ -745,7 +746,8 @@ class GroupPrefetcher:
self._staging().clear() # nothing is staged under capture; the synchronize below lands the rest
self.staged_bytes = 0
try:
- torch.cuda.synchronize(self.device)
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(self.device)
except Exception: # noqa: BLE001 - the copies below are what matters
pass
for group in self.groups:
diff --git a/studio/backend/core/inference/diffusion_stable_arena.py b/studio/backend/core/inference/diffusion_stable_arena.py
index 3fc0ac2d57d..cd8c7b7f779 100644
--- a/studio/backend/core/inference/diffusion_stable_arena.py
+++ b/studio/backend/core/inference/diffusion_stable_arena.py
@@ -301,14 +301,47 @@ class VmmArena:
return
lib = _driver()
with torch.cuda.device(self.device_index):
- torch.cuda.synchronize(self.device_index)
- self._base = None
- _check(lib.cuMemUnmap(self.va, self.size), "cuMemUnmap")
- _check(lib.cuMemRelease(self._handle), "cuMemRelease")
+ from .diffusion_cuda_graph import synchronize_outside_capture
+ synchronize_outside_capture(self.device_index)
+ self._release_synced(lib)
+
+ def _release_synced(self, lib: Any) -> None:
+ """Unmap and release, the card already synchronized with no capture recording."""
+ self._base = None
+ _check(lib.cuMemUnmap(self.va, self.size), "cuMemUnmap")
+ _check(lib.cuMemRelease(self._handle), "cuMemRelease")
self._handle = None
_count_mapped(self.device_index, -self.size)
self.unmaps += 1
+ def _close_idle(self) -> None:
+ """``close`` for ``release_when_capture_idle``: run under the capture lock with no capture recording."""
+ import torch
+
+ if self.closed:
+ return
+ try:
+ if self._handle is not None:
+ with torch.cuda.device(self.device_index):
+ torch.cuda.synchronize(self.device_index)
+ self._release_synced(_driver())
+ finally:
+ self.closed = True
+ try:
+ _driver().cuMemAddressFree(self.va, self.size)
+ except Exception: # noqa: BLE001 - process teardown
+ pass
+
+ def close_from_finalizer(self) -> None:
+ """``close`` for weakref finalizers: never blocks on the capture gate (the finalizer's thread may hold its
+ lock). Runs now when no capture records and the lock is free, else at the next gated point; the memory is
+ freed only after a device sync with no capture recording, as ``unmap`` does."""
+ if self.closed:
+ return
+ from .diffusion_cuda_graph import release_when_capture_idle
+
+ release_when_capture_idle(self._close_idle)
+
def close(self) -> None:
if self.closed:
return
@@ -419,7 +452,7 @@ class ModuleArena:
arena = state.get("arena")
if arena is not None:
try:
- arena.close()
+ arena.close_from_finalizer()
except Exception: # noqa: BLE001 - process teardown
pass
@@ -798,7 +831,7 @@ class GroupRange:
def _close_ranges(ranges: list) -> None:
for rng in ranges:
try:
- rng.arena.close()
+ rng.arena.close_from_finalizer()
except Exception: # noqa: BLE001 - process teardown
pass
diff --git a/studio/backend/tests/test_diffusion_capture_serialises.py b/studio/backend/tests/test_diffusion_capture_serialises.py
new file mode 100644
index 00000000000..c11b74310fb
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_capture_serialises.py
@@ -0,0 +1,462 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A CUDA graph recording on one Studio thread survives the CUDA work every other Studio thread does meanwhile.
+
+Measured (torch 2.14, CUDA 13): while a global-mode capture records, another thread's event query, host read, pinned
+alloc, ``empty_cache``, stream sync or first cuBLAS / cuDNN call invalidates it (and raises in that thread); in
+thread_local mode only a device-wide synchronize does, so captures record thread_local, exclude each other, and
+Studio's device syncs wait for another thread's recording."""
+
+from __future__ import annotations
+
+import threading
+import time
+
+import pytest
+import torch
+
+import core.inference.diffusion_cuda_graph as cg
+
+_CUDA = torch.cuda.is_available()
+# Long enough that the other thread's call lands inside the recording; short, because the fixed code holds it out.
+WINDOW_S = 1.0
+
+
[email protected](autouse = True)
+def _fresh_gate(monkeypatch):
+ monkeypatch.setattr(cg, "_FAILED_GRAPHS", [])
+ monkeypatch.setattr(cg, "_POOL_BOX", [None])
+ monkeypatch.setattr(cg, "_OUTSIDE_STUCK", [False])
+ yield
+ assert cg._CAPTURE_DEPTH == 0 and cg._OUTSIDE_JOBS == 0
+
+
+class _Step(torch.nn.Module):
+ """A denoiser step; while it records it opens ``window`` and holds the recording open until ``done`` (or 1 s)."""
+
+ def __init__(
+ self,
+ window = None,
+ done = None,
+ ) -> None:
+ super().__init__()
+ torch.manual_seed(0)
+ self.lin = torch.nn.Linear(64, 64)
+ self.window, self.done = window, done
+
+ def forward(
+ self,
+ x,
+ return_dict = True,
+ ):
+ y = self.lin(x)
+ if self.window is not None and torch.cuda.is_current_stream_capturing():
+ self.window.set()
+ self.done.wait(WINDOW_S)
+ y = torch.nn.functional.gelu(y) * 2
+ return (y,) if return_dict is False else y
+
+
+def _run_in_thread(fn):
+ box = {}
+
+ def target():
+ try:
+ torch.cuda.set_device(0)
+ with torch.inference_mode():
+ box["out"] = fn()
+ except BaseException as exc: # noqa: BLE001
+ box["error"] = f"{type(exc).__name__}: {exc}"
+
+ thread = threading.Thread(target = target, name = "studio-other-thread")
+ return thread, box
+
+
+def _record_while(other):
+ """Record ``_Step`` on this thread while ``other`` runs on a fresh thread inside the recording window."""
+ window, done = threading.Event(), threading.Event()
+ step = _Step(window, done).cuda()
+ handle = cg.GraphedForward(step).enable()
+ x = torch.randn(8, 64, device = "cuda")
+
+ def gated():
+ assert window.wait(300) # first CUDA use can take minutes on a loaded host
+ try:
+ return other()
+ finally:
+ done.set()
+
+ thread, box = _run_in_thread(gated)
+ thread.start()
+ try:
+ with torch.inference_mode():
+ first = step(x, return_dict = False)[0].clone()
+ again = step(x, return_dict = False)[0].clone()
+ finally:
+ done.set()
+ thread.join(60)
+ return step, handle, x, first, again, box
+
+
+def _expect_graph_kept(step, handle, x, first, again):
+ try:
+ assert handle.capture_error is None, handle.capture_error
+ assert not handle.poisoned
+ assert handle.stats["captures"] == 1 and handle.stats["replays"] >= 1, handle.stats
+ with torch.inference_mode():
+ ref = torch.nn.functional.gelu(step.lin(x)) * 2
+ assert torch.equal(first, ref) and torch.equal(again, ref)
+ finally:
+ handle.free()
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_a_recording_survives_another_threads_decode_tail():
+ """Another thread's decode (first cuBLAS/cuDNN use, host read, event query, pinned alloc) during a recording.
+
+ Its noise comes from an explicit generator, as every Studio render's does (see the default-generator test below)."""
+
+ def decode_tail():
+ gen = torch.Generator(device = "cuda").manual_seed(0)
+ w = torch.randn(64, 64, device = "cuda", generator = gen)
+ img = torch.nn.functional.conv2d(
+ torch.randn(1, 4, 32, 32, device = "cuda", generator = gen),
+ torch.randn(3, 4, 3, 3, device = "cuda", generator = gen),
+ )
+ host = (img.float().clamp(-1, 1) * 127.5 + 127.5).to(torch.uint8).cpu()
+ torch.cuda.Event().query()
+ torch.empty(1 << 16).pin_memory()
+ return (w @ w).sum().item(), host.shape
+
+ step, handle, x, first, again, box = _record_while(decode_tail)
+ assert "error" not in box, box["error"]
+ _expect_graph_kept(step, handle, x, first, again)
+
+
+def _torch_at_least(major: int, minor: int) -> bool:
+ return tuple(int(p) for p in torch.__version__.split("+")[0].split(".")[:2]) >= (major, minor)
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_another_threads_default_generator_draw_during_a_recording():
+ """Up to torch 2.11 a capture flags the device's DEFAULT generator as capturing for every thread, so another
+ thread's ``torch.randn(device = "cuda")`` raises "Offset increment outside graph capture encountered
+ unexpectedly" in THAT thread; the recording itself is kept. torch 2.12 keys the RNG capture state by capture id
+ (pytorch#176752), so the draw just runs. Studio draws its noise from explicit generators for that reason."""
+
+ def default_draw():
+ return torch.randn(64, device = "cuda").sum().item()
+
+ step, handle, x, first, again, box = _record_while(default_draw)
+ if _torch_at_least(2, 12):
+ assert "error" not in box, box["error"]
+ else:
+ assert "Offset increment outside graph capture" in box.get("error", ""), box
+ _expect_graph_kept(step, handle, x, first, again)
+ # The default generator still draws once the recording ended.
+ torch.randn(4, device = "cuda").sum().item()
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_two_threads_recording_at_once_both_keep_their_graphs():
+ """A second capture's entry waits out the first's recording instead of breaking it."""
+ other = _Step().cuda()
+ other_handle = cg.GraphedForward(other).enable()
+ y = torch.randn(8, 64, device = "cuda")
+
+ def second_capture():
+ out = other(y, return_dict = False)[0].clone()
+ return out, other(y, return_dict = False)[0].clone()
+
+ step, handle, x, first, again, box = _record_while(second_capture)
+ assert "error" not in box, box["error"]
+ try:
+ assert other_handle.capture_error is None, other_handle.capture_error
+ assert other_handle.stats["captures"] == 1 and other_handle.stats["replays"] >= 1
+ with torch.inference_mode():
+ ref = torch.nn.functional.gelu(other.lin(y)) * 2
+ assert all(torch.equal(t, ref) for t in box["out"])
+ finally:
+ other_handle.free()
+ _expect_graph_kept(step, handle, x, first, again)
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_a_studio_device_sync_on_another_thread_waits_for_the_recording():
+ """A device-wide synchronize breaks a recording in any capture mode: Studio's sync waits the recording out."""
+ seen = {}
+
+ def studio_sync():
+ t0 = time.perf_counter()
+ cg.synchronize_outside_capture(torch.device("cuda", 0))
+ seen["waited_s"] = time.perf_counter() - t0
+
+ step, handle, x, first, again, box = _record_while(studio_sync)
+ assert "error" not in box, box["error"]
+ _expect_graph_kept(step, handle, x, first, again)
+ assert seen["waited_s"] >= WINDOW_S * 0.5 # held out until the recording ended
+
+
+def test_the_whole_step_records_thread_local():
+ assert cg.CAPTURE_MODE == "thread_local"
+
+
+def _hold_capture_open(entered, release):
+ def target():
+ with cg._capturing():
+ entered.set()
+ release.wait(10)
+
+ return threading.Thread(target = target)
+
+
+def test_a_capture_waits_for_another_threads_recording():
+ entered, release = threading.Event(), threading.Event()
+ first = _hold_capture_open(entered, release)
+ first.start()
+ assert entered.wait(10)
+ second_in = threading.Event()
+
+ def second():
+ with cg._capturing():
+ second_in.set()
+
+ t = threading.Thread(target = second)
+ t.start()
+ assert not second_in.wait(0.3) # held out while the first records
+ release.set()
+ first.join(10)
+ assert second_in.wait(10)
+ t.join(10)
+
+
+def test_a_threads_own_nested_capture_does_not_wait_on_itself():
+ with cg._capturing():
+ assert cg.recording_here()
+ with cg._capturing():
+ assert cg._CAPTURE_DEPTH == 2
+ assert not cg.recording_here()
+
+
+def test_a_capture_held_out_too_long_is_refused_not_recorded(monkeypatch):
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.2)
+ entered, release = threading.Event(), threading.Event()
+ first = _hold_capture_open(entered, release)
+ first.start()
+ assert entered.wait(10)
+ try:
+ with pytest.raises(cg.CaptureRefused):
+ with cg._capturing():
+ pass
+ finally:
+ release.set()
+ first.join(10)
+
+
+def test_synchronize_outside_capture_waits_and_holds_captures_off(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ torch.cuda,
+ "synchronize",
+ lambda device = None: calls.append((device, cg.capture_in_progress(), cg._OUTSIDE_JOBS)),
+ )
+ entered, release = threading.Event(), threading.Event()
+ first = _hold_capture_open(entered, release)
+ first.start()
+ assert entered.wait(10)
+ t = threading.Thread(target = cg.synchronize_outside_capture, args = ("cuda:0",))
+ t.start()
+ time.sleep(0.3)
+ assert calls == [] # waiting for the recording to end
+ release.set()
+ first.join(10)
+ t.join(10)
+ # ran with no recording, and counted, so no capture could enter during it
+ assert calls == [("cuda:0", False, 1)]
+
+
+def test_synchronize_inside_this_threads_recording_does_not_wait_on_itself(monkeypatch):
+ calls = []
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: calls.append(device))
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 30.0)
+ t0 = time.perf_counter()
+ with cg._capturing():
+ cg.synchronize_outside_capture()
+ assert calls == [None] and time.perf_counter() - t0 < 5
+
+
+def test_synchronize_under_hold_off_capture_does_not_retake_the_lock(monkeypatch):
+ """The ring drop syncs under ``hold_off_capture`` (the lock is not reentrant): that sync runs straight away."""
+ calls = []
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: calls.append(device))
+ done = threading.Event()
+
+ def ring_drop():
+ with cg.hold_off_capture() as safe:
+ assert safe
+ cg.synchronize_outside_capture("cuda:0")
+ done.set()
+
+ t = threading.Thread(target = ring_drop, daemon = True)
+ t.start()
+ assert done.wait(5), "deadlocked on the capture lock"
+ t.join(5)
+ assert calls == ["cuda:0"]
+
+
+def test_a_captures_own_sync_is_refused_but_a_plain_sync_still_runs_after_the_wait(monkeypatch):
+ """Past the wait a capture's pre-record sync refuses; any other sync still runs."""
+ calls = []
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: calls.append(device))
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.2)
+ entered, release = threading.Event(), threading.Event()
+ first = _hold_capture_open(entered, release)
+ first.start()
+ assert entered.wait(10)
+ try:
+ with pytest.raises(cg.CaptureRefused):
+ cg.synchronize_outside_capture(refuse = True)
+ assert calls == []
+ cg.synchronize_outside_capture("cuda:0")
+ assert calls == ["cuda:0"]
+ finally:
+ release.set()
+ first.join(10)
+
+
+class _FakeDriver:
+ def __init__(self, calls):
+ self.calls = calls
+
+ def cuMemUnmap(self, va, size):
+ self.calls.append("unmap")
+ return 0
+
+ def cuMemRelease(self, handle):
+ self.calls.append("release")
+ return 0
+
+ def cuMemAddressFree(self, va, size):
+ self.calls.append("free")
+ return 0
+
+
+def _fake_arena(monkeypatch, calls):
+ """A mapped ``VmmArena`` over a fake driver, so the finalizer path runs on any host."""
+ import contextlib
+
+ import core.inference.diffusion_stable_arena as sa
+
+ monkeypatch.setattr(sa, "_driver", lambda: _FakeDriver(calls))
+ monkeypatch.setattr(torch.cuda, "device", lambda _index: contextlib.nullcontext())
+ monkeypatch.setattr(
+ torch.cuda,
+ "synchronize",
+ lambda device = None: calls.append(
+ ("sync", cg._CAPTURE_DEPTH > 0)
+ ), # lock-free: may run under it
+ )
+ arena = object.__new__(sa.VmmArena)
+ arena.size, arena.device_index, arena.va = 1 << 21, 0, 1 << 40
+ arena._handle, arena._base = 7, object()
+ arena.maps, arena.unmaps, arena.closed = 1, 0, False
+ sa._count_mapped(0, arena.size)
+ return sa, arena
+
+
+def _on_thread_with_timeout(fn, timeout = 5.0):
+ """Run ``fn`` on a fresh thread: a deadlock fails the test instead of hanging it."""
+ box = {}
+
+ def target():
+ try:
+ box["out"] = fn()
+ except BaseException as exc: # noqa: BLE001
+ box["error"] = exc
+
+ t = threading.Thread(target = target, daemon = True)
+ t.start()
+ t.join(timeout)
+ assert not t.is_alive(), "deadlocked on the capture lock"
+ if "error" in box:
+ raise box["error"]
+ return box.get("out")
+
+
+def test_an_arena_finalizer_under_this_threads_capture_lock_defers_not_deadlocks(monkeypatch):
+ """GC can run an arena finalizer on a thread inside the gate's own ``Condition`` setup (lock held, no flag set):
+ it must queue the release, not re-take the non-reentrant lock; the next gated sync frees it after a device sync."""
+ monkeypatch.setattr(cg, "_DEFERRED_RELEASES", __import__("collections").deque(), raising = False)
+ calls = []
+ sa, arena = _fake_arena(monkeypatch, calls)
+
+ def finalizer_under_lock():
+ with cg._CAPTURE_IDLE: # what synchronize_outside_capture holds around wait_for
+ sa.ModuleArena._finalize({"arena": arena})
+ assert calls == [] and not arena.closed and arena.mapped
+ assert len(cg._DEFERRED_RELEASES) == 1
+
+ _on_thread_with_timeout(finalizer_under_lock)
+ _on_thread_with_timeout(lambda: cg.synchronize_outside_capture("cuda:0"))
+ assert arena.closed and not arena.mapped and len(cg._DEFERRED_RELEASES) == 0
+ # the gated sync, then the deferred close: its own sync with no capture recording, before the memory goes
+ assert calls == [("sync", False), ("sync", False), "unmap", "release", "free"]
+
+
+def test_an_arena_finalizer_during_another_threads_recording_waits_for_the_capture_exit(
+ monkeypatch,
+):
+ monkeypatch.setattr(cg, "_DEFERRED_RELEASES", __import__("collections").deque(), raising = False)
+ calls = []
+ sa, arena = _fake_arena(monkeypatch, calls)
+ entered, release = threading.Event(), threading.Event()
+ first = _hold_capture_open(entered, release)
+ first.start()
+ assert entered.wait(10)
+ try:
+ _on_thread_with_timeout(lambda: sa._close_ranges([types_ns(arena = arena)]))
+ assert calls == [] and arena.mapped # never freed under a recording
+ finally:
+ release.set()
+ first.join(10)
+ assert not first.is_alive()
+ # drained by the capture's exit, with no capture recording
+ assert arena.closed and not arena.mapped
+ assert calls == [("sync", False), "unmap", "release", "free"]
+
+
+def test_an_arena_finalizer_with_the_gate_free_closes_at_once(monkeypatch):
+ monkeypatch.setattr(cg, "_DEFERRED_RELEASES", __import__("collections").deque(), raising = False)
+ calls = []
+ sa, arena = _fake_arena(monkeypatch, calls)
+ _on_thread_with_timeout(lambda: sa.ModuleArena._finalize({"arena": arena}))
+ assert arena.closed and len(cg._DEFERRED_RELEASES) == 0
+ assert calls == [("sync", False), "unmap", "release", "free"]
+
+
+def types_ns(**kw):
+ import types
+ return types.SimpleNamespace(**kw)
+
+
[email protected](not _CUDA, reason = "needs CUDA")
+def test_a_real_arena_finalizer_under_the_capture_lock_frees_on_the_next_gated_sync():
+ import core.inference.diffusion_stable_arena as sa
+
+ torch.cuda.set_device(0)
+ module = torch.nn.Linear(8, 8)
+ marena = sa.ModuleArena(module, "cuda:0")
+ module.to("cuda:0")
+ arena = marena.arena
+ assert arena is not None and arena.mapped
+
+ def finalizer_under_lock():
+ with cg._CAPTURE_IDLE:
+ sa.ModuleArena._finalize({"arena": arena})
+ return arena.mapped
+
+ assert _on_thread_with_timeout(finalizer_under_lock) is True
+ _on_thread_with_timeout(cg.synchronize_outside_capture)
+ assert arena.closed and not arena.mapped
+ del module, marena
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index 34cb8748862..43290bac6c4 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -141,8 +141,10 @@ def _build_stub_torch():
graph,
pool = None,
stream = None,
+ capture_error_mode = "global",
):
records["graphs"].append((graph, pool))
+ records["capture_mode"] = capture_error_mode
records["capture_streams"].append(stream)
yield
diff --git a/studio/backend/tests/test_diffusion_nvfp4_speed.py b/studio/backend/tests/test_diffusion_nvfp4_speed.py
index bd9ec695eb2..72cc1933ae3 100644
--- a/studio/backend/tests/test_diffusion_nvfp4_speed.py
+++ b/studio/backend/tests/test_diffusion_nvfp4_speed.py
@@ -175,6 +175,12 @@ def _fake_torch():
torch.randn = _fake_alloc("randn")
torch.isfinite = lambda t: t
+ class _Generator:
+ def manual_seed(self, _seed):
+ return self
+
+ torch.Generator = _Generator
+
def _device(kind, index = None):
if isinstance(kind, _FakeDevice):
return kind
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 07945e250ed..416e9a9dbd0 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2207,7 +2207,13 @@ def clear_gpu_cache():
if torch.cuda.is_available() and any(
torch.cuda.memory_reserved(i) for i in range(torch.cuda.device_count())
):
- torch.cuda.synchronize()
+ # An unload beside another backend's render: a device sync must not land inside its graph recording.
+ sync = torch.cuda.synchronize
+ try:
+ from core.inference.diffusion_cuda_graph import synchronize_outside_capture as sync
+ except Exception: # noqa: BLE001 - no Studio diffusion backend here: a plain sync
+ pass
+ sync()
# Both are no-ops without an allocator and neither creates a context.
torch.cuda.empty_cache()
torch.cuda.ipc_collect()