Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: stop flushing the CUDA allocator on every prompt encode under block offload (#14777)

Daniel Han9f5010c423c3
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index 80302c9bd08..02e006e59fd 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -249,7 +249,7 @@ _UNSEEN = object()
class _Entry:
- __slots__ = ("graph", "static_in", "static_out", "out_spec", "replays", "slots")
+ __slots__ = ("graph", "static_in", "static_out", "out_spec", "replays", "slots", "pool")
import weakref
@@ -607,8 +607,9 @@ class BlockGraph:
graph = torch.cuda.CUDAGraph()
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
- before = torch.cuda.memory_reserved()
with _capturing():
+ # read after the entry, which first hands back any retired graph pool
+ before = torch.cuda.memory_reserved()
try:
# Inside the try: a capture_begin that raised after taking the pool is abandoned too.
graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
@@ -642,6 +643,7 @@ class BlockGraph:
entry.slots = (slot, out_slot)
self.shared.hold(entry.slots)
entry.graph = graph
+ entry.pool = pool
entry.out_spec = out_spec
entry.replays = 0
if any(k[0] == full[0] for k in self.cache):
@@ -656,8 +658,7 @@ class BlockGraph:
note_capture(self._unsloth_capture_device)
_LIVE_BLOCKS.add(self)
while len(self.cache) > self.max_graphs:
- _, evicted = self.cache.popitem(last = False)
- self._forget(evicted)
+ _drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)
self.stats["evictions"] += 1
return entry
@@ -666,9 +667,9 @@ class BlockGraph:
self.shared.drop(getattr(entry, "slots", ()))
def _forget_all(self) -> None:
- for entry in self.cache.values():
- self._forget(entry)
+ entries = list(self.cache.values())
self.cache.clear()
+ _drop_entries(self, entries)
def _replay(self, entry: _Entry, live: list) -> Any:
for dst, src in zip(entry.static_in, live):
@@ -722,7 +723,7 @@ class _SegRecord:
class _SegEntry:
- __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays")
+ __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays", "pool")
def _root_key_extra(root: Any) -> Any:
@@ -815,9 +816,9 @@ class SegmentGraph:
self.versions: list = []
def reset(self) -> None:
- for entry in self.cache.values():
- self._forget(entry)
+ entries = list(self.cache.values())
self.cache.clear()
+ _drop_entries(self, entries)
self.seen.clear()
self.refused.clear()
self._reset_forward()
@@ -1049,8 +1050,9 @@ class SegmentGraph:
graph = torch.cuda.CUDAGraph()
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
- before = torch.cuda.memory_reserved()
with _capturing():
+ # read after the entry, which first hands back any retired graph pool
+ before = torch.cuda.memory_reserved()
try:
graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
try:
@@ -1081,6 +1083,7 @@ class SegmentGraph:
)
return None
entry.graph = graph
+ entry.pool = pool
entry.outputs = outputs
entry.record = rec
entry.slots = (slot,)
@@ -1093,8 +1096,7 @@ class SegmentGraph:
self._unsloth_capture_device = capture_device()
note_capture(self._unsloth_capture_device)
while len(self.cache) > self.max_graphs:
- _, evicted = self.cache.popitem(last = False)
- self._forget(evicted)
+ _drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)
return entry
@@ -1413,14 +1415,37 @@ class BlockGraphSet:
return int(sum(st.pool_bytes + st.static_bytes for st in self._all_shared()) + slots)
-def _release_cached() -> None:
+def _drop_entries(
+ owner: Any,
+ entries: list,
+ evicting: bool = False,
+) -> None:
+ """The one way a block or segment graph drops recordings (eviction, poison, reset): forget them, then count their
+ pool as retired, so no capture records beside its cached segments before a flush. An eviction from the current
+ pool is not counted (the entry just recorded into it keeps it alive); any other drop is."""
+ if not entries:
+ return
+ for entry in entries:
+ owner._forget(entry)
+ current = getattr(getattr(owner, "shared", None), "pool", None)
+ if (
+ evicting
+ and current is not None
+ and all(getattr(e, "pool", None) == current for e in entries)
+ ):
+ return
try:
- import gc
+ from .diffusion_cuda_graph import note_pool_retired
+ note_pool_retired()
+ except Exception: # noqa: BLE001
+ pass
- gc.collect()
- torch = _torch()
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
+
+def _release_cached() -> None:
+ try:
+ from .diffusion_cuda_graph import release_cached_outside_capture
+ if _torch().cuda.is_available():
+ release_cached_outside_capture()
except Exception: # noqa: BLE001
pass
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 1a319ba973c..8dc6fed8d92 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -217,12 +217,29 @@ def _capturing():
The depth MUST be raised before ``torch.cuda.graph`` is entered, which is what makes the
ordering above hold; keep this the outermost of the two context managers. Raises
``CaptureRefused`` (depth untouched, nothing recorded) when replica-side work did not drain in time.
+
+ Every capture (whole step, planned or offloaded re-record, per block, per segment) enters here, so this is also
+ where a retired graph pool is handed back before anything records (``_settle_retired_pools``).
"""
global _CAPTURE_DEPTH
own = getattr(_OUTSIDE_LOCAL, "depth", 0)
+ seen = _retired_pools_seen()
with _CAPTURE_IDLE:
- _wait_outside_drained(own)
- _CAPTURE_DEPTH += 1
+ for _ in range(_ENTRY_SETTLE_TRIES):
+ if seen is not None:
+ _settle_retired_pools(seen)
+ _wait_outside_drained(own)
+ # Retirement takes _RETIRE_LOCK, so one is either counted before this check (flush again, still under
+ # the capture lock) or lands once the depth is up (due for the next capture): never in between.
+ with _RETIRE_LOCK:
+ if _RETIRED_POOLS[1] >= _RETIRED_POOLS[0]:
+ _CAPTURE_DEPTH += 1
+ break
+ seen = _RETIRED_POOLS[0]
+ else:
+ raise CaptureRefused(
+ "graph pools kept retiring while this capture entered; capture deferred"
+ )
try:
yield
finally:
@@ -293,6 +310,149 @@ def hold_off_capture():
_CAPTURE_LOCK.release()
+# A retired pool's segments stay cached until empty_cache, which cannot run while a capture records, so a capture
+# beside them can OOM. Rule: no capture records until every pool retired before its entry was flushed. Counted, not
+# flagged, so a retirement landing mid-flush stays due.
+_RETIRE_LOCK = threading.Lock()
+_RETIRED_POOLS = [0, 0] # [retirements counted, retirements flushed]
+_POOL_FLUSHES = [0] # retired-pool flushes performed (diagnostics, tests)
+# A deferred release owes a cuBLAS workspace clear; the next entry flush does it.
+_CUBLAS_CLEAR_DUE = [False]
+# Wait (not eager fallback) for another thread's recording to end: eager CUDA work mid-capture can invalidate it.
+RETIRED_FLUSH_WAIT_S = OUTSIDE_WAIT_S
+# Entry flush attempts before the capture is refused.
+_ENTRY_SETTLE_TRIES = 8
+
+
+def note_pool_retired() -> None:
+ """Count a retired graph pool. Call once the graphs that held it are dropped, never before."""
+ with _RETIRE_LOCK:
+ _RETIRED_POOLS[0] += 1
+
+
+def retired_pool_due() -> bool:
+ with _RETIRE_LOCK:
+ return _RETIRED_POOLS[0] != _RETIRED_POOLS[1]
+
+
+def _retired_pools_seen() -> Optional[int]:
+ """The retirement count a flush starting now covers, None when nothing is due. Read BEFORE ``gc.collect``, so
+ every graph counted is already unreferenced (or in a cycle that collect frees) when the flush runs."""
+ with _RETIRE_LOCK:
+ seen = _RETIRED_POOLS[0]
+ if seen == _RETIRED_POOLS[1]:
+ return None
+ gc.collect() # outside _CAPTURE_LOCK: a finalizer may take it
+ return seen
+
+
+def _flush_retired_locked(seen: int) -> bool:
+ """Under ``_CAPTURE_LOCK`` with no capture recording: synchronize, clear the cuBLAS workspaces a deferred release
+ owed, flush, acknowledge up to ``seen``."""
+ with _RETIRE_LOCK:
+ clear_cublas, _CUBLAS_CLEAR_DUE[0] = _CUBLAS_CLEAR_DUE[0], False
+ try:
+ torch = _torch()
+ cuda = getattr(torch, "cuda", None)
+ if cuda is not None and cuda.is_available():
+ sync = getattr(cuda, "synchronize", None)
+ if callable(sync):
+ sync()
+ if clear_cublas:
+ clear = getattr(getattr(torch, "_C", None), "_cuda_clearCublasWorkspaces", None)
+ if callable(clear):
+ clear()
+ empty = getattr(cuda, "empty_cache", None)
+ if callable(empty):
+ empty()
+ except Exception: # noqa: BLE001 - not flushed: stays due, the cuBLAS clear with it
+ if clear_cublas:
+ with _RETIRE_LOCK:
+ _CUBLAS_CLEAR_DUE[0] = True
+ return False
+ with _RETIRE_LOCK:
+ _RETIRED_POOLS[1] = max(_RETIRED_POOLS[1], seen)
+ _POOL_FLUSHES[0] += 1
+ return True
+
+
+def _settle_retired_pools(seen: int) -> None:
+ """Under ``_CAPTURE_IDLE``, from ``_capturing``: flush the pools retired up to ``seen`` before this capture records.
+ Waits (lock released) for another thread's recording to end, since that one prohibits the flush; raises
+ ``CaptureRefused`` (depth untouched, nothing recorded) when it does not end in time or the flush failed."""
+ with _RETIRE_LOCK:
+ if _RETIRED_POOLS[1] >= seen:
+ return # another capture flushed them already: no wait on its recording
+ if not _CAPTURE_IDLE.wait_for(lambda: _CAPTURE_DEPTH == 0, timeout = RETIRED_FLUSH_WAIT_S):
+ raise CaptureRefused(
+ "a retired graph pool could not be flushed (another capture is recording); capture deferred"
+ )
+ with _RETIRE_LOCK:
+ if _RETIRED_POOLS[1] >= seen:
+ return # another capture flushed them while this one waited
+ if not _flush_retired_locked(seen):
+ raise CaptureRefused("a retired graph pool could not be flushed; capture deferred")
+
+
+def flush_retired_pools() -> bool:
+ """Best effort, outside any capture: hand retired pools back now rather than at the next capture's entry. False
+ (still due) while another thread records."""
+ seen = _retired_pools_seen()
+ if seen is None:
+ return True
+ with hold_off_capture() as safe:
+ if not safe:
+ return False
+ return _flush_retired_locked(seen)
+
+
+def flush_now_locked(clear_cublas: bool = False) -> bool:
+ """For a caller already holding ``hold_off_capture()`` (True): flush now, and acknowledge every retirement counted
+ before it, so the next capture's entry does not flush again."""
+ with _RETIRE_LOCK:
+ seen = _RETIRED_POOLS[0]
+ if clear_cublas:
+ _CUBLAS_CLEAR_DUE[0] = True
+ gc.collect()
+ return _flush_retired_locked(seen)
+
+
+def release_cached_outside_capture(clear_cublas: bool = False) -> bool:
+ """The one way graph memory goes back outside a capture's entry: ``empty_cache`` (and the cuBLAS workspaces) now,
+ under the capture lock with no capture recording, acknowledging the retirements counted before it; or, while
+ another thread records (either call is prohibited inside its recording and can invalidate it), counted as a
+ retired pool with the cuBLAS clear owed, so the next capture's entry flush does both."""
+ # Owed first, in one step with the cuBLAS debt: a capture entering from here on sees it due, whether or not this
+ # flush gets the lock; a successful flush below acknowledges it.
+ with _RETIRE_LOCK:
+ _RETIRED_POOLS[0] += 1
+ seen = _RETIRED_POOLS[0]
+ if clear_cublas:
+ _CUBLAS_CLEAR_DUE[0] = True
+ gc.collect() # outside _CAPTURE_LOCK: a finalizer may take it
+ with hold_off_capture() as safe:
+ if safe and _flush_retired_locked(seen):
+ return True
+ return False
+
+
+def _note_dropped_pools(tokens: list) -> None:
+ """Graph entries just left a step cache: count a retirement when one of their pools no live graph records into
+ (a pool still shared with a live graph keeps its freed blocks for that pool's next capture)."""
+ try:
+ if not tokens:
+ return
+ live = [
+ getattr(entry, "pool_token", None)
+ for wrapper in tuple(_LIVE_WRAPPERS)
+ for entry in tuple(wrapper.cache.values())
+ ]
+ if any(token is None or token not in live for token in tokens):
+ note_pool_retired()
+ except Exception: # noqa: BLE001 - unsure: count it, the cost is one flush
+ note_pool_retired()
+
+
# Raised while a whole step warms up or records: per-block graphs then run their compute, not nested replays.
_STEP_RECORDING = [0]
@@ -545,6 +705,8 @@ def retire_failed_capture(
(_COLLIDED_GRAPHS if begun_elsewhere else _FAILED_GRAPHS).append(graph)
if pool is not None and _POOL_BOX[0] == pool:
_POOL_BOX[0] = None
+ if not begun_elsewhere:
+ note_pool_retired() # whatever the failed recording took stays cached until the next capture's entry flush
def _nvfp4_flashinfer_linears(module: Any) -> list:
@@ -985,23 +1147,15 @@ class GraphedForward:
def _release(self) -> None:
"""Return the dropped graphs' pool segments to the device; ``gc.collect`` alone leaves them in the private pool."""
- try:
- gc.collect()
- torch = _torch()
- cuda = getattr(torch, "cuda", None)
- if getattr(self, "placement", None) is not None:
- # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them
- clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)
- if callable(clear):
- clear()
- if cuda is not None and hasattr(cuda, "empty_cache"):
- cuda.empty_cache()
- except Exception: # noqa: BLE001 - best effort
- pass
+ # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them
+ release_cached_outside_capture(clear_cublas = getattr(self, "placement", None) is not None)
def reset(self) -> "GraphedForward":
"""Drop every captured graph (weights changed under us: LoRA load, unload, adapter switch)."""
+ dropped = bool(self.cache)
self.cache.clear()
+ if dropped:
+ note_pool_retired() # counted where it drops; the _release below flushes and acknowledges it
self._recorded.clear()
# Nothing recorded is left to replay: every scope's first render starts over (captures counted afresh).
self._sealed_scopes.clear()
@@ -1027,7 +1181,9 @@ class GraphedForward:
# entry is now unreachable and only pins its statics, outputs and slice of the pool for the
# life of the load. Freeing is left to the caller's ``_release``, which runs once the failed
# capture's own frames are gone, so both go back in one pass.
+ dropped = [getattr(e, "pool_token", None) for e in self.cache.values()]
self.cache.clear()
+ _note_dropped_pools(dropped)
_drop_pool_if_unused()
return self
@@ -1130,11 +1286,15 @@ class GraphedForward:
type(self.module).__name__,
reason,
)
- if self.cache.pop(key, None) is not None:
+ dropped_entry = self.cache.pop(key, None)
+ if dropped_entry is not None:
try:
_torch().cuda.current_stream().synchronize() # no replay still running on what is freed
except Exception: # noqa: BLE001
pass
+ gone = getattr(dropped_entry, "pool_token", None)
+ del dropped_entry # counted once the graph is gone, never before
+ _note_dropped_pools([gone])
_drop_pool_if_unused()
if not self.cache:
self.capture_error = {"type": "Refused", "msg": reason}
@@ -1268,7 +1428,9 @@ class GraphedForward:
self.stats["invalidations"] += 1
# Not reset(): its empty_cache on every model-offload onload makes the next onload pay the
# cudaMallocs again. The dropped pool's blocks stay with the allocator.
+ dropped = [getattr(e, "pool_token", None) for e in self.cache.values()]
self.cache.clear()
+ _note_dropped_pools(dropped) # the next capture flushes it before it records
_drop_pool_if_unused() # the next capture must not record into a pool no graph holds
self._warmed.clear()
self.valid_token = token
@@ -1388,8 +1550,9 @@ class GraphedForward:
return call(*warm_args, **warm_kwargs)
if len(self.cache) >= self.max_graphs and self.plan is not None:
# A planned call keys on the prompt length, so evict the least recently replayed graph.
- self.cache.pop(next(iter(self.cache)))
+ gone = getattr(self.cache.pop(next(iter(self.cache))), "pool_token", None)
self.stats["evictions"] += 1
+ _note_dropped_pools([gone])
# The last graph on the shared pool takes the pool with it; never hand its id to the next capture.
_drop_pool_if_unused()
if len(self.cache) >= self.max_graphs:
@@ -1639,6 +1802,7 @@ class GraphedForward:
pool = torch.cuda.graph_pool_handle()
recorder = _graph_without_flush if recapture else torch.cuda.graph
try:
+ # _capturing hands back any retired pool first, so the no-flush re-record never records beside one.
with _capturing(), recorder(graph, pool = pool):
out = record(*static_args, **static_kwargs)
except BaseException as exc:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index ffae0f233e8..57883d7982b 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4189,10 +4189,8 @@ def install_encode_release(
restore = release_resident_groups(
owner, surplus, logger, denoisers_only = True, reason = "the prompt encode"
)
+ # No empty_cache: the allocator frees cached segments and retries on OOM; flushing per prompt cost a sync.
pending.append(restore)
- if restore is not None and torch.cuda.is_available():
- # freed blocks sit in the default stream's pool; the encoder onloads on the copy stream
- torch.cuda.empty_cache()
def _after(module: Any, args: Any, output: Any) -> None:
if not pending:
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 69ead084a71..1774c2396f4 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -620,9 +620,7 @@ class GroupPrefetcher:
self._drop_ring()
def _drop_ring(self) -> None:
- import torch
-
- from .diffusion_cuda_graph import hold_off_capture
+ from .diffusion_cuda_graph import flush_now_locked, hold_off_capture
with hold_off_capture() as safe:
if not safe:
# Another thread is recording: its capture prohibits these syncs and frees. Retry after a later forward.
@@ -630,16 +628,9 @@ class GroupPrefetcher:
return
self._drop_slots_at_end = False
self.disable_slots()
- try:
- import gc
-
- gc.collect() # a declined step graph's recordings, whose pool is flushed below with the ring
- clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)
- if callable(clear):
- clear() # and the workspace cuBLAS made for their capture stream
- torch.cuda.empty_cache() # the ring's segments go back to the device, not just to the cache
- except Exception: # noqa: BLE001
- pass
+ # The ring's segments go back to the device, not just to the cache, with a declined step graph's
+ # recordings (their pool, and the workspace cuBLAS made for their capture stream): acknowledged as flushed.
+ flush_now_locked(clear_cublas = True)
def disable_slots(self) -> None:
"""Drop the ring. Only between forwards: every streamed group is back on the host by then."""
diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.py
index 05260c524d4..f089def3f92 100644
--- a/studio/backend/tests/test_diffusion_block_graph.py
+++ b/studio/backend/tests/test_diffusion_block_graph.py
@@ -863,6 +863,139 @@ def test_compiled_streamed_blocks_record_below_their_hooks_with_no_graph_break()
handle.free()
+def _reset_retired(monkeypatch):
+ if hasattr(cg, "_RETIRED_POOLS"):
+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [0, 0])
+ else:
+ monkeypatch.setattr(cg, "_RETIRED_POOL_DUE", [False], raising = False)
+
+
+def test_a_block_capture_after_a_failed_one_flushes_the_retired_pool_first(monkeypatch):
+ """A failed per-block capture retires its pool. When it cannot be flushed then (another thread started recording,
+ which prohibits the flush), the pool stays due: the next capture hands it back before it records, so it never
+ records beside the retired segments. Every block still runs its compute exactly."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ net = _net(blocks = 2).cuda()
+ ref = copy.deepcopy(net)
+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)
+ monkeypatch.setattr(cg, "_FAILED_GRAPHS", [])
+ events = []
+ real_flush = torch.cuda.empty_cache
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: events.append("flush") or real_flush())
+ real_begin = torch.cuda.CUDAGraph.capture_begin
+ other = {"recording": False}
+
+ def begin(self, *args, **kwargs):
+ if "fail" not in events:
+ events.append("fail")
+ cg._CAPTURE_DEPTH += 1 # another render thread starts recording as this capture fails
+ other["recording"] = True
+ raise RuntimeError("capture failed")
+ events.append("begin")
+ return real_begin(self, *args, **kwargs)
+
+ real_refuse = cg.refuse_if_exhausted
+
+ def refuse(*args, **kwargs):
+ if other["recording"]: # ... and is done before the next block records
+ other["recording"] = False
+ cg._CAPTURE_DEPTH -= 1
+ return real_refuse(*args, **kwargs)
+
+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_begin", begin)
+ monkeypatch.setattr(cg, "refuse_if_exhausted", refuse)
+ x = torch.randn(7, 16, device = "cuda")
+ try:
+ with torch.inference_mode():
+ for _ in range(4):
+ assert torch.equal(net(x), ref(x))
+ finally:
+ if other["recording"]:
+ cg._CAPTURE_DEPTH -= 1
+ start = events.index("fail")
+ assert "begin" in events[start:]
+ assert "flush" in events[start : events.index("begin", start)]
+ if hasattr(cg, "retired_pool_due"):
+ assert not cg.retired_pool_due()
+ handle.free()
+
+
+def test_a_block_poisoned_before_capture_counts_the_graphs_it_dropped(monkeypatch):
+ """A block with recorded layouts that fails before capture_begin on a new one (budget, warm-up, statics) is
+ poisoned and drops every graph it held: their pool counts as retired, so the next capture flushes it first."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ net = _net(blocks = 2).cuda()
+ ref = copy.deepcopy(net)
+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)
+ x = torch.randn(7, 16, device = "cuda")
+ with torch.inference_mode():
+ for _ in range(4):
+ assert torch.equal(net(x), ref(x))
+ held = sum(len(g.cache) for g in handle.graphs)
+ assert held >= 1
+
+ def refuse(*args, **kwargs):
+ raise RuntimeError("no room for the statics")
+
+ monkeypatch.setattr(cg, "refuse_if_exhausted", refuse)
+ y = torch.randn(5, 16, device = "cuda")
+ with torch.inference_mode():
+ for _ in range(4):
+ assert torch.equal(net(y), ref(y))
+ assert any(g.poisoned for g in handle.graphs)
+ due = cg.retired_pool_due() if hasattr(cg, "retired_pool_due") else False
+ assert due
+ handle.free()
+
+
+def test_evicting_a_block_graph_from_a_retired_pool_counts_it(monkeypatch):
+ """After a failed capture replaced the shared pool, older recordings still hold the old one. When the LRU evicts
+ the last of them, that pool retires: it is counted, so the next capture flushes it before recording beside it."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ net = _net(blocks = 1).cuda()
+ ref = copy.deepcopy(net)
+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)
+ block = handle.graphs[0]
+ limit = block.max_graphs
+ with torch.inference_mode():
+ for rows in range(2, 2 + limit):
+ x = torch.randn(rows, 16, device = "cuda")
+ for _ in range(3):
+ assert torch.equal(net(x), ref(x))
+ assert len(block.cache) == limit
+ block.shared.pool = None # what retire_failed_capture leaves: later recordings get a fresh pool
+ _reset_retired(monkeypatch)
+ with torch.inference_mode():
+ x = torch.randn(2 + limit, 16, device = "cuda")
+ for _ in range(3):
+ assert torch.equal(net(x), ref(x))
+ assert block.stats["evictions"] >= 1
+ due = cg.retired_pool_due() if hasattr(cg, "retired_pool_due") else False
+ assert due
+ handle.free()
+
+
+def test_a_block_capture_refused_before_recording_flushes_nothing(monkeypatch):
+ """Past the failure budget every block fails before capture_begin: no pool was retired, so no allocator flush."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ net = _net(blocks = 2).cuda()
+ ref = copy.deepcopy(net)
+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)
+ flushes = cg._POOL_FLUSHES[0]
+ monkeypatch.setattr(cg, "captures_exhausted", lambda: True)
+ x = torch.randn(7, 16, device = "cuda")
+ with torch.inference_mode():
+ for _ in range(3):
+ assert torch.equal(net(x), ref(x))
+ assert handle.stats["fallbacks"] + handle.stats.get("segment_fallbacks", 0) >= 1
+ assert cg._POOL_FLUSHES[0] == flushes and not cg.retired_pool_due()
+ handle.free()
+
+
def test_alternating_layouts_both_record():
"""True CFG calls each block with two text lengths in turn; each layout must still record and replay."""
_cuda()
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index 4559f6c9f2b..145b676a9a3 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -807,6 +807,29 @@ def test_encode_streams_the_whole_resident_denoiser_back_to_the_flat_room(monkey
assert torch.equal(net(x.cuda()).cpu(), ref)
+def test_encode_release_does_not_flush_the_allocator_per_prompt(monkeypatch):
+ """The encode release streams groups and pins them back on every new prompt. It leaves the freed blocks cached:
+ an encoder allocation that needs the room gets it back from the allocator itself. An empty_cache per prompt cost a
+ device sync plus fresh cudaMallocs for the encoder on every render."""
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+ block_mib = 4
+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 2 * block_mib), None) == 1
+ flushes = {"n": 0}
+ real = torch.cuda.empty_cache
+
+ def counted():
+ flushes["n"] += 1
+ real()
+
+ monkeypatch.setattr(torch.cuda, "empty_cache", counted)
+ for _ in range(3):
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2 # the release still ran
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert flushes["n"] == 0
+
+
def test_encode_release_restores_after_a_failed_encode(monkeypatch):
torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
dm.install_encode_release(pipe, _whole_plan(1024, 0), None)
diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
index b13d1d47050..13b5897543f 100644
--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
@@ -167,6 +167,394 @@ def test_release_and_restore_records_again_and_stays_exact():
assert handles[0].stats["invalidations"] >= 1 and handles[0].stats["captures"] >= 2
+def _reset_retired(monkeypatch):
+ if hasattr(cg, "_RETIRED_POOLS"):
+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [0, 0])
+ else:
+ monkeypatch.setattr(cg, "_RETIRED_POOL_DUE", [False], raising = False)
+
+
+def _retire():
+ """Another wrapper's invalidation retired a pool."""
+ if hasattr(cg, "note_pool_retired"):
+ cg.note_pool_retired()
+ else:
+ cg._RETIRED_POOL_DUE[0] = True
+
+
+def _due():
+ if hasattr(cg, "retired_pool_due"):
+ return cg.retired_pool_due()
+ return cg._RETIRED_POOL_DUE[0]
+
+
+def test_an_offloaded_re_record_hands_the_dropped_pool_back_first(monkeypatch):
+ """Each new prompt encode moves the resident groups (release, the encoder takes the room, re-pin elsewhere), so
+ the step records again. The encode release no longer flushes the allocator; the re-record itself hands the
+ dropped recording's pool back before it records, so the reserved memory stays flat instead of growing by a pool
+ per prompt, and every output stays exact."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ seeds = list(range(6))
+ want = _eager_reference(3, seeds)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ pipe = _pipe(net)
+ handles, _ = cg.arm_after_placement(pipe)
+ got = _run(net, seeds[:2])
+ flushes = cg._POOL_FLUSHES[0]
+ reserved = []
+ for cycle in range(4):
+ restore = dm.release_resident_groups(pipe, 10_000, None)
+ hold = torch.empty(
+ 16 << 20, dtype = torch.uint8, device = "cuda"
+ ) # the encoder takes the freed room
+ restore()
+ del hold
+ got += _run(net, seeds[2 + cycle : 3 + cycle])
+ reserved.append(torch.cuda.memory_reserved())
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ records = handles[0].stats["captures"]
+ assert records >= 2 # the step recorded again after the moves
+ assert cg._POOL_FLUSHES[0] - flushes >= records - 1
+ assert reserved[-1] <= reserved[0]
+ cg.uninstall_all(handles)
+
+
+def _two_shapes(net, shapes = (16, 8)):
+ outs = []
+ with torch.no_grad():
+ for rows in shapes:
+ g = torch.Generator(device = "cpu").manual_seed(rows)
+ x = torch.randn(rows, 64, generator = g).cuda()
+ t = torch.rand(1, generator = g).cuda()
+ outs.append(net(x, t, return_dict = False)[0].clone())
+ torch.cuda.synchronize()
+ return outs
+
+
+def test_one_invalidation_flushes_the_retired_pool_once_for_every_re_recorded_key(monkeypatch):
+ """Two keys re-recorded under one new placement share one retired pool: the first re-record flushes it, the second
+ does not flush the allocator again."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ pipe = _pipe(net)
+ handles, _ = cg.arm_after_placement(pipe)
+ _two_shapes(net)
+ _two_shapes(net)
+ assert handles[0].stats["captures"] == 2
+ flushes = cg._POOL_FLUSHES[0]
+ for cycle in range(2):
+ restore = dm.release_resident_groups(pipe, 10_000, None)
+ hold = torch.empty(16 << 20, dtype = torch.uint8, device = "cuda")
+ restore()
+ del hold
+ _two_shapes(net)
+ _two_shapes(net)
+ assert cg._POOL_FLUSHES[0] - flushes == cycle + 1
+ assert handles[0].stats["captures"] == 6
+ cg.uninstall_all(handles)
+
+
+def test_a_re_record_waits_for_the_retired_pool_flush(monkeypatch):
+ """While the flush is prohibited (another thread records), the re-record is deferred and the step runs eager; it
+ records once that recording ended and the flush went through."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)
+ seeds = list(range(5))
+ want = _eager_reference(3, seeds)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ pipe = _pipe(net)
+ handles, _ = cg.arm_after_placement(pipe)
+ got = _run(net, seeds[:2])
+ assert handles[0].stats["captures"] == 1
+ restore = dm.release_resident_groups(pipe, 10_000, None)
+ hold = torch.empty(16 << 20, dtype = torch.uint8, device = "cuda")
+ restore()
+ del hold
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording
+ got += _run(net, seeds[2:3])
+ assert handles[0].stats["captures"] == 1
+ assert handles[0].stats.get("capture_refusals", 0) >= 1 and cg.retired_pool_due()
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ got += _run(net, seeds[3:])
+ assert handles[0].stats["captures"] == 2 and not cg.retired_pool_due()
+ for a, b in zip(got, want):
+ assert torch.equal(a, b)
+ cg.uninstall_all(handles)
+
+
+def test_a_capture_refused_while_another_records_never_empties_the_cache(monkeypatch):
+ """A capture refused because another thread is recording (a retired pool is due and cannot be flushed) runs eager
+ without touching the allocator: an empty_cache or a cuBLAS workspace clear inside that recording could invalidate
+ it. The cleanup stays due and the next capture's entry flush does it once the recording ended."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ _two_shapes(net)
+ before = handles[0].stats["captures"]
+ _retire() # another wrapper retired a pool
+ real = torch.cuda.empty_cache
+ during = []
+ monkeypatch.setattr(
+ torch.cuda, "empty_cache", lambda: during.append(cg._CAPTURE_DEPTH) or real()
+ )
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording
+ _two_shapes(net, (12, 4)) # new keys: each capture is refused at its entry
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ assert handles[0].stats["captures"] == before
+ assert handles[0].stats.get("capture_refusals", 0) >= 1
+ assert during == [] # nothing flushed while the other capture recorded
+ _two_shapes(net, (12, 4))
+ assert handles[0].stats["captures"] == before + 2 and not _due()
+ cg.uninstall_all(handles)
+
+
+def test_a_deferred_release_still_clears_the_cublas_workspaces_before_the_next_recording(
+ monkeypatch,
+):
+ """An offloaded wrapper's release also clears the cuBLAS workspaces its recordings left on their capture stream.
+ Deferred because another thread was recording, that clear is owed with the retired pool: the next capture's entry
+ flush does it under the capture lock (no capture recording), once, before it records."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ monkeypatch.setattr(cg, "_CUBLAS_CLEAR_DUE", [False], raising = False)
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ _two_shapes(net)
+ before = handles[0].stats["captures"]
+ _retire()
+ real = getattr(torch._C, "_cuda_clearCublasWorkspaces")
+ clears = []
+
+ def clear():
+ clears.append(cg._CAPTURE_DEPTH)
+ return real()
+
+ monkeypatch.setattr(torch._C, "_cuda_clearCublasWorkspaces", clear)
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording
+ _two_shapes(net, (12, 4)) # refused at entry: the release after it is deferred
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ assert handles[0].stats["captures"] == before and clears == []
+ _two_shapes(net, (12, 4))
+ assert handles[0].stats["captures"] == before + 2
+ assert clears == [0] # once, outside any recording
+ assert not cg._CUBLAS_CLEAR_DUE[0] and not _due()
+ cg.uninstall_all(handles)
+
+
+def test_a_settled_retirement_never_waits_on_another_capture(monkeypatch):
+ """Two threads saw the same retirement; the first flushed it and is now recording. The second must not wait on
+ that recording (and be refused after the timeout): its retirements are already flushed."""
+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 1], raising = False)
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 5.0)
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # the first thread is recording
+ import time
+
+ start = time.monotonic()
+ with cg._CAPTURE_IDLE:
+ cg._settle_retired_pools(1)
+ assert time.monotonic() - start < 1.0
+
+
+def test_an_immediate_release_acknowledges_the_retirement_it_flushed(monkeypatch):
+ """A speed drop or a reset counts the dropped pool and then releases it at once (no capture recording): that flush
+ acknowledges it, so the next capture does not synchronize and flush the allocator a second time."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ torch.manual_seed(0)
+ net = torch.nn.Linear(64, 64).cuda()
+ handle = cg.GraphedForward(net)
+ _retire() # the drop just counted its pool
+ handle._release() # ... and released it right away
+ assert not _due()
+
+
+def test_a_capture_with_a_pool_due_waits_out_another_recording_instead_of_running_eager(
+ monkeypatch,
+):
+ """PyTorch runs no non-captured CUDA work on any thread while a capture is underway. A capture whose retired pool
+ must be flushed while another thread records therefore waits for that recording to end, then flushes and records;
+ it is not refused into an eager step that would run beside the recording."""
+ import threading
+ import time
+
+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 0], raising = False)
+ flushed = []
+ monkeypatch.setattr(
+ cg, "_flush_retired_locked", lambda seen: flushed.append(cg._CAPTURE_DEPTH) or True
+ )
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording
+
+ def other_recording_ends():
+ time.sleep(0.6)
+ with cg._CAPTURE_IDLE:
+ cg._CAPTURE_DEPTH = 0
+ cg._CAPTURE_IDLE.notify_all()
+
+ ender = threading.Thread(target = other_recording_ends)
+ ender.start()
+ try:
+ with cg._CAPTURE_IDLE:
+ cg._settle_retired_pools(1)
+ finally:
+ ender.join()
+ assert flushed == [0]
+
+
+def test_a_deferred_release_is_owed_before_it_tries_the_lock(monkeypatch):
+ """The release counts its pool (and the cuBLAS clear) before gc and the lock attempt: a capture entering meanwhile
+ already sees it due instead of recording beside the unflushed segments."""
+ _reset_retired(monkeypatch)
+ monkeypatch.setattr(cg, "_CUBLAS_CLEAR_DUE", [False], raising = False)
+ seen_during_gc = []
+
+ class _Gc:
+ @staticmethod
+ def collect():
+ seen_during_gc.append((cg.retired_pool_due(), cg._CUBLAS_CLEAR_DUE[0]))
+
+ @__import__("contextlib").contextmanager
+ def recording_elsewhere():
+ yield False
+
+ monkeypatch.setattr(cg, "gc", _Gc)
+ monkeypatch.setattr(cg, "hold_off_capture", recording_elsewhere)
+ assert cg.release_cached_outside_capture(clear_cublas = True) is False
+ assert seen_during_gc == [(True, True)]
+ assert cg.retired_pool_due() and cg._CUBLAS_CLEAR_DUE[0]
+
+
+def test_a_pool_retired_while_a_capture_enters_is_flushed_before_it_records(monkeypatch):
+ """Another thread retires a pool after this capture read the counters but before it raised the depth (here, while
+ it waits for replica-side work to drain): the entry flushes that one too, so nothing records while it is due."""
+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 0], raising = False)
+ flushes = []
+
+ def flush(seen):
+ flushes.append(seen)
+ cg._RETIRED_POOLS[1] = max(cg._RETIRED_POOLS[1], seen)
+ return True
+
+ late = []
+
+ def drained(own):
+ if not late:
+ late.append(1)
+ cg.note_pool_retired() # lands after the entry flush, before the depth goes up
+
+ monkeypatch.setattr(cg, "_flush_retired_locked", flush)
+ monkeypatch.setattr(cg, "_wait_outside_drained", drained)
+ monkeypatch.setattr(cg.gc, "collect", lambda *a: 0)
+ with cg._capturing():
+ due_while_recording = cg.retired_pool_due()
+ assert late and not due_while_recording
+ assert flushes == [1, 2]
+
+
+def test_a_first_capture_refused_before_entry_keeps_the_retired_pool_due(monkeypatch):
+ """Another wrapper retired a pool; this wrapper's first capture is refused in _capturing (replica-side work still
+ held) before anything flushes: the flush stays due for the next capture."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ _retire()
+ monkeypatch.setattr(cg, "capture_refused_now", lambda: False)
+
+ def refused(own):
+ raise cg.CaptureRefused("replica-side CUDA work still running; capture refused")
+
+ monkeypatch.setattr(cg, "_wait_outside_drained", refused)
+ monkeypatch.setattr(cg, "await_outside_drained", lambda: None)
+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # the flush cannot run either
+ _run(net, [0, 1])
+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)
+ assert handles[0].stats["captures"] == 0
+ assert handles[0].stats.get("capture_refusals", 0) >= 1
+ assert _due()
+ cg.uninstall_all(handles)
+
+
+def test_a_pool_retired_while_the_capture_flushes_is_flushed_before_it_records(monkeypatch):
+ """Retirement is not serialized with captures: another thread's invalidation can retire a pool while this
+ capture's entry flush runs. That flush acknowledges only what was retired before it began, and the entry rechecks
+ before it raises the depth, so the newly retired pool is flushed too before anything records."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ torch.manual_seed(0)
+ net = _streamed(_Net(), 3)
+ handles, _ = cg.arm_after_placement(_pipe(net))
+ _retire()
+ real = torch.cuda.empty_cache
+ raced = []
+ due_at_record = []
+
+ def flush():
+ real()
+ if not raced:
+ raced.append(1)
+ _retire() # lands after this flush freed what it could, before the capture records
+
+ real_begin = torch.cuda.CUDAGraph.capture_begin
+
+ def begin(self, *args, **kwargs):
+ due_at_record.append(_due())
+ return real_begin(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.cuda, "empty_cache", flush)
+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_begin", begin)
+ flushes = cg._POOL_FLUSHES[0]
+ _run(net, [0, 1])
+ assert raced and handles[0].stats["captures"] >= 1
+ assert due_at_record and not any(due_at_record)
+ assert cg._POOL_FLUSHES[0] - flushes == 2 and not _due()
+ cg.uninstall_all(handles)
+
+
+def test_a_resident_planned_re_record_flushes_a_pool_another_wrapper_retired(monkeypatch):
+ """The retired-pool rule is process-wide, not per offload placement: a resident wrapper with a capture-safe plan
+ re-records later keys without torch.cuda.graph's flush, so a pool an offloaded wrapper retired must be handed back
+ before that re-record too, or it records a new prompt shape beside the reserved segments."""
+ _cuda()
+ _reset_retired(monkeypatch)
+ torch.manual_seed(0)
+ net = torch.nn.Linear(64, 64).cuda()
+ handle = cg.GraphedForward(net)
+ handle.plan = (
+ lambda *a: None
+ ) # a planned step: keys after the first re-record with no flush of their own
+ real = torch.cuda.empty_cache
+ flushes = []
+ with torch.no_grad():
+ first = handle._capture((torch.randn(16, 64, device = "cuda"),), {}, key = "a")
+ _retire()
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: flushes.append(1) or real())
+ x = torch.randn(8, 64, device = "cuda")
+ second = handle._capture((x,), {}, key = "b")
+ second.static[0].copy_(x)
+ second.graph.replay()
+ torch.cuda.synchronize()
+ out = second.out_tensors[0].clone()
+ assert handle.stats["captures"] == 2
+ assert flushes and not _due()
+ assert torch.equal(out, net(x))
+ del first, second
+ cg._POOL_BOX[0] = None
+
+
def test_unpinned_host_copies_keep_the_forward_eager_with_the_reason(monkeypatch):
_cuda()
monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")